From 13b3418a7f8cf24bd2706fb8dc8a4530d8ef9e0d Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Tue, 15 Jul 2025 04:25:07 +0000 Subject: [PATCH 1/5] Split `abc_module()` into `prepare_module()` and `run_abc()` `prepare_module()` will have to run on the main thread. --- passes/techmap/abc.cc | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/passes/techmap/abc.cc b/passes/techmap/abc.cc index 7d81b52d257..680428096e2 100644 --- a/passes/techmap/abc.cc +++ b/passes/techmap/abc.cc @@ -180,8 +180,9 @@ struct AbcModuleState { std::string remap_name(RTLIL::IdString abc_name, RTLIL::Wire **orig_wire = nullptr); void dump_loop_graph(FILE *f, int &nr, dict> &edges, pool &workpool, std::vector &in_counts); void handle_loops(AbcSigMap &assign_map, RTLIL::Module *module); - void abc_module(RTLIL::Design *design, RTLIL::Module *module, AbcSigMap &assign_map, const std::vector &cells, + void prepare_module(RTLIL::Design *design, RTLIL::Module *module, AbcSigMap &assign_map, const std::vector &cells, bool dff_mode, std::string clk_str); + void run_abc(); void extract(AbcSigMap &assign_map, RTLIL::Design *design, RTLIL::Module *module); void finish(); }; @@ -775,7 +776,7 @@ struct abc_output_filter } }; -void AbcModuleState::abc_module(RTLIL::Design *design, RTLIL::Module *module, AbcSigMap &assign_map, const std::vector &cells, +void AbcModuleState::prepare_module(RTLIL::Design *design, RTLIL::Module *module, AbcSigMap &assign_map, const std::vector &cells, bool dff_mode, std::string clk_str) { map_autoidx = autoidx++; @@ -987,9 +988,12 @@ void AbcModuleState::abc_module(RTLIL::Design *design, RTLIL::Module *module, Ab mark_port(assign_map, srst_sig); handle_loops(assign_map, module); +} - buffer = stringf("%s/input.blif", tempdir_name); - f = fopen(buffer.c_str(), "wt"); +void AbcModuleState::run_abc() +{ + std::string buffer = stringf("%s/input.blif", tempdir_name); + FILE *f = fopen(buffer.c_str(), "wt"); if (f == nullptr) log_error("Opening %s for writing failed: %s\n", buffer, strerror(errno)); @@ -1111,8 +1115,6 @@ void AbcModuleState::abc_module(RTLIL::Design *design, RTLIL::Module *module, Ab log_push(); if (count_output > 0) { - log_header(design, "Executing ABC.\n"); - auto &cell_cost = cmos_cost ? CellCosts::cmos_gate_cost() : CellCosts::default_gate_cost(); buffer = stringf("%s/stdcells.genlib", tempdir_name); @@ -1522,8 +1524,6 @@ void AbcModuleState::finish() log("Removing temp directory.\n"); remove_directory(tempdir_name); } - - log_pop(); } // For every signal that connects cells from different sets, or a cell in a set to a cell not in any set, @@ -2163,7 +2163,8 @@ struct AbcPass : public Pass { assign_cell_connection_ports(mod, {&cells}, assign_map); AbcModuleState state(config, initvals); - state.abc_module(design, mod, assign_map, cells, dff_mode, clk_str); + state.prepare_module(design, mod, assign_map, cells, dff_mode, clk_str); + state.run_abc(); state.extract(assign_map, design, mod); state.finish(); continue; @@ -2333,7 +2334,8 @@ struct AbcPass : public Pass { state.arst_sig = assign_map(std::get<5>(it.first)); state.srst_polarity = std::get<6>(it.first); state.srst_sig = assign_map(std::get<7>(it.first)); - state.abc_module(design, mod, assign_map, it.second, !state.clk_sig.empty(), "$"); + state.prepare_module(design, mod, assign_map, it.second, !state.clk_sig.empty(), "$"); + state.run_abc(); state.extract(assign_map, design, mod); state.finish(); } From 222f457a045e474bcf6712ad167a85d688b55b48 Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Tue, 15 Jul 2025 05:33:09 +0000 Subject: [PATCH 2/5] Only write out stdcells/lutcosts once for all ABC runs --- passes/techmap/abc.cc | 141 ++++++++++++++++++++++++------------------ 1 file changed, 82 insertions(+), 59 deletions(-) diff --git a/passes/techmap/abc.cc b/passes/techmap/abc.cc index 680428096e2..03417b7f22c 100644 --- a/passes/techmap/abc.cc +++ b/passes/techmap/abc.cc @@ -119,6 +119,7 @@ bool cmos_cost; struct AbcConfig { + std::string global_tempdir_name; std::string script_file; std::string exe_file; std::vector liberty_files; @@ -878,9 +879,9 @@ void AbcModuleState::prepare_module(RTLIL::Design *design, RTLIL::Module *module abc_script += stringf("read_constr -v \"%s\"; ", config.constr_file); } else if (!config.lut_costs.empty()) - abc_script += stringf("read_lut %s/lutdefs.txt; ", tempdir_name); + abc_script += stringf("read_lut %s/lutdefs.txt; ", config.global_tempdir_name); else - abc_script += stringf("read_library %s/stdcells.genlib; ", tempdir_name); + abc_script += stringf("read_library %s/stdcells.genlib; ", config.global_tempdir_name); if (!config.script_file.empty()) { const std::string &script_file = config.script_file; @@ -1112,65 +1113,8 @@ void AbcModuleState::run_abc() log("Extracted %d gates and %d wires to a netlist network with %d inputs and %d outputs.\n", count_gates, GetSize(signal_list), count_input, count_output); - log_push(); if (count_output > 0) { - auto &cell_cost = cmos_cost ? CellCosts::cmos_gate_cost() : CellCosts::default_gate_cost(); - - buffer = stringf("%s/stdcells.genlib", tempdir_name); - f = fopen(buffer.c_str(), "wt"); - if (f == nullptr) - log_error("Opening %s for writing failed: %s\n", buffer, strerror(errno)); - fprintf(f, "GATE ZERO 1 Y=CONST0;\n"); - fprintf(f, "GATE ONE 1 Y=CONST1;\n"); - fprintf(f, "GATE BUF %d Y=A; PIN * NONINV 1 999 1 0 1 0\n", cell_cost.at(ID($_BUF_))); - fprintf(f, "GATE NOT %d Y=!A; PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_NOT_))); - if (enabled_gates.count("AND")) - fprintf(f, "GATE AND %d Y=A*B; PIN * NONINV 1 999 1 0 1 0\n", cell_cost.at(ID($_AND_))); - if (enabled_gates.count("NAND")) - fprintf(f, "GATE NAND %d Y=!(A*B); PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_NAND_))); - if (enabled_gates.count("OR")) - fprintf(f, "GATE OR %d Y=A+B; PIN * NONINV 1 999 1 0 1 0\n", cell_cost.at(ID($_OR_))); - if (enabled_gates.count("NOR")) - fprintf(f, "GATE NOR %d Y=!(A+B); PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_NOR_))); - if (enabled_gates.count("XOR")) - fprintf(f, "GATE XOR %d Y=(A*!B)+(!A*B); PIN * UNKNOWN 1 999 1 0 1 0\n", cell_cost.at(ID($_XOR_))); - if (enabled_gates.count("XNOR")) - fprintf(f, "GATE XNOR %d Y=(A*B)+(!A*!B); PIN * UNKNOWN 1 999 1 0 1 0\n", cell_cost.at(ID($_XNOR_))); - if (enabled_gates.count("ANDNOT")) - fprintf(f, "GATE ANDNOT %d Y=A*!B; PIN * UNKNOWN 1 999 1 0 1 0\n", cell_cost.at(ID($_ANDNOT_))); - if (enabled_gates.count("ORNOT")) - fprintf(f, "GATE ORNOT %d Y=A+!B; PIN * UNKNOWN 1 999 1 0 1 0\n", cell_cost.at(ID($_ORNOT_))); - if (enabled_gates.count("AOI3")) - fprintf(f, "GATE AOI3 %d Y=!((A*B)+C); PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_AOI3_))); - if (enabled_gates.count("OAI3")) - fprintf(f, "GATE OAI3 %d Y=!((A+B)*C); PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_OAI3_))); - if (enabled_gates.count("AOI4")) - fprintf(f, "GATE AOI4 %d Y=!((A*B)+(C*D)); PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_AOI4_))); - if (enabled_gates.count("OAI4")) - fprintf(f, "GATE OAI4 %d Y=!((A+B)*(C+D)); PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_OAI4_))); - if (enabled_gates.count("MUX")) - fprintf(f, "GATE MUX %d Y=(A*B)+(S*B)+(!S*A); PIN * UNKNOWN 1 999 1 0 1 0\n", cell_cost.at(ID($_MUX_))); - if (enabled_gates.count("NMUX")) - fprintf(f, "GATE NMUX %d Y=!((A*B)+(S*B)+(!S*A)); PIN * UNKNOWN 1 999 1 0 1 0\n", cell_cost.at(ID($_NMUX_))); - if (map_mux4) - fprintf(f, "GATE MUX4 %d Y=(!S*!T*A)+(S*!T*B)+(!S*T*C)+(S*T*D); PIN * UNKNOWN 1 999 1 0 1 0\n", 2*cell_cost.at(ID($_MUX_))); - if (map_mux8) - fprintf(f, "GATE MUX8 %d Y=(!S*!T*!U*A)+(S*!T*!U*B)+(!S*T*!U*C)+(S*T*!U*D)+(!S*!T*U*E)+(S*!T*U*F)+(!S*T*U*G)+(S*T*U*H); PIN * UNKNOWN 1 999 1 0 1 0\n", 4*cell_cost.at(ID($_MUX_))); - if (map_mux16) - fprintf(f, "GATE MUX16 %d Y=(!S*!T*!U*!V*A)+(S*!T*!U*!V*B)+(!S*T*!U*!V*C)+(S*T*!U*!V*D)+(!S*!T*U*!V*E)+(S*!T*U*!V*F)+(!S*T*U*!V*G)+(S*T*U*!V*H)+(!S*!T*!U*V*I)+(S*!T*!U*V*J)+(!S*T*!U*V*K)+(S*T*!U*V*L)+(!S*!T*U*V*M)+(S*!T*U*V*N)+(!S*T*U*V*O)+(S*T*U*V*P); PIN * UNKNOWN 1 999 1 0 1 0\n", 8*cell_cost.at(ID($_MUX_))); - fclose(f); - - if (!config.lut_costs.empty()) { - buffer = stringf("%s/lutdefs.txt", tempdir_name); - f = fopen(buffer.c_str(), "wt"); - if (f == nullptr) - log_error("Opening %s for writing failed: %s\n", buffer, strerror(errno)); - for (int i = 0; i < GetSize(config.lut_costs); i++) - fprintf(f, "%d %d.00 1.00\n", i+1, config.lut_costs.at(i)); - fclose(f); - } - buffer = stringf("\"%s\" -s -f %s/abc.script 2>&1", config.exe_file, tempdir_name); log("Running ABC command: %s\n", replace_tempdir(buffer, tempdir_name, config.show_tempdir)); @@ -1231,6 +1175,65 @@ void AbcModuleState::run_abc() log("Don't call ABC as there is nothing to map.\n"); } +void emit_global_input_files(const AbcConfig &config) +{ + if (!config.lut_costs.empty()) { + std::string buffer = stringf("%s/lutdefs.txt", config.global_tempdir_name.c_str()); + FILE *f = fopen(buffer.c_str(), "wt"); + if (f == nullptr) + log_error("Opening %s for writing failed: %s\n", buffer.c_str(), strerror(errno)); + for (int i = 0; i < GetSize(config.lut_costs); i++) + fprintf(f, "%d %d.00 1.00\n", i+1, config.lut_costs.at(i)); + fclose(f); + } else { + auto &cell_cost = cmos_cost ? CellCosts::cmos_gate_cost() : CellCosts::default_gate_cost(); + + std::string buffer = stringf("%s/stdcells.genlib", config.global_tempdir_name.c_str()); + FILE *f = fopen(buffer.c_str(), "wt"); + if (f == nullptr) + log_error("Opening %s for writing failed: %s\n", buffer.c_str(), strerror(errno)); + fprintf(f, "GATE ZERO 1 Y=CONST0;\n"); + fprintf(f, "GATE ONE 1 Y=CONST1;\n"); + fprintf(f, "GATE BUF %d Y=A; PIN * NONINV 1 999 1 0 1 0\n", cell_cost.at(ID($_BUF_))); + fprintf(f, "GATE NOT %d Y=!A; PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_NOT_))); + if (enabled_gates.count("AND")) + fprintf(f, "GATE AND %d Y=A*B; PIN * NONINV 1 999 1 0 1 0\n", cell_cost.at(ID($_AND_))); + if (enabled_gates.count("NAND")) + fprintf(f, "GATE NAND %d Y=!(A*B); PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_NAND_))); + if (enabled_gates.count("OR")) + fprintf(f, "GATE OR %d Y=A+B; PIN * NONINV 1 999 1 0 1 0\n", cell_cost.at(ID($_OR_))); + if (enabled_gates.count("NOR")) + fprintf(f, "GATE NOR %d Y=!(A+B); PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_NOR_))); + if (enabled_gates.count("XOR")) + fprintf(f, "GATE XOR %d Y=(A*!B)+(!A*B); PIN * UNKNOWN 1 999 1 0 1 0\n", cell_cost.at(ID($_XOR_))); + if (enabled_gates.count("XNOR")) + fprintf(f, "GATE XNOR %d Y=(A*B)+(!A*!B); PIN * UNKNOWN 1 999 1 0 1 0\n", cell_cost.at(ID($_XNOR_))); + if (enabled_gates.count("ANDNOT")) + fprintf(f, "GATE ANDNOT %d Y=A*!B; PIN * UNKNOWN 1 999 1 0 1 0\n", cell_cost.at(ID($_ANDNOT_))); + if (enabled_gates.count("ORNOT")) + fprintf(f, "GATE ORNOT %d Y=A+!B; PIN * UNKNOWN 1 999 1 0 1 0\n", cell_cost.at(ID($_ORNOT_))); + if (enabled_gates.count("AOI3")) + fprintf(f, "GATE AOI3 %d Y=!((A*B)+C); PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_AOI3_))); + if (enabled_gates.count("OAI3")) + fprintf(f, "GATE OAI3 %d Y=!((A+B)*C); PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_OAI3_))); + if (enabled_gates.count("AOI4")) + fprintf(f, "GATE AOI4 %d Y=!((A*B)+(C*D)); PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_AOI4_))); + if (enabled_gates.count("OAI4")) + fprintf(f, "GATE OAI4 %d Y=!((A+B)*(C+D)); PIN * INV 1 999 1 0 1 0\n", cell_cost.at(ID($_OAI4_))); + if (enabled_gates.count("MUX")) + fprintf(f, "GATE MUX %d Y=(A*B)+(S*B)+(!S*A); PIN * UNKNOWN 1 999 1 0 1 0\n", cell_cost.at(ID($_MUX_))); + if (enabled_gates.count("NMUX")) + fprintf(f, "GATE NMUX %d Y=!((A*B)+(S*B)+(!S*A)); PIN * UNKNOWN 1 999 1 0 1 0\n", cell_cost.at(ID($_NMUX_))); + if (map_mux4) + fprintf(f, "GATE MUX4 %d Y=(!S*!T*A)+(S*!T*B)+(!S*T*C)+(S*T*D); PIN * UNKNOWN 1 999 1 0 1 0\n", 2*cell_cost.at(ID($_MUX_))); + if (map_mux8) + fprintf(f, "GATE MUX8 %d Y=(!S*!T*!U*A)+(S*!T*!U*B)+(!S*T*!U*C)+(S*T*!U*D)+(!S*!T*U*E)+(S*!T*U*F)+(!S*T*U*G)+(S*T*U*H); PIN * UNKNOWN 1 999 1 0 1 0\n", 4*cell_cost.at(ID($_MUX_))); + if (map_mux16) + fprintf(f, "GATE MUX16 %d Y=(!S*!T*!U*!V*A)+(S*!T*!U*!V*B)+(!S*T*!U*!V*C)+(S*T*!U*!V*D)+(!S*!T*U*!V*E)+(S*!T*U*!V*F)+(!S*T*U*!V*G)+(S*T*U*!V*H)+(!S*!T*!U*V*I)+(S*!T*!U*V*J)+(!S*T*!U*V*K)+(S*T*!U*V*L)+(!S*!T*U*V*M)+(S*!T*U*V*N)+(!S*T*U*V*O)+(S*T*U*V*P); PIN * UNKNOWN 1 999 1 0 1 0\n", 8*cell_cost.at(ID($_MUX_))); + fclose(f); + } +} + void AbcModuleState::extract(AbcSigMap &assign_map, RTLIL::Design *design, RTLIL::Module *module) { if (!did_run_abc) { @@ -1811,6 +1814,13 @@ struct AbcPass : public Pass { config.show_tempdir = design->scratchpad_get_bool("abc.showtmp", false); markgroups = design->scratchpad_get_bool("abc.markgroups", markgroups); + if (config.cleanup) + config.global_tempdir_name = get_base_tmpdir() + "/"; + else + config.global_tempdir_name = "_tmp_"; + config.global_tempdir_name += proc_program_prefix() + "yosys-abc-XXXXXX"; + config.global_tempdir_name = make_temp_dir(config.global_tempdir_name); + if (design->scratchpad_get_bool("abc.debug")) { config.cleanup = false; config.show_tempdir = true; @@ -2139,6 +2149,8 @@ struct AbcPass : public Pass { // enabled_gates.insert("NMUX"); } + emit_global_input_files(config); + for (auto mod : design->selected_modules()) { if (mod->processes.size() > 0) { @@ -2164,9 +2176,12 @@ struct AbcPass : public Pass { AbcModuleState state(config, initvals); state.prepare_module(design, mod, assign_map, cells, dff_mode, clk_str); + log_push(); + log_header(design, "Executing ABC.\n"); state.run_abc(); state.extract(assign_map, design, mod); state.finish(); + log_pop(); continue; } @@ -2335,12 +2350,20 @@ struct AbcPass : public Pass { state.srst_polarity = std::get<6>(it.first); state.srst_sig = assign_map(std::get<7>(it.first)); state.prepare_module(design, mod, assign_map, it.second, !state.clk_sig.empty(), "$"); + log_push(); + log_header(design, "Executing ABC.\n"); state.run_abc(); state.extract(assign_map, design, mod); state.finish(); + log_pop(); } } + if (config.cleanup) { + log("Removing global temp directory.\n"); + remove_directory(config.global_tempdir_name); + } + log_pop(); } } AbcPass; From 38f8165c8079bf9f1e853ef1ff15237aa48d4527 Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Wed, 10 Sep 2025 22:53:53 +0000 Subject: [PATCH 3/5] Remove direct RTLIL access from gate_t --- passes/techmap/abc.cc | 54 ++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/passes/techmap/abc.cc b/passes/techmap/abc.cc index 03417b7f22c..a0521e26956 100644 --- a/passes/techmap/abc.cc +++ b/passes/techmap/abc.cc @@ -104,8 +104,10 @@ struct gate_t gate_type_t type; int in1, in2, in3, in4; bool is_port; - RTLIL::SigBit bit; + bool bit_is_wire; + bool bit_is_1; RTLIL::State init; + std::string bit_str; }; bool map_mux4; @@ -156,6 +158,7 @@ struct AbcModuleState { int map_autoidx = 0; std::vector signal_list; + std::vector signal_bits; dict signal_map; FfInitVals &initvals; bool had_init = false; @@ -204,10 +207,13 @@ int AbcModuleState::map_signal(const AbcSigMap &assign_map, RTLIL::SigBit bit, g gate.in3 = -1; gate.in4 = -1; gate.is_port = bit.wire != nullptr && val.is_port; - gate.bit = bit; + gate.bit_is_wire = bit.wire != nullptr; + gate.bit_is_1 = bit == State::S1; gate.init = initvals(bit); - signal_list.push_back(gate); + gate.bit_str = std::string(log_signal(bit)); signal_map[bit] = gate.id; + signal_list.push_back(std::move(gate)); + signal_bits.push_back(bit); } gate_t &gate = signal_list[signal_map[bit]]; @@ -463,17 +469,17 @@ std::string AbcModuleState::remap_name(RTLIL::IdString abc_name, RTLIL::Wire **o if (sid < GetSize(signal_list)) { - auto sig = signal_list.at(sid); - if (sig.bit.wire != nullptr) + const auto &bit = signal_bits.at(sid); + if (bit.wire != nullptr) { - std::string s = stringf("$abc$%d$%s", map_autoidx, sig.bit.wire->name.c_str()+1); - if (sig.bit.wire->width != 1) - s += stringf("[%d]", sig.bit.offset); + std::string s = stringf("$abc$%d$%s", map_autoidx, bit.wire->name.c_str()+1); + if (bit.wire->width != 1) + s += stringf("[%d]", bit.offset); if (isnew) s += "_new"; s += postfix; if (orig_wire != nullptr) - *orig_wire = sig.bit.wire; + *orig_wire = bit.wire; return s; } } @@ -501,7 +507,7 @@ void AbcModuleState::dump_loop_graph(FILE *f, int &nr, dict> &edg } for (auto n : nodes) - fprintf(f, " ys__n%d [label=\"%s\\nid=%d, count=%d\"%s];\n", n, log_signal(signal_list[n].bit), + fprintf(f, " ys__n%d [label=\"%s\\nid=%d, count=%d\"%s];\n", n, signal_list[n].bit_str.c_str(), n, in_counts[n], workpool.count(n) ? ", shape=box" : ""); for (auto &e : edges) @@ -562,7 +568,7 @@ void AbcModuleState::handle_loops(AbcSigMap &assign_map, RTLIL::Module *module) int id = *workpool.begin(); workpool.erase(id); - // log("Removing non-loop node %d from graph: %s\n", id, log_signal(signal_list[id].bit)); + // log("Removing non-loop node %d from graph: %s\n", id, signal_list[id].bit_str); for (int id2 : edges[id]) { log_assert(in_edges_count[id2] > 0); @@ -582,8 +588,8 @@ void AbcModuleState::handle_loops(AbcSigMap &assign_map, RTLIL::Module *module) for (auto &edge_it : edges) { int id2 = edge_it.first; - RTLIL::Wire *w1 = signal_list[id1].bit.wire; - RTLIL::Wire *w2 = signal_list[id2].bit.wire; + RTLIL::Wire *w1 = signal_bits[id1].wire; + RTLIL::Wire *w2 = signal_bits[id2].wire; if (w1 == nullptr) id1 = id2; else if (w2 == nullptr) @@ -605,7 +611,7 @@ void AbcModuleState::handle_loops(AbcSigMap &assign_map, RTLIL::Module *module) continue; } - log_assert(signal_list[id1].bit.wire != nullptr); + log_assert(signal_bits[id1].wire != nullptr); std::stringstream sstr; sstr << "$abcloop$" << (autoidx++); @@ -615,10 +621,10 @@ void AbcModuleState::handle_loops(AbcSigMap &assign_map, RTLIL::Module *module) for (int id2 : edges[id1]) { if (first_line) log("Breaking loop using new signal %s: %s -> %s\n", log_signal(RTLIL::SigSpec(wire)), - log_signal(signal_list[id1].bit), log_signal(signal_list[id2].bit)); + signal_list[id1].bit_str, signal_list[id2].bit_str); else log(" %*s %s -> %s\n", int(strlen(log_signal(RTLIL::SigSpec(wire)))), "", - log_signal(signal_list[id1].bit), log_signal(signal_list[id2].bit)); + signal_list[id1].bit_str, signal_list[id2].bit_str); first_line = false; } @@ -641,7 +647,7 @@ void AbcModuleState::handle_loops(AbcSigMap &assign_map, RTLIL::Module *module) } edges[id1].swap(edges[id3]); - connect(assign_map, module, RTLIL::SigSig(signal_list[id3].bit, signal_list[id1].bit)); + connect(assign_map, module, RTLIL::SigSig(signal_bits[id3], signal_bits[id1])); dump_loop_graph(dot_f, dot_nr, edges, workpool, in_edges_count); } } @@ -1006,7 +1012,7 @@ void AbcModuleState::run_abc() if (!si.is_port || si.type != G(NONE)) continue; fprintf(f, " ys__n%d", si.id); - pi_map[count_input++] = log_signal(si.bit); + pi_map[count_input++] = si.bit_str; } if (count_input == 0) fprintf(f, " dummy_input\n"); @@ -1018,17 +1024,17 @@ void AbcModuleState::run_abc() if (!si.is_port || si.type == G(NONE)) continue; fprintf(f, " ys__n%d", si.id); - po_map[count_output++] = log_signal(si.bit); + po_map[count_output++] = si.bit_str; } fprintf(f, "\n"); for (auto &si : signal_list) - fprintf(f, "# ys__n%-5d %s\n", si.id, log_signal(si.bit)); + fprintf(f, "# ys__n%-5d %s\n", si.id, si.bit_str.c_str()); for (auto &si : signal_list) { - if (si.bit.wire == nullptr) { + if (!si.bit_is_wire) { fprintf(f, ".names ys__n%d\n", si.id); - if (si.bit == RTLIL::State::S1) + if (si.bit_is_1) fprintf(f, "1\n"); } } @@ -1503,12 +1509,12 @@ void AbcModuleState::extract(AbcSigMap &assign_map, RTLIL::Design *design, RTLIL snprintf(buffer, 100, "\\ys__n%d", si.id); RTLIL::SigSig conn; if (si.type != G(NONE)) { - conn.first = si.bit; + conn.first = signal_bits[si.id]; conn.second = module->wire(remap_name(buffer)); out_wires++; } else { conn.first = module->wire(remap_name(buffer)); - conn.second = si.bit; + conn.second = signal_bits[si.id]; in_wires++; } connect(assign_map, module, conn); From 27462da20895706b6a9b362c8daa042a06d222a8 Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Sat, 26 Jul 2025 02:45:21 +0000 Subject: [PATCH 4/5] Run ABCs in parallel. Large circuits can run hundreds or thousands of ABCs in a single AbcPass. For some circuits, some of those ABC runs can run for hundreds of seconds. Running ABCs in parallel with each other and in parallel with main-thread processing (reading and writing BLIF files, copying ABC BLIF output into the design) can give large speedups. --- Makefile | 14 ++- kernel/threading.cc | 45 +++++++++ kernel/threading.h | 159 +++++++++++++++++++++++++++++ kernel/yosys.cc | 2 +- misc/create_vcxsrc.sh | 1 + passes/techmap/abc.cc | 229 ++++++++++++++++++++++++++---------------- 6 files changed, 362 insertions(+), 88 deletions(-) create mode 100644 kernel/threading.cc create mode 100644 kernel/threading.h diff --git a/Makefile b/Makefile index d565ead16bf..4aae25c3c84 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,12 @@ LINK_ABC := 0 # Needed for environments that can't run executables (i.e. emscripten, wasm) DISABLE_SPAWN := 0 # Needed for environments that don't have proper thread support (i.e. emscripten, wasm--for now) +ENABLE_THREADS := 1 +ifeq ($(ENABLE_THREADS),1) DISABLE_ABC_THREADS := 0 +else +DISABLE_ABC_THREADS := 1 +endif # clang sanitizers SANITIZER = @@ -300,6 +305,7 @@ DISABLE_SPAWN := 1 ifeq ($(ENABLE_ABC),1) LINK_ABC := 1 +ENABLE_THREADS := 0 DISABLE_ABC_THREADS := 1 endif @@ -457,6 +463,11 @@ CXXFLAGS := -Og -DDEBUG $(filter-out $(OPT_LEVEL),$(CXXFLAGS)) STRIP := endif +ifeq ($(ENABLE_THREADS),1) +CXXFLAGS += -DYOSYS_ENABLE_THREADS +LIBS += -lpthread +endif + ifeq ($(ENABLE_ABC),1) CXXFLAGS += -DYOSYS_ENABLE_ABC ifeq ($(LINK_ABC),1) @@ -612,6 +623,7 @@ $(eval $(call add_include_file,kernel/satgen.h)) $(eval $(call add_include_file,kernel/scopeinfo.h)) $(eval $(call add_include_file,kernel/sexpr.h)) $(eval $(call add_include_file,kernel/sigtools.h)) +$(eval $(call add_include_file,kernel/threading.h)) $(eval $(call add_include_file,kernel/timinginfo.h)) $(eval $(call add_include_file,kernel/utils.h)) $(eval $(call add_include_file,kernel/yosys.h)) @@ -638,7 +650,7 @@ OBJS += kernel/log_compat.o endif OBJS += kernel/binding.o kernel/tclapi.o OBJS += kernel/cellaigs.o kernel/celledges.o kernel/cost.o kernel/satgen.o kernel/scopeinfo.o kernel/qcsat.o kernel/mem.o kernel/ffmerge.o kernel/ff.o kernel/yw.o kernel/json.o kernel/fmt.o kernel/sexpr.o -OBJS += kernel/drivertools.o kernel/functional.o +OBJS += kernel/drivertools.o kernel/functional.o kernel/threading.o ifeq ($(ENABLE_ZLIB),1) OBJS += kernel/fstdata.o endif diff --git a/kernel/threading.cc b/kernel/threading.cc new file mode 100644 index 00000000000..49fddaa7c0e --- /dev/null +++ b/kernel/threading.cc @@ -0,0 +1,45 @@ +#include "kernel/yosys_common.h" +#include "kernel/threading.h" + +YOSYS_NAMESPACE_BEGIN + +void DeferredLogs::flush() +{ + for (auto &m : logs) + if (m.error) + YOSYS_NAMESPACE_PREFIX log_error("%s", m.text.c_str()); + else + YOSYS_NAMESPACE_PREFIX log("%s", m.text.c_str()); +} + +int ThreadPool::pool_size(int reserved_cores, int max_threads) +{ +#ifdef YOSYS_ENABLE_THREADS + int num_threads = std::min(std::thread::hardware_concurrency() - reserved_cores, max_threads); + return std::max(0, num_threads); +#else + return 0; +#endif +} + +ThreadPool::ThreadPool(int pool_size, std::function b) + : body(std::move(b)) +{ +#ifdef YOSYS_ENABLE_THREADS + threads.reserve(pool_size); + for (int i = 0; i < pool_size; i++) + threads.emplace_back([i, this]{ body(i); }); +#else + log_assert(pool_size == 0); +#endif +} + +ThreadPool::~ThreadPool() +{ +#ifdef YOSYS_ENABLE_THREADS + for (auto &t : threads) + t.join(); +#endif +} + +YOSYS_NAMESPACE_END diff --git a/kernel/threading.h b/kernel/threading.h new file mode 100644 index 00000000000..8c08d670cad --- /dev/null +++ b/kernel/threading.h @@ -0,0 +1,159 @@ +#include + +#ifdef YOSYS_ENABLE_THREADS +#include +#include +#include +#endif + +#include "kernel/yosys_common.h" +#include "kernel/log.h" + +#ifndef YOSYS_THREADING_H +#define YOSYS_THREADING_H + +YOSYS_NAMESPACE_BEGIN + +// Concurrent queue implementation. Not fast, but simple. +// Multi-producer, multi-consumer, optionally bounded. +// When YOSYS_ENABLE_THREADS is not defined, this is just a non-thread-safe non-blocking deque. +template +class ConcurrentQueue +{ +public: + ConcurrentQueue(int capacity = INT_MAX) + : capacity(capacity) {} + // Push an element into the queue. If it's at capacity, block until there is room. + void push_back(T t) + { +#ifdef YOSYS_ENABLE_THREADS + std::unique_lock lock(mutex); + not_full_condition.wait(lock, [this] { return static_cast(contents.size()) < capacity; }); + if (contents.empty()) + not_empty_condition.notify_one(); +#endif + log_assert(!closed); + contents.push_back(std::move(t)); +#ifdef YOSYS_ENABLE_THREADS + if (static_cast(contents.size()) < capacity) + not_full_condition.notify_one(); +#endif + } + // Signal that no more elements will be produced. `pop_front()` will return nullopt. + void close() + { +#ifdef YOSYS_ENABLE_THREADS + std::unique_lock lock(mutex); + not_empty_condition.notify_all(); +#endif + closed = true; + } + // Pop an element from the queue. Blocks until an element is available + // or the queue is closed and empty. + std::optional pop_front() + { + return pop_front_internal(true); + } + // Pop an element from the queue. Does not block, just returns nullopt if the + // queue is empty. + std::optional try_pop_front() + { + return pop_front_internal(false); + } +private: +#ifdef YOSYS_ENABLE_THREADS + std::optional pop_front_internal(bool wait) + { + std::unique_lock lock(mutex); + if (wait) { + not_empty_condition.wait(lock, [this] { return !contents.empty() || closed; }); + } +#else + std::optional pop_front_internal(bool) + { +#endif + if (contents.empty()) + return std::nullopt; +#ifdef YOSYS_ENABLE_THREADS + if (static_cast(contents.size()) == capacity) + not_full_condition.notify_one(); +#endif + T result = std::move(contents.front()); + contents.pop_front(); +#ifdef YOSYS_ENABLE_THREADS + if (!contents.empty()) + not_empty_condition.notify_one(); +#endif + return std::move(result); + } + +#ifdef YOSYS_ENABLE_THREADS + std::mutex mutex; + // Signals one waiter thread when the queue changes and is not full. + std::condition_variable not_full_condition; + // Signals one waiter thread when the queue changes and is not empty. + std::condition_variable not_empty_condition; +#endif + std::deque contents; + int capacity; + bool closed = false; +}; + +class DeferredLogs +{ +public: + template + void log(FmtString...> fmt, Args... args) + { + logs.push_back({fmt.format(args...), false}); + } + template + void log_error(FmtString...> fmt, Args... args) + { + logs.push_back({fmt.format(args...), true}); + } + void flush(); +private: + struct Message + { + std::string text; + bool error; + }; + std::vector logs; +}; + +class ThreadPool +{ +public: + // Computes the number of worker threads to use. + // `reserved_cores` cores are set aside for other threads (e.g. work on the main thread). + // `max_threads` --- don't return more workers than this. + // The result may be 0. + static int pool_size(int reserved_cores, int max_threads); + + // Create a pool of threads running the given closure (parameterized by thread number). + // `pool_size` must be the result of a `pool_size()` call. + ThreadPool(int pool_size, std::function b); + ThreadPool(ThreadPool &&other) = delete; + // Waits for all threads to terminate. Make sure those closures return! + ~ThreadPool(); + + // Return the number of threads in the pool. + int num_threads() const + { +#ifdef YOSYS_ENABLE_THREADS + return threads.size(); +#else + return 0; +#endif + } +private: + std::function body; +#ifdef YOSYS_ENABLE_THREADS + std::vector threads; +#endif +}; + +YOSYS_NAMESPACE_END + +#endif // YOSYS_THREADING_H diff --git a/kernel/yosys.cc b/kernel/yosys.cc index dc640fae9b9..95beca75c49 100644 --- a/kernel/yosys.cc +++ b/kernel/yosys.cc @@ -177,7 +177,7 @@ int run_command(const std::string &command, std::function "$vcxsrc"/YosysVS/YosysVS.vcxproj.new sed -i 's,,\n stdcpp17\n /Zc:__cplusplus %(AdditionalOptions),g' "$vcxsrc"/YosysVS/YosysVS.vcxproj.new +sed -i 's,,YOSYS_ENABLE_THREADS;,g' "$vcxsrc"/YosysVS/YosysVS.vcxproj.new if [ -f "/usr/include/FlexLexer.h" ] ; then sed -i 's,,;..\\yosys\\libs\\flex,g' "$vcxsrc"/YosysVS/YosysVS.vcxproj.new fi diff --git a/passes/techmap/abc.cc b/passes/techmap/abc.cc index a0521e26956..4a65671a28b 100644 --- a/passes/techmap/abc.cc +++ b/passes/techmap/abc.cc @@ -48,6 +48,7 @@ #include "kernel/ff.h" #include "kernel/cost.h" #include "kernel/log.h" +#include "kernel/threading.h" #include #include #include @@ -55,6 +56,7 @@ #include #include #include +#include #include #ifndef _WIN32 @@ -153,30 +155,41 @@ struct AbcSigVal { using AbcSigMap = SigValMap; -struct AbcModuleState { +// Used by off-main-threads. Contains no direct or indirect access to RTLIL. +struct RunAbcState { const AbcConfig &config; - int map_autoidx = 0; + std::string tempdir_name; std::vector signal_list; + bool did_run = false; + bool err = false; + DeferredLogs logs; + dict pi_map, po_map; + + RunAbcState(const AbcConfig &config) : config(config) {} + void run(); +}; + +struct AbcModuleState { + RunAbcState run_abc; + + int map_autoidx = 0; std::vector signal_bits; dict signal_map; FfInitVals &initvals; bool had_init = false; - bool did_run_abc = false; bool clk_polarity = false; bool en_polarity = false; bool arst_polarity = false; bool srst_polarity = false; RTLIL::SigSpec clk_sig, en_sig, arst_sig, srst_sig; - dict pi_map, po_map; int undef_bits_lost = 0; - std::string tempdir_name; - AbcModuleState(const AbcConfig &config, FfInitVals &initvals) - : config(config), initvals(initvals) {} + : run_abc(config), initvals(initvals) {} + AbcModuleState(AbcModuleState&&) = delete; int map_signal(const AbcSigMap &assign_map, RTLIL::SigBit bit, gate_type_t gate_type = G(NONE), int in1 = -1, int in2 = -1, int in3 = -1, int in4 = -1); void mark_port(const AbcSigMap &assign_map, RTLIL::SigSpec sig); @@ -186,7 +199,6 @@ struct AbcModuleState { void handle_loops(AbcSigMap &assign_map, RTLIL::Module *module); void prepare_module(RTLIL::Design *design, RTLIL::Module *module, AbcSigMap &assign_map, const std::vector &cells, bool dff_mode, std::string clk_str); - void run_abc(); void extract(AbcSigMap &assign_map, RTLIL::Design *design, RTLIL::Module *module); void finish(); }; @@ -200,7 +212,7 @@ int AbcModuleState::map_signal(const AbcSigMap &assign_map, RTLIL::SigBit bit, g if (signal_map.count(bit) == 0) { gate_t gate; - gate.id = signal_list.size(); + gate.id = run_abc.signal_list.size(); gate.type = G(NONE); gate.in1 = -1; gate.in2 = -1; @@ -212,11 +224,11 @@ int AbcModuleState::map_signal(const AbcSigMap &assign_map, RTLIL::SigBit bit, g gate.init = initvals(bit); gate.bit_str = std::string(log_signal(bit)); signal_map[bit] = gate.id; - signal_list.push_back(std::move(gate)); + run_abc.signal_list.push_back(std::move(gate)); signal_bits.push_back(bit); } - gate_t &gate = signal_list[signal_map[bit]]; + gate_t &gate = run_abc.signal_list[signal_map[bit]]; if (gate_type != G(NONE)) gate.type = gate_type; @@ -236,7 +248,7 @@ void AbcModuleState::mark_port(const AbcSigMap &assign_map, RTLIL::SigSpec sig) { for (auto &bit : assign_map(sig)) if (bit.wire != nullptr && signal_map.count(bit) > 0) - signal_list[signal_map[bit]].is_port = true; + run_abc.signal_list[signal_map[bit]].is_port = true; } bool AbcModuleState::extract_cell(const AbcSigMap &assign_map, RTLIL::Module *module, RTLIL::Cell *cell, bool keepff) @@ -315,7 +327,7 @@ bool AbcModuleState::extract_cell(const AbcSigMap &assign_map, RTLIL::Module *mo if (keepff) { SigBit bit = ff.sig_q; if (assign_map(bit).wire != nullptr) { - signal_list[gate_id].is_port = true; + run_abc.signal_list[gate_id].is_port = true; } if (bit.wire != nullptr) bit.wire->attributes[ID::keep] = 1; @@ -467,7 +479,7 @@ std::string AbcModuleState::remap_name(RTLIL::IdString abc_name, RTLIL::Wire **o size_t postfix_start = abc_sname.find_first_not_of("0123456789"); std::string postfix = postfix_start != std::string::npos ? abc_sname.substr(postfix_start) : ""; - if (sid < GetSize(signal_list)) + if (sid < GetSize(run_abc.signal_list)) { const auto &bit = signal_bits.at(sid); if (bit.wire != nullptr) @@ -507,7 +519,7 @@ void AbcModuleState::dump_loop_graph(FILE *f, int &nr, dict> &edg } for (auto n : nodes) - fprintf(f, " ys__n%d [label=\"%s\\nid=%d, count=%d\"%s];\n", n, signal_list[n].bit_str.c_str(), + fprintf(f, " ys__n%d [label=\"%s\\nid=%d, count=%d\"%s];\n", n, run_abc.signal_list[n].bit_str.c_str(), n, in_counts[n], workpool.count(n) ? ", shape=box" : ""); for (auto &e : edges) @@ -529,7 +541,7 @@ void AbcModuleState::handle_loops(AbcSigMap &assign_map, RTLIL::Module *module) // (Kahn, Arthur B. (1962), "Topological sorting of large networks") dict> edges; - std::vector in_edges_count(signal_list.size()); + std::vector in_edges_count(run_abc.signal_list.size()); pool workpool; FILE *dot_f = nullptr; @@ -538,7 +550,7 @@ void AbcModuleState::handle_loops(AbcSigMap &assign_map, RTLIL::Module *module) // uncomment for troubleshooting the loop detection code // dot_f = fopen("test.dot", "w"); - for (auto &g : signal_list) { + for (auto &g : run_abc.signal_list) { if (g.type == G(NONE) || g.type == G(FF) || g.type == G(FF0) || g.type == G(FF1)) { workpool.insert(g.id); } else { @@ -621,29 +633,29 @@ void AbcModuleState::handle_loops(AbcSigMap &assign_map, RTLIL::Module *module) for (int id2 : edges[id1]) { if (first_line) log("Breaking loop using new signal %s: %s -> %s\n", log_signal(RTLIL::SigSpec(wire)), - signal_list[id1].bit_str, signal_list[id2].bit_str); + run_abc.signal_list[id1].bit_str, run_abc.signal_list[id2].bit_str); else log(" %*s %s -> %s\n", int(strlen(log_signal(RTLIL::SigSpec(wire)))), "", - signal_list[id1].bit_str, signal_list[id2].bit_str); + run_abc.signal_list[id1].bit_str, run_abc.signal_list[id2].bit_str); first_line = false; } int id3 = map_signal(assign_map, RTLIL::SigSpec(wire)); - signal_list[id1].is_port = true; - signal_list[id3].is_port = true; + run_abc.signal_list[id1].is_port = true; + run_abc.signal_list[id3].is_port = true; log_assert(id3 == int(in_edges_count.size())); in_edges_count.push_back(0); workpool.insert(id3); for (int id2 : edges[id1]) { - if (signal_list[id2].in1 == id1) - signal_list[id2].in1 = id3; - if (signal_list[id2].in2 == id1) - signal_list[id2].in2 = id3; - if (signal_list[id2].in3 == id1) - signal_list[id2].in3 = id3; - if (signal_list[id2].in4 == id1) - signal_list[id2].in4 = id3; + if (run_abc.signal_list[id2].in1 == id1) + run_abc.signal_list[id2].in1 = id3; + if (run_abc.signal_list[id2].in2 == id1) + run_abc.signal_list[id2].in2 = id3; + if (run_abc.signal_list[id2].in3 == id1) + run_abc.signal_list[id2].in3 = id3; + if (run_abc.signal_list[id2].in4 == id1) + run_abc.signal_list[id2].in4 = id3; } edges[id1].swap(edges[id3]); @@ -724,14 +736,14 @@ std::string replace_tempdir(std::string text, std::string tempdir_name, bool sho struct abc_output_filter { - const AbcModuleState &state; + RunAbcState &state; bool got_cr; int escape_seq_state; std::string linebuf; std::string tempdir_name; bool show_tempdir; - abc_output_filter(const AbcModuleState& state, std::string tempdir_name, bool show_tempdir) + abc_output_filter(RunAbcState& state, std::string tempdir_name, bool show_tempdir) : state(state), tempdir_name(tempdir_name), show_tempdir(show_tempdir) { got_cr = false; @@ -759,7 +771,7 @@ struct abc_output_filter return; } if (ch == '\n') { - log("ABC: %s\n", replace_tempdir(linebuf, tempdir_name, show_tempdir)); + state.logs.log("ABC: %s\n", replace_tempdir(linebuf, tempdir_name, show_tempdir)); got_cr = false, linebuf.clear(); return; } @@ -772,7 +784,7 @@ struct abc_output_filter { int pi, po; if (sscanf(line.c_str(), "Start-point = pi%d. End-point = po%d.", &pi, &po) == 2) { - log("ABC: Start-point = pi%d (%s). End-point = po%d (%s).\n", + state.logs.log("ABC: Start-point = pi%d (%s). End-point = po%d (%s).\n", pi, state.pi_map.count(pi) ? state.pi_map.at(pi).c_str() : "???", po, state.po_map.count(po) ? state.po_map.at(po).c_str() : "???"); return; @@ -858,16 +870,17 @@ void AbcModuleState::prepare_module(RTLIL::Design *design, RTLIL::Module *module if (dff_mode && clk_sig.empty()) log_cmd_error("Clock domain %s not found.\n", clk_str.c_str()); + const AbcConfig &config = run_abc.config; if (config.cleanup) - tempdir_name = get_base_tmpdir() + "/"; + run_abc.tempdir_name = get_base_tmpdir() + "/"; else - tempdir_name = "_tmp_"; - tempdir_name += proc_program_prefix() + "yosys-abc-XXXXXX"; - tempdir_name = make_temp_dir(tempdir_name); + run_abc.tempdir_name = "_tmp_"; + run_abc.tempdir_name += proc_program_prefix() + "yosys-abc-XXXXXX"; + run_abc.tempdir_name = make_temp_dir(run_abc.tempdir_name); log_header(design, "Extracting gate netlist of module `%s' to `%s/input.blif'..\n", - module->name.c_str(), replace_tempdir(tempdir_name, tempdir_name, config.show_tempdir).c_str()); + module->name.c_str(), replace_tempdir(run_abc.tempdir_name, run_abc.tempdir_name, config.show_tempdir).c_str()); - std::string abc_script = stringf("read_blif \"%s/input.blif\"; ", tempdir_name); + std::string abc_script = stringf("read_blif \"%s/input.blif\"; ", run_abc.tempdir_name); if (!config.liberty_files.empty() || !config.genlib_files.empty()) { std::string dont_use_args; @@ -933,15 +946,15 @@ void AbcModuleState::prepare_module(RTLIL::Design *design, RTLIL::Module *module for (size_t pos = abc_script.find("{S}"); pos != std::string::npos; pos = abc_script.find("{S}", pos)) abc_script = abc_script.substr(0, pos) + config.lutin_shared + abc_script.substr(pos+3); if (config.abc_dress) - abc_script += stringf("; dress \"%s/input.blif\"", tempdir_name); - abc_script += stringf("; write_blif %s/output.blif", tempdir_name); + abc_script += stringf("; dress \"%s/input.blif\"", run_abc.tempdir_name); + abc_script += stringf("; write_blif %s/output.blif", run_abc.tempdir_name); abc_script = add_echos_to_abc_cmd(abc_script); for (size_t i = 0; i+1 < abc_script.size(); i++) if (abc_script[i] == ';' && abc_script[i+1] == ' ') abc_script[i+1] = '\n'; - std::string buffer = stringf("%s/abc.script", tempdir_name); + std::string buffer = stringf("%s/abc.script", run_abc.tempdir_name); FILE *f = fopen(buffer.c_str(), "wt"); if (f == nullptr) log_error("Opening %s for writing failed: %s\n", buffer, strerror(errno)); @@ -997,12 +1010,15 @@ void AbcModuleState::prepare_module(RTLIL::Design *design, RTLIL::Module *module handle_loops(assign_map, module); } -void AbcModuleState::run_abc() +void RunAbcState::run() { std::string buffer = stringf("%s/input.blif", tempdir_name); FILE *f = fopen(buffer.c_str(), "wt"); - if (f == nullptr) - log_error("Opening %s for writing failed: %s\n", buffer, strerror(errno)); + if (f == nullptr) { + logs.log("Opening %s for writing failed: %s\n", buffer, strerror(errno)); + err = true; + return; + } fprintf(f, ".model netlist\n"); @@ -1117,13 +1133,14 @@ void AbcModuleState::run_abc() fprintf(f, ".end\n"); fclose(f); - log("Extracted %d gates and %d wires to a netlist network with %d inputs and %d outputs.\n", + logs.log("Extracted %d gates and %d wires to a netlist network with %d inputs and %d outputs.\n", count_gates, GetSize(signal_list), count_input, count_output); if (count_output > 0) { buffer = stringf("\"%s\" -s -f %s/abc.script 2>&1", config.exe_file, tempdir_name); - log("Running ABC command: %s\n", replace_tempdir(buffer, tempdir_name, config.show_tempdir)); + logs.log("Running ABC command: %s\n", replace_tempdir(buffer, tempdir_name, config.show_tempdir)); + errno = 0; #ifndef YOSYS_LINK_ABC abc_output_filter filt(*this, tempdir_name, config.show_tempdir); int ret = run_command(buffer, std::bind(&abc_output_filter::next_line, filt, std::placeholders::_1)); @@ -1172,10 +1189,10 @@ void AbcModuleState::run_abc() temp_stdouterr_r.close(); #endif if (ret != 0) { - log_error("ABC: execution of command \"%s\" failed: return code %d.\n", buffer, ret); + logs.log_error("ABC: execution of command \"%s\" failed: return code %d (errno=%d).\n", buffer, ret, errno); return; } - did_run_abc = true; + did_run = true; return; } log("Don't call ABC as there is nothing to map.\n"); @@ -1242,19 +1259,23 @@ void emit_global_input_files(const AbcConfig &config) void AbcModuleState::extract(AbcSigMap &assign_map, RTLIL::Design *design, RTLIL::Module *module) { - if (!did_run_abc) { + log_push(); + log_header(design, "Executed ABC.\n"); + run_abc.logs.flush(); + if (!run_abc.did_run) { + finish(); return; } - std::string buffer = stringf("%s/%s", tempdir_name, "output.blif"); + std::string buffer = stringf("%s/%s", run_abc.tempdir_name, "output.blif"); std::ifstream ifs; ifs.open(buffer); if (ifs.fail()) log_error("Can't open ABC output file `%s'.\n", buffer); - bool builtin_lib = config.liberty_files.empty() && config.genlib_files.empty(); + bool builtin_lib = run_abc.config.liberty_files.empty() && run_abc.config.genlib_files.empty(); RTLIL::Design *mapped_design = new RTLIL::Design; - parse_blif(mapped_design, ifs, builtin_lib ? ID(DFF) : ID(_dff_), false, config.sop_mode); + parse_blif(mapped_design, ifs, builtin_lib ? ID(DFF) : ID(_dff_), false, run_abc.config.sop_mode); ifs.close(); @@ -1503,7 +1524,7 @@ void AbcModuleState::extract(AbcSigMap &assign_map, RTLIL::Design *design, RTLIL for (auto &it : cell_stats) log("ABC RESULTS: %15s cells: %8d\n", it.first, it.second); int in_wires = 0, out_wires = 0; - for (auto &si : signal_list) + for (auto &si : run_abc.signal_list) if (si.is_port) { char buffer[100]; snprintf(buffer, 100, "\\ys__n%d", si.id); @@ -1519,20 +1540,22 @@ void AbcModuleState::extract(AbcSigMap &assign_map, RTLIL::Design *design, RTLIL } connect(assign_map, module, conn); } - log("ABC RESULTS: internal signals: %8d\n", int(signal_list.size()) - in_wires - out_wires); + log("ABC RESULTS: internal signals: %8d\n", int(run_abc.signal_list.size()) - in_wires - out_wires); log("ABC RESULTS: input signals: %8d\n", in_wires); log("ABC RESULTS: output signals: %8d\n", out_wires); delete mapped_design; + finish(); } void AbcModuleState::finish() { - if (config.cleanup) + if (run_abc.config.cleanup) { log("Removing temp directory.\n"); - remove_directory(tempdir_name); + remove_directory(run_abc.tempdir_name); } + log_pop(); } // For every signal that connects cells from different sets, or a cell in a set to a cell not in any set, @@ -2182,12 +2205,8 @@ struct AbcPass : public Pass { AbcModuleState state(config, initvals); state.prepare_module(design, mod, assign_map, cells, dff_mode, clk_str); - log_push(); - log_header(design, "Executing ABC.\n"); - state.run_abc(); + state.run_abc.run(); state.extract(assign_map, design, mod); - state.finish(); - log_pop(); continue; } @@ -2332,36 +2351,74 @@ struct AbcPass : public Pass { } log_header(design, "Summary of detected clock domains:\n"); - for (auto &it : assigned_cells) - log(" %d cells in clk=%s%s, en=%s%s, arst=%s%s, srst=%s%s\n", GetSize(it.second), - std::get<0>(it.first) ? "" : "!", log_signal(std::get<1>(it.first)), - std::get<2>(it.first) ? "" : "!", log_signal(std::get<3>(it.first)), - std::get<4>(it.first) ? "" : "!", log_signal(std::get<5>(it.first)), - std::get<6>(it.first) ? "" : "!", log_signal(std::get<7>(it.first))); - { std::vector*> cell_sets; - for (auto &it : assigned_cells) + for (auto &it : assigned_cells) { + log(" %d cells in clk=%s%s, en=%s%s, arst=%s%s, srst=%s%s\n", GetSize(it.second), + std::get<0>(it.first) ? "" : "!", log_signal(std::get<1>(it.first)), + std::get<2>(it.first) ? "" : "!", log_signal(std::get<3>(it.first)), + std::get<4>(it.first) ? "" : "!", log_signal(std::get<5>(it.first)), + std::get<6>(it.first) ? "" : "!", log_signal(std::get<7>(it.first))); cell_sets.push_back(&it.second); + } assign_cell_connection_ports(mod, cell_sets, assign_map); } + + // Reserve one core for our main thread, and don't create more worker threads + // than ABC runs. + int max_threads = assigned_cells.size(); + if (max_threads <= 1) { + // Just do everything on the main thread. + max_threads = 0; + } +#ifdef YOSYS_LINK_ABC + // ABC does't support multithreaded calls so don't call it off the main thread. + max_threads = 0; +#endif + int num_worker_threads = ThreadPool::pool_size(1, max_threads); + ConcurrentQueue> work_queue(num_worker_threads); + ConcurrentQueue> work_finished_queue; + int work_finished_count = 0; + ThreadPool worker_threads(num_worker_threads, [&](int){ + while (std::optional> work = + work_queue.pop_front()) { + // Only the `run_abc` component is safe to touch here! + (*work)->run_abc.run(); + work_finished_queue.push_back(std::move(*work)); + } + }); for (auto &it : assigned_cells) { - AbcModuleState state(config, initvals); - state.clk_polarity = std::get<0>(it.first); - state.clk_sig = assign_map(std::get<1>(it.first)); - state.en_polarity = std::get<2>(it.first); - state.en_sig = assign_map(std::get<3>(it.first)); - state.arst_polarity = std::get<4>(it.first); - state.arst_sig = assign_map(std::get<5>(it.first)); - state.srst_polarity = std::get<6>(it.first); - state.srst_sig = assign_map(std::get<7>(it.first)); - state.prepare_module(design, mod, assign_map, it.second, !state.clk_sig.empty(), "$"); - log_push(); - log_header(design, "Executing ABC.\n"); - state.run_abc(); - state.extract(assign_map, design, mod); - state.finish(); - log_pop(); + // Process ABC results that have already finished before queueing another ABC. + // This should keep our memory usage down. + while (std::optional> work = + work_finished_queue.try_pop_front()) { + (*work)->extract(assign_map, design, mod); + ++work_finished_count; + } + std::unique_ptr state = std::make_unique(config, initvals); + state->clk_polarity = std::get<0>(it.first); + state->clk_sig = assign_map(std::get<1>(it.first)); + state->en_polarity = std::get<2>(it.first); + state->en_sig = assign_map(std::get<3>(it.first)); + state->arst_polarity = std::get<4>(it.first); + state->arst_sig = assign_map(std::get<5>(it.first)); + state->srst_polarity = std::get<6>(it.first); + state->srst_sig = assign_map(std::get<7>(it.first)); + state->prepare_module(design, mod, assign_map, it.second, !state->clk_sig.empty(), "$"); + if (num_worker_threads > 0) { + work_queue.push_back(std::move(state)); + } else { + // Just run everything on the main thread. + state->run_abc.run(); + work_finished_queue.push_back(std::move(state)); + } + } + work_queue.close(); + while (work_finished_count < static_cast(assigned_cells.size())) { + std::optional> work = + work_finished_queue.pop_front(); + (*work)->extract(assign_map, design, mod); + ++work_finished_count; } } From ae0ca7578a4450347dc7a545073315f72eaec19a Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Fri, 8 Aug 2025 05:26:30 +0000 Subject: [PATCH 5/5] Use a pool of ABC processes. Doing ABC runs in parallel can actually make things slower when every ABC run requires spawning an ABC subprocess --- especially when using popen(), which on glibc does not use vfork(). What seems to happen is that constant fork()ing keeps making the main process data pages copy-on-write, so the main process code that is setting up each ABC call takes a lot of minor page-faults, slowing it down. The solution is pretty straightforward although a little tricky to implement. We just reuse ABC subprocesses. Instead of passing the ABC script name on the command line, we spawn an ABC REPL and pipe a command into it to source the script. When that's done we echo an `ABC_DONE` token instead of exiting. Yosys then puts the ABC process onto a stack which we can pull from the next time we do an ABC run. For one of our large designs, this is an additional 5x speedup of the primary AbcPass. It does 5155 ABC runs, all very small; runtime of the AbcPass goes from 760s to 149s (not very scientific benchmarking but the effect size is large). --- kernel/threading.h | 27 ++++++ passes/techmap/abc.cc | 207 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 221 insertions(+), 13 deletions(-) diff --git a/kernel/threading.h b/kernel/threading.h index 8c08d670cad..c34abf8507d 100644 --- a/kernel/threading.h +++ b/kernel/threading.h @@ -154,6 +154,33 @@ class ThreadPool #endif }; +template +class ConcurrentStack +{ +public: + void push_back(T &&t) { +#ifdef YOSYS_ENABLE_THREADS + std::lock_guard lock(mutex); +#endif + contents.push_back(std::move(t)); + } + std::optional try_pop_back() { +#ifdef YOSYS_ENABLE_THREADS + std::lock_guard lock(mutex); +#endif + if (contents.empty()) + return std::nullopt; + T result = std::move(contents.back()); + contents.pop_back(); + return result; + } +private: +#ifdef YOSYS_ENABLE_THREADS + std::mutex mutex; +#endif + std::vector contents; +}; + YOSYS_NAMESPACE_END #endif // YOSYS_THREADING_H diff --git a/passes/techmap/abc.cc b/passes/techmap/abc.cc index 4a65671a28b..82a5124ae1d 100644 --- a/passes/techmap/abc.cc +++ b/passes/techmap/abc.cc @@ -59,6 +59,11 @@ #include #include +#ifdef __linux__ +# include +# include +# include +#endif #ifndef _WIN32 # include # include @@ -153,6 +158,121 @@ struct AbcSigVal { } }; +#if defined(__linux__) && !defined(YOSYS_DISABLE_SPAWN) +struct AbcProcess +{ + pid_t pid; + int to_child_pipe; + int from_child_pipe; + + AbcProcess() : pid(0), to_child_pipe(-1), from_child_pipe(-1) {} + AbcProcess(AbcProcess &&other) { + pid = other.pid; + to_child_pipe = other.to_child_pipe; + from_child_pipe = other.from_child_pipe; + other.pid = 0; + other.to_child_pipe = other.from_child_pipe = -1; + } + AbcProcess &operator=(AbcProcess &&other) { + if (this != &other) { + pid = other.pid; + to_child_pipe = other.to_child_pipe; + from_child_pipe = other.from_child_pipe; + other.pid = 0; + other.to_child_pipe = other.from_child_pipe = -1; + } + return *this; + } + ~AbcProcess() { + if (pid == 0) + return; + if (to_child_pipe >= 0) + close(to_child_pipe); + int status; + int ret = waitpid(pid, &status, 0); + if (ret != pid) { + log_error("waitpid(%d) failed", pid); + } + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + log_error("ABC failed with status %X", status); + } + if (from_child_pipe >= 0) + close(from_child_pipe); + } +}; + +std::optional spawn_abc(const char* abc_exe, DeferredLogs &logs) { + // Open pipes O_CLOEXEC so we don't leak any of the fds into racing + // fork()s. + int to_child_pipe[2]; + if (pipe2(to_child_pipe, O_CLOEXEC) != 0) { + logs.log_error("pipe failed"); + return std::nullopt; + } + int from_child_pipe[2]; + if (pipe2(from_child_pipe, O_CLOEXEC) != 0) { + logs.log_error("pipe failed"); + return std::nullopt; + } + + AbcProcess result; + result.to_child_pipe = to_child_pipe[1]; + result.from_child_pipe = from_child_pipe[0]; + // Allow the child side of the pipes to be inherited. + fcntl(to_child_pipe[0], F_SETFD, 0); + fcntl(from_child_pipe[1], F_SETFD, 0); + + posix_spawn_file_actions_t file_actions; + if (posix_spawn_file_actions_init(&file_actions) != 0) { + logs.log_error("posix_spawn_file_actions_init failed"); + return std::nullopt; + } + + if (posix_spawn_file_actions_addclose(&file_actions, to_child_pipe[1]) != 0) { + logs.log_error("posix_spawn_file_actions_addclose failed"); + return std::nullopt; + } + if (posix_spawn_file_actions_addclose(&file_actions, from_child_pipe[0]) != 0) { + logs.log_error("posix_spawn_file_actions_addclose failed"); + return std::nullopt; + } + if (posix_spawn_file_actions_adddup2(&file_actions, to_child_pipe[0], STDIN_FILENO) != 0) { + logs.log_error("posix_spawn_file_actions_adddup2 failed"); + return std::nullopt; + } + if (posix_spawn_file_actions_adddup2(&file_actions, from_child_pipe[1], STDOUT_FILENO) != 0) { + logs.log_error("posix_spawn_file_actions_adddup2 failed"); + return std::nullopt; + } + if (posix_spawn_file_actions_adddup2(&file_actions, from_child_pipe[1], STDERR_FILENO) != 0) { + logs.log_error("posix_spawn_file_actions_adddup2 failed"); + return std::nullopt; + } + if (posix_spawn_file_actions_addclose(&file_actions, to_child_pipe[0]) != 0) { + logs.log_error("posix_spawn_file_actions_addclose failed"); + return std::nullopt; + } + if (posix_spawn_file_actions_addclose(&file_actions, from_child_pipe[1]) != 0) { + logs.log_error("posix_spawn_file_actions_addclose failed"); + return std::nullopt; + } + + char arg1[] = "-s"; + char* argv[] = { strdup(abc_exe), arg1, nullptr }; + if (0 != posix_spawn(&result.pid, abc_exe, &file_actions, nullptr, argv, environ)) { + logs.log_error("posix_spawn %s failed", abc_exe); + return std::nullopt; + } + free(argv[0]); + posix_spawn_file_actions_destroy(&file_actions); + close(to_child_pipe[0]); + close(from_child_pipe[1]); + return result; +} +#else +struct AbcProcess {}; +#endif + using AbcSigMap = SigValMap; // Used by off-main-threads. Contains no direct or indirect access to RTLIL. @@ -167,7 +287,7 @@ struct RunAbcState { dict pi_map, po_map; RunAbcState(const AbcConfig &config) : config(config) {} - void run(); + void run(ConcurrentStack &process_pool); }; struct AbcModuleState { @@ -1010,7 +1130,42 @@ void AbcModuleState::prepare_module(RTLIL::Design *design, RTLIL::Module *module handle_loops(assign_map, module); } -void RunAbcState::run() +bool read_until_abc_done(abc_output_filter &filt, int fd, DeferredLogs &logs) { + std::string line; + char buf[1024]; + while (true) { + int ret = read(fd, buf, sizeof(buf) - 1); + if (ret < 0) { + logs.log_error("Failed to read from ABC, errno=%d", errno); + return false; + } + if (ret == 0) { + logs.log_error("ABC exited prematurely"); + return false; + } + char *start = buf; + char *end = buf + ret; + while (start < end) { + char *p = static_cast(memchr(start, '\n', end - start)); + if (p == nullptr) { + break; + } + line.append(start, p + 1 - start); + // ABC seems to actually print "ABC_DONE \n", but we probably shouldn't + // rely on that extra space being output. + if (line.substr(0, 8) == "ABC_DONE") { + // Ignore any leftover output, there should only be a prompt perhaps + return true; + } + filt.next_line(line); + line.clear(); + start = p + 1; + } + line.append(start, end - start); + } +} + +void RunAbcState::run(ConcurrentStack &process_pool) { std::string buffer = stringf("%s/input.blif", tempdir_name); FILE *f = fopen(buffer.c_str(), "wt"); @@ -1137,14 +1292,12 @@ void RunAbcState::run() count_gates, GetSize(signal_list), count_input, count_output); if (count_output > 0) { - buffer = stringf("\"%s\" -s -f %s/abc.script 2>&1", config.exe_file, tempdir_name); - logs.log("Running ABC command: %s\n", replace_tempdir(buffer, tempdir_name, config.show_tempdir)); + std::string tmp_script_name = stringf("%s/abc.script", tempdir_name); + logs.log("Running ABC script: %s\n", replace_tempdir(tmp_script_name, tempdir_name, config.show_tempdir)); errno = 0; -#ifndef YOSYS_LINK_ABC abc_output_filter filt(*this, tempdir_name, config.show_tempdir); - int ret = run_command(buffer, std::bind(&abc_output_filter::next_line, filt, std::placeholders::_1)); -#else +#ifdef YOSYS_LINK_ABC string temp_stdouterr_name = stringf("%s/stdouterr.txt", tempdir_name); FILE *temp_stdouterr_w = fopen(temp_stdouterr_name.c_str(), "w"); if (temp_stdouterr_w == NULL) @@ -1165,7 +1318,6 @@ void RunAbcState::run() fclose(temp_stdouterr_w); // These needs to be mutable, supposedly due to getopt char *abc_argv[5]; - string tmp_script_name = stringf("%s/abc.script", tempdir_name); abc_argv[0] = strdup(config.exe_file.c_str()); abc_argv[1] = strdup("-s"); abc_argv[2] = strdup("-f"); @@ -1183,13 +1335,40 @@ void RunAbcState::run() fclose(old_stdout); fclose(old_stderr); std::ifstream temp_stdouterr_r(temp_stdouterr_name); - abc_output_filter filt(*this, tempdir_name, config.show_tempdir); for (std::string line; std::getline(temp_stdouterr_r, line); ) filt.next_line(line + "\n"); temp_stdouterr_r.close(); +#elif defined(__linux__) && !defined(YOSYS_DISABLE_SPAWN) + AbcProcess process; + if (std::optional process_opt = process_pool.try_pop_back()) { + process = std::move(process_opt.value()); + } else if (std::optional process_opt = spawn_abc(config.exe_file.c_str(), logs)) { + process = std::move(process_opt.value()); + } else { + return; + } + std::string cmd = stringf( + // This makes ABC switch stdout to line buffering, which we need + // to see our ABC_DONE message. + "set abcout /dev/stdout\n" + "empty\n" + "source %s\n" + "echo \"ABC_DONE\"\n", tmp_script_name); + int ret = write(process.to_child_pipe, cmd.c_str(), cmd.size()); + if (ret != static_cast(cmd.size())) { + logs.log_error("write failed"); + return; + } + ret = read_until_abc_done(filt, process.from_child_pipe, logs) ? 0 : 1; + if (ret == 0) { + process_pool.push_back(std::move(process)); + } +#else + std::string cmd = stringf("\"%s\" -s -f %s/abc.script 2>&1", config.exe_file.c_str(), tempdir_name.c_str()); + int ret = run_command(cmd, std::bind(&abc_output_filter::next_line, filt, std::placeholders::_1)); #endif if (ret != 0) { - logs.log_error("ABC: execution of command \"%s\" failed: return code %d (errno=%d).\n", buffer, ret, errno); + logs.log_error("ABC: execution of script \"%s\" failed: return code %d (errno=%d).\n", tmp_script_name, ret, errno); return; } did_run = true; @@ -2205,7 +2384,8 @@ struct AbcPass : public Pass { AbcModuleState state(config, initvals); state.prepare_module(design, mod, assign_map, cells, dff_mode, clk_str); - state.run_abc.run(); + ConcurrentStack process_pool; + state.run_abc.run(process_pool); state.extract(assign_map, design, mod); continue; } @@ -2379,11 +2559,12 @@ struct AbcPass : public Pass { ConcurrentQueue> work_queue(num_worker_threads); ConcurrentQueue> work_finished_queue; int work_finished_count = 0; + ConcurrentStack process_pool; ThreadPool worker_threads(num_worker_threads, [&](int){ while (std::optional> work = work_queue.pop_front()) { // Only the `run_abc` component is safe to touch here! - (*work)->run_abc.run(); + (*work)->run_abc.run(process_pool); work_finished_queue.push_back(std::move(*work)); } }); @@ -2409,7 +2590,7 @@ struct AbcPass : public Pass { work_queue.push_back(std::move(state)); } else { // Just run everything on the main thread. - state->run_abc.run(); + state->run_abc.run(process_pool); work_finished_queue.push_back(std::move(state)); } }