Support NBAs to arrays inside loops

For NBAs that might execute a dynamic number of times in a single
evaluation (specifically: those that assign to array elements inside
loops), we introduce a new run-time VlNBACommitQueue data-structure
(currently a vector), which stores all pending updates and the necessary
info to reconstruct the LHS reference of the AstAssignDly at run-time.

All variables needing a commit queue has their corresponding unique
commit queue.

All NBAs to a variable that requires a commit queue go through the
commit queue. This is necessary to preserve update order in sequential
code, e.g.:
 a[7] <= 10
 for (int i = 1 ; i < 10; ++i) a[i] <= i;
 a[2] <= 10
needs to end with array elements 1..9 being 1, 10, 3, 4, 5, 6, 7, 8, 9.

This enables supporting common forms of NBAs to arrays on the left hand
side of <= in non-suspendable/non-fork code. (Suspendable/fork
implementation is unclear to me so I left it unchanged, see verilator#5084).

Any NBA that does not need a commit queue (i.e.: those that were
supported before), use the same scheme as before, and this patch should
have no effect on the generated code for those NBAs.

include/verilated_types.h

@@ -415,8 +415,20 @@ class VlWriteMem final {
 
 static int _vl_cmp_w(int words, WDataInP const lwp, WDataInP const rwp) VL_PURE;
 
+template <std::size_t T_Words>
+struct VlWide;
+
+// Type trait to check if a type is VlWide
+template <typename>
+struct IsVlWide : public std::false_type {};
+
+template <std::size_t T_Words>
+struct IsVlWide<VlWide<T_Words>> : public std::true_type {};
+
 template <std::size_t T_Words>
 struct VlWide final {
+    static constexpr size_t Words = T_Words;
+
     // MEMBERS
     // This should be the only data member, otherwise generated static initializers need updating
     EData m_storage[T_Words];  // Contents of the packed array
@@ -1274,14 +1286,41 @@ void VL_WRITEMEM_N(bool hex, int bits, const std::string& filename,
 /// This class may get exposed to a Verilated Model's top I/O, if the top
 /// IO has an unpacked array.
 
-template <class T_Value, std::size_t T_Depth>
+template <typename T_Value, std::size_t T_Depth>
+struct VlUnpacked;
+
+// Type trait to check if a type is VlUnpacked
+template <typename>
+struct IsVlUnpacked : public std::false_type {};
+
+template <typename T_Value, std::size_t T_Depth>
+struct IsVlUnpacked<VlUnpacked<T_Value, T_Depth>> : public std::true_type {};
+
+template <typename T_Value, std::size_t T_Depth>
 struct VlUnpacked final {
 private:
     // TYPES
     using T_Key = IData;  // Index type, for uniformity with other containers
     using Unpacked = T_Value[T_Depth];
 
+    struct RankBaseCase final {
+        static constexpr size_t Rank = 0;
+    };
+
+    template <typename T>
+    struct BaseElementBaseCase final {
+        using BaseElement = T;
+    };
+
 public:
+    // Rank of this array. E.g.: VlUnpacked<VlUnpacked<_: not VlUnpacked, _>, _> have rank '2'
+    static constexpr size_t Rank
+        = std::conditional<IsVlUnpacked<T_Value>::value, T_Value, RankBaseCase>::type::Rank + 1;
+
+    // Final non-array element type.
+    using BaseElement = typename std::conditional<IsVlUnpacked<T_Value>::value, T_Value,
+                                                  BaseElementBaseCase<T_Value>>::type::BaseElement;
+
     // MEMBERS
     // This should be the only data member, otherwise generated static initializers need updating
     Unpacked m_storage;  // Contents of the unpacked array
@@ -1511,6 +1550,154 @@ std::string VL_TO_STRING(const VlUnpacked<T_Value, T_Depth>& obj) {
     return obj.to_string();
 }
 
+//===================================================================
+// Helper to apply the given indices to a target expression
+
+template <size_t Curr, size_t Rank, typename T_Target>
+struct VlApplyIndices final {
+    VL_ATTR_ALWINLINE
+    static auto& apply(T_Target& target, const size_t* indicesp) {
+        return VlApplyIndices<Curr + 1, Rank, decltype(target[indicesp[Curr]])>::apply(
+            target[indicesp[Curr]], indicesp);
+    }
+};
+
+template <size_t Rank, typename T_Target>
+struct VlApplyIndices<Rank, Rank, T_Target> final {
+    VL_ATTR_ALWINLINE
+    static T_Target& apply(T_Target& target, const size_t*) { return target; }
+};
+
+//===================================================================
+// Commit queue for NBAs - currently only for unpacked arrays
+
+template <class T_Target, bool Partial>
+class VlNBACommitQueue;
+
+// Whole element updates only
+template <class T_Target>
+class VlNBACommitQueue<T_Target, /* Partial: */ false> final {
+    static_assert(IsVlUnpacked<T_Target>::value, "'VlNBACommitQueue' only supports 'VlUnpacked'");
+
+    static constexpr size_t Rank = T_Target::Rank;
+    using Element = typename T_Target::BaseElement;
+
+    struct Entry final {
+        Element value;
+        size_t indices[Rank];
+    };
+
+    std::vector<Entry> m_pending;  // Pending updates, in program order
+
+public:
+    VlNBACommitQueue() = default;
+    ~VlNBACommitQueue() = default;
+
+    template <typename... Args>
+    void enqueue(const Element& value, Args... indices) {
+        m_pending.emplace_back(Entry{value, {indices...}});
+    }
+
+    // Note: T_Commit might be different from T_Target. Specifically, when the signal is a
+    // top-level IO port, T_Commit will be a native C array, while T_Target, willbe a VlUnpacked
+    template <typename T_Commit>
+    void commit(T_Commit& target) {
+        if (m_pending.empty()) return;
+        for (const Entry& entry : m_pending) {
+            VlApplyIndices<0, Rank, T_Commit>::apply(target, entry.indices) = entry.value;
+        }
+        m_pending.clear();
+    }
+};
+
+// With partial element updates
+template <class T_Target>
+class VlNBACommitQueue<T_Target, /* Partial: */ true> final {
+    static_assert(IsVlUnpacked<T_Target>::value, "'VlNBACommitQueue' only supports 'VlUnpacked'");
+
+    static constexpr size_t Rank = T_Target::Rank;
+    using Element = typename T_Target::BaseElement;
+
+    // Queue entry when partial elements might need updating
+    struct Entry final {
+        Element value;
+        Element mask;
+        size_t indices[Rank];
+    };
+
+    std::vector<Entry> m_pending;  // Pending updates, in program order
+
+    // Binary & | ~ for elements to use for masking in partial updates. Sorry for the templates.
+    template <typename T_Elem>
+    VL_ATTR_ALWINLINE static typename std::enable_if<!IsVlWide<T_Elem>::value, T_Elem>::type
+    bAnd(const T_Elem& a, const T_Elem& b) {
+        return a & b;
+    }
+
+    template <typename T_Elem>
+    VL_ATTR_ALWINLINE static typename std::enable_if<IsVlWide<T_Elem>::value, T_Elem>::type
+    bAnd(const T_Elem& a, const T_Elem& b) {
+        T_Elem result;
+        for (size_t i = 0; i < T_Elem::Words; ++i) {
+            result.m_storage[i] = a.m_storage[i] & b.m_storage[i];
+        }
+        return result;
+    }
+
+    template <typename T_Elem>
+    VL_ATTR_ALWINLINE static typename std::enable_if<!IsVlWide<T_Elem>::value, T_Elem>::type
+    bOr(const T_Elem& a, const T_Elem& b) {
+        return a | b;
+    }
+
+    template <typename T_Elem>
+    VL_ATTR_ALWINLINE static typename std::enable_if<IsVlWide<T_Elem>::value, T_Elem>::type
+    bOr(const T_Elem& a, const T_Elem& b) {
+        T_Elem result;
+        for (size_t i = 0; i < T_Elem::Words; ++i) {
+            result.m_storage[i] = a.m_storage[i] | b.m_storage[i];
+        }
+        return result;
+    }
+
+    template <typename T_Elem>
+    VL_ATTR_ALWINLINE static typename std::enable_if<!IsVlWide<T_Elem>::value, T_Elem>::type
+    bNot(const T_Elem& a) {
+        return ~a;
+    }
+
+    template <typename T_Elem>
+    VL_ATTR_ALWINLINE static typename std::enable_if<IsVlWide<T_Elem>::value, T_Elem>::type
+    bNot(const T_Elem& a) {
+        T_Elem result;
+        for (size_t i = 0; i < T_Elem::Words; ++i) result.m_storage[i] = ~a.m_storage[i];
+        return result;
+    }
+
+public:
+    VlNBACommitQueue() = default;
+    ~VlNBACommitQueue() = default;
+
+    template <typename... Args>
+    void enqueue(const Element& value, const Element& mask, Args... indices) {
+        m_pending.emplace_back(Entry{value, mask, {indices...}});
+    }
+
+    // Note: T_Commit might be different from T_Target. Specifically, when the signal is a
+    // top-level IO port, T_Commit will be a native C array, while T_Target, willbe a VlUnpacked
+    template <typename T_Commit>
+    void commit(T_Commit& target) {
+        if (m_pending.empty()) return;
+        for (const Entry& entry : m_pending) {  //
+            auto& ref = VlApplyIndices<0, Rank, T_Commit>::apply(target, entry.indices);
+            // Maybe inefficient, but it works for now ...
+            const auto oldValue = ref;
+            ref = bOr(bAnd(entry.value, entry.mask), bAnd(oldValue, bNot(entry.mask)));
+        }
+        m_pending.clear();
+    }
+};
+
 //===================================================================
 // Object that VlDeleter is capable of deleting
 

src/V3AstNodeDType.h

@@ -959,6 +959,30 @@ class AstMemberDType final : public AstNodeDType {
         return false;
     }
 };
+class AstNBACommitQueueDType final : public AstNodeDType {
+    // @astgen ptr := m_targetDTypep : AstNodeDType  // Type of the corresponding variable
+    const bool m_partial;  // Partial element update required
+
+public:
+    AstNBACommitQueueDType(FileLine* fl, AstNodeDType* targetDTypep, bool partial)
+        : ASTGEN_SUPER_NBACommitQueueDType(fl)
+        , m_partial{partial}
+        , m_targetDTypep{targetDTypep} {
+        dtypep(this);
+    }
+    ASTGEN_MEMBERS_AstNBACommitQueueDType;
+
+    AstNodeDType* targetDTypep() const { return m_targetDTypep; }
+    bool partial() const { return m_partial; }
+    bool similarDType(const AstNodeDType* samep) const override { return this == samep; }
+    AstBasicDType* basicp() const override { return nullptr; }
+    AstNodeDType* skipRefp() const override { return (AstNodeDType*)this; }
+    AstNodeDType* skipRefToConstp() const override { return (AstNodeDType*)this; }
+    AstNodeDType* skipRefToEnump() const override { return (AstNodeDType*)this; }
+    int widthAlignBytes() const override { return 1; }
+    int widthTotalBytes() const override { return 24; }
+    bool isCompound() const override { return true; }
+};
 class AstParamTypeDType final : public AstNodeDType {
     // Parents: MODULE
     // A parameter type statement; much like a var or typedef

src/V3AstNodes.cpp

@@ -788,6 +788,12 @@ AstNodeDType::CTypeRecursed AstNodeDType::cTypeRecurse(bool compound, bool packe
         info.m_type = "VlUnpacked<" + sub.m_type;
         info.m_type += ", " + cvtToStr(adtypep->declRange().elements());
         info.m_type += ">";
+    } else if (const auto* const adtypep = VN_CAST(dtypep, NBACommitQueueDType)) {
+        UASSERT_OBJ(!packed, this, "Unsupported type for packed struct or union");
+        compound = true;
+        const CTypeRecursed sub = adtypep->targetDTypep()->cTypeRecurse(compound, false);
+        const char* const partialp = adtypep->partial() ? "true" : "false";
+        info.m_type = "VlNBACommitQueue<" + sub.m_type + ", " + partialp + ">";
     } else if (packed && (VN_IS(dtypep, PackArrayDType))) {
         const AstPackArrayDType* const adtypep = VN_CAST(dtypep, PackArrayDType);
         const CTypeRecursed sub = adtypep->subDTypep()->cTypeRecurse(false, true);
@@ -2683,6 +2689,7 @@ void AstCMethodHard::setPurity() {
                                                           {"commit", false},
                                                           {"delay", false},
                                                           {"done", false},
+                                                          {"enqueue", false},
                                                           {"erase", false},
                                                           {"evaluate", false},
                                                           {"evaluation", false},