Add JSON AST dumps

Makefile.in

@@ -410,6 +410,7 @@ PY_PROGRAMS = \
 	src/cppcheck_filtered \
 	src/flexfix \
 	src/vlcovgen \
+	src/.gdbinit.py \
 	test_regress/t/*.pf \
 	nodist/clang_check_attributes \
 	nodist/code_coverage \

docs/CONTRIBUTORS

@@ -157,6 +157,7 @@ Srinivasan Venkataramanan
 Stefan Wallentowitz
 Stephen Henry
 Steven Hugg
+Szymon Gizler
 Sören Tempel
 Teng Huang
 Tim Hutt

docs/internals.rst

@@ -1696,6 +1696,76 @@ Similarly, the ``NETLIST`` has a list of modules referred to by its
 ``op1p()`` pointer.
 
 
+.tree.json Output
+-----------------
+
+``.tree.json``` is alternative dump format (enabled with ``--dump-tree-json``)
+that is meant for programatic processing (see e.g. `verilator_jsontree` tool)
+
+Structure (note: plain output is unformatted):
+::
+
+  {
+    /* here go atributes that are common to all types of nodes */
+    "type": "VAR",
+    "name": "out_wide",
+    "addr": "0x91a780",
+    "file": "t_do_while.v:18:4", /* <filename>:<linenum>:<colnum> */
+    "editNum": 634,
+    /* here go fields that are specific to VAR nodes: */
+    "isSc": false,
+    "ioDirection": "NONE",
+    "isConst": false,
+    "isPullup": false,
+    "isPulldown": false,
+    "isUsedClock": false,
+    "isSigPublic": false,
+    "isLatched": false,
+    "isUsedLoopIdx": false,
+    "noReset": false,
+    "attrIsolateAssign": false,
+    "attrFileDescr": false,
+    "isDpiOpenArray": false,
+    "isFuncReturn": false,
+    "isFuncLocal": false,
+    "attrClocker": "UNKNOWN",
+    "lifetime": "NONE",
+    "varType": "VAR",
+    /* here go lists of child nodes (which use similar structure as their parrent): */
+    op1: [ /* ... */ ],
+    op2: [ /* ... */ ],
+    op3: [ /* ... */ ],
+    op4: [ /* ... */ ]
+  }
+
+verilator_jsontree
+------------------
+``verilator_jsontree`` is tool that pretty prints, filters and diffs ``.tree.json`` files.
+The "pretty-print format" looks like this
+
+Structure:
+::
+
+  ...
+  MODULE "t" t_do_while.v:16:8 0x558a88b8ae90 <e1152> timeunit:1ps
+    op2:
+      VAR "a" t_do_while.v:17:8 0x558a88b8b370 <e633> attrClocker:UNKNOWN, ioDirection:NONE, lifetime:NONE, varType:VAR
+       op1:
+         BASICDTYPE "int" t_do_while.v:17:4 0x558a88b8b2a0 <e634> keyword:int, range:31:0
+      INITIAL "" t_do_while.v:18:4 0x558a88b96420 <e948>
+      ...
+
+* Order of fields: ``type``, ``name``, ``file``, ``addr``, ``editNum``, <node-specific-fields>, ``op1``, ``op2``, ``op3``, ``op4``.
+* For the sake of brevity, names of common fields (type, name, file, addr and editNum) are omitted.
+* Unless ``--verbose`` is used, node-specific fields with the value of false are omitted.
+* If there are no children in a given ``op*`` array then it is skipped.
+* ``name`` is often an empty string, so it is enclosed in quotes to make this case apparent.
+* Unwanted fields can be filtered out using the ``-d``` flag, for example ``-d '.file, .editNum'``.
+* In diffs, removed/added lines are colored, and replacements of inline fields are signaled with the ``->`` marker.
+* Unless ``--verbose`` is used, chunks of unmodified nodes are replaced with a ``...`` in diff
+
+To use jsontree, ``jq`` (a CLI utility) and ``DeepDiff`` (a Python library) have to be installed.
+
 .tree.dot Output
 ----------------
 
@@ -1768,6 +1838,18 @@ To print a node:
    pnt nodep
    # or: call dumpTreeGdb(nodep)  # aliased to "pnt" in src/.gdbinit
 
+``src/.gdbinit`` and ``src/.gdbinit.py`` define handy utils for working with
+JSON dumps. For example
+
+* ``jstash nodep`` - Do an unformatted JSON dump and save it into value history (e.g. ``$1``)
+* ``jtree nodep`` - Do a JSON dump and pretty print it using ``verilator_jsontree``.
+* ``jtree $1`` - Pretty print a dump that was previously saved by ``jstash``.
+* ``jtree nodep -d '.file, .timeunit'`` - Do a JSON dump, filter out some fields and pretty print it.
+* ``jtree 0x55555613dca0`` - Pretty print using address literal (rather than actual pointer).
+* ``jtree $1 nodep`` - Diff ``nodep`` against its older dump.
+
+A detailed description of ``jstash`` and ``jtree`` can be displayed using ``gdb``'s ``help`` command.
+
 When GDB halts, it is useful to understand that the backtrace will commonly
 show the iterator functions between each invocation of ``visit`` in the
 backtrace. You will typically see a frame sequence something like:

src/.gdbinit

@@ -20,6 +20,26 @@ document pnt
   Verilator: Print AstNode NODEP's tree
 end
 
+# source python-based gdb config with jshow/jdiff definitions
+# (we store it in separate file, so it can be highlighted/linted/formatted as python)
+python
+import os
+if "VERILATOR_ROOT" in os.environ:
+  gdbinit_py = os.environ["VERILATOR_ROOT"] + "/src/.gdbinit.py"
+  gdb.execute("source" + gdbinit_py)
+end
+
+define jstash
+  call (char*) &(AstNode::dumpJsonTreeGdb($arg0)[0])
+end
+document jstash
+Verilator: Do json dump of the given nodep and save it in value history (e.g. $1) for later
+inspection using jtree. nodep can be actual pointer or adress literal (like 0x55555613dca0).
+end
+
+alias -a js=jstash
+alias -a jt=jtree
+
 define dtf
   call AstNode::dumpTreeFileGdb($arg0, 0)
 end

src/.gdbinit.py

@@ -0,0 +1,68 @@
+# pylint: disable=line-too-long,invalid-name,multiple-statements,missing-function-docstring,missing-class-docstring,missing-module-docstring,no-else-return,too-few-public-methods,unused-argument
+import os
+import sys
+import tempfile
+
+import gdb  # pylint: disable=import-error
+
+# add dir with verilator_jsontree to import path
+sys.path.append(os.environ["VERILATOR_ROOT"] + "/bin")
+
+
+def _get_dump(node):
+    return gdb.execute(f'printf "%s", &(AstNode::dumpJsonTreeGdb({node})[0])',
+                       to_string=True)
+
+
+def _tmpfile():
+    return tempfile.NamedTemporaryFile(mode="wt")  # write, text mode
+
+
+def _fwrite(file, s):
+    """write to file and flush buf to make sure that data is written before passing file to jsontree"""
+    file.write(s)
+    file.flush()
+
+
+class JsonTreeCmd(gdb.Command):
+    """Verilator: Pretty print or diff node(s) using jsontree. Node is allowed to be:
+    * an actual pointer,
+    * an address literal (like 0x55555613dca0),
+    * a gdb value (like $1) that stores a dump previously done by the jstash command.
+    Besides not taking input from file, it works exactly like `verilator_jsontree`:
+    * passing one node gives you a pretty print,
+    * passing two nodes gives you a diff,
+    * flags like -d work like expected.
+    """
+
+    def __init__(self):
+        super().__init__("jtree", gdb.COMMAND_USER, gdb.COMPLETE_EXPRESSION)
+
+    def _null_check(self, old, new):
+        err = ""
+        if old == "<nullptr>\n": err += "old == <nullptr>\n"
+        if new == "<nullptr>\n": err += "new == <nullptr>"
+        if err: raise gdb.GdbError(err.strip("\n"))
+
+    def invoke(self, arg_str, from_tty):
+        import verilator_jsontree as jsontree  # pylint: disable=wrong-import-position
+        # jsontree import may fail so we do it here rather than in outer scope
+
+        # We abuse verilator_jsontree's arg parser to find arguments with nodes
+        # After finding them, we replace them with proper files
+        jsontree_args = jsontree.parser.parse_args(gdb.string_to_argv(arg_str))
+        self._null_check(jsontree_args.file, jsontree_args.newfile)
+        with _tmpfile() as oldfile, _tmpfile() as newfile:
+            if jsontree_args.file:
+                _fwrite(oldfile, _get_dump(jsontree_args.file))
+                jsontree_args.file = oldfile.name
+            if jsontree_args.newfile:
+                _fwrite(newfile, _get_dump(jsontree_args.newfile))
+                jsontree_args.newfile = newfile.name
+            try:
+                jsontree.main(jsontree_args)
+            except SystemExit:  # jsontree prints nice errmsgs on exit(), so we just catch it to suppress cryptic python trace
+                return
+
+
+JsonTreeCmd()

src/V3Ast.cpp

@@ -22,6 +22,7 @@
 
 #include <iomanip>
 #include <memory>
+#include <sstream>
 
 VL_DEFINE_DEBUG_FUNCTIONS;
 
@@ -1170,6 +1171,22 @@ void AstNode::checkTreeIter(const AstNode* prevBackp) const VL_MT_STABLE {
     }
 }
 
+// cppcheck-suppress unusedFunction  // Debug only
+std::string AstNode::dumpJsonTreeGdb(const AstNode* nodep) {
+    if (!nodep) return "<nullptr>\n";
+    std::stringstream nodepStream;
+    nodep->dumpTreeJson(nodepStream);
+    return nodepStream.rdbuf()->str();
+}
+// cppcheck-suppress unusedFunction  // Debug only
+// identity func to allow for passing already done dumps to jtree
+const char* AstNode::dumpJsonTreeGdb(const char* str) { return str; }
+// cppcheck-suppress unusedFunction  // Debug only
+// allow for passing pointer literals like 0x42.. without manual cast
+std::string AstNode::dumpJsonTreeGdb(intptr_t nodep) {
+    if (!nodep) return "<nullptr>\n";
+    return dumpJsonTreeGdb((const AstNode*)nodep);
+}
 // cppcheck-suppress unusedFunction  // Debug only
 void AstNode::dumpGdb(const AstNode* nodep) {  // For GDB only  // LCOV_EXCL_LINE
     if (!nodep) {
@@ -1328,6 +1345,15 @@ void AstNode::dumpTreeDot(std::ostream& os) const {
     drawChildren(os, this, m_op4p, "op4");
 }
 
+void AstNode::dumpTreeJsonFile(const string& filename, bool append, bool doDump) {
+    if (!doDump) return;
+    UINFO(2, "Dumping " << filename << endl);
+    const std::unique_ptr<std::ofstream> treejsonp{V3File::new_ofstream(filename, append)};
+    if (treejsonp->fail()) v3fatal("Can't write " << filename);
+    dumpTreeJson(*treejsonp);
+    *treejsonp << "\n";
+}
+
 void AstNode::dumpTreeDotFile(const string& filename, bool append, bool doDump) {
     if (doDump) {
         UINFO(2, "Dumping " << filename << endl);

src/V3Ast.h

@@ -31,6 +31,7 @@
 #include "V3Ast__gen_forward_class_decls.h"  // From ./astgen
 
 #include <cmath>
+#include <cstdint>
 #include <functional>
 #include <map>
 #include <set>
@@ -2161,6 +2162,9 @@ class AstNode VL_NOT_FINAL {
     string warnOther() const VL_REQUIRES(V3Error::s().m_mutex) { return fileline()->warnOther(); }
 
     virtual void dump(std::ostream& str = std::cout) const;
+    static std::string dumpJsonTreeGdb(const AstNode* nodep);  // For GDB only
+    static const char* dumpJsonTreeGdb(const char* str);  // For GDB only
+    static std::string dumpJsonTreeGdb(intptr_t nodep);  // For GDB only
     static void dumpGdb(const AstNode* nodep);  // For GDB only
     void dumpGdbHeader() const;
 
@@ -2216,6 +2220,9 @@ class AstNode VL_NOT_FINAL {
     static void dumpTreeFileGdb(const AstNode* nodep, const char* filenamep = nullptr);
     void dumpTreeDot(std::ostream& os = std::cout) const;
     void dumpTreeDotFile(const string& filename, bool append = false, bool doDump = true);
+    virtual void dumpExtraJson(std::ostream& os) const {};  // node specific fields
+    void dumpTreeJson(std::ostream& os) const;
+    void dumpTreeJsonFile(const string& filename, bool append = false, bool doDump = true);
 
     // METHODS - static advancement
     static AstNode* afterCommentp(AstNode* nodep) {

src/V3AstNodeDType.h

@@ -52,6 +52,7 @@ class AstNodeDType VL_NOT_FINAL : public AstNode {
     ASTGEN_MEMBERS_AstNodeDType;
     // ACCESSORS
     void dump(std::ostream& str) const override;
+    void dumpExtraJson(std::ostream& str) const override;
     virtual void dumpSmall(std::ostream& str) const;
     bool hasDType() const override { return true; }
     /// Require VlUnpacked, instead of [] for POD elements.
@@ -148,6 +149,7 @@ class AstNodeArrayDType VL_NOT_FINAL : public AstNodeDType {
 public:
     ASTGEN_MEMBERS_AstNodeArrayDType;
     void dump(std::ostream& str) const override;
+    void dumpExtraJson(std::ostream& str) const override;
     void dumpSmall(std::ostream& str) const override;
     const char* broken() const override {
         BROKEN_RTN(!((m_refDTypep && !childDTypep() && m_refDTypep->brokeExists())
@@ -217,6 +219,7 @@ class AstNodeUOrStructDType VL_NOT_FINAL : public AstNodeDType {
     int uniqueNum() const { return m_uniqueNum; }
     const char* broken() const override;
     void dump(std::ostream& str) const override;
+    void dumpExtraJson(std::ostream& str) const override;
     bool isCompound() const override { return !packed(); }
     // For basicp() we reuse the size to indicate a "fake" basic type of same size
     AstBasicDType* basicp() const override {
@@ -387,6 +390,7 @@ class AstBasicDType final : public AstNodeDType {
 public:
     ASTGEN_MEMBERS_AstBasicDType;
     void dump(std::ostream& str) const override;
+    void dumpExtraJson(std::ostream& str) const override;
     // width/widthMin/numeric compared elsewhere
     bool same(const AstNode* samep) const override;
     bool similarDType(const AstNodeDType* samep) const override {
@@ -548,6 +552,7 @@ class AstClassRefDType final : public AstNodeDType {
         return this == samep || (type() == samep->type() && same(samep));
     }
     void dump(std::ostream& str = std::cout) const override;
+    void dumpExtraJson(std::ostream& str = std::cout) const override;
     void dumpSmall(std::ostream& str) const override;
     string name() const override VL_MT_STABLE;
     AstBasicDType* basicp() const override VL_MT_STABLE { return nullptr; }
@@ -813,6 +818,7 @@ class AstEnumDType final : public AstNodeDType {
     string name() const override VL_MT_STABLE { return m_name; }
     void name(const string& flag) override { m_name = flag; }
     void dump(std::ostream& str = std::cout) const override;
+    void dumpExtraJson(std::ostream& str = std::cout) const override;
     void dumpSmall(std::ostream& str) const override;
     // METHODS
     AstBasicDType* basicp() const override VL_MT_STABLE { return subDTypep()->basicp(); }
@@ -867,6 +873,7 @@ class AstIfaceRefDType final : public AstNodeDType {
     // METHODS
     const char* broken() const override;
     void dump(std::ostream& str = std::cout) const override;
+    void dumpExtraJson(std::ostream& str = std::cout) const override;
     void dumpSmall(std::ostream& str) const override;
     void cloneRelink() override;
     AstBasicDType* basicp() const override VL_MT_STABLE { return nullptr; }
@@ -982,6 +989,7 @@ class AstParamTypeDType final : public AstNodeDType {
     }
     ASTGEN_MEMBERS_AstParamTypeDType;
     void dump(std::ostream& str = std::cout) const override;
+    void dumpExtraJson(std::ostream& str = std::cout) const override;
     AstNodeDType* getChildDTypep() const override { return childDTypep(); }
     AstNodeDType* subDTypep() const override VL_MT_STABLE {
         return dtypep() ? dtypep() : childDTypep();
@@ -1135,6 +1143,7 @@ class AstRefDType final : public AstNodeDType {
         return skipRefp()->similarDType(samep->skipRefp());
     }
     void dump(std::ostream& str = std::cout) const override;
+    void dumpExtraJson(std::ostream& str = std::cout) const override;
     void dumpSmall(std::ostream& str) const override;
     string name() const override VL_MT_STABLE { return m_name; }
     string prettyDTypeName() const override {