Merge branch 'AlexanderArvidsson-feature/bindgen-docs'

CHANGELOG.md

@@ -1,5 +1,16 @@
 ## Updates
 
+### 11-Jan-2024
+
+The language bindings code-generation can now extract comments from the C headers
+and include them into the language bindings. Currently this is supported in
+the Odin and Zig bindings, but adding comments to the other language bindings
+is fairly easy since the bulk of the work happens in the common `gen_ir.py` script
+which parses the C API declarations into a JSON tree.
+
+Related PR: https://github.com/floooh/sokol/pull/1176, many thanks to
+@AlexanderArvidsson!
+
 ### 17-Dec-2024
 
 - sokol_imgui.h (breaking change): user-provided images and samplers are now

bindgen/gen_ir.py

@@ -1,17 +1,27 @@
 #-------------------------------------------------------------------------------
 #   Generate an intermediate representation of a clang AST dump.
 #-------------------------------------------------------------------------------
-import json, sys, subprocess
+import re, json, sys, subprocess
 
 def is_api_decl(decl, prefix):
     if 'name' in decl:
         return decl['name'].startswith(prefix)
     elif decl['kind'] == 'EnumDecl':
         # an anonymous enum, check if the items start with the prefix
-        return decl['inner'][0]['name'].lower().startswith(prefix)
+        first = get_first_non_comment(decl['inner'])
+        return first['name'].lower().startswith(prefix)
     else:
         return False
 
+def get_first_non_comment(items):
+    return next(i for i in items if i['kind'] != 'FullComment')
+
+def strip_comments(items):
+    return [i for i in items if i['kind'] != 'FullComment']
+
+def extract_comment(comment, source):
+    return source[comment['range']['begin']['offset']:comment['range']['end']['offset']]
+
 def is_dep_decl(decl, dep_prefixes):
     for prefix in dep_prefixes:
         if is_api_decl(decl, prefix):
@@ -27,12 +37,15 @@ def dep_prefix(decl, dep_prefixes):
 def filter_types(str):
     return str.replace('_Bool', 'bool')
 
-def parse_struct(decl):
+def parse_struct(decl, source):
     outp = {}
     outp['kind'] = 'struct'
     outp['name'] = decl['name']
     outp['fields'] = []
     for item_decl in decl['inner']:
+        if item_decl['kind'] == 'FullComment':
+            outp['comment'] = extract_comment(item_decl, source)
+            continue
         if item_decl['kind'] != 'FieldDecl':
             sys.exit(f"ERROR: Structs must only contain simple fields ({decl['name']})")
         item = {}
@@ -42,7 +55,7 @@ def parse_struct(decl):
         outp['fields'].append(item)
     return outp
 
-def parse_enum(decl):
+def parse_enum(decl, source):
     outp = {}
     if 'name' in decl:
         outp['kind'] = 'enum'
@@ -53,31 +66,40 @@ def parse_enum(decl):
         needs_value = True
     outp['items'] = []
     for item_decl in decl['inner']:
+        if item_decl['kind'] == 'FullComment':
+            outp['comment'] = extract_comment(item_decl, source)
+            continue
         if item_decl['kind'] == 'EnumConstantDecl':
             item = {}
             item['name'] = item_decl['name']
             if 'inner' in item_decl:
-                const_expr = item_decl['inner'][0]
-                if const_expr['kind'] != 'ConstantExpr':
-                    sys.exit(f"ERROR: Enum values must be a ConstantExpr ({item_decl['name']}), is '{const_expr['kind']}'")
-                if const_expr['valueCategory'] != 'rvalue' and const_expr['valueCategory'] != 'prvalue':
-                    sys.exit(f"ERROR: Enum value ConstantExpr must be 'rvalue' or 'prvalue' ({item_decl['name']}), is '{const_expr['valueCategory']}'")
-                if not ((len(const_expr['inner']) == 1) and (const_expr['inner'][0]['kind'] == 'IntegerLiteral')):
-                    sys.exit(f"ERROR: Enum value ConstantExpr must have exactly one IntegerLiteral ({item_decl['name']})")
-                item['value'] = const_expr['inner'][0]['value']
+                exprs = strip_comments(item_decl['inner'])
+                if len(exprs) > 0:
+                    const_expr = exprs[0]
+                    if const_expr['kind'] != 'ConstantExpr':
+                        sys.exit(f"ERROR: Enum values must be a ConstantExpr ({item_decl['name']}), is '{const_expr['kind']}'")
+                    if const_expr['valueCategory'] != 'rvalue' and const_expr['valueCategory'] != 'prvalue':
+                        sys.exit(f"ERROR: Enum value ConstantExpr must be 'rvalue' or 'prvalue' ({item_decl['name']}), is '{const_expr['valueCategory']}'")
+                    const_expr_inner = strip_comments(const_expr['inner'])
+                    if not ((len(const_expr_inner) == 1) and (const_expr_inner[0]['kind'] == 'IntegerLiteral')):
+                        sys.exit(f"ERROR: Enum value ConstantExpr must have exactly one IntegerLiteral ({item_decl['name']})")
+                    item['value'] = const_expr_inner[0]['value']
             if needs_value and 'value' not in item:
-                sys.exit(f"ERROR: anonymous enum items require an explicit value")
+                sys.exit("ERROR: anonymous enum items require an explicit value")
             outp['items'].append(item)
     return outp
 
-def parse_func(decl):
+def parse_func(decl, source):
     outp = {}
     outp['kind'] = 'func'
     outp['name'] = decl['name']
     outp['type'] = filter_types(decl['type']['qualType'])
     outp['params'] = []
     if 'inner' in decl:
         for param in decl['inner']:
+            if param['kind'] == 'FullComment':
+                outp['comment'] = extract_comment(param, source)
+                continue
             if param['kind'] != 'ParmVarDecl':
                 print(f"  >> warning: ignoring func {decl['name']} (unsupported parameter type)")
                 return None
@@ -87,38 +109,44 @@ def parse_func(decl):
             outp['params'].append(outp_param)
     return outp
 
-def parse_decl(decl):
+def parse_decl(decl, source):
     kind = decl['kind']
     if kind == 'RecordDecl':
-        return parse_struct(decl)
+        return parse_struct(decl, source)
     elif kind == 'EnumDecl':
-        return parse_enum(decl)
+        return parse_enum(decl, source)
     elif kind == 'FunctionDecl':
-        return parse_func(decl)
+        return parse_func(decl, source)
     else:
         return None
 
-def clang(csrc_path):
-    cmd = ['clang', '-Xclang', '-ast-dump=json', '-c' ]
-    cmd.append(csrc_path)
+def clang(csrc_path, with_comments=False):
+    cmd = ['clang', '-Xclang', '-ast-dump=json', "-c", csrc_path]
+    if with_comments:
+        cmd.append('-fparse-all-comments')
     return subprocess.check_output(cmd)
 
-def gen(header_path, source_path, module, main_prefix, dep_prefixes):
-    ast = clang(source_path)
+def gen(header_path, source_path, module, main_prefix, dep_prefixes, with_comments=False):
+    ast = clang(source_path, with_comments=with_comments)
     inp = json.loads(ast)
     outp = {}
     outp['module'] = module
     outp['prefix'] = main_prefix
     outp['dep_prefixes'] = dep_prefixes
     outp['decls'] = []
-    for decl in inp['inner']:
-        is_dep = is_dep_decl(decl, dep_prefixes)
-        if is_api_decl(decl, main_prefix) or is_dep:
-            outp_decl = parse_decl(decl)
-            if outp_decl is not None:
-                outp_decl['is_dep'] = is_dep
-                outp_decl['dep_prefix'] = dep_prefix(decl, dep_prefixes)
-                outp['decls'].append(outp_decl)
+    with open(header_path, 'r') as f:
+        source = f.read()
+        first_comment = re.search(r"/\*(.*?)\*/", source, re.S).group(1)
+        if first_comment and "Project URL" in first_comment:
+            outp['comment'] = first_comment
+        for decl in inp['inner']:
+            is_dep = is_dep_decl(decl, dep_prefixes)
+            if is_api_decl(decl, main_prefix) or is_dep:
+                outp_decl = parse_decl(decl, source)
+                if outp_decl is not None:
+                    outp_decl['is_dep'] = is_dep
+                    outp_decl['dep_prefix'] = dep_prefix(decl, dep_prefixes)
+                    outp['decls'].append(outp_decl)
     with open(f'{module}.json', 'w') as f:
         f.write(json.dumps(outp, indent=2));
     return outp

bindgen/gen_odin.py

@@ -3,6 +3,7 @@
 #
 #   Generate Odin bindings.
 #-------------------------------------------------------------------------------
+import textwrap
 import gen_ir
 import gen_util as util
 import os, shutil, sys
@@ -21,6 +22,7 @@
     'sdtx_':    'debugtext',
     'sshape_':  'shape',
     'sglue_':   'glue',
+    'simgui_':   'imgui',
 }
 
 system_libs = {
@@ -76,6 +78,7 @@
     'sdtx_':    'sokol_debugtext.c',
     'sshape_':  'sokol_shape.c',
     'sglue_':   'sokol_glue.c',
+    'simgui_':  'sokol_imgui.c',
 }
 
 ignores = [
@@ -147,6 +150,16 @@ def l(s):
     global out_lines
     out_lines += s + '\n'
 
+def c(s, indent=""):
+    if not s:
+        return
+    if '\n' in s:
+        l(f'{indent}/*')
+        l(textwrap.indent(textwrap.dedent(s), prefix=f"    {indent}"))
+        l(f'{indent}*/')
+    else:
+        l(f'{indent}// {s.strip()}')
+
 def check_override(name, default=None):
     if name in overrides:
         return overrides[name]
@@ -425,6 +438,8 @@ def gen_c_imports(inp, c_prefix, prefix):
             args = funcdecl_args_c(decl, prefix)
             res_type = funcdecl_result_c(decl, prefix)
             res_str = '' if res_type == '' else f'-> {res_type}'
+            if decl.get('comment'):
+                c(decl['comment'], indent="    ")
             # Need to special case sapp_sg to avoid Odin's context keyword
             if c_prefix == "sapp_sg":
                 l(f'    @(link_name="{decl["name"]}")')
@@ -435,14 +450,18 @@ def gen_c_imports(inp, c_prefix, prefix):
     l('')
 
 def gen_consts(decl, prefix):
+    c(decl.get('comment'))
     for item in decl['items']:
         item_name = check_override(item['name'])
+        c(item.get('comment'))
         l(f"{as_snake_case(item_name, prefix)} :: {item['value']}")
     l('')
 
 def gen_struct(decl, prefix):
     c_struct_name = check_override(decl['name'])
     struct_name = as_struct_or_enum_type(c_struct_name, prefix)
+    if decl.get('comment'):
+        c(decl['comment'])
     l(f'{struct_name} :: struct {{')
     for field in decl['fields']:
         field_name = check_override(field['name'])
@@ -457,6 +476,8 @@ def gen_struct(decl, prefix):
 
 def gen_enum(decl, prefix):
     enum_name = check_override(decl['name'])
+    if decl.get('comment'):
+        c(decl['comment'])
     l(f'{as_struct_or_enum_type(enum_name, prefix)} :: enum i32 {{')
     for item in decl['items']:
         item_name = as_enum_item_name(check_override(item['name']))
@@ -488,6 +509,9 @@ def gen_module(inp, c_prefix, dep_prefixes):
     l('// machine generated, do not edit')
     l('')
     l(f"package sokol_{inp['module']}")
+    if inp.get('comment'):
+        l('')
+        c(inp['comment'])
     gen_imports(dep_prefixes)
     gen_helpers(inp)
     prefix = inp['prefix']
@@ -534,7 +558,7 @@ def gen(c_header_path, c_prefix, dep_c_prefixes):
     shutil.copyfile(c_header_path, f'{c_root}/{os.path.basename(c_header_path)}')
     csource_path = get_csource_path(c_prefix)
     module_name = module_names[c_prefix]
-    ir = gen_ir.gen(c_header_path, csource_path, module_name, c_prefix, dep_c_prefixes)
+    ir = gen_ir.gen(c_header_path, csource_path, module_name, c_prefix, dep_c_prefixes, with_comments=True)
     gen_module(ir, c_prefix, dep_c_prefixes)
     with open(f"{module_root}/{ir['module']}/{ir['module']}.odin", 'w', newline='\n') as f_outp:
         f_outp.write(out_lines)

bindgen/gen_zig.py

@@ -8,6 +8,7 @@
 #-------------------------------------------------------------------------------
 import gen_ir
 import os, shutil, sys
+import textwrap
 
 import gen_util as util
 
@@ -125,6 +126,13 @@ def l(s):
     global out_lines
     out_lines += s + '\n'
 
+def c(s, indent="", comment="///"):
+    if not s:
+        return
+    prefix = f"{indent}{comment}"
+    for line in textwrap.dedent(s).splitlines():
+        l(f"{prefix} {line}" if line else prefix )
+
 def as_zig_prim_type(s):
     return prim_types[s]
 
@@ -324,6 +332,7 @@ def funcdecl_result_zig(decl, prefix):
 def gen_struct(decl, prefix):
     struct_name = check_override(decl['name'])
     zig_type = as_zig_struct_type(struct_name, prefix)
+    c(decl.get('comment'))
     l(f"pub const {zig_type} = extern struct {{")
     for field in decl['fields']:
         field_name = check_override(field['name'])
@@ -382,14 +391,19 @@ def gen_struct(decl, prefix):
         else:
             sys.exit(f"ERROR gen_struct: {field_name}: {field_type};")
     l("};")
+    l("")
 
 def gen_consts(decl, prefix):
+    c(decl.get('comment'))
     for item in decl['items']:
         item_name = check_override(item['name'])
+        c(item.get('comment'))
         l(f"pub const {util.as_lower_snake_case(item_name, prefix)} = {item['value']};")
+    l("")
 
 def gen_enum(decl, prefix):
     enum_name = check_override(decl['name'])
+    c(decl.get('comment'))
     l(f"pub const {as_zig_enum_type(enum_name, prefix)} = enum(i32) {{")
     for item in decl['items']:
         item_name = as_enum_item_name(check_override(item['name']))
@@ -399,13 +413,17 @@ def gen_enum(decl, prefix):
             else:
                 l(f"    {item_name},")
     l("};")
+    l("")
 
 def gen_func_c(decl, prefix):
+    c(decl.get('comment'))
     l(f"pub extern fn {decl['name']}({funcdecl_args_c(decl, prefix)}) {funcdecl_result_c(decl, prefix)};")
+    l('')
 
 def gen_func_zig(decl, prefix):
     c_func_name = decl['name']
     zig_func_name = util.as_lower_camel_case(check_override(decl['name']), prefix)
+    c(decl.get('comment'))
     if c_func_name in c_callbacks:
         # a simple forwarded C callback function
         l(f"pub const {zig_func_name} = {c_func_name};")
@@ -435,6 +453,7 @@ def gen_func_zig(decl, prefix):
         s += ");"
         l(s)
         l("}")
+        l("")
 
 def pre_parse(inp):
     global struct_types
@@ -535,6 +554,9 @@ def gen_helpers(inp):
 
 def gen_module(inp, dep_prefixes):
     l('// machine generated, do not edit')
+    if inp.get('comment'):
+        l('')
+        c(inp['comment'], comment="//")
     l('')
     gen_imports(inp, dep_prefixes)
     gen_helpers(inp)
@@ -570,8 +592,8 @@ def gen(c_header_path, c_prefix, dep_c_prefixes):
     print(f'  {c_header_path} => {module_name}')
     reset_globals()
     shutil.copyfile(c_header_path, f'sokol-zig/src/sokol/c/{os.path.basename(c_header_path)}')
-    ir = gen_ir.gen(c_header_path, c_source_path, module_name, c_prefix, dep_c_prefixes)
-    gen_module(ir, dep_c_prefixes)
+    ir = gen_ir.gen(c_header_path, c_source_path, module_name, c_prefix, dep_c_prefixes, with_comments=True)
+    gen_module(ir, dep_c_prefixes,)
     output_path = f"sokol-zig/src/sokol/{ir['module']}.zig"
     with open(output_path, 'w', newline='\n') as f_outp:
         f_outp.write(out_lines)