py2lcov

#!/usr/bin/env python3

#   Copyright (c) MediaTek USA Inc., 2020-2024
#
#   This program is free software;  you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or (at
#   your option) any later version.
#
#   This program is distributed in the hope that it will be useful, but
#   WITHOUT ANY WARRANTY;  without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#   General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program;  if not, see
#   <http://www.gnu.org/licenses/>.
#
#
# This script traverses Python coverage data in one or more coverage
# data file (generated by the Coverage.py module) and translates it into
# LCOV .info format.
#
#   py2lcov [--output mydata.info] [--test-name name] [options] coverage.dat+
#
# See 'py2lcov --help' for more usage information
#
# See https://coverage.readthedocs.io for directions on how to use Coverage.py
# to generate Python coverage data.
#
#
# arguably, should do this in Perl so we could use the lcovutil module utilities
# In the meantime:  suggested use model is to translate the python XML using this
# utility, then read it back into lcov for additional processing.
#
# @todo figure out/enhance Coverage.py to characterize branch expressions
# @todo perhaps this should be integrated into the Coverage.py module itself.
#       This might no longer be a good idea as XML translation is no longer limited
#       to the Coverage.py module.

import os
import os.path
import sys
import re
import argparse
import xml.etree.ElementTree as ET
import fnmatch
import subprocess
import copy
import base64
import hashlib
import pdb
from xml2lcovutil import ProcessFile

def main():
    usageString="""py2lcov: Translate Python coverage data to LCOV .info format.
Please also see https://coverage.readthedocs.io

Note that the '--no-functions' argument may result in subtly inconsistent coverage
data if a 'no-functions' coverage DB is merged with one which contains derived
function data because the 'def myFunc(...)' line will acquire a 'hit' count
of 1 because the python interpreter considers the 'def' to have been executed
when the line is interpreted (i.e., when the function is defined).
This will generate an 'inconsistent' error if the function is not executed in
your tests because the (derived) function will have a zero hit count but the
first line of the function has a non-zero count.
Best practice is to either always specify '--no-functions' or never specify
'--no-functions'.

py2lcov uses Coverage.py to extract coverage data.
Note that the name of the Coverage.py executable my differ on your platform.
By default, py2lcov uses 'coverage' (which it expects to be in your path).
You can use a different executable, either:
  - through your COVERAGE_COMMAND environment variable, or
  - via the 'py2lcov --cmd exename ..' command line option.

py2lcov does not implement the full suite of LCOV features (e.g., filtering,
substitutions, etc.).
Please generate the translated LCOV format file and then read the data
back in to lcov to use any of those features.
%(usage)s

Example:
   $ export PYCOV_DATA=path/to/pydata

     For 'coverage' versions 6.6.1 and higher (which support "--data-file"):
   $ coverage run --data-file=${PYCOV_DATA} --append --branch \\
       `which myPthonScript.py` args_to_my_python_script

     For older versions which don't support "--data-files":
        use COVERAGE_FILE environment variable to specify data file
   $ COVERAGE_FILE=${PYCOV_DATA} coverage run --append --branch \\
       `which myPthonScript.py` args_to_my_python_script

     # now use py2lcov to translate the XML to INFO file format -
     # also include version information in the generated coverage data.
   $ py2lcov -o pydata.info ${PYCOV_DATA}

    # apply some filtering
   $ lcov -a pydata.info -o filtered.info --filter branch,blank

     # and use genhtml to produce an HTML coverage report:
   $ genhtml -o html_report pydata.info ....
     # the filtered result.
   $ genhtml -o html_filtered filtered.info ....
     # use differential coverage to see exactly what filtering did
   $ genhtml -o html_differential --baseline-file mydata.info filtered.info ...

   Deprecated feature:
   For backward compatibility, py2lcov also supports translation to LCOV
   format from intermediate XML:

       # first translate from Python coerage data to XML:
     $ coverage xml --data-file=${PYCOV_DATA} -o pydata.xml |& tee pydata.log
       # or - if your Coverage.py module is too old to support '--data-file':
     $ COVERAGE_FILE=${PYCOV_DATA} coverage xml -o pydata.xml |& tee pydata.log

       # then translate XML to LCOV format:
     $ py2lcov -i pydata.xml -o pydata.info --version-script myCovScript

   """ % {
       'usage' : ProcessFile.usageNote,
       }

    from_env = ''
    cover_cmd = 'coverage'
    if 'COVERAGE_COMMAND' in os.environ:
        cover_cmd = os.environ['COVERAGE_COMMAND']
        from_env = ' (from your COVERAGE_COMMAND environment variable)'

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=usageString)

    parser.add_argument('-i', '--input', dest='input', default=None,
                        help="DEPRECATED: specify the input xml file from coverage.py")
    parser.add_argument('-o', '--output', dest='output', default='py2lcov.info',
                        help="specify the out LCOV .info file, default: py2lcov.info")
    parser.add_argument('-t', '--test-name', '--testname', dest='testName', default='',
                        help="specify the test name for the TN: entry in LCOV .info file")
    parser.add_argument('-e', '--exclude', dest='excludePatterns', default='',
                        help="specify the exclude file patterns separated by ','")
    parser.add_argument('-v', '--verbose', dest='verbose', default=0, action='count',
                        help="print debug messages")
    parser.add_argument('--version-script', dest='version',
                        help="version extract callback")
    parser.add_argument('--checksum', dest='checksum', action='store_true',
                        default=False,
                        help="compute line checksum - see 'man lcov'")
    parser.add_argument("--no-functions", dest='deriveFunctions',
                        default=True, action='store_false',
                        help="do not derive function coverpoints")
    parser.add_argument("--tabwidth", dest='tabwidth', default=8, type=int,
                        help='tabsize when computing indent')
    parser.add_argument('-k', "--keep-going", dest='keepGoing', default=False, action='store_true',
                        help="ignore errors")
    parser.add_argument('--cmd', dest='cover_cmd', default=cover_cmd,
                        help='executable used to extract python data - e.g., "python3-coverage".  Default is "%s"%s.' % (cover_cmd, from_env))
    parser.add_argument('inputs', nargs='*',
                        help="list of python coverage data input files - expected to be XML or Python .dat format")

    args = parser.parse_args()

    if args.input:
        if not args.keepGoing:
            print("--input is deprecated - please use 'py2lcov ... %s" % (args.input))
        args.inputs.append(args.input);


    if not args.inputs:
        # no input file - see if COVERAGE_FILE environment variable is set
        try:
            args.inputs.append(os.environ['COVERAGE_FILE'])
            print("reading input from default COVERAGE_FILE '%s'" % args.inputs[0])
        except:
            print("Error:  no input files")
            sys.exit(1)

    args.isPython = True
    p = ProcessFile(args)

    for f in args.inputs:
        base, ext = os.path.splitext(f)
        if ext == '.xml':
            p.process_xml_file(f)
            continue

        # assume that anything not ending in .xml is a Coverage.py data file
        xml = base + '.xml'
        suffix = 1
        while os.path.exists(xml):
            xml = base + '.xml%d' % suffix
            suffix += 1
        env = os.environ.copy()
        env["COVERAGE_FILE"] = f
        cmd = [args.cover_cmd, "xml", "-o", xml]
        try:
            x = subprocess.run(cmd, shell=False, check=True, stdout=True, stderr=True, env=env)
        except subprocess.CalledProcessError as err:
            print("Error:  error during XML conversion of %s: %s" % (
                f, str(err)));
            if not args.keepGoing:
                sys.exit(1)
            continue
        p.process_xml_file(xml)
        os.unlink(xml)

    p.close()


if __name__ == '__main__':
    main()