1# Copyright 2023 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Generate a coverage report using llvm-cov.""" 15 16import argparse 17import json 18import logging 19import sys 20import subprocess 21from pathlib import Path 22from typing import Any 23 24_LOG = logging.getLogger(__name__) 25 26 27def _parser_args() -> dict[str, Any]: 28 parser = argparse.ArgumentParser(description=__doc__) 29 parser.add_argument( 30 '--llvm-cov-path', 31 type=Path, 32 required=True, 33 help='Path to the llvm-cov binary to use to generate coverage reports.', 34 ) 35 parser.add_argument( 36 '--format', 37 dest='format_type', 38 type=str, 39 choices=['text', 'html', 'lcov', 'json'], 40 required=True, 41 help='Desired output format of the code coverage report.', 42 ) 43 parser.add_argument( 44 '--test-metadata-path', 45 type=Path, 46 required=True, 47 help='Path to the *.test_metadata.json file that describes all of the ' 48 'tests being used to generate a coverage report.', 49 ) 50 parser.add_argument( 51 '--profdata-path', 52 type=Path, 53 required=True, 54 help='Path for the output merged profdata file to use with generating a' 55 ' coverage report for the tests described in --test-metadata.', 56 ) 57 parser.add_argument( 58 '--root-dir', 59 type=Path, 60 required=True, 61 help='Path to the project\'s root directory.', 62 ) 63 parser.add_argument( 64 '--build-dir', 65 type=Path, 66 required=True, 67 help='Path to the ninja build directory.', 68 ) 69 parser.add_argument( 70 '--output-dir', 71 type=Path, 72 required=True, 73 help='Path to where the output report should be placed. This must be a ' 74 'relative path (from the current working directory) to ensure the ' 75 'depfiles are generated correctly.', 76 ) 77 parser.add_argument( 78 '--depfile-path', 79 type=Path, 80 required=True, 81 help='Path for the output depfile to convey the extra input ' 82 'requirements from parsing --test-metadata.', 83 ) 84 parser.add_argument( 85 '--filter-path', 86 dest='filter_paths', 87 type=str, 88 action='append', 89 default=[], 90 help='Only these folder paths or files will be included in the output. ' 91 'To work properly, these must be aboslute paths or relative paths from ' 92 'the current working directory. No globs or regular expression features' 93 ' are supported.', 94 ) 95 parser.add_argument( 96 '--ignore-filename-pattern', 97 dest='ignore_filename_patterns', 98 type=str, 99 action='append', 100 default=[], 101 help='Any file path that matches one of these regular expression ' 102 'patterns will be excluded from the output report (possibly even if ' 103 'that path was included in --filter-paths). The regular expression ' 104 'engine for these is somewhat primitive and does not support things ' 105 'like look-ahead or look-behind.', 106 ) 107 return vars(parser.parse_args()) 108 109 110def generate_report( 111 llvm_cov_path: Path, 112 format_type: str, 113 test_metadata_path: Path, 114 profdata_path: Path, 115 root_dir: Path, 116 build_dir: Path, 117 output_dir: Path, 118 depfile_path: Path, 119 filter_paths: list[str], 120 ignore_filename_patterns: list[str], 121) -> int: 122 """Generate a coverage report using llvm-cov.""" 123 124 # Ensure directories that need to be absolute are. 125 root_dir = root_dir.resolve() 126 build_dir = build_dir.resolve() 127 128 # Open the test_metadata_path, parse it to JSON, and extract out the 129 # test binaries. 130 test_metadata = json.loads(test_metadata_path.read_text()) 131 test_binaries = [ 132 Path(obj['test_directory']) / obj['test_name'] 133 for obj in test_metadata 134 if 'test_type' in obj and obj['test_type'] == 'unit_test' 135 ] 136 137 # llvm-cov export does not create an output file, so we mimic it by creating 138 # the directory structure and writing to file outself after we run the 139 # command. 140 if format_type in ['lcov', 'json']: 141 export_output_path = ( 142 output_dir / 'report.lcov' 143 if format_type == 'lcov' 144 else output_dir / 'report.json' 145 ) 146 output_dir.mkdir(parents=True, exist_ok=True) 147 148 # Build the command to the llvm-cov subtool based on provided arguments. 149 command = [str(llvm_cov_path)] 150 if format_type in ['html', 'text']: 151 command += [ 152 'show', 153 '--format', 154 format_type, 155 '--output-dir', 156 str(output_dir), 157 ] 158 else: # format_type in ['lcov', 'json'] 159 command += [ 160 'export', 161 '--format', 162 # `text` is JSON format when using `llvm-cov`. 163 format_type if format_type == 'lcov' else 'text', 164 ] 165 # We really need two `--path-equivalence` options to be able to map both the 166 # root directory for coverage files to the absolute path of the project 167 # root_dir and to be able to map "out/" prefix to the provided build_dir. 168 # 169 # llvm-cov does not currently support two `--path-equivalence` options, so 170 # we use `--compilation-dir` and `--path-equivalence` together. This has the 171 # unfortunate consequence of showing file paths as absolute in the JSON, 172 # LCOV, and text reports. 173 # 174 # An unwritten assumption here is that root_dir must be an 175 # absolute path to enable file-path-based filtering. 176 # 177 # This is due to turning all file paths into absolute files here: 178 # https://github.com/llvm-mirror/llvm/blob/2c4ca6832fa6b306ee6a7010bfb80a3f2596f824/tools/llvm-cov/CodeCoverage.cpp#L188. 179 command += [ 180 '--compilation-dir', 181 str(root_dir), 182 ] 183 # Pigweed maps any build directory to out, which causes generated files to 184 # be reported to exist under the out directory, which may not exist if the 185 # build directory is not exactly out. This maps out back to the build 186 # directory so generated files can be found. 187 command += [ 188 '--path-equivalence', 189 f'{str(root_dir)}/out,{str(build_dir)}', 190 ] 191 command += [ 192 '--instr-profile', 193 str(profdata_path), 194 ] 195 command += [ 196 f'--ignore-filename-regex={path}' for path in ignore_filename_patterns 197 ] 198 # The test binary positional argument MUST appear before the filter path 199 # positional arguments. llvm-cov is a horrible interface. 200 command += [str(test_binaries[0])] 201 command += [f'--object={binary}' for binary in test_binaries[1:]] 202 command += [ 203 str(Path(filter_path).resolve()) for filter_path in filter_paths 204 ] 205 206 _LOG.info('') 207 _LOG.info(' '.join(command)) 208 _LOG.info('') 209 210 # Generate the coverage report by invoking the command. 211 if format_type in ['html', 'text']: 212 output = subprocess.run(command) 213 if output.returncode != 0: 214 return output.returncode 215 else: # format_type in ['lcov', 'json'] 216 output = subprocess.run(command, capture_output=True) 217 if output.returncode != 0: 218 _LOG.error(output.stderr) 219 return output.returncode 220 export_output_path.write_bytes(output.stdout) 221 222 # Generate the depfile that describes the dependency on the test binaries 223 # used to create the report output. 224 depfile_target = Path('.') 225 if format_type in ['lcov', 'json']: 226 depfile_target = export_output_path 227 elif format_type == 'text': 228 depfile_target = output_dir / 'index.txt' 229 else: # format_type == 'html' 230 depfile_target = output_dir / 'index.html' 231 depfile_path.write_text( 232 ''.join( 233 [ 234 str(depfile_target), 235 ': \\\n', 236 *[str(binary) + ' \\\n' for binary in test_binaries], 237 ] 238 ) 239 ) 240 241 return 0 242 243 244def main() -> int: 245 return generate_report(**_parser_args()) 246 247 248if __name__ == "__main__": 249 sys.exit(main()) 250