1# Copyright 2023 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Checks that source files are listed in build files, such as BUILD.bazel.""" 15 16import logging 17from pathlib import Path 18from typing import Callable, Sequence 19 20from pw_cli.file_filter import FileFilter 21from pw_presubmit import build, format_code, git_repo, presubmit_context 22from pw_presubmit.presubmit import Check, filter_paths 23from pw_presubmit.presubmit_context import ( 24 PresubmitContext, 25 PresubmitFailure, 26) 27 28_LOG: logging.Logger = logging.getLogger(__name__) 29 30# The filter is used twice for each source_is_in_* check. First to decide 31# whether the check should be run. Once it's running, we use ctx.all_paths 32# instead of ctx.paths since we want to check that all files are in the build, 33# not just changed files, but we need to run ctx.all_paths through the same 34# filter within the check or we won't properly ignore files that the caller 35# asked to be ignored. 36 37_DEFAULT_BAZEL_EXTENSIONS = (*format_code.C_FORMAT.extensions,) 38 39 40def bazel( 41 source_filter: FileFilter, 42 files_and_extensions_to_check: Sequence[str] = _DEFAULT_BAZEL_EXTENSIONS, 43) -> Check: 44 """Create a presubmit check that ensures source files are in Bazel files. 45 46 Args: 47 source_filter: filter that selects files that must be in the Bazel build 48 files_and_extensions_to_check: files and extensions to look for (the 49 source_filter might match build files that won't be in the build but 50 this should only match source files) 51 """ 52 53 @filter_paths(file_filter=source_filter) 54 def source_is_in_bazel_build(ctx: PresubmitContext): 55 """Checks that source files are in the Bazel build.""" 56 57 paths = source_filter.filter(ctx.all_paths) 58 paths = presubmit_context.apply_exclusions(ctx, paths) 59 60 missing = build.check_bazel_build_for_files( 61 files_and_extensions_to_check, 62 paths, 63 bazel_dirs=[ctx.root], 64 ) 65 66 if missing: 67 with ctx.failure_summary_log.open('w') as outs: 68 print('Missing files:', file=outs) 69 for miss in missing: 70 print(miss, file=outs) 71 72 _LOG.warning('All source files must appear in BUILD.bazel files') 73 raise PresubmitFailure 74 75 return source_is_in_bazel_build 76 77 78_DEFAULT_GN_EXTENSIONS = ( 79 'setup.cfg', 80 '.toml', 81 '.rst', 82 '.py', 83 *format_code.C_FORMAT.extensions, 84) 85 86 87def gn( # pylint: disable=invalid-name 88 source_filter: FileFilter, 89 files_and_extensions_to_check: Sequence[str] = _DEFAULT_GN_EXTENSIONS, 90) -> Check: 91 """Create a presubmit check that ensures source files are in GN files. 92 93 Args: 94 source_filter: filter that selects files that must be in the GN build 95 files_and_extensions_to_check: files and extensions to look for (the 96 source_filter might match build files that won't be in the build but 97 this should only match source files) 98 """ 99 100 @filter_paths(file_filter=source_filter) 101 def source_is_in_gn_build(ctx: PresubmitContext): 102 """Checks that source files are in the GN build.""" 103 104 paths = source_filter.filter(ctx.all_paths) 105 paths = presubmit_context.apply_exclusions(ctx, paths) 106 107 missing = build.check_gn_build_for_files( 108 files_and_extensions_to_check, 109 paths, 110 gn_build_files=git_repo.list_files( 111 pathspecs=['BUILD.gn', '*BUILD.gn'], repo_path=ctx.root 112 ), 113 ) 114 115 if missing: 116 with ctx.failure_summary_log.open('w') as outs: 117 print('Missing files:', file=outs) 118 for miss in missing: 119 print(miss, file=outs) 120 121 _LOG.warning('All source files must appear in BUILD.gn files') 122 raise PresubmitFailure 123 124 return source_is_in_gn_build 125 126 127_DEFAULT_CMAKE_EXTENSIONS = (*format_code.C_FORMAT.extensions,) 128 129 130def cmake( 131 source_filter: FileFilter, 132 run_cmake: Callable[[PresubmitContext], None], 133 files_and_extensions_to_check: Sequence[str] = _DEFAULT_CMAKE_EXTENSIONS, 134) -> Check: 135 """Create a presubmit check that ensures source files are in CMake files. 136 137 Args: 138 source_filter: filter that selects files that must be in the CMake build 139 run_cmake: callable that takes a PresubmitContext and invokes CMake 140 files_and_extensions_to_check: files and extensions to look for (the 141 source_filter might match build files that won't be in the build but 142 this should only match source files) 143 """ 144 145 to_check = tuple(files_and_extensions_to_check) 146 147 @filter_paths(file_filter=source_filter) 148 def source_is_in_cmake_build(ctx: PresubmitContext): 149 """Checks that source files are in the CMake build.""" 150 151 paths = source_filter.filter(ctx.all_paths) 152 paths = presubmit_context.apply_exclusions(ctx, paths) 153 154 run_cmake(ctx) 155 missing = build.check_compile_commands_for_files( 156 ctx.output_dir / 'compile_commands.json', 157 (f for f in paths if str(f).endswith(to_check)), 158 ) 159 160 if missing: 161 with ctx.failure_summary_log.open('w') as outs: 162 print('Missing files:', file=outs) 163 for miss in missing: 164 print(miss, file=outs) 165 166 _LOG.warning( 167 'Files missing from CMake:\n%s', 168 '\n'.join(str(f) for f in missing), 169 ) 170 raise PresubmitFailure 171 172 return source_is_in_cmake_build 173 174 175_DEFAULT_SOONG_EXTENSIONS = (*format_code.CPP_SOURCE_EXTS, '.proto') 176 177 178def soong( # pylint: disable=invalid-name 179 source_filter: FileFilter, 180 files_and_extensions_to_check: Sequence[str] = _DEFAULT_SOONG_EXTENSIONS, 181) -> Check: 182 """Create a presubmit check that ensures sources are in Android.bp files. 183 184 Args: 185 source_filter: filter that selects files that must be in the Soong files 186 files_and_extensions_to_check: files and extensions to look for (the 187 source_filter might match build files that won't be in the build but 188 this should only match source files) 189 """ 190 191 @filter_paths(file_filter=source_filter) 192 def source_is_in_soong_build(ctx: PresubmitContext): 193 """Checks that source files are in the Soong build.""" 194 195 paths = source_filter.filter(ctx.all_paths) 196 paths = presubmit_context.apply_exclusions(ctx, paths) 197 198 # For now, only check modules where there is an Android.bp file. 199 relevant_paths: list[Path] = [] 200 for path in paths: 201 # For now, don't require tests be included in Android.bp files. 202 split = path.stem.split('_') 203 if 'test' in split or 'mock' in split: 204 continue 205 206 if (path.parent / 'Android.bp').is_file(): 207 relevant_paths.append(path) 208 209 missing = build.check_soong_build_for_files( 210 files_and_extensions_to_check, 211 relevant_paths, 212 soong_build_files=git_repo.list_files( 213 pathspecs=['Android.bp', '*Android.bp'], repo_path=ctx.root 214 ), 215 ) 216 217 if missing: 218 with ctx.failure_summary_log.open('w') as outs: 219 print('Missing files:', file=outs) 220 for miss in missing: 221 print(miss, file=outs) 222 223 _LOG.warning('All source files must appear in Android.bp files') 224 raise PresubmitFailure 225 226 return source_is_in_soong_build 227