1// Copyright 2024 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package android 16 17import ( 18 "bytes" 19 "encoding/csv" 20 "fmt" 21 "slices" 22 "strconv" 23 "strings" 24 25 "github.com/google/blueprint" 26 "github.com/google/blueprint/gobtools" 27) 28 29var ( 30 // Constants of property names used in compliance metadata of modules 31 ComplianceMetadataProp = struct { 32 NAME string 33 PACKAGE string 34 MODULE_TYPE string 35 OS string 36 ARCH string 37 IS_PRIMARY_ARCH string 38 VARIANT string 39 IS_STATIC_LIB string 40 INSTALLED_FILES string 41 BUILT_FILES string 42 STATIC_DEPS string 43 STATIC_DEP_FILES string 44 WHOLE_STATIC_DEPS string 45 WHOLE_STATIC_DEP_FILES string 46 LICENSES string 47 48 // module_type=package 49 PKG_DEFAULT_APPLICABLE_LICENSES string 50 51 // module_type=license 52 LIC_LICENSE_KINDS string 53 LIC_LICENSE_TEXT string 54 LIC_PACKAGE_NAME string 55 56 // module_type=license_kind 57 LK_CONDITIONS string 58 LK_URL string 59 }{ 60 "name", 61 "package", 62 "module_type", 63 "os", 64 "arch", 65 "is_primary_arch", 66 "variant", 67 "is_static_lib", 68 "installed_files", 69 "built_files", 70 "static_deps", 71 "static_dep_files", 72 "whole_static_deps", 73 "whole_static_dep_files", 74 "licenses", 75 76 "pkg_default_applicable_licenses", 77 78 "lic_license_kinds", 79 "lic_license_text", 80 "lic_package_name", 81 82 "lk_conditions", 83 "lk_url", 84 } 85 86 // A constant list of all property names in compliance metadata 87 // Order of properties here is the order of columns in the exported CSV file. 88 COMPLIANCE_METADATA_PROPS = []string{ 89 ComplianceMetadataProp.NAME, 90 ComplianceMetadataProp.PACKAGE, 91 ComplianceMetadataProp.MODULE_TYPE, 92 ComplianceMetadataProp.OS, 93 ComplianceMetadataProp.ARCH, 94 ComplianceMetadataProp.VARIANT, 95 ComplianceMetadataProp.IS_STATIC_LIB, 96 ComplianceMetadataProp.IS_PRIMARY_ARCH, 97 // Space separated installed files 98 ComplianceMetadataProp.INSTALLED_FILES, 99 // Space separated built files 100 ComplianceMetadataProp.BUILT_FILES, 101 // Space separated module names of static dependencies 102 ComplianceMetadataProp.STATIC_DEPS, 103 // Space separated file paths of static dependencies 104 ComplianceMetadataProp.STATIC_DEP_FILES, 105 // Space separated module names of whole static dependencies 106 ComplianceMetadataProp.WHOLE_STATIC_DEPS, 107 // Space separated file paths of whole static dependencies 108 ComplianceMetadataProp.WHOLE_STATIC_DEP_FILES, 109 ComplianceMetadataProp.LICENSES, 110 // module_type=package 111 ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES, 112 // module_type=license 113 ComplianceMetadataProp.LIC_LICENSE_KINDS, 114 ComplianceMetadataProp.LIC_LICENSE_TEXT, // resolve to file paths 115 ComplianceMetadataProp.LIC_PACKAGE_NAME, 116 // module_type=license_kind 117 ComplianceMetadataProp.LK_CONDITIONS, 118 ComplianceMetadataProp.LK_URL, 119 } 120) 121 122// ComplianceMetadataInfo provides all metadata of a module, e.g. name, module type, package, license, 123// dependencies, built/installed files, etc. It is a wrapper on a map[string]string with some utility 124// methods to get/set properties' values. 125type ComplianceMetadataInfo struct { 126 properties map[string]string 127} 128 129type complianceMetadataInfoGob struct { 130 Properties map[string]string 131} 132 133func NewComplianceMetadataInfo() *ComplianceMetadataInfo { 134 return &ComplianceMetadataInfo{ 135 properties: map[string]string{}, 136 } 137} 138 139func (m *ComplianceMetadataInfo) ToGob() *complianceMetadataInfoGob { 140 return &complianceMetadataInfoGob{ 141 Properties: m.properties, 142 } 143} 144 145func (m *ComplianceMetadataInfo) FromGob(data *complianceMetadataInfoGob) { 146 m.properties = data.Properties 147} 148 149func (c *ComplianceMetadataInfo) GobEncode() ([]byte, error) { 150 return gobtools.CustomGobEncode[complianceMetadataInfoGob](c) 151} 152 153func (c *ComplianceMetadataInfo) GobDecode(data []byte) error { 154 return gobtools.CustomGobDecode[complianceMetadataInfoGob](data, c) 155} 156 157func (c *ComplianceMetadataInfo) SetStringValue(propertyName string, value string) { 158 if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) { 159 panic(fmt.Errorf("Unknown metadata property: %s.", propertyName)) 160 } 161 c.properties[propertyName] = value 162} 163 164func (c *ComplianceMetadataInfo) SetListValue(propertyName string, value []string) { 165 c.SetStringValue(propertyName, strings.TrimSpace(strings.Join(value, " "))) 166} 167 168func (c *ComplianceMetadataInfo) getStringValue(propertyName string) string { 169 if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) { 170 panic(fmt.Errorf("Unknown metadata property: %s.", propertyName)) 171 } 172 return c.properties[propertyName] 173} 174 175func (c *ComplianceMetadataInfo) getAllValues() map[string]string { 176 return c.properties 177} 178 179var ( 180 ComplianceMetadataProvider = blueprint.NewProvider[*ComplianceMetadataInfo]() 181) 182 183// buildComplianceMetadataProvider starts with the ModuleContext.ComplianceMetadataInfo() and fills in more common metadata 184// for different module types without accessing their private fields but through android.Module interface 185// and public/private fields of package android. The final metadata is stored to a module's ComplianceMetadataProvider. 186func buildComplianceMetadataProvider(ctx *moduleContext, m *ModuleBase) { 187 complianceMetadataInfo := ctx.ComplianceMetadataInfo() 188 complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.NAME, m.Name()) 189 complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.PACKAGE, ctx.ModuleDir()) 190 complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.MODULE_TYPE, ctx.ModuleType()) 191 192 switch ctx.ModuleType() { 193 case "license": 194 licenseModule := m.module.(*licenseModule) 195 complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_KINDS, licenseModule.properties.License_kinds) 196 complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_TEXT, PathsForModuleSrc(ctx, licenseModule.properties.License_text).Strings()) 197 complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LIC_PACKAGE_NAME, String(licenseModule.properties.Package_name)) 198 case "license_kind": 199 licenseKindModule := m.module.(*licenseKindModule) 200 complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LK_CONDITIONS, licenseKindModule.properties.Conditions) 201 complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LK_URL, licenseKindModule.properties.Url) 202 default: 203 complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.OS, ctx.Os().String()) 204 complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.ARCH, ctx.Arch().String()) 205 complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.IS_PRIMARY_ARCH, strconv.FormatBool(ctx.PrimaryArch())) 206 complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.VARIANT, ctx.ModuleSubDir()) 207 if m.primaryLicensesProperty != nil && m.primaryLicensesProperty.getName() == "licenses" { 208 complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LICENSES, m.primaryLicensesProperty.getStrings()) 209 } 210 211 var installed InstallPaths 212 installed = append(installed, ctx.installFiles...) 213 installed = append(installed, ctx.katiInstalls.InstallPaths()...) 214 installed = append(installed, ctx.katiSymlinks.InstallPaths()...) 215 installed = append(installed, ctx.katiInitRcInstalls.InstallPaths()...) 216 installed = append(installed, ctx.katiVintfInstalls.InstallPaths()...) 217 complianceMetadataInfo.SetListValue(ComplianceMetadataProp.INSTALLED_FILES, FirstUniqueStrings(installed.Strings())) 218 } 219 ctx.setProvider(ComplianceMetadataProvider, complianceMetadataInfo) 220} 221 222func init() { 223 RegisterComplianceMetadataSingleton(InitRegistrationContext) 224} 225 226func RegisterComplianceMetadataSingleton(ctx RegistrationContext) { 227 ctx.RegisterParallelSingletonType("compliance_metadata_singleton", complianceMetadataSingletonFactory) 228} 229 230var ( 231 // sqlite3 command line tool 232 sqlite3 = pctx.HostBinToolVariable("sqlite3", "sqlite3") 233 234 // Command to import .csv files to sqlite3 database 235 importCsv = pctx.AndroidStaticRule("importCsv", 236 blueprint.RuleParams{ 237 Command: `rm -rf $out && ` + 238 `${sqlite3} $out ".import --csv $in modules" && ` + 239 `${sqlite3} $out ".import --csv ${make_metadata} make_metadata" && ` + 240 `${sqlite3} $out ".import --csv ${make_modules} make_modules"`, 241 CommandDeps: []string{"${sqlite3}"}, 242 }, "make_metadata", "make_modules") 243) 244 245func complianceMetadataSingletonFactory() Singleton { 246 return &complianceMetadataSingleton{} 247} 248 249type complianceMetadataSingleton struct { 250} 251 252func writerToCsv(csvWriter *csv.Writer, row []string) { 253 err := csvWriter.Write(row) 254 if err != nil { 255 panic(err) 256 } 257} 258 259// Collect compliance metadata from all Soong modules, write to a CSV file and 260// import compliance metadata from Make and Soong to a sqlite3 database. 261func (c *complianceMetadataSingleton) GenerateBuildActions(ctx SingletonContext) { 262 if !ctx.Config().HasDeviceProduct() { 263 return 264 } 265 var buffer bytes.Buffer 266 csvWriter := csv.NewWriter(&buffer) 267 268 // Collect compliance metadata of modules in Soong and write to out/soong/compliance-metadata/<product>/soong-modules.csv file. 269 columnNames := []string{"id"} 270 columnNames = append(columnNames, COMPLIANCE_METADATA_PROPS...) 271 writerToCsv(csvWriter, columnNames) 272 273 rowId := -1 274 ctx.VisitAllModules(func(module Module) { 275 if !module.Enabled(ctx) { 276 return 277 } 278 moduleType := ctx.ModuleType(module) 279 if moduleType == "package" { 280 metadataMap := map[string]string{ 281 ComplianceMetadataProp.NAME: ctx.ModuleName(module), 282 ComplianceMetadataProp.MODULE_TYPE: ctx.ModuleType(module), 283 ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES: strings.Join(module.base().primaryLicensesProperty.getStrings(), " "), 284 } 285 rowId = rowId + 1 286 metadata := []string{strconv.Itoa(rowId)} 287 for _, propertyName := range COMPLIANCE_METADATA_PROPS { 288 metadata = append(metadata, metadataMap[propertyName]) 289 } 290 writerToCsv(csvWriter, metadata) 291 return 292 } 293 if provider, ok := ctx.otherModuleProvider(module, ComplianceMetadataProvider); ok { 294 metadataInfo := provider.(*ComplianceMetadataInfo) 295 rowId = rowId + 1 296 metadata := []string{strconv.Itoa(rowId)} 297 for _, propertyName := range COMPLIANCE_METADATA_PROPS { 298 metadata = append(metadata, metadataInfo.getStringValue(propertyName)) 299 } 300 writerToCsv(csvWriter, metadata) 301 return 302 } 303 }) 304 csvWriter.Flush() 305 306 deviceProduct := ctx.Config().DeviceProduct() 307 modulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "soong-modules.csv") 308 WriteFileRuleVerbatim(ctx, modulesCsv, buffer.String()) 309 310 // Metadata generated in Make 311 makeMetadataCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-metadata.csv") 312 makeModulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-modules.csv") 313 314 // Import metadata from Make and Soong to sqlite3 database 315 complianceMetadataDb := PathForOutput(ctx, "compliance-metadata", deviceProduct, "compliance-metadata.db") 316 ctx.Build(pctx, BuildParams{ 317 Rule: importCsv, 318 Input: modulesCsv, 319 Implicits: []Path{ 320 makeMetadataCsv, 321 makeModulesCsv, 322 }, 323 Output: complianceMetadataDb, 324 Args: map[string]string{ 325 "make_metadata": makeMetadataCsv.String(), 326 "make_modules": makeModulesCsv.String(), 327 }, 328 }) 329 330 // Phony rule "compliance-metadata.db". "m compliance-metadata.db" to create the compliance metadata database. 331 ctx.Build(pctx, BuildParams{ 332 Rule: blueprint.Phony, 333 Inputs: []Path{complianceMetadataDb}, 334 Output: PathForPhony(ctx, "compliance-metadata.db"), 335 }) 336 337} 338