1// Copyright 2022 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package main 16 17import ( 18 "bytes" 19 "crypto/sha1" 20 "encoding/hex" 21 "flag" 22 "fmt" 23 "io" 24 "io/fs" 25 "os" 26 "path/filepath" 27 "sort" 28 "strings" 29 "time" 30 31 "android/soong/response" 32 "android/soong/tools/compliance" 33 "android/soong/tools/compliance/projectmetadata" 34 35 "github.com/google/blueprint/deptools" 36 37 "github.com/spdx/tools-golang/builder/builder2v2" 38 spdx_json "github.com/spdx/tools-golang/json" 39 "github.com/spdx/tools-golang/spdx/common" 40 spdx "github.com/spdx/tools-golang/spdx/v2_2" 41 "github.com/spdx/tools-golang/spdxlib" 42) 43 44var ( 45 failNoneRequested = fmt.Errorf("\nNo license metadata files requested") 46 failNoLicenses = fmt.Errorf("No licenses found") 47) 48 49const NOASSERTION = "NOASSERTION" 50 51type context struct { 52 stdout io.Writer 53 stderr io.Writer 54 rootFS fs.FS 55 product string 56 stripPrefix []string 57 creationTime creationTimeGetter 58 buildid string 59} 60 61func (ctx context) strip(installPath string) string { 62 for _, prefix := range ctx.stripPrefix { 63 if strings.HasPrefix(installPath, prefix) { 64 p := strings.TrimPrefix(installPath, prefix) 65 if 0 == len(p) { 66 p = ctx.product 67 } 68 if 0 == len(p) { 69 continue 70 } 71 return p 72 } 73 } 74 return installPath 75} 76 77// newMultiString creates a flag that allows multiple values in an array. 78func newMultiString(flags *flag.FlagSet, name, usage string) *multiString { 79 var f multiString 80 flags.Var(&f, name, usage) 81 return &f 82} 83 84// multiString implements the flag `Value` interface for multiple strings. 85type multiString []string 86 87func (ms *multiString) String() string { return strings.Join(*ms, ", ") } 88func (ms *multiString) Set(s string) error { *ms = append(*ms, s); return nil } 89 90func main() { 91 var expandedArgs []string 92 for _, arg := range os.Args[1:] { 93 if strings.HasPrefix(arg, "@") { 94 f, err := os.Open(strings.TrimPrefix(arg, "@")) 95 if err != nil { 96 fmt.Fprintln(os.Stderr, err.Error()) 97 os.Exit(1) 98 } 99 100 respArgs, err := response.ReadRspFile(f) 101 f.Close() 102 if err != nil { 103 fmt.Fprintln(os.Stderr, err.Error()) 104 os.Exit(1) 105 } 106 expandedArgs = append(expandedArgs, respArgs...) 107 } else { 108 expandedArgs = append(expandedArgs, arg) 109 } 110 } 111 112 flags := flag.NewFlagSet("flags", flag.ExitOnError) 113 114 flags.Usage = func() { 115 fmt.Fprintf(os.Stderr, `Usage: %s {options} file.meta_lic {file.meta_lic...} 116 117Outputs an SBOM.spdx. 118 119Options: 120`, filepath.Base(os.Args[0])) 121 flags.PrintDefaults() 122 } 123 124 outputFile := flags.String("o", "-", "Where to write the SBOM spdx file. (default stdout)") 125 depsFile := flags.String("d", "", "Where to write the deps file") 126 product := flags.String("product", "", "The name of the product for which the notice is generated.") 127 stripPrefix := newMultiString(flags, "strip_prefix", "Prefix to remove from paths. i.e. path to root (multiple allowed)") 128 buildid := flags.String("build_id", "", "Uniquely identifies the build. (default timestamp)") 129 130 flags.Parse(expandedArgs) 131 132 // Must specify at least one root target. 133 if flags.NArg() == 0 { 134 flags.Usage() 135 os.Exit(2) 136 } 137 138 if len(*outputFile) == 0 { 139 flags.Usage() 140 fmt.Fprintf(os.Stderr, "must specify file for -o; use - for stdout\n") 141 os.Exit(2) 142 } else { 143 dir, err := filepath.Abs(filepath.Dir(*outputFile)) 144 if err != nil { 145 fmt.Fprintf(os.Stderr, "cannot determine path to %q: %s\n", *outputFile, err) 146 os.Exit(1) 147 } 148 fi, err := os.Stat(dir) 149 if err != nil { 150 fmt.Fprintf(os.Stderr, "cannot read directory %q of %q: %s\n", dir, *outputFile, err) 151 os.Exit(1) 152 } 153 if !fi.IsDir() { 154 fmt.Fprintf(os.Stderr, "parent %q of %q is not a directory\n", dir, *outputFile) 155 os.Exit(1) 156 } 157 } 158 159 var ofile io.Writer 160 ofile = os.Stdout 161 var obuf *bytes.Buffer 162 if *outputFile != "-" { 163 obuf = &bytes.Buffer{} 164 ofile = obuf 165 } 166 167 ctx := &context{ofile, os.Stderr, compliance.FS, *product, *stripPrefix, actualTime, *buildid} 168 169 spdxDoc, deps, err := sbomGenerator(ctx, flags.Args()...) 170 171 if err != nil { 172 if err == failNoneRequested { 173 flags.Usage() 174 } 175 fmt.Fprintf(os.Stderr, "%s\n", err.Error()) 176 os.Exit(1) 177 } 178 179 // writing the spdx Doc created 180 if err := spdx_json.Save2_2(spdxDoc, ofile); err != nil { 181 fmt.Fprintf(os.Stderr, "failed to write document to %v: %v", *outputFile, err) 182 os.Exit(1) 183 } 184 185 if *outputFile != "-" { 186 err := os.WriteFile(*outputFile, obuf.Bytes(), 0666) 187 if err != nil { 188 fmt.Fprintf(os.Stderr, "could not write output to %q: %s\n", *outputFile, err) 189 os.Exit(1) 190 } 191 } 192 193 if *depsFile != "" { 194 err := deptools.WriteDepFile(*depsFile, *outputFile, deps) 195 if err != nil { 196 fmt.Fprintf(os.Stderr, "could not write deps to %q: %s\n", *depsFile, err) 197 os.Exit(1) 198 } 199 } 200 os.Exit(0) 201} 202 203type creationTimeGetter func() string 204 205// actualTime returns current time in UTC 206func actualTime() string { 207 t := time.Now().UTC() 208 return t.UTC().Format("2006-01-02T15:04:05Z") 209} 210 211// replaceSlashes replaces "/" by "-" for the library path to be used for packages & files SPDXID 212func replaceSlashes(x string) string { 213 return strings.ReplaceAll(x, "/", "-") 214} 215 216// stripDocName removes the outdir prefix and meta_lic suffix from a target Name 217func stripDocName(name string) string { 218 // remove outdir prefix 219 if strings.HasPrefix(name, "out/") { 220 name = name[4:] 221 } 222 223 // remove suffix 224 if strings.HasSuffix(name, ".meta_lic") { 225 name = name[:len(name)-9] 226 } else if strings.HasSuffix(name, "/meta_lic") { 227 name = name[:len(name)-9] + "/" 228 } 229 230 return name 231} 232 233// getPackageName returns a package name of a target Node 234func getPackageName(_ *context, tn *compliance.TargetNode) string { 235 return replaceSlashes(tn.Name()) 236} 237 238// getDocumentName returns a package name of a target Node 239func getDocumentName(ctx *context, tn *compliance.TargetNode, pm *projectmetadata.ProjectMetadata) string { 240 if len(ctx.product) > 0 { 241 return replaceSlashes(ctx.product) 242 } 243 if len(tn.ModuleName()) > 0 { 244 if pm != nil { 245 return replaceSlashes(pm.Name() + ":" + tn.ModuleName()) 246 } 247 return replaceSlashes(tn.ModuleName()) 248 } 249 250 return stripDocName(replaceSlashes(tn.Name())) 251} 252 253// getDownloadUrl returns the download URL if available (GIT, SVN, etc..), 254// or NOASSERTION if not available, none determined or ambiguous 255func getDownloadUrl(_ *context, pm *projectmetadata.ProjectMetadata) string { 256 if pm == nil { 257 return NOASSERTION 258 } 259 260 urlsByTypeName := pm.UrlsByTypeName() 261 if urlsByTypeName == nil { 262 return NOASSERTION 263 } 264 265 url := urlsByTypeName.DownloadUrl() 266 if url == "" { 267 return NOASSERTION 268 } 269 return url 270} 271 272// getProjectMetadata returns the optimal project metadata for the target node 273func getProjectMetadata(_ *context, pmix *projectmetadata.Index, 274 tn *compliance.TargetNode) (*projectmetadata.ProjectMetadata, error) { 275 pms, err := pmix.MetadataForProjects(tn.Projects()...) 276 if err != nil { 277 return nil, fmt.Errorf("Unable to read projects for %q: %w\n", tn.Name(), err) 278 } 279 if len(pms) == 0 { 280 return nil, nil 281 } 282 283 // Getting the project metadata that contains most of the info needed for sbomGenerator 284 score := -1 285 index := -1 286 for i := 0; i < len(pms); i++ { 287 tempScore := 0 288 if pms[i].Name() != "" { 289 tempScore += 1 290 } 291 if pms[i].Version() != "" { 292 tempScore += 1 293 } 294 if pms[i].UrlsByTypeName().DownloadUrl() != "" { 295 tempScore += 1 296 } 297 298 if tempScore == score { 299 if pms[i].Project() < pms[index].Project() { 300 index = i 301 } 302 } else if tempScore > score { 303 score = tempScore 304 index = i 305 } 306 } 307 return pms[index], nil 308} 309 310// inputFiles returns the complete list of files read 311func inputFiles(lg *compliance.LicenseGraph, pmix *projectmetadata.Index, licenseTexts []string) []string { 312 projectMeta := pmix.AllMetadataFiles() 313 targets := lg.TargetNames() 314 files := make([]string, 0, len(licenseTexts)+len(targets)+len(projectMeta)) 315 files = append(files, licenseTexts...) 316 files = append(files, targets...) 317 files = append(files, projectMeta...) 318 return files 319} 320 321// generateSPDXNamespace generates a unique SPDX Document Namespace using a SHA1 checksum 322func generateSPDXNamespace(buildid string, created string, files ...string) string { 323 324 seed := strings.Join(files, "") 325 326 if buildid == "" { 327 seed += created 328 } else { 329 seed += buildid 330 } 331 332 // Compute a SHA1 checksum of the seed. 333 hash := sha1.Sum([]byte(seed)) 334 uuid := hex.EncodeToString(hash[:]) 335 336 namespace := fmt.Sprintf("SPDXRef-DOCUMENT-%s", uuid) 337 338 return namespace 339} 340 341// sbomGenerator implements the spdx bom utility 342 343// SBOM is part of the new government regulation issued to improve national cyber security 344// and enhance software supply chain and transparency, see https://www.cisa.gov/sbom 345 346// sbomGenerator uses the SPDX standard, see the SPDX specification (https://spdx.github.io/spdx-spec/) 347// sbomGenerator is also following the internal google SBOM styleguide (http://goto.google.com/spdx-style-guide) 348func sbomGenerator(ctx *context, files ...string) (*spdx.Document, []string, error) { 349 // Must be at least one root file. 350 if len(files) < 1 { 351 return nil, nil, failNoneRequested 352 } 353 354 pmix := projectmetadata.NewIndex(ctx.rootFS) 355 356 lg, err := compliance.ReadLicenseGraph(ctx.rootFS, ctx.stderr, files) 357 358 if err != nil { 359 return nil, nil, fmt.Errorf("Unable to read license text file(s) for %q: %v\n", files, err) 360 } 361 362 // creating the packages section 363 pkgs := []*spdx.Package{} 364 365 // creating the relationship section 366 relationships := []*spdx.Relationship{} 367 368 // creating the license section 369 otherLicenses := []*spdx.OtherLicense{} 370 371 // spdx document name 372 var docName string 373 374 // main package name 375 var mainPkgName string 376 377 // implementing the licenses references for the packages 378 licenses := make(map[string]string) 379 concludedLicenses := func(licenseTexts []string) string { 380 licenseRefs := make([]string, 0, len(licenseTexts)) 381 for _, licenseText := range licenseTexts { 382 license := strings.SplitN(licenseText, ":", 2)[0] 383 if _, ok := licenses[license]; !ok { 384 licenseRef := "LicenseRef-" + replaceSlashes(license) 385 licenses[license] = licenseRef 386 } 387 388 licenseRefs = append(licenseRefs, licenses[license]) 389 } 390 if len(licenseRefs) > 1 { 391 return "(" + strings.Join(licenseRefs, " AND ") + ")" 392 } else if len(licenseRefs) == 1 { 393 return licenseRefs[0] 394 } 395 return "NONE" 396 } 397 398 isMainPackage := true 399 visitedNodes := make(map[*compliance.TargetNode]struct{}) 400 401 // performing a Breadth-first top down walk of licensegraph and building package information 402 compliance.WalkTopDownBreadthFirst(nil, lg, 403 func(lg *compliance.LicenseGraph, tn *compliance.TargetNode, path compliance.TargetEdgePath) bool { 404 if err != nil { 405 return false 406 } 407 var pm *projectmetadata.ProjectMetadata 408 pm, err = getProjectMetadata(ctx, pmix, tn) 409 if err != nil { 410 return false 411 } 412 413 if isMainPackage { 414 docName = getDocumentName(ctx, tn, pm) 415 mainPkgName = replaceSlashes(getPackageName(ctx, tn)) 416 isMainPackage = false 417 } 418 419 if len(path) == 0 { 420 // Add the describe relationship for the main package 421 rln := &spdx.Relationship{ 422 RefA: common.MakeDocElementID("" /* this document */, "DOCUMENT"), 423 RefB: common.MakeDocElementID("", mainPkgName), 424 Relationship: "DESCRIBES", 425 } 426 relationships = append(relationships, rln) 427 428 } else { 429 // Check parent and identify annotation 430 parent := path[len(path)-1] 431 targetEdge := parent.Edge() 432 if targetEdge.IsRuntimeDependency() { 433 // Adding the dynamic link annotation RUNTIME_DEPENDENCY_OF relationship 434 rln := &spdx.Relationship{ 435 RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))), 436 RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))), 437 Relationship: "RUNTIME_DEPENDENCY_OF", 438 } 439 relationships = append(relationships, rln) 440 441 } else if targetEdge.IsDerivation() { 442 // Adding the derivation annotation as a CONTAINS relationship 443 rln := &spdx.Relationship{ 444 RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))), 445 RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))), 446 Relationship: "CONTAINS", 447 } 448 relationships = append(relationships, rln) 449 450 } else if targetEdge.IsBuildTool() { 451 // Adding the toolchain annotation as a BUILD_TOOL_OF relationship 452 rln := &spdx.Relationship{ 453 RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))), 454 RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))), 455 Relationship: "BUILD_TOOL_OF", 456 } 457 relationships = append(relationships, rln) 458 459 } else { 460 panic(fmt.Errorf("Unknown dependency type: %v", targetEdge.Annotations())) 461 } 462 } 463 464 if _, alreadyVisited := visitedNodes[tn]; alreadyVisited { 465 return false 466 } 467 visitedNodes[tn] = struct{}{} 468 pkgName := getPackageName(ctx, tn) 469 470 // Making an spdx package and adding it to pkgs 471 pkg := &spdx.Package{ 472 PackageName: replaceSlashes(pkgName), 473 PackageDownloadLocation: getDownloadUrl(ctx, pm), 474 PackageSPDXIdentifier: common.ElementID(replaceSlashes(pkgName)), 475 PackageLicenseConcluded: concludedLicenses(tn.LicenseTexts()), 476 } 477 478 if pm != nil && pm.Version() != "" { 479 pkg.PackageVersion = pm.Version() 480 } else { 481 pkg.PackageVersion = NOASSERTION 482 } 483 484 pkgs = append(pkgs, pkg) 485 486 return true 487 }) 488 489 // Adding Non-standard licenses 490 491 licenseTexts := make([]string, 0, len(licenses)) 492 493 for licenseText := range licenses { 494 licenseTexts = append(licenseTexts, licenseText) 495 } 496 497 sort.Strings(licenseTexts) 498 499 for _, licenseText := range licenseTexts { 500 // open the file 501 f, err := ctx.rootFS.Open(filepath.Clean(licenseText)) 502 if err != nil { 503 return nil, nil, fmt.Errorf("error opening license text file %q: %w", licenseText, err) 504 } 505 506 // read the file 507 text, err := io.ReadAll(f) 508 if err != nil { 509 return nil, nil, fmt.Errorf("error reading license text file %q: %w", licenseText, err) 510 } 511 // Making an spdx License and adding it to otherLicenses 512 otherLicenses = append(otherLicenses, &spdx.OtherLicense{ 513 LicenseName: strings.Replace(licenses[licenseText], "LicenseRef-", "", -1), 514 LicenseIdentifier: string(licenses[licenseText]), 515 ExtractedText: string(text), 516 }) 517 } 518 519 deps := inputFiles(lg, pmix, licenseTexts) 520 sort.Strings(deps) 521 522 // Making the SPDX doc 523 ci, err := builder2v2.BuildCreationInfoSection2_2("Organization", "Google LLC", nil) 524 if err != nil { 525 return nil, nil, fmt.Errorf("Unable to build creation info section for SPDX doc: %v\n", err) 526 } 527 528 ci.Created = ctx.creationTime() 529 530 doc := &spdx.Document{ 531 SPDXVersion: "SPDX-2.2", 532 DataLicense: "CC0-1.0", 533 SPDXIdentifier: "DOCUMENT", 534 DocumentName: docName, 535 DocumentNamespace: generateSPDXNamespace(ctx.buildid, ci.Created, files...), 536 CreationInfo: ci, 537 Packages: pkgs, 538 Relationships: relationships, 539 OtherLicenses: otherLicenses, 540 } 541 542 if err := spdxlib.ValidateDocument2_2(doc); err != nil { 543 return nil, nil, fmt.Errorf("Unable to validate the SPDX doc: %v\n", err) 544 } 545 546 return doc, deps, nil 547} 548