1# Copyright (C) 2024 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15upstream_git_repo = 'https://gitlab.com/kernel-firmware/linux-firmware.git' 16 17# ########################################################## 18# Parsing of linux-firmware WHENCE file 19# ########################################################## 20 21section_divider = '--------------------------------------------------------------------------' 22 23# Handle paths that might be quoted or use backslashes to escape spaces. 24# 25# For some reason some of the paths listed in WHENCE seem to be quoted or 26# use '\ ' to escape spaces in the files, even though this doesn't seem like 27# it should be necessary. Handle it. This takes a path `p`. If it's surrounded 28# by quotes it just strips the quotes off. If it isn't quoted but we see 29# '\ ' we'll transform that to just simple spaces. 30def unquote_path(p): 31 if p.startswith('"') and p.endswith('"'): 32 p = p[1:-1] 33 else: 34 p = p.replace('\\ ', ' ') 35 36 return p 37 38# Parse WHENCE from the upstream repository and return dict w/ info about files. 39# 40# This will read the upstream whence and return a dictionary keyed by files 41# referenced in the upstream WHENCE (anything tagged 'File:', 'RawFile:', or 42# 'Link:'). Values in the dictionary will be another dict that looks like: 43# { 44# 'driver': Name of the driver line associated with this file, 45# 'kind': The kind of file ('File', 'RawFile', or 'Link') 46# 'license': If this key is present it's the text of the license; if this 47# key is not present then the license is unknown. 48# 'link': Only present for 'kind' == 'Link'. This is where the link 49# should point to. 50# } 51def parse_whence(whence_text): 52 file_info_map = {} 53 54 driver = 'UNKNOWN' 55 unlicensed_files = [] 56 license_text = None 57 58 for line in whence_text.splitlines() + [section_divider]: 59 # Take out trailing spaces / carriage returns 60 line = line.rstrip() 61 62 # Look for tags, which are lines that look like: 63 # tag: other stuff 64 # 65 # Tags always need to start the line and can't have any spaces in 66 # them, which helps ID them as tags. 67 # 68 # Note that normally we require a space after the colon which keeps 69 # us from getting confused when we're parsing license text that 70 # has a URL. We special-case allow lines to end with a colon 71 # since some tags (like "License:") are sometimes multiline tags 72 # and the first line might just be blank. 73 if ': ' in line or line.endswith(':'): 74 tag, _, rest = line.partition(': ') 75 tag = tag.rstrip(':') 76 if ' ' in tag or '\t' in tag: 77 tag = None 78 else: 79 rest = rest.lstrip() 80 else: 81 tag = None 82 83 # Any new tag or a full separator ends a license. 84 if line == section_divider or (tag and license_text): 85 if license_text: 86 for f in unlicensed_files: 87 # Clear out blank lines at the start and end 88 for i, text in enumerate(license_text): 89 if text: 90 break 91 for j, text in reversed(list(enumerate(license_text))): 92 if text: 93 break 94 license_text = license_text[i:j+1] 95 96 file_info_map[f]['license'] = '\n'.join(license_text) 97 unlicensed_files = [] 98 license_text = None 99 100 if line == section_divider: 101 driver = 'UNKNOWN' 102 103 if tag == 'Driver': 104 driver = rest 105 elif tag in ['File', 'RawFile', 'Link']: 106 if tag == 'Link': 107 rest, _, link_dest = rest.partition(' -> ') 108 rest = rest.rstrip() 109 link_dest = unquote_path(link_dest.lstrip()) 110 111 rest = unquote_path(rest) 112 113 file_info_map[rest] = { 'driver': driver, 'kind': tag } 114 if tag == 'Link': 115 file_info_map[rest]['link'] = link_dest 116 117 unlicensed_files.append(rest) 118 elif tag in ['License', 'Licence']: 119 license_text = [rest] 120 elif license_text: 121 license_text.append(line) 122 123 return file_info_map 124 125# Look at the license and see if it references other files. 126# 127# Many of the licenses in WHENCE refer to other files in the same directory. 128# This will detect those and return a list of indirectly referenced files. 129def find_indirect_files(license_text): 130 license_files = [] 131 132 # Our regex match works better if there are no carriage returns, so 133 # split everything else and join with spaces. All we care about is 134 # detecting indirectly referenced files anyway. 135 license_text_oneline = ' '.join(license_text.splitlines()) 136 137 # The only phrasing that appears present right now refer to one or two 138 # other files and looks like: 139 # See <filename> for details 140 # See <filename1> and <filename2> for details 141 # 142 # Detect those two. More can be added later. 143 pattern = re2.compile(r'.*[Ss]ee (.*) for details.*') 144 matcher = pattern.matcher(license_text_oneline) 145 if matcher.matches(): 146 for i in range(matcher.group_count()): 147 license_files.extend(matcher.group(i + 1).split(' and ')) 148 149 return license_files 150 151# ########################################################## 152# Templates for generated files 153# ########################################################## 154 155# NOTES: 156# - Right now license_type is always BY_EXCEPTION_ONLY. If this is 157# ever not right we can always add a lookup table by license_kind. 158metadata_template = \ 159'''name: "linux-firmware-{name}" 160description: 161 "Contains an import of upstream linux-firmware for {name}." 162 163third_party {{ 164 homepage: "{upstream_git_repo}" 165 identifier {{ 166 type: "Git" 167 value: "{upstream_git_repo}" 168 primary_source: true 169 version: "{version}" 170 }} 171 version: "{version}" 172 last_upgrade_date {{ year: {year} month: {month} day: {day} }} 173 license_type: BY_EXCEPTION_ONLY 174}} 175''' 176 177# Automatically create the METADATA file under the directory `name`. 178def create_metadata_file(ctx, name): 179 output = metadata_template.format( 180 name = name, 181 year = ctx.now_as_string('yyyy'), 182 month = ctx.now_as_string('M'), 183 day = ctx.now_as_string('d'), 184 version = ctx.fill_template('${GIT_SHA1}'), 185 upstream_git_repo = upstream_git_repo, 186 ) 187 188 ctx.write_path(ctx.new_path(name + '/METADATA'), output) 189 190android_bp_template = \ 191'''// Copyright (C) {year} The Android Open Source Project 192// 193// Licensed under the Apache License, Version 2.0 (the "License"); 194// you may not use this file except in compliance with the License. 195// You may obtain a copy of the License at 196// 197// http://www.apache.org/licenses/LICENSE-2.0 198// 199// Unless required by applicable law or agreed to in writing, software 200// distributed under the License is distributed on an "AS IS" BASIS, 201// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 202// See the License for the specific language governing permissions and 203// limitations under the License. 204// 205// This file is autogenerated by copybara, please do not edit. 206 207license {{ 208 name: "linux_firmware_{name}_license", 209 // Private visibility because nothing links against this. The kernel just 210 // asks for it to be loaded from disk by name. 211 visibility: ["//visibility:private"], 212 license_kinds: [{license_kinds}], 213 license_text: [{license_files}], 214}} 215 216prebuilt_firmware {{ 217 name: "linux_firmware_{name}", 218 licenses: ["linux_firmware_{name}_license"], 219 srcs: [{fw_files}], 220 dsts: [{fw_files}], 221 vendor: true, 222}} 223''' 224 225# Format the an array of strings to go into Android.bp 226def format_android_bp_string_arr(a): 227 if len(a) == 1: 228 return '"%s"' % a[0] 229 230 indent_per = ' ' 231 indent = indent_per * 2 232 return '\n' + indent + \ 233 (',\n' + indent).join(['"%s"' % s for s in a]) + \ 234 ',\n' + indent_per 235 236# Automatically create the Android.bp file under the directory `name`. 237def create_android_bp_file(ctx, name, license_kind, fw_files): 238 output = android_bp_template.format( 239 name = name, 240 license_kinds = format_android_bp_string_arr([license_kind]), 241 license_files = format_android_bp_string_arr(['LICENSE']), 242 fw_files = format_android_bp_string_arr(fw_files), 243 year = ctx.now_as_string('yyyy'), 244 ) 245 ctx.write_path(ctx.new_path(name + '/Android.bp'), output) 246 247# Create the LICENSE file containing all relevant license text. 248def create_license_file(ctx, name, license_text, license_files, fw_files): 249 license_header_strs = [] 250 license_header_strs.append("For the files:") 251 for f in fw_files: 252 license_header_strs.append("- %s" % f) 253 license_header_strs.append("\nThe license is as follows:\n") 254 license_header_strs.append(license_text) 255 256 # Even though the indrectly referenced files are copied to the directory 257 # too, policy says to copy them into the main LICENSE file for easy 258 # reference. 259 license_strs = ['\n'.join(license_header_strs)] 260 for f in license_files: 261 license_strs.append("The text of %s is:\n\n%s" % 262 (f, ctx.read_path(ctx.new_path(name + '/' + f)))) 263 264 ctx.write_path(ctx.new_path(name + '/LICENSE'), 265 '\n\n---\n\n'.join(license_strs)) 266 267commit_message_template = \ 268'''IMPORT: {name} 269 270Import firmware "{name}" using copybara. 271 272Third-Party Import of: {upstream_git_repo} 273Request Document: go/android3p 274For CL Reviewers: go/android3p#reviewing-a-cl 275For Build Team: go/ab-third-party-imports 276Security Questionnaire: REPLACE WITH bug filed by go/android3p process 277Bug: REPLACE WITH motivation bug 278Bug: REPLACE WITH bug filed by go/android3p process 279Test: None 280''' 281 282# Automatically set the commit message. 283def set_commit_message(ctx, name): 284 output = commit_message_template.format( 285 name = name, 286 upstream_git_repo = upstream_git_repo, 287 ) 288 ctx.set_message(output) 289 290# ########################################################## 291# Main transformation 292# ########################################################## 293 294def _firmware_import_transform(ctx): 295 name = ctx.params['name'] 296 license_kind = ctx.params['license_kind'] 297 expected_license_files = ctx.params['license_files'] 298 fw_files = ctx.params['fw_files'] 299 300 # We will read the WHENCE to validate that the license upstream lists for 301 # the fw_files matches the license we think we have. 302 whence_text = ctx.read_path(ctx.new_path('WHENCE')) 303 file_info_map = parse_whence(whence_text) 304 305 # To be a valid import then every fw_file we're importing must have the 306 # same license. Validate that. 307 license_text = file_info_map[fw_files[0]].get('license', '') 308 if not license_text: 309 ctx.console.error('Missing license for "%s"' % fw_files[0]) 310 311 bad_licenses = [f for f in fw_files 312 if file_info_map[f].get('license', '') != license_text] 313 if bad_licenses: 314 ctx.console.error( 315 ('All files in a module must have a matching license. ' + 316 'The license(s) for "%s" don\'t match the license for "%s".') % 317 (', '.join(bad_licenses), fw_files[0]) 318 ) 319 320 for f in expected_license_files: 321 ctx.run(core.move(f, name + '/' + f)) 322 323 for f in fw_files: 324 if file_info_map[f]['kind'] == 'Link': 325 dirname = (name + '/' + f).rsplit('/', 1)[0] 326 ctx.create_symlink(ctx.new_path(name + '/' + f), 327 ctx.new_path(dirname + '/' + file_info_map[f]['link'])) 328 else: 329 ctx.run(core.move(f, name + '/' + f)) 330 331 # Look for indirectly referenced license files since we'll need those too. 332 license_files = find_indirect_files(license_text) 333 334 # copybara required us to specify all the origin files. To make this work 335 # firmware_import_workflow() requires the callers to provide the list of 336 # indirectly referenced license files. Those should match what we detected. 337 if tuple(sorted(license_files)) != tuple(sorted(expected_license_files)): 338 ctx.console.error( 339 ('Upstream WHENCE specified that licenses were %r but we expected %r.') % 340 (license_files, expected_license_files) 341 ) 342 343 create_license_file(ctx, name, license_text, license_files, fw_files) 344 create_metadata_file(ctx, name) 345 create_android_bp_file(ctx, name, license_kind, fw_files) 346 set_commit_message(ctx, name) 347 348 # We don't actually want 'WHENCE' in the destination but we need to read it 349 # so we need to list it in the input files. We can't core.remove() since 350 # that yells at us. Just replace it with some placeholder text. 351 ctx.write_path(ctx.new_path('WHENCE'), 352 'Upstream WHENCE is parsed to confirm licenses; not copied here.\n') 353 354# Create a workflow for the given files. 355def firmware_import_workflow(name, license_kind, license_files, fw_files): 356 return core.workflow( 357 name = name, 358 authoring = authoring.overwrite('linux-firmware importer <[email protected]>'), 359 360 origin = git.origin( 361 url = upstream_git_repo, 362 ref = 'main', 363 ), 364 365 # The below is just a placeholder and will be overridden by command 366 # line arguments passed by `run_copybara.sh`. The script is used 367 # because our copybara flow is different than others. Our flow is: 368 # 1. Add the new firmware import to copy.bara.sky and create a 369 # 'ANDROID:' CL for this. 370 # 2. Run copybara locally which creates an 'IMPORT:' CL. Validate 371 # that it looks OK. 372 # 3. Upload both CLs in a chain and get review. 373 destination = git.destination( 374 url = 'please-use-run_copybara.sh', 375 push = 'please-use-run_copybara.sh', 376 ), 377 378 origin_files = glob( 379 include = license_files + fw_files + ['WHENCE'], 380 ), 381 destination_files = glob( 382 include = [name + '/**'] + ['WHENCE'], 383 ), 384 mode = 'SQUASH', 385 transformations = [ 386 core.dynamic_transform( 387 impl = _firmware_import_transform, 388 params = { 389 'name': name, 390 'license_kind': license_kind, 391 'license_files': license_files, 392 'fw_files': fw_files, 393 }, 394 ), 395 ] 396 ) 397 398# ########################################################## 399# Firmware that we manage, sorted alphabetically 400# ########################################################## 401 402# Realtek r8152 (and related) USB Ethernet adapters 403firmware_import_workflow( 404 name = 'r8152', 405 license_kind = 'BSD-Binary-Only', 406 license_files = [ 407 'LICENCE.rtlwifi_firmware.txt', 408 ], 409 fw_files = [ 410 'rtl_nic/rtl8153a-2.fw', 411 'rtl_nic/rtl8153a-3.fw', 412 'rtl_nic/rtl8153a-4.fw', 413 'rtl_nic/rtl8153b-2.fw', 414 'rtl_nic/rtl8153c-1.fw', 415 'rtl_nic/rtl8156a-2.fw', 416 'rtl_nic/rtl8156b-2.fw', 417 ], 418) 419 420# Ralink RT2800 (and related) USB wireless MACs 421firmware_import_workflow( 422 name = 'rt2800usb', 423 license_kind = 'BSD-Binary-Only', 424 license_files = [ 425 'LICENCE.ralink-firmware.txt', 426 ], 427 fw_files = [ 428 'rt2870.bin', 429 ], 430) 431