1#!/usr/bin/env python 2""" 3This script is used to build "official" universal installers on macOS. 4 5NEW for 3.10 and backports: 6- support universal2 variant with arm64 and x86_64 archs 7- enable clang optimizations when building on 10.15+ 8 9NEW for 3.9.0 and backports: 10- 2.7 end-of-life issues: 11 - Python 3 installs now update the Current version link 12 in /Library/Frameworks/Python.framework/Versions 13- fully support running under Python 3 as well as 2.7 14- support building on newer macOS systems with SIP 15- fully support building on macOS 10.9+ 16- support 10.6+ on best effort 17- support bypassing docs build by supplying a prebuilt 18 docs html tarball in the third-party source library, 19 in the format and filename conventional of those 20 downloadable from python.org: 21 python-3.x.y-docs-html.tar.bz2 22 23NEW for 3.7.0: 24- support Intel 64-bit-only () and 32-bit-only installer builds 25- build and use internal Tcl/Tk 8.6 for 10.6+ builds 26- deprecate use of explicit SDK (--sdk-path=) since all but the oldest 27 versions of Xcode support implicit setting of an SDK via environment 28 variables (SDKROOT and friends, see the xcrun man page for more info). 29 The SDK stuff was primarily needed for building universal installers 30 for 10.4; so as of 3.7.0, building installers for 10.4 is no longer 31 supported with build-installer. 32- use generic "gcc" as compiler (CC env var) rather than "gcc-4.2" 33 34TODO: 35- test building with SDKROOT and DEVELOPER_DIR xcrun env variables 36 37Usage: see USAGE variable in the script. 38""" 39import platform, os, sys, getopt, textwrap, shutil, stat, time, pwd, grp 40try: 41 import urllib2 as urllib_request 42except ImportError: 43 import urllib.request as urllib_request 44 45STAT_0o755 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR 46 | stat.S_IRGRP | stat.S_IXGRP 47 | stat.S_IROTH | stat.S_IXOTH ) 48 49STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR 50 | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP 51 | stat.S_IROTH | stat.S_IXOTH ) 52 53INCLUDE_TIMESTAMP = 1 54VERBOSE = 1 55 56RUNNING_ON_PYTHON2 = sys.version_info.major == 2 57 58if RUNNING_ON_PYTHON2: 59 from plistlib import writePlist 60else: 61 from plistlib import dump 62 def writePlist(path, plist): 63 with open(plist, 'wb') as fp: 64 dump(path, fp) 65 66def shellQuote(value): 67 """ 68 Return the string value in a form that can safely be inserted into 69 a shell command. 70 """ 71 return "'%s'"%(value.replace("'", "'\"'\"'")) 72 73def grepValue(fn, variable): 74 """ 75 Return the unquoted value of a variable from a file.. 76 QUOTED_VALUE='quotes' -> str('quotes') 77 UNQUOTED_VALUE=noquotes -> str('noquotes') 78 """ 79 variable = variable + '=' 80 for ln in open(fn, 'r'): 81 if ln.startswith(variable): 82 value = ln[len(variable):].strip() 83 return value.strip("\"'") 84 raise RuntimeError("Cannot find variable %s" % variable[:-1]) 85 86_cache_getVersion = None 87 88def getVersion(): 89 global _cache_getVersion 90 if _cache_getVersion is None: 91 _cache_getVersion = grepValue( 92 os.path.join(SRCDIR, 'configure'), 'PACKAGE_VERSION') 93 return _cache_getVersion 94 95def getVersionMajorMinor(): 96 return tuple([int(n) for n in getVersion().split('.', 2)]) 97 98_cache_getFullVersion = None 99 100def getFullVersion(): 101 global _cache_getFullVersion 102 if _cache_getFullVersion is not None: 103 return _cache_getFullVersion 104 fn = os.path.join(SRCDIR, 'Include', 'patchlevel.h') 105 for ln in open(fn): 106 if 'PY_VERSION' in ln: 107 _cache_getFullVersion = ln.split()[-1][1:-1] 108 return _cache_getFullVersion 109 raise RuntimeError("Cannot find full version??") 110 111FW_PREFIX = ["Library", "Frameworks", "Python.framework"] 112FW_VERSION_PREFIX = "--undefined--" # initialized in parseOptions 113FW_SSL_DIRECTORY = "--undefined--" # initialized in parseOptions 114 115# The directory we'll use to create the build (will be erased and recreated) 116WORKDIR = "/tmp/_py" 117 118# The directory we'll use to store third-party sources. Set this to something 119# else if you don't want to re-fetch required libraries every time. 120DEPSRC = os.path.join(WORKDIR, 'third-party') 121DEPSRC = os.path.expanduser('~/Universal/other-sources') 122 123universal_opts_map = { 'universal2': ('arm64', 'x86_64'), 124 '32-bit': ('i386', 'ppc',), 125 '64-bit': ('x86_64', 'ppc64',), 126 'intel': ('i386', 'x86_64'), 127 'intel-32': ('i386',), 128 'intel-64': ('x86_64',), 129 '3-way': ('ppc', 'i386', 'x86_64'), 130 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } 131default_target_map = { 132 'universal2': '10.9', 133 '64-bit': '10.5', 134 '3-way': '10.5', 135 'intel': '10.5', 136 'intel-32': '10.4', 137 'intel-64': '10.5', 138 'all': '10.5', 139} 140 141UNIVERSALOPTS = tuple(universal_opts_map.keys()) 142 143UNIVERSALARCHS = '32-bit' 144 145ARCHLIST = universal_opts_map[UNIVERSALARCHS] 146 147# Source directory (assume we're in Mac/BuildScript) 148SRCDIR = os.path.dirname( 149 os.path.dirname( 150 os.path.dirname( 151 os.path.abspath(__file__ 152 )))) 153 154# $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level 155DEPTARGET = '10.5' 156 157def getDeptargetTuple(): 158 return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) 159 160def getBuildTuple(): 161 return tuple([int(n) for n in platform.mac_ver()[0].split('.')[0:2]]) 162 163def getTargetCompilers(): 164 target_cc_map = { 165 '10.4': ('gcc-4.0', 'g++-4.0'), 166 '10.5': ('gcc', 'g++'), 167 '10.6': ('gcc', 'g++'), 168 '10.7': ('gcc', 'g++'), 169 '10.8': ('gcc', 'g++'), 170 } 171 return target_cc_map.get(DEPTARGET, ('clang', 'clang++') ) 172 173CC, CXX = getTargetCompilers() 174 175PYTHON_3 = getVersionMajorMinor() >= (3, 0) 176 177USAGE = textwrap.dedent("""\ 178 Usage: build_python [options] 179 180 Options: 181 -? or -h: Show this message 182 -b DIR 183 --build-dir=DIR: Create build here (default: %(WORKDIR)r) 184 --third-party=DIR: Store third-party sources here (default: %(DEPSRC)r) 185 --sdk-path=DIR: Location of the SDK (deprecated, use SDKROOT env variable) 186 --src-dir=DIR: Location of the Python sources (default: %(SRCDIR)r) 187 --dep-target=10.n macOS deployment target (default: %(DEPTARGET)r) 188 --universal-archs=x universal architectures (options: %(UNIVERSALOPTS)r, default: %(UNIVERSALARCHS)r) 189""")% globals() 190 191# Dict of object file names with shared library names to check after building. 192# This is to ensure that we ended up dynamically linking with the shared 193# library paths and versions we expected. For example: 194# EXPECTED_SHARED_LIBS['_tkinter.so'] = [ 195# '/Library/Frameworks/Tcl.framework/Versions/8.5/Tcl', 196# '/Library/Frameworks/Tk.framework/Versions/8.5/Tk'] 197EXPECTED_SHARED_LIBS = {} 198 199# Are we building and linking with our own copy of Tcl/TK? 200# For now, do so if deployment target is 10.6+. 201def internalTk(): 202 return getDeptargetTuple() >= (10, 6) 203 204# Do we use 8.6.8 when building our own copy 205# of Tcl/Tk or a modern version. 206# We use the old version when buildin on 207# old versions of macOS due to build issues. 208def useOldTk(): 209 return getBuildTuple() < (10, 15) 210 211 212def tweak_tcl_build(basedir, archList): 213 with open("Makefile", "r") as fp: 214 contents = fp.readlines() 215 216 # For reasons I don't understand the tcl configure script 217 # decides that some stdlib symbols aren't present, before 218 # deciding that strtod is broken. 219 new_contents = [] 220 for line in contents: 221 if line.startswith("COMPAT_OBJS"): 222 # note: the space before strtod.o is intentional, 223 # the detection of a broken strtod results in 224 # "fixstrod.o" on this line. 225 for nm in ("strstr.o", "strtoul.o", " strtod.o"): 226 line = line.replace(nm, "") 227 new_contents.append(line) 228 229 with open("Makefile", "w") as fp: 230 fp.writelines(new_contents) 231 232# List of names of third party software built with this installer. 233# The names will be inserted into the rtf version of the License. 234THIRD_PARTY_LIBS = [] 235 236# Instructions for building libraries that are necessary for building a 237# batteries included python. 238# [The recipes are defined here for convenience but instantiated later after 239# command line options have been processed.] 240def library_recipes(): 241 result = [] 242 243 # Since Apple removed the header files for the deprecated system 244 # OpenSSL as of the Xcode 7 release (for OS X 10.10+), we do not 245 # have much choice but to build our own copy here, too. 246 247 result.extend([ 248 dict( 249 name="OpenSSL 1.1.1u", 250 url="https://www.openssl.org/source/openssl-1.1.1u.tar.gz", 251 checksum='e2f8d84b523eecd06c7be7626830370300fbcc15386bf5142d72758f6963ebc6', 252 buildrecipe=build_universal_openssl, 253 configure=None, 254 install=None, 255 ), 256 ]) 257 258 if internalTk(): 259 if useOldTk(): 260 tcl_tk_ver='8.6.8' 261 tcl_checksum='81656d3367af032e0ae6157eff134f89' 262 263 tk_checksum='5e0faecba458ee1386078fb228d008ba' 264 tk_patches = ['tk868_on_10_8_10_9.patch'] 265 266 else: 267 tcl_tk_ver='8.6.12' 268 tcl_checksum='87ea890821d2221f2ab5157bc5eb885f' 269 270 tk_checksum='1d6dcf6120356e3d211e056dff5e462a' 271 tk_patches = [ ] 272 273 274 result.extend([ 275 dict( 276 name="Tcl %s"%(tcl_tk_ver,), 277 url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tcl%s-src.tar.gz"%(tcl_tk_ver,), 278 checksum=tcl_checksum, 279 buildDir="unix", 280 configure_pre=[ 281 '--enable-shared', 282 '--enable-threads', 283 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), 284 ], 285 useLDFlags=False, 286 buildrecipe=tweak_tcl_build, 287 install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ 288 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), 289 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), 290 }, 291 ), 292 dict( 293 name="Tk %s"%(tcl_tk_ver,), 294 url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tk%s-src.tar.gz"%(tcl_tk_ver,), 295 checksum=tk_checksum, 296 patches=tk_patches, 297 buildDir="unix", 298 configure_pre=[ 299 '--enable-aqua', 300 '--enable-shared', 301 '--enable-threads', 302 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), 303 ], 304 useLDFlags=False, 305 install='make TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ 306 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), 307 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), 308 "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.6'%(getVersion())), 309 }, 310 ), 311 ]) 312 313 if PYTHON_3: 314 result.extend([ 315 dict( 316 name="XZ 5.2.3", 317 url="http://tukaani.org/xz/xz-5.2.3.tar.gz", 318 checksum='ef68674fb47a8b8e741b34e429d86e9d', 319 configure_pre=[ 320 '--disable-dependency-tracking', 321 ] 322 ), 323 ]) 324 325 result.extend([ 326 dict( 327 name="NCurses 5.9", 328 url="http://ftp.gnu.org/pub/gnu/ncurses/ncurses-5.9.tar.gz", 329 checksum='8cb9c412e5f2d96bc6f459aa8c6282a1', 330 configure_pre=[ 331 "--enable-widec", 332 "--without-cxx", 333 "--without-cxx-binding", 334 "--without-ada", 335 "--without-curses-h", 336 "--enable-shared", 337 "--with-shared", 338 "--without-debug", 339 "--without-normal", 340 "--without-tests", 341 "--without-manpages", 342 "--datadir=/usr/share", 343 "--sysconfdir=/etc", 344 "--sharedstatedir=/usr/com", 345 "--with-terminfo-dirs=/usr/share/terminfo", 346 "--with-default-terminfo-dir=/usr/share/terminfo", 347 "--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib"%(getVersion(),), 348 ], 349 patchscripts=[ 350 ("ftp://ftp.invisible-island.net/ncurses//5.9/ncurses-5.9-20120616-patch.sh.bz2", 351 "f54bf02a349f96a7c4f0d00922f3a0d4"), 352 ], 353 useLDFlags=False, 354 install='make && make install DESTDIR=%s && cd %s/usr/local/lib && ln -fs ../../../Library/Frameworks/Python.framework/Versions/%s/lib/lib* .'%( 355 shellQuote(os.path.join(WORKDIR, 'libraries')), 356 shellQuote(os.path.join(WORKDIR, 'libraries')), 357 getVersion(), 358 ), 359 ), 360 dict( 361 name="SQLite 3.42.0", 362 url="https://sqlite.org/2023/sqlite-autoconf-3420000.tar.gz", 363 checksum="0c5a92bc51cf07cae45b4a1e94653dea", 364 extra_cflags=('-Os ' 365 '-DSQLITE_ENABLE_FTS5 ' 366 '-DSQLITE_ENABLE_FTS4 ' 367 '-DSQLITE_ENABLE_FTS3_PARENTHESIS ' 368 '-DSQLITE_ENABLE_RTREE ' 369 '-DSQLITE_OMIT_AUTOINIT ' 370 '-DSQLITE_TCL=0 ' 371 ), 372 configure_pre=[ 373 '--enable-threadsafe', 374 '--enable-shared=no', 375 '--enable-static=yes', 376 '--disable-readline', 377 '--disable-dependency-tracking', 378 ] 379 ), 380 ]) 381 382 if not PYTHON_3: 383 result.extend([ 384 dict( 385 name="Sleepycat DB 4.7.25", 386 url="http://download.oracle.com/berkeley-db/db-4.7.25.tar.gz", 387 checksum='ec2b87e833779681a0c3a814aa71359e', 388 buildDir="build_unix", 389 configure="../dist/configure", 390 configure_pre=[ 391 '--includedir=/usr/local/include/db4', 392 ] 393 ), 394 ]) 395 396 return result 397 398def compilerCanOptimize(): 399 """ 400 Return True iff the default Xcode version can use PGO and LTO 401 """ 402 # bpo-42235: The version check is pretty conservative, can be 403 # adjusted after testing 404 mac_ver = tuple(map(int, platform.mac_ver()[0].split('.'))) 405 return mac_ver >= (10, 15) 406 407# Instructions for building packages inside the .mpkg. 408def pkg_recipes(): 409 unselected_for_python3 = ('selected', 'unselected')[PYTHON_3] 410 result = [ 411 dict( 412 name="PythonFramework", 413 long_name="Python Framework", 414 source="/Library/Frameworks/Python.framework", 415 readme="""\ 416 This package installs Python.framework, that is the python 417 interpreter and the standard library. 418 """, 419 postflight="scripts/postflight.framework", 420 selected='selected', 421 ), 422 dict( 423 name="PythonApplications", 424 long_name="GUI Applications", 425 source="/Applications/Python %(VER)s", 426 readme="""\ 427 This package installs IDLE (an interactive Python IDE), 428 Python Launcher and Build Applet (create application bundles 429 from python scripts). 430 431 It also installs a number of examples and demos. 432 """, 433 required=False, 434 selected='selected', 435 ), 436 dict( 437 name="PythonUnixTools", 438 long_name="UNIX command-line tools", 439 source="/usr/local/bin", 440 readme="""\ 441 This package installs the unix tools in /usr/local/bin for 442 compatibility with older releases of Python. This package 443 is not necessary to use Python. 444 """, 445 required=False, 446 selected='selected', 447 ), 448 dict( 449 name="PythonDocumentation", 450 long_name="Python Documentation", 451 topdir="/Library/Frameworks/Python.framework/Versions/%(VER)s/Resources/English.lproj/Documentation", 452 source="/pydocs", 453 readme="""\ 454 This package installs the python documentation at a location 455 that is usable for pydoc and IDLE. 456 """, 457 postflight="scripts/postflight.documentation", 458 required=False, 459 selected='selected', 460 ), 461 dict( 462 name="PythonProfileChanges", 463 long_name="Shell profile updater", 464 readme="""\ 465 This packages updates your shell profile to make sure that 466 the Python tools are found by your shell in preference of 467 the system provided Python tools. 468 469 If you don't install this package you'll have to add 470 "/Library/Frameworks/Python.framework/Versions/%(VER)s/bin" 471 to your PATH by hand. 472 """, 473 postflight="scripts/postflight.patch-profile", 474 topdir="/Library/Frameworks/Python.framework", 475 source="/empty-dir", 476 required=False, 477 selected='selected', 478 ), 479 dict( 480 name="PythonInstallPip", 481 long_name="Install or upgrade pip", 482 readme="""\ 483 This package installs (or upgrades from an earlier version) 484 pip, a tool for installing and managing Python packages. 485 """, 486 postflight="scripts/postflight.ensurepip", 487 topdir="/Library/Frameworks/Python.framework", 488 source="/empty-dir", 489 required=False, 490 selected='selected', 491 ), 492 ] 493 494 return result 495 496def fatal(msg): 497 """ 498 A fatal error, bail out. 499 """ 500 sys.stderr.write('FATAL: ') 501 sys.stderr.write(msg) 502 sys.stderr.write('\n') 503 sys.exit(1) 504 505def fileContents(fn): 506 """ 507 Return the contents of the named file 508 """ 509 return open(fn, 'r').read() 510 511def runCommand(commandline): 512 """ 513 Run a command and raise RuntimeError if it fails. Output is suppressed 514 unless the command fails. 515 """ 516 fd = os.popen(commandline, 'r') 517 data = fd.read() 518 xit = fd.close() 519 if xit is not None: 520 sys.stdout.write(data) 521 raise RuntimeError("command failed: %s"%(commandline,)) 522 523 if VERBOSE: 524 sys.stdout.write(data); sys.stdout.flush() 525 526def captureCommand(commandline): 527 fd = os.popen(commandline, 'r') 528 data = fd.read() 529 xit = fd.close() 530 if xit is not None: 531 sys.stdout.write(data) 532 raise RuntimeError("command failed: %s"%(commandline,)) 533 534 return data 535 536def getTclTkVersion(configfile, versionline): 537 """ 538 search Tcl or Tk configuration file for version line 539 """ 540 try: 541 f = open(configfile, "r") 542 except OSError: 543 fatal("Framework configuration file not found: %s" % configfile) 544 545 for l in f: 546 if l.startswith(versionline): 547 f.close() 548 return l 549 550 fatal("Version variable %s not found in framework configuration file: %s" 551 % (versionline, configfile)) 552 553def checkEnvironment(): 554 """ 555 Check that we're running on a supported system. 556 """ 557 558 if sys.version_info[0:2] < (2, 7): 559 fatal("This script must be run with Python 2.7 (or later)") 560 561 if platform.system() != 'Darwin': 562 fatal("This script should be run on a macOS 10.5 (or later) system") 563 564 if int(platform.release().split('.')[0]) < 8: 565 fatal("This script should be run on a macOS 10.5 (or later) system") 566 567 # Because we only support dynamic load of only one major/minor version of 568 # Tcl/Tk, if we are not using building and using our own private copy of 569 # Tcl/Tk, ensure: 570 # 1. there is a user-installed framework (usually ActiveTcl) in (or linked 571 # in) SDKROOT/Library/Frameworks. As of Python 3.7.0, we no longer 572 # enforce that the version of the user-installed framework also 573 # exists in the system-supplied Tcl/Tk frameworks. Time to support 574 # Tcl/Tk 8.6 even if Apple does not. 575 if not internalTk(): 576 frameworks = {} 577 for framework in ['Tcl', 'Tk']: 578 fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework 579 libfw = os.path.join('/', fwpth) 580 usrfw = os.path.join(os.getenv('HOME'), fwpth) 581 frameworks[framework] = os.readlink(libfw) 582 if not os.path.exists(libfw): 583 fatal("Please install a link to a current %s %s as %s so " 584 "the user can override the system framework." 585 % (framework, frameworks[framework], libfw)) 586 if os.path.exists(usrfw): 587 fatal("Please rename %s to avoid possible dynamic load issues." 588 % usrfw) 589 590 if frameworks['Tcl'] != frameworks['Tk']: 591 fatal("The Tcl and Tk frameworks are not the same version.") 592 593 print(" -- Building with external Tcl/Tk %s frameworks" 594 % frameworks['Tk']) 595 596 # add files to check after build 597 EXPECTED_SHARED_LIBS['_tkinter.so'] = [ 598 "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl" 599 % frameworks['Tcl'], 600 "/Library/Frameworks/Tk.framework/Versions/%s/Tk" 601 % frameworks['Tk'], 602 ] 603 else: 604 print(" -- Building private copy of Tcl/Tk") 605 print("") 606 607 # Remove inherited environment variables which might influence build 608 environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_', 609 'LD_', 'LIBRARY_', 'PATH', 'PYTHON'] 610 for ev in list(os.environ): 611 for prefix in environ_var_prefixes: 612 if ev.startswith(prefix) : 613 print("INFO: deleting environment variable %s=%s" % ( 614 ev, os.environ[ev])) 615 del os.environ[ev] 616 617 base_path = '/bin:/sbin:/usr/bin:/usr/sbin' 618 if 'SDK_TOOLS_BIN' in os.environ: 619 base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path 620 # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin; 621 # add its fixed location here if it exists 622 OLD_DEVELOPER_TOOLS = '/Developer/Tools' 623 if os.path.isdir(OLD_DEVELOPER_TOOLS): 624 base_path = base_path + ':' + OLD_DEVELOPER_TOOLS 625 os.environ['PATH'] = base_path 626 print("Setting default PATH: %s"%(os.environ['PATH'])) 627 628def parseOptions(args=None): 629 """ 630 Parse arguments and update global settings. 631 """ 632 global WORKDIR, DEPSRC, SRCDIR, DEPTARGET 633 global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX 634 global FW_VERSION_PREFIX 635 global FW_SSL_DIRECTORY 636 637 if args is None: 638 args = sys.argv[1:] 639 640 try: 641 options, args = getopt.getopt(args, '?hb', 642 [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=', 643 'dep-target=', 'universal-archs=', 'help' ]) 644 except getopt.GetoptError: 645 print(sys.exc_info()[1]) 646 sys.exit(1) 647 648 if args: 649 print("Additional arguments") 650 sys.exit(1) 651 652 deptarget = None 653 for k, v in options: 654 if k in ('-h', '-?', '--help'): 655 print(USAGE) 656 sys.exit(0) 657 658 elif k in ('-d', '--build-dir'): 659 WORKDIR=v 660 661 elif k in ('--third-party',): 662 DEPSRC=v 663 664 elif k in ('--sdk-path',): 665 print(" WARNING: --sdk-path is no longer supported") 666 667 elif k in ('--src-dir',): 668 SRCDIR=v 669 670 elif k in ('--dep-target', ): 671 DEPTARGET=v 672 deptarget=v 673 674 elif k in ('--universal-archs', ): 675 if v in UNIVERSALOPTS: 676 UNIVERSALARCHS = v 677 ARCHLIST = universal_opts_map[UNIVERSALARCHS] 678 if deptarget is None: 679 # Select alternate default deployment 680 # target 681 DEPTARGET = default_target_map.get(v, '10.5') 682 else: 683 raise NotImplementedError(v) 684 685 else: 686 raise NotImplementedError(k) 687 688 SRCDIR=os.path.abspath(SRCDIR) 689 WORKDIR=os.path.abspath(WORKDIR) 690 DEPSRC=os.path.abspath(DEPSRC) 691 692 CC, CXX = getTargetCompilers() 693 694 FW_VERSION_PREFIX = FW_PREFIX[:] + ["Versions", getVersion()] 695 FW_SSL_DIRECTORY = FW_VERSION_PREFIX[:] + ["etc", "openssl"] 696 697 print("-- Settings:") 698 print(" * Source directory: %s" % SRCDIR) 699 print(" * Build directory: %s" % WORKDIR) 700 print(" * Third-party source: %s" % DEPSRC) 701 print(" * Deployment target: %s" % DEPTARGET) 702 print(" * Universal archs: %s" % str(ARCHLIST)) 703 print(" * C compiler: %s" % CC) 704 print(" * C++ compiler: %s" % CXX) 705 print("") 706 print(" -- Building a Python %s framework at patch level %s" 707 % (getVersion(), getFullVersion())) 708 print("") 709 710def extractArchive(builddir, archiveName): 711 """ 712 Extract a source archive into 'builddir'. Returns the path of the 713 extracted archive. 714 715 XXX: This function assumes that archives contain a toplevel directory 716 that is has the same name as the basename of the archive. This is 717 safe enough for almost anything we use. Unfortunately, it does not 718 work for current Tcl and Tk source releases where the basename of 719 the archive ends with "-src" but the uncompressed directory does not. 720 For now, just special case Tcl and Tk tar.gz downloads. 721 """ 722 curdir = os.getcwd() 723 try: 724 os.chdir(builddir) 725 if archiveName.endswith('.tar.gz'): 726 retval = os.path.basename(archiveName[:-7]) 727 if ((retval.startswith('tcl') or retval.startswith('tk')) 728 and retval.endswith('-src')): 729 retval = retval[:-4] 730 # Strip rcxx suffix from Tcl/Tk release candidates 731 retval_rc = retval.find('rc') 732 if retval_rc > 0: 733 retval = retval[:retval_rc] 734 if os.path.exists(retval): 735 shutil.rmtree(retval) 736 fp = os.popen("tar zxf %s 2>&1"%(shellQuote(archiveName),), 'r') 737 738 elif archiveName.endswith('.tar.bz2'): 739 retval = os.path.basename(archiveName[:-8]) 740 if os.path.exists(retval): 741 shutil.rmtree(retval) 742 fp = os.popen("tar jxf %s 2>&1"%(shellQuote(archiveName),), 'r') 743 744 elif archiveName.endswith('.tar'): 745 retval = os.path.basename(archiveName[:-4]) 746 if os.path.exists(retval): 747 shutil.rmtree(retval) 748 fp = os.popen("tar xf %s 2>&1"%(shellQuote(archiveName),), 'r') 749 750 elif archiveName.endswith('.zip'): 751 retval = os.path.basename(archiveName[:-4]) 752 if os.path.exists(retval): 753 shutil.rmtree(retval) 754 fp = os.popen("unzip %s 2>&1"%(shellQuote(archiveName),), 'r') 755 756 data = fp.read() 757 xit = fp.close() 758 if xit is not None: 759 sys.stdout.write(data) 760 raise RuntimeError("Cannot extract %s"%(archiveName,)) 761 762 return os.path.join(builddir, retval) 763 764 finally: 765 os.chdir(curdir) 766 767def downloadURL(url, fname): 768 """ 769 Download the contents of the url into the file. 770 """ 771 fpIn = urllib_request.urlopen(url) 772 fpOut = open(fname, 'wb') 773 block = fpIn.read(10240) 774 try: 775 while block: 776 fpOut.write(block) 777 block = fpIn.read(10240) 778 fpIn.close() 779 fpOut.close() 780 except: 781 try: 782 os.unlink(fname) 783 except OSError: 784 pass 785 786def verifyThirdPartyFile(url, checksum, fname): 787 """ 788 Download file from url to filename fname if it does not already exist. 789 Abort if file contents does not match supplied md5 checksum. 790 """ 791 name = os.path.basename(fname) 792 if os.path.exists(fname): 793 print("Using local copy of %s"%(name,)) 794 else: 795 print("Did not find local copy of %s"%(name,)) 796 print("Downloading %s"%(name,)) 797 downloadURL(url, fname) 798 print("Archive for %s stored as %s"%(name, fname)) 799 if len(checksum) == 32: 800 algo = 'md5' 801 elif len(checksum) == 64: 802 algo = 'sha256' 803 else: 804 raise ValueError(checksum) 805 if os.system( 806 'CHECKSUM=$(openssl %s %s) ; test "${CHECKSUM##*= }" = "%s"' 807 % (algo, shellQuote(fname), checksum) ): 808 fatal('%s checksum mismatch for file %s' % (algo, fname)) 809 810def build_universal_openssl(basedir, archList): 811 """ 812 Special case build recipe for universal build of openssl. 813 814 The upstream OpenSSL build system does not directly support 815 OS X universal builds. We need to build each architecture 816 separately then lipo them together into fat libraries. 817 """ 818 819 # OpenSSL fails to build with Xcode 2.5 (on OS X 10.4). 820 # If we are building on a 10.4.x or earlier system, 821 # unilaterally disable assembly code building to avoid the problem. 822 no_asm = int(platform.release().split(".")[0]) < 9 823 824 def build_openssl_arch(archbase, arch): 825 "Build one architecture of openssl" 826 arch_opts = { 827 "i386": ["darwin-i386-cc"], 828 "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"], 829 "arm64": ["darwin64-arm64-cc"], 830 "ppc": ["darwin-ppc-cc"], 831 "ppc64": ["darwin64-ppc-cc"], 832 } 833 834 # Somewhere between OpenSSL 1.1.0j and 1.1.1c, changes cause the 835 # "enable-ec_nistp_64_gcc_128" option to get compile errors when 836 # building on our 10.6 gcc-4.2 environment. There have been other 837 # reports of projects running into this when using older compilers. 838 # So, for now, do not try to use "enable-ec_nistp_64_gcc_128" when 839 # building for 10.6. 840 if getDeptargetTuple() == (10, 6): 841 arch_opts['x86_64'].remove('enable-ec_nistp_64_gcc_128') 842 843 configure_opts = [ 844 "no-idea", 845 "no-mdc2", 846 "no-rc5", 847 "no-zlib", 848 "no-ssl3", 849 # "enable-unit-test", 850 "shared", 851 "--prefix=%s"%os.path.join("/", *FW_VERSION_PREFIX), 852 "--openssldir=%s"%os.path.join("/", *FW_SSL_DIRECTORY), 853 ] 854 if no_asm: 855 configure_opts.append("no-asm") 856 runCommand(" ".join(["perl", "Configure"] 857 + arch_opts[arch] + configure_opts)) 858 runCommand("make depend") 859 runCommand("make all") 860 runCommand("make install_sw DESTDIR=%s"%shellQuote(archbase)) 861 # runCommand("make test") 862 return 863 864 srcdir = os.getcwd() 865 universalbase = os.path.join(srcdir, "..", 866 os.path.basename(srcdir) + "-universal") 867 os.mkdir(universalbase) 868 archbasefws = [] 869 for arch in archList: 870 # fresh copy of the source tree 871 archsrc = os.path.join(universalbase, arch, "src") 872 shutil.copytree(srcdir, archsrc, symlinks=True) 873 # install base for this arch 874 archbase = os.path.join(universalbase, arch, "root") 875 os.mkdir(archbase) 876 # Python framework base within install_prefix: 877 # the build will install into this framework.. 878 # This is to ensure that the resulting shared libs have 879 # the desired real install paths built into them. 880 archbasefw = os.path.join(archbase, *FW_VERSION_PREFIX) 881 882 # build one architecture 883 os.chdir(archsrc) 884 build_openssl_arch(archbase, arch) 885 os.chdir(srcdir) 886 archbasefws.append(archbasefw) 887 888 # copy arch-independent files from last build into the basedir framework 889 basefw = os.path.join(basedir, *FW_VERSION_PREFIX) 890 shutil.copytree( 891 os.path.join(archbasefw, "include", "openssl"), 892 os.path.join(basefw, "include", "openssl") 893 ) 894 895 shlib_version_number = grepValue(os.path.join(archsrc, "Makefile"), 896 "SHLIB_VERSION_NUMBER") 897 # e.g. -> "1.0.0" 898 libcrypto = "libcrypto.dylib" 899 libcrypto_versioned = libcrypto.replace(".", "."+shlib_version_number+".") 900 # e.g. -> "libcrypto.1.0.0.dylib" 901 libssl = "libssl.dylib" 902 libssl_versioned = libssl.replace(".", "."+shlib_version_number+".") 903 # e.g. -> "libssl.1.0.0.dylib" 904 905 try: 906 os.mkdir(os.path.join(basefw, "lib")) 907 except OSError: 908 pass 909 910 # merge the individual arch-dependent shared libs into a fat shared lib 911 archbasefws.insert(0, basefw) 912 for (lib_unversioned, lib_versioned) in [ 913 (libcrypto, libcrypto_versioned), 914 (libssl, libssl_versioned) 915 ]: 916 runCommand("lipo -create -output " + 917 " ".join(shellQuote( 918 os.path.join(fw, "lib", lib_versioned)) 919 for fw in archbasefws)) 920 # and create an unversioned symlink of it 921 os.symlink(lib_versioned, os.path.join(basefw, "lib", lib_unversioned)) 922 923 # Create links in the temp include and lib dirs that will be injected 924 # into the Python build so that setup.py can find them while building 925 # and the versioned links so that the setup.py post-build import test 926 # does not fail. 927 relative_path = os.path.join("..", "..", "..", *FW_VERSION_PREFIX) 928 for fn in [ 929 ["include", "openssl"], 930 ["lib", libcrypto], 931 ["lib", libssl], 932 ["lib", libcrypto_versioned], 933 ["lib", libssl_versioned], 934 ]: 935 os.symlink( 936 os.path.join(relative_path, *fn), 937 os.path.join(basedir, "usr", "local", *fn) 938 ) 939 940 return 941 942def buildRecipe(recipe, basedir, archList): 943 """ 944 Build software using a recipe. This function does the 945 'configure;make;make install' dance for C software, with a possibility 946 to customize this process, basically a poor-mans DarwinPorts. 947 """ 948 curdir = os.getcwd() 949 950 name = recipe['name'] 951 THIRD_PARTY_LIBS.append(name) 952 url = recipe['url'] 953 configure = recipe.get('configure', './configure') 954 buildrecipe = recipe.get('buildrecipe', None) 955 install = recipe.get('install', 'make && make install DESTDIR=%s'%( 956 shellQuote(basedir))) 957 958 archiveName = os.path.split(url)[-1] 959 sourceArchive = os.path.join(DEPSRC, archiveName) 960 961 if not os.path.exists(DEPSRC): 962 os.mkdir(DEPSRC) 963 964 verifyThirdPartyFile(url, recipe['checksum'], sourceArchive) 965 print("Extracting archive for %s"%(name,)) 966 buildDir=os.path.join(WORKDIR, '_bld') 967 if not os.path.exists(buildDir): 968 os.mkdir(buildDir) 969 970 workDir = extractArchive(buildDir, sourceArchive) 971 os.chdir(workDir) 972 973 for patch in recipe.get('patches', ()): 974 if isinstance(patch, tuple): 975 url, checksum = patch 976 fn = os.path.join(DEPSRC, os.path.basename(url)) 977 verifyThirdPartyFile(url, checksum, fn) 978 else: 979 # patch is a file in the source directory 980 fn = os.path.join(curdir, patch) 981 runCommand('patch -p%s < %s'%(recipe.get('patchlevel', 1), 982 shellQuote(fn),)) 983 984 for patchscript in recipe.get('patchscripts', ()): 985 if isinstance(patchscript, tuple): 986 url, checksum = patchscript 987 fn = os.path.join(DEPSRC, os.path.basename(url)) 988 verifyThirdPartyFile(url, checksum, fn) 989 else: 990 # patch is a file in the source directory 991 fn = os.path.join(curdir, patchscript) 992 if fn.endswith('.bz2'): 993 runCommand('bunzip2 -fk %s' % shellQuote(fn)) 994 fn = fn[:-4] 995 runCommand('sh %s' % shellQuote(fn)) 996 os.unlink(fn) 997 998 if 'buildDir' in recipe: 999 os.chdir(recipe['buildDir']) 1000 1001 if configure is not None: 1002 configure_args = [ 1003 "--prefix=/usr/local", 1004 "--enable-static", 1005 "--disable-shared", 1006 #"CPP=gcc -arch %s -E"%(' -arch '.join(archList,),), 1007 ] 1008 1009 if 'configure_pre' in recipe: 1010 args = list(recipe['configure_pre']) 1011 if '--disable-static' in args: 1012 configure_args.remove('--enable-static') 1013 if '--enable-shared' in args: 1014 configure_args.remove('--disable-shared') 1015 configure_args.extend(args) 1016 1017 if recipe.get('useLDFlags', 1): 1018 configure_args.extend([ 1019 "CFLAGS=%s-mmacosx-version-min=%s -arch %s " 1020 "-I%s/usr/local/include"%( 1021 recipe.get('extra_cflags', ''), 1022 DEPTARGET, 1023 ' -arch '.join(archList), 1024 shellQuote(basedir)[1:-1],), 1025 "LDFLAGS=-mmacosx-version-min=%s -L%s/usr/local/lib -arch %s"%( 1026 DEPTARGET, 1027 shellQuote(basedir)[1:-1], 1028 ' -arch '.join(archList)), 1029 ]) 1030 else: 1031 configure_args.extend([ 1032 "CFLAGS=%s-mmacosx-version-min=%s -arch %s " 1033 "-I%s/usr/local/include"%( 1034 recipe.get('extra_cflags', ''), 1035 DEPTARGET, 1036 ' -arch '.join(archList), 1037 shellQuote(basedir)[1:-1],), 1038 ]) 1039 1040 if 'configure_post' in recipe: 1041 configure_args = configure_args + list(recipe['configure_post']) 1042 1043 configure_args.insert(0, configure) 1044 configure_args = [ shellQuote(a) for a in configure_args ] 1045 1046 print("Running configure for %s"%(name,)) 1047 runCommand(' '.join(configure_args) + ' 2>&1') 1048 1049 if buildrecipe is not None: 1050 # call special-case build recipe, e.g. for openssl 1051 buildrecipe(basedir, archList) 1052 1053 if install is not None: 1054 print("Running install for %s"%(name,)) 1055 runCommand('{ ' + install + ' ;} 2>&1') 1056 1057 print("Done %s"%(name,)) 1058 print("") 1059 1060 os.chdir(curdir) 1061 1062def buildLibraries(): 1063 """ 1064 Build our dependencies into $WORKDIR/libraries/usr/local 1065 """ 1066 print("") 1067 print("Building required libraries") 1068 print("") 1069 universal = os.path.join(WORKDIR, 'libraries') 1070 os.mkdir(universal) 1071 os.makedirs(os.path.join(universal, 'usr', 'local', 'lib')) 1072 os.makedirs(os.path.join(universal, 'usr', 'local', 'include')) 1073 1074 for recipe in library_recipes(): 1075 buildRecipe(recipe, universal, ARCHLIST) 1076 1077 1078 1079def buildPythonDocs(): 1080 # This stores the documentation as Resources/English.lproj/Documentation 1081 # inside the framework. pydoc and IDLE will pick it up there. 1082 print("Install python documentation") 1083 rootDir = os.path.join(WORKDIR, '_root') 1084 buildDir = os.path.join('../../Doc') 1085 docdir = os.path.join(rootDir, 'pydocs') 1086 curDir = os.getcwd() 1087 os.chdir(buildDir) 1088 runCommand('make clean') 1089 1090 # Search third-party source directory for a pre-built version of the docs. 1091 # Use the naming convention of the docs.python.org html downloads: 1092 # python-3.9.0b1-docs-html.tar.bz2 1093 doctarfiles = [ f for f in os.listdir(DEPSRC) 1094 if f.startswith('python-'+getFullVersion()) 1095 if f.endswith('-docs-html.tar.bz2') ] 1096 if doctarfiles: 1097 doctarfile = doctarfiles[0] 1098 if not os.path.exists('build'): 1099 os.mkdir('build') 1100 # if build directory existed, it was emptied by make clean, above 1101 os.chdir('build') 1102 # Extract the first archive found for this version into build 1103 runCommand('tar xjf %s'%shellQuote(os.path.join(DEPSRC, doctarfile))) 1104 # see if tar extracted a directory ending in -docs-html 1105 archivefiles = [ f for f in os.listdir('.') 1106 if f.endswith('-docs-html') 1107 if os.path.isdir(f) ] 1108 if archivefiles: 1109 archivefile = archivefiles[0] 1110 # make it our 'Docs/build/html' directory 1111 print(' -- using pre-built python documentation from %s'%archivefile) 1112 os.rename(archivefile, 'html') 1113 os.chdir(buildDir) 1114 1115 htmlDir = os.path.join('build', 'html') 1116 if not os.path.exists(htmlDir): 1117 # Create virtual environment for docs builds with blurb and sphinx 1118 runCommand('make venv') 1119 runCommand('make html PYTHON=venv/bin/python') 1120 os.rename(htmlDir, docdir) 1121 os.chdir(curDir) 1122 1123 1124def buildPython(): 1125 print("Building a universal python for %s architectures" % UNIVERSALARCHS) 1126 1127 buildDir = os.path.join(WORKDIR, '_bld', 'python') 1128 rootDir = os.path.join(WORKDIR, '_root') 1129 1130 if os.path.exists(buildDir): 1131 shutil.rmtree(buildDir) 1132 if os.path.exists(rootDir): 1133 shutil.rmtree(rootDir) 1134 os.makedirs(buildDir) 1135 os.makedirs(rootDir) 1136 os.makedirs(os.path.join(rootDir, 'empty-dir')) 1137 curdir = os.getcwd() 1138 os.chdir(buildDir) 1139 1140 # Extract the version from the configure file, needed to calculate 1141 # several paths. 1142 version = getVersion() 1143 1144 # Since the extra libs are not in their installed framework location 1145 # during the build, augment the library path so that the interpreter 1146 # will find them during its extension import sanity checks. 1147 1148 print("Running configure...") 1149 runCommand("%s -C --enable-framework --enable-universalsdk=/ " 1150 "--with-universal-archs=%s " 1151 "%s " 1152 "%s " 1153 "%s " 1154 "%s " 1155 "%s " 1156 "%s " 1157 "LDFLAGS='-g -L%s/libraries/usr/local/lib' " 1158 "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%( 1159 shellQuote(os.path.join(SRCDIR, 'configure')), 1160 UNIVERSALARCHS, 1161 (' ', '--with-computed-gotos ')[PYTHON_3], 1162 (' ', '--without-ensurepip ')[PYTHON_3], 1163 (' ', "--with-openssl='%s/libraries/usr/local'"%( 1164 shellQuote(WORKDIR)[1:-1],))[PYTHON_3], 1165 (' ', "--enable-optimizations --with-lto")[compilerCanOptimize()], 1166 (' ', "TCLTK_CFLAGS='-I%s/libraries/usr/local/include'"%( 1167 shellQuote(WORKDIR)[1:-1],))[internalTk()], 1168 (' ', "TCLTK_LIBS='-L%s/libraries/usr/local/lib -ltcl8.6 -ltk8.6'"%( 1169 shellQuote(WORKDIR)[1:-1],))[internalTk()], 1170 shellQuote(WORKDIR)[1:-1], 1171 shellQuote(WORKDIR)[1:-1])) 1172 1173 # As of macOS 10.11 with SYSTEM INTEGRITY PROTECTION, DYLD_* 1174 # environment variables are no longer automatically inherited 1175 # by child processes from their parents. We used to just set 1176 # DYLD_LIBRARY_PATH, pointing to the third-party libs, 1177 # in build-installer.py's process environment and it was 1178 # passed through the make utility into the environment of 1179 # setup.py. Instead, we now append DYLD_LIBRARY_PATH to 1180 # the existing RUNSHARED configuration value when we call 1181 # make for extension module builds. 1182 1183 runshared_for_make = "".join([ 1184 " RUNSHARED=", 1185 "'", 1186 grepValue("Makefile", "RUNSHARED"), 1187 ' DYLD_LIBRARY_PATH=', 1188 os.path.join(WORKDIR, 'libraries', 'usr', 'local', 'lib'), 1189 "'" ]) 1190 1191 # Look for environment value BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS 1192 # and, if defined, append its value to the make command. This allows 1193 # us to pass in version control tags, like GITTAG, to a build from a 1194 # tarball rather than from a vcs checkout, thus eliminating the need 1195 # to have a working copy of the vcs program on the build machine. 1196 # 1197 # A typical use might be: 1198 # export BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS=" \ 1199 # GITVERSION='echo 123456789a' \ 1200 # GITTAG='echo v3.6.0' \ 1201 # GITBRANCH='echo 3.6'" 1202 1203 make_extras = os.getenv("BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS") 1204 if make_extras: 1205 make_cmd = "make " + make_extras + runshared_for_make 1206 else: 1207 make_cmd = "make" + runshared_for_make 1208 print("Running " + make_cmd) 1209 runCommand(make_cmd) 1210 1211 make_cmd = "make install DESTDIR=%s %s"%( 1212 shellQuote(rootDir), 1213 runshared_for_make) 1214 print("Running " + make_cmd) 1215 runCommand(make_cmd) 1216 1217 make_cmd = "make frameworkinstallextras DESTDIR=%s %s"%( 1218 shellQuote(rootDir), 1219 runshared_for_make) 1220 print("Running " + make_cmd) 1221 runCommand(make_cmd) 1222 1223 print("Copying required shared libraries") 1224 if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')): 1225 build_lib_dir = os.path.join( 1226 WORKDIR, 'libraries', 'Library', 'Frameworks', 1227 'Python.framework', 'Versions', getVersion(), 'lib') 1228 fw_lib_dir = os.path.join( 1229 WORKDIR, '_root', 'Library', 'Frameworks', 1230 'Python.framework', 'Versions', getVersion(), 'lib') 1231 if internalTk(): 1232 # move Tcl and Tk pkgconfig files 1233 runCommand("mv %s/pkgconfig/* %s/pkgconfig"%( 1234 shellQuote(build_lib_dir), 1235 shellQuote(fw_lib_dir) )) 1236 runCommand("rm -r %s/pkgconfig"%( 1237 shellQuote(build_lib_dir), )) 1238 runCommand("mv %s/* %s"%( 1239 shellQuote(build_lib_dir), 1240 shellQuote(fw_lib_dir) )) 1241 1242 frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework') 1243 frmDirVersioned = os.path.join(frmDir, 'Versions', version) 1244 path_to_lib = os.path.join(frmDirVersioned, 'lib', 'python%s'%(version,)) 1245 # create directory for OpenSSL certificates 1246 sslDir = os.path.join(frmDirVersioned, 'etc', 'openssl') 1247 os.makedirs(sslDir) 1248 1249 print("Fix file modes") 1250 gid = grp.getgrnam('admin').gr_gid 1251 1252 shared_lib_error = False 1253 for dirpath, dirnames, filenames in os.walk(frmDir): 1254 for dn in dirnames: 1255 os.chmod(os.path.join(dirpath, dn), STAT_0o775) 1256 os.chown(os.path.join(dirpath, dn), -1, gid) 1257 1258 for fn in filenames: 1259 if os.path.islink(fn): 1260 continue 1261 1262 # "chmod g+w $fn" 1263 p = os.path.join(dirpath, fn) 1264 st = os.stat(p) 1265 os.chmod(p, stat.S_IMODE(st.st_mode) | stat.S_IWGRP) 1266 os.chown(p, -1, gid) 1267 1268 if fn in EXPECTED_SHARED_LIBS: 1269 # check to see that this file was linked with the 1270 # expected library path and version 1271 data = captureCommand("otool -L %s" % shellQuote(p)) 1272 for sl in EXPECTED_SHARED_LIBS[fn]: 1273 if ("\t%s " % sl) not in data: 1274 print("Expected shared lib %s was not linked with %s" 1275 % (sl, p)) 1276 shared_lib_error = True 1277 1278 if shared_lib_error: 1279 fatal("Unexpected shared library errors.") 1280 1281 if PYTHON_3: 1282 LDVERSION=None 1283 VERSION=None 1284 ABIFLAGS=None 1285 1286 fp = open(os.path.join(buildDir, 'Makefile'), 'r') 1287 for ln in fp: 1288 if ln.startswith('VERSION='): 1289 VERSION=ln.split()[1] 1290 if ln.startswith('ABIFLAGS='): 1291 ABIFLAGS=ln.split() 1292 ABIFLAGS=ABIFLAGS[1] if len(ABIFLAGS) > 1 else '' 1293 if ln.startswith('LDVERSION='): 1294 LDVERSION=ln.split()[1] 1295 fp.close() 1296 1297 LDVERSION = LDVERSION.replace('$(VERSION)', VERSION) 1298 LDVERSION = LDVERSION.replace('$(ABIFLAGS)', ABIFLAGS) 1299 config_suffix = '-' + LDVERSION 1300 if getVersionMajorMinor() >= (3, 6): 1301 config_suffix = config_suffix + '-darwin' 1302 else: 1303 config_suffix = '' # Python 2.x 1304 1305 # We added some directories to the search path during the configure 1306 # phase. Remove those because those directories won't be there on 1307 # the end-users system. Also remove the directories from _sysconfigdata.py 1308 # (added in 3.3) if it exists. 1309 1310 include_path = '-I%s/libraries/usr/local/include' % (WORKDIR,) 1311 lib_path = '-L%s/libraries/usr/local/lib' % (WORKDIR,) 1312 1313 # fix Makefile 1314 path = os.path.join(path_to_lib, 'config' + config_suffix, 'Makefile') 1315 fp = open(path, 'r') 1316 data = fp.read() 1317 fp.close() 1318 1319 for p in (include_path, lib_path): 1320 data = data.replace(" " + p, '') 1321 data = data.replace(p + " ", '') 1322 1323 fp = open(path, 'w') 1324 fp.write(data) 1325 fp.close() 1326 1327 # fix _sysconfigdata 1328 # 1329 # TODO: make this more robust! test_sysconfig_module of 1330 # distutils.tests.test_sysconfig.SysconfigTestCase tests that 1331 # the output from get_config_var in both sysconfig and 1332 # distutils.sysconfig is exactly the same for both CFLAGS and 1333 # LDFLAGS. The fixing up is now complicated by the pretty 1334 # printing in _sysconfigdata.py. Also, we are using the 1335 # pprint from the Python running the installer build which 1336 # may not cosmetically format the same as the pprint in the Python 1337 # being built (and which is used to originally generate 1338 # _sysconfigdata.py). 1339 1340 import pprint 1341 if getVersionMajorMinor() >= (3, 6): 1342 # XXX this is extra-fragile 1343 path = os.path.join(path_to_lib, 1344 '_sysconfigdata_%s_darwin_darwin.py' % (ABIFLAGS,)) 1345 else: 1346 path = os.path.join(path_to_lib, '_sysconfigdata.py') 1347 fp = open(path, 'r') 1348 data = fp.read() 1349 fp.close() 1350 # create build_time_vars dict 1351 if RUNNING_ON_PYTHON2: 1352 exec(data) 1353 else: 1354 g_dict = {} 1355 l_dict = {} 1356 exec(data, g_dict, l_dict) 1357 build_time_vars = l_dict['build_time_vars'] 1358 vars = {} 1359 for k, v in build_time_vars.items(): 1360 if type(v) == type(''): 1361 for p in (include_path, lib_path): 1362 v = v.replace(' ' + p, '') 1363 v = v.replace(p + ' ', '') 1364 vars[k] = v 1365 1366 fp = open(path, 'w') 1367 # duplicated from sysconfig._generate_posix_vars() 1368 fp.write('# system configuration generated and used by' 1369 ' the sysconfig module\n') 1370 fp.write('build_time_vars = ') 1371 pprint.pprint(vars, stream=fp) 1372 fp.close() 1373 1374 # Add symlinks in /usr/local/bin, using relative links 1375 usr_local_bin = os.path.join(rootDir, 'usr', 'local', 'bin') 1376 to_framework = os.path.join('..', '..', '..', 'Library', 'Frameworks', 1377 'Python.framework', 'Versions', version, 'bin') 1378 if os.path.exists(usr_local_bin): 1379 shutil.rmtree(usr_local_bin) 1380 os.makedirs(usr_local_bin) 1381 for fn in os.listdir( 1382 os.path.join(frmDir, 'Versions', version, 'bin')): 1383 os.symlink(os.path.join(to_framework, fn), 1384 os.path.join(usr_local_bin, fn)) 1385 1386 os.chdir(curdir) 1387 1388def patchFile(inPath, outPath): 1389 data = fileContents(inPath) 1390 data = data.replace('$FULL_VERSION', getFullVersion()) 1391 data = data.replace('$VERSION', getVersion()) 1392 data = data.replace('$MACOSX_DEPLOYMENT_TARGET', ''.join((DEPTARGET, ' or later'))) 1393 data = data.replace('$ARCHITECTURES', ", ".join(universal_opts_map[UNIVERSALARCHS])) 1394 data = data.replace('$INSTALL_SIZE', installSize()) 1395 data = data.replace('$THIRD_PARTY_LIBS', "\\\n".join(THIRD_PARTY_LIBS)) 1396 1397 # This one is not handy as a template variable 1398 data = data.replace('$PYTHONFRAMEWORKINSTALLDIR', '/Library/Frameworks/Python.framework') 1399 fp = open(outPath, 'w') 1400 fp.write(data) 1401 fp.close() 1402 1403def patchScript(inPath, outPath): 1404 major, minor = getVersionMajorMinor() 1405 data = fileContents(inPath) 1406 data = data.replace('@PYMAJOR@', str(major)) 1407 data = data.replace('@PYVER@', getVersion()) 1408 fp = open(outPath, 'w') 1409 fp.write(data) 1410 fp.close() 1411 os.chmod(outPath, STAT_0o755) 1412 1413 1414 1415def packageFromRecipe(targetDir, recipe): 1416 curdir = os.getcwd() 1417 try: 1418 # The major version (such as 2.5) is included in the package name 1419 # because having two version of python installed at the same time is 1420 # common. 1421 pkgname = '%s-%s'%(recipe['name'], getVersion()) 1422 srcdir = recipe.get('source') 1423 pkgroot = recipe.get('topdir', srcdir) 1424 postflight = recipe.get('postflight') 1425 readme = textwrap.dedent(recipe['readme']) 1426 isRequired = recipe.get('required', True) 1427 1428 print("- building package %s"%(pkgname,)) 1429 1430 # Substitute some variables 1431 textvars = dict( 1432 VER=getVersion(), 1433 FULLVER=getFullVersion(), 1434 ) 1435 readme = readme % textvars 1436 1437 if pkgroot is not None: 1438 pkgroot = pkgroot % textvars 1439 else: 1440 pkgroot = '/' 1441 1442 if srcdir is not None: 1443 srcdir = os.path.join(WORKDIR, '_root', srcdir[1:]) 1444 srcdir = srcdir % textvars 1445 1446 if postflight is not None: 1447 postflight = os.path.abspath(postflight) 1448 1449 packageContents = os.path.join(targetDir, pkgname + '.pkg', 'Contents') 1450 os.makedirs(packageContents) 1451 1452 if srcdir is not None: 1453 os.chdir(srcdir) 1454 runCommand("pax -wf %s . 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),)) 1455 runCommand("gzip -9 %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),)) 1456 runCommand("mkbom . %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.bom')),)) 1457 1458 fn = os.path.join(packageContents, 'PkgInfo') 1459 fp = open(fn, 'w') 1460 fp.write('pmkrpkg1') 1461 fp.close() 1462 1463 rsrcDir = os.path.join(packageContents, "Resources") 1464 os.mkdir(rsrcDir) 1465 fp = open(os.path.join(rsrcDir, 'ReadMe.txt'), 'w') 1466 fp.write(readme) 1467 fp.close() 1468 1469 if postflight is not None: 1470 patchScript(postflight, os.path.join(rsrcDir, 'postflight')) 1471 1472 vers = getFullVersion() 1473 major, minor = getVersionMajorMinor() 1474 pl = dict( 1475 CFBundleGetInfoString="Python.%s %s"%(pkgname, vers,), 1476 CFBundleIdentifier='org.python.Python.%s'%(pkgname,), 1477 CFBundleName='Python.%s'%(pkgname,), 1478 CFBundleShortVersionString=vers, 1479 IFMajorVersion=major, 1480 IFMinorVersion=minor, 1481 IFPkgFormatVersion=0.10000000149011612, 1482 IFPkgFlagAllowBackRev=False, 1483 IFPkgFlagAuthorizationAction="RootAuthorization", 1484 IFPkgFlagDefaultLocation=pkgroot, 1485 IFPkgFlagFollowLinks=True, 1486 IFPkgFlagInstallFat=True, 1487 IFPkgFlagIsRequired=isRequired, 1488 IFPkgFlagOverwritePermissions=False, 1489 IFPkgFlagRelocatable=False, 1490 IFPkgFlagRestartAction="NoRestart", 1491 IFPkgFlagRootVolumeOnly=True, 1492 IFPkgFlagUpdateInstalledLangauges=False, 1493 ) 1494 writePlist(pl, os.path.join(packageContents, 'Info.plist')) 1495 1496 pl = dict( 1497 IFPkgDescriptionDescription=readme, 1498 IFPkgDescriptionTitle=recipe.get('long_name', "Python.%s"%(pkgname,)), 1499 IFPkgDescriptionVersion=vers, 1500 ) 1501 writePlist(pl, os.path.join(packageContents, 'Resources', 'Description.plist')) 1502 1503 finally: 1504 os.chdir(curdir) 1505 1506 1507def makeMpkgPlist(path): 1508 1509 vers = getFullVersion() 1510 major, minor = getVersionMajorMinor() 1511 1512 pl = dict( 1513 CFBundleGetInfoString="Python %s"%(vers,), 1514 CFBundleIdentifier='org.python.Python', 1515 CFBundleName='Python', 1516 CFBundleShortVersionString=vers, 1517 IFMajorVersion=major, 1518 IFMinorVersion=minor, 1519 IFPkgFlagComponentDirectory="Contents/Packages", 1520 IFPkgFlagPackageList=[ 1521 dict( 1522 IFPkgFlagPackageLocation='%s-%s.pkg'%(item['name'], getVersion()), 1523 IFPkgFlagPackageSelection=item.get('selected', 'selected'), 1524 ) 1525 for item in pkg_recipes() 1526 ], 1527 IFPkgFormatVersion=0.10000000149011612, 1528 IFPkgFlagBackgroundScaling="proportional", 1529 IFPkgFlagBackgroundAlignment="left", 1530 IFPkgFlagAuthorizationAction="RootAuthorization", 1531 ) 1532 1533 writePlist(pl, path) 1534 1535 1536def buildInstaller(): 1537 1538 # Zap all compiled files 1539 for dirpath, _, filenames in os.walk(os.path.join(WORKDIR, '_root')): 1540 for fn in filenames: 1541 if fn.endswith('.pyc') or fn.endswith('.pyo'): 1542 os.unlink(os.path.join(dirpath, fn)) 1543 1544 outdir = os.path.join(WORKDIR, 'installer') 1545 if os.path.exists(outdir): 1546 shutil.rmtree(outdir) 1547 os.mkdir(outdir) 1548 1549 pkgroot = os.path.join(outdir, 'Python.mpkg', 'Contents') 1550 pkgcontents = os.path.join(pkgroot, 'Packages') 1551 os.makedirs(pkgcontents) 1552 for recipe in pkg_recipes(): 1553 packageFromRecipe(pkgcontents, recipe) 1554 1555 rsrcDir = os.path.join(pkgroot, 'Resources') 1556 1557 fn = os.path.join(pkgroot, 'PkgInfo') 1558 fp = open(fn, 'w') 1559 fp.write('pmkrpkg1') 1560 fp.close() 1561 1562 os.mkdir(rsrcDir) 1563 1564 makeMpkgPlist(os.path.join(pkgroot, 'Info.plist')) 1565 pl = dict( 1566 IFPkgDescriptionTitle="Python", 1567 IFPkgDescriptionVersion=getVersion(), 1568 ) 1569 1570 writePlist(pl, os.path.join(pkgroot, 'Resources', 'Description.plist')) 1571 for fn in os.listdir('resources'): 1572 if fn == '.svn': continue 1573 if fn.endswith('.jpg'): 1574 shutil.copy(os.path.join('resources', fn), os.path.join(rsrcDir, fn)) 1575 else: 1576 patchFile(os.path.join('resources', fn), os.path.join(rsrcDir, fn)) 1577 1578 1579def installSize(clear=False, _saved=[]): 1580 if clear: 1581 del _saved[:] 1582 if not _saved: 1583 data = captureCommand("du -ks %s"%( 1584 shellQuote(os.path.join(WORKDIR, '_root')))) 1585 _saved.append("%d"%((0.5 + (int(data.split()[0]) / 1024.0)),)) 1586 return _saved[0] 1587 1588 1589def buildDMG(): 1590 """ 1591 Create DMG containing the rootDir. 1592 """ 1593 outdir = os.path.join(WORKDIR, 'diskimage') 1594 if os.path.exists(outdir): 1595 shutil.rmtree(outdir) 1596 1597 # We used to use the deployment target as the last characters of the 1598 # installer file name. With the introduction of weaklinked installer 1599 # variants, we may have two variants with the same file name, i.e. 1600 # both ending in '10.9'. To avoid this, we now use the major/minor 1601 # version numbers of the macOS version we are building on. 1602 # Also, as of macOS 11, operating system version numbering has 1603 # changed from three components to two, i.e. 1604 # 10.14.1, 10.14.2, ... 1605 # 10.15.1, 10.15.2, ... 1606 # 11.1, 11.2, ... 1607 # 12.1, 12.2, ... 1608 # (A further twist is that, when running on macOS 11, binaries built 1609 # on older systems may be shown an operating system version of 10.16 1610 # instead of 11. We should not run into that situation here.) 1611 # Also we should use "macos" instead of "macosx" going forward. 1612 # 1613 # To maintain compatibility for legacy variants, the file name for 1614 # builds on macOS 10.15 and earlier remains: 1615 # python-3.x.y-macosx10.z.{dmg->pkg} 1616 # e.g. python-3.9.4-macosx10.9.{dmg->pkg} 1617 # and for builds on macOS 11+: 1618 # python-3.x.y-macosz.{dmg->pkg} 1619 # e.g. python-3.9.4-macos11.{dmg->pkg} 1620 1621 build_tuple = getBuildTuple() 1622 if build_tuple[0] < 11: 1623 os_name = 'macosx' 1624 build_system_version = '%s.%s' % build_tuple 1625 else: 1626 os_name = 'macos' 1627 build_system_version = str(build_tuple[0]) 1628 imagepath = os.path.join(outdir, 1629 'python-%s-%s%s'%(getFullVersion(),os_name,build_system_version)) 1630 if INCLUDE_TIMESTAMP: 1631 imagepath = imagepath + '-%04d-%02d-%02d'%(time.localtime()[:3]) 1632 imagepath = imagepath + '.dmg' 1633 1634 os.mkdir(outdir) 1635 1636 # Try to mitigate race condition in certain versions of macOS, e.g. 10.9, 1637 # when hdiutil create fails with "Resource busy". For now, just retry 1638 # the create a few times and hope that it eventually works. 1639 1640 volname='Python %s'%(getFullVersion()) 1641 cmd = ("hdiutil create -format UDRW -volname %s -srcfolder %s -size 100m %s"%( 1642 shellQuote(volname), 1643 shellQuote(os.path.join(WORKDIR, 'installer')), 1644 shellQuote(imagepath + ".tmp.dmg" ))) 1645 for i in range(5): 1646 fd = os.popen(cmd, 'r') 1647 data = fd.read() 1648 xit = fd.close() 1649 if not xit: 1650 break 1651 sys.stdout.write(data) 1652 print(" -- retrying hdiutil create") 1653 time.sleep(5) 1654 else: 1655 raise RuntimeError("command failed: %s"%(cmd,)) 1656 1657 if not os.path.exists(os.path.join(WORKDIR, "mnt")): 1658 os.mkdir(os.path.join(WORKDIR, "mnt")) 1659 runCommand("hdiutil attach %s -mountroot %s"%( 1660 shellQuote(imagepath + ".tmp.dmg"), shellQuote(os.path.join(WORKDIR, "mnt")))) 1661 1662 # Custom icon for the DMG, shown when the DMG is mounted. 1663 shutil.copy("../Icons/Disk Image.icns", 1664 os.path.join(WORKDIR, "mnt", volname, ".VolumeIcon.icns")) 1665 runCommand("SetFile -a C %s/"%( 1666 shellQuote(os.path.join(WORKDIR, "mnt", volname)),)) 1667 1668 runCommand("hdiutil detach %s"%(shellQuote(os.path.join(WORKDIR, "mnt", volname)))) 1669 1670 setIcon(imagepath + ".tmp.dmg", "../Icons/Disk Image.icns") 1671 runCommand("hdiutil convert %s -format UDZO -o %s"%( 1672 shellQuote(imagepath + ".tmp.dmg"), shellQuote(imagepath))) 1673 setIcon(imagepath, "../Icons/Disk Image.icns") 1674 1675 os.unlink(imagepath + ".tmp.dmg") 1676 1677 return imagepath 1678 1679 1680def setIcon(filePath, icnsPath): 1681 """ 1682 Set the custom icon for the specified file or directory. 1683 """ 1684 1685 dirPath = os.path.normpath(os.path.dirname(__file__)) 1686 toolPath = os.path.join(dirPath, "seticon.app/Contents/MacOS/seticon") 1687 if not os.path.exists(toolPath) or os.stat(toolPath).st_mtime < os.stat(dirPath + '/seticon.m').st_mtime: 1688 # NOTE: The tool is created inside an .app bundle, otherwise it won't work due 1689 # to connections to the window server. 1690 appPath = os.path.join(dirPath, "seticon.app/Contents/MacOS") 1691 if not os.path.exists(appPath): 1692 os.makedirs(appPath) 1693 runCommand("cc -o %s %s/seticon.m -framework Cocoa"%( 1694 shellQuote(toolPath), shellQuote(dirPath))) 1695 1696 runCommand("%s %s %s"%(shellQuote(os.path.abspath(toolPath)), shellQuote(icnsPath), 1697 shellQuote(filePath))) 1698 1699def main(): 1700 # First parse options and check if we can perform our work 1701 parseOptions() 1702 checkEnvironment() 1703 1704 os.environ['MACOSX_DEPLOYMENT_TARGET'] = DEPTARGET 1705 os.environ['CC'] = CC 1706 os.environ['CXX'] = CXX 1707 1708 if os.path.exists(WORKDIR): 1709 shutil.rmtree(WORKDIR) 1710 os.mkdir(WORKDIR) 1711 1712 os.environ['LC_ALL'] = 'C' 1713 1714 # Then build third-party libraries such as sleepycat DB4. 1715 buildLibraries() 1716 1717 # Now build python itself 1718 buildPython() 1719 1720 # And then build the documentation 1721 # Remove the Deployment Target from the shell 1722 # environment, it's no longer needed and 1723 # an unexpected build target can cause problems 1724 # when Sphinx and its dependencies need to 1725 # be (re-)installed. 1726 del os.environ['MACOSX_DEPLOYMENT_TARGET'] 1727 buildPythonDocs() 1728 1729 1730 # Prepare the applications folder 1731 folder = os.path.join(WORKDIR, "_root", "Applications", "Python %s"%( 1732 getVersion(),)) 1733 fn = os.path.join(folder, "License.rtf") 1734 patchFile("resources/License.rtf", fn) 1735 fn = os.path.join(folder, "ReadMe.rtf") 1736 patchFile("resources/ReadMe.rtf", fn) 1737 fn = os.path.join(folder, "Update Shell Profile.command") 1738 patchScript("scripts/postflight.patch-profile", fn) 1739 fn = os.path.join(folder, "Install Certificates.command") 1740 patchScript("resources/install_certificates.command", fn) 1741 os.chmod(folder, STAT_0o755) 1742 setIcon(folder, "../Icons/Python Folder.icns") 1743 1744 # Create the installer 1745 buildInstaller() 1746 1747 # And copy the readme into the directory containing the installer 1748 patchFile('resources/ReadMe.rtf', 1749 os.path.join(WORKDIR, 'installer', 'ReadMe.rtf')) 1750 1751 # Ditto for the license file. 1752 patchFile('resources/License.rtf', 1753 os.path.join(WORKDIR, 'installer', 'License.rtf')) 1754 1755 fp = open(os.path.join(WORKDIR, 'installer', 'Build.txt'), 'w') 1756 fp.write("# BUILD INFO\n") 1757 fp.write("# Date: %s\n" % time.ctime()) 1758 fp.write("# By: %s\n" % pwd.getpwuid(os.getuid()).pw_gecos) 1759 fp.close() 1760 1761 # And copy it to a DMG 1762 buildDMG() 1763 1764if __name__ == "__main__": 1765 main() 1766