From 6c6d31f8533c70f14dfe095f63e41efd75e20a8a Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 29 Aug 2022 08:50:22 +0200 Subject: [PATCH 01/19] Download osx packages and setup script --- builders/macos-python-builder.psm1 | 140 +++++++++++++++++++---------- installers/macos-setup-template.sh | 64 +++++++++++++ 2 files changed, 158 insertions(+), 46 deletions(-) create mode 100644 installers/macos-setup-template.sh diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index 235888185..ecb10aafd 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -1,6 +1,6 @@ -using module "./nix-python-builder.psm1" +using module "./python-builder.psm1" -class macOSPythonBuilder : NixPythonBuilder { +class macOSPythonBuilder : PythonBuilder { <# .SYNOPSIS MacOS Python builder class. @@ -20,67 +20,115 @@ class macOSPythonBuilder : NixPythonBuilder { #> + [string] $InstallationTemplateName + [string] $InstallationScriptName + [string] $OutputArtifactName + macOSPythonBuilder( [semver] $version, [string] $architecture, [string] $platform - ) : Base($version, $architecture, $platform) { } + ) : Base($version, $architecture, $platform) { + $this.InstallationTemplateName = "macos-setup-template.ps1" + $this.InstallationScriptName = "setup.sh" + $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.zip" + } - [void] PrepareEnvironment() { + [string] GetPythonExtension() { <# .SYNOPSIS - Prepare system environment by installing dependencies and required packages. + Return extension for required version of Python package. #> + + $extension = ".pkg" + + return $extension } - [void] Configure() { + [string] GetArchitectureExtension() { <# .SYNOPSIS - Execute configure script with required parameters. + Return architecture suffix for Python package. #> - $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation() - $configureString = "./configure" - $configureString += " --prefix=$pythonBinariesLocation" - $configureString += " --enable-optimizations" - $configureString += " --enable-shared" - $configureString += " --with-lto" - - ### For Python versions which support it, compile a universal2 (arm64 + x86_64 hybrid) build. The arm64 slice - ### will never be used itself by a Github Actions runner but using a universal2 Python is the only way to build - ### universal2 C extensions and wheels. This is supported by Python >= 3.10 and was backported to Python >= - ### 3.9.1 and >= 3.8.10. - ### Disabled, discussion: https://github.com/actions/python-versions/pull/114 - # if ($this.Version -ge "3.8.10" -and $this.Version -ne "3.8.13" -and $this.Version -ne "3.9.0" ) { - # $configureString += " --enable-universalsdk --with-universal-archs=universal2" - # } - - ### OS X 10.11, Apple no longer provides header files for the deprecated system version of OpenSSL. - ### Solution is to install these libraries from a third-party package manager, - ### and then add the appropriate paths for the header and library files to configure command. - ### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies) - if ($this.Version -lt "3.7.0") { - $env:LDFLAGS = "-L/usr/local/opt/openssl@1.1/lib -L/usr/local/opt/zlib/lib" - $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/zlib/include" - } else { - $configureString += " --with-openssl=/usr/local/opt/openssl@1.1" - if ($this.Version -gt "3.7.12") { - $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'" - } - } + $ArchitectureExtension = "-macos11" + + return $ArchitectureExtension + } + + [uri] GetSourceUri() { + <# + .SYNOPSIS + Get base Python URI and return complete URI for Python installation executable. + #> + + $base = $this.GetBaseUri() + $versionName = $this.GetBaseVersion() + $nativeVersion = Convert-Version -version $this.Version + $architecture = $this.GetArchitectureExtension() + $extension = $this.GetPythonExtension() + + $uri = "${base}/${versionName}/python-${nativeVersion}${architecture}${extension}" + + return $uri + } + + [string] Download() { + <# + .SYNOPSIS + Download Python installation executable into artifact location. + #> - ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.* - ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension) - if ($this.Version -ge "3.2.0") { - $configureString += " --enable-loadable-sqlite-extensions" - $env:LDFLAGS += " -L$(brew --prefix sqlite3)/lib" - $env:CFLAGS += " -I$(brew --prefix sqlite3)/include" - $env:CPPFLAGS += "-I$(brew --prefix sqlite3)/include" + $sourceUri = $this.GetSourceUri() + + Write-Host "Sources URI: $sourceUri" + $sourcesLocation = Download-File -Uri $sourceUri -OutputFolder $this.WorkFolderLocation + Write-Debug "Done; Sources location: $sourcesLocation" + + return $sourcesLocation + } + + [void] CreateInstallationScript() { + <# + .SYNOPSIS + Create Python artifact installation script based on specified template. + #> + + $sourceUri = $this.GetSourceUri() + $pythonExecName = [IO.path]::GetFileName($sourceUri.AbsoluteUri) + $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath $this.InstallationTemplateName + $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw + $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName -ItemType File + + $variablesToReplace = @{ + "{{__ARCHITECTURE__}}" = $this.Architecture; + "{{__VERSION__}}" = $this.Version; + "{{__PYTHON_EXEC_NAME__}}" = $pythonExecName } - Write-Host "The passed configure options are: " - Write-Host $configureString + $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) } + $installationTemplateContent | Out-File -FilePath $installationScriptLocation + Write-Debug "Done; Installation script location: $installationScriptLocation)" + } + + [void] ArchiveArtifact() { + $OutputPath = Join-Path $this.ArtifactFolderLocation $this.OutputArtifactName + Create-SevenZipArchive -SourceFolder $this.WorkFolderLocation -ArchivePath $OutputPath + } + + [void] Build() { + <# + .SYNOPSIS + Generates Python artifact from downloaded Python installation executable. + #> + + Write-Host "Download Python $($this.Version) [$($this.Architecture)] executable..." + $this.Download() + + Write-Host "Create installation script..." + $this.CreateInstallationScript() - Execute-Command -Command $configureString + Write-Host "Archive artifact" + $this.ArchiveArtifact() } } diff --git a/installers/macos-setup-template.sh b/installers/macos-setup-template.sh new file mode 100644 index 000000000..6c9402e95 --- /dev/null +++ b/installers/macos-setup-template.sh @@ -0,0 +1,64 @@ +set -e + +PYTHON_FULL_VERSION="{{__VERSION_FULL__}}" +MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1) +MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2) + +PYTHON_MAJOR=python$MAJOR_VERSION +PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION +PYTHON_MAJORMINOR=python$MAJOR_VERSION$MINOR_VERSION + +if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then + # No AGENT_TOOLSDIRECTORY on GitHub images + TOOLCACHE_ROOT=$RUNNER_TOOL_CACHE +else + TOOLCACHE_ROOT=$AGENT_TOOLSDIRECTORY +fi + +PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python +PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION +PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64 + +echo "Check if Python hostedtoolcache folder exist..." +if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then + echo "Creating Python hostedtoolcache folder..." + mkdir -p $PYTHON_TOOLCACHE_PATH +elif [ -d $PYTHON_TOOLCACHE_VERSION_PATH ]; then + # TODO: remove ALL other directories for same MAJOR_VERSION.$MINOR_VERSION + echo "Deleting Python $PYTHON_FULL_VERSION" + rm -rf $PYTHON_TOOLCACHE_VERSION_PATH +fi + +echo "Create Python $PYTHON_FULL_VERSION folder" +mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH + +echo "Copy Python binaries to hostedtoolcache folder" +#cp -R ./* $PYTHON_TOOLCACHE_VERSION_ARCH_PATH +sudo installer -pkg "python-${PYTHON_FULL_VERSION}-macos11.pkg" -target / +rm $PYTHON_TOOLCACHE_VERSION_ARCH_PATH/setup.sh + +cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH + +echo "Create hostedtoolcach symlinks (Required for the backward compatibility)" +ln -s /Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}/bin bin +ln -s /Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}/include include +ln -s /Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}/share share +ln -s /Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}/lib lib + +echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)" +ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python + +cd bin/ +ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR +if [ ! -f python ]; then + ln -s $PYTHON_MAJOR_DOT_MINOR python +fi + +chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR python + +echo "Upgrading pip..." +./python -m ensurepip +./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location + +echo "Create complete file" +touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete From ea7dfb480e8122a2753f5219346ab1774a6dde71 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Wed, 31 Aug 2022 15:37:18 +0200 Subject: [PATCH 02/19] install from pkg for 3.11 only --- builders/macos-python-builder.psm1 | 119 +++++++++++------- ...emplate.sh => macos-pkg-setup-template.sh} | 23 ++-- 2 files changed, 86 insertions(+), 56 deletions(-) rename installers/{macos-setup-template.sh => macos-pkg-setup-template.sh} (83%) diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index ecb10aafd..c6ddc382e 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -1,6 +1,6 @@ -using module "./python-builder.psm1" +using module "./nix-python-builder.psm1" -class macOSPythonBuilder : PythonBuilder { +class macOSPythonBuilder : NixPythonBuilder { <# .SYNOPSIS MacOS Python builder class. @@ -20,43 +20,71 @@ class macOSPythonBuilder : PythonBuilder { #> - [string] $InstallationTemplateName - [string] $InstallationScriptName - [string] $OutputArtifactName - macOSPythonBuilder( [semver] $version, [string] $architecture, [string] $platform - ) : Base($version, $architecture, $platform) { - $this.InstallationTemplateName = "macos-setup-template.ps1" - $this.InstallationScriptName = "setup.sh" - $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.zip" - } + ) : Base($version, $architecture, $platform) { } - [string] GetPythonExtension() { + [void] PrepareEnvironment() { <# .SYNOPSIS - Return extension for required version of Python package. + Prepare system environment by installing dependencies and required packages. #> - - $extension = ".pkg" - - return $extension } - [string] GetArchitectureExtension() { + [void] Configure() { <# .SYNOPSIS - Return architecture suffix for Python package. + Execute configure script with required parameters. #> - $ArchitectureExtension = "-macos11" + $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation() + $configureString = "./configure" + $configureString += " --prefix=$pythonBinariesLocation" + $configureString += " --enable-optimizations" + $configureString += " --enable-shared" + $configureString += " --with-lto" + + ### For Python versions which support it, compile a universal2 (arm64 + x86_64 hybrid) build. The arm64 slice + ### will never be used itself by a Github Actions runner but using a universal2 Python is the only way to build + ### universal2 C extensions and wheels. This is supported by Python >= 3.10 and was backported to Python >= + ### 3.9.1 and >= 3.8.10. + ### Disabled, discussion: https://github.com/actions/python-versions/pull/114 + # if ($this.Version -ge "3.8.10" -and $this.Version -ne "3.8.13" -and $this.Version -ne "3.9.0" ) { + # $configureString += " --enable-universalsdk --with-universal-archs=universal2" + # } + + ### OS X 10.11, Apple no longer provides header files for the deprecated system version of OpenSSL. + ### Solution is to install these libraries from a third-party package manager, + ### and then add the appropriate paths for the header and library files to configure command. + ### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies) + if ($this.Version -lt "3.7.0") { + $env:LDFLAGS = "-L/usr/local/opt/openssl@1.1/lib -L/usr/local/opt/zlib/lib" + $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/zlib/include" + } else { + $configureString += " --with-openssl=/usr/local/opt/openssl@1.1" + if ($this.Version -gt "3.7.12") { + $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'" + } + } + + ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.* + ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension) + if ($this.Version -ge "3.2.0") { + $configureString += " --enable-loadable-sqlite-extensions" + $env:LDFLAGS += " -L$(brew --prefix sqlite3)/lib" + $env:CFLAGS += " -I$(brew --prefix sqlite3)/include" + $env:CPPFLAGS += "-I$(brew --prefix sqlite3)/include" + } + + Write-Host "The passed configure options are: " + Write-Host $configureString - return $ArchitectureExtension + Execute-Command -Command $configureString } - [uri] GetSourceUri() { + [uri] GetPkgUri() { <# .SYNOPSIS Get base Python URI and return complete URI for Python installation executable. @@ -65,45 +93,41 @@ class macOSPythonBuilder : PythonBuilder { $base = $this.GetBaseUri() $versionName = $this.GetBaseVersion() $nativeVersion = Convert-Version -version $this.Version - $architecture = $this.GetArchitectureExtension() - $extension = $this.GetPythonExtension() + $architecture = "-macos11" + $extension = ".pkg" $uri = "${base}/${versionName}/python-${nativeVersion}${architecture}${extension}" return $uri } - [string] Download() { + [string] DownloadPkg() { <# .SYNOPSIS Download Python installation executable into artifact location. #> - $sourceUri = $this.GetSourceUri() + $pkgUri = $this.GetPkgUri() - Write-Host "Sources URI: $sourceUri" - $sourcesLocation = Download-File -Uri $sourceUri -OutputFolder $this.WorkFolderLocation - Write-Debug "Done; Sources location: $sourcesLocation" + Write-Host "Sources URI: $pkgUri" + $pkgLocation = Download-File -Uri $pkgUri -OutputFolder $this.WorkFolderLocation + Write-Debug "Done; Package location: $pkgLocation" - return $sourcesLocation + return $pkgLocation } - [void] CreateInstallationScript() { + [void] CreateInstallationScriptPkg() { <# .SYNOPSIS Create Python artifact installation script based on specified template. #> - $sourceUri = $this.GetSourceUri() - $pythonExecName = [IO.path]::GetFileName($sourceUri.AbsoluteUri) $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath $this.InstallationTemplateName $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw - $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName -ItemType File + $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name "macos-pkg-setup-template.sh" -ItemType File $variablesToReplace = @{ - "{{__ARCHITECTURE__}}" = $this.Architecture; - "{{__VERSION__}}" = $this.Version; - "{{__PYTHON_EXEC_NAME__}}" = $pythonExecName + "{{__VERSION_FULL__}}" = $this.Version; } $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) } @@ -111,22 +135,25 @@ class macOSPythonBuilder : PythonBuilder { Write-Debug "Done; Installation script location: $installationScriptLocation)" } - [void] ArchiveArtifact() { - $OutputPath = Join-Path $this.ArtifactFolderLocation $this.OutputArtifactName - Create-SevenZipArchive -SourceFolder $this.WorkFolderLocation -ArchivePath $OutputPath - } - [void] Build() { <# .SYNOPSIS Generates Python artifact from downloaded Python installation executable. #> - Write-Host "Download Python $($this.Version) [$($this.Architecture)] executable..." - $this.Download() + if ($this.Version -ge "3.11.0") { + Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..." + $this.DownloadPkg() + + Write-Host "Create installation script..." + $this.CreateInstallationScriptPkg() + } else { + Write-Host "Download Python $($this.Version) [$($this.Architecture)] executable..." + $this.Download() - Write-Host "Create installation script..." - $this.CreateInstallationScript() + Write-Host "Create installation script..." + $this.CreateInstallationScript() + } Write-Host "Archive artifact" $this.ArchiveArtifact() diff --git a/installers/macos-setup-template.sh b/installers/macos-pkg-setup-template.sh similarity index 83% rename from installers/macos-setup-template.sh rename to installers/macos-pkg-setup-template.sh index 6c9402e95..45f2cd3d8 100644 --- a/installers/macos-setup-template.sh +++ b/installers/macos-pkg-setup-template.sh @@ -23,23 +23,26 @@ echo "Check if Python hostedtoolcache folder exist..." if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then echo "Creating Python hostedtoolcache folder..." mkdir -p $PYTHON_TOOLCACHE_PATH -elif [ -d $PYTHON_TOOLCACHE_VERSION_PATH ]; then - # TODO: remove ALL other directories for same MAJOR_VERSION.$MINOR_VERSION - echo "Deleting Python $PYTHON_FULL_VERSION" - rm -rf $PYTHON_TOOLCACHE_VERSION_PATH +else + # remove ALL other directories for same major.minor python versions + find $PYTHON_TOOLCACHE_PATH -name "${MAJOR_VERSION}.${MINOR_VERSION}.*"|while read python_version;do + python_version_x64="$python_version/x64" + if [ -e "$python_version_x64" ];then + echo "Deleting Python $python_version_x64" + rm -rf "$python_version_x64" + fi + done fi -echo "Create Python $PYTHON_FULL_VERSION folder" -mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH - -echo "Copy Python binaries to hostedtoolcache folder" -#cp -R ./* $PYTHON_TOOLCACHE_VERSION_ARCH_PATH +echo "Install Python binaries from prebuilt package" sudo installer -pkg "python-${PYTHON_FULL_VERSION}-macos11.pkg" -target / rm $PYTHON_TOOLCACHE_VERSION_ARCH_PATH/setup.sh +echo "Create hostedtoolcach symlinks (Required for the backward compatibility)" +echo "Create Python $PYTHON_FULL_VERSION folder" +mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH -echo "Create hostedtoolcach symlinks (Required for the backward compatibility)" ln -s /Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}/bin bin ln -s /Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}/include include ln -s /Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}/share share From 38ecfc57fce7d9d2ae6f158cd0eb02e61f6a4373 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Wed, 31 Aug 2022 20:12:16 +0200 Subject: [PATCH 03/19] More debug --- builders/macos-python-builder.psm1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index c6ddc382e..7ecf53dec 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -141,6 +141,8 @@ class macOSPythonBuilder : NixPythonBuilder { Generates Python artifact from downloaded Python installation executable. #> + Write-Host "Build Python $($this.Version) [$($this.Architecture)]" + if ($this.Version -ge "3.11.0") { Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..." $this.DownloadPkg() From 39fd250531f4b420d5dfa8950033b68b8a0b3f46 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Wed, 31 Aug 2022 22:35:30 +0200 Subject: [PATCH 04/19] More debug --- builders/macos-python-builder.psm1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index 7ecf53dec..f1c88b38c 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -141,7 +141,8 @@ class macOSPythonBuilder : NixPythonBuilder { Generates Python artifact from downloaded Python installation executable. #> - Write-Host "Build Python $($this.Version) [$($this.Architecture)]" + Write-Host "Build Python *$($this.Version)* [$($this.Architecture)]" + Write-Host ($this.Version -ge "3.11.0") if ($this.Version -ge "3.11.0") { Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..." From 49cfaf868ce5000f89a0f7ce2c828a8ab8a3363f Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Thu, 1 Sep 2022 09:48:42 +0200 Subject: [PATCH 05/19] fix version check --- builders/macos-python-builder.psm1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index f1c88b38c..348bed35a 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -142,9 +142,9 @@ class macOSPythonBuilder : NixPythonBuilder { #> Write-Host "Build Python *$($this.Version)* [$($this.Architecture)]" - Write-Host ($this.Version -ge "3.11.0") + Write-Host ($this.Version -ge 3.11.0) - if ($this.Version -ge "3.11.0") { + if ($this.Version -ge 3.11.0) { Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..." $this.DownloadPkg() From f0ce8e128c118d291f95b534635021aea9dbe461 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Fri, 2 Sep 2022 04:21:35 +0200 Subject: [PATCH 06/19] New-Item build_output.txt --- builders/macos-python-builder.psm1 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index 348bed35a..3d04e19b3 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -113,6 +113,7 @@ class macOSPythonBuilder : NixPythonBuilder { $pkgLocation = Download-File -Uri $pkgUri -OutputFolder $this.WorkFolderLocation Write-Debug "Done; Package location: $pkgLocation" + New-Item "build_output.txt" return $pkgLocation } @@ -141,9 +142,6 @@ class macOSPythonBuilder : NixPythonBuilder { Generates Python artifact from downloaded Python installation executable. #> - Write-Host "Build Python *$($this.Version)* [$($this.Architecture)]" - Write-Host ($this.Version -ge 3.11.0) - if ($this.Version -ge 3.11.0) { Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..." $this.DownloadPkg() From 2c77830ae8e611ccc63b739aedd41cdd88d2c25e Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Fri, 2 Sep 2022 04:46:21 +0200 Subject: [PATCH 07/19] installationTemplateLocation --- builders/macos-python-builder.psm1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index 3d04e19b3..096efec31 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -113,7 +113,7 @@ class macOSPythonBuilder : NixPythonBuilder { $pkgLocation = Download-File -Uri $pkgUri -OutputFolder $this.WorkFolderLocation Write-Debug "Done; Package location: $pkgLocation" - New-Item "build_output.txt" + New-Item -Path $this.WorkFolderLocation -Name "build_output.txt" -ItemType File return $pkgLocation } @@ -123,9 +123,9 @@ class macOSPythonBuilder : NixPythonBuilder { Create Python artifact installation script based on specified template. #> - $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath $this.InstallationTemplateName + $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath "macos-pkg-setup-template.sh" $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw - $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name "macos-pkg-setup-template.sh" -ItemType File + $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName -ItemType File $variablesToReplace = @{ "{{__VERSION_FULL__}}" = $this.Version; From 92e5a5e0735791cfdce14ab785e35f28c6052d46 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 5 Sep 2022 10:19:51 +0200 Subject: [PATCH 08/19] fix version --- builders/macos-python-builder.psm1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index 096efec31..c7566058a 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -142,7 +142,9 @@ class macOSPythonBuilder : NixPythonBuilder { Generates Python artifact from downloaded Python installation executable. #> - if ($this.Version -ge 3.11.0) { + $PkgVersion = [semver]"3.11.0" + + if ($this.Version -ge $PkgVersion) { Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..." $this.DownloadPkg() From 7e276fa6c778cfc306a4afc20fdad364559c15de Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 5 Sep 2022 10:30:56 +0200 Subject: [PATCH 09/19] fix beta version --- builders/macos-python-builder.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index c7566058a..297d938ca 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -142,7 +142,7 @@ class macOSPythonBuilder : NixPythonBuilder { Generates Python artifact from downloaded Python installation executable. #> - $PkgVersion = [semver]"3.11.0" + $PkgVersion = [semver]"3.11.0-beta.1" if ($this.Version -ge $PkgVersion) { Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..." From e54b0cc7bf47f9f4fd62584d8b835b50c270779a Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 5 Sep 2022 10:43:41 +0200 Subject: [PATCH 10/19] fix building from source --- builders/macos-python-builder.psm1 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index 297d938ca..73c3197e6 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -151,11 +151,7 @@ class macOSPythonBuilder : NixPythonBuilder { Write-Host "Create installation script..." $this.CreateInstallationScriptPkg() } else { - Write-Host "Download Python $($this.Version) [$($this.Architecture)] executable..." - $this.Download() - - Write-Host "Create installation script..." - $this.CreateInstallationScript() + super().Build() } Write-Host "Archive artifact" From 7af2663514f24cfd215f922fd30bc7d8cc2f48fb Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 5 Sep 2022 11:37:49 +0200 Subject: [PATCH 11/19] fix building from source --- builders/macos-python-builder.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index 73c3197e6..0f1c1b923 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -151,7 +151,7 @@ class macOSPythonBuilder : NixPythonBuilder { Write-Host "Create installation script..." $this.CreateInstallationScriptPkg() } else { - super().Build() + ([NixPythonBuilder]$this).Build() } Write-Host "Archive artifact" From 6a0e66549676a4086fc48b3f08bd7201e809b755 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 5 Sep 2022 12:39:51 +0200 Subject: [PATCH 12/19] fix pkg name --- builders/macos-python-builder.psm1 | 24 +++++++++++++++++++----- installers/macos-pkg-setup-template.sh | 3 ++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index 0f1c1b923..f31d38c7a 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -84,19 +84,32 @@ class macOSPythonBuilder : NixPythonBuilder { Execute-Command -Command $configureString } - [uri] GetPkgUri() { + [string] GetPkgName() { <# .SYNOPSIS - Get base Python URI and return complete URI for Python installation executable. + Return Python installation Package. #> - $base = $this.GetBaseUri() - $versionName = $this.GetBaseVersion() $nativeVersion = Convert-Version -version $this.Version $architecture = "-macos11" $extension = ".pkg" - $uri = "${base}/${versionName}/python-${nativeVersion}${architecture}${extension}" + $pkg = "python-${nativeVersion}${architecture}${extension}" + + return $pkg + } + + [uri] GetPkgUri() { + <# + .SYNOPSIS + Get base Python URI and return complete URI for Python installation package. + #> + + $base = $this.GetBaseUri() + $versionName = $this.GetBaseVersion() + $pkg = $this.GetPkgName() + + $uri = "${base}/${versionName}/${pkg}" return $uri } @@ -129,6 +142,7 @@ class macOSPythonBuilder : NixPythonBuilder { $variablesToReplace = @{ "{{__VERSION_FULL__}}" = $this.Version; + "{{__PKG_NAME__}}" = $this.GetPkgName(); } $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) } diff --git a/installers/macos-pkg-setup-template.sh b/installers/macos-pkg-setup-template.sh index 45f2cd3d8..af6244208 100644 --- a/installers/macos-pkg-setup-template.sh +++ b/installers/macos-pkg-setup-template.sh @@ -1,6 +1,7 @@ set -e PYTHON_FULL_VERSION="{{__VERSION_FULL__}}" +PYTHON_PKG_NAME="{{__PKG_NAME__}}" MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1) MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2) @@ -35,7 +36,7 @@ else fi echo "Install Python binaries from prebuilt package" -sudo installer -pkg "python-${PYTHON_FULL_VERSION}-macos11.pkg" -target / +sudo installer -pkg $PYTHON_PKG_NAME -target / rm $PYTHON_TOOLCACHE_VERSION_ARCH_PATH/setup.sh echo "Create hostedtoolcach symlinks (Required for the backward compatibility)" From 0918f49149e34efb4617b0ca9aebf81f9b96e3bd Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 5 Sep 2022 13:42:01 +0200 Subject: [PATCH 13/19] fix setup.sh --- installers/macos-pkg-setup-template.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/installers/macos-pkg-setup-template.sh b/installers/macos-pkg-setup-template.sh index af6244208..0446ee97b 100644 --- a/installers/macos-pkg-setup-template.sh +++ b/installers/macos-pkg-setup-template.sh @@ -37,7 +37,6 @@ fi echo "Install Python binaries from prebuilt package" sudo installer -pkg $PYTHON_PKG_NAME -target / -rm $PYTHON_TOOLCACHE_VERSION_ARCH_PATH/setup.sh echo "Create hostedtoolcach symlinks (Required for the backward compatibility)" echo "Create Python $PYTHON_FULL_VERSION folder" From ea3e94b774c99ca35d131ac29050b3e84b3de21f Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 5 Sep 2022 17:12:18 +0200 Subject: [PATCH 14/19] fix test --- tests/sources/python-config-test.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/sources/python-config-test.py b/tests/sources/python-config-test.py index a53ad6c83..0454a170e 100644 --- a/tests/sources/python-config-test.py +++ b/tests/sources/python-config-test.py @@ -4,12 +4,16 @@ import sys import platform import os +import semver # Define variables os_type = platform.system() version = sys.argv[1] nativeVersion = sys.argv[2] +version_sv = semver.VersionInfo.parse(version) +pkg_installer = version_sv.major > 3 or version_sv.major == 3 and version_sv.minor >= 11 + lib_dir_path = sysconfig.get_config_var('LIBDIR') ld_library_name = sysconfig.get_config_var('LDLIBRARY') @@ -19,7 +23,11 @@ ### Define expected variables if os_type == 'Linux': expected_ld_library_extension = 'so' if os_type == 'Darwin': expected_ld_library_extension = 'dylib' -expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version) + +if pkg_installer: + expected_lib_dir_path = '/Library/Frameworks/Python.framework/Versions/{0}.{1}/lib'.format(version_sv.major, version_sv.minor) +else: + expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version) # Check modules ### Validate libraries path From 7156cc7087934d8aef448e62e56d4064101b6c93 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 6 Sep 2022 08:53:11 +0200 Subject: [PATCH 15/19] Fix config test with semver --- tests/sources/python-config-test.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/sources/python-config-test.py b/tests/sources/python-config-test.py index 0454a170e..b327327c8 100644 --- a/tests/sources/python-config-test.py +++ b/tests/sources/python-config-test.py @@ -4,15 +4,17 @@ import sys import platform import os -import semver # Define variables os_type = platform.system() version = sys.argv[1] nativeVersion = sys.argv[2] -version_sv = semver.VersionInfo.parse(version) -pkg_installer = version_sv.major > 3 or version_sv.major == 3 and version_sv.minor >= 11 +versions=version.split(".") +version_major=int(versions[0]) +version_minor=int(versions[1]) + +pkg_installer = os_type == 'Darwin' and (version_major > 3 or version_major == 3 and version_minor >= 11) lib_dir_path = sysconfig.get_config_var('LIBDIR') ld_library_name = sysconfig.get_config_var('LDLIBRARY') @@ -25,7 +27,7 @@ if os_type == 'Darwin': expected_ld_library_extension = 'dylib' if pkg_installer: - expected_lib_dir_path = '/Library/Frameworks/Python.framework/Versions/{0}.{1}/lib'.format(version_sv.major, version_sv.minor) + expected_lib_dir_path = '/Library/Frameworks/Python.framework/Versions/{0}.{1}/lib'.format(version_major, version_minor) else: expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version) @@ -46,7 +48,8 @@ exit(1) else: print('%s was built without shared extensions' % ld_library_name) - exit(1) + if not pkg_installer: + exit(1) ### Validate macOS if os_type == 'Darwin': @@ -67,12 +70,14 @@ if openssl_includes != expected_openssl_includes: print('Invalid openssl_includes: %s; Expected: %s' % (openssl_includes, expected_openssl_includes)) - exit(1) + if not pkg_installer: + exit(1) if openssl_ldflags != expected_openssl_ldflags: print('Invalid openssl_ldflags: %s; Expected: %s' % (openssl_ldflags, expected_openssl_ldflags)) - exit(1) + if not pkg_installer: + exit(1) ### Validate libreadline if not have_libreadline: print('Missing libreadline') - exit(1) \ No newline at end of file + exit(1) From d89512ec36a00b223c4d7ee206debeafba7ff479 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Wed, 14 Sep 2022 10:06:20 +0200 Subject: [PATCH 16/19] Fix PYTHON_MAJOR_MINOR --- installers/macos-pkg-setup-template.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/installers/macos-pkg-setup-template.sh b/installers/macos-pkg-setup-template.sh index 0446ee97b..590679673 100644 --- a/installers/macos-pkg-setup-template.sh +++ b/installers/macos-pkg-setup-template.sh @@ -7,7 +7,7 @@ MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2) PYTHON_MAJOR=python$MAJOR_VERSION PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION -PYTHON_MAJORMINOR=python$MAJOR_VERSION$MINOR_VERSION +PYTHON_MAJOR_MINOR=python$MAJOR_VERSION$MINOR_VERSION if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then # No AGENT_TOOLSDIRECTORY on GitHub images @@ -52,12 +52,12 @@ echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipeli ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python cd bin/ -ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR +ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR if [ ! -f python ]; then ln -s $PYTHON_MAJOR_DOT_MINOR python fi -chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR python +chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python echo "Upgrading pip..." ./python -m ensurepip From 1b9b897ff211934af96af6e10a95d397a3fa4579 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Wed, 14 Sep 2022 10:13:30 +0200 Subject: [PATCH 17/19] migrate from .format to interpolation --- tests/sources/python-config-test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/sources/python-config-test.py b/tests/sources/python-config-test.py index b327327c8..928fa2bac 100644 --- a/tests/sources/python-config-test.py +++ b/tests/sources/python-config-test.py @@ -27,9 +27,9 @@ if os_type == 'Darwin': expected_ld_library_extension = 'dylib' if pkg_installer: - expected_lib_dir_path = '/Library/Frameworks/Python.framework/Versions/{0}.{1}/lib'.format(version_major, version_minor) + expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib' else: - expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version) + expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/x64/lib' # Check modules ### Validate libraries path From 372758f2377b9befdca57329c5cf82c7507711b4 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Wed, 14 Sep 2022 10:19:34 +0200 Subject: [PATCH 18/19] add PYTHON_FRAMEWORK_PATH variable --- installers/macos-pkg-setup-template.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/installers/macos-pkg-setup-template.sh b/installers/macos-pkg-setup-template.sh index 590679673..67c57c860 100644 --- a/installers/macos-pkg-setup-template.sh +++ b/installers/macos-pkg-setup-template.sh @@ -19,6 +19,7 @@ fi PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64 +PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}" echo "Check if Python hostedtoolcache folder exist..." if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then @@ -43,10 +44,10 @@ echo "Create Python $PYTHON_FULL_VERSION folder" mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH -ln -s /Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}/bin bin -ln -s /Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}/include include -ln -s /Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}/share share -ln -s /Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}/lib lib +ln -s "${PYTHON_FRAMEWORK_PATH}/bin" bin +ln -s "${PYTHON_FRAMEWORK_PATH}/include" include +ln -s "${PYTHON_FRAMEWORK_PATH}/share" share +ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)" ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python From be69151d155f0ef32b58dce7f62e4f39c5b07f6b Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Wed, 14 Sep 2022 10:59:56 +0200 Subject: [PATCH 19/19] improve pkg condition --- tests/sources/python-config-test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sources/python-config-test.py b/tests/sources/python-config-test.py index 928fa2bac..9ebcf242b 100644 --- a/tests/sources/python-config-test.py +++ b/tests/sources/python-config-test.py @@ -14,7 +14,7 @@ version_major=int(versions[0]) version_minor=int(versions[1]) -pkg_installer = os_type == 'Darwin' and (version_major > 3 or version_major == 3 and version_minor >= 11) +pkg_installer = os_type == 'Darwin' and (version_major == 3 and version_minor >= 11) lib_dir_path = sysconfig.get_config_var('LIBDIR') ld_library_name = sysconfig.get_config_var('LDLIBRARY')