Publish PyKotor Tools CI/CD #333
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish PyKotor Tools CI/CD | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: false | |
| env: | |
| OS_RUNNERS_JSON: '["windows-latest", "ubuntu-latest", "macos-latest"]' | |
| PYTHON_VERSIONS_JSON: '["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]' | |
| ARCHITECTURES_JSON: '["x86", "x64"]' | |
| UPX_VERSION: "4.2.2" | |
| USE_UPX: true | |
| BUILD_BOOTLOADERS: false # Build pyinstaller bootloaders. Pretty much unusable due to AV heuristics. | |
| on: | |
| # push: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 6 * * 1" # Runs every Monday at 06:00 UTC | |
| permissions: | |
| contents: write | |
| jobs: | |
| detect-tools: | |
| name: Detect Tools | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tools_matrix: ${{ steps.detect.outputs.tools_matrix }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Discover tools and compile scripts | |
| id: detect | |
| run: | | |
| python3 << 'PYTHON_SCRIPT' | |
| import json | |
| import os | |
| import subprocess | |
| # Check if we're in test mode (limit to one tool) | |
| test_mode = os.environ.get("ACT_TEST_MODE", "").lower() == "true" | |
| test_tool = os.environ.get("ACT_TEST_TOOL", "").strip() | |
| tools = json.loads(subprocess.check_output([ | |
| "python3", | |
| ".github/scripts/discover_tools.py", | |
| "--format", | |
| "json", | |
| ], text=True)) | |
| # In test mode, only process the specified tool (or first tool if none specified) | |
| if test_mode: | |
| if test_tool: | |
| tools = [ | |
| t for t in tools | |
| if test_tool.lower() in { | |
| t["directory"].lower(), | |
| (t.get("build_name") or t.get("name") or t["directory"]).lower(), | |
| } | |
| ] | |
| else: | |
| tools = tools[:1] | |
| print("Detected tools:") | |
| for t in tools: | |
| build_name = (t.get("build_name") or t.get("name") or t["directory"]).lower() | |
| print(f" - {t['display_name']} (build: {build_name})") | |
| tools_json = json.dumps(tools) | |
| print(f"Tools matrix JSON: {tools_json}") | |
| print(f"Number of tools: {len(tools)}") | |
| print(f"JSON length: {len(tools_json)}") | |
| # GitHub Actions outputs: use simple single-line format (same as build-pr.yml) | |
| # fromJson() can parse single-line JSON arrays | |
| # CRITICAL: Ensure no newlines in the JSON string | |
| tools_json_clean = tools_json.replace("\n", "").replace("\r", "") | |
| print(f"Cleaned JSON length: {len(tools_json_clean)}") | |
| with open(os.environ["GITHUB_OUTPUT"], "a") as fh: | |
| fh.write(f"tools_matrix={tools_json_clean}\n") | |
| print(f"✓ Output written to GITHUB_OUTPUT") | |
| print(f" Key: tools_matrix") | |
| print(f" Value (first 200 chars): {tools_json_clean[:200]}...") | |
| # Also generate full build matrix for easier fromJSON() parsing | |
| # This creates a single matrix with all combinations | |
| import os as os_module | |
| os_runners = json.loads(os_module.environ.get("OS_RUNNERS_JSON", '["windows-latest", "ubuntu-latest", "macos-latest"]')) | |
| python_versions = json.loads(os_module.environ.get("PYTHON_VERSIONS_JSON", '["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]')) | |
| architectures = json.loads(os_module.environ.get("ARCHITECTURES_JSON", '["x86", "x64"]')) | |
| # Generate full matrix | |
| full_matrix = [] | |
| for tool in tools: | |
| build_name = (tool.get("build_name") or tool.get("name") or tool["directory"]).lower() | |
| for os_name in os_runners: | |
| for py_ver in python_versions: | |
| for arch in architectures: | |
| # Skip unsupported combinations | |
| if os_name in ["ubuntu-latest", "macos-latest"] and arch == "x86": | |
| continue | |
| # Only add Windows entries to full_matrix (for VC redist URLs) | |
| # Other OS entries are handled by the main matrix | |
| if os_name == "windows-latest": | |
| full_matrix.append({ | |
| "tool": tool, | |
| "os": os_name, | |
| "python-version": py_ver, | |
| "architecture": arch, | |
| "build_name": build_name, | |
| "qt_version": "pyqt6", | |
| "arch": arch, | |
| "vc_redist2015": f"https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.{'x86' if arch == 'x86' else 'x64'}.exe", | |
| "vc_redist-latest": f"https://aka.ms/vs/17/release/vc_redist.{'x86' if arch == 'x86' else 'x64'}.exe", | |
| "vc_redist2019": f"https://aka.ms/vs/17/release/vc_redist.{'x86' if arch == 'x86' else 'x64'}.exe" | |
| }) | |
| full_matrix_json = json.dumps(full_matrix) | |
| full_matrix_json_clean = full_matrix_json.replace("\n", "").replace("\r", "") | |
| print(f"Full matrix: {len(full_matrix)} combinations") | |
| with open(os.environ["GITHUB_OUTPUT"], "a") as fh: | |
| fh.write(f"full_matrix={full_matrix_json_clean}\n") | |
| print(f"✓ Full matrix written to GITHUB_OUTPUT") | |
| PYTHON_SCRIPT | |
| env: | |
| OS_RUNNERS_JSON: ${{ env.OS_RUNNERS_JSON }} | |
| PYTHON_VERSIONS_JSON: ${{ env.PYTHON_VERSIONS_JSON }} | |
| ARCHITECTURES_JSON: ${{ env.ARCHITECTURES_JSON }} | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| os: ${{ steps.set_env.outputs.os }} | |
| python-version: ${{ steps.set_env.outputs.python-version }} | |
| architecture: ${{ steps.set_env.outputs.architecture }} | |
| #qt_version: ${#{ steps.set_env.outputs.qt_version }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Install PowerShell on non-Windows | |
| if: ${{ (success() || failure()) && runner.os != 'Windows' }} | |
| shell: bash | |
| run: | | |
| chmod +x ./install_powershell.sh | |
| bash ./install_powershell.sh | |
| - name: Set environment variables | |
| id: set_env | |
| if: ${{ (success() || failure()) }} | |
| env: | |
| ACT_TEST_MODE: ${{ env.ACT_TEST_MODE }} | |
| run: | | |
| # In test mode, limit matrix to speed up testing | |
| $testMode = $Env:ACT_TEST_MODE -eq "true" | |
| if ($testMode) { | |
| $osJson = '["ubuntu-latest"]' | |
| $pythonJson = '["3.13"]' | |
| $archJson = '["x64"]' | |
| Write-Host "Test mode: limiting matrix to ubuntu-latest, Python 3.13, x64" | |
| } else { | |
| $osJson = '${{ env.OS_RUNNERS_JSON }}' | |
| $pythonJson = '${{ env.PYTHON_VERSIONS_JSON }}' | |
| $archJson = '${{ env.ARCHITECTURES_JSON }}' | |
| } | |
| # Use simple format (no EOF delimiters) to match detect-tools job format | |
| # This ensures fromJSON() can parse the outputs correctly | |
| $singleLineJson = $osJson -replace "`r", "" -replace "`n", "" | |
| echo "os=$singleLineJson" >> $Env:GITHUB_OUTPUT | |
| $singleLineJson = $pythonJson -replace "`r", "" -replace "`n", "" | |
| echo "python-version=$singleLineJson" >> $Env:GITHUB_OUTPUT | |
| $singleLineJson = $archJson -replace "`r", "" -replace "`n", "" | |
| echo "architecture=$singleLineJson" >> $Env:GITHUB_OUTPUT | |
| shell: pwsh | |
| build: | |
| needs: [setup, detect-tools] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tool: ${{ fromJson(needs.detect-tools.outputs.tools_matrix) }} | |
| os: ${{ fromJson(needs.setup.outputs.os) }} | |
| python-version: ${{ fromJson(needs.setup.outputs.python-version) }} | |
| architecture: ${{ fromJson(needs.setup.outputs.architecture) }} | |
| qt_version: ["pyqt6"] | |
| include: | |
| - architecture: x86 | |
| vc_redist2015: "https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.x86.exe" | |
| vc_redist-latest: "https://aka.ms/vs/17/release/vc_redist.x86.exe" | |
| vc_redist2019: "https://aka.ms/vs/17/release/vc_redist.x86.exe" | |
| - architecture: x64 | |
| vc_redist2015: "https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.x64.exe" | |
| vc_redist-latest: "https://aka.ms/vs/17/release/vc_redist.x64.exe" | |
| vc_redist2019: "https://aka.ms/vs/17/release/vc_redist.x64.exe" | |
| exclude: | |
| # unix x86 is definitely not supported. | |
| - os: ubuntu-latest | |
| architecture: x86 | |
| - os: macos-latest | |
| architecture: x86 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Install PowerShell on non-Windows | |
| if: ${{ (success() || failure()) && runner.os != 'Windows' }} | |
| shell: bash | |
| run: | | |
| chmod +x ./install_powershell.sh | |
| bash ./install_powershell.sh | |
| - name: Determine Python version string | |
| id: set-python-version | |
| if: ${{ (success() || failure()) }} | |
| run: | | |
| Add-Content -Path $Env:GITHUB_ENV -Value "PYTHON_VERSION=${{ matrix.python-version }}" | |
| shell: pwsh | |
| env: | |
| MATRIX_ARCH: ${{ matrix.architecture }} | |
| - name: Reset APT sources to default | |
| if: ${{ (success() || failure()) && runner.os == 'Linux' }} | |
| run: | | |
| echo "Resetting APT sources to default Ubuntu repositories" | |
| sudo rm /etc/apt/sources.list | |
| echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs) main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list | |
| echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list | |
| echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-backports main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list | |
| echo "deb http://security.ubuntu.com/ubuntu $(lsb_release -cs)-security main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list | |
| sudo apt-get update -y | |
| shell: bash | |
| - name: Add deadsnakes PPA for older Python versions (Linux) | |
| if: ${{ (success() || failure()) && runner.os == 'Linux' }} | |
| run: | | |
| sudo apt-get install -y software-properties-common | |
| sudo add-apt-repository -y ppa:deadsnakes/ppa | |
| sudo apt-get update -y | |
| shell: bash | |
| - name: Cache UPX | |
| if: ${{ (success() || failure()) && runner.os != 'macOS' }} | |
| uses: actions/cache@v5.0.5 | |
| id: cache-upx | |
| with: | |
| path: upx-dir | |
| key: upx-${{ env.UPX_VERSION }}-${{ runner.os }}-${{ matrix.architecture }} | |
| restore-keys: | | |
| upx-${{ env.UPX_VERSION }}-${{ runner.os }}- | |
| - name: Set UPX download URL | |
| # upx docs express that crashes are happening on ventura and above with upx, don't use on mac. | |
| if: ${{ (success() || failure()) && runner.os != 'macOS' }} | |
| id: upx_setup | |
| run: | | |
| $build = "no" | |
| $archiveName = "" | |
| if ("${{ runner.os }}" -eq "Windows") { | |
| if ("${{ matrix.architecture }}" -eq "x86") { | |
| $archiveName = "upx-${{ env.UPX_VERSION }}-win32.zip" | |
| } else { | |
| $archiveName = "upx-${{ env.UPX_VERSION }}-win64.zip" | |
| } | |
| } elseif ("${{ runner.os }}" -eq "Linux") { | |
| $archiveName = "upx-${{ env.UPX_VERSION }}-amd64_linux.tar.xz" | |
| } elseif ("${{ runner.os }}" -eq "macOS") { | |
| $build = "yes" | |
| $archiveName = "upx-${{ env.UPX_VERSION }}-src.tar.xz" | |
| } | |
| $url = "https://github.com/upx/upx/releases/download/v${{ env.UPX_VERSION }}/$archiveName" | |
| Add-Content -Path $Env:GITHUB_OUTPUT -Value "build=$build" | |
| Add-Content -Path $Env:GITHUB_OUTPUT -Value "url=$url" | |
| Add-Content -Path $Env:GITHUB_OUTPUT -Value "archiveName=$archiveName" | |
| shell: pwsh | |
| - name: Download and prepare UPX | |
| if: ${{ (success() || failure()) && runner.os != 'macOS' && steps.cache-upx.outputs.cache-hit != 'true' }} | |
| run: | | |
| $ext = "${{ runner.os }}" -eq "Windows" ? "zip" : "tar.xz" | |
| $url = "${{ steps.upx_setup.outputs.url }}" | |
| $archiveName = "${{ steps.upx_setup.outputs.archiveName }}" | |
| $outputPath = "upx-dir" | |
| # Use Invoke-WebRequest or curl depending on the OS | |
| if ("${{ runner.os }}" -eq "Windows") { | |
| Invoke-WebRequest -Uri $url -OutFile $archiveName | |
| } elseif ("${{ runner.os }}" -eq "Linux") { | |
| curl -L $url -o $archiveName | |
| } | |
| New-Item -ItemType Directory -Force -Path "upx-dir" -ErrorAction SilentlyContinue | |
| if ("${{ runner.os }}" -ne "macOS") { | |
| if ($ext -eq "zip") { | |
| $fileNameWithoutExtension = [System.IO.Path]::GetFileNameWithoutExtension($archiveName) | |
| Expand-Archive -Path $archiveName -DestinationPath "temp_folder_upx" | |
| if (-not (Test-Path -Path "upx-dir")) { | |
| New-Item -ItemType Directory -Path "upx-dir" | |
| } | |
| Get-ChildItem -LiteralPath "temp_folder_upx/$fileNameWithoutExtension" -Recurse | Move-Item -Destination "upx-dir" | |
| Remove-Item "temp_folder_upx" -Recurse -Force -ErrorAction SilentlyContinue | |
| } else { | |
| tar -xvf $archiveName --strip-components=1 -C "upx-dir" | |
| } | |
| Remove-Item $archiveName # Clean up downloaded archive | |
| } | |
| shell: pwsh | |
| - name: Set UPX directory path | |
| if: ${{ (success() || failure()) && runner.os != 'macOS' }} | |
| id: upx_dir | |
| run: | | |
| $upx_dir = "./upx-dir" | |
| $upx_dir = $([System.IO.Path]::GetFullPath('./upx-dir')) | |
| Dir -Recurse $upx_dir | Get-Childitem | |
| echo "UPX_DIR=$upx_dir" | Out-File -FilePath $Env:GITHUB_ENV -Append | |
| Write-Output "UPX_DIR set to '$upx_dir'" | |
| shell: pwsh | |
| - name: Install Visual Studio 2015 C++ Redistributable | |
| if: ${{ (success() || failure()) && runner.os == 'Windows' }} | |
| run: | | |
| $url = "${{ matrix.vc_redist2015 }}" | |
| $output = "vc_redist.exe" | |
| Invoke-WebRequest -Uri $url -OutFile $output | |
| Start-Process $output -ArgumentList '/install', '/quiet', '/norestart' -Wait | |
| Remove-Item -Path $output | |
| #choco install vcredist2015 -y | |
| shell: pwsh | |
| - name: Install Visual Studio 2019 C++ Redistributable | |
| if: ${{ (success() || failure()) && runner.os == 'Windows' }} | |
| run: | | |
| $url = "${{ matrix.vc_redist2019 }}" | |
| $output = "vc_redist.exe" | |
| Invoke-WebRequest -Uri $url -OutFile $output | |
| Start-Process $output -ArgumentList '/install', '/quiet', '/norestart' -Wait | |
| Remove-Item -Path $output | |
| #choco install vcredist2019 -y | |
| shell: pwsh | |
| - name: Install Visual Studio latest C++ Redistributable | |
| if: ${{ (success() || failure()) && runner.os == 'Windows' }} | |
| run: | | |
| $url = "${{ matrix.vc_redist-latest }}" | |
| $output = "vc_redist.exe" | |
| Invoke-WebRequest -Uri $url -OutFile $output | |
| Start-Process $output -ArgumentList '/install', '/quiet', '/norestart' -Wait | |
| Remove-Item -Path $output | |
| shell: pwsh | |
| - name: Locate compile and dependency scripts | |
| id: locate-scripts | |
| if: ${{ (success() || failure()) }} | |
| shell: pwsh | |
| run: | | |
| # Matrix now contains tool object directly | |
| $buildName = "${{ matrix.tool.build_name }}" | |
| $toolDir = "${{ matrix.tool.directory }}" | |
| $toolPath = "${{ matrix.tool.path }}" | |
| $compileScript = "compile/compile_tool.ps1" | |
| $depsScript = "compile/deps_tool.ps1" | |
| if (-not (Test-Path $compileScript)) { | |
| Write-Error "Required build wrapper missing: $compileScript" | |
| exit 1 | |
| } | |
| if (-not (Test-Path $depsScript)) { | |
| Write-Error "Required build wrapper missing: $depsScript" | |
| exit 1 | |
| } | |
| echo "compile_script=$compileScript" >> $Env:GITHUB_OUTPUT | |
| echo "deps_script=$depsScript" >> $Env:GITHUB_OUTPUT | |
| echo "use_compile_tool=True" >> $Env:GITHUB_OUTPUT | |
| echo "tool_dir=$toolDir" >> $Env:GITHUB_OUTPUT | |
| echo "tool_path=$toolPath" >> $Env:GITHUB_OUTPUT | |
| echo "Using compile script $compileScript" | |
| if ($depsScript) { echo "Using deps script $depsScript" } else { echo "No deps script found" } | |
| - name: Pre-install Python packages (Linux workaround) | |
| if: ${{ (success() || failure()) && runner.os == 'Linux' }} | |
| shell: bash | |
| run: | | |
| # Workaround: Pre-install Python without pip package (python3.X-pip doesn't exist in deadsnakes PPA) | |
| # This prevents venv creation from failing and prompting for input | |
| PYTHON_VERSION="${{ matrix.python-version }}" | |
| PYTHON_MAJOR_MINOR=$(echo $PYTHON_VERSION | cut -d. -f1,2) | |
| echo "Adding deadsnakes PPA for Python $PYTHON_VERSION..." | |
| sudo apt-get update -qq | |
| sudo apt-get install -y software-properties-common | |
| sudo add-apt-repository -y ppa:deadsnakes/ppa || true | |
| sudo apt-get update -qq | |
| echo "Pre-installing Python $PYTHON_VERSION packages (without pip package)..." | |
| sudo apt-get install -y \ | |
| python${PYTHON_MAJOR_MINOR} \ | |
| python${PYTHON_MAJOR_MINOR}-dev \ | |
| python${PYTHON_MAJOR_MINOR}-venv \ | |
| libpython${PYTHON_MAJOR_MINOR}-dev || true | |
| # Verify Python is available | |
| python${PYTHON_MAJOR_MINOR} --version || (echo "Failed to install Python $PYTHON_VERSION" && exit 1) | |
| - name: Cache pip packages (Linux/macOS) | |
| if: ${{ (success() || failure()) && runner.os != 'Windows' }} | |
| uses: actions/cache@v5.0.5 | |
| id: cache-pip-unix | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/Library/Caches/pip | |
| key: pip-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.architecture }}-${{ hashFiles('**/requirements.txt', '**/pyproject.toml', '**/setup.py') }} | |
| restore-keys: | | |
| pip-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.architecture }}- | |
| pip-${{ runner.os }}-${{ matrix.python-version }}- | |
| pip-${{ runner.os }}- | |
| - name: Get Windows pip cache path | |
| if: ${{ (success() || failure()) && runner.os == 'Windows' }} | |
| id: pip-cache-path | |
| shell: pwsh | |
| run: | | |
| $pipCache = "$Env:LOCALAPPDATA\pip\Cache" | |
| echo "path=$pipCache" >> $Env:GITHUB_OUTPUT | |
| - name: Cache pip packages (Windows) | |
| if: ${{ (success() || failure()) && runner.os == 'Windows' }} | |
| uses: actions/cache@v5.0.5 | |
| id: cache-pip-windows | |
| with: | |
| path: ${{ steps.pip-cache-path.outputs.path }} | |
| key: pip-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.architecture }}-${{ hashFiles('**/requirements.txt', '**/pyproject.toml', '**/setup.py') }} | |
| restore-keys: | | |
| pip-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.architecture }}- | |
| pip-${{ runner.os }}-${{ matrix.python-version }}- | |
| pip-${{ runner.os }}- | |
| - name: Create virtual environment | |
| shell: pwsh | |
| if: ${{ (success() || failure()) }} | |
| env: | |
| MATRIX_ARCH: ${{ matrix.architecture }} | |
| run: | | |
| $ErrorActionPreference = "Stop" | |
| $buildName = "${{ matrix.tool.build_name }}" | |
| $toolDir = "${{ matrix.tool.directory }}" | |
| $toolPath = "${{ matrix.tool.path }}" | |
| $venvName = ".venv_${buildName}_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}" | |
| $Env:PYKOTOR_VENV_NAME = $venvName | |
| # Set LD_LIBRARY_PATH for Linux (needed for some Python installations) | |
| if ("${{ runner.os }}" -eq "Linux") { | |
| $Env:LD_LIBRARY_PATH = "/usr/local/lib" + ":" + $Env:LD_LIBRARY_PATH | |
| } | |
| $pyinstallerSpec = "pyinstaller" | |
| if ("${{ runner.os }}" -eq "Windows") { | |
| if ("${{ matrix.python-version }}" -eq "3.12" -or "${{ matrix.python-version }}" -eq "3.11") { | |
| $pyinstallerSpec = "pyinstaller==5.13.2" | |
| } else { | |
| $pyinstallerSpec = "pyinstaller==5.4" | |
| } | |
| } | |
| $depsArgs = @( | |
| "--tool-path", $toolPath, | |
| "--venv-name", $venvName, | |
| "--noprompt", | |
| "--force-python-version", "${{ matrix.python-version }}", | |
| "--pip-requirements", "./Libraries/PyKotor/requirements.txt", | |
| "--pip-install-pyinstaller", $pyinstallerSpec | |
| ) | |
| if ("${{ matrix.tool.requires_qt }}" -eq "true") { | |
| $depsArgs += @("--qt-api", "PyQt6") | |
| } | |
| Write-Host "Running deps_tool.ps1: $($depsArgs -join ' ')" | |
| & ./compile/deps_tool.ps1 @depsArgs | |
| if ($LASTEXITCODE -ne 0) { | |
| throw "deps_tool.ps1 failed with exit code $LASTEXITCODE" | |
| } | |
| $repoRoot = (Get-Location).Path | |
| $venvPython = if ("${{ runner.os }}" -eq "Windows") { | |
| Join-Path $repoRoot "$venvName/Scripts/python.exe" | |
| } else { | |
| Join-Path $repoRoot "$venvName/bin/python" | |
| } | |
| if (-not (Test-Path -LiteralPath $venvPython)) { | |
| throw "Venv python not found at $venvPython" | |
| } | |
| Write-Host "pythonExePath=$venvPython" | |
| & $venvPython --version | |
| # Persist across subsequent steps (each step is a fresh process) | |
| Add-Content -Path $Env:GITHUB_ENV -Value "pythonExePath=$venvPython" | |
| Add-Content -Path $Env:GITHUB_ENV -Value "PYKOTOR_VENV_NAME=$venvName" | |
| Add-Content -Path $Env:GITHUB_STEP_SUMMARY -Value "### Python/Venv" | |
| Add-Content -Path $Env:GITHUB_STEP_SUMMARY -Value ("- Venv: `{0}`" -f $venvName) | |
| Add-Content -Path $Env:GITHUB_STEP_SUMMARY -Value ("- Python: `{0}`" -f $venvPython) | |
| - name: Compile tool | |
| shell: pwsh | |
| env: | |
| MATRIX_ARCH: ${{ matrix.architecture }} | |
| run: | | |
| $ErrorActionPreference = "Stop" | |
| $buildName = "${{ matrix.tool.build_name }}" | |
| $venvName = ".venv_${buildName}_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}" | |
| $upxDir = $Env:UPX_DIR | |
| $Env:PYTHONOPTIMIZE = "1" | |
| # Set LD_LIBRARY_PATH for Linux | |
| if ("${{ runner.os }}" -eq "Linux") { | |
| $Env:LD_LIBRARY_PATH = "/usr/local/lib" + ":" + $Env:LD_LIBRARY_PATH | |
| } | |
| if (-not $Env:pythonExePath) { | |
| Write-Error "env:pythonExePath is missing. Expected Create virtual environment step to populate GITHUB_ENV." | |
| Write-Host "Available environment variables:" -ForegroundColor Yellow | |
| Get-ChildItem Env: | Where-Object { $_.Name -like "*python*" -or $_.Name -like "*venv*" } | ForEach-Object { Write-Host " $($_.Name)=$($_.Value)" } | |
| exit 1 | |
| } | |
| # Add Python to PATH for compilation | |
| $pythonDir = Split-Path -Parent $Env:pythonExePath | |
| if ("${{ runner.os }}" -eq "Windows") { | |
| $Env:PATH = "$pythonDir;$Env:PATH" | |
| } else { | |
| $Env:PATH = "$pythonDir" + ":" + $Env:PATH | |
| } | |
| Write-Host "Using compile wrapper compile/compile_tool.ps1" | |
| Write-Host "Venv name: $venvName" | |
| Write-Host "UPX directory: $upxDir" | |
| Write-Host "pythonExePath=$Env:pythonExePath" | |
| Write-Host "Python directory added to PATH: $pythonDir" | |
| $toolPath = "${{ steps.locate-scripts.outputs.tool_path }}" | |
| $compileArgs = @( | |
| "--tool-path", $toolPath, | |
| "--venv-name", $venvName, | |
| "--skip-venv", | |
| "--python-exe", $Env:pythonExePath, | |
| "--noprompt" | |
| ) | |
| if ("${{ matrix.tool.requires_qt }}" -eq "true") { | |
| $compileArgs += @("--qt-api", "PyQt6", "--exclude-other-qt") | |
| } | |
| if ($upxDir) { $compileArgs += @("--upx-dir", $upxDir) } | |
| Write-Host "Running: ./compile/compile_tool.ps1 $($compileArgs -join ' ')" | |
| & ./compile/compile_tool.ps1 @compileArgs | |
| if ($LASTEXITCODE -ne 0) { | |
| Write-Error "Compilation failed with exit code $LASTEXITCODE" | |
| exit $LASTEXITCODE | |
| } | |
| - name: Upload compiled binaries attempt 1 | |
| if: ${{ (success() || failure()) && !env.ACT }} | |
| id: upload_attempt_1 | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: publish_${{ matrix.tool.build_name }}_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| path: ./dist/** | |
| retention-days: 90 | |
| continue-on-error: true | |
| - name: Upload compiled binaries attempt 2 | |
| id: upload_attempt_2 | |
| if: ${{ (success() || failure()) && !env.ACT && steps.upload_attempt_1.outcome == 'failure' }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: publish_${{ matrix.tool.build_name }}_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| path: ./dist/** | |
| retention-days: 90 | |
| continue-on-error: true | |
| - name: Upload compiled binaries attempt 3 | |
| id: upload_attempt_3 | |
| if: ${{ (success() || failure()) && !env.ACT && steps.upload_attempt_2.outcome == 'failure' }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: publish_${{ matrix.tool.build_name }}_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| path: ./dist/** | |
| retention-days: 90 | |
| continue-on-error: true | |
| - name: Upload compiled binaries attempt 4 | |
| id: upload_attempt_4 | |
| if: ${{ (success() || failure()) && !env.ACT && steps.upload_attempt_3.outcome == 'failure' }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: publish_${{ matrix.tool.build_name }}_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| path: ./dist/** | |
| retention-days: 90 | |
| continue-on-error: true | |
| - name: Upload compiled binaries attempt 5 | |
| id: upload_attempt_5 | |
| if: ${{ (success() || failure()) && !env.ACT && steps.upload_attempt_4.outcome == 'failure' }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: publish_${{ matrix.tool.build_name }}_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| path: ./dist/** | |
| retention-days: 90 | |
| - name: Run tests for compiled tool | |
| shell: pwsh | |
| env: | |
| MATRIX_ARCH: ${{ matrix.architecture }} | |
| run: | | |
| $ErrorActionPreference = "Stop" | |
| $buildName = "${{ matrix.tool.build_name }}" | |
| $venvName = ".venv_${buildName}_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}" | |
| $toolDir = "${{ matrix.tool.path }}" | |
| # Set LD_LIBRARY_PATH for Linux | |
| if ("${{ runner.os }}" -eq "Linux") { | |
| $Env:LD_LIBRARY_PATH = "/usr/local/lib" + ":" + $Env:LD_LIBRARY_PATH | |
| } | |
| # Use pythonExePath from GITHUB_ENV (set by Create virtual environment step) | |
| if (-not $Env:pythonExePath) { | |
| Write-Host "env:pythonExePath not found, constructing from venv name..." -ForegroundColor Yellow | |
| $venvPath = $venvName | |
| if ("${{ runner.os }}" -eq "Windows") { | |
| $pythonExePath = Join-Path $venvPath "Scripts\python.exe" | |
| } else { | |
| $pythonExePath = Join-Path $venvPath "bin\python" | |
| } | |
| } else { | |
| $pythonExePath = $Env:pythonExePath | |
| } | |
| if (-not (Test-Path $pythonExePath)) { | |
| Write-Host "Python executable not found at $pythonExePath, skipping tests" -ForegroundColor Yellow | |
| exit 0 | |
| } | |
| # Add Python to PATH | |
| $pythonDir = Split-Path -Parent $pythonExePath | |
| if ("${{ runner.os }}" -eq "Windows") { | |
| $Env:PATH = "$pythonDir;$Env:PATH" | |
| } else { | |
| $Env:PATH = "$pythonDir" + ":" + $Env:PATH | |
| } | |
| Write-Host "Running tests for ${{ matrix.tool.display_name }}" | |
| Write-Host "Tool directory: $toolDir" | |
| Write-Host "Python executable: $pythonExePath" | |
| # Check if tests directory exists | |
| $testsPath = Join-Path $toolDir "tests" | |
| if (-not (Test-Path $testsPath)) { | |
| Write-Host "No tests directory found at $testsPath, skipping tests" | |
| exit 0 | |
| } | |
| # Install test dependencies if needed | |
| $pyprojectPath = Join-Path $toolDir "pyproject.toml" | |
| if (Test-Path $pyprojectPath) { | |
| Write-Host "Installing test dependencies from pyproject.toml..." | |
| & $pythonExePath -m pip install -e "$toolDir[dev]" --quiet 2>&1 | Out-Null | |
| } | |
| # Install pytest-timeout if available | |
| & $pythonExePath -m pip install pytest-timeout --quiet 2>&1 | Out-Null | |
| # Run tests with timeout (2 minutes for individual tests) | |
| Write-Host "Running pytest tests..." | |
| try { | |
| $pytestArgs = @("$testsPath", "-v", "--tb=short") | |
| # Add timeout if pytest-timeout is available | |
| $timeoutCheck = & $pythonExePath -m pytest --help 2>&1 | Select-String "timeout" | |
| if ($timeoutCheck) { | |
| $pytestArgs += @("--timeout=120", "--timeout-method=thread") | |
| } | |
| if ("${{ runner.os }}" -eq "Linux") { | |
| # Use xvfb for GUI tests on Linux if available | |
| if (Get-Command xvfb-run -ErrorAction SilentlyContinue) { | |
| Write-Host "Using xvfb-run for GUI tests..." | |
| xvfb-run -a & $pythonExePath -m pytest $pytestArgs | |
| } else { | |
| & $pythonExePath -m pytest $pytestArgs | |
| } | |
| } else { | |
| & $pythonExePath -m pytest $pytestArgs | |
| } | |
| if ($LASTEXITCODE -ne 0) { | |
| Write-Host -ForegroundColor Yellow "Tests completed with exit code $LASTEXITCODE (some tests may have failed)" | |
| # Don't fail the build if tests fail, but report it | |
| } else { | |
| Write-Host -ForegroundColor Green "All tests passed!" | |
| } | |
| } catch { | |
| Write-Host -ForegroundColor Yellow "Error running tests: $($_.Exception.Message)" | |
| Write-Host "Continuing despite test errors..." | |
| } | |
| package: # The goal of this job is to repackage by toolname, rather than all tools by os_pyversion_arch | |
| needs: build # do not start this job until all 'build' jobs complete | |
| if: ${{ always() && (needs.build.result == 'success' || needs.build.result == 'failure') }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| filesToArchive: ${{ steps.set-matrix.outputs.matrixJson }} | |
| steps: | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| path: published_workflow_builds/ | |
| pattern: publish_* | |
| - name: Re-package artifacts by Python version and architecture | |
| shell: pwsh | |
| run: | | |
| $here_dirpath = (Get-Location) | |
| Write-Output "here: '$here_dirpath'" | |
| $packagedArtifactsPath = Join-Path -Path $here_dirpath -ChildPath "packaged_artifacts" | |
| New-Item -ItemType Directory -Force -Path $packagedArtifactsPath | |
| # Check if artifacts directory exists | |
| $artifactsDir = "published_workflow_builds" | |
| if (-not (Test-Path $artifactsDir)) { | |
| Write-Host "No artifacts directory found - creating empty directory" -ForegroundColor Yellow | |
| New-Item -ItemType Directory -Force -Path $artifactsDir | |
| } | |
| # Check if there are any artifacts | |
| $artifactCount = (Get-ChildItem -Path $artifactsDir -Filter "publish_*" -ErrorAction SilentlyContinue | Measure-Object).Count | |
| if ($artifactCount -eq 0) { | |
| Write-Host "No build artifacts found - this may indicate all build jobs failed" -ForegroundColor Yellow | |
| Write-Host "Creating empty artifact-names.txt to allow workflow to continue" -ForegroundColor Yellow | |
| @() | Out-File -FilePath "artifact-names.txt" -Encoding UTF8 | |
| exit 0 | |
| } | |
| # Navigate to the directory with downloaded artifacts | |
| Set-Location -Path $artifactsDir | |
| $artifactNames = New-Object 'System.Collections.Generic.HashSet[string]' | |
| # Iterate through each tool_os_pythonversion_arch folder. | |
| Get-ChildItem -Filter "publish_*" | ForEach-Object { | |
| $matrixPackagePath = $_.FullName | |
| Write-Output "Found matrix package '$matrixPackagePath'" | |
| $toolName, $os, $pythonVersion, $architecture = $_.BaseName -replace '^publish_', '' -split '_',4 | |
| $shortOsName = $os | |
| if ($os -eq "ubuntu-latest") { | |
| $shortOsName = "Linux" | |
| } elseif ($os -eq "macos-latest") { | |
| $shortOsName = "Mac" | |
| } elseif ($os -eq "windows-latest") { | |
| $shortOsName = "Win" | |
| } | |
| # Iterate through each tool in the folder | |
| Get-ChildItem -LiteralPath $matrixPackagePath | ForEach-Object { | |
| $toolExePath = $_.FullName | |
| $toolFileName = $_.Name | |
| $fileBaseName = [IO.Path]::GetFileNameWithoutExtension($_.Name) | |
| $fileExtension = $_.Extension | |
| $outerArchiveName = "${toolName}_$fileBaseName`_$pythonVersion" | |
| $outerZipPath = Join-Path -Path $packagedArtifactsPath -ChildPath $outerArchiveName | |
| $osSpecificArchiveName = "${toolName}_$fileBaseName`_$shortOsName-$architecture" | |
| $osSpecificArchivePath = Join-Path -Path $outerZipPath -ChildPath $osSpecificArchiveName | |
| Write-Output " ToolFileName: '$toolFileName'" | |
| Write-Output " Tool original filepath: '$toolExePath'" | |
| Write-Output "outerArchiveName: '$outerArchiveName'" | |
| Write-Output " - osSpecificArchiveName: '$osSpecificArchiveName'" | |
| Write-Output "outerZipPath: '$outerZipPath'" | |
| Write-Output " - osSpecificArchivePath: '$osSpecificArchivePath'" | |
| New-Item -ItemType Directory -Force -Path $outerZipPath | |
| chmod 777 -R $toolExePath | |
| $toolExeParentDirPath = Split-Path -Parent $toolExePath | |
| Write-Output "Creating archive for '$toolFileName' at '$toolExeParentDirPath'..." | |
| Write-Output ("Push-Location to start in '$toolExeParentDirPath' (originally at '$(Get-Location)')") | |
| Push-Location -Path $toolExeParentDirPath | |
| # Archive the tool by os identifier. | |
| if ((-not $fileExtension) -or (-not $fileExtension.Trim()) -or ($fileExtension.ToLower().Trim() -eq ".app")) { | |
| $osSpecificArchivePath = "$osSpecificArchivePath.tar.gz" | |
| if (Test-Path $toolExePath -PathType Container -ErrorAction SilentlyContinue) { # It's a directory | |
| tar -czf "$osSpecificArchivePath" -C "$toolExePath" . | |
| } else { # It's a file | |
| tar -czf "$osSpecificArchivePath" -C "$toolExeParentDirPath" $toolFileName | |
| } | |
| } else { | |
| $osSpecificArchivePath = "$osSpecificArchivePath.zip" | |
| zip -r -9 "$osSpecificArchivePath" $toolFileName | |
| } | |
| Pop-Location | |
| Write-Output ("Pop-Location to return to $(Get-Location) (was pushed to '$toolExeParentDirPath')") | |
| Write-Output "Compressed archive saved to '$osSpecificArchivePath'" | |
| chmod 777 -R "$osSpecificArchivePath" | |
| $artifactNames.Add($outerArchiveName) | |
| } | |
| } | |
| # Save artifact names to a file | |
| $artifactNames | Out-File -FilePath "../artifact-names.txt" -Encoding UTF8 | |
| # use for debug | |
| # Get-ChildItem -Force | ForEach-Object { | |
| # $size = if ($_.PSIsContainer) { "N/A" } else { $_.Length } | |
| # $attrs = $_.Attributes.ToString() -replace 'ReadOnly', 'RO' -replace 'Hidden', 'H' -replace 'System', 'S' -replace 'Archive', 'A' -replace 'Directory', 'D' -replace ', ', '|' | |
| # [PSCustomObject]@{ | |
| # Name = $_.Name | |
| # Size = $size | |
| # Attributes = $attrs | |
| # } | |
| # } | Format-Table -AutoSize | |
| # Navigate back to the root of the workspace | |
| Set-Location -Path "../" | |
| - name: Generate matrix for uploading | |
| if: ${{ success() || failure() }} | |
| id: set-matrix | |
| shell: pwsh | |
| run: | | |
| $artifactNames = Get-Content 'artifact-names.txt' -ReadCount 0 | |
| # Initialize an array to hold the artifact names directly | |
| $matrixArray = @() | |
| foreach ($name in $artifactNames) { | |
| if (-not [string]::IsNullOrEmpty($name)) { | |
| Write-Host "Processing artifact name: $name" | |
| # Trim the name and add directly to the array | |
| $matrixArray += $name.trim() | |
| } | |
| } | |
| # Convert the array directly to JSON | |
| $jsonMatrix = $matrixArray | ConvertTo-Json -Depth 5 -Compress | |
| # Use a single line of JSON for the matrix to avoid issues | |
| $singleLineJsonMatrix = $jsonMatrix -replace "`r", "" | |
| $singleLineJsonMatrix = $singleLineJsonMatrix -replace "`n", "" | |
| Write-Host "Matrix JSON:" | |
| Write-Host $singleLineJsonMatrix | |
| echo "matrixJson<<EOF" >> $Env:GITHUB_OUTPUT | |
| echo $singleLineJsonMatrix >> $Env:GITHUB_OUTPUT | |
| echo "EOF" >> $Env:GITHUB_OUTPUT | |
| - name: Upload all repackages for next job | |
| if: ${{ success() || failure() }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: all_tool_dists_onearchive | |
| path: packaged_artifacts/** | |
| retention-days: 1 # only used for the next job. | |
| compression-level: 0 | |
| upload: | |
| needs: package | |
| if: ${{ success() || failure() }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| artifactb: ${{ fromJson(needs.package.outputs.filesToArchive) }} | |
| steps: | |
| - name: Download repackages from package job | |
| if: ${{ success() || failure() }} | |
| uses: actions/download-artifact@v8 | |
| with: | |
| path: all_tool_dists_onearchive | |
| pattern: all_tool_dists* | |
| - name: Upload re-packaged artifact | |
| if: ${{ success() || failure() }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ${{ matrix.artifactb }} | |
| path: all_tool_dists_onearchive/all_tool_dists_onearchive/${{ matrix.artifactb }} | |
| compression-level: 9 |