Merge pull request #177 from aminya/arch-python

This commit is contained in:
Amin Yahyaabadi 2023-06-29 14:51:29 -07:00 committed by GitHub
commit ec7b72626c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 1550 additions and 1469 deletions

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -68,14 +68,14 @@
"@actions/exec": "^1.1.1",
"@actions/io": "^1.1.3",
"@actions/tool-cache": "^2.0.1",
"@babel/cli": "^7.21.5",
"@babel/cli": "^7.22.5",
"@types/cross-spawn": "^6.0.2",
"@types/eslint": "^8.40.0",
"@types/jest": "^29.5.1",
"@types/eslint": "^8.40.2",
"@types/jest": "^29.5.2",
"@types/mri": "^1.1.1",
"@types/node": "^20.2.4",
"@types/node": "^20.3.2",
"@types/npmcli__ci-detect": "^2.0.0",
"@types/prettier": "2.7.2",
"@types/prettier": "2.7.3",
"@types/semver": "^7.5.0",
"@types/which": "^3.0.0",
"@upleveled/babel-plugin-remove-node-prefix": "github:aminya/babel-plugin-remove-node-prefix#95fcbd92405b99a6eece48c493548996f12e6519",
@ -89,8 +89,8 @@
"escape-path-with-spaces": "^1.0.2",
"escape-quotes": "^1.0.2",
"escape-string-regexp": "^5.0.0",
"eslint": "^8.41.0",
"eslint-config-atomic": "^1.19.1",
"eslint": "^8.43.0",
"eslint-config-atomic": "^1.19.3",
"exec-powershell": "workspace:*",
"execa": "^7.1.1",
"fast-glob": "^3.2.12",
@ -98,12 +98,13 @@
"gen-readme": "^1.6.0",
"is-url-online": "^1.5.0",
"jest": "^29.5.0",
"micro-memoize": "^4.1.2",
"mri": "^1.2.0",
"msvc-dev-cmd": "github:aminya/msvc-dev-cmd#9f672c1",
"npm-check-updates": "^16.10.12",
"npm-check-updates": "^16.10.13",
"npm-run-all2": "^6.0.5",
"numerous": "1.0.3",
"parcel": "2.9.0",
"parcel": "2.9.3",
"path-exists": "^5.0.0",
"patha": "^0.4.1",
"prettier": "2.8.8",
@ -111,15 +112,15 @@
"quote-unquote": "^1.0.0",
"readme-md-generator": "^1.0.0",
"retry-as-promised": "^7.0.4",
"semver": "7.5.1",
"semver": "7.5.3",
"setup-python": "github:actions/setup-python#v4.6.1",
"shx": "0.3.4",
"simple-update-notifier": "^1.1.0",
"simple-update-notifier": "^2.0.0",
"time-delta": "github:aminya/time-delta#69d91a41cef28e569be9a2991129f5f7d1f0d00e",
"ts-jest": "^29.1.0",
"ts-node": "^10.9.1",
"ts-readme": "^1.1.3",
"typescript": "^5.0.4",
"typescript": "^5.1.5",
"ubuntu-version": "^2.0.0",
"untildify-user": "workspace:*",
"user-access": "workspace:*",

File diff suppressed because it is too large Load Diff

View File

@ -16,6 +16,7 @@ import { setupDnfPack } from "../utils/setup/setupDnfPack"
import { isUbuntu } from "../utils/env/isUbuntu"
import { pathExists } from "path-exists"
import retry from "retry-as-promised"
import { ubuntuVersion } from "../utils/env/ubuntu_version"
/** Get the platform data for cmake */
// eslint-disable-next-line @typescript-eslint/no-unused-vars
@ -90,7 +91,7 @@ export async function setupDoxygen(version: string, setupDir: string, arch: stri
} else {
throw new Error(`Unsupported linux distributions`)
}
await setupGraphviz(getVersion("graphviz", undefined), "", arch)
await setupGraphviz(getVersion("graphviz", undefined, await ubuntuVersion()), "", arch)
return installationInfo
}
default: {

View File

@ -2,6 +2,7 @@ import { setupGraphviz } from "../graphviz"
import { cleanupTmpDir, setupTmpDir, testBin } from "../../utils/tests/test-helpers"
import { InstallationInfo } from "../../utils/setup/setupBin"
import { getVersion } from "../../versions/versions"
import { ubuntuVersion } from "../../utils/env/ubuntu_version"
jest.setTimeout(300000)
describe("setup-graphviz", () => {
@ -11,7 +12,11 @@ describe("setup-graphviz", () => {
})
it("should setup graphviz", async () => {
const installInfo = await setupGraphviz(getVersion("graphviz", undefined), directory, process.arch)
const installInfo = await setupGraphviz(
getVersion("graphviz", undefined, await ubuntuVersion()),
directory,
process.arch
)
await testBin("dot", ["-V"], (installInfo as InstallationInfo | undefined)?.binDir)
})

View File

@ -15,6 +15,7 @@ import { addVPrefix, removeVPrefix } from "../utils/setup/version"
import { info } from "ci-log"
import { untildifyUser } from "untildify-user"
import { setupNinja } from "../ninja/ninja"
import { ubuntuVersion } from "../utils/env/ubuntu_version"
function getDownloadKcovPackageInfo(version: string): PackageInfo {
return {
@ -79,12 +80,16 @@ async function buildKcov(file: string, dest: string) {
async function getCmake() {
let cmake = which.sync("cmake", { nothrow: true })
if (cmake === null) {
const { binDir } = await setupCmake(getVersion("cmake", undefined), join(untildifyUser(""), "cmake"), "")
const { binDir } = await setupCmake(
getVersion("cmake", undefined, await ubuntuVersion()),
join(untildifyUser(""), "cmake"),
""
)
cmake = join(binDir, "cmake")
}
const ninja = which.sync("ninja", { nothrow: true })
if (ninja === null) {
await setupNinja(getVersion("ninja", undefined), join(untildifyUser(""), "ninja"), "")
await setupNinja(getVersion("ninja", undefined, await ubuntuVersion()), join(untildifyUser(""), "ninja"), "")
}
return cmake
}

View File

@ -46,8 +46,7 @@ async function setupLLVMWithoutActivation(version: string, setupDir: string, arc
async function setupLLVMDeps(arch: string, version: string) {
if (process.platform === "linux") {
// install llvm build dependencies
const osVersion = await ubuntuVersion()
await setupGcc(getVersion("gcc", undefined, osVersion), "", arch) // using llvm requires ld, an up to date libstdc++, etc. So, install gcc first
await setupGcc(getVersion("gcc", undefined, await ubuntuVersion()), "", arch) // using llvm requires ld, an up to date libstdc++, etc. So, install gcc first
if (isUbuntu()) {
const majorVersion = parseInt(version.split(".")[0], 10)

View File

@ -24,6 +24,7 @@ describe("setup-python", () => {
it("should setup python via system", async () => {
process.env.CI = "false"
process.env.GITHUB_ACTIONS = "false"
const installInfo = await setupPython(getVersion("python", "true", await ubuntuVersion()), directory, process.arch)

View File

@ -1,45 +1,87 @@
/* eslint-disable require-atomic-updates */
import { getExecOutput } from "@actions/exec"
import assert from "assert"
import { GITHUB_ACTIONS } from "ci-info"
import { info, warning } from "ci-log"
import { execaSync } from "execa"
import memoize from "micro-memoize"
import { addExeExt, dirname, join } from "patha"
import which from "which"
import { addPath } from "../utils/env/addEnv"
import { hasDnf } from "../utils/env/hasDnf"
import { isArch } from "../utils/env/isArch"
import { isUbuntu } from "../utils/env/isUbuntu"
import { setupAptPack } from "../utils/setup/setupAptPack"
import { setupPacmanPack } from "../utils/setup/setupPacmanPack"
import { InstallationInfo } from "../utils/setup/setupBin"
import { setupBrewPack } from "../utils/setup/setupBrewPack"
import { setupChocoPack } from "../utils/setup/setupChocoPack"
import { GITHUB_ACTIONS } from "ci-info"
import { warning, info } from "ci-log"
import { isArch } from "../utils/env/isArch"
import which from "which"
import { InstallationInfo } from "../utils/setup/setupBin"
import { dirname, join } from "patha"
import { hasDnf } from "../utils/env/hasDnf"
import { setupDnfPack } from "../utils/setup/setupDnfPack"
import { isUbuntu } from "../utils/env/isUbuntu"
import { getExecOutput } from "@actions/exec"
import { setupPacmanPack } from "../utils/setup/setupPacmanPack"
import { isBinUptoDate } from "../utils/setup/version"
import { getVersion } from "../versions/versions"
import assert from "assert"
import { execaSync } from "execa"
import { unique } from "../utils/std"
import { MinVersions } from "../versions/default_versions"
import { pathExists } from "path-exists"
export async function setupPython(version: string, setupDir: string, arch: string) {
if (!GITHUB_ACTIONS) {
// TODO parse version
return setupPythonViaSystem(version, setupDir, arch)
export async function setupPython(version: string, setupDir: string, arch: string): Promise<InstallationInfo> {
const installInfo = await findOrSetupPython(version, setupDir, arch)
assert(installInfo.bin !== undefined)
const foundPython = installInfo.bin
// setup pip
const foundPip = await findOrSetupPip(foundPython)
if (foundPip === undefined) {
throw new Error("pip was not installed correctly")
}
// setup wheel
try {
info("Installing python in GitHub Actions")
const { setupActionsPython } = await import("./actions_python")
return setupActionsPython(version, setupDir, arch)
setupWheel(foundPython)
} catch (err) {
warning((err as Error).toString())
return setupPythonViaSystem(version, setupDir, arch)
warning(`Failed to install wheels: ${(err as Error).toString()}. Ignoring...`)
}
return installInfo
}
export async function setupPythonViaSystem(
version: string,
setupDir: string,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
_arch: string
): Promise<InstallationInfo> {
async function findOrSetupPython(version: string, setupDir: string, arch: string) {
let installInfo: InstallationInfo | undefined
let foundPython = await findPython(setupDir)
if (foundPython !== undefined) {
const binDir = dirname(foundPython)
installInfo = { bin: foundPython, installDir: binDir, binDir }
} else {
// if python is not found, try to install it
if (GITHUB_ACTIONS) {
// install python in GitHub Actions
try {
info("Installing python in GitHub Actions")
const { setupActionsPython } = await import("./actions_python")
await setupActionsPython(version, setupDir, arch)
foundPython = (await findPython(setupDir))!
const binDir = dirname(foundPython)
installInfo = { bin: foundPython, installDir: binDir, binDir }
} catch (err) {
warning((err as Error).toString())
}
}
if (installInfo === undefined) {
// install python via system package manager
installInfo = await setupPythonSystem(setupDir, version)
}
}
if (foundPython === undefined || installInfo.bin === undefined) {
foundPython = (await findPython(setupDir))!
installInfo.bin = foundPython
}
return installInfo
}
async function setupPythonSystem(setupDir: string, version: string) {
let installInfo: InstallationInfo | undefined
switch (process.platform) {
case "win32": {
if (setupDir) {
@ -48,86 +90,157 @@ export async function setupPythonViaSystem(
await setupChocoPack("python3", version)
}
// Adding the bin dir to the path
const pythonBinPath =
which.sync("python3.exe", { nothrow: true }) ??
which.sync("python.exe", { nothrow: true }) ??
join(setupDir, "python.exe")
const pythonSetupDir = dirname(pythonBinPath)
const bin = (await findPython(setupDir))!
const binDir = dirname(bin)
/** The directory which the tool is installed to */
await addPath(pythonSetupDir)
return { installDir: pythonSetupDir, binDir: pythonSetupDir }
await addPath(binDir)
installInfo = { installDir: binDir, binDir, bin }
break
}
case "darwin": {
return setupBrewPack("python3", version)
installInfo = await setupBrewPack("python3", version)
break
}
case "linux": {
let installInfo: InstallationInfo
if (isArch()) {
installInfo = await setupPacmanPack("python", version)
await setupPacmanPack("python-pip")
} else if (hasDnf()) {
installInfo = setupDnfPack("python3", version)
setupDnfPack("python3-pip")
} else if (isUbuntu()) {
installInfo = await setupAptPack([{ name: "python3", version }, { name: "python3-pip" }])
installInfo = await setupAptPack([{ name: "python3", version }])
} else {
throw new Error("Unsupported linux distributions")
}
return installInfo
break
}
default: {
throw new Error("Unsupported platform")
}
}
return installInfo
}
let setupPythonAndPipTried = false
/// setup python and pip if needed
export async function setupPythonAndPip(): Promise<string> {
let foundPython: string
// install python
if (which.sync("python3", { nothrow: true }) !== null) {
foundPython = "python3"
} else if (which.sync("python", { nothrow: true }) !== null && (await isBinUptoDate("python", "3.0.0"))) {
foundPython = "python"
} else {
info("python3 was not found. Installing python")
await setupPython(getVersion("python", undefined), "", process.arch)
// try again
if (setupPythonAndPipTried) {
throw new Error("Failed to install python")
async function findPython(binDir?: string) {
for (const pythonBin of ["python3", "python"]) {
// eslint-disable-next-line no-await-in-loop
const foundPython = await isPythonUpToDate(pythonBin, binDir)
if (foundPython !== undefined) {
return foundPython
}
setupPythonAndPipTried = true
return setupPythonAndPip() // recurse
}
return undefined
}
async function isPythonUpToDate(candidate: string, binDir?: string) {
try {
if (binDir !== undefined) {
const pythonBinPath = join(binDir, addExeExt(candidate))
if (await pathExists(pythonBinPath)) {
if (await isBinUptoDate(pythonBinPath, MinVersions.python!)) {
return pythonBinPath
}
}
}
const pythonBinPaths = (await which(candidate, { nothrow: true, all: true })) ?? []
for (const pythonBinPath of pythonBinPaths) {
// eslint-disable-next-line no-await-in-loop
if (await isBinUptoDate(pythonBinPath, MinVersions.python!)) {
return pythonBinPath
}
}
} catch {
// fall through
}
return undefined
}
async function findOrSetupPip(foundPython: string) {
const maybePip = await findPip()
if (maybePip === undefined) {
// install pip if not installed
info("pip was not found. Installing pip")
await setupPip(foundPython)
return findPip() // recurse to check if pip is on PATH and up-to-date
}
assert(typeof foundPython === "string")
return maybePip
}
// install pip
if (process.platform === "win32") {
// downgrade pip on Windows
// https://github.com/pypa/pip/issues/10875#issuecomment-1030293005
execaSync(foundPython, ["-m", "pip", "install", "-U", "pip==21.3.1"], { stdio: "inherit" })
} else if (process.platform === "linux") {
async function findPip() {
for (const pipCandidate of ["pip3", "pip"]) {
// eslint-disable-next-line no-await-in-loop
const maybePip = await isPipUptoDate(pipCandidate)
if (maybePip !== undefined) {
return maybePip
}
}
return undefined
}
async function isPipUptoDate(pip: string) {
try {
const pipPaths = (await which(pip, { nothrow: true, all: true })) ?? []
for (const pipPath of pipPaths) {
// eslint-disable-next-line no-await-in-loop
if (pipPath !== null && (await isBinUptoDate(pipPath, MinVersions.pip!))) {
return pipPath
}
}
} catch {
// fall through
}
return undefined
}
async function setupPip(foundPython: string) {
const upgraded = ensurePipUpgrade(foundPython)
if (!upgraded) {
await setupPipSystem()
// upgrade pip
ensurePipUpgrade(foundPython)
}
}
function ensurePipUpgrade(foundPython: string) {
try {
execaSync(foundPython, ["-m", "ensurepip", "-U", "--upgrade"], { stdio: "inherit" })
return true
} catch (err1) {
info((err1 as Error)?.toString?.())
try {
// ensure pip is disabled on Ubuntu
execaSync(foundPython, ["-m", "pip", "install", "--upgrade", "pip"], { stdio: "inherit" })
return true
} catch (err2) {
info((err2 as Error)?.toString?.())
// pip module not found
}
}
// all methods failed
return false
}
function setupPipSystem() {
if (process.platform === "linux") {
// ensure that pip is installed on Linux (happens when python is found but pip not installed)
if (isArch()) {
await setupPacmanPack("python-pip")
return setupPacmanPack("python-pip")
} else if (hasDnf()) {
setupDnfPack("python3-pip")
return setupDnfPack("python3-pip")
} else if (isUbuntu()) {
await setupAptPack([{ name: "python3-pip" }])
return setupAptPack([{ name: "python3-pip" }])
}
}
// install wheel (required for Conan, Meson, etc.)
execaSync(foundPython, ["-m", "pip", "install", "-U", "wheel"], { stdio: "inherit" })
return foundPython
throw new Error(`Could not install pip on ${process.platform}`)
}
export async function addPythonBaseExecPrefix(python: string) {
/** Install wheel (required for Conan, Meson, etc.) */
function setupWheel(foundPython: string) {
execaSync(foundPython, ["-m", "pip", "install", "-U", "wheel"], { stdio: "inherit" })
}
async function addPythonBaseExecPrefix_raw(python: string) {
const dirs: string[] = []
// detection based on the platform
@ -145,3 +258,10 @@ export async function addPythonBaseExecPrefix(python: string) {
// remove duplicates
return unique(dirs)
}
/**
* Add the base exec prefix to the PATH. This is required for Conan, Meson, etc. to work properly.
*
* The answer is cached for subsequent calls
*/
export const addPythonBaseExecPrefix = memoize(addPythonBaseExecPrefix_raw)

View File

@ -3,18 +3,24 @@ import { getUbuntuVersion } from "ubuntu-version"
import which from "which"
import { setupAptPack } from "../setup/setupAptPack"
import { isUbuntu } from "./isUbuntu"
import os from "os"
import memoize from "micro-memoize"
export async function ubuntuVersion(): Promise<number[] | null> {
async function ubuntuVersion_raw(): Promise<number[] | null> {
try {
if (isUbuntu()) {
if (which.sync("lsb_release", { nothrow: true }) === null) {
await setupAptPack([{ name: "lsb-release" }])
try {
if (which.sync("lsb_release", { nothrow: true }) === null) {
await setupAptPack([{ name: "lsb-release" }])
}
} catch {
return detectUsingOsVersion()
}
const versionSplitted = await getUbuntuVersion()
if (versionSplitted.length === 0) {
warning("Failed to get the ubuntu major version.")
return null
return detectUsingOsVersion()
}
return versionSplitted
@ -26,3 +32,18 @@ export async function ubuntuVersion(): Promise<number[] | null> {
return null
}
}
/** Detect Ubuntu version */
export const ubuntuVersion = memoize(ubuntuVersion_raw)
/** Detect Ubuntu version using os.version() for Ubuntu based distros */
function detectUsingOsVersion() {
// #46~22.04.1-Ubuntu SMP ...
const osVersion = os.version()
const versionSplitted = osVersion.split(".")
const majorVersion = parseInt(versionSplitted[0].replace("#", ""), 10)
const minorVersion = parseInt(versionSplitted[1].replace("~", ""), 10)
const patchVersion = parseInt(versionSplitted[2].split("-")[0], 10)
return [majorVersion, minorVersion, patchVersion]
}

View File

@ -123,26 +123,31 @@ function initGpg() {
}
export async function addAptKeyViaServer(keys: string[], name: string, server = "keyserver.ubuntu.com") {
const fileName = `/etc/apt/trusted.gpg.d/${name}`
if (!(await pathExists(fileName))) {
initGpg()
try {
const fileName = `/etc/apt/trusted.gpg.d/${name}`
if (!(await pathExists(fileName))) {
initGpg()
await Promise.all(
keys.map(async (key) => {
await execRoot("gpg", [
"--no-default-keyring",
"--keyring",
`gnupg-ring:${fileName}`,
"--keyserver",
server,
"--recv-keys",
key,
])
await execRoot("chmod", ["644", fileName])
})
)
await Promise.all(
keys.map(async (key) => {
await execRoot("gpg", [
"--no-default-keyring",
"--keyring",
`gnupg-ring:${fileName}`,
"--keyserver",
server,
"--recv-keys",
key,
])
await execRoot("chmod", ["644", fileName])
})
)
}
return fileName
} catch (err) {
warning(`Failed to add apt key via server ${server}: ${err}`)
return undefined
}
return fileName
}
export async function addAptKeyViaDownload(name: string, url: string) {

View File

@ -34,6 +34,7 @@ export type InstallationInfo = {
/** The top install dir */
installDir?: string
binDir: string
bin?: string
}
let didInit: boolean = false

View File

@ -3,31 +3,29 @@ import { execaSync } from "execa"
import { pathExists } from "path-exists"
import { addExeExt, dirname, join } from "patha"
import which from "which"
import { addPythonBaseExecPrefix, setupPythonAndPip } from "../../python/python"
import { addPythonBaseExecPrefix, setupPython } from "../../python/python"
import { addPath } from "../env/addEnv"
import { InstallationInfo } from "./setupBin"
import { getVersion } from "../../versions/versions"
import { ubuntuVersion } from "../env/ubuntu_version"
/* eslint-disable require-atomic-updates */
let python: string | undefined
let binDirs: string[] | undefined
/** A function that installs a package using pip */
export async function setupPipPack(name: string, version?: string): Promise<InstallationInfo> {
info(`Installing ${name} ${version ?? ""} via pip`)
if (python === undefined) {
python = await setupPythonAndPip()
python = (await setupPython(getVersion("python", undefined, await ubuntuVersion()), "", process.arch)).bin!
}
execaSync(python, ["-m", "pip", "install", version !== undefined && version !== "" ? `${name}==${version}` : name], {
stdio: "inherit",
})
if (binDirs === undefined) {
binDirs = await addPythonBaseExecPrefix(python)
}
const binDir = await findBinDir(binDirs, name)
const execPaths = await addPythonBaseExecPrefix(python)
const binDir = await findBinDir(execPaths, name)
await addPath(binDir)

View File

@ -30,6 +30,11 @@ export const DefaultVersions: Record<string, string | undefined> = {
gcc: isArch() ? "13.1.1-1" : "13", // https://github.com/brechtsanders/winlibs_mingw/releases and // https://packages.ubuntu.com/search?suite=all&arch=any&searchon=names&keywords=gcc
}
export const MinVersions: Record<string, string | undefined> = {
pip: "22.3.1",
python: "3.7.9",
}
/// If an ubuntu versions is not in this map:
// - the newer ubuntu versions use the first entry (e.g. v20),
// - the older ones use ""