Merge pull request #177 from aminya/arch-python

package.json

@@ -68,14 +68,14 @@
     "@actions/exec": "^1.1.1",
     "@actions/io": "^1.1.3",
     "@actions/tool-cache": "^2.0.1",
-    "@babel/cli": "^7.21.5",
+    "@babel/cli": "^7.22.5",
     "@types/cross-spawn": "^6.0.2",
-    "@types/eslint": "^8.40.0",
+    "@types/eslint": "^8.40.2",
-    "@types/jest": "^29.5.1",
+    "@types/jest": "^29.5.2",
     "@types/mri": "^1.1.1",
-    "@types/node": "^20.2.4",
+    "@types/node": "^20.3.2",
     "@types/npmcli__ci-detect": "^2.0.0",
-    "@types/prettier": "2.7.2",
+    "@types/prettier": "2.7.3",
     "@types/semver": "^7.5.0",
     "@types/which": "^3.0.0",
     "@upleveled/babel-plugin-remove-node-prefix": "github:aminya/babel-plugin-remove-node-prefix#95fcbd92405b99a6eece48c493548996f12e6519",
@@ -89,8 +89,8 @@
     "escape-path-with-spaces": "^1.0.2",
     "escape-quotes": "^1.0.2",
     "escape-string-regexp": "^5.0.0",
-    "eslint": "^8.41.0",
+    "eslint": "^8.43.0",
-    "eslint-config-atomic": "^1.19.1",
+    "eslint-config-atomic": "^1.19.3",
     "exec-powershell": "workspace:*",
     "execa": "^7.1.1",
     "fast-glob": "^3.2.12",
@@ -98,12 +98,13 @@
     "gen-readme": "^1.6.0",
     "is-url-online": "^1.5.0",
     "jest": "^29.5.0",
+    "micro-memoize": "^4.1.2",
     "mri": "^1.2.0",
     "msvc-dev-cmd": "github:aminya/msvc-dev-cmd#9f672c1",
-    "npm-check-updates": "^16.10.12",
+    "npm-check-updates": "^16.10.13",
     "npm-run-all2": "^6.0.5",
     "numerous": "1.0.3",
-    "parcel": "2.9.0",
+    "parcel": "2.9.3",
     "path-exists": "^5.0.0",
     "patha": "^0.4.1",
     "prettier": "2.8.8",
@@ -111,15 +112,15 @@
     "quote-unquote": "^1.0.0",
     "readme-md-generator": "^1.0.0",
     "retry-as-promised": "^7.0.4",
-    "semver": "7.5.1",
+    "semver": "7.5.3",
     "setup-python": "github:actions/setup-python#v4.6.1",
     "shx": "0.3.4",
-    "simple-update-notifier": "^1.1.0",
+    "simple-update-notifier": "^2.0.0",
     "time-delta": "github:aminya/time-delta#69d91a41cef28e569be9a2991129f5f7d1f0d00e",
     "ts-jest": "^29.1.0",
     "ts-node": "^10.9.1",
     "ts-readme": "^1.1.3",
-    "typescript": "^5.0.4",
+    "typescript": "^5.1.5",
     "ubuntu-version": "^2.0.0",
     "untildify-user": "workspace:*",
     "user-access": "workspace:*",

src/doxygen/doxygen.ts

@@ -16,6 +16,7 @@ import { setupDnfPack } from "../utils/setup/setupDnfPack"
 import { isUbuntu } from "../utils/env/isUbuntu"
 import { pathExists } from "path-exists"
 import retry from "retry-as-promised"
+import { ubuntuVersion } from "../utils/env/ubuntu_version"
 
 /** Get the platform data for cmake */
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -90,7 +91,7 @@ export async function setupDoxygen(version: string, setupDir: string, arch: stri
       } else {
         throw new Error(`Unsupported linux distributions`)
       }
-      await setupGraphviz(getVersion("graphviz", undefined), "", arch)
+      await setupGraphviz(getVersion("graphviz", undefined, await ubuntuVersion()), "", arch)
       return installationInfo
     }
     default: {

src/graphviz/__tests__/graphviz.test.ts

@@ -2,6 +2,7 @@ import { setupGraphviz } from "../graphviz"
 import { cleanupTmpDir, setupTmpDir, testBin } from "../../utils/tests/test-helpers"
 import { InstallationInfo } from "../../utils/setup/setupBin"
 import { getVersion } from "../../versions/versions"
+import { ubuntuVersion } from "../../utils/env/ubuntu_version"
 
 jest.setTimeout(300000)
 describe("setup-graphviz", () => {
@@ -11,7 +12,11 @@ describe("setup-graphviz", () => {
   })
 
   it("should setup graphviz", async () => {
-    const installInfo = await setupGraphviz(getVersion("graphviz", undefined), directory, process.arch)
+    const installInfo = await setupGraphviz(
+      getVersion("graphviz", undefined, await ubuntuVersion()),
+      directory,
+      process.arch
+    )
 
     await testBin("dot", ["-V"], (installInfo as InstallationInfo | undefined)?.binDir)
   })

src/kcov/kcov.ts

@@ -15,6 +15,7 @@ import { addVPrefix, removeVPrefix } from "../utils/setup/version"
 import { info } from "ci-log"
 import { untildifyUser } from "untildify-user"
 import { setupNinja } from "../ninja/ninja"
+import { ubuntuVersion } from "../utils/env/ubuntu_version"
 
 function getDownloadKcovPackageInfo(version: string): PackageInfo {
   return {
@@ -79,12 +80,16 @@ async function buildKcov(file: string, dest: string) {
 async function getCmake() {
   let cmake = which.sync("cmake", { nothrow: true })
   if (cmake === null) {
-    const { binDir } = await setupCmake(getVersion("cmake", undefined), join(untildifyUser(""), "cmake"), "")
+    const { binDir } = await setupCmake(
+      getVersion("cmake", undefined, await ubuntuVersion()),
+      join(untildifyUser(""), "cmake"),
+      ""
+    )
     cmake = join(binDir, "cmake")
   }
   const ninja = which.sync("ninja", { nothrow: true })
   if (ninja === null) {
-    await setupNinja(getVersion("ninja", undefined), join(untildifyUser(""), "ninja"), "")
+    await setupNinja(getVersion("ninja", undefined, await ubuntuVersion()), join(untildifyUser(""), "ninja"), "")
   }
   return cmake
 }

src/llvm/llvm.ts

@@ -46,8 +46,7 @@ async function setupLLVMWithoutActivation(version: string, setupDir: string, arc
 async function setupLLVMDeps(arch: string, version: string) {
   if (process.platform === "linux") {
     // install llvm build dependencies
-    const osVersion = await ubuntuVersion()
+    await setupGcc(getVersion("gcc", undefined, await ubuntuVersion()), "", arch) // using llvm requires ld, an up to date libstdc++, etc. So, install gcc first
-    await setupGcc(getVersion("gcc", undefined, osVersion), "", arch) // using llvm requires ld, an up to date libstdc++, etc. So, install gcc first
 
     if (isUbuntu()) {
       const majorVersion = parseInt(version.split(".")[0], 10)

src/python/__tests__/python.test.ts

@@ -24,6 +24,7 @@ describe("setup-python", () => {
 
   it("should setup python via system", async () => {
     process.env.CI = "false"
+    process.env.GITHUB_ACTIONS = "false"
 
     const installInfo = await setupPython(getVersion("python", "true", await ubuntuVersion()), directory, process.arch)
 

src/python/python.ts

@@ -1,45 +1,87 @@
+/* eslint-disable require-atomic-updates */
+import { getExecOutput } from "@actions/exec"
+import assert from "assert"
+import { GITHUB_ACTIONS } from "ci-info"
+import { info, warning } from "ci-log"
+import { execaSync } from "execa"
+import memoize from "micro-memoize"
+import { addExeExt, dirname, join } from "patha"
+import which from "which"
 import { addPath } from "../utils/env/addEnv"
+import { hasDnf } from "../utils/env/hasDnf"
+import { isArch } from "../utils/env/isArch"
+import { isUbuntu } from "../utils/env/isUbuntu"
 import { setupAptPack } from "../utils/setup/setupAptPack"
-import { setupPacmanPack } from "../utils/setup/setupPacmanPack"
+import { InstallationInfo } from "../utils/setup/setupBin"
 import { setupBrewPack } from "../utils/setup/setupBrewPack"
 import { setupChocoPack } from "../utils/setup/setupChocoPack"
-import { GITHUB_ACTIONS } from "ci-info"
-import { warning, info } from "ci-log"
-import { isArch } from "../utils/env/isArch"
-import which from "which"
-import { InstallationInfo } from "../utils/setup/setupBin"
-import { dirname, join } from "patha"
-import { hasDnf } from "../utils/env/hasDnf"
 import { setupDnfPack } from "../utils/setup/setupDnfPack"
-import { isUbuntu } from "../utils/env/isUbuntu"
+import { setupPacmanPack } from "../utils/setup/setupPacmanPack"
-import { getExecOutput } from "@actions/exec"
 import { isBinUptoDate } from "../utils/setup/version"
-import { getVersion } from "../versions/versions"
-import assert from "assert"
-import { execaSync } from "execa"
 import { unique } from "../utils/std"
+import { MinVersions } from "../versions/default_versions"
+import { pathExists } from "path-exists"
 
-export async function setupPython(version: string, setupDir: string, arch: string) {
+export async function setupPython(version: string, setupDir: string, arch: string): Promise<InstallationInfo> {
-  if (!GITHUB_ACTIONS) {
+  const installInfo = await findOrSetupPython(version, setupDir, arch)
-    // TODO parse version
+  assert(installInfo.bin !== undefined)
-    return setupPythonViaSystem(version, setupDir, arch)
+  const foundPython = installInfo.bin
+
+  // setup pip
+  const foundPip = await findOrSetupPip(foundPython)
+  if (foundPip === undefined) {
+    throw new Error("pip was not installed correctly")
   }
+
+  // setup wheel
+  try {
+    setupWheel(foundPython)
+  } catch (err) {
+    warning(`Failed to install wheels: ${(err as Error).toString()}. Ignoring...`)
+  }
+
+  return installInfo
+}
+
+async function findOrSetupPython(version: string, setupDir: string, arch: string) {
+  let installInfo: InstallationInfo | undefined
+  let foundPython = await findPython(setupDir)
+
+  if (foundPython !== undefined) {
+    const binDir = dirname(foundPython)
+    installInfo = { bin: foundPython, installDir: binDir, binDir }
+  } else {
+    // if python is not found, try to install it
+    if (GITHUB_ACTIONS) {
+      // install python in GitHub Actions
       try {
         info("Installing python in GitHub Actions")
         const { setupActionsPython } = await import("./actions_python")
-    return setupActionsPython(version, setupDir, arch)
+        await setupActionsPython(version, setupDir, arch)
+
+        foundPython = (await findPython(setupDir))!
+        const binDir = dirname(foundPython)
+        installInfo = { bin: foundPython, installDir: binDir, binDir }
       } catch (err) {
         warning((err as Error).toString())
-    return setupPythonViaSystem(version, setupDir, arch)
+      }
+    }
+    if (installInfo === undefined) {
+      // install python via system package manager
+      installInfo = await setupPythonSystem(setupDir, version)
     }
   }
 
-export async function setupPythonViaSystem(
+  if (foundPython === undefined || installInfo.bin === undefined) {
-  version: string,
+    foundPython = (await findPython(setupDir))!
-  setupDir: string,
+    installInfo.bin = foundPython
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  }
-  _arch: string
+
-): Promise<InstallationInfo> {
+  return installInfo
+}
+
+async function setupPythonSystem(setupDir: string, version: string) {
+  let installInfo: InstallationInfo | undefined
   switch (process.platform) {
     case "win32": {
       if (setupDir) {
@@ -48,86 +90,157 @@ export async function setupPythonViaSystem(
         await setupChocoPack("python3", version)
       }
       // Adding the bin dir to the path
-      const pythonBinPath =
+      const bin = (await findPython(setupDir))!
-        which.sync("python3.exe", { nothrow: true }) ??
+      const binDir = dirname(bin)
-        which.sync("python.exe", { nothrow: true }) ??
-        join(setupDir, "python.exe")
-      const pythonSetupDir = dirname(pythonBinPath)
       /** The directory which the tool is installed to */
-      await addPath(pythonSetupDir)
+      await addPath(binDir)
-      return { installDir: pythonSetupDir, binDir: pythonSetupDir }
+      installInfo = { installDir: binDir, binDir, bin }
+      break
     }
     case "darwin": {
-      return setupBrewPack("python3", version)
+      installInfo = await setupBrewPack("python3", version)
+      break
     }
     case "linux": {
-      let installInfo: InstallationInfo
       if (isArch()) {
         installInfo = await setupPacmanPack("python", version)
-        await setupPacmanPack("python-pip")
       } else if (hasDnf()) {
         installInfo = setupDnfPack("python3", version)
-        setupDnfPack("python3-pip")
       } else if (isUbuntu()) {
-        installInfo = await setupAptPack([{ name: "python3", version }, { name: "python3-pip" }])
+        installInfo = await setupAptPack([{ name: "python3", version }])
       } else {
         throw new Error("Unsupported linux distributions")
       }
-      return installInfo
+      break
     }
     default: {
       throw new Error("Unsupported platform")
     }
   }
+  return installInfo
 }
 
-let setupPythonAndPipTried = false
+async function findPython(binDir?: string) {
-
+  for (const pythonBin of ["python3", "python"]) {
-/// setup python and pip if needed
+    // eslint-disable-next-line no-await-in-loop
-export async function setupPythonAndPip(): Promise<string> {
+    const foundPython = await isPythonUpToDate(pythonBin, binDir)
-  let foundPython: string
+    if (foundPython !== undefined) {
-
-  // install python
-  if (which.sync("python3", { nothrow: true }) !== null) {
-    foundPython = "python3"
-  } else if (which.sync("python", { nothrow: true }) !== null && (await isBinUptoDate("python", "3.0.0"))) {
-    foundPython = "python"
-  } else {
-    info("python3 was not found. Installing python")
-    await setupPython(getVersion("python", undefined), "", process.arch)
-    // try again
-    if (setupPythonAndPipTried) {
-      throw new Error("Failed to install python")
-    }
-    setupPythonAndPipTried = true
-    return setupPythonAndPip() // recurse
-  }
-
-  assert(typeof foundPython === "string")
-
-  // install pip
-  if (process.platform === "win32") {
-    // downgrade pip on Windows
-    // https://github.com/pypa/pip/issues/10875#issuecomment-1030293005
-    execaSync(foundPython, ["-m", "pip", "install", "-U", "pip==21.3.1"], { stdio: "inherit" })
-  } else if (process.platform === "linux") {
-    // ensure that pip is installed on Linux (happens when python is found but pip not installed)
-    if (isArch()) {
-      await setupPacmanPack("python-pip")
-    } else if (hasDnf()) {
-      setupDnfPack("python3-pip")
-    } else if (isUbuntu()) {
-      await setupAptPack([{ name: "python3-pip" }])
-    }
-  }
-
-  // install wheel (required for Conan, Meson, etc.)
-  execaSync(foundPython, ["-m", "pip", "install", "-U", "wheel"], { stdio: "inherit" })
-
       return foundPython
     }
+  }
+  return undefined
+}
 
-export async function addPythonBaseExecPrefix(python: string) {
+async function isPythonUpToDate(candidate: string, binDir?: string) {
+  try {
+    if (binDir !== undefined) {
+      const pythonBinPath = join(binDir, addExeExt(candidate))
+      if (await pathExists(pythonBinPath)) {
+        if (await isBinUptoDate(pythonBinPath, MinVersions.python!)) {
+          return pythonBinPath
+        }
+      }
+    }
+    const pythonBinPaths = (await which(candidate, { nothrow: true, all: true })) ?? []
+    for (const pythonBinPath of pythonBinPaths) {
+      // eslint-disable-next-line no-await-in-loop
+      if (await isBinUptoDate(pythonBinPath, MinVersions.python!)) {
+        return pythonBinPath
+      }
+    }
+  } catch {
+    // fall through
+  }
+  return undefined
+}
+
+async function findOrSetupPip(foundPython: string) {
+  const maybePip = await findPip()
+
+  if (maybePip === undefined) {
+    // install pip if not installed
+    info("pip was not found. Installing pip")
+    await setupPip(foundPython)
+    return findPip() // recurse to check if pip is on PATH and up-to-date
+  }
+
+  return maybePip
+}
+
+async function findPip() {
+  for (const pipCandidate of ["pip3", "pip"]) {
+    // eslint-disable-next-line no-await-in-loop
+    const maybePip = await isPipUptoDate(pipCandidate)
+    if (maybePip !== undefined) {
+      return maybePip
+    }
+  }
+  return undefined
+}
+
+async function isPipUptoDate(pip: string) {
+  try {
+    const pipPaths = (await which(pip, { nothrow: true, all: true })) ?? []
+    for (const pipPath of pipPaths) {
+      // eslint-disable-next-line no-await-in-loop
+      if (pipPath !== null && (await isBinUptoDate(pipPath, MinVersions.pip!))) {
+        return pipPath
+      }
+    }
+  } catch {
+    // fall through
+  }
+  return undefined
+}
+
+async function setupPip(foundPython: string) {
+  const upgraded = ensurePipUpgrade(foundPython)
+  if (!upgraded) {
+    await setupPipSystem()
+    // upgrade pip
+    ensurePipUpgrade(foundPython)
+  }
+}
+
+function ensurePipUpgrade(foundPython: string) {
+  try {
+    execaSync(foundPython, ["-m", "ensurepip", "-U", "--upgrade"], { stdio: "inherit" })
+    return true
+  } catch (err1) {
+    info((err1 as Error)?.toString?.())
+    try {
+      // ensure pip is disabled on Ubuntu
+      execaSync(foundPython, ["-m", "pip", "install", "--upgrade", "pip"], { stdio: "inherit" })
+      return true
+    } catch (err2) {
+      info((err2 as Error)?.toString?.())
+      // pip module not found
+    }
+  }
+  // all methods failed
+  return false
+}
+
+function setupPipSystem() {
+  if (process.platform === "linux") {
+    // ensure that pip is installed on Linux (happens when python is found but pip not installed)
+    if (isArch()) {
+      return setupPacmanPack("python-pip")
+    } else if (hasDnf()) {
+      return setupDnfPack("python3-pip")
+    } else if (isUbuntu()) {
+      return setupAptPack([{ name: "python3-pip" }])
+    }
+  }
+  throw new Error(`Could not install pip on ${process.platform}`)
+}
+
+/** Install wheel (required for Conan, Meson, etc.) */
+function setupWheel(foundPython: string) {
+  execaSync(foundPython, ["-m", "pip", "install", "-U", "wheel"], { stdio: "inherit" })
+}
+
+async function addPythonBaseExecPrefix_raw(python: string) {
   const dirs: string[] = []
 
   // detection based on the platform
@@ -145,3 +258,10 @@ export async function addPythonBaseExecPrefix(python: string) {
   // remove duplicates
   return unique(dirs)
 }
+
+/**
+ * Add the base exec prefix to the PATH. This is required for Conan, Meson, etc. to work properly.
+ *
+ * The answer is cached for subsequent calls
+ */
+export const addPythonBaseExecPrefix = memoize(addPythonBaseExecPrefix_raw)

src/utils/env/ubuntu_version.ts

@@ -3,18 +3,24 @@ import { getUbuntuVersion } from "ubuntu-version"
 import which from "which"
 import { setupAptPack } from "../setup/setupAptPack"
 import { isUbuntu } from "./isUbuntu"
+import os from "os"
+import memoize from "micro-memoize"
 
-export async function ubuntuVersion(): Promise<number[] | null> {
+async function ubuntuVersion_raw(): Promise<number[] | null> {
   try {
     if (isUbuntu()) {
+      try {
         if (which.sync("lsb_release", { nothrow: true }) === null) {
           await setupAptPack([{ name: "lsb-release" }])
         }
+      } catch {
+        return detectUsingOsVersion()
+      }
+
       const versionSplitted = await getUbuntuVersion()
 
       if (versionSplitted.length === 0) {
-        warning("Failed to get the ubuntu major version.")
+        return detectUsingOsVersion()
-        return null
       }
 
       return versionSplitted
@@ -26,3 +32,18 @@ export async function ubuntuVersion(): Promise<number[] | null> {
     return null
   }
 }
+
+/** Detect Ubuntu version */
+export const ubuntuVersion = memoize(ubuntuVersion_raw)
+
+/** Detect Ubuntu version using os.version() for Ubuntu based distros */
+function detectUsingOsVersion() {
+  // #46~22.04.1-Ubuntu SMP ...
+  const osVersion = os.version()
+  const versionSplitted = osVersion.split(".")
+  const majorVersion = parseInt(versionSplitted[0].replace("#", ""), 10)
+  const minorVersion = parseInt(versionSplitted[1].replace("~", ""), 10)
+  const patchVersion = parseInt(versionSplitted[2].split("-")[0], 10)
+
+  return [majorVersion, minorVersion, patchVersion]
+}

src/utils/setup/setupAptPack.ts

@@ -123,6 +123,7 @@ function initGpg() {
 }
 
 export async function addAptKeyViaServer(keys: string[], name: string, server = "keyserver.ubuntu.com") {
+  try {
     const fileName = `/etc/apt/trusted.gpg.d/${name}`
     if (!(await pathExists(fileName))) {
       initGpg()
@@ -143,6 +144,10 @@ export async function addAptKeyViaServer(keys: string[], name: string, server =
       )
     }
     return fileName
+  } catch (err) {
+    warning(`Failed to add apt key via server ${server}: ${err}`)
+    return undefined
+  }
 }
 
 export async function addAptKeyViaDownload(name: string, url: string) {

src/utils/setup/setupBin.ts

@@ -34,6 +34,7 @@ export type InstallationInfo = {
   /** The top install dir */
   installDir?: string
   binDir: string
+  bin?: string
 }
 
 let didInit: boolean = false

src/utils/setup/setupPipPack.ts

@@ -3,31 +3,29 @@ import { execaSync } from "execa"
 import { pathExists } from "path-exists"
 import { addExeExt, dirname, join } from "patha"
 import which from "which"
-import { addPythonBaseExecPrefix, setupPythonAndPip } from "../../python/python"
+import { addPythonBaseExecPrefix, setupPython } from "../../python/python"
 import { addPath } from "../env/addEnv"
 import { InstallationInfo } from "./setupBin"
+import { getVersion } from "../../versions/versions"
+import { ubuntuVersion } from "../env/ubuntu_version"
 
 /* eslint-disable require-atomic-updates */
 let python: string | undefined
-let binDirs: string[] | undefined
 
 /** A function that installs a package using pip */
 export async function setupPipPack(name: string, version?: string): Promise<InstallationInfo> {
   info(`Installing ${name} ${version ?? ""} via pip`)
 
   if (python === undefined) {
-    python = await setupPythonAndPip()
+    python = (await setupPython(getVersion("python", undefined, await ubuntuVersion()), "", process.arch)).bin!
   }
 
   execaSync(python, ["-m", "pip", "install", version !== undefined && version !== "" ? `${name}==${version}` : name], {
     stdio: "inherit",
   })
 
-  if (binDirs === undefined) {
+  const execPaths = await addPythonBaseExecPrefix(python)
-    binDirs = await addPythonBaseExecPrefix(python)
+  const binDir = await findBinDir(execPaths, name)
-  }
-
-  const binDir = await findBinDir(binDirs, name)
 
   await addPath(binDir)
 

src/versions/default_versions.ts

@@ -30,6 +30,11 @@ export const DefaultVersions: Record<string, string | undefined> = {
   gcc: isArch() ? "13.1.1-1" : "13", // https://github.com/brechtsanders/winlibs_mingw/releases and // https://packages.ubuntu.com/search?suite=all&arch=any&searchon=names&keywords=gcc
 }
 
+export const MinVersions: Record<string, string | undefined> = {
+  pip: "22.3.1",
+  python: "3.7.9",
+}
+
 /// If an ubuntu versions is not in this map:
 // - the newer ubuntu versions use the first entry (e.g. v20),
 // - the older ones use ""