Merge pull request #183 from aminya/llvm-conflicts [skip test]

cspell.config.yaml

@@ -12,10 +12,6 @@ ignorePaths:
   - .vscode/extensions.json
 words:
   - aarch
-  - clangd
-  - Trofimovich
-  - cobertura
-  - whatwg
   - aminya
   - applellvm
   - bazel
@@ -24,7 +20,9 @@ words:
   - caxa
   - ccache
   - choco
+  - clangd
   - cmake
+  - cobertura
   - copr
   - CPATH
   - Cppcheck
@@ -50,6 +48,7 @@ words:
   - LDFLAGS
   - lefticus
   - libbinutils
+  - libc
   - libdw
   - libstdc
   - libtinfo
@@ -77,6 +76,7 @@ words:
   - setx
   - Syuu
   - terserrc
+  - Trofimovich
   - tsbuildinfo
   - ucrt
   - untildify
@@ -87,6 +87,7 @@ words:
   - visualc
   - visualcpp
   - vsversion
+  - whatwg
   - xcrun
   - Yahyaabadi
 ignoreWords: []

src/llvm/llvm.ts

@@ -1,22 +1,19 @@
-import { execRoot } from "admina"
 import { GITHUB_ACTIONS } from "ci-info"
 import { info, warning } from "ci-log"
-import { execa } from "execa"
-import { promises } from "fs"
-const { readFile, writeFile, chmod } = promises
 import memoize from "micro-memoize"
 import { delimiter } from "path"
 import { pathExists } from "path-exists"
 import { addExeExt, join } from "patha"
 import { setupGcc } from "../gcc/gcc"
 import { setupMacOSSDK } from "../macos-sdk/macos-sdk"
-import { addEnv, addPath } from "../utils/env/addEnv"
+import { addEnv } from "../utils/env/addEnv"
 import { isUbuntu } from "../utils/env/isUbuntu"
 import { ubuntuVersion } from "../utils/env/ubuntu_version"
-import { hasNala, setupAptPack, updateAptAlternatives } from "../utils/setup/setupAptPack"
+import { setupAptPack, updateAptAlternatives } from "../utils/setup/setupAptPack"
 import { InstallationInfo, setupBin } from "../utils/setup/setupBin"
 import { semverCoerceIfInvalid } from "../utils/setup/version"
 import { getVersion } from "../versions/versions"
+import { setupLLVMApt } from "./llvm_installer"
 import { getLLVMPackageInfo } from "./llvm_url"
 
 export async function setupLLVM(version: string, setupDir: string, arch: string): Promise<InstallationInfo> {
@@ -55,50 +52,6 @@ async function setupLLVMOnly(version: string, setupDir: string, arch: string) {
   return installationInfo
 }
 
-async function setupLLVMApt(majorVersion: number): Promise<InstallationInfo> {
-  // TODO for older versions, this also includes the minor version
-  const installationFolder = `/usr/lib/llvm-${majorVersion}`
-
-  await setupAptPack([{ name: "curl" }])
-  await execa("curl", ["-LJO", "https://apt.llvm.org/llvm.sh"], { cwd: "/tmp" })
-  const neededPackages = await patchAptLLVMScript("/tmp/llvm.sh", "/tmp/llvm-setup-cpp.sh")
-  await setupAptPack(neededPackages)
-  await chmod("/tmp/llvm-setup-cpp.sh", "755")
-  await execRoot("bash", ["/tmp/llvm-setup-cpp.sh", `${majorVersion}`, "all"], {
-    stdio: "inherit",
-    shell: true,
-  })
-
-  await addPath(`${installationFolder}/bin`)
-
-  return {
-    installDir: `${installationFolder}`,
-    binDir: `${installationFolder}/bin`,
-    bin: `${installationFolder}/bin/clang++`,
-  }
-}
-
-async function patchAptLLVMScript(path: string, target_path: string) {
-  let script = await readFile(path, "utf-8")
-  // make the scirpt non-interactive and fix broken packages
-  script = script
-    .replace(
-      /add-apt-repository "\${REPO_NAME}"/g,
-      // eslint-disable-next-line no-template-curly-in-string
-      'add-apt-repository -y "${REPO_NAME}"'
-    )
-    // fix conflicts between libclang-rt and libclang
-    .replace(/apt-get install -y/g, 'apt-get install -o Dpkg::Options::="--force-overwrite" -y --fix-broken')
-  // use nala if it is available
-  if (hasNala()) {
-    script = script.replace(/apt-get/g, "nala")
-  }
-  await writeFile(target_path, script)
-
-  // the packages needed by the script
-  return [{ name: "lsb-release" }, { name: "wget" }, { name: "software-properties-common" }, { name: "gnupg" }]
-}
-
 async function llvmBinaryDeps_raw(majorVersion: number) {
   if (isUbuntu()) {
     if (majorVersion <= 10) {

src/llvm/llvm_installer.ts

@@ -0,0 +1,81 @@
+import { execRoot } from "admina"
+import { execa } from "execa"
+import { addPath } from "../utils/env/addEnv"
+import { hasNala, isPackageInstalled, setupAptPack } from "../utils/setup/setupAptPack"
+import { InstallationInfo } from "../utils/setup/setupBin"
+import { promises } from "fs"
+import { info } from "console"
+const { readFile, writeFile, chmod } = promises
+
+export async function setupLLVMApt(majorVersion: number): Promise<InstallationInfo> {
+  // TODO for older versions, this also includes the minor version
+  const installationFolder = `/usr/lib/llvm-${majorVersion}`
+
+  await setupAptPack([{ name: "curl" }])
+  await execa("curl", ["-LJO", "https://apt.llvm.org/llvm.sh"], { cwd: "/tmp" })
+  const neededPackages = await patchAptLLVMScript("/tmp/llvm.sh", "/tmp/llvm-setup-cpp.sh")
+  await setupAptPack(neededPackages)
+  await chmod("/tmp/llvm-setup-cpp.sh", "755")
+  await execRoot("bash", ["/tmp/llvm-setup-cpp.sh", `${majorVersion}`, "all"], {
+    stdio: "inherit",
+    shell: true,
+  })
+
+  await addPath(`${installationFolder}/bin`)
+
+  return {
+    installDir: `${installationFolder}`,
+    binDir: `${installationFolder}/bin`,
+    bin: `${installationFolder}/bin/clang++`,
+  }
+}
+
+async function patchAptLLVMScript(path: string, target_path: string) {
+  let script = await readFile(path, "utf-8")
+
+  script = nonInteractiveScript(script)
+  script = await removeConflictingPAckages(script)
+  script = useNalaScript(script)
+
+  await writeFile(target_path, script)
+
+  // the packages needed by the script
+  return [{ name: "lsb-release" }, { name: "wget" }, { name: "software-properties-common" }, { name: "gnupg" }]
+}
+function nonInteractiveScript(givenScript: string) {
+  // make the scirpt non-interactive and fix broken packages
+  return givenScript.replace(
+    /add-apt-repository "\${REPO_NAME}"/g,
+    // eslint-disable-next-line no-template-curly-in-string
+    'add-apt-repository -y "${REPO_NAME}"'
+  )
+}
+
+async function removeConflictingPAckages(givenScript: string) {
+  // fix conflicts between libclang-rt and libclang
+  let script = givenScript.replace(
+    /apt-get install -y/g,
+    'apt-get install -o Dpkg::Options::="--force-overwrite" -y --fix-broken'
+  )
+
+  // check if these are installed and if so, remove them from the script as they conflict
+  const conflictingPackages = ["libc++-$LLVM_VERSION-dev", "libc++abi-$LLVM_VERSION-dev", "libunwind-$LLVM_VERSION-dev"]
+  await Promise.all(
+    conflictingPackages.map(async (pack) => {
+      const installingPack = pack.replace("$LLVM_VERSION", "*")
+      if (await isPackageInstalled(installingPack)) {
+        info(`Removing conflicting package ${installingPack}`)
+        script = script.replace(pack, "")
+      }
+    })
+  )
+  return script
+}
+
+function useNalaScript(script: string) {
+  // use nala if it is available
+  if (hasNala()) {
+    return script.replace(/apt-get/g, "nala")
+  }
+  return script
+}

src/utils/setup/setupAptPack.ts

@@ -20,6 +20,12 @@ export type AptPackage = {
   repositories?: string[]
 }
 
+const retryErrors = [
+  "E: Could not get lock",
+  "dpkg: error processing archive",
+  "dpkg: error: dpkg status database is locked by another process",
+]
+
 /** A function that installs a package using apt */
 export async function setupAptPack(packages: AptPackage[], update = false): Promise<InstallationInfo> {
   const apt: string = getApt()
@@ -57,7 +63,7 @@ export async function setupAptPack(packages: AptPackage[], update = false): Prom
   } catch (err) {
     if ("stderr" in (err as ExecaError)) {
       const stderr = (err as ExecaError).stderr
-      if (stderr.includes("E: Could not get lock") || stderr.includes("dpkg: error processing archive")) {
+      if (retryErrors.some((error) => stderr.includes(error))) {
         warning(`Failed to install packages ${aptArgs}. Retrying...`)
         execRootSync(apt, ["install", "--fix-broken", "-y", ...aptArgs])
       }
@@ -224,3 +230,15 @@ export async function updateAptAlternatives(name: string, path: string) {
     )
   }
 }
+
+export async function isPackageInstalled(regexp: string) {
+  try {
+    // check if a package matching the regexp is installed
+    const { stdout } = await execa("dpkg", ["-l", regexp])
+    const lines = stdout.split("\n")
+    // check if the output contains any lines that start with "ii"
+    return lines.some((line) => line.startsWith("ii"))
+  } catch {
+    return false
+  }
+}