Merge pull request #257 from aminya/vcpkg-version

This commit is contained in:
Amin Yahyaabadi 2024-08-12 11:05:41 -07:00 committed by GitHub
commit bcd2a428a8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 165 additions and 120 deletions

2
.gitignore vendored
View File

@ -17,3 +17,5 @@ exe/
*.log
*.exe
.cache/
coverage

View File

@ -7,7 +7,8 @@
"**/dist/**",
"dev/cpp_vcpkg_project/**/*",
"**/.venv/",
"**/.*cache/"
"**/.*cache/",
"**/coverage/"
],
"ignoreUnknown": true
},

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -23,13 +23,15 @@ export async function setupLLVM(version: string, setupDir: string, arch: string)
}
async function setupLLVMWithoutActivation_raw(version: string, setupDir: string, arch: string) {
// install LLVM and its dependencies in parallel
const [installationInfo, _1, _2] = await Promise.all([
// install LLVM
const [installationInfo, _1] = await Promise.all([
setupLLVMOnly(version, setupDir, arch),
setupLLVMDeps(arch),
addLLVMLoggingMatcher(),
])
// install LLVM dependencies
await setupLLVMDeps(arch)
return installationInfo
}
const setupLLVMWithoutActivation = memoize(setupLLVMWithoutActivation_raw, { isPromise: true })

View File

@ -4,7 +4,7 @@ import { execa } from "execa"
import { chmod, readFile, writeFile } from "fs/promises"
import { DEFAULT_TIMEOUT } from "../installTool"
import { addPath } from "../utils/env/addEnv"
import { hasNala, isPackageRegexInstalled, setupAptPack } from "../utils/setup/setupAptPack"
import { aptTimeout, hasNala, isPackageRegexInstalled, setupAptPack } from "../utils/setup/setupAptPack"
import type { InstallationInfo } from "../utils/setup/setupBin"
export enum LLVMPackages {
@ -70,8 +70,8 @@ function nonInteractiveScript(script: string) {
// make the scirpt non-interactive and fix broken packages
return script.replace(
/add-apt-repository "\${REPO_NAME}"/g,
// eslint-disable-next-line no-template-curly-in-string
"add-apt-repository -y \"${REPO_NAME}\"",
`add-apt-repository -y -n "\${REPO_NAME}"
apt-get update -o ${aptTimeout} -y`,
)
}
@ -79,7 +79,7 @@ async function removeConflictingPackages(givenScript: string) {
// fix conflicts between libclang-rt and libclang
let script = givenScript.replace(
/apt-get install -y/g,
"apt-get install -o Dpkg::Options::=\"--force-overwrite\" -y --fix-broken",
`apt-get install -o Dpkg::Options::="--force-overwrite" -o ${aptTimeout} -y --fix-broken`,
)
// check if these are installed and if so, remove them from the script as they conflict

View File

@ -13,6 +13,9 @@ import type { InstallationInfo } from "./setupBin"
let didUpdate: boolean = false
let didInit: boolean = false
// wait up to 300 seconds if the apt-get lock is held
export const aptTimeout = "DPkg::Lock::Timeout=300"
export type AptPackage = {
name: string
version?: string
@ -44,13 +47,9 @@ export async function setupAptPack(packages: AptPackage[], update = false): Prom
// Add the repos if needed
await addRepositories(apt, packages)
// Qualify the packages into full package name/version
let qualifiedPacks = await Promise.all(packages.map((pack) => getAptArg(pack.name, pack.version)))
const needToInstall = await filterAndQualifyAptPackages(packages)
// find the packages that are not installed
qualifiedPacks = await Promise.all(qualifiedPacks.filter(async (pack) => !(await isPackageInstalled(pack))))
if (qualifiedPacks.length === 0) {
if (needToInstall.length === 0) {
info("All packages are already installed")
return { binDir: "/usr/bin/" }
}
@ -63,13 +62,13 @@ export async function setupAptPack(packages: AptPackage[], update = false): Prom
// Install
try {
execRootSync(apt, ["install", "--fix-broken", "-y", ...qualifiedPacks])
execRootSync(apt, ["install", "--fix-broken", "-y", ...needToInstall])
} catch (err) {
if ("stderr" in (err as ExecaError)) {
const stderr = (err as ExecaError).stderr
if (retryErrors.some((error) => stderr.includes(error))) {
warning(`Failed to install packages ${qualifiedPacks}. Retrying...`)
execRootSync(apt, ["install", "--fix-broken", "-y", ...qualifiedPacks])
warning(`Failed to install packages ${needToInstall}. Retrying...`)
execRootSync(apt, ["install", "--fix-broken", "-y", "-o", aptTimeout, ...needToInstall])
}
} else {
throw err
@ -86,6 +85,21 @@ export enum AptPackageType {
None = 3,
}
/**
* Filter out the packages that are already installed and qualify the packages into a full package name/version
*/
async function filterAndQualifyAptPackages(packages: AptPackage[]) {
return (await Promise.all(packages.map(qualifiedNeededAptPackage)))
.filter((pack) => pack !== undefined)
}
async function qualifiedNeededAptPackage(pack: AptPackage) {
// Qualify the packages into full package name/version
const qualified = await getAptArg(pack.name, pack.version)
// filter out the packages that are already installed
return (await isPackageInstalled(qualified)) ? undefined : qualified
}
async function addRepositories(apt: string, packages: AptPackage[]) {
const allRepositories = [...new Set(packages.flatMap((pack) => pack.repositories ?? []))]
if (allRepositories.length !== 0) {
@ -93,10 +107,10 @@ async function addRepositories(apt: string, packages: AptPackage[]) {
await initApt(apt)
didInit = true
}
await installAddAptRepo()
await installAddAptRepo(apt)
for (const repo of allRepositories) {
// eslint-disable-next-line no-await-in-loop
execRootSync("add-apt-repository", ["-y", repo])
execRootSync("add-apt-repository", ["-y", "--no-update", repo])
}
updateRepos(apt)
didUpdate = true
@ -163,7 +177,7 @@ export function hasNala() {
return which.sync("nala", { nothrow: true }) !== null
}
function getApt() {
export function getApt() {
let apt: string
if (hasNala()) {
apt = "nala"
@ -174,14 +188,14 @@ function getApt() {
}
function updateRepos(apt: string) {
execRootSync(apt, apt !== "nala" ? ["update", "-y"] : ["update"])
execRootSync(apt, apt !== "nala" ? ["update", "-y", "-o", aptTimeout] : ["update", "-o", aptTimeout])
}
async function installAddAptRepo() {
async function installAddAptRepo(apt: string) {
if (await isPackageInstalled("software-properties-common")) {
return
}
execRootSync("apt-get", ["install", "-y", "--fix-broken", "software-properties-common"])
execRootSync(apt, ["install", "-y", "--fix-broken", "-o", aptTimeout, "software-properties-common"])
}
/** Install gnupg and certificates (usually missing from docker containers) */
@ -192,10 +206,14 @@ async function initApt(apt: string) {
didUpdate = true
}
const toInstall = ["ca-certificates", "gnupg", "apt-utils"].filter(async (pack) => !(await isPackageInstalled(pack)))
const toInstall = await filterAndQualifyAptPackages([
{ name: "ca-certificates" },
{ name: "gnupg" },
{ name: "apt-utils" },
])
if (toInstall.length !== 0) {
execRootSync(apt, ["install", "-y", "--fix-broken", ...toInstall])
execRootSync(apt, ["install", "-y", "--fix-broken", "-o", aptTimeout, ...toInstall])
}
const promises: Promise<string | void>[] = [

View File

@ -1,20 +1,28 @@
import { setupTmpDir, testBin } from "../../utils/tests/test-helpers"
import { cleanupTmpDir, setupTmpDir, testBin } from "../../utils/tests/test-helpers"
import { setupVcpkg } from "../vcpkg"
jest.setTimeout(300000)
async function testVcpkg(directory: string) {
const { binDir } = await setupVcpkg("", directory, "")
await testBin("vcpkg", ["--version"], binDir)
return binDir
}
describe("setup-vcpkg", () => {
let directory: string
beforeAll(async () => {
beforeEach(async () => {
directory = await setupTmpDir("vcpkg")
})
it("should setup vcpkg", async () => {
await testVcpkg(directory)
console.log(!("true" in ["", "true"]))
const { binDir } = await setupVcpkg("", directory, "")
await testBin("vcpkg", ["--version"], binDir)
return binDir
})
it("should setup vcpkg with specific version", async () => {
const { binDir } = await setupVcpkg("e590c2b30c08caf1dd8d612ec602a003f9784b7d", directory, "")
await testBin("vcpkg", ["--version"], binDir)
return binDir
})
afterEach(async () => {
await cleanupTmpDir(directory)
})
})

View File

@ -1,5 +1,5 @@
import { grantUserWriteAccess } from "admina"
import { notice } from "ci-log"
import { info, notice } from "ci-log"
import { execaSync } from "execa"
import { pathExists } from "path-exists"
import { addShExt, addShRelativePrefix, dirname, join } from "patha"
@ -16,7 +16,7 @@ import { setupPacmanPack } from "../utils/setup/setupPacmanPack"
let hasVCPKG = false
// eslint-disable-next-line @typescript-eslint/no-unused-vars
export async function setupVcpkg(_version: string, setupDir: string, _arch: string): Promise<InstallationInfo> {
export async function setupVcpkg(version: string, setupDir: string, _arch: string): Promise<InstallationInfo> {
if (!hasVCPKG || which.sync("vcpkg", { nothrow: true }) === null) {
if (process.platform === "linux") {
// vcpkg download and extraction dependencies
@ -50,12 +50,23 @@ export async function setupVcpkg(_version: string, setupDir: string, _arch: stri
}
}
// clone if not already exists
if (!(await pathExists(join(setupDir, addShExt("bootstrap-vcpkg", ".bat"))))) {
execaSync("git", ["clone", "https://github.com/microsoft/vcpkg"], { cwd: dirname(setupDir), stdio: "inherit" })
} else {
notice(`Vcpkg folder already exists at ${setupDir}. This might mean that ~/vcpkg is restored from the cache.`)
notice(`Vcpkg folder already exists at ${setupDir}. Skipping the clone`)
}
// if version specified, checkout the version
if (version !== "" && version !== "true") {
info(`Checking out vcpkg version ${version}`)
execaSync("git", ["checkout", version], {
cwd: setupDir,
stdio: "inherit",
})
}
// bootstrap vcpkg
execaSync(addShExt(addShRelativePrefix("bootstrap-vcpkg"), ".bat"), {
cwd: setupDir,
shell: true,