Merge pull request #257 from aminya/vcpkg-version

This commit is contained in:
Amin Yahyaabadi 2024-08-12 11:05:41 -07:00 committed by GitHub
commit bcd2a428a8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 165 additions and 120 deletions

2
.gitignore vendored
View File

@ -17,3 +17,5 @@ exe/
*.log *.log
*.exe *.exe
.cache/ .cache/
coverage

View File

@ -7,7 +7,8 @@
"**/dist/**", "**/dist/**",
"dev/cpp_vcpkg_project/**/*", "dev/cpp_vcpkg_project/**/*",
"**/.venv/", "**/.venv/",
"**/.*cache/" "**/.*cache/",
"**/coverage/"
], ],
"ignoreUnknown": true "ignoreUnknown": true
}, },

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -23,13 +23,15 @@ export async function setupLLVM(version: string, setupDir: string, arch: string)
} }
async function setupLLVMWithoutActivation_raw(version: string, setupDir: string, arch: string) { async function setupLLVMWithoutActivation_raw(version: string, setupDir: string, arch: string) {
// install LLVM and its dependencies in parallel // install LLVM
const [installationInfo, _1, _2] = await Promise.all([ const [installationInfo, _1] = await Promise.all([
setupLLVMOnly(version, setupDir, arch), setupLLVMOnly(version, setupDir, arch),
setupLLVMDeps(arch),
addLLVMLoggingMatcher(), addLLVMLoggingMatcher(),
]) ])
// install LLVM dependencies
await setupLLVMDeps(arch)
return installationInfo return installationInfo
} }
const setupLLVMWithoutActivation = memoize(setupLLVMWithoutActivation_raw, { isPromise: true }) const setupLLVMWithoutActivation = memoize(setupLLVMWithoutActivation_raw, { isPromise: true })

View File

@ -4,7 +4,7 @@ import { execa } from "execa"
import { chmod, readFile, writeFile } from "fs/promises" import { chmod, readFile, writeFile } from "fs/promises"
import { DEFAULT_TIMEOUT } from "../installTool" import { DEFAULT_TIMEOUT } from "../installTool"
import { addPath } from "../utils/env/addEnv" import { addPath } from "../utils/env/addEnv"
import { hasNala, isPackageRegexInstalled, setupAptPack } from "../utils/setup/setupAptPack" import { aptTimeout, hasNala, isPackageRegexInstalled, setupAptPack } from "../utils/setup/setupAptPack"
import type { InstallationInfo } from "../utils/setup/setupBin" import type { InstallationInfo } from "../utils/setup/setupBin"
export enum LLVMPackages { export enum LLVMPackages {
@ -70,8 +70,8 @@ function nonInteractiveScript(script: string) {
// make the scirpt non-interactive and fix broken packages // make the scirpt non-interactive and fix broken packages
return script.replace( return script.replace(
/add-apt-repository "\${REPO_NAME}"/g, /add-apt-repository "\${REPO_NAME}"/g,
// eslint-disable-next-line no-template-curly-in-string `add-apt-repository -y -n "\${REPO_NAME}"
"add-apt-repository -y \"${REPO_NAME}\"", apt-get update -o ${aptTimeout} -y`,
) )
} }
@ -79,7 +79,7 @@ async function removeConflictingPackages(givenScript: string) {
// fix conflicts between libclang-rt and libclang // fix conflicts between libclang-rt and libclang
let script = givenScript.replace( let script = givenScript.replace(
/apt-get install -y/g, /apt-get install -y/g,
"apt-get install -o Dpkg::Options::=\"--force-overwrite\" -y --fix-broken", `apt-get install -o Dpkg::Options::="--force-overwrite" -o ${aptTimeout} -y --fix-broken`,
) )
// check if these are installed and if so, remove them from the script as they conflict // check if these are installed and if so, remove them from the script as they conflict

View File

@ -13,6 +13,9 @@ import type { InstallationInfo } from "./setupBin"
let didUpdate: boolean = false let didUpdate: boolean = false
let didInit: boolean = false let didInit: boolean = false
// wait up to 300 seconds if the apt-get lock is held
export const aptTimeout = "DPkg::Lock::Timeout=300"
export type AptPackage = { export type AptPackage = {
name: string name: string
version?: string version?: string
@ -44,13 +47,9 @@ export async function setupAptPack(packages: AptPackage[], update = false): Prom
// Add the repos if needed // Add the repos if needed
await addRepositories(apt, packages) await addRepositories(apt, packages)
// Qualify the packages into full package name/version const needToInstall = await filterAndQualifyAptPackages(packages)
let qualifiedPacks = await Promise.all(packages.map((pack) => getAptArg(pack.name, pack.version)))
// find the packages that are not installed if (needToInstall.length === 0) {
qualifiedPacks = await Promise.all(qualifiedPacks.filter(async (pack) => !(await isPackageInstalled(pack))))
if (qualifiedPacks.length === 0) {
info("All packages are already installed") info("All packages are already installed")
return { binDir: "/usr/bin/" } return { binDir: "/usr/bin/" }
} }
@ -63,13 +62,13 @@ export async function setupAptPack(packages: AptPackage[], update = false): Prom
// Install // Install
try { try {
execRootSync(apt, ["install", "--fix-broken", "-y", ...qualifiedPacks]) execRootSync(apt, ["install", "--fix-broken", "-y", ...needToInstall])
} catch (err) { } catch (err) {
if ("stderr" in (err as ExecaError)) { if ("stderr" in (err as ExecaError)) {
const stderr = (err as ExecaError).stderr const stderr = (err as ExecaError).stderr
if (retryErrors.some((error) => stderr.includes(error))) { if (retryErrors.some((error) => stderr.includes(error))) {
warning(`Failed to install packages ${qualifiedPacks}. Retrying...`) warning(`Failed to install packages ${needToInstall}. Retrying...`)
execRootSync(apt, ["install", "--fix-broken", "-y", ...qualifiedPacks]) execRootSync(apt, ["install", "--fix-broken", "-y", "-o", aptTimeout, ...needToInstall])
} }
} else { } else {
throw err throw err
@ -86,6 +85,21 @@ export enum AptPackageType {
None = 3, None = 3,
} }
/**
* Filter out the packages that are already installed and qualify the packages into a full package name/version
*/
async function filterAndQualifyAptPackages(packages: AptPackage[]) {
return (await Promise.all(packages.map(qualifiedNeededAptPackage)))
.filter((pack) => pack !== undefined)
}
async function qualifiedNeededAptPackage(pack: AptPackage) {
// Qualify the packages into full package name/version
const qualified = await getAptArg(pack.name, pack.version)
// filter out the packages that are already installed
return (await isPackageInstalled(qualified)) ? undefined : qualified
}
async function addRepositories(apt: string, packages: AptPackage[]) { async function addRepositories(apt: string, packages: AptPackage[]) {
const allRepositories = [...new Set(packages.flatMap((pack) => pack.repositories ?? []))] const allRepositories = [...new Set(packages.flatMap((pack) => pack.repositories ?? []))]
if (allRepositories.length !== 0) { if (allRepositories.length !== 0) {
@ -93,10 +107,10 @@ async function addRepositories(apt: string, packages: AptPackage[]) {
await initApt(apt) await initApt(apt)
didInit = true didInit = true
} }
await installAddAptRepo() await installAddAptRepo(apt)
for (const repo of allRepositories) { for (const repo of allRepositories) {
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
execRootSync("add-apt-repository", ["-y", repo]) execRootSync("add-apt-repository", ["-y", "--no-update", repo])
} }
updateRepos(apt) updateRepos(apt)
didUpdate = true didUpdate = true
@ -163,7 +177,7 @@ export function hasNala() {
return which.sync("nala", { nothrow: true }) !== null return which.sync("nala", { nothrow: true }) !== null
} }
function getApt() { export function getApt() {
let apt: string let apt: string
if (hasNala()) { if (hasNala()) {
apt = "nala" apt = "nala"
@ -174,14 +188,14 @@ function getApt() {
} }
function updateRepos(apt: string) { function updateRepos(apt: string) {
execRootSync(apt, apt !== "nala" ? ["update", "-y"] : ["update"]) execRootSync(apt, apt !== "nala" ? ["update", "-y", "-o", aptTimeout] : ["update", "-o", aptTimeout])
} }
async function installAddAptRepo() { async function installAddAptRepo(apt: string) {
if (await isPackageInstalled("software-properties-common")) { if (await isPackageInstalled("software-properties-common")) {
return return
} }
execRootSync("apt-get", ["install", "-y", "--fix-broken", "software-properties-common"]) execRootSync(apt, ["install", "-y", "--fix-broken", "-o", aptTimeout, "software-properties-common"])
} }
/** Install gnupg and certificates (usually missing from docker containers) */ /** Install gnupg and certificates (usually missing from docker containers) */
@ -192,10 +206,14 @@ async function initApt(apt: string) {
didUpdate = true didUpdate = true
} }
const toInstall = ["ca-certificates", "gnupg", "apt-utils"].filter(async (pack) => !(await isPackageInstalled(pack))) const toInstall = await filterAndQualifyAptPackages([
{ name: "ca-certificates" },
{ name: "gnupg" },
{ name: "apt-utils" },
])
if (toInstall.length !== 0) { if (toInstall.length !== 0) {
execRootSync(apt, ["install", "-y", "--fix-broken", ...toInstall]) execRootSync(apt, ["install", "-y", "--fix-broken", "-o", aptTimeout, ...toInstall])
} }
const promises: Promise<string | void>[] = [ const promises: Promise<string | void>[] = [

View File

@ -1,20 +1,28 @@
import { setupTmpDir, testBin } from "../../utils/tests/test-helpers" import { cleanupTmpDir, setupTmpDir, testBin } from "../../utils/tests/test-helpers"
import { setupVcpkg } from "../vcpkg" import { setupVcpkg } from "../vcpkg"
jest.setTimeout(300000) jest.setTimeout(300000)
async function testVcpkg(directory: string) {
const { binDir } = await setupVcpkg("", directory, "")
await testBin("vcpkg", ["--version"], binDir)
return binDir
}
describe("setup-vcpkg", () => { describe("setup-vcpkg", () => {
let directory: string let directory: string
beforeAll(async () => { beforeEach(async () => {
directory = await setupTmpDir("vcpkg") directory = await setupTmpDir("vcpkg")
}) })
it("should setup vcpkg", async () => { it("should setup vcpkg", async () => {
await testVcpkg(directory) console.log(!("true" in ["", "true"]))
const { binDir } = await setupVcpkg("", directory, "")
await testBin("vcpkg", ["--version"], binDir)
return binDir
})
it("should setup vcpkg with specific version", async () => {
const { binDir } = await setupVcpkg("e590c2b30c08caf1dd8d612ec602a003f9784b7d", directory, "")
await testBin("vcpkg", ["--version"], binDir)
return binDir
})
afterEach(async () => {
await cleanupTmpDir(directory)
}) })
}) })

View File

@ -1,5 +1,5 @@
import { grantUserWriteAccess } from "admina" import { grantUserWriteAccess } from "admina"
import { notice } from "ci-log" import { info, notice } from "ci-log"
import { execaSync } from "execa" import { execaSync } from "execa"
import { pathExists } from "path-exists" import { pathExists } from "path-exists"
import { addShExt, addShRelativePrefix, dirname, join } from "patha" import { addShExt, addShRelativePrefix, dirname, join } from "patha"
@ -16,7 +16,7 @@ import { setupPacmanPack } from "../utils/setup/setupPacmanPack"
let hasVCPKG = false let hasVCPKG = false
// eslint-disable-next-line @typescript-eslint/no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars
export async function setupVcpkg(_version: string, setupDir: string, _arch: string): Promise<InstallationInfo> { export async function setupVcpkg(version: string, setupDir: string, _arch: string): Promise<InstallationInfo> {
if (!hasVCPKG || which.sync("vcpkg", { nothrow: true }) === null) { if (!hasVCPKG || which.sync("vcpkg", { nothrow: true }) === null) {
if (process.platform === "linux") { if (process.platform === "linux") {
// vcpkg download and extraction dependencies // vcpkg download and extraction dependencies
@ -50,12 +50,23 @@ export async function setupVcpkg(_version: string, setupDir: string, _arch: stri
} }
} }
// clone if not already exists
if (!(await pathExists(join(setupDir, addShExt("bootstrap-vcpkg", ".bat"))))) { if (!(await pathExists(join(setupDir, addShExt("bootstrap-vcpkg", ".bat"))))) {
execaSync("git", ["clone", "https://github.com/microsoft/vcpkg"], { cwd: dirname(setupDir), stdio: "inherit" }) execaSync("git", ["clone", "https://github.com/microsoft/vcpkg"], { cwd: dirname(setupDir), stdio: "inherit" })
} else { } else {
notice(`Vcpkg folder already exists at ${setupDir}. This might mean that ~/vcpkg is restored from the cache.`) notice(`Vcpkg folder already exists at ${setupDir}. Skipping the clone`)
} }
// if version specified, checkout the version
if (version !== "" && version !== "true") {
info(`Checking out vcpkg version ${version}`)
execaSync("git", ["checkout", version], {
cwd: setupDir,
stdio: "inherit",
})
}
// bootstrap vcpkg
execaSync(addShExt(addShRelativePrefix("bootstrap-vcpkg"), ".bat"), { execaSync(addShExt(addShRelativePrefix("bootstrap-vcpkg"), ".bat"), {
cwd: setupDir, cwd: setupDir,
shell: true, shell: true,