perf: parallelized checking of the file existence

This commit is contained in:
Amin Yahyaabadi 2022-11-22 20:51:18 -08:00
parent 43c67b8a9c
commit ebd90dd38d
16 changed files with 90 additions and 68 deletions

View File

@ -65,6 +65,7 @@
"mri": "^1.2.0",
"msvc-dev-cmd": "github:aminya/msvc-dev-cmd#9f672c1",
"numerous": "1.0.3",
"path-exists": "^5.0.0",
"patha": "^0.4.1",
"quote-unquote": "^1.0.0",
"semver": "7.3.8",

View File

@ -44,6 +44,7 @@ importers:
npm-run-all2: ^6.0.4
numerous: 1.0.3
parcel: 2.8.0
path-exists: ^5.0.0
patha: ^0.4.1
prettier: 2.7.1
prettier-config-atomic: ^3.1.0
@ -78,6 +79,7 @@ importers:
mri: 1.2.0
msvc-dev-cmd: github.com/aminya/msvc-dev-cmd/9f672c1
numerous: 1.0.3
path-exists: 5.0.0
patha: 0.4.1
quote-unquote: 1.0.0
semver: 7.3.8
@ -7824,6 +7826,11 @@ packages:
engines: {node: '>=8'}
dev: true
/path-exists/5.0.0:
resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
dev: false
/path-is-absolute/1.0.1:
resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==}
engines: {node: '>=0.10.0'}

View File

@ -1,6 +1,7 @@
/* eslint-disable require-atomic-updates */
import execa from "execa"
import { existsSync } from "fs"
import { pathExists } from "path-exists"
import { dirname } from "patha"
import which from "which"
import { addPath } from "../utils/env/addEnv"
@ -63,7 +64,7 @@ export async function setupChocolatey(
binDir = `${process.env.ChocolateyInstall ?? "C:/ProgramData/chocolatey"}/bin`
}
if (existsSync(binDir)) {
if (await pathExists(binDir)) {
return { binDir }
}
return undefined

View File

@ -9,11 +9,12 @@ import { extractTar, extractZip } from "../utils/setup/extract"
import { notice } from "ci-log"
import { setupGraphviz } from "../graphviz/graphviz"
import { getVersion } from "../versions/versions"
import { existsSync } from "fs"
import { isArch } from "../utils/env/isArch"
import { hasDnf } from "../utils/env/hasDnf"
import { setupDnfPack } from "../utils/setup/setupDnfPack"
import { isUbuntu } from "../utils/env/isUbuntu"
import { pathExists } from "path-exists"
/** Get the platform data for cmake */
// eslint-disable-next-line @typescript-eslint/no-unused-vars
@ -99,7 +100,7 @@ async function activateWinDoxygen() {
"C:/Program Files/doxygen/bin",
"C:/Program Files (x86)/doxygen",
]) {
if (existsSync(join(binDir, "doxygen.exe"))) {
if (await pathExists(join(binDir, "doxygen.exe"))) {
// eslint-disable-next-line no-await-in-loop
await addPath(binDir)
return binDir

View File

@ -1,5 +1,5 @@
import { addPath, addEnv } from "../utils/env/addEnv"
import { existsSync } from "fs"
import { setupAptPack, updateAptAlternatives } from "../utils/setup/setupAptPack"
import { setupPacmanPack } from "../utils/setup/setupPacmanPack"
import { setupBrewPack } from "../utils/setup/setupBrewPack"
@ -16,6 +16,7 @@ import { isArch } from "../utils/env/isArch"
import { isUbuntu } from "../utils/env/isUbuntu"
import { hasDnf } from "../utils/env/hasDnf"
import { setupDnfPack } from "../utils/setup/setupDnfPack"
import { pathExists } from "path-exists"
interface MingwInfo {
releaseName: string
@ -133,13 +134,13 @@ export async function setupGcc(version: string, setupDir: string, arch: string)
async function setupChocoMingw(version: string, arch: string): Promise<InstallationInfo | undefined> {
await setupChocoPack("mingw", version)
let binDir: string | undefined
if (arch === "x64" && existsSync("C:/tools/mingw64/bin")) {
if (arch === "x64" && (await pathExists("C:/tools/mingw64/bin"))) {
binDir = "C:/tools/mingw64/bin"
await addPath(binDir)
} else if (arch === "ia32" && existsSync("C:/tools/mingw32/bin")) {
} else if (arch === "ia32" && (await pathExists("C:/tools/mingw32/bin"))) {
binDir = "C:/tools/mingw32/bin"
await addPath(binDir)
} else if (existsSync(`${process.env.ChocolateyInstall ?? "C:/ProgramData/chocolatey"}/bin/g++.exe`)) {
} else if (await pathExists(`${process.env.ChocolateyInstall ?? "C:/ProgramData/chocolatey"}/bin/g++.exe`)) {
binDir = `${process.env.ChocolateyInstall ?? "C:/ProgramData/chocolatey"}/bin`
}
if (binDir !== undefined) {
@ -171,19 +172,19 @@ async function activateGcc(version: string, binDir: string) {
promises.push(addEnv("CC", `${binDir}/gcc-${majorVersion}`), addEnv("CXX", `${binDir}/g++-${majorVersion}`))
if (isUbuntu()) {
updateAptAlternatives("cc", `${binDir}/gcc-${majorVersion}`)
updateAptAlternatives("cxx", `${binDir}/g++-${majorVersion}`)
updateAptAlternatives("gcc", `${binDir}/gcc-${majorVersion}`)
updateAptAlternatives("g++", `${binDir}/g++-${majorVersion}`)
await updateAptAlternatives("cc", `${binDir}/gcc-${majorVersion}`)
await updateAptAlternatives("cxx", `${binDir}/g++-${majorVersion}`)
await updateAptAlternatives("gcc", `${binDir}/gcc-${majorVersion}`)
await updateAptAlternatives("g++", `${binDir}/g++-${majorVersion}`)
}
} else {
promises.push(addEnv("CC", `${binDir}/gcc-${version}`), addEnv("CXX", `${binDir}/g++-${version}`))
if (isUbuntu()) {
updateAptAlternatives("cc", `${binDir}/gcc-${version}`)
updateAptAlternatives("cxx", `${binDir}/g++-${version}`)
updateAptAlternatives("gcc", `${binDir}/gcc-${version}`)
updateAptAlternatives("g++", `${binDir}/g++-${version}`)
await updateAptAlternatives("cc", `${binDir}/gcc-${version}`)
await updateAptAlternatives("cxx", `${binDir}/g++-${version}`)
await updateAptAlternatives("gcc", `${binDir}/gcc-${version}`)
await updateAptAlternatives("g++", `${binDir}/g++-${version}`)
}
}
}
@ -191,15 +192,15 @@ async function activateGcc(version: string, binDir: string) {
promises.push(setupMacOSSDK())
if (ciDetect() === "github-actions") {
addGccLoggingMatcher()
await addGccLoggingMatcher()
}
await Promise.all(promises)
}
function addGccLoggingMatcher() {
async function addGccLoggingMatcher() {
const matcherPath = join(__dirname, "gcc_matcher.json")
if (!existsSync(matcherPath)) {
if (!(await pathExists(matcherPath))) {
return warning("the gcc_matcher.json file does not exist in the same folder as setup_cpp.js")
}
info(`::add-matcher::${matcherPath}`)

View File

@ -7,13 +7,14 @@ import { setupMacOSSDK } from "../macos-sdk/macos-sdk"
import { addEnv } from "../utils/env/addEnv"
import { setupAptPack, updateAptAlternatives } from "../utils/setup/setupAptPack"
import { info, warning } from "ci-log"
import { existsSync } from "fs"
import ciDetect from "@npmcli/ci-detect"
import { setupGcc } from "../gcc/gcc"
import { getVersion } from "../versions/versions"
import { isUbuntu } from "../utils/env/isUbuntu"
import { getLLVMPackageInfo } from "./llvm_url"
import { ubuntuVersion } from "../utils/env/ubuntu_version"
import { pathExists } from "path-exists"
export async function setupLLVM(version: string, setupDir: string, arch: string): Promise<InstallationInfo> {
const installationInfo = await setupLLVMWithoutActivation(version, setupDir, arch)
@ -94,41 +95,41 @@ export async function activateLLVM(directory: string, versionGiven: string) {
// windows builds fail with llvm's CPATH
if (process.platform !== "win32") {
const llvmMajor = semverMajor(version)
if (existsSync(`${directory}/lib/clang/${version}/include`)) {
if (await pathExists(`${directory}/lib/clang/${version}/include`)) {
promises.push(addEnv("CPATH", `${directory}/lib/clang/${version}/include`))
} else if (existsSync(`${directory}/lib/clang/${llvmMajor}/include`)) {
} else if (await pathExists(`${directory}/lib/clang/${llvmMajor}/include`)) {
promises.push(addEnv("CPATH", `${directory}/lib/clang/${llvmMajor}/include`))
}
}
if (isUbuntu()) {
updateAptAlternatives("cc", `${directory}/bin/clang`)
updateAptAlternatives("cxx", `${directory}/bin/clang++`)
updateAptAlternatives("clang", `${directory}/bin/clang`)
updateAptAlternatives("clang++", `${directory}/bin/clang++`)
updateAptAlternatives("lld", `${directory}/bin/lld`)
updateAptAlternatives("ld.lld", `${directory}/bin/ld.lld`)
updateAptAlternatives("llvm-ar", `${directory}/bin/llvm-ar`)
await updateAptAlternatives("cc", `${directory}/bin/clang`)
await updateAptAlternatives("cxx", `${directory}/bin/clang++`)
await updateAptAlternatives("clang", `${directory}/bin/clang`)
await updateAptAlternatives("clang++", `${directory}/bin/clang++`)
await updateAptAlternatives("lld", `${directory}/bin/lld`)
await updateAptAlternatives("ld.lld", `${directory}/bin/ld.lld`)
await updateAptAlternatives("llvm-ar", `${directory}/bin/llvm-ar`)
}
if (ciDetect() === "github-actions") {
addLLVMLoggingMatcher()
await addLLVMLoggingMatcher()
}
await Promise.all(promises)
}
/** Setup llvm tools (clang tidy, clang format, etc) without activating llvm and using it as the compiler */
export function setupClangTools(version: string, setupDir: string, arch: string): Promise<InstallationInfo> {
export async function setupClangTools(version: string, setupDir: string, arch: string): Promise<InstallationInfo> {
if (ciDetect() === "github-actions") {
addLLVMLoggingMatcher()
await addLLVMLoggingMatcher()
}
return setupLLVMWithoutActivation(version, setupDir, arch)
}
function addLLVMLoggingMatcher() {
async function addLLVMLoggingMatcher() {
const matcherPath = join(__dirname, "llvm_matcher.json")
if (!existsSync(matcherPath)) {
if (!(await pathExists(matcherPath))) {
return warning("the llvm_matcher.json file does not exist in the same folder as setup_cpp.js")
}
info(`::add-matcher::${matcherPath}`)

View File

@ -270,7 +270,7 @@ export async function main(args: string[]): Promise<number> {
info(`took ${timeFormatter.format(time1, time2) || "0 seconds"}`)
}
finalizeCpprc()
await finalizeCpprc()
if (successMessages.length === 0 && errorMessages.length === 0) {
warning("setup_cpp was called without any arguments. Nothing to do.")

View File

@ -5,8 +5,9 @@ import { setupVCVarsall } from "../vcvarsall/vcvarsall"
import { vsversion_to_versionnumber, findVcvarsall } from "msvc-dev-cmd/lib.js"
import ciDetect from "@npmcli/ci-detect"
import { join } from "patha"
import { existsSync } from "fs"
import { error, info, warning } from "ci-log"
import { pathExists } from "path-exists"
type MSVCVersion = "2022" | "17.0" | "2019" | "16.0" | "2017" | "15.0" | "2015" | "14.0" | "2013" | "12.0" | string
@ -66,13 +67,13 @@ export async function setupMSVC(
await setupVCVarsall(version, VCTargetsPath, arch, toolset, sdk, uwp, spectre)
if (ciDetect() === "github-actions") {
addMSVCLoggingMatcher()
await addMSVCLoggingMatcher()
}
}
function addMSVCLoggingMatcher() {
async function addMSVCLoggingMatcher() {
const matcherPath = join(__dirname, "msvc_matcher.json")
if (!existsSync(matcherPath)) {
if (!(await pathExists(matcherPath))) {
return warning("the msvc_matcher.json file does not exist in the same folder as setup_cpp.js")
}
info(`::add-matcher::${matcherPath}`)

View File

@ -1,12 +1,13 @@
import { useCpythonVersion } from "setup-python/src/find-python"
import { findPyPyVersion } from "setup-python/src/find-pypy"
import { existsSync } from "fs"
import { info, warning } from "ci-log"
import { debug } from "@actions/core"
import { join } from "patha"
import ciDetect from "@npmcli/ci-detect"
import { isCacheFeatureAvailable, IS_MAC } from "setup-python/src/utils"
import { getCacheDistributor } from "setup-python/src/cache-distributions/cache-factory"
import { pathExists } from "path-exists"
function isPyPyVersion(versionSpec: string) {
return versionSpec.startsWith("pypy")
@ -51,15 +52,15 @@ export async function setupActionsPython(version: string, _setupDir: string, arc
}
if (ciDetect() === "github-actions") {
addPythonLoggingMatcher()
await addPythonLoggingMatcher()
}
return undefined
}
function addPythonLoggingMatcher() {
async function addPythonLoggingMatcher() {
const matcherPath = join(__dirname, "python_matcher.json")
if (!existsSync(matcherPath)) {
if (!(await pathExists(matcherPath))) {
return warning("the python_matcher.json file does not exist in the same folder as setup_cpp.js")
}
info(`::add-matcher::${matcherPath}`)

View File

@ -1,13 +1,14 @@
import { exportVariable, addPath as ghAddPath, info, setFailed } from "@actions/core"
import ciDetect from "@npmcli/ci-detect"
import { untildifyUser } from "untildify-user"
import { appendFileSync, existsSync, readFileSync, writeFileSync } from "fs"
import { appendFileSync, readFileSync, writeFileSync } from "fs"
import { error, warning } from "ci-log"
import { execPowershell } from "exec-powershell"
import { delimiter } from "path"
import escapeSpace from "escape-path-with-spaces"
import { giveUserAccess } from "user-access"
import escapeQuote from "escape-quotes"
import { pathExists } from "path-exists"
/**
* Add an environment variable.
@ -75,7 +76,7 @@ async function addEnvSystem(name: string, valGiven: string | undefined) {
}
case "linux":
case "darwin": {
setupCppInProfile()
await setupCppInProfile()
appendFileSync(cpprc_path, `\nexport ${name}="${val}"\n`)
info(`${name}="${val}" was added to "${cpprc_path}`)
return
@ -99,7 +100,7 @@ async function addPathSystem(path: string) {
}
case "linux":
case "darwin": {
setupCppInProfile()
await setupCppInProfile()
appendFileSync(cpprc_path, `\nexport PATH="${path}:$PATH"\n`)
info(`"${path}" was added to "${cpprc_path}"`)
return
@ -113,7 +114,7 @@ async function addPathSystem(path: string) {
let setupCppInProfile_called = false
/// handles adding conditions to source .cpprc file from .bashrc and .profile
export function setupCppInProfile() {
export async function setupCppInProfile() {
if (setupCppInProfile_called) {
return
}
@ -121,7 +122,7 @@ export function setupCppInProfile() {
// a variable that prevents source_cpprc from being called from .bashrc and .profile
const source_cpprc_str = "# Automatically Generated by setup-cpp\nexport SOURCE_CPPRC=0"
if (existsSync(cpprc_path)) {
if (await pathExists(cpprc_path)) {
const cpprc_content = readFileSync(cpprc_path, "utf8")
if (cpprc_content.includes(source_cpprc_str)) {
// already executed setupCppInProfile
@ -153,8 +154,8 @@ export function setupCppInProfile() {
setupCppInProfile_called = true
}
export function finalizeCpprc() {
if (existsSync(cpprc_path)) {
export async function finalizeCpprc() {
if (await pathExists(cpprc_path)) {
const entries = readFileSync(cpprc_path, "utf-8").split("\n")
const unique_entries = [...new Set(entries.reverse())].reverse() // remove duplicates, keeping the latest entry

View File

@ -79,8 +79,8 @@ async function initApt(apt: string) {
"ca-certificates",
"gnupg",
])
addAptKeyViaServer(["3B4FE6ACC0B21F32", "40976EAF437D05B5"], "setup-cpp-ubuntu-archive.gpg")
addAptKeyViaServer(["1E9377A2BA9EF27F"], "launchpad-toolchain.gpg")
await addAptKeyViaServer(["3B4FE6ACC0B21F32", "40976EAF437D05B5"], "setup-cpp-ubuntu-archive.gpg")
await addAptKeyViaServer(["1E9377A2BA9EF27F"], "launchpad-toolchain.gpg")
if (apt === "nala") {
// enable utf8 otherwise it fails because of the usage of ASCII encoding
await addEnv("LANG", "C.UTF-8")
@ -92,9 +92,9 @@ function initGpg() {
execRootSync("gpg", ["-k"])
}
export function addAptKeyViaServer(keys: string[], name: string, server = "keyserver.ubuntu.com") {
export async function addAptKeyViaServer(keys: string[], name: string, server = "keyserver.ubuntu.com") {
const fileName = `/etc/apt/trusted.gpg.d/${name}`
if (!existsSync(fileName)) {
if (!(await pathExists(fileName))) {
initGpg()
for (const key of keys) {
execRootSync("gpg", [
@ -114,7 +114,7 @@ export function addAptKeyViaServer(keys: string[], name: string, server = "keyse
export async function addAptKeyViaDownload(name: string, url: string) {
const fileName = `/etc/apt/trusted.gpg.d/${name}`
if (!existsSync(fileName)) {
if (!(await pathExists(fileName))) {
initGpg()
await setupAptPack("curl", undefined)
execRootSync("bash", ["-c", `curl -s ${url} | gpg --no-default-keyring --keyring gnupg-ring:${fileName} --import`])

View File

@ -2,7 +2,7 @@ import { find, downloadTool, cacheDir } from "@actions/tool-cache"
import { info } from "@actions/core"
import { addPath } from "../env/addEnv"
import { join } from "patha"
import { existsSync } from "fs"
import { tmpdir } from "os"
import ciDetect from "@npmcli/ci-detect"
import { setupAptPack } from "./setupAptPack"
@ -11,6 +11,7 @@ import { isArch } from "../env/isArch"
import { hasDnf } from "../env/hasDnf"
import { setupDnfPack } from "./setupDnfPack"
import { isUbuntu } from "../env/isUbuntu"
import { pathExists } from "path-exists"
/** A type that describes a package */
export type PackageInfo = {
@ -70,7 +71,7 @@ export async function setupBin(
if (dir) {
const installDir = join(dir, extractedFolderName)
const binDir = join(installDir, binRelativeDir)
if (existsSync(binDir) && existsSync(join(binDir, binFileName))) {
if (await pathExists(join(binDir, binFileName))) {
info(`${name} ${version} was found in the cache at ${binDir}.`)
await addPath(binDir)
@ -87,7 +88,7 @@ export async function setupBin(
const binFile = join(binDir, binFileName)
// download ane extract the package into the installation directory.
if (!existsSync(binDir) || !existsSync(binFile)) {
if ((await Promise.all([pathExists(binDir), pathExists(binFile)])).includes(false)) {
info(`Download and extract ${name} ${version}`)
if (!didInit) {

View File

@ -3,10 +3,11 @@ import execa from "execa"
import { info } from "@actions/core"
import { addPythonBaseExecPrefix, setupPythonAndPip } from "../../python/python"
import { InstallationInfo } from "./setupBin"
import { existsSync } from "fs"
import { addExeExt, dirname, join } from "patha"
import { addPath } from "../env/addEnv"
import which from "which"
import { pathExists } from "path-exists"
let python: string | undefined
let binDirs: string[] | undefined
@ -27,15 +28,18 @@ export async function setupPipPack(name: string, version?: string): Promise<Inst
binDirs = await addPythonBaseExecPrefix(python)
}
const binDir = findBinDir(binDirs, name)
const binDir = await findBinDir(binDirs, name)
await addPath(binDir)
return { binDir }
}
function findBinDir(dirs: string[], name: string) {
const foundDir = dirs.find((dir) => existsSync(join(dir, addExeExt(name))))
async function findBinDir(dirs: string[], name: string) {
const exists = await Promise.all(dirs.map((dir) => pathExists(join(dir, addExeExt(name)))))
const dirIndex = exists.findIndex((exist) => exist)
const foundDir = dirs[dirIndex]
if (foundDir !== undefined) {
return foundDir
}

View File

@ -3,7 +3,8 @@ import { tmpdir } from "os"
import * as path from "patha"
import { addExeExt, join } from "patha"
import spawn from "cross-spawn"
import { existsSync } from "fs"
import { pathExists } from "path-exists"
export async function setupTmpDir(testName: string) {
const tempDirectory = path.join(tmpdir(), "setup cpp temp", testName)
@ -39,7 +40,7 @@ export async function testBin(
console.log(`Testing the existence of ${binDir}`)
expect(binDir).toBeDefined()
expect(binDir).not.toHaveLength(0)
expect(existsSync(binDir)).toBeTruthy()
expect(await pathExists(binDir)).toBeTruthy()
bin = join(binDir, addExeExt(name))
}

View File

@ -1,5 +1,5 @@
import execa from "execa"
import { existsSync } from "fs"
import { dirname, join, addShExt, addShRelativePrefix } from "patha"
import which from "which"
import { addPath } from "../utils/env/addEnv"
@ -12,6 +12,7 @@ import { hasDnf } from "../utils/env/hasDnf"
import { setupDnfPack } from "../utils/setup/setupDnfPack"
import { isUbuntu } from "../utils/env/isUbuntu"
import { giveUserAccess } from "user-access"
import { pathExists } from "path-exists"
let hasVCPKG = false
@ -44,7 +45,7 @@ export async function setupVcpkg(_version: string, setupDir: string, _arch: stri
}
}
if (!existsSync(join(setupDir, addShExt("bootstrap-vcpkg", ".bat")))) {
if (!(await pathExists(join(setupDir, addShExt("bootstrap-vcpkg", ".bat"))))) {
execa.sync("git", ["clone", "https://github.com/microsoft/vcpkg"], { cwd: dirname(setupDir), stdio: "inherit" })
} else {
notice(`Vcpkg folder already exists at ${setupDir}. This might mean that ~/vcpkg is restored from the cache.`)

View File

@ -1,9 +1,9 @@
import { existsSync } from "fs"
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import { setupMSVCDevCmd } from "msvc-dev-cmd/lib.js"
import { addEnv } from "../utils/env/addEnv"
import { info } from "ci-log"
import { pathExists } from "path-exists"
function getArch(arch: string): string {
switch (arch) {
@ -30,7 +30,7 @@ export async function setupVCVarsall(
uwp?: boolean,
spectre?: boolean
) {
if (VCTargetsPath !== undefined && existsSync(VCTargetsPath)) {
if (VCTargetsPath !== undefined && (await pathExists(VCTargetsPath))) {
info(`Adding ${VCTargetsPath} to PATH`)
await addEnv("VCTargetsPath", VCTargetsPath)
}