Merge pull request #141 from aminya/faster-llvm [skip ci]

This commit is contained in:
Amin Yahyaabadi 2022-11-08 16:47:28 -08:00 committed by GitHub
commit 8c9d48b9aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 319 additions and 304 deletions

View File

@ -8,6 +8,7 @@ ignorePaths:
- .git/
- dist/
- dev/cpp_vcpkg_project
- "**/node_modules/"
words:
- aarch
- aminya

View File

@ -1,3 +1,4 @@
/* eslint-disable import/no-extraneous-dependencies */
import { node } from "execa"
function getPlatformName() {

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -29,9 +29,9 @@
"docs": "shx rm -rf packages/*/README.md && pnpm -r exec readme --path ../../dev/readme/template.md -y && pnpm -r exec ts-readme",
"format": "run-s lint.prettier",
"lint": "run-p --aggregate-output --continue-on-error lint.cspell lint.eslint lint.prettier lint.tsc",
"lint.cspell": "cspell lint --no-progress --show-suggestions",
"lint.cspell": "cspell lint --no-progress --show-suggestions --cache --cache-location ./.cache/cspell/.cspellcache",
"lint.eslint": "eslint **/*.{ts,tsx,js,jsx,cjs,mjs,json,yaml} --no-error-on-unmatched-pattern --cache --cache-location ./.cache/eslint/ --fix",
"lint.prettier": "prettier --write .",
"lint.prettier": "prettier --list-different --write .",
"lint.tsc": "tsc --noEmit",
"pack.exe": "shx rm -rf ./dist/tsconfig.tsbuildinfo && ts-node --esm ./dev/scripts/pack-exe.ts",
"prepare": "pnpm run -r build && pnpm run -w build",

View File

@ -1,4 +1,4 @@
import { setupLLVM, VERSIONS, getUrl, setupClangTools, getLinuxUrl } from "../llvm"
import { setupLLVM, setupClangTools } from "../llvm"
import { getSpecificVersionAndUrl } from "../../utils/setup/version"
import { isUrlOnline } from "is-url-online"
import { setupTmpDir, testBin } from "../../utils/tests/test-helpers"
@ -9,6 +9,7 @@ import { chmodSync } from "fs"
import { getVersion } from "../../versions/versions"
import { ubuntuVersion } from "../../utils/env/ubuntu_version"
import * as io from "@actions/io"
import { getLinuxUrl, getUrl, VERSIONS } from "../llvm_url"
jest.setTimeout(400000)
async function testUrl(version: string) {

View File

@ -1,321 +1,59 @@
import { join, addExeExt } from "patha"
import { delimiter } from "path"
import semverLte from "semver/functions/lte"
import semverMajor from "semver/functions/major"
import { isUrlOnline } from "is-url-online"
import { InstallationInfo, PackageInfo, setupBin } from "../utils/setup/setupBin"
import { extractExe, extractTarByExe } from "../utils/setup/extract"
import {
getSpecificVersionAndUrl,
getSpecificVersions,
getVersions,
semverCoerceIfInvalid,
} from "../utils/setup/version"
import { InstallationInfo, setupBin } from "../utils/setup/setupBin"
import { semverCoerceIfInvalid } from "../utils/setup/version"
import { setupMacOSSDK } from "../macos-sdk/macos-sdk"
import { addEnv } from "../utils/env/addEnv"
import { setOutput } from "@actions/core"
import { setupAptPack, updateAptAlternatives } from "../utils/setup/setupAptPack"
import { info, warning } from "ci-log"
import { existsSync } from "fs"
import ciDetect from "@npmcli/ci-detect"
import { setupGcc } from "../gcc/gcc"
import { getVersion } from "../versions/versions"
import { isArch } from "../utils/env/isArch"
import { isUbuntu } from "../utils/env/isUbuntu"
//================================================
// Version
//================================================
/** The specific and minimum LLVM versions supported by this action. */
export const VERSIONS: Set<string> = getVersions([
"3.5.0",
"3.5.1",
"3.5.2",
"3.6.0",
"3.6.1",
"3.6.2",
"3.7.0",
"3.7.1",
"3.8.0",
"3.8.1",
"3.9.0",
"3.9.1",
"4.0.0",
"4.0.1",
"5.0.0",
"5.0.1",
"5.0.2",
"6.0.0",
"6.0.1",
"7.0.0",
"7.0.1",
"7.1.0",
"8.0.0",
"8.0.1",
"9.0.0",
"9.0.1",
"10.0.0",
"10.0.1",
"11.0.0",
"11.0.1",
"11.1.0",
"12.0.0",
"12.0.1",
"13.0.0",
"13.0.1",
"14.0.0",
"14.0.1",
"14.0.2",
"14.0.3",
"14.0.4",
"14.0.5",
"14.0.6",
"15.0.0",
"15.0.1",
"15.0.2",
])
//================================================
// URL
//================================================
/** Gets a LLVM download URL for GitHub. */
function getGitHubUrl(version: string, prefix: string, suffix: string): string {
const file = `${prefix}${version}${suffix}`
return `https://github.com/llvm/llvm-project/releases/download/llvmorg-${version}/${file}`
}
/** Gets a LLVM download URL for https://releases.llvm.org. */
function getReleaseUrl(version: string, prefix: string, suffix: string): string {
const file = `${prefix}${version}${suffix}`
return `https://releases.llvm.org/${version}/${file}`
}
/** The LLVM versions that were never released for the Darwin platform. */
const DARWIN_MISSING: Set<string> = new Set([
"3.5.1",
"3.6.1",
"3.6.2",
"3.7.1",
"3.8.1",
"3.9.1",
"6.0.1",
"7.0.1",
"7.1.0",
"8.0.1",
"11.0.1",
"11.1.0",
"12.0.1",
])
/** Gets an LLVM download URL for the Darwin platform. */
function getDarwinUrl(version: string): string | null {
if (DARWIN_MISSING.has(version)) {
return null
}
const darwin = version === "9.0.0" ? "-darwin-apple" : "-apple-darwin"
const prefix = "clang+llvm-"
const suffix = `-x86_64${darwin}.tar.xz`
if (semverLte(version, "9.0.1")) {
return getReleaseUrl(version, prefix, suffix)
} else {
return getGitHubUrl(version, prefix, suffix)
}
}
/**
* The LLVM versions that should use the last RC version instead of the release version for the Linux (Ubuntu) platform.
* This is useful when there were binaries released for the Linux (Ubuntu) platform for the last RC version but not for
* the actual release version.
*/
const UBUNTU_RC: Map<string, string> = new Map()
/**
* The (latest) Ubuntu versions for each LLVM version.
*
* https://github.com/llvm/llvm-project/releases/tag/llvmorg-14.0.1 or https://releases.llvm.org/14.0.1
*/
// TODO change based on ubuntu version
const UBUNTU_SUFFIX_MAP: { [key: string]: string } = {
"3.5.0": "-ubuntu-14.04",
"3.5.1": "",
"3.5.2": "-ubuntu-14.04",
"3.6.0": "-ubuntu-14.04",
"3.6.1": "-ubuntu-14.04",
"3.6.2": "-ubuntu-14.04",
"3.7.0": "-ubuntu-14.04",
"3.7.1": "-ubuntu-14.04",
"3.8.0": "-ubuntu-16.04",
"3.8.1": "-ubuntu-16.04",
"3.9.0": "-ubuntu-16.04",
"3.9.1": "-ubuntu-16.04",
"4.0.0": "-ubuntu-16.04",
"5.0.0": "-ubuntu16.04",
"5.0.1": "-ubuntu-16.04",
"5.0.2": "-ubuntu-16.04",
"6.0.0": "-ubuntu-16.04",
"6.0.1": "-ubuntu-16.04",
"7.0.0": "-ubuntu-16.04",
"7.0.1": "-ubuntu-18.04",
"7.1.0": "-ubuntu-14.04",
"8.0.0": "-ubuntu-18.04",
"9.0.0": "-ubuntu-18.04",
"9.0.1": "-ubuntu-16.04",
"10.0.0": "-ubuntu-18.04",
"10.0.1": "-ubuntu-16.04",
"11.0.0": "-ubuntu-20.04",
"11.0.1": "-ubuntu-16.04",
"11.1.0": "-ubuntu-16.04",
"12.0.0": "-ubuntu-20.04",
"12.0.1": "-ubuntu-16.04",
"13.0.0": "-ubuntu-20.04",
"13.0.0-ubuntu-16.04": "-ubuntu-16.04",
"13.0.0-ubuntu-20.04": "-ubuntu-20.04",
"13.0.1": "-ubuntu-18.04",
"13.0.1-ubuntu-18.04": "-ubuntu-18.04",
"14.0.0": "-ubuntu-18.04",
// "14.0.1": "-ubuntu-18.04", // only available for powerpc64le
"15.0.2": "-rhel86",
}
/** The latest supported LLVM version for the Linux (Ubuntu) platform. */
const MAX_UBUNTU: string = "15.0.2"
/** Gets an LLVM download URL for the Linux (Ubuntu) platform. */
export function getLinuxUrl(versionGiven: string): string {
let version = versionGiven
const rc = UBUNTU_RC.get(version)
if (rc !== undefined) {
version = rc
}
let linuxVersion: string
// ubuntu-version is specified
if (version.includes("ubuntu")) {
const givenUbuntuVersion = version.replace(/-ubuntu-.*/, "")
if (!VERSIONS.has(givenUbuntuVersion)) {
throw new Error(`Unsupported Ubuntu version: ${givenUbuntuVersion}`)
}
linuxVersion = version.replace(givenUbuntuVersion, "")
version = getSpecificVersions(VERSIONS, givenUbuntuVersion)[0]
} else if (version !== "" && version in UBUNTU_SUFFIX_MAP) {
linuxVersion = UBUNTU_SUFFIX_MAP[version]
} else {
// default to the maximum version
linuxVersion = UBUNTU_SUFFIX_MAP[MAX_UBUNTU]
warning(`Falling back to LLVM version ${MAX_UBUNTU} ${linuxVersion} for the Ubuntu.`)
}
const prefix = "clang+llvm-"
let suffix: string
if (version === "5.0.0") {
suffix = `-linux-x86_64${linuxVersion}.tar.xz`
} else if (linuxVersion.includes("-rhel86")) {
suffix = `-x86_64-unknown-linux-gnu${linuxVersion}.tar.xz`
} else {
suffix = `-x86_64-linux-gnu${linuxVersion}.tar.xz`
}
if (semverLte(version, "9.0.1")) {
return getReleaseUrl(version, prefix, suffix)
} else {
return getGitHubUrl(version, prefix, suffix)
}
}
/** The LLVM versions that were never released for the Windows platform. */
const WIN32_MISSING: Set<string> = new Set(["10.0.1"])
/** Gets an LLVM download URL for the Windows platform. */
async function getWin32Url(version: string): Promise<string | null> {
if (WIN32_MISSING.has(version)) {
return null
}
const prefix = "LLVM-"
const suffix = semverLte(version, "3.7.0") ? "-win32.exe" : "-win64.exe"
const olderThan9_1 = semverLte(version, "9.0.1")
let url: string
let fallback = false
if (olderThan9_1) {
url = getReleaseUrl(version, prefix, suffix)
if (!(await isUrlOnline(url))) {
fallback = true // fallback to github
}
}
if (fallback || !olderThan9_1) {
url = getGitHubUrl(version, prefix, suffix)
}
return url!
}
/** Gets an LLVM download URL. */
export function getUrl(platform: string, version: string): string | null | Promise<string | null> {
switch (platform) {
case "darwin":
return getDarwinUrl(version)
case "linux":
return getLinuxUrl(version)
case "win32":
return getWin32Url(version)
default:
return null
}
}
//================================================
// Exports
//================================================
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async function getLLVMPackageInfo(version: string, platform: NodeJS.Platform, _arch: string): Promise<PackageInfo> {
const [specificVersion, url] = await getSpecificVersionAndUrl(VERSIONS, platform, version, getUrl)
setOutput("version", specificVersion)
return {
url,
extractedFolderName: "",
binRelativeDir: "bin",
binFileName: addExeExt("clang"),
extractFunction:
platform === "win32"
? extractExe
: (file: string, dest: string) => {
return extractTarByExe(file, dest, ["--strip-components=1"])
},
}
}
import { getLLVMPackageInfo } from "./llvm_url"
export async function setupLLVM(version: string, setupDir: string, arch: string): Promise<InstallationInfo> {
const installationInfo = await _setupLLVM(version, setupDir, arch)
const installationInfo = await setupLLVMWithoutActivation(version, setupDir, arch)
await activateLLVM(installationInfo.installDir ?? setupDir, version)
return installationInfo
}
let didInit = false
async function _setupLLVM(version: string, setupDir: string, arch: string) {
const installationInfo = await setupBin("llvm", version, getLLVMPackageInfo, setupDir, arch)
if (!didInit) {
if (process.platform === "linux") {
// install llvm build dependencies
await setupGcc(getVersion("gcc", undefined), "", arch) // using llvm requires ld, an up to date libstdc++, etc. So, install gcc first
if (isArch()) {
// setupPacmanPack("ncurses")
// TODO: install libtinfo ?
} else if (isUbuntu()) {
await setupAptPack("libtinfo-dev")
}
}
let installedDeps = false
async function setupLLVMWithoutActivation(version: string, setupDir: string, arch: string) {
const installationInfoPromise = setupBin("llvm", version, getLLVMPackageInfo, setupDir, arch)
let depsPromise: Promise<void>
if (!installedDeps) {
depsPromise = setupLLVMDeps(arch)
// eslint-disable-next-line require-atomic-updates
didInit = true
installedDeps = true
} else {
depsPromise = Promise.resolve()
}
// install LLVM and its dependencies in parallel
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const [installationInfo, _] = await Promise.all([installationInfoPromise, depsPromise])
return installationInfo
}
async function setupLLVMDeps(arch: string) {
if (process.platform === "linux") {
// install llvm build dependencies
await setupGcc(getVersion("gcc", undefined), "", arch) // using llvm requires ld, an up to date libstdc++, etc. So, install gcc first
if (isUbuntu()) {
await setupAptPack("libtinfo-dev")
}
// TODO: install libtinfo on other distros
// setupPacmanPack("ncurses")
}
}
export async function activateLLVM(directory: string, versionGiven: string) {
const version = semverCoerceIfInvalid(versionGiven)
@ -378,7 +116,7 @@ export function setupClangTools(version: string, setupDir: string, arch: string)
if (ciDetect() === "github-actions") {
addLLVMLoggingMatcher()
}
return _setupLLVM(version, setupDir, arch)
return setupLLVMWithoutActivation(version, setupDir, arch)
}
function addLLVMLoggingMatcher() {

275
src/llvm/llvm_url.ts Normal file
View File

@ -0,0 +1,275 @@
import semverLte from "semver/functions/lte"
import { isUrlOnline } from "is-url-online"
import { getSpecificVersionAndUrl, getSpecificVersions, getVersions } from "../utils/setup/version"
import { info, warning } from "ci-log"
import { PackageInfo } from "../utils/setup/setupBin"
import { addExeExt } from "patha"
import { extractExe, extractTarByExe } from "../utils/setup/extract"
//================================================
// Version
//================================================
/** The specific and minimum LLVM versions supported by this action. */
export const VERSIONS: Set<string> = getVersions([
"3.5.0",
"3.5.1",
"3.5.2",
"3.6.0",
"3.6.1",
"3.6.2",
"3.7.0",
"3.7.1",
"3.8.0",
"3.8.1",
"3.9.0",
"3.9.1",
"4.0.0",
"4.0.1",
"5.0.0",
"5.0.1",
"5.0.2",
"6.0.0",
"6.0.1",
"7.0.0",
"7.0.1",
"7.1.0",
"8.0.0",
"8.0.1",
"9.0.0",
"9.0.1",
"10.0.0",
"10.0.1",
"11.0.0",
"11.0.1",
"11.1.0",
"12.0.0",
"12.0.1",
"13.0.0",
"13.0.1",
"14.0.0",
"14.0.1",
"14.0.2",
"14.0.3",
"14.0.4",
"14.0.5",
"14.0.6",
"15.0.0",
"15.0.1",
"15.0.2",
])
/** The LLVM versions that were never released for the Darwin platform. */
const DARWIN_MISSING: Set<string> = new Set([
"3.5.1",
"3.6.1",
"3.6.2",
"3.7.1",
"3.8.1",
"3.9.1",
"6.0.1",
"7.0.1",
"7.1.0",
"8.0.1",
"11.0.1",
"11.1.0",
"12.0.1",
])
/**
* The LLVM versions that should use the last RC version instead of the release version for the Linux (Ubuntu) platform.
* This is useful when there were binaries released for the Linux (Ubuntu) platform for the last RC version but not for
* the actual release version.
*/
const UBUNTU_RC: Map<string, string> = new Map()
/**
* The (latest) Ubuntu versions for each LLVM version.
*
* https://github.com/llvm/llvm-project/releases/tag/llvmorg-14.0.1 or https://releases.llvm.org/14.0.1
*/
// TODO change based on ubuntu version
const UBUNTU_SUFFIX_MAP: { [key: string]: string } = {
"3.5.0": "-ubuntu-14.04",
"3.5.1": "",
"3.5.2": "-ubuntu-14.04",
"3.6.0": "-ubuntu-14.04",
"3.6.1": "-ubuntu-14.04",
"3.6.2": "-ubuntu-14.04",
"3.7.0": "-ubuntu-14.04",
"3.7.1": "-ubuntu-14.04",
"3.8.0": "-ubuntu-16.04",
"3.8.1": "-ubuntu-16.04",
"3.9.0": "-ubuntu-16.04",
"3.9.1": "-ubuntu-16.04",
"4.0.0": "-ubuntu-16.04",
"5.0.0": "-ubuntu16.04",
"5.0.1": "-ubuntu-16.04",
"5.0.2": "-ubuntu-16.04",
"6.0.0": "-ubuntu-16.04",
"6.0.1": "-ubuntu-16.04",
"7.0.0": "-ubuntu-16.04",
"7.0.1": "-ubuntu-18.04",
"7.1.0": "-ubuntu-14.04",
"8.0.0": "-ubuntu-18.04",
"9.0.0": "-ubuntu-18.04",
"9.0.1": "-ubuntu-16.04",
"10.0.0": "-ubuntu-18.04",
"10.0.1": "-ubuntu-16.04",
"11.0.0": "-ubuntu-20.04",
"11.0.1": "-ubuntu-16.04",
"11.1.0": "-ubuntu-16.04",
"12.0.0": "-ubuntu-20.04",
"12.0.1": "-ubuntu-16.04",
"13.0.0": "-ubuntu-20.04",
"13.0.0-ubuntu-16.04": "-ubuntu-16.04",
"13.0.0-ubuntu-20.04": "-ubuntu-20.04",
"13.0.1": "-ubuntu-18.04",
"13.0.1-ubuntu-18.04": "-ubuntu-18.04",
"14.0.0": "-ubuntu-18.04",
// "14.0.1": "-ubuntu-18.04", // only available for powerpc64le
"15.0.2": "-rhel86",
}
/** The latest supported LLVM version for the Linux (Ubuntu) platform. */
const MAX_UBUNTU: string = "15.0.2"
//================================================
// URL
//================================================
/** Gets a LLVM download URL for GitHub. */
function getGitHubUrl(version: string, prefix: string, suffix: string): string {
const file = `${prefix}${version}${suffix}`
return `https://github.com/llvm/llvm-project/releases/download/llvmorg-${version}/${file}`
}
/** Gets a LLVM download URL for https://releases.llvm.org. */
function getReleaseUrl(version: string, prefix: string, suffix: string): string {
const file = `${prefix}${version}${suffix}`
return `https://releases.llvm.org/${version}/${file}`
}
/** Gets an LLVM download URL for the Darwin platform. */
function getDarwinUrl(version: string): string | null {
if (DARWIN_MISSING.has(version)) {
return null
}
const darwin = version === "9.0.0" ? "-darwin-apple" : "-apple-darwin"
const prefix = "clang+llvm-"
const suffix = `-x86_64${darwin}.tar.xz`
if (semverLte(version, "9.0.1")) {
return getReleaseUrl(version, prefix, suffix)
} else {
return getGitHubUrl(version, prefix, suffix)
}
}
/** Gets an LLVM download URL for the Linux (Ubuntu) platform. */
export function getLinuxUrl(versionGiven: string): string {
let version = versionGiven
const rc = UBUNTU_RC.get(version)
if (rc !== undefined) {
version = rc
}
let linuxVersion: string
// ubuntu-version is specified
if (version.includes("ubuntu")) {
const givenUbuntuVersion = version.replace(/-ubuntu-.*/, "")
if (!VERSIONS.has(givenUbuntuVersion)) {
throw new Error(`Unsupported Ubuntu version: ${givenUbuntuVersion}`)
}
linuxVersion = version.replace(givenUbuntuVersion, "")
version = getSpecificVersions(VERSIONS, givenUbuntuVersion)[0]
} else if (version !== "" && version in UBUNTU_SUFFIX_MAP) {
linuxVersion = UBUNTU_SUFFIX_MAP[version]
} else {
// default to the maximum version
linuxVersion = UBUNTU_SUFFIX_MAP[MAX_UBUNTU]
warning(`Falling back to LLVM version ${MAX_UBUNTU} ${linuxVersion} for the Ubuntu.`)
}
const prefix = "clang+llvm-"
let suffix: string
if (version === "5.0.0") {
suffix = `-linux-x86_64${linuxVersion}.tar.xz`
} else if (linuxVersion.includes("-rhel86")) {
suffix = `-x86_64-unknown-linux-gnu${linuxVersion}.tar.xz`
} else {
suffix = `-x86_64-linux-gnu${linuxVersion}.tar.xz`
}
if (semverLte(version, "9.0.1")) {
return getReleaseUrl(version, prefix, suffix)
} else {
return getGitHubUrl(version, prefix, suffix)
}
}
/** The LLVM versions that were never released for the Windows platform. */
const WIN32_MISSING: Set<string> = new Set(["10.0.1"])
/** Gets an LLVM download URL for the Windows platform. */
async function getWin32Url(version: string): Promise<string | null> {
if (WIN32_MISSING.has(version)) {
return null
}
const prefix = "LLVM-"
const suffix = semverLte(version, "3.7.0") ? "-win32.exe" : "-win64.exe"
const olderThan9_1 = semverLte(version, "9.0.1")
let url: string
let fallback = false
if (olderThan9_1) {
url = getReleaseUrl(version, prefix, suffix)
if (!(await isUrlOnline(url))) {
fallback = true // fallback to github
}
}
if (fallback || !olderThan9_1) {
url = getGitHubUrl(version, prefix, suffix)
}
return url!
}
/** Gets an LLVM download URL. */
export function getUrl(platform: string, version: string): string | null | Promise<string | null> {
switch (platform) {
case "darwin":
return getDarwinUrl(version)
case "linux":
return getLinuxUrl(version)
case "win32":
return getWin32Url(version)
default:
return null
}
}
export async function getLLVMPackageInfo(
version: string,
platform: NodeJS.Platform,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
_arch: string
): Promise<PackageInfo> {
const [specificVersion, url] = await getSpecificVersionAndUrl(VERSIONS, platform, version, getUrl)
info(`specific llvm version: ${specificVersion}`)
return {
url,
extractedFolderName: "",
binRelativeDir: "bin",
binFileName: addExeExt("clang"),
extractFunction:
platform === "win32"
? extractExe
: (file: string, dest: string) => {
return extractTarByExe(file, dest, ["--strip-components=1"])
},
}
}

View File

@ -13,7 +13,6 @@ import { hasDnf } from "../utils/env/hasDnf"
import { setupDnfPack } from "../utils/setup/setupDnfPack"
import { isUbuntu } from "../utils/env/isUbuntu"
import { getExecOutput } from "@actions/exec"
import { existsSync } from "fs"
import { isBinUptoDate } from "../utils/setup/version"
import { getVersion } from "../versions/versions"
import assert from "assert"