Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/build-macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
- name: Compute cache keys
id: cache-key
run: |
llvm_file="cmake/llvm-hash.txt"
llvm_file="cmake/llvm-info.json"
nvidia_file="cmake/nvidia-toolchain-version.json"
json_file="cmake/json-version.txt"

Expand All @@ -41,7 +41,7 @@ jobs:
fi

# Process the files if they exist
echo "llvm=$(cat $llvm_file | cut -c 1-8)" >> $GITHUB_OUTPUT
echo "llvm=$(sha256sum $llvm_file | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
echo "nvidia=$(sha256sum $nvidia_file | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
echo "json=$(cat $json_file)" >> $GITHUB_OUTPUT
echo "datetime=$(date -u -Iseconds)" >> $GITHUB_OUTPUT
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/integration-tests-amd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ jobs:
- name: Compute cache keys
id: cache-key
run: |
llvm_file="cmake/llvm-hash.txt"
llvm_file="cmake/llvm-info.json"
nvidia_file="cmake/nvidia-toolchain-version.json"
json_file="cmake/json-version.txt"

Expand All @@ -70,7 +70,7 @@ jobs:
fi

# Process the files if they exist
echo "llvm=$(cat $llvm_file | cut -c 1-8)" >> $GITHUB_OUTPUT
echo "llvm=$(sha256sum $llvm_file | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
echo "nvidia=$(sha256sum $nvidia_file | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
echo "json=$(cat $json_file)" >> $GITHUB_OUTPUT
shell: bash
Expand Down Expand Up @@ -236,7 +236,7 @@ jobs:
- name: Compute cache keys
id: cache-key
run: |
llvm_file="cmake/llvm-hash.txt"
llvm_file="cmake/llvm-info.json"
nvidia_file="cmake/nvidia-toolchain-version.json"
json_file="cmake/json-version.txt"

Expand All @@ -245,7 +245,7 @@ jobs:
exit 1
fi

echo "llvm=$(cat $llvm_file | cut -c 1-8)" >> $GITHUB_OUTPUT
echo "llvm=$(sha256sum $llvm_file | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
echo "nvidia=$(sha256sum $nvidia_file | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
echo "json=$(cat $json_file)" >> $GITHUB_OUTPUT
shell: bash
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/integration-tests-nvidia.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
- name: Compute cache keys
id: cache-key
run: |
llvm_file="cmake/llvm-hash.txt"
llvm_file="cmake/llvm-info.json"
nvidia_file="cmake/nvidia-toolchain-version.json"
json_file="cmake/json-version.txt"

Expand All @@ -45,7 +45,7 @@ jobs:
fi

# Process the files if they exist
echo "llvm=$(cat $llvm_file | cut -c 1-8)" >> $GITHUB_OUTPUT
echo "llvm=$(sha256sum $llvm_file | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
echo "nvidia=$(sha256sum $nvidia_file | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
echo "json=$(cat $json_file)" >> $GITHUB_OUTPUT
shell: bash
Expand Down
35 changes: 20 additions & 15 deletions .github/workflows/llvm-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on:
branches:
- llvm-head
paths:
- cmake/llvm-hash.txt
- cmake/llvm-info.json
pull_request:
paths:
- .github/workflows/llvm-build.yml
Expand Down Expand Up @@ -44,18 +44,30 @@ jobs:
with:
path: llvm-build

- name: Fetch LLVM Commit Hash
- name: Install System Prerequisites (AlmaLinux)
if: matrix.config.target-os == 'almalinux'
run: |
rpm --import https://packages.microsoft.com/keys/microsoft.asc
dnf install --assumeyes https://packages.microsoft.com/config/rhel/8/packages-microsoft-prod.rpm
dnf install --assumeyes azure-cli llvm-toolset-20.1.8 python38-pip python38-devel git jq tar gzip
alternatives --set python3 /usr/bin/python3.8
echo "extra_cmake_flags=-DCMAKE_ASM_COMPILER=clang -DCMAKE_CXX_FLAGS=-Wno-everything -DPython3_EXECUTABLE=/usr/bin/python3.8 -DPython_EXECUTABLE=/usr/bin/python3.8" >> "$GITHUB_ENV"

- name: Fetch LLVM Build Metadata
shell: bash
run: |
LLVM_COMMIT_HASH="$(cat llvm-build/cmake/llvm-hash.txt)"
LLVM_COMMIT_HASH="$(jq -r '.llvm_hash' llvm-build/cmake/llvm-info.json)"
echo "Found LLVM commit hash: ${LLVM_COMMIT_HASH}"
echo "llvm_commit_hash=${LLVM_COMMIT_HASH}" >> ${GITHUB_ENV}

LLVM_BUILD_NUMBER="$(jq -r '.build_number' llvm-build/cmake/llvm-info.json)"
echo "Found LLVM build number: ${LLVM_BUILD_NUMBER}"

SHORT_LLVM_COMMIT_HASH="${LLVM_COMMIT_HASH:0:8}"
echo "Short LLVM commit hash: ${SHORT_LLVM_COMMIT_HASH}"
echo "short_llvm_commit_hash=${SHORT_LLVM_COMMIT_HASH}" >> ${GITHUB_ENV}

INSTALL_DIR="llvm-${SHORT_LLVM_COMMIT_HASH}-${{ matrix.config.target-os }}-${{ matrix.config.arch }}"
INSTALL_DIR="llvm-${SHORT_LLVM_COMMIT_HASH}-${{ matrix.config.target-os }}-${{ matrix.config.arch }}-${LLVM_BUILD_NUMBER}"
echo "LLVM installation directory name: ${INSTALL_DIR}"
echo "llvm_install_dir=${INSTALL_DIR}" >> ${GITHUB_ENV}

Expand All @@ -78,15 +90,6 @@ jobs:
with:
arch: amd64

- name: Install System Prerequisites (AlmaLinux)
if: matrix.config.target-os == 'almalinux'
run: |
rpm --import https://packages.microsoft.com/keys/microsoft.asc
dnf install --assumeyes https://packages.microsoft.com/config/rhel/8/packages-microsoft-prod.rpm
dnf install --assumeyes azure-cli llvm-toolset python38-pip python38-devel git tar gzip
alternatives --set python3 /usr/bin/python3.8
echo "extra_cmake_flags=-DCMAKE_ASM_COMPILER=clang -DCMAKE_CXX_FLAGS=-Wno-everything -DPython3_EXECUTABLE=/usr/bin/python3.8 -DPython_EXECUTABLE=/usr/bin/python3.8" >> "$GITHUB_ENV"

- name: Install Prerequisites
shell: bash
run: |
Expand Down Expand Up @@ -164,7 +167,7 @@ jobs:
with:
name: llvm-${{ matrix.config.target-os }}-${{ matrix.config.arch }}
path: |
${{ github.workspace }}/llvm-*-${{ matrix.config.target-os }}-${{ matrix.config.arch }}.tar.gz
${{ github.workspace }}/llvm-*-${{ matrix.config.target-os }}-${{ matrix.config.arch }}-*.tar.gz

- name: Azure login
if: ${{ (github.repository == 'triton-lang/triton') && github.ref_name == 'llvm-head' }}
Expand All @@ -178,7 +181,9 @@ jobs:
if: ${{ (github.repository == 'triton-lang/triton') && github.ref_name == 'llvm-head' }}
shell: bash -el {0}
run: |
az storage blob upload --account-name oaitriton --auth-mode login --container-name public --file "${{ env.llvm_install_dir }}.tar.gz" --name "llvm-builds/${{ env.llvm_install_dir }}.tar.gz" --overwrite
sha256sum "${{ env.llvm_install_dir }}.tar.gz"

az storage blob upload --account-name oaitriton --auth-mode login --container-name public --file "${{ env.llvm_install_dir }}.tar.gz" --name "llvm-builds/${{ env.llvm_install_dir }}.tar.gz"

URL=$(az storage blob url --account-name oaitriton --auth-mode login --container-name public --name "llvm-builds/${{ env.llvm_install_dir }}.tar.gz")
echo "Blob URL: ${URL}"
Expand Down
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ set_property(
APPEND
PROPERTY CMAKE_CONFIGURE_DEPENDS
"${CMAKE_CURRENT_SOURCE_DIR}/python/build_helpers.py"
"${CMAKE_CURRENT_SOURCE_DIR}/cmake/llvm-hash.txt"
"${CMAKE_CURRENT_SOURCE_DIR}/cmake/llvm-info.json"
"${CMAKE_CURRENT_SOURCE_DIR}/cmake/json-version.txt"
)
find_package(Python3 REQUIRED COMPONENTS Interpreter)
Expand Down
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,10 @@ make dev-install-llvm
Alternatively, follow these steps to build LLVM from source manually.
</summary>

1. Find the version of LLVM that Triton builds against. Check
`cmake/llvm-hash.txt` to see the current version. For example, if it says:
49af6502c6dcb4a7f7520178bd14df396f78240c.
1. Find the version of LLVM that Triton builds against. Check the `llvm_hash`
field in `cmake/llvm-info.json` to see the current version. For example, if it
says:
"llvm_hash": "49af6502c6dcb4a7f7520178bd14df396f78240c"

This means that the version of Triton you have builds against
[LLVM](https://github.com/llvm/llvm-project) 49af6502.
Expand Down
1 change: 0 additions & 1 deletion cmake/llvm-hash.txt

This file was deleted.

13 changes: 13 additions & 0 deletions cmake/llvm-info.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"llvm_hash": "87717bf9f81f7b29466c5d9a30a3453bdfc93941",
"build_number": 1,
"sha256sum": {
"almalinux-arm64": "83acad6d0bb6a540700e0bdf2d7ebfa71df1b5b7df29d007c3b4724e3228d493",
"almalinux-x64": "3010e7959f02032a16b3229b3a2079cd470cf4429b370dea5aa17d2482a5b4d1",
"macos-arm64": "c2e4219f2e04f21ee07832ae319b69726acf3ee496edb8d1786d2095d06ce004",
"macos-x64": "8c939eace38bbd0ce303ef01e6a3112d37740f2283b1209de10d59680df4ebd4",
"ubuntu-arm64": "cb35d89c202748a2d09c28f72c481b1e15b7f9407a16b348e6a04bf4e8c00116",
"ubuntu-x64": "7889df00f0dbbceb8e45774362b0478590029400a3abd58bdabba83d92fb2bbe",
"windows-x64": "9b2acb214d635bde0a5500a95186f99540e6aa5ac36f58a5a843f4fdb22a6a8a"
}
}
53 changes: 44 additions & 9 deletions python/build_helpers.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import argparse
import contextlib
import hashlib
import json
import os
import io
Expand All @@ -10,6 +11,7 @@
import sys
import sysconfig
import tarfile
import tempfile
import time
import urllib.request
import zipfile
Expand Down Expand Up @@ -61,6 +63,11 @@ def _normalize_optional(value: str) -> Optional[str]:
return value if value else None


# Taken from https://github.com/pytorch/pytorch/blob/master/tools/setup_helpers/env.py
def check_env_flag(name: str, default: str = "") -> bool:
return os.getenv(name, default).upper() in ["ON", "1", "YES", "TRUE", "Y"]


def _normalize_optional_path(value: str) -> Optional[str]:
normalized = _normalize_optional(value)
if normalized is None:
Expand Down Expand Up @@ -170,7 +177,7 @@ def _download_file(url: str, label: str):
return file_bytes


def _download_and_extract(url, download_dir, label):
def _download_and_extract(url, download_dir, label, expected_sha256=None):
label = f"downloading {label}"
os.makedirs(download_dir, exist_ok=True)
with contextlib.ExitStack() as stack:
Expand All @@ -180,10 +187,25 @@ def _download_and_extract(url, download_dir, label):
file = stack.enter_context(zipfile.ZipFile(file_bytes, "r"))
file.extractall(path=download_dir)
else:
# tar files can be streamed directly into untar
response = stack.enter_context(open_url(url))
progress_reader = DownloadProgressReader(response, label)
file = stack.enter_context(tarfile.open(fileobj=progress_reader, mode="r|*"))
tar_fileobj = progress_reader
if expected_sha256 is not None:
tar_fileobj = stack.enter_context(tempfile.TemporaryFile())
digest = hashlib.sha256()
while chunk := progress_reader.read(io.DEFAULT_BUFFER_SIZE):
tar_fileobj.write(chunk)
digest.update(chunk)
actual_sha256 = digest.hexdigest()
if actual_sha256 != expected_sha256:
message = (f"LLVM download from {url} failed checksum validation. "
f"Expected SHA256 {expected_sha256}, got {actual_sha256}.")
if check_env_flag("TRITON_UNSAFE_DISABLE_SHA_CHECK"):
print(f"WARNING: {message}", file=sys.stderr)
else:
raise RuntimeError(message)
tar_fileobj.seek(0)
file = stack.enter_context(tarfile.open(fileobj=tar_fileobj, mode="r|*"))
# Use extractall without filter for Python version < 3.12 compatibility
if hasattr(tarfile, "data_filter"):
file.extractall(path=download_dir, filter="data")
Expand Down Expand Up @@ -217,6 +239,7 @@ class Package:
lib_flag: str
syspath_var_name: str
sym_name: Optional[str] = None
sha256sum: Optional[str] = None


def get_json_package_info():
Expand Down Expand Up @@ -272,14 +295,26 @@ def get_llvm_package_info(helper_args: BuildHelperArgs):
f"LLVM pre-compiled image is not available for {system}-{arch}. Proceeding with user-configured LLVM from source build."
)
return Package("llvm", "LLVM-C.lib", "", "LLVM_INCLUDE_DIRS", "LLVM_LIBRARY_DIR", "LLVM_SYSPATH")
llvm_hash_path = os.path.join(get_base_dir(), "cmake", "llvm-hash.txt")
with open(llvm_hash_path, "r") as llvm_hash_file:
rev = llvm_hash_file.read(8)
name = f"llvm-{rev}-{system_suffix}"
llvm_info_path = os.path.join(get_base_dir(), "cmake", "llvm-info.json")
with open(llvm_info_path, "r") as llvm_info_file:
llvm_info = json.load(llvm_info_file)
rev = llvm_info["llvm_hash"][:8]
build_number = llvm_info["build_number"]
name = f"llvm-{rev}-{system_suffix}-{build_number}"
# Create a stable symlink that doesn't include revision
sym_name = f"llvm-{system_suffix}"
url = f"https://oaitriton.blob.core.windows.net/public/llvm-builds/{name}.tar.gz"
return Package("llvm", name, url, "LLVM_INCLUDE_DIRS", "LLVM_LIBRARY_DIR", "LLVM_SYSPATH", sym_name=sym_name)
sha256sum = llvm_info["sha256sum"][system_suffix]
return Package(
"llvm",
name,
url,
"LLVM_INCLUDE_DIRS",
"LLVM_LIBRARY_DIR",
"LLVM_SYSPATH",
sym_name=sym_name,
sha256sum=sha256sum,
)


def _get_syspath_override(package_syspath_var_name: str, helper_args: BuildHelperArgs) -> Optional[str]:
Expand Down Expand Up @@ -308,7 +343,7 @@ def _get_thirdparty_package_cmake_vars(package: Package, helper_args: BuildHelpe
if not helper_args.offline_build and not input_defined and not input_compatible:
with contextlib.suppress(Exception):
shutil.rmtree(package_root_dir)
_download_and_extract(package.url, package_root_dir, package.name)
_download_and_extract(package.url, package_root_dir, package.name, package.sha256sum)
# write version url to package_dir
with open(os.path.join(package_dir, "version.txt"), "w") as file:
file.write(package.url)
Expand Down
2 changes: 1 addition & 1 deletion scripts/build-llvm-project.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ REPO_ROOT="$(git rev-parse --show-toplevel)"
LLVM_TARGETS=${LLVM_TARGETS:-Native;NVPTX;AMDGPU}
LLVM_PROJECTS=${LLVM_PROJECTS:-mlir;llvm;lld;clang}
LLVM_BUILD_TYPE=${LLVM_BUILD_TYPE:-RelWithDebInfo}
LLVM_COMMIT_HASH=${LLVM_COMMIT_HASH:-$(cat "$REPO_ROOT/cmake/llvm-hash.txt")}
LLVM_COMMIT_HASH=${LLVM_COMMIT_HASH:-$(jq -r '.llvm_hash' "$REPO_ROOT/cmake/llvm-info.json")}
LLVM_PROJECT_PATH=${LLVM_PROJECT_PATH:-"$REPO_ROOT/llvm-project"}
LLVM_BUILD_PATH=${LLVM_BUILD_PATH:-"$LLVM_PROJECT_PATH/build"}
LLVM_INSTALL_PATH=${LLVM_INSTALL_PATH:-"$LLVM_PROJECT_PATH/install"}
Expand Down
7 changes: 1 addition & 6 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ class editable_wheel:

sys.path.insert(0, os.path.dirname(__file__))

from python.build_helpers import get_base_dir, get_cmake_dir
from python.build_helpers import check_env_flag, get_base_dir, get_cmake_dir


def is_git_repo() -> bool:
Expand Down Expand Up @@ -126,11 +126,6 @@ def copy_externals():
]


# Taken from https://github.com/pytorch/pytorch/blob/master/tools/setup_helpers/env.py
def check_env_flag(name: str, default: str = "") -> bool:
return os.getenv(name, default).upper() in ["ON", "1", "YES", "TRUE", "Y"]


def get_build_type():
if check_env_flag("DEBUG"):
return "Debug"
Expand Down
Loading