Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmake/llvm-vglibc/almalinux-x64.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2.27
1 change: 1 addition & 0 deletions cmake/llvm-vglibc/ubuntu-x64.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2.34
137 changes: 137 additions & 0 deletions cmake/llvm-vglibc/update.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
#!/bin/bash
Comment thread
HollowMan6 marked this conversation as resolved.

# Script to automatically update LLVM glibc version requirements
# This script downloads LLVM pre-compiled builds and extracts their glibc dependencies

set -e # Exit on error

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CMAKE_DIR="$(dirname "$SCRIPT_DIR")"
TEMP_DIR=$(mktemp -d)

# Cleanup function
cleanup() {
echo "Cleaning up temporary files..."
rm -rf "$TEMP_DIR"
}
trap cleanup EXIT

# Read LLVM hash
LLVM_HASH_FILE="${CMAKE_DIR}/llvm-hash.txt"
if [[ ! -f "$LLVM_HASH_FILE" ]]; then
echo "Error: llvm-hash.txt not found at $LLVM_HASH_FILE"
exit 1
fi

# Read first 8 characters of the hash
LLVM_REV=$(head -c 8 "$LLVM_HASH_FILE")
echo "LLVM revision: $LLVM_REV"

# Base URL for LLVM builds
BASE_URL="https://oaitriton.blob.core.windows.net/public/llvm-builds"

# List of system suffixes to check (Linux x64 only)
SYSTEM_SUFFIXES=(
"ubuntu-x64"
"almalinux-x64"
)

echo "Downloading and analyzing LLVM builds..."
echo "========================================="

# Function to get glibc version from a binary
get_glibc_version() {
local binary_path="$1"

if [[ ! -f "$binary_path" ]]; then
echo "Error: Binary not found at $binary_path" >&2
return 1
fi

# Extract GLIBC versions using objdump
local versions=$(objdump -T "$binary_path" 2>/dev/null | grep GLIBC_ | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -V -u)

if [[ -z "$versions" ]]; then
echo "Error: No GLIBC versions found in $binary_path" >&2
return 1
fi

# Get the highest (last) version
local max_version=$(echo "$versions" | tail -n 1)
echo "$max_version"
}

# Process each system suffix
for suffix in "${SYSTEM_SUFFIXES[@]}"; do
echo ""
echo "Processing: $suffix"
echo "-------------------"

LLVM_NAME="llvm-${LLVM_REV}-${suffix}"
LLVM_URL="${BASE_URL}/${LLVM_NAME}.tar.gz"
DOWNLOAD_PATH="${TEMP_DIR}/${LLVM_NAME}.tar.gz"
EXTRACT_PATH="${TEMP_DIR}/${LLVM_NAME}"

echo "Downloading from: $LLVM_URL"

# Download the archive
if ! curl -f -L -o "$DOWNLOAD_PATH" "$LLVM_URL" 2>/dev/null; then
echo "Warning: Failed to download $LLVM_URL"
echo "Skipping $suffix"
continue
fi

echo "Download complete. Extracting bin/lld..."

# Extract only bin/lld from the archive
mkdir -p "$EXTRACT_PATH"
if ! tar -xzf "$DOWNLOAD_PATH" -C "$EXTRACT_PATH" --wildcards '*/bin/lld' 2>/dev/null; then
echo "Warning: Failed to extract bin/lld from $DOWNLOAD_PATH"
echo "Skipping $suffix"
continue
fi

echo "Extraction complete. Analyzing binary..."

# Find the extracted lld binary
BINARY_TO_CHECK=$(find "$EXTRACT_PATH" -type f -name "lld" 2>/dev/null | head -n 1)

if [[ -z "$BINARY_TO_CHECK" ]]; then
echo "Warning: bin/lld not found in $LLVM_NAME"
echo "Skipping $suffix"
continue
fi

echo "Checking binary: $(basename "$BINARY_TO_CHECK")"

# Get GLIBC version
GLIBC_VERSION=$(get_glibc_version "$BINARY_TO_CHECK")

if [[ $? -ne 0 ]] || [[ -z "$GLIBC_VERSION" ]]; then
echo "Warning: Failed to determine GLIBC version for $suffix"
echo "Skipping $suffix"
continue
fi

echo "Detected minimum GLIBC version: $GLIBC_VERSION"

# Write to file
OUTPUT_FILE="${SCRIPT_DIR}/${suffix}.txt"
echo "$GLIBC_VERSION" > "$OUTPUT_FILE"
echo "Updated: $OUTPUT_FILE"
done

echo ""
echo "========================================="
echo "Update complete!"
echo ""
echo "Current LLVM glibc version requirements:"
for suffix in "${SYSTEM_SUFFIXES[@]}"; do
VERSION_FILE="${SCRIPT_DIR}/${suffix}.txt"
if [[ -f "$VERSION_FILE" ]]; then
VERSION=$(cat "$VERSION_FILE")
printf " %-20s: %s\n" "$suffix" "$VERSION"
else
printf " %-20s: (not set)\n" "$suffix"
fi
done
47 changes: 36 additions & 11 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,30 +200,55 @@ def get_llvm_package_info():
arch = {"x86_64": "x64", "arm64": "arm64", "aarch64": "arm64"}[platform.machine()]
except KeyError:
arch = platform.machine()
system_suffix = ""
if (env_system_suffix := os.environ.get("TRITON_LLVM_SYSTEM_SUFFIX", None)):
system_suffix = env_system_suffix
elif system == "Darwin":
system_suffix = f"macos-{arch}"
elif system == "Linux":
vglibc = 0
if arch == 'arm64' and is_linux_os('almalinux'):
system_suffix = 'almalinux-arm64'
elif arch == 'arm64':
system_suffix = 'ubuntu-arm64'
elif arch == 'x64':
vglibc = tuple(map(int, platform.libc_ver()[1].split('.')))
vglibc = vglibc[0] * 100 + vglibc[1]
if vglibc > 228:
# Ubuntu 24 LTS (v2.39)
# Ubuntu 22 LTS (v2.35)
# Ubuntu 20 LTS (v2.31)
system_suffix = "ubuntu-x64"
else:
# Manylinux_2.28 (v2.28)
# AlmaLinux 8 (v2.28)
system_suffix = "almalinux-x64"
else:

# NOTE: Update using `cmake/llvm-vglibc/update.sh` when new precompiled
# llvm builds are available
vglibc_dir = os.path.join(get_base_dir(), "cmake", "llvm-vglibc")

# Dynamically discover available system suffixes and
# read glibc version requirements by reading the folder
# Files should be named as <system-suffix>.txt (e.g., ubuntu-x64.txt)
configs_with_versions = []
if os.path.exists(vglibc_dir):
for filename in sorted(os.listdir(vglibc_dir)):
Comment thread
HollowMan6 marked this conversation as resolved.
if filename.endswith('.txt'):
# Extract system suffix from filename (e.g., ubuntu-x64.txt -> ubuntu-x64)
suffix = filename[:-4]
config_path = os.path.join(vglibc_dir, filename)
try:
with open(config_path, 'r') as f:
required_version = f.read().strip()
required_parts = tuple(map(int, required_version.split('.')))
required_vglibc = required_parts[0] * 100 + required_parts[1]
configs_with_versions.append((suffix, required_vglibc))
except (FileNotFoundError, ValueError, IndexError) as e:
print(f"Warning: Could not read {filename}: {e}", file=sys.stderr)

# Sort by version (descending) to check highest version requirements first
Comment thread
HollowMan6 marked this conversation as resolved.
configs_with_versions.sort(key=lambda x: x[1], reverse=True)

for suffix, required_vglibc in configs_with_versions:
if vglibc >= required_vglibc:
system_suffix = suffix
break

if len(system_suffix) == 0:
print(
f"LLVM pre-compiled image is not available for {system}-{arch}. Proceeding with user-configured LLVM from source build."
f"LLVM pre-compiled image is not available for {arch} with glibc version {vglibc}. Proceeding with user-configured LLVM from source build."
)
Comment thread
HollowMan6 marked this conversation as resolved.
return Package("llvm", "LLVM-C.lib", "", "LLVM_INCLUDE_DIRS", "LLVM_LIBRARY_DIR", "LLVM_SYSPATH")
Comment thread
HollowMan6 marked this conversation as resolved.
else:
Expand Down