fix: Propagate version tag to WebUI asset download in self-hosted CI (#23051)

* fix: Propagate version tag to WebUI asset download in self-hosted CI

* refactor: Apply suggestions from @CISC

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>

* fix: Skip npm build when Node.js is not installed

Avoid 'no such file or directory' errors on CI runners that lack
Node.js. Check if npm is available via find_program before attempting
npm install + npm run build. Falls back to HF Bucket download.

* fix: Use + separator for ASSETS list to fix Windows build

Replace fragile \; escaping with a + separator when passing the
WebUI asset list via -DASSETS to the download script. On Windows,
the \; escaping was not reliably preserved through the CMake build
system, causing all asset filenames to be concatenated into one
(e.g., 'index.html;bundle.js;bundle.css;loading.html' as a single
file), which broke the HF Bucket download and subsequent xxd.cmake
step.

+ is safe because it is not special in cmd.exe (unlike | which is a
pipe operator), not special in CMake's -D argument parser, and not
a valid Windows filename character. CMakeLists.txt joins assets
with + and webui-download.cmake splits them back via regex.

* fix: Validate HF_WEBUI_VERSION environment variable with regex

Add input validation for the HF_WEBUI_VERSION env var to prevent
CMake list separator or path-traversal issues in stamp filenames
and download URLs. Rejects non-conforming characters early.

* fix: Remove 'latest' fallback for HF_WEBUI_VERSION

When needs.determine-tag.outputs.tag_name is empty, let CMake's
default resolution handle it (empty -> git-based version lookup)
instead of falling back to 'latest'. This ensures the sentinel
stamp file is consistent with CMake's resolution logic.

* fix: Demote checksum verification failure to warning instead of hard gate

* fix: End line character

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
This commit is contained in:
Aleksander Grygier
2026-05-14 17:57:20 +02:00
committed by GitHub
parent 97b658cee8
commit 0c3e4fccca
3 changed files with 104 additions and 46 deletions
+40
View File
@@ -55,7 +55,22 @@ env:
LLAMA_LOG_TIMESTAMPS: 1 LLAMA_LOG_TIMESTAMPS: 1
jobs: jobs:
determine-tag:
name: Determine tag name
runs-on: ubuntu-slim
outputs:
tag_name: ${{ steps.tag.outputs.name }}
steps:
- name: Clone
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Determine tag name
id: tag
uses: ./.github/actions/get-tag-name
ggml-ci-nvidia-cuda: ggml-ci-nvidia-cuda:
needs: determine-tag
runs-on: [self-hosted, Linux, NVIDIA] runs-on: [self-hosted, Linux, NVIDIA]
steps: steps:
@@ -65,11 +80,14 @@ jobs:
- name: Test - name: Test
id: ggml-ci id: ggml-ci
env:
HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
run: | run: |
nvidia-smi nvidia-smi
GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
ggml-ci-nvidia-vulkan-cm: ggml-ci-nvidia-vulkan-cm:
needs: determine-tag
runs-on: [self-hosted, Linux, NVIDIA] runs-on: [self-hosted, Linux, NVIDIA]
steps: steps:
@@ -79,11 +97,14 @@ jobs:
- name: Test - name: Test
id: ggml-ci id: ggml-ci
env:
HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
run: | run: |
vulkaninfo --summary vulkaninfo --summary
GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
ggml-ci-nvidia-vulkan-cm2: ggml-ci-nvidia-vulkan-cm2:
needs: determine-tag
runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2] runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2]
steps: steps:
@@ -93,6 +114,8 @@ jobs:
- name: Test - name: Test
id: ggml-ci id: ggml-ci
env:
HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
run: | run: |
vulkaninfo --summary vulkaninfo --summary
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
@@ -172,6 +195,7 @@ jobs:
# GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp # GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
ggml-ci-mac-metal: ggml-ci-mac-metal:
needs: determine-tag
runs-on: [self-hosted, macOS, ARM64] runs-on: [self-hosted, macOS, ARM64]
steps: steps:
@@ -181,10 +205,13 @@ jobs:
- name: Test - name: Test
id: ggml-ci id: ggml-ci
env:
HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
run: | run: |
GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
ggml-ci-mac-webgpu: ggml-ci-mac-webgpu:
needs: determine-tag
runs-on: [self-hosted, macOS, ARM64] runs-on: [self-hosted, macOS, ARM64]
steps: steps:
@@ -207,11 +234,14 @@ jobs:
- name: Test - name: Test
id: ggml-ci id: ggml-ci
env:
HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
run: | run: |
GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \ GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
ggml-ci-mac-vulkan: ggml-ci-mac-vulkan:
needs: determine-tag
runs-on: [self-hosted, macOS, ARM64] runs-on: [self-hosted, macOS, ARM64]
steps: steps:
@@ -221,11 +251,14 @@ jobs:
- name: Test - name: Test
id: ggml-ci id: ggml-ci
env:
HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
run: | run: |
vulkaninfo --summary vulkaninfo --summary
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
ggml-ci-linux-intel-vulkan: ggml-ci-linux-intel-vulkan:
needs: determine-tag
runs-on: [self-hosted, Linux, Intel] runs-on: [self-hosted, Linux, Intel]
steps: steps:
@@ -237,11 +270,14 @@ jobs:
- name: Test - name: Test
id: ggml-ci id: ggml-ci
env:
HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
run: | run: |
vulkaninfo --summary vulkaninfo --summary
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
ggml-ci-win-intel-vulkan: ggml-ci-win-intel-vulkan:
needs: determine-tag
runs-on: [self-hosted, Windows, X64, Intel] runs-on: [self-hosted, Windows, X64, Intel]
steps: steps:
@@ -256,6 +292,7 @@ jobs:
MSYSTEM: UCRT64 MSYSTEM: UCRT64
CHERE_INVOKING: 1 CHERE_INVOKING: 1
PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }} PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }}
HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
run: | run: |
vulkaninfo --summary vulkaninfo --summary
# Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create # Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create
@@ -263,6 +300,7 @@ jobs:
LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp
ggml-ci-intel-openvino-gpu-low-perf: ggml-ci-intel-openvino-gpu-low-perf:
needs: determine-tag
runs-on: [self-hosted, Linux, Intel, OpenVINO] runs-on: [self-hosted, Linux, Intel, OpenVINO]
concurrency: concurrency:
@@ -294,6 +332,8 @@ jobs:
- name: Test - name: Test
id: ggml-ci id: ggml-ci
env:
HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
run: | run: |
source ./openvino_toolkit/setupvars.sh source ./openvino_toolkit/setupvars.sh
GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
+49 -40
View File
@@ -11,7 +11,7 @@ cmake_minimum_required(VERSION 3.16)
set(PUBLIC_DIR "" CACHE STRING "Directory to store/download assets") set(PUBLIC_DIR "" CACHE STRING "Directory to store/download assets")
set(HF_BUCKET "" CACHE STRING "Hugging Face bucket name") set(HF_BUCKET "" CACHE STRING "Hugging Face bucket name")
set(HF_VERSION "" CACHE STRING "Version to download (empty = resolve from git)") set(HF_VERSION "" CACHE STRING "Version to download (empty = resolve from git)")
set(ASSETS "" CACHE STRING "Semicolon-separated list of asset filenames") set(ASSETS "" CACHE STRING "Plus-separated list of asset filenames (+)")
set(STAMP_FILE "" CACHE STRING "Stamp file to create on success (optional)") set(STAMP_FILE "" CACHE STRING "Stamp file to create on success (optional)")
set(SOURCE_DIR "" CACHE STRING "Project source root (to resolve version from git)") set(SOURCE_DIR "" CACHE STRING "Project source root (to resolve version from git)")
set(NPM_DIR "" CACHE STRING "WebUI source directory (to run npm build)") set(NPM_DIR "" CACHE STRING "WebUI source directory (to run npm build)")
@@ -31,6 +31,10 @@ if("${RESOLVED_VERSION}" STREQUAL "" AND NOT "${SOURCE_DIR}" STREQUAL "")
endif() endif()
endif() endif()
# Convert + back to CMake list (+ is used as separator instead of ; to
# avoid platform-specific escaping issues when passing via -D arguments)
string(REGEX REPLACE "\\+" ";" ASSETS "${ASSETS}")
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# 2. Check stamp freshness — re-download if resolved version changed # 2. Check stamp freshness — re-download if resolved version changed
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -69,52 +73,58 @@ set(PROVISION_SUCCESS FALSE)
if(NOT PROVISION_SUCCESS AND NOT "${NPM_DIR}" STREQUAL "") if(NOT PROVISION_SUCCESS AND NOT "${NPM_DIR}" STREQUAL "")
if(EXISTS "${NPM_DIR}/package.json") if(EXISTS "${NPM_DIR}/package.json")
message(STATUS "WebUI: building from source in ${NPM_DIR}") # Check if npm is available before attempting npm build
find_program(NPM_EXECUTABLE npm)
if(NPM_EXECUTABLE)
message(STATUS "WebUI: building from source in ${NPM_DIR}")
# Run npm install if node_modules is missing # Run npm install if node_modules is missing
if(NOT EXISTS "${NPM_DIR}/node_modules") if(NOT EXISTS "${NPM_DIR}/node_modules")
message(STATUS "WebUI: running npm install (first time)") message(STATUS "WebUI: running npm install (first time)")
execute_process(
COMMAND npm install
WORKING_DIRECTORY "${NPM_DIR}"
RESULT_VARIABLE NPM_INSTALL_RESULT
OUTPUT_VARIABLE NPM_OUT
ERROR_VARIABLE NPM_ERR
)
if(NOT NPM_INSTALL_RESULT EQUAL 0)
message(STATUS "WebUI: npm install failed (${NPM_INSTALL_RESULT}), falling back to download")
message(STATUS " stderr: ${NPM_ERR}")
endif()
endif()
# Run the build
execute_process( execute_process(
COMMAND npm install COMMAND npm run build
WORKING_DIRECTORY "${NPM_DIR}" WORKING_DIRECTORY "${NPM_DIR}"
RESULT_VARIABLE NPM_INSTALL_RESULT RESULT_VARIABLE NPM_BUILD_RESULT
OUTPUT_VARIABLE NPM_OUT OUTPUT_VARIABLE NPM_OUT
ERROR_VARIABLE NPM_ERR ERROR_VARIABLE NPM_ERR
) )
if(NOT NPM_INSTALL_RESULT EQUAL 0)
message(STATUS "WebUI: npm install failed (${NPM_INSTALL_RESULT}), falling back to download") if(NPM_BUILD_RESULT EQUAL 0)
# Verify that the expected assets were produced
set(ALL_BUILT TRUE)
foreach(asset ${ASSETS})
if(NOT EXISTS "${PUBLIC_DIR}/${asset}")
set(ALL_BUILT FALSE)
break()
endif()
endforeach()
if(ALL_BUILT)
message(STATUS "WebUI: local npm build succeeded")
set(PROVISION_SUCCESS TRUE)
else()
message(STATUS "WebUI: npm build completed but assets missing from ${PUBLIC_DIR}, falling back to download")
endif()
else()
message(STATUS "WebUI: npm build failed (${NPM_BUILD_RESULT}), falling back to download")
message(STATUS " stderr: ${NPM_ERR}") message(STATUS " stderr: ${NPM_ERR}")
endif() endif()
endif()
# Run the build
execute_process(
COMMAND npm run build
WORKING_DIRECTORY "${NPM_DIR}"
RESULT_VARIABLE NPM_BUILD_RESULT
OUTPUT_VARIABLE NPM_OUT
ERROR_VARIABLE NPM_ERR
)
if(NPM_BUILD_RESULT EQUAL 0)
# Verify that the expected assets were produced
set(ALL_BUILT TRUE)
foreach(asset ${ASSETS})
if(NOT EXISTS "${PUBLIC_DIR}/${asset}")
set(ALL_BUILT FALSE)
break()
endif()
endforeach()
if(ALL_BUILT)
message(STATUS "WebUI: local npm build succeeded")
set(PROVISION_SUCCESS TRUE)
else()
message(STATUS "WebUI: npm build completed but assets missing from ${PUBLIC_DIR}, falling back to download")
endif()
else() else()
message(STATUS "WebUI: npm build failed (${NPM_BUILD_RESULT}), falling back to download") message(STATUS "WebUI: npm not found, skipping npm build and trying HF Bucket download")
message(STATUS " stderr: ${NPM_ERR}")
endif() endif()
else() else()
message(STATUS "WebUI: NPM_DIR (${NPM_DIR}) has no package.json, skipping npm build") message(STATUS "WebUI: NPM_DIR (${NPM_DIR}) has no package.json, skipping npm build")
@@ -178,8 +188,7 @@ if(NOT PROVISION_SUCCESS AND HF_ENABLED)
string(REGEX MATCH "${EXPECTED_HASH_UPPER}[ \\t]+${asset}" CHECKSUM_LINE "${CHECKSUMS_CONTENT}") string(REGEX MATCH "${EXPECTED_HASH_UPPER}[ \\t]+${asset}" CHECKSUM_LINE "${CHECKSUMS_CONTENT}")
if(NOT CHECKSUM_LINE) if(NOT CHECKSUM_LINE)
message(WARNING "WebUI: checksum verification failed for ${asset}") message(WARNING "WebUI: checksum verification failed for ${asset}")
set(ALL_OK FALSE) message(WARNING " downloaded file may not match expected checksum, but will be used")
break()
endif() endif()
endforeach() endforeach()
if(ALL_OK) if(ALL_OK)
+15 -6
View File
@@ -76,12 +76,21 @@ if (LLAMA_BUILD_WEBUI)
# Priority 2: Build-time asset provisioning (npm build → HF Bucket fallback) # Priority 2: Build-time asset provisioning (npm build → HF Bucket fallback)
if(NOT WEBUI_SOURCE_DIR) if(NOT WEBUI_SOURCE_DIR)
if(DEFINED LLAMA_BUILD_NUMBER) # Environment variable takes precedence (e.g., from CI workflows)
set(HF_WEBUI_VERSION "${LLAMA_BUILD_NUMBER}") if(DEFINED ENV{HF_WEBUI_VERSION})
set(HF_WEBUI_VERSION "$ENV{HF_WEBUI_VERSION}")
# Validate against allowed characters to prevent CMake list separator
# or path-traversal issues in stamp filenames and download URLs
if(NOT HF_WEBUI_VERSION MATCHES "^[A-Za-z0-9._-]+$")
message(FATAL_ERROR "WebUI: invalid HF_WEBUI_VERSION='${HF_WEBUI_VERSION}' - must match ^[A-Za-z0-9._-]+$")
endif()
message(STATUS "WebUI: using HF_WEBUI_VERSION from environment=${HF_WEBUI_VERSION}")
elseif(DEFINED LLAMA_BUILD_NUMBER)
set(HF_WEBUI_VERSION "b${LLAMA_BUILD_NUMBER}")
message(STATUS "WebUI: using LLAMA_BUILD_NUMBER=${HF_WEBUI_VERSION}") message(STATUS "WebUI: using LLAMA_BUILD_NUMBER=${HF_WEBUI_VERSION}")
else() else()
set(HF_WEBUI_VERSION "") set(HF_WEBUI_VERSION "")
message(STATUS "WebUI: LLAMA_BUILD_NUMBER not defined") message(STATUS "WebUI: version not specified (will use HF 'latest')")
endif() endif()
# Stamp file embeds the version tag so a changed build number triggers # Stamp file embeds the version tag so a changed build number triggers
@@ -93,8 +102,8 @@ if (LLAMA_BUILD_WEBUI)
endif() endif()
set(WEBUI_STAMP "${CMAKE_CURRENT_BINARY_DIR}/.webui-${WEBUI_VERSION_TAG}.stamp") set(WEBUI_STAMP "${CMAKE_CURRENT_BINARY_DIR}/.webui-${WEBUI_VERSION_TAG}.stamp")
# Escape semicolons so the CMake list is passed as a single -D argument # Join assets with + separator (safe across all platforms, unlike ; and |)
string(REPLACE ";" "\\;" PUBLIC_ASSETS_ESC "${PUBLIC_ASSETS}") string(REPLACE ";" "+" PUBLIC_ASSETS_JOINED "${PUBLIC_ASSETS}")
add_custom_command( add_custom_command(
OUTPUT ${WEBUI_STAMP} OUTPUT ${WEBUI_STAMP}
@@ -104,7 +113,7 @@ if (LLAMA_BUILD_WEBUI)
"-DHF_BUCKET=${LLAMA_WEBUI_HF_BUCKET}" "-DHF_BUCKET=${LLAMA_WEBUI_HF_BUCKET}"
"-DHF_VERSION=${HF_WEBUI_VERSION}" "-DHF_VERSION=${HF_WEBUI_VERSION}"
"-DHF_ENABLED=${LLAMA_USE_PREBUILT_WEBUI}" "-DHF_ENABLED=${LLAMA_USE_PREBUILT_WEBUI}"
"-DASSETS=${PUBLIC_ASSETS_ESC}" "-DASSETS=${PUBLIC_ASSETS_JOINED}"
"-DSTAMP_FILE=${WEBUI_STAMP}" "-DSTAMP_FILE=${WEBUI_STAMP}"
"-DNPM_DIR=${CMAKE_CURRENT_SOURCE_DIR}/webui" "-DNPM_DIR=${CMAKE_CURRENT_SOURCE_DIR}/webui"
-P ${PROJECT_SOURCE_DIR}/scripts/webui-download.cmake -P ${PROJECT_SOURCE_DIR}/scripts/webui-download.cmake