Scripts IA

Test de vitesse batch / ubatch via llama.cpp

#!/usr/bin/env bash

set -u

LLAMA_BENCH="/opt/llama.cpp/build/bin/llama-bench"
MODEL="/opt/models/Qwen3.8-27B/UD-Q4_K_XL/Qwen3.8-27B-UD-Q4_K_XL.gguf"

OUT="/tmp/muse_batch_bench.csv"

# Taille du prompt de benchmark.
# 8192 suffit largement pour comparer b/ub sans faire durer chaque test des minutes.
PROMPT=8192

# Nombre de répétitions par couple.
REPS=3

BATCHES=(512 1024 2048)
UBATCHES=(128 256 512 1024)

echo "batch,ubatch,pp_tps,gpu0_peak_mib,gpu1_peak_mib,status" > "$OUT"

echo
echo "============================================================"
echo " Muse-Glimmer batch / ubatch benchmark"
echo "============================================================"
echo "Model : $MODEL"
echo "Prompt: $PROMPT tokens"
echo "Runs  : $REPS"
echo


for B in "${BATCHES[@]}"; do
    for UB in "${UBATCHES[@]}"; do

        # ubatch ne doit pas être supérieur au batch.
        if (( UB > B )); then
            continue
        fi

        echo
        echo "============================================================"
        echo "b=$B  ub=$UB"
        echo "============================================================"

        TMP_JSON=$(mktemp)
        TMP_VRAM=$(mktemp)

        #
        # Surveillance VRAM pendant le benchmark
        #
        (
            while true; do
                nvidia-smi \
                    --query-gpu=memory.used \
                    --format=csv,noheader,nounits \
                    >> "$TMP_VRAM"

                sleep 0.1
            done
        ) &

        MON_PID=$!

        #
        # Benchmark
        #
        "$LLAMA_BENCH" \
            -m "$MODEL" \
            -p "$PROMPT" \
            -n 0 \
            -r "$REPS" \
            -b "$B" \
            -ub "$UB" \
            -ngl 999 \
            -sm tensor \
            -ts 1/1 \
            -ctk q8_0 \
            -ctv q8_0 \
            -fa on \
            -o json \
            > "$TMP_JSON" 2>/tmp/muse_bench_error.log

        RC=$?

        kill "$MON_PID" 2>/dev/null
        wait "$MON_PID" 2>/dev/null

        #
        # Pic VRAM
        #
        GPU0_PEAK=$(awk 'NR%2==1 {if ($1>m) m=$1} END {print m+0}' "$TMP_VRAM")
        GPU1_PEAK=$(awk 'NR%2==0 {if ($1>m) m=$1} END {print m+0}' "$TMP_VRAM")

        if (( RC != 0 )); then
            echo "ECHEC / OOM"
            cat /tmp/muse_bench_error.log

            echo "$B,$UB,0,$GPU0_PEAK,$GPU1_PEAK,FAIL" >> "$OUT"

            rm -f "$TMP_JSON" "$TMP_VRAM"
            continue
        fi

        #
        # Récupération du débit prompt processing depuis le JSON
        #
        PP=$(python3 - "$TMP_JSON" <<'PY'
import json
import sys

with open(sys.argv[1], "r") as f:
    data = json.load(f)

if isinstance(data, dict):
    data = [data]

vals = []

for x in data:
    if x.get("n_prompt", 0) > 0:
        v = x.get("avg_ts")
        if v is not None:
            vals.append(float(v))

if vals:
    print(f"{sum(vals)/len(vals):.2f}")
else:
    print("0")
PY
)

        echo
        echo "PP       : $PP tok/s"
        echo "GPU0 peak: $GPU0_PEAK MiB"
        echo "GPU1 peak: $GPU1_PEAK MiB"

        echo "$B,$UB,$PP,$GPU0_PEAK,$GPU1_PEAK,OK" >> "$OUT"

        rm -f "$TMP_JSON" "$TMP_VRAM"

        # petite pause thermique
        sleep 3
    done
done


echo
echo
echo "============================================================"
echo " RÉSULTATS"
echo "============================================================"

column -s, -t "$OUT"

echo
echo "CSV : $OUT"