Strix Halo AI Toolboxes

Toolboxes for GenAI on AMD Ryzen AI MAX+ Containerized environments for LLMs, Image Generation, and Fine-tuning.

• https://strix-halo-toolboxes.com/

ai-toolbox-cockpit

# toolbox
ai-toolbox-cockpit

# enter
distrobox enter llama-rocm-10.0

• https://github.com/kyuz0/ai-toolbox-cockpit

-m /models/Muse-Glimmer-30B-GGUF/Muse-Glimmer-30B-UD-Q4_K_XL.gguf 
-c 126976 
-ngl 999 
--host 0.0.0.0 
--port 8080 
--load-mode none 
-fa 1 
-b 2048 
-ub 2048 
--jinja 
--temp 1.0 
--top-p 0.95 
--top-k 64

bits

• https://github.com/kyuz0/amd-strix-halo-toolboxes#4-run-inference

gguf-vram-estimator.py models/my-model.gguf --contexts 32768

llama.cpp

bash-5.3# printenv
HOSTNAME=334da2659057
PWD=/
container=oci
HOME=/root
LANG=C.UTF-8
HIP_PATH=/opt/rocm
HF_TOKEN=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
HIP_VISIBLE_DEVICES=0
TERM=xterm-256color
SHLVL=1
ROCM_PATH=/opt/rocm
LD_LIBRARY_PATH=/opt/rocm/core/lib/rocm_sysdeps/lib:/opt/rocm/core/lib
LC_ALL=C.UTF-8
PATH=/opt/rocm/bin:/opt/rocm/core/bin:/opt/rocm/core/lib/llvm/bin:/usr/local/bin:/usr/bin
HSA_OVERRIDE_GFX_VERSION=11.5.1
_=/usr/bin/printenv
bash-5.3# llama server --version
version: 0.3.0-dev (build 10759, commit b81c99b47)
built with GNU 16.2.1 for Linux x86_64