DGX Spark 安裝使用 NIM

參考文件 https://build.nvidia.com/spark/nim-llm/overview

下載 ngc

$ wget --content-disposition \

https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/4.11.1/files/ngccli_arm64.zip \

-O ngccli_arm64.zip && unzip ngccli_arm64.zip

$ find ngc-cli/ -type f -exec md5sum {} + | LC_ALL=C sort | md5sum -c ngc-cli.md5

$ sha256sum ngccli_arm64.zip

9c285ef8250c30c21e70a836700121945b296a8441005b829179bde6106ce6d1

$ chmod u+x ngc-cli/ngc

$ echo "export PATH=\"\$PATH:$(pwd)/ngc-cli\"" >> ~/.bash_profile && source ~/.bash_profile

$ ngc config set

Enter API key [no-apikey]. Choices: [<VALID_APIKEY>, 'no-apikey']: 

Enter CLI output format type [ascii]. Choices: ['ascii', 'csv', 'json']: 

Validating configuration...

Successfully validated configuration.

Saving configuration...

Successfully saved NGC configuration to /home/spark/.ngc/config

$ docker login nvcr.io

Username: $oauthtoken

Password: Y6NWM0NjYzYjYtMzczMy00MjVkLTg1NWQtZTE2MDNmZTAxNDBj

$ newgrp docker

$ export NGC_API_KEY="Y6NWM0NjYzYjYtMzczMy00MjVkLTg1NWQtZTE2MDNmZTAxNDBj"

$ echo "$NGC_API_KEY" | docker login nvcr.io --username '$oauthtoken' --password-stdin

$ export CONTAINER_NAME="nim-llm-demo"

$ export IMG_NAME="nvcr.io/nim/meta/llama-3.1-8b-instruct-dgx-spark:latest"

$ export LOCAL_NIM_CACHE=$(pwd)/cache/nim

$ export LOCAL_NIM_WORKSPACE=~/workspace

$ mkdir -p "$LOCAL_NIM_WORKSPACE"

$ chmod -R a+w "$LOCAL_NIM_WORKSPACE"

$ mkdir -p "$LOCAL_NIM_CACHE"

$ chmod -R a+w "$LOCAL_NIM_CACHE"

$ docker run -it --rm --name=$CONTAINER_NAME \

  --gpus all \

  --shm-size=16GB \

  -e NGC_API_KEY=$NGC_API_KEY \

  -v "$LOCAL_NIM_CACHE:/opt/nim/.cache" \

  -v "$LOCAL_NIM_WORKSPACE:/opt/nim/workspace" \

  -p 8000:8000 \

  $IMG_NAME

$ curl -X 'POST' \

    'http://0.0.0.0:8000/v1/chat/completions' \

    -H 'accept: application/json' \

    -H 'Content-Type: application/json' \

    -d '{

      "model": "meta/llama-3.1-8b-instruct",

      "messages": [

        {

          "role":"system",

          "content":"detailed thinking on"

        },

        {

          "role":"user",

          "content":"Can you write me a song?"

        }

      ],

      "top_p": 1,

      "n": 1,

      "max_tokens": 15,

      "frequency_penalty": 1.0,

      "stop": ["hello"]

    }'

    

$ docker stop $CONTAINER_NAME

$ docker rm $CONTAINER_NAME