export NGC_API_KEY=<your-ngc-api-key>kubectl create secret docker-registry ngc-secret \ --docker-server=nvcr.io\ --docker-username='$oauthtoken'\ --docker-password=${NGC_API_KEY}

kubectl apply -f-<<EOFapiVersion: v1kind: Secretmetadata:name: nvidia-nim-secretsstringData:NGC_API_KEY: <your-ngc-api-key>EOF

arena serve kserve \--name=llama3-8b-instruct \--image=nvcr.io/nim/meta/llama3-8b-instruct:1.0.0 \--image-pull-secret=ngc-secret \--gpus=1 \--cpu=8 \--memory=32Gi \--share-memory=32Gi \--port=8000 \--security-context runAsUser=0 \--annotation=serving.kserve.io/autoscalerClass=external \--env NIM_CACHE_PATH=/mnt/models \--env-from-secret NGC_API_KEY=nvidia-nim-secrets \--enable-prometheus=true \--metrics-port=8000 \--data=nim-model:/mnt/models

INFO[0004] The Job llama3-8b-instruct has been submitted successfullyINFO[0004] You can run `arena serve get llama3-8b-instruct --type kserve -n default` to check the job status

arena serve get llama3-8b-instruct

Name:       llama3-8b-instructNamespace:defaultType: KServeVersion:1Desired:1Available:1Age:24mAddress:http://llama3-8b-instruct-default.example.comPort: :80GPU:1

Instances:NAME STATUS AGEREADYRESTARTSGPUNODE---- ------ -----------------------llama3-8b-instruct-predictor-545445b4bc-97qc5Running24m1/10 1ap-southeast-1.172.16.xx.xxx

# Obtain the IP address of the Nginx ingress.NGINX_INGRESS_IP=$(kubectl -n kube-system get svc nginx-ingress-lb -ojsonpath='{.status.loadBalancer.ingress[0].ip}')# Obtain the Hostname of the Inference Service.SERVICE_HOSTNAME=$(kubectl get inferenceservice llama3-8b-instruct -o jsonpath='{.status.url}' | cut -d "/" -f 3)# Send a request to access the inference service.curl -H "Host: $SERVICE_HOSTNAME" -H "Content-Type: application/json" http://$NGINX_INGRESS_IP:80/v1/chat/completions -d '{"model": "meta/llama3-8b-instruct", "messages": [{"role": "user", "content": "Once upon a time"}], "max_tokens": 64, "temperature": 0.7, "top_p": 0.9, "seed": 10}'

{"id":"cmpl-70af7fa8c5ba4fe7b903835e326325ce","object":"chat.completion","created":1721557865,"model":"meta/llama3-8b-instruct","choices":[{"index":0,"message":{"role":"assistant","content":"It sounds like you're about to tell a story! I'd love to hear it. Please go ahead and continue with \"Once upon a time...\""},"logprobs":null,"finish_reason":"stop","stop_reason":128009}],"usage":{"prompt_tokens":14,"total_tokens":45,"completion_tokens":31}}

- seriesQuery: num_requests_waiting{namespace!="",pod!=""}resources:overrides:namespace: {resource: "namespace"}pod: {resource: "pod"}metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)

apiVersion: autoscaling/v2kind: HorizontalPodAutoscalermetadata:name: llama3-8b-instruct-hpanamespace: defaultspec:# HPA的最小Pod数量和最大Pod数量。minReplicas: 1maxReplicas: 3# 监控的指标数组，支持多种类型的指标共存。metrics:- pods:metric:name: num_requests_waitingtarget:averageValue: 10type: AverageValuetype: Pods# HPA的伸缩对象描述，HPA会动态修改该对象的Pod数量。scaleTargetRef:apiVersion: apps/v1kind: Deploymentname: llama3-8b-instruct-predictor

kubectl apply -f hpa.yaml

kubectl  get hpa llama3-8b-instruct-hpa

NAME                     REFERENCE                                 TARGETS   MINPODS   MAXPODS   REPLICAS   AGEllama3-8b-instruct-hpa Deployment/llama3-8b-instruct-predictor 0/101 3 134s

hey -z 5m -c 400 -m POST -host $SERVICE_HOSTNAME -H "Content-Type: application/json" -d '{"model": "meta/llama3-8b-instruct", "messages": [{"role": "user", "content": "Once upon a time"}], "max_tokens": 64}' http://$NGINX_INGRESS_IP:80/v1/chat/completions

kubectl describe hpa llama3-8b-instruct-hpa

Events:TypeReason Age From Message---------- -------- -------NormalSuccessfulRescale52s horizontal-pod-autoscalerNew size: 3; reason: pods metric num_requests_waiting above target