from PIL import Imagefrom transformers import AutoTokenizerfrom vllm import LLM, SamplingParams
# 图像文件路径列表IMAGES = ["/root/ld/ld_project/MiniCPM-V/assets/airplane.jpeg",# 本地图片路径]
# 模型名称或路径MODEL_NAME = "/root/ld/ld_model_pretrained/Minicpmv2_6"# 本地模型路径或Hugging Face模型名称
# 打开并转换图像image = Image.open(IMAGES[0]).convert("RGB")
# 初始化分词器tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
# 初始化语言模型llm = LLM(model=MODEL_NAME, gpu_memory_utilization=1,# 使用全部GPU内存 trust_remote_code=True, max_model_len=2048)# 根据内存状况可调整此值
# 构建对话消息messages = [{'role': 'user', 'content': '(<image>./</image>)\n' + '请描述这张图片'}]
# 应用对话模板到消息prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
# 设置停止符ID# 2.0# stop_token_ids = [tokenizer.eos_id]# 2.5#stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]# 2.6 stop_tokens = ['<|im_end|>', '<|endoftext|>']stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
# 设置生成参数sampling_params = SamplingParams(stop_token_ids=stop_token_ids,# temperature=0.7,# top_p=0.8,# top_k=100,# seed=3472,max_tokens=1024,# min_tokens=150,temperature=0,use_beam_search=True,# length_penalty=1.2,best_of=3)
# 获取模型输出outputs = llm.generate({"prompt": prompt,"multi_modal_data": {"image": image}}, sampling_params=sampling_params)print(outputs[0].outputs[0].text)

from transformers import AutoTokenizerfrom decord import VideoReader, cpufrom PIL import Imagefrom vllm import LLM, SamplingParams
# 进行图片推理MAX_NUM_FRAMES = 64def encode_video(filepath):def uniform_sample(l, n):gap = len(l) / nidxs = [int(i * gap + gap / 2) for i in range(n)]return [l[i] for i in idxs]vr = VideoReader(filepath, ctx=cpu(0))sample_fps = round(vr.get_avg_fps() / 1)# FPSframe_idx = [i for i in range(0, len(vr), sample_fps)]if len(frame_idx)>MAX_NUM_FRAMES:frame_idx = uniform_sample(frame_idx, MAX_NUM_FRAMES)video = vr.get_batch(frame_idx).asnumpy()video = [Image.fromarray(v.astype('uint8')) for v in video]return video
MODEL_NAME = "openbmb/MiniCPM-V-2_6" # or local model path
video = encode_video("xxx.mp4")messages = [{"role":"user","content":"".join(["(<image>./</image>)"] * len(video)) + \"\nPlease describe this video."}]prompt = tokenizer.apply_chat_template(messages,tokenize=False,add_generation_prompt=True)tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)llm = LLM(model=MODEL_NAME,gpu_memory_utilization=1,max_model_len=4096)
stop_tokens = ['<|im_end|>', '<|endoftext|>']stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
sampling_params = SamplingParams(stop_token_ids=stop_token_ids, use_beam_search=Truetemperature=0, max_tokens=64)
outputs = llm.generate({"prompt": prompt,"multi_modal_data": {"image": {"images": video,"use_image_id": False,"max_slice_nums": 1 if len(video) > 16 else 2}}}, sampling_params=sampling_params_beam)

vllm serve /root/ld/ld_model_pretrained/Minicpmv2_6 --dtype auto --max-model-len 2048 --api-key token-abc123 --gpu_memory_utilization 1 --trust-remote-code

from openai import OpenAIopenai_api_key = "token-abc123" # your api key set in launch serveropenai_api_base = "http://localhost:8000/v1" # http idclient = OpenAI(api_key=openai_api_key,base_url=openai_api_base,)
chat_response = client.chat.completions.create(model="/root/ld/ld_model_pretrained/Minicpmv2_6", # model_local_path or huggingface idmessages=[{"role": "user","content": [# NOTE: 使用图像令牌 <image> 的提示格式是不必要的，因为提示将由API服务器自动处理。# 由于提示将由API服务器自动处理，因此不需要使用包含 <image> 图像令牌的提示格式。{"type": "text", "text": "请描述这张图片"},{"type": "image_url","image_url": {"url": "https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign.jpg",},},],}],extra_body={"stop_token_ids": [151645, 151643]})print("Chat response:", chat_response)print("Chat response content:", chat_response.choices[0].message.content)

from openai import OpenAIopenai_api_key = "token-abc123" # your api key set in launch serveropenai_api_base = "http://localhost:8000/v1" # http idclient = OpenAI(api_key=openai_api_key,base_url=openai_api_base,)
# 用于传本地图片with open('your/local/pic/path','rb') as file:image = "data:image/jpeg;base64,"+ base64.b64encode(file.read()).decode('utf-8')
chat_response = client.chat.completions.create(model="/root/ld/ld_model_pretrained/Minicpmv2_6", # model_local_path or huggingface idmessages=[{"role": "user","content": [# NOTE: 使用图像令牌 <image> 的提示格式是不必要的，因为提示将由API服务器自动处理。# 由于提示将由API服务器自动处理，因此不需要使用包含 <image> 图像令牌的提示格式。{"type": "text", "text": "请描述这张图片"},{"type": "image_url","image_url": {"url": image,},},],}],extra_body={"stop_token_ids": [151645, 151643]})print("Chat response:", chat_response)print("Chat response content:", chat_response.choices[0].message.content)

brew install ffmpegbrew install pkg-config

git clone -b minicpm-v2.5 https://github.com/OpenBMB/llama.cpp.git

cd llama.cppmake

git clone https://huggingface.co/openbmb/MiniCPM-V-2_6

# quantize int4 version./llama-quantize ../MiniCPM-V-2_6/model/ggml-model-f16.gguf ../MiniCPM-V-2_6/model/ggml-model-Q4_K_M.gguf Q4_K_M

./llama-minicpmv-cli -m /Users/liudan/Downloads/Minicpmv2_6gguf/ggml-model-Q4_K_M.gguf --mmproj /Users/liudan/Downloads/Minicpmv2_6gguf/mmproj-model-f16.gguf -c 8192 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --video ./Minicpmv2_6gguf/test_vedieo.mp4 -p "我接下来会给你一个视频，请告诉我视频中描述了什么"

git clone -b minicpm-v2.6 https://github.com/OpenBMB/ollama.gitcd ollama/llm

brew install go cmake gcc

What is described in this picture? /Users/liudan/Desktop/11.jpg