modelscope download --model=qwen/Qwen2-VL-7B-Instruct --local_dir ./Qwen2-VL-7B-Instruct

pip install git+https://github.com/huggingface/transformers
pip install qwen-vl-utils

from transformers import Qwen2VLForConditionalGeneration, AutoTokenizer, AutoProcessorfrom qwen_vl_utils import process_vision_info# default: Load the model on the available device(s)# model = Qwen2VLForConditionalGeneration.from_pretrained(# "./Qwen2-VL-7B-Instruct", torch_dtype="auto", device_map="auto"# )import torch
model = Qwen2VLForConditionalGeneration.from_pretrained("./Qwen2-VL-7B-Instruct",torch_dtype=torch.bfloat16,attn_implementation="flash_attention_2",).to("cuda:0")
# We recommend enabling flash_attention_2 for better acceleration and memory saving, especially in multi-image and video scenarios.# model = Qwen2VLForConditionalGeneration.from_pretrained(# "Qwen/Qwen2-VL-7B-Instruct",# torch_dtype=torch.bfloat16,# attn_implementation="flash_attention_2",# device_map="auto",# )# default processerprocessor = AutoProcessor.from_pretrained("./Qwen2-VL-7B-Instruct")# The default range for the number of visual tokens per image in the model is 4-16384.# You can set min_pixels and max_pixels according to your needs, such as a token range of 256-1280, to balance performance and cost.# min_pixels = 256*28*28# max_pixels = 1280*28*28# processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct", min_pixels=min_pixels, max_pixels=max_pixels)
messages = [{    "role": "user","content": [{"type": "image","image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg",},{"type": "text", "text": "Describe this image."},],}]# Preparation for inferencetext = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)image_inputs, video_inputs = process_vision_info(messages)inputs = processor(text=[text],images=image_inputs,videos=video_inputs,padding=True,return_tensors="pt",)inputs = inputs.to("cuda")# Inference: Generation of the outputgenerated_ids = model.generate(**inputs, max_new_tokens=128)generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]output_text = processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)print(output_text)

git clone https://github.com/modelscope/swift.gitcd swiftpip install -e .[llm]pip install pyav qwen_vl_utils

# 默认会将lora_target_modules设置为llm的所有linearCUDA_VISIBLE_DEVICES=0,1,2,3 NPROC_PER_NODE=4 swift sft \--model_type qwen2-vl-7b-instruct \--model_id_or_path qwen/Qwen2-VL-7B-Instruct \--sft_type lora \--dataset coco-en-mini#20000 \--deepspeed default-zero2

--dataset train.jsonl \--val_dataset val.jsonl \

{"query": "<image>55555", "response": "66666", "images": ["image_path"]}{"query": "eeeee<image>eeeee<image>eeeee", "response": "fffff", "history": [], "images": ["image_path1", "image_path2"]}{"query": "EEEEE", "response": "FFFFF", "history": [["query1", "response2"

CUDA_VISIBLE_DEVICES=0 swift infer \--ckpt_dir output/qwen2-vl-7b-instruct/vx-xxx/checkpoint-xxx \--load_dataset_config true --merge_lora true