pip install -U optimum[neural-compressor] intel-extension-for-transformers

def quantize(model_name: str, output_path: str, calibration_set: "datasets.Dataset"):model = AutoModel.from_pretrained(model_name)tokenizer = AutoTokenizer.from_pretrained(model_name)
def preprocess_function(examples):return tokenizer(examples["text"], padding="max_length", max_length=512, truncation=True)
vectorized_ds = calibration_set.map(preprocess_function, num_proc=10)vectorized_ds = vectorized_ds.remove_columns(["text"])
quantizer = INCQuantizer.from_pretrained(model)quantization_config = PostTrainingQuantConfig(approach="static", backend="ipex", domain="nlp")quantizer.quantize(quantization_config=quantization_config,calibration_dataset=vectorized_ds,save_directory=output_path,batch_size=1,)tokenizer.save_pretrained(output_path)

# 数据集地址https://huggingface.co/datasets/allenai/qasper

from optimum.intel import IPEXModelmodel = IPEXModel.from_pretrained("Intel/bge-small-en-v1.5-rag-int8-static")

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("Intel/bge-small-en-v1.5-rag-int8-static")inputs = tokenizer(sentences, return_tensors="pt")
with torch.no_grad():outputs = model(**inputs)# get the [CLS] tokenembeddings = outputs[0][:, 0]