model_name = "./models/gpt2"tokenizer = AutoTokenizer.from_pretrained(model_name)model = AutoModelForCausalLM.from_pretrained(model_name)
# fix dtype post quantization to "pretend" to be fp32def get_float32_dtype(self):return torch.float32GPT2Model.dtype = property(get_float32_dtype)
# This will return the memory footprint of the current model in bytesmodel.get_memory_footprint() # 510342192 => 0.47GB

r_min = min(x_1, x_2, ..., x_n)r_max = max(x_1, x_2, ..., x_n)

s = (r_max - r_min) / (q_max - q_min)

z = q_min - r_min / s

q = round(x / s + z)

def quantize(t):# obtain range of values in the tensor to map between 0 and 255min_val, max_val = t.min(), t.max()
# determine the "zero-point", or value in the tensor to map to 0scale = (max_val - min_val) / 255zero_point = min_val
# quantize and clamp to ensure we're in [0, 255]t_quant = (t - zero_point) / scalet_quant = torch.clamp(t_quant, min=0, max=255)
# keep track of scale and zero_point for reversing quantizationstate = (scale, zero_point)
# cast to uint8 and returnt_quant = t_quant.type(torch.uint8)return t_quant, state    quant_model, states = quantize_model(model)quant_model.get_memory_footprint() # 137022768 => 0.12G