from unsloth import FastLanguageModelimport torchfrom trl import SFTTrainerfrom transformers import TrainingArgumentsfrom datasets import load_datasetmax_seq_length = 2048 # Supports RoPE Scaling interally, so choose any!# Get LAION dataseturl = "https://huggingface.co/datasets/laion/OIG/resolve/main/unified_chip2.jsonl"dataset = load_dataset("json", data_files = {"train" : url}, split = "train")
# 4bit pre quantized models we support for 4x faster downloading + no OOMs.fourbit_models = ["unsloth/mistral-7b-bnb-4bit","unsloth/mistral-7b-instruct-v0.2-bnb-4bit","unsloth/llama-2-7b-bnb-4bit","unsloth/gemma-7b-bnb-4bit","unsloth/gemma-7b-it-bnb-4bit", # Instruct version of Gemma 7b"unsloth/gemma-2b-bnb-4bit","unsloth/gemma-2b-it-bnb-4bit", # Instruct version of Gemma 2b"unsloth/llama-3-8b-bnb-4bit", # [NEW] 15 Trillion token Llama-3"unsloth/Phi-3-mini-4k-instruct-bnb-4bit",] # More models at https://huggingface.co/unsloth
model, tokenizer = FastLanguageModel.from_pretrained(model_name = "unsloth/llama-3-8b-bnb-4bit",max_seq_length = max_seq_length,dtype = None,load_in_4bit = True,)
# Do model patching and add fast LoRA weightsmodel = FastLanguageModel.get_peft_model(model,r = 16,target_modules = ["q_proj", "k_proj", "v_proj", "o_proj","gate_proj", "up_proj", "down_proj",],lora_alpha = 16,lora_dropout = 0, # Supports any, but = 0 is optimizedbias = "none",# Supports any, but = "none" is optimized# [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long contextrandom_state = 3407,max_seq_length = max_seq_length,use_rslora = False,# We support rank stabilized LoRAloftq_config = None, # And LoftQ)
trainer = SFTTrainer(model = model,train_dataset = dataset,dataset_text_field = "text",max_seq_length = max_seq_length,tokenizer = tokenizer,args = TrainingArguments(per_device_train_batch_size = 2,gradient_accumulation_steps = 4,warmup_steps = 10,max_steps = 60,fp16 = not torch.cuda.is_bf16_supported(),bf16 = torch.cuda.is_bf16_supported(),logging_steps = 1,output_dir = "outputs",optim = "adamw_8bit",seed = 3407,),)trainer.train()

git clone https://github.com/yangjianxin1/Firefly.git

pip install git+https://github.com/unslothai/unsloth.gitpip install transformers==4.37pip install bitsandbytes==0.43.1pip install peft==0.10.0pip install torch==2.2.2pip install xformers==0.0.25.post1

python train.py --train_args_file train_args/sft/qlora/llama3-8b-sft-qlora.json