https://github.com/microsoft/DeepSpeed

from datasets import load_dataset


dataset = load_dataset("teknium/OpenHermes-2.5")

def _return_header(message)-> str:role = message["from"]header = ""if role == "system":header = "system"elif role == "gpt":header = "assistant"elif role == "human":header = "user"return header


def encode_header(message):text = ''text = text + "<|start_header_id|>"header = _return_header(message)text = text + headertext = text + "<|end_header_id|>"text = text + "\n\n"return text


def encode_message(message)->str:text = encode_header(message)text = text + message["value"].strip()text = text + "<|eot_id|>"return text


def encode_dialog_prompt(dialog):text = ''text = text + "<|begin_of_text|>"for message in dialog:text = text + encode_message(message)return textds = dataset.map(lambda x: {"content":encode_dialog_prompt(x['conversations'])}, num_proc=10)

ds = ds.remove_columns(['custom_instruction', 'topic', 'model_name', 'model', 'skip_prompt_formatting', 'category', 'conversations', 'views', 'language', 'id', 'title', 'idx', 'hash', 'avatarUrl', 'system_prompt', 'source'])train_test_split = ds["train"].train_test_split(test_size=0.3)

train_test_split.push_to_hub("sumandas/openhermes-2.5-llama3")

<|begin_of_text|><|start_header_id|>system<|end_header_id|> You are an AI assistant. Provide a detailed answer so user don't need to search outside to understand the answer.<|eot_id|><|start_header_id|>user<|end_header_id|> Instructions: Given a sentence, generate what should be the most likely next statement. The next statement should be reasonable and logically correct. Input: The screen is full of white bubbles and words, while a pair of hands plays the piano. The bubbles and words disappear and it Output:<|eot_id|><|start_header_id|>assistant<|end_header_id|> Output: becomes apparent that the hands are creating a visual representation of the music being played, captivating the audience with this unique sensory experience.<|eot_id|>

git+https://github.com/huggingface/transformersgit+https://github.com/huggingface/accelerategit+https://github.com/huggingface/peftgit+https://github.com/huggingface/trlhuggingface-hubbitsandbytesevaluatedatasetseinopswandbtiktokenxformerssentencepiecedeepspeedtorch==2.2.2

accelerate launch --config_file "deepspeed_config.yaml"train.py \--seed 100 \--model_name_or_path "meta-llama/Meta-Llama-3-8B-Instruct" \--dataset_name "sumandas/openhermes-2.5-llama3" \--chat_template_format "none" \--add_special_tokens False \--append_concat_token False \--splits "train,test" \--max_seq_len 2048 \--num_train_epochs 1 \--logging_steps 5 \--log_level "info" \--logging_strategy "steps" \--evaluation_strategy "epoch" \--save_strategy "steps" \--push_to_hub \--hub_private_repo True \--report_to "wandb" \--hub_strategy "every_save" \--bf16 True \--packing True \--learning_rate 1e-4 \--lr_scheduler_type "cosine" \--weight_decay 1e-4 \--warmup_ratio 0.0 \--max_grad_norm 1.0 \--output_dir "llama3-openhermes-2.5" \--per_device_train_batch_size 4\--per_device_eval_batch_size 4\--gradient_accumulation_steps 2 \--gradient_checkpointing True \--use_reentrant True \--dataset_text_field "content" \--use_flash_attn True \--use_peft_lora True \--lora_r 8 \--lora_alpha 16 \--lora_dropout 0.1 \--lora_target_modules "all-linear" \--use_4bit_quantization True \--use_nested_quant True \--bnb_4bit_compute_dtype "bfloat16" \--bnb_4bit_quant_storage_dtype "bfloat16"

{"train/learning_rate": 0.00004551803455482833,"eval/steps_per_second": 0.893,"_wandb.runtime": 51487,"_runtime": 51480.36651659012,"_timestamp": 1713698971.6200776,"train/epoch": 1.0571428571428572,"train/grad_norm": 0.14189070214353952,"train/global_step": 8325,"eval/samples_per_second": 7.141,"_step": 1665,"eval/loss": 0.963840126991272,"train/loss": 0.9674,"eval/runtime": 7532.9797}

from peft import PeftModelfrom transformers import AutoModelForCausalLM


base_model = AutoModelForCausalLM.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct",torch_dtype=torch.bfloat16, device_map="auto")adapter_model = PeftModel.from_pretrained(base_model, "sumandas/llama3-openhermes-2.5")adapter_model.merge_and_unload()

adapter_model.push_to_hub("sumandas/llama3-openhermes-2.5")

import torchimport transformersfrom transformers import pipeline


pipe = pipeline("text-generation", model="sumandas/llama3-openhermes-2.5",model_kwargs={"torch_dtype": torch.bfloat16},device="cuda")


generate_kwargs = {"do_sample": True,"temperature": 0.7,"max_new_tokens": 35,}pipe("<|begin_of_text|><|start_header_id|>user<|end_header_id|> Describe the food of Thailand in Chinese<|eot_id|><|start_header_id|>assistant<|end_header_id|>",max_length=2048)