base_model: mistralai/Mistral-7B-v0.3model_type: AutoModelForCausalLMtokenizer_type: AutoTokenizer load_in_8bit: false# load_in_4bit: truestrict: false datasets: - path: /workspace/datasets/dolphin-2.9.3/dolphin201-sharegpt2.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/SystemChat_filtered_sharegpt.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/SystemChat_multilingual_sharegpt.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/dolphin-coder-translate-sharegpt2.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/dolphin-coder-codegen-sharegpt2.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/m-a-p_Code-Feedback-sharegpt-unfiltered.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/m-a-p_CodeFeedback-Filtered-Instruction-sharegpt-unfiltered.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/not_samantha_norefusals.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/Orca-Math-resort-unfiltered.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/agent_instruct_react_unfiltered.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/toolbench_instruct_j1s1_3k_unfiltered.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/toolbench_negative_unfiltered.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/toolbench_react_10p_unfiltered.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/toolbench_tflan_cot_30p_unfiltered.jsonl type: sharegpt conversation: chatml - path: /workspace/datasets/dolphin-2.9.3/openhermes200k_unfiltered.jsonl type: sharegpt conversation: chatml chat_template: chatml# adapter: qlora# lora_r: 128# lora_alpha: 16# lora_modules_to_save: [embed_tokens, lm_head]# lora_dropout: 0.05# lora_target_linear: true dataset_prepared_path: /workspace/axolotl/dolph-2.9.3-preparedval_set_size: 0.01output_dir: /workspace/axolotl/dolphin-2.9.3-mistral-7B sequence_len: 8192sample_packing: truepad_to_sequence_len: true wandb_project: dolphin-2.9.3-Mistral-7Bwandb_watch:wandb_run_id:wandb_log_model: gradient_accumulation_steps: 16micro_batch_size: 1num_epochs: 3optimizer: adamw_8bitlr_scheduler: cosinelearning_rate: 5e-6train_on_inputs: falsegroup_by_length: falsebf16: autofp16:tf32: gradient_checkpointing: truegradient_checkpointing_kwargs: use_reentrant: falseearly_stopping_patience:resume_from_checkpoint:logging_steps: 1xformers_attention:flash_attention: true warmup_steps: 100# evals_per_epoch: 4eval_table_size:saves_per_epoch: 1save_total_limit: 2save_steps:debug:deepspeed: /workspace/axolotl/deepspeed_configs/zero3_bf16.jsonweight_decay: 0.1fsdp:fsdp_config:special_tokens: eos_token: "<|im_end|>"tokens: - "<|im_start|>"