-
Notifications
You must be signed in to change notification settings - Fork 952
Description
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
CUDA_VISIBLE_DEVICES=4,5
NPROC_PER_NODE=2
MASTER_PORT=29502
HF_DEVICE_MAP="null"
MAX_PIXELS=546560
swift sft
--model '/data0/helizhi/modelscope/models/Qwen/Qwen2.5-VL-32B-Instruct'
--dataset 'prompt/train.jsonl'
--val_dataset 'prompt/val.jsonl'
--train_type lora
--torch_dtype bfloat16
--num_train_epochs 25
--per_device_train_batch_size 1
--per_device_eval_batch_size 1
--learning_rate 2e-5
--lora_rank 16
--lora_alpha 32
--target_modules all-linear
--freeze_vit false
--gradient_accumulation_steps 8
--eval_steps 20
--save_steps 20
--save_total_limit 2
--logging_steps 5
--max_length 32768
--output_dir output/ultrasound_qwen2__5-vl-32b-full-multi-gpu
--warmup_ratio 0.1
--dataloader_num_workers 4
--dataset_num_proc 8
--trainable_parameters "model.visual.blocks.28, model.visual.blocks.29, model.visual.blocks.30, model.visual.blocks.31, model.visual.merger"
lora微调并解冻vit最后4层,需要如何修改上述参数?