Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +2 -0
- docs/resources/web-ui-en.jpg +3 -0
- docs/resources/web-ui.jpg +3 -0
- examples/train/seq_cls/qwen2_5/deploy.sh +8 -0
- examples/train/seq_cls/qwen2_5/infer.sh +5 -0
- examples/train/seq_cls/qwen2_5/sft.sh +28 -0
- examples/train/streaming/train.sh +17 -0
- examples/train/think_model/qwen3_demo1.sh +30 -0
- examples/train/think_model/qwen3_demo2.sh +29 -0
- examples/train/tuners/adalora/train.sh +16 -0
- examples/train/tuners/adapter/train.sh +16 -0
- examples/train/tuners/boft/train.sh +16 -0
- examples/train/tuners/bone/train.sh +16 -0
- examples/train/tuners/dora/train.sh +19 -0
- examples/train/tuners/galore/train_galore.sh +18 -0
- examples/train/tuners/galore/train_qgalore.sh +20 -0
- examples/train/tuners/lisa/train.sh +17 -0
- examples/train/tuners/llamapro/train.sh +17 -0
- examples/train/tuners/longlora/train.sh +16 -0
- examples/train/tuners/lora-ga/train.sh +33 -0
- examples/train/tuners/lora/train.sh +18 -0
- examples/train/tuners/neftune/train.sh +19 -0
- examples/train/tuners/olora/train.sh +19 -0
- examples/train/tuners/pissa/train.sh +19 -0
- examples/train/tuners/qlora/train.sh +19 -0
- examples/train/tuners/reft/train.sh +17 -0
- examples/train/tuners/unsloth/train.sh +19 -0
- requirements/docs.txt +8 -0
- requirements/eval.txt +2 -0
- requirements/framework.txt +38 -0
- requirements/install_all.sh +12 -0
- requirements/seq_parallel.txt +1 -0
- requirements/swanlab.txt +1 -0
- requirements/tests.txt +6 -0
- scripts/benchmark/config/tuner.json +301 -0
- scripts/benchmark/exp.py +50 -0
- scripts/benchmark/exp_utils.py +391 -0
- scripts/benchmark/generate_report.py +433 -0
- scripts/utils/plot_loss.py +9 -0
- scripts/utils/run_dataset_info.py +106 -0
- scripts/utils/run_model_info.py +101 -0
- scripts/utils/run_template.py +8 -0
- swift/__init__.py +55 -0
- swift/cli/__init__.py +0 -0
- swift/cli/_megatron/__init__.py +0 -0
- swift/cli/_megatron/main.py +20 -0
- swift/cli/_megatron/pt.py +4 -0
- swift/cli/_megatron/sft.py +4 -0
- swift/cli/app.py +4 -0
- swift/cli/deploy.py +5 -0
.gitattributes
CHANGED
|
@@ -43,3 +43,5 @@ docs/resources/grpo_countdown.png filter=lfs diff=lfs merge=lfs -text
|
|
| 43 |
docs/resources/grpo_geoqa.png filter=lfs diff=lfs merge=lfs -text
|
| 44 |
docs/resources/grpo_openr1_multimodal.png filter=lfs diff=lfs merge=lfs -text
|
| 45 |
docs/resources/kto_data.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 43 |
docs/resources/grpo_geoqa.png filter=lfs diff=lfs merge=lfs -text
|
| 44 |
docs/resources/grpo_openr1_multimodal.png filter=lfs diff=lfs merge=lfs -text
|
| 45 |
docs/resources/kto_data.png filter=lfs diff=lfs merge=lfs -text
|
| 46 |
+
docs/resources/web-ui.jpg filter=lfs diff=lfs merge=lfs -text
|
| 47 |
+
docs/resources/web-ui-en.jpg filter=lfs diff=lfs merge=lfs -text
|
docs/resources/web-ui-en.jpg
ADDED
|
Git LFS Details
|
docs/resources/web-ui.jpg
ADDED
|
Git LFS Details
|
examples/train/seq_cls/qwen2_5/deploy.sh
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 2 |
+
swift deploy \
|
| 3 |
+
--adapters output/vx-xxx/checkpoint-xxx
|
| 4 |
+
|
| 5 |
+
# curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/json" -d '{
|
| 6 |
+
# "model": "Qwen2.5-0.5B",
|
| 7 |
+
# "messages": [{"role": "user", "content": "包装差,容易被调包。"}]
|
| 8 |
+
# }'
|
examples/train/seq_cls/qwen2_5/infer.sh
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 2 |
+
swift infer \
|
| 3 |
+
--adapters output/vx-xxx/checkpoint-xxx \
|
| 4 |
+
--load_data_args true \
|
| 5 |
+
--max_batch_size 16
|
examples/train/seq_cls/qwen2_5/sft.sh
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# If `num_labels` is provided, it will be considered a classification task,
|
| 2 |
+
# and AutoModelForSequenceClassification will be used to load the model.
|
| 3 |
+
# You can also specify `--model Qwen/Qwen2.5-0.5B-Instruct --use_chat_template true`.
|
| 4 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 5 |
+
swift sft \
|
| 6 |
+
--model Qwen/Qwen2.5-0.5B \
|
| 7 |
+
--train_type lora \
|
| 8 |
+
--dataset 'DAMO_NLP/jd:cls#2000' \
|
| 9 |
+
--torch_dtype bfloat16 \
|
| 10 |
+
--num_train_epochs 1 \
|
| 11 |
+
--per_device_train_batch_size 1 \
|
| 12 |
+
--per_device_eval_batch_size 1 \
|
| 13 |
+
--learning_rate 1e-4 \
|
| 14 |
+
--lora_rank 8 \
|
| 15 |
+
--lora_alpha 32 \
|
| 16 |
+
--target_modules all-linear \
|
| 17 |
+
--gradient_accumulation_steps 16 \
|
| 18 |
+
--eval_steps 50 \
|
| 19 |
+
--save_steps 50 \
|
| 20 |
+
--save_total_limit 2 \
|
| 21 |
+
--logging_steps 5 \
|
| 22 |
+
--max_length 2048 \
|
| 23 |
+
--output_dir output \
|
| 24 |
+
--warmup_ratio 0.05 \
|
| 25 |
+
--dataloader_num_workers 4 \
|
| 26 |
+
--num_labels 2 \
|
| 27 |
+
--task_type seq_cls \
|
| 28 |
+
--use_chat_template false
|
examples/train/streaming/train.sh
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 2 |
+
swift sft \
|
| 3 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 4 |
+
--train_type lora \
|
| 5 |
+
--dataset 'swift/self-cognition#1000' \
|
| 6 |
+
--streaming true \
|
| 7 |
+
--max_steps 1000 \
|
| 8 |
+
--learning_rate 1e-4 \
|
| 9 |
+
--lora_rank 8 \
|
| 10 |
+
--lora_alpha 32 \
|
| 11 |
+
--gradient_accumulation_steps 16 \
|
| 12 |
+
--eval_steps 100 \
|
| 13 |
+
--save_steps 100 \
|
| 14 |
+
--save_total_limit 2 \
|
| 15 |
+
--logging_steps 5 \
|
| 16 |
+
--model_author swift \
|
| 17 |
+
--model_name swift-robot
|
examples/train/think_model/qwen3_demo1.sh
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# use `--loss_scale ignore_empty_think`
|
| 2 |
+
# Avoid losing the think capability by ignoring the loss of empty `<think>\n\n</think>\n\n`
|
| 3 |
+
# This method is also applicable to the Deepseek-R1 series of models.
|
| 4 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 5 |
+
swift sft \
|
| 6 |
+
--model Qwen/Qwen3-8B \
|
| 7 |
+
--train_type lora \
|
| 8 |
+
--dataset 'swift/Qwen3-SFT-Mixin#2000' \
|
| 9 |
+
'swift/self-cognition:empty_think#600' \
|
| 10 |
+
--torch_dtype bfloat16 \
|
| 11 |
+
--num_train_epochs 1 \
|
| 12 |
+
--per_device_train_batch_size 1 \
|
| 13 |
+
--per_device_eval_batch_size 1 \
|
| 14 |
+
--learning_rate 1e-4 \
|
| 15 |
+
--lora_rank 8 \
|
| 16 |
+
--lora_alpha 32 \
|
| 17 |
+
--target_modules all-linear \
|
| 18 |
+
--gradient_accumulation_steps 16 \
|
| 19 |
+
--eval_steps 50 \
|
| 20 |
+
--save_steps 50 \
|
| 21 |
+
--save_total_limit 2 \
|
| 22 |
+
--logging_steps 5 \
|
| 23 |
+
--max_length 2048 \
|
| 24 |
+
--output_dir output \
|
| 25 |
+
--warmup_ratio 0.05 \
|
| 26 |
+
--dataloader_num_workers 4 \
|
| 27 |
+
--use_liger_kernel true \
|
| 28 |
+
--loss_scale ignore_empty_think \
|
| 29 |
+
--model_author swift \
|
| 30 |
+
--model_name swift-robot
|
examples/train/think_model/qwen3_demo2.sh
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# use `swift/self-cognition:qwen3`
|
| 2 |
+
# Avoid losing the thinking capability by appending `/no_think` to the dataset query.
|
| 3 |
+
# https://github.com/modelscope/ms-swift/blob/77985c2ccdac8ed4037174ee222e79d1f1d5059d/swift/llm/dataset/dataset/llm.py#L835
|
| 4 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 5 |
+
swift sft \
|
| 6 |
+
--model Qwen/Qwen3-8B \
|
| 7 |
+
--train_type lora \
|
| 8 |
+
--dataset 'swift/Qwen3-SFT-Mixin#2000' \
|
| 9 |
+
'swift/self-cognition:qwen3#600' \
|
| 10 |
+
--torch_dtype bfloat16 \
|
| 11 |
+
--num_train_epochs 1 \
|
| 12 |
+
--per_device_train_batch_size 1 \
|
| 13 |
+
--per_device_eval_batch_size 1 \
|
| 14 |
+
--learning_rate 1e-4 \
|
| 15 |
+
--lora_rank 8 \
|
| 16 |
+
--lora_alpha 32 \
|
| 17 |
+
--target_modules all-linear \
|
| 18 |
+
--gradient_accumulation_steps 16 \
|
| 19 |
+
--eval_steps 50 \
|
| 20 |
+
--save_steps 50 \
|
| 21 |
+
--save_total_limit 2 \
|
| 22 |
+
--logging_steps 5 \
|
| 23 |
+
--max_length 2048 \
|
| 24 |
+
--output_dir output \
|
| 25 |
+
--warmup_ratio 0.05 \
|
| 26 |
+
--dataloader_num_workers 4 \
|
| 27 |
+
--use_liger_kernel true \
|
| 28 |
+
--model_author swift \
|
| 29 |
+
--model_name swift-robot
|
examples/train/tuners/adalora/train.sh
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 17GiB
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 5 |
+
--train_type adalora \
|
| 6 |
+
--dataset 'swift/self-cognition#1000' \
|
| 7 |
+
--num_train_epochs 1 \
|
| 8 |
+
--per_device_train_batch_size 1 \
|
| 9 |
+
--learning_rate 1e-4 \
|
| 10 |
+
--gradient_accumulation_steps 16 \
|
| 11 |
+
--eval_steps 100 \
|
| 12 |
+
--save_steps 100 \
|
| 13 |
+
--save_total_limit 2 \
|
| 14 |
+
--logging_steps 5 \
|
| 15 |
+
--model_author swift \
|
| 16 |
+
--model_name swift-robot
|
examples/train/tuners/adapter/train.sh
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 17GiB
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 5 |
+
--train_type adapter \
|
| 6 |
+
--dataset 'swift/self-cognition#1000' \
|
| 7 |
+
--num_train_epochs 1 \
|
| 8 |
+
--per_device_train_batch_size 1 \
|
| 9 |
+
--learning_rate 1e-4 \
|
| 10 |
+
--gradient_accumulation_steps 16 \
|
| 11 |
+
--eval_steps 100 \
|
| 12 |
+
--save_steps 100 \
|
| 13 |
+
--save_total_limit 2 \
|
| 14 |
+
--logging_steps 5 \
|
| 15 |
+
--model_author swift \
|
| 16 |
+
--model_name swift-robot
|
examples/train/tuners/boft/train.sh
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 17GiB
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 5 |
+
--train_type boft \
|
| 6 |
+
--dataset 'swift/self-cognition#1000' \
|
| 7 |
+
--num_train_epochs 1 \
|
| 8 |
+
--per_device_train_batch_size 1 \
|
| 9 |
+
--learning_rate 1e-4 \
|
| 10 |
+
--gradient_accumulation_steps 16 \
|
| 11 |
+
--eval_steps 100 \
|
| 12 |
+
--save_steps 100 \
|
| 13 |
+
--save_total_limit 2 \
|
| 14 |
+
--logging_steps 5 \
|
| 15 |
+
--model_author swift \
|
| 16 |
+
--model_name swift-robot
|
examples/train/tuners/bone/train.sh
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 17.3GiB
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 5 |
+
--train_type bone \
|
| 6 |
+
--dataset 'swift/self-cognition#1000' \
|
| 7 |
+
--num_train_epochs 1 \
|
| 8 |
+
--per_device_train_batch_size 1 \
|
| 9 |
+
--learning_rate 1e-4 \
|
| 10 |
+
--gradient_accumulation_steps 16 \
|
| 11 |
+
--eval_steps 100 \
|
| 12 |
+
--save_steps 100 \
|
| 13 |
+
--save_total_limit 2 \
|
| 14 |
+
--logging_steps 5 \
|
| 15 |
+
--model_author swift \
|
| 16 |
+
--model_name swift-robot
|
examples/train/tuners/dora/train.sh
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 17.2GiB
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 5 |
+
--train_type lora \
|
| 6 |
+
--use_dora true \
|
| 7 |
+
--dataset 'swift/self-cognition#1000' \
|
| 8 |
+
--num_train_epochs 1 \
|
| 9 |
+
--per_device_train_batch_size 1 \
|
| 10 |
+
--learning_rate 1e-4 \
|
| 11 |
+
--lora_rank 8 \
|
| 12 |
+
--lora_alpha 32 \
|
| 13 |
+
--gradient_accumulation_steps 16 \
|
| 14 |
+
--eval_steps 100 \
|
| 15 |
+
--save_steps 100 \
|
| 16 |
+
--save_total_limit 2 \
|
| 17 |
+
--logging_steps 5 \
|
| 18 |
+
--model_author swift \
|
| 19 |
+
--model_name swift-robot
|
examples/train/tuners/galore/train_galore.sh
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 38GiB
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 5 |
+
--train_type full \
|
| 6 |
+
--dataset 'swift/self-cognition#1000' \
|
| 7 |
+
--num_train_epochs 1 \
|
| 8 |
+
--per_device_train_batch_size 1 \
|
| 9 |
+
--learning_rate 1e-5 \
|
| 10 |
+
--gradient_accumulation_steps 16 \
|
| 11 |
+
--eval_steps 100 \
|
| 12 |
+
--save_steps 100 \
|
| 13 |
+
--save_total_limit 2 \
|
| 14 |
+
--logging_steps 5 \
|
| 15 |
+
--model_author swift \
|
| 16 |
+
--model_name swift-robot \
|
| 17 |
+
--use_galore true \
|
| 18 |
+
--galore_optim_per_parameter true
|
examples/train/tuners/galore/train_qgalore.sh
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 35GiB
|
| 2 |
+
# pip install bitsandbytes==0.40.0
|
| 3 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 4 |
+
swift sft \
|
| 5 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 6 |
+
--train_type full \
|
| 7 |
+
--torch_dtype bfloat16 \
|
| 8 |
+
--dataset 'lvjianjin/AdvertiseGen#1000' \
|
| 9 |
+
--num_train_epochs 1 \
|
| 10 |
+
--per_device_train_batch_size 1 \
|
| 11 |
+
--learning_rate 1e-5 \
|
| 12 |
+
--gradient_accumulation_steps 16 \
|
| 13 |
+
--eval_steps 100 \
|
| 14 |
+
--save_steps 100 \
|
| 15 |
+
--save_total_limit 2 \
|
| 16 |
+
--logging_steps 5 \
|
| 17 |
+
--model_author swift \
|
| 18 |
+
--model_name swift-robot \
|
| 19 |
+
--use_galore true \
|
| 20 |
+
--galore_quantization true
|
examples/train/tuners/lisa/train.sh
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 29GiB
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 5 |
+
--train_type full \
|
| 6 |
+
--dataset 'swift/self-cognition#1000' \
|
| 7 |
+
--lisa_activated_layers 2 \
|
| 8 |
+
--num_train_epochs 1 \
|
| 9 |
+
--per_device_train_batch_size 1 \
|
| 10 |
+
--learning_rate 1e-5 \
|
| 11 |
+
--gradient_accumulation_steps 16 \
|
| 12 |
+
--eval_steps 100 \
|
| 13 |
+
--save_steps 100 \
|
| 14 |
+
--save_total_limit 2 \
|
| 15 |
+
--logging_steps 5 \
|
| 16 |
+
--model_author swift \
|
| 17 |
+
--model_name swift-robot
|
examples/train/tuners/llamapro/train.sh
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 25.4GiB
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 5 |
+
--train_type llamapro \
|
| 6 |
+
--dataset 'swift/self-cognition#1000' \
|
| 7 |
+
--llamapro_num_new_blocks 4 \
|
| 8 |
+
--num_train_epochs 1 \
|
| 9 |
+
--per_device_train_batch_size 1 \
|
| 10 |
+
--learning_rate 1e-4 \
|
| 11 |
+
--gradient_accumulation_steps 16 \
|
| 12 |
+
--eval_steps 100 \
|
| 13 |
+
--save_steps 100 \
|
| 14 |
+
--save_total_limit 2 \
|
| 15 |
+
--logging_steps 5 \
|
| 16 |
+
--model_author swift \
|
| 17 |
+
--model_name swift-robot
|
examples/train/tuners/longlora/train.sh
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 2 |
+
swift sft \
|
| 3 |
+
--model LLM-Research/Meta-Llama-3.1-8B-Instruct \
|
| 4 |
+
--train_type longlora \
|
| 5 |
+
--dataset 'AI-ModelScope/LongAlpaca-12k#1000' \
|
| 6 |
+
--num_train_epochs 1 \
|
| 7 |
+
--learning_rate 1e-4 \
|
| 8 |
+
--attn_impl flash_attn \
|
| 9 |
+
--gradient_accumulation_steps 16 \
|
| 10 |
+
--lora_rank 8 \
|
| 11 |
+
--lora_alpha 32 \
|
| 12 |
+
--eval_steps 100 \
|
| 13 |
+
--save_steps 100 \
|
| 14 |
+
--max_length 10000 \
|
| 15 |
+
--save_total_limit 2 \
|
| 16 |
+
--logging_steps 5
|
examples/train/tuners/lora-ga/train.sh
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Train
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2-1.5B-Instruct \
|
| 5 |
+
--train_type lora \
|
| 6 |
+
--dataset 'swift/self-cognition#1000' \
|
| 7 |
+
--num_train_epochs 1 \
|
| 8 |
+
--per_device_train_batch_size 1 \
|
| 9 |
+
--learning_rate 1e-4 \
|
| 10 |
+
--lora_rank 8 \
|
| 11 |
+
--lora_alpha 32 \
|
| 12 |
+
--init_weights lora-ga \
|
| 13 |
+
--lora_ga_batch_size 2 \
|
| 14 |
+
--lora_ga_iters 2 \
|
| 15 |
+
--lora_ga_max_length 1024 \
|
| 16 |
+
--lora_ga_direction ArB2r \
|
| 17 |
+
--lora_ga_scale stable \
|
| 18 |
+
--lora_ga_stable_gamma 16 \
|
| 19 |
+
--gradient_accumulation_steps 16 \
|
| 20 |
+
--eval_steps 100 \
|
| 21 |
+
--save_steps 100 \
|
| 22 |
+
--save_total_limit 2 \
|
| 23 |
+
--logging_steps 5 \
|
| 24 |
+
--model_author swift \
|
| 25 |
+
--model_name swift-robot
|
| 26 |
+
|
| 27 |
+
# Infer
|
| 28 |
+
# swift infer \
|
| 29 |
+
# --model Qwen/Qwen2-1.5B-Instruct \
|
| 30 |
+
# --ckpt_dir ./output/Qwen2-1.5B-Instruct/v0-20241214-191235/checkpoint-62/converted/default \
|
| 31 |
+
# --infer_backend pt \
|
| 32 |
+
# --stream true \
|
| 33 |
+
# --max_new_tokens 2048
|
examples/train/tuners/lora/train.sh
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 17.2GiB
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 5 |
+
--train_type lora \
|
| 6 |
+
--dataset 'swift/self-cognition#1000' \
|
| 7 |
+
--num_train_epochs 1 \
|
| 8 |
+
--per_device_train_batch_size 1 \
|
| 9 |
+
--learning_rate 1e-4 \
|
| 10 |
+
--lora_rank 8 \
|
| 11 |
+
--lora_alpha 32 \
|
| 12 |
+
--gradient_accumulation_steps 16 \
|
| 13 |
+
--eval_steps 100 \
|
| 14 |
+
--save_steps 100 \
|
| 15 |
+
--save_total_limit 2 \
|
| 16 |
+
--logging_steps 5 \
|
| 17 |
+
--model_author swift \
|
| 18 |
+
--model_name swift-robot
|
examples/train/tuners/neftune/train.sh
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 17GiB
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 5 |
+
--train_type lora \
|
| 6 |
+
--dataset 'swift/self-cognition#1000' \
|
| 7 |
+
--num_train_epochs 1 \
|
| 8 |
+
--per_device_train_batch_size 1 \
|
| 9 |
+
--neftune_noise_alpha 15 \
|
| 10 |
+
--learning_rate 1e-4 \
|
| 11 |
+
--lora_rank 8 \
|
| 12 |
+
--lora_alpha 32 \
|
| 13 |
+
--gradient_accumulation_steps 16 \
|
| 14 |
+
--eval_steps 100 \
|
| 15 |
+
--save_steps 100 \
|
| 16 |
+
--save_total_limit 2 \
|
| 17 |
+
--logging_steps 5 \
|
| 18 |
+
--model_author swift \
|
| 19 |
+
--model_name swift-robot
|
examples/train/tuners/olora/train.sh
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 17GiB
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 5 |
+
--train_type lora \
|
| 6 |
+
--dataset 'swift/self-cognition#1000' \
|
| 7 |
+
--num_train_epochs 1 \
|
| 8 |
+
--per_device_train_batch_size 1 \
|
| 9 |
+
--learning_rate 1e-4 \
|
| 10 |
+
--lora_rank 8 \
|
| 11 |
+
--lora_alpha 32 \
|
| 12 |
+
--init_lora_weights olora \
|
| 13 |
+
--gradient_accumulation_steps 16 \
|
| 14 |
+
--eval_steps 100 \
|
| 15 |
+
--save_steps 100 \
|
| 16 |
+
--save_total_limit 2 \
|
| 17 |
+
--logging_steps 5 \
|
| 18 |
+
--model_author swift \
|
| 19 |
+
--model_name swift-robot
|
examples/train/tuners/pissa/train.sh
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 17GiB
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 5 |
+
--train_type lora \
|
| 6 |
+
--dataset 'swift/self-cognition#1000' \
|
| 7 |
+
--num_train_epochs 1 \
|
| 8 |
+
--per_device_train_batch_size 1 \
|
| 9 |
+
--learning_rate 1e-4 \
|
| 10 |
+
--lora_rank 8 \
|
| 11 |
+
--lora_alpha 32 \
|
| 12 |
+
--init_lora_weights pissa \
|
| 13 |
+
--gradient_accumulation_steps 16 \
|
| 14 |
+
--eval_steps 100 \
|
| 15 |
+
--save_steps 100 \
|
| 16 |
+
--save_total_limit 2 \
|
| 17 |
+
--logging_steps 5 \
|
| 18 |
+
--model_author swift \
|
| 19 |
+
--model_name swift-robot
|
examples/train/tuners/qlora/train.sh
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 2 |
+
swift sft \
|
| 3 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 4 |
+
--train_type lora \
|
| 5 |
+
--dataset 'swift/self-cognition#1000' \
|
| 6 |
+
--num_train_epochs 1 \
|
| 7 |
+
--per_device_train_batch_size 1 \
|
| 8 |
+
--learning_rate 1e-4 \
|
| 9 |
+
--lora_rank 8 \
|
| 10 |
+
--lora_alpha 32 \
|
| 11 |
+
--gradient_accumulation_steps 16 \
|
| 12 |
+
--eval_steps 100 \
|
| 13 |
+
--save_steps 100 \
|
| 14 |
+
--save_total_limit 2 \
|
| 15 |
+
--logging_steps 5 \
|
| 16 |
+
--model_author swift \
|
| 17 |
+
--model_name swift-robot \
|
| 18 |
+
--quant_bits 4 \
|
| 19 |
+
--quant_method bnb
|
examples/train/tuners/reft/train.sh
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 2 |
+
swift sft \
|
| 3 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 4 |
+
--train_type reft \
|
| 5 |
+
--dataset 'swift/self-cognition#1000' \
|
| 6 |
+
--reft_intervention_type 'LoreftIntervention' \
|
| 7 |
+
--num_train_epochs 1 \
|
| 8 |
+
--per_device_train_batch_size 1 \
|
| 9 |
+
--learning_rate 1e-4 \
|
| 10 |
+
--gradient_checkpointing false \
|
| 11 |
+
--gradient_accumulation_steps 16 \
|
| 12 |
+
--eval_steps 100 \
|
| 13 |
+
--save_steps 100 \
|
| 14 |
+
--save_total_limit 2 \
|
| 15 |
+
--logging_steps 5 \
|
| 16 |
+
--model_author swift \
|
| 17 |
+
--model_name swift-robot
|
examples/train/tuners/unsloth/train.sh
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 17GiB
|
| 2 |
+
CUDA_VISIBLE_DEVICES=0 \
|
| 3 |
+
swift sft \
|
| 4 |
+
--model Qwen/Qwen2.5-7B-Instruct \
|
| 5 |
+
--tuner_backend unsloth \
|
| 6 |
+
--train_type lora \
|
| 7 |
+
--dataset 'swift/self-cognition#1000' \
|
| 8 |
+
--num_train_epochs 1 \
|
| 9 |
+
--per_device_train_batch_size 1 \
|
| 10 |
+
--learning_rate 1e-4 \
|
| 11 |
+
--lora_rank 8 \
|
| 12 |
+
--lora_alpha 32 \
|
| 13 |
+
--gradient_accumulation_steps 16 \
|
| 14 |
+
--eval_steps 100 \
|
| 15 |
+
--save_steps 100 \
|
| 16 |
+
--save_total_limit 2 \
|
| 17 |
+
--logging_steps 5 \
|
| 18 |
+
--model_author swift \
|
| 19 |
+
--model_name swift-robot
|
requirements/docs.txt
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
docutils>=0.16.0
|
| 2 |
+
myst_parser
|
| 3 |
+
recommonmark
|
| 4 |
+
sphinx>=5.3.0
|
| 5 |
+
sphinx-book-theme
|
| 6 |
+
sphinx-copybutton
|
| 7 |
+
sphinx-rtd-theme
|
| 8 |
+
sphinx_markdown_tables
|
requirements/eval.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
evalscope[opencompass]
|
| 2 |
+
evalscope[vlmeval]
|
requirements/framework.txt
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
accelerate
|
| 2 |
+
addict
|
| 3 |
+
aiohttp
|
| 4 |
+
attrdict
|
| 5 |
+
binpacking
|
| 6 |
+
charset_normalizer
|
| 7 |
+
cpm_kernels
|
| 8 |
+
dacite
|
| 9 |
+
datasets>=3.0,<3.4
|
| 10 |
+
einops
|
| 11 |
+
fastapi
|
| 12 |
+
gradio>=3.40.0
|
| 13 |
+
importlib_metadata
|
| 14 |
+
jieba
|
| 15 |
+
matplotlib
|
| 16 |
+
modelscope>=1.23
|
| 17 |
+
nltk
|
| 18 |
+
numpy<2.0
|
| 19 |
+
openai
|
| 20 |
+
oss2
|
| 21 |
+
pandas
|
| 22 |
+
peft>=0.11,<0.16
|
| 23 |
+
pillow
|
| 24 |
+
requests
|
| 25 |
+
rouge
|
| 26 |
+
safetensors
|
| 27 |
+
scipy
|
| 28 |
+
sentencepiece
|
| 29 |
+
simplejson>=3.3.0
|
| 30 |
+
sortedcontainers>=1.5.9
|
| 31 |
+
tensorboard
|
| 32 |
+
tiktoken
|
| 33 |
+
tqdm
|
| 34 |
+
transformers>=4.33,<4.53
|
| 35 |
+
transformers_stream_generator
|
| 36 |
+
trl>=0.13,<0.18
|
| 37 |
+
uvicorn
|
| 38 |
+
zstandard
|
requirements/install_all.sh
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# please use python=3.10, cuda12.*
|
| 2 |
+
# sh requirements/install_all.sh
|
| 3 |
+
pip install "vllm>=0.5.1" -U
|
| 4 |
+
pip install "lmdeploy>=0.5" -U --no-deps
|
| 5 |
+
pip install autoawq -U --no-deps
|
| 6 |
+
pip install auto_gptq optimum bitsandbytes -U
|
| 7 |
+
pip install git+https://github.com/modelscope/ms-swift.git
|
| 8 |
+
pip install timm -U
|
| 9 |
+
pip install deepspeed -U
|
| 10 |
+
pip install qwen_vl_utils qwen_omni_utils decord librosa pyav icecream soundfile -U
|
| 11 |
+
pip install liger_kernel nvitop pre-commit -U
|
| 12 |
+
# flash-attn: https://github.com/Dao-AILab/flash-attention/releases
|
requirements/seq_parallel.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
xtuner
|
requirements/swanlab.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
swanlab
|
requirements/tests.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
expecttest
|
| 2 |
+
flake8
|
| 3 |
+
isort>=4.3.21
|
| 4 |
+
modelscope
|
| 5 |
+
pre-commit
|
| 6 |
+
yapf==0.30.0 # use fix version to ensure consistent auto-styling
|
scripts/benchmark/config/tuner.json
ADDED
|
@@ -0,0 +1,301 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cmd": "sft",
|
| 3 |
+
"requirements":{
|
| 4 |
+
"gpu": "1",
|
| 5 |
+
"ddp": "1"
|
| 6 |
+
},
|
| 7 |
+
"eval_requirements": {
|
| 8 |
+
"gpu": "1"
|
| 9 |
+
},
|
| 10 |
+
"eval_dataset": ["ceval", "gsm8k", "arc"],
|
| 11 |
+
"args": {
|
| 12 |
+
"model": "Qwen/Qwen-7B-Chat",
|
| 13 |
+
"dataset": "iic/ms_agent",
|
| 14 |
+
"per_device_train_batch_size": 1,
|
| 15 |
+
"max_length": 2048,
|
| 16 |
+
"loss_scale": "react",
|
| 17 |
+
"gradient_accumulation_steps": 16,
|
| 18 |
+
"learning_rate": 5e-5,
|
| 19 |
+
"attn_impl": "flash_attn",
|
| 20 |
+
"eval_steps": 2000,
|
| 21 |
+
"save_steps": 2000,
|
| 22 |
+
"num_train_epochs": 2,
|
| 23 |
+
"gradient_checkpointing": true,
|
| 24 |
+
"weight_decay": 0.01,
|
| 25 |
+
"warmup_ratio": 0.03,
|
| 26 |
+
"save_total_limit": 2,
|
| 27 |
+
"logging_steps": 10
|
| 28 |
+
},
|
| 29 |
+
"experiment": [
|
| 30 |
+
{
|
| 31 |
+
"name": "lora",
|
| 32 |
+
"args": {
|
| 33 |
+
"train_type": "lora",
|
| 34 |
+
"lora_rank": 8,
|
| 35 |
+
"lora_alpha": 32
|
| 36 |
+
}
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
"name": "lora+packing",
|
| 40 |
+
"args": {
|
| 41 |
+
"train_type": "lora",
|
| 42 |
+
"lora_rank": 8,
|
| 43 |
+
"lora_alpha": 32,
|
| 44 |
+
"packing": true,
|
| 45 |
+
"eval_steps": 200,
|
| 46 |
+
"save_steps": 200
|
| 47 |
+
}
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"name": "lora+packing+ddp",
|
| 51 |
+
"requirements":{
|
| 52 |
+
"gpu": "2",
|
| 53 |
+
"ddp": "2"
|
| 54 |
+
},
|
| 55 |
+
"args": {
|
| 56 |
+
"train_type": "lora",
|
| 57 |
+
"lora_rank": 8,
|
| 58 |
+
"lora_alpha": 32,
|
| 59 |
+
"packing": true,
|
| 60 |
+
"eval_steps": 100,
|
| 61 |
+
"save_steps": 100
|
| 62 |
+
}
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"name": "lora+packing+lazytokenize",
|
| 66 |
+
"args": {
|
| 67 |
+
"train_type": "lora",
|
| 68 |
+
"lora_rank": 8,
|
| 69 |
+
"lora_alpha": 32,
|
| 70 |
+
"packing": true,
|
| 71 |
+
"lazy_tokenize": true,
|
| 72 |
+
"eval_steps": 200,
|
| 73 |
+
"save_steps": 200
|
| 74 |
+
}
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"name": "lora+",
|
| 78 |
+
"args": {
|
| 79 |
+
"train_type": "lora",
|
| 80 |
+
"lora_rank": 8,
|
| 81 |
+
"lora_alpha": 32,
|
| 82 |
+
"lorap_lr_ratio": 16.0
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "rslora",
|
| 87 |
+
"args": {
|
| 88 |
+
"train_type": "lora",
|
| 89 |
+
"lora_rank": 8,
|
| 90 |
+
"lora_alpha": 32,
|
| 91 |
+
"use_rslora": true
|
| 92 |
+
}
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"name": "dora",
|
| 96 |
+
"args": {
|
| 97 |
+
"train_type": "lora",
|
| 98 |
+
"lora_rank": 8,
|
| 99 |
+
"lora_alpha": 32,
|
| 100 |
+
"use_dora": true
|
| 101 |
+
}
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"name": "lora+neftune",
|
| 105 |
+
"args": {
|
| 106 |
+
"train_type": "lora",
|
| 107 |
+
"lora_rank": 8,
|
| 108 |
+
"lora_alpha": 32,
|
| 109 |
+
"neftune_noise_alpha": 15.0
|
| 110 |
+
}
|
| 111 |
+
},
|
| 112 |
+
{
|
| 113 |
+
"name": "llamapro",
|
| 114 |
+
"args": {
|
| 115 |
+
"train_type": "llamapro",
|
| 116 |
+
"llamapro_num_new_blocks": "4"
|
| 117 |
+
}
|
| 118 |
+
},
|
| 119 |
+
{
|
| 120 |
+
"name": "full",
|
| 121 |
+
"requirements":{
|
| 122 |
+
"gpu": "1",
|
| 123 |
+
"ddp": "1"
|
| 124 |
+
},
|
| 125 |
+
"args": {
|
| 126 |
+
"train_type": "full"
|
| 127 |
+
}
|
| 128 |
+
},
|
| 129 |
+
{
|
| 130 |
+
"name": "reft",
|
| 131 |
+
"requirements":{
|
| 132 |
+
"gpu": "1",
|
| 133 |
+
"ddp": "1"
|
| 134 |
+
},
|
| 135 |
+
"args": {
|
| 136 |
+
"train_type": "reft",
|
| 137 |
+
"gradient_checkpointing": "false",
|
| 138 |
+
"loss_scale": "default"
|
| 139 |
+
}
|
| 140 |
+
},
|
| 141 |
+
{
|
| 142 |
+
"name": "full+galore128+quantize",
|
| 143 |
+
"requirements":{
|
| 144 |
+
"gpu": "1",
|
| 145 |
+
"ddp": "1"
|
| 146 |
+
},
|
| 147 |
+
"args": {
|
| 148 |
+
"train_type": "full",
|
| 149 |
+
"use_galore": "true",
|
| 150 |
+
"galore_rank": "128",
|
| 151 |
+
"galore_update_proj_gap": "200",
|
| 152 |
+
"galore_optim_per_parameter": "false",
|
| 153 |
+
"galore_with_embedding": "false",
|
| 154 |
+
"galore_quantization": "true"
|
| 155 |
+
}
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"name": "full+galore128+quantize+proj_quant",
|
| 159 |
+
"requirements":{
|
| 160 |
+
"gpu": "1",
|
| 161 |
+
"ddp": "1"
|
| 162 |
+
},
|
| 163 |
+
"args": {
|
| 164 |
+
"train_type": "full",
|
| 165 |
+
"use_galore": "true",
|
| 166 |
+
"galore_rank": "128",
|
| 167 |
+
"galore_update_proj_gap": "200",
|
| 168 |
+
"galore_optim_per_parameter": "false",
|
| 169 |
+
"galore_with_embedding": "false",
|
| 170 |
+
"galore_quantization": "true",
|
| 171 |
+
"galore_proj_quant": "true"
|
| 172 |
+
}
|
| 173 |
+
},
|
| 174 |
+
{
|
| 175 |
+
"name": "full+galore128",
|
| 176 |
+
"requirements":{
|
| 177 |
+
"gpu": "1",
|
| 178 |
+
"ddp": "1"
|
| 179 |
+
},
|
| 180 |
+
"args": {
|
| 181 |
+
"train_type": "full",
|
| 182 |
+
"use_galore": "true",
|
| 183 |
+
"galore_rank": "128",
|
| 184 |
+
"galore_update_proj_gap": "200",
|
| 185 |
+
"galore_optim_per_parameter": "false",
|
| 186 |
+
"galore_with_embedding": "false"
|
| 187 |
+
}
|
| 188 |
+
},
|
| 189 |
+
{
|
| 190 |
+
"name": "full+galore64",
|
| 191 |
+
"requirements":{
|
| 192 |
+
"gpu": "1",
|
| 193 |
+
"ddp": "1"
|
| 194 |
+
},
|
| 195 |
+
"args": {
|
| 196 |
+
"train_type": "full",
|
| 197 |
+
"use_galore": "true",
|
| 198 |
+
"galore_rank": "64",
|
| 199 |
+
"galore_update_proj_gap": "200",
|
| 200 |
+
"galore_optim_per_parameter": "false",
|
| 201 |
+
"galore_with_embedding": "false"
|
| 202 |
+
}
|
| 203 |
+
},
|
| 204 |
+
{
|
| 205 |
+
"name": "full+galore32",
|
| 206 |
+
"requirements":{
|
| 207 |
+
"gpu": "1",
|
| 208 |
+
"ddp": "1"
|
| 209 |
+
},
|
| 210 |
+
"args": {
|
| 211 |
+
"train_type": "full",
|
| 212 |
+
"use_galore": "true",
|
| 213 |
+
"galore_rank": "32",
|
| 214 |
+
"galore_update_proj_gap": "200",
|
| 215 |
+
"galore_optim_per_parameter": "false",
|
| 216 |
+
"galore_with_embedding": "false"
|
| 217 |
+
}
|
| 218 |
+
},
|
| 219 |
+
{
|
| 220 |
+
"name": "full+galore_emb",
|
| 221 |
+
"requirements":{
|
| 222 |
+
"gpu": "1",
|
| 223 |
+
"ddp": "1"
|
| 224 |
+
},
|
| 225 |
+
"args": {
|
| 226 |
+
"train_type": "full",
|
| 227 |
+
"use_galore": "true",
|
| 228 |
+
"galore_rank": "128",
|
| 229 |
+
"galore_update_proj_gap": "200",
|
| 230 |
+
"galore_optim_per_parameter": "false",
|
| 231 |
+
"galore_with_embedding": "true"
|
| 232 |
+
}
|
| 233 |
+
},
|
| 234 |
+
{
|
| 235 |
+
"name": "full+galore_perparam",
|
| 236 |
+
"requirements":{
|
| 237 |
+
"gpu": "1",
|
| 238 |
+
"ddp": "1"
|
| 239 |
+
},
|
| 240 |
+
"args": {
|
| 241 |
+
"train_type": "full",
|
| 242 |
+
"use_galore": "true",
|
| 243 |
+
"galore_rank": "128",
|
| 244 |
+
"galore_update_proj_gap": "200",
|
| 245 |
+
"galore_optim_per_parameter": "true",
|
| 246 |
+
"galore_with_embedding": "false"
|
| 247 |
+
}
|
| 248 |
+
},
|
| 249 |
+
{
|
| 250 |
+
"name": "adalora",
|
| 251 |
+
"args": {
|
| 252 |
+
"train_type": "adalora",
|
| 253 |
+
"lora_rank": 8,
|
| 254 |
+
"lora_alpha": 32
|
| 255 |
+
}
|
| 256 |
+
},
|
| 257 |
+
{
|
| 258 |
+
"name": "adapter",
|
| 259 |
+
"args": {
|
| 260 |
+
"train_type": "adapter"
|
| 261 |
+
}
|
| 262 |
+
},
|
| 263 |
+
{
|
| 264 |
+
"name": "full+lisa_2",
|
| 265 |
+
"info": "lisa 2layers + full",
|
| 266 |
+
"args": {
|
| 267 |
+
"train_type": "full",
|
| 268 |
+
"lisa_activated_layers": 2,
|
| 269 |
+
"lisa_step_interval": 20
|
| 270 |
+
}
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"name": "full+lisa_4",
|
| 274 |
+
"info": "lisa 4layers + full",
|
| 275 |
+
"args": {
|
| 276 |
+
"train_type": "full",
|
| 277 |
+
"lisa_activated_layers": 4,
|
| 278 |
+
"lisa_step_interval": 20
|
| 279 |
+
}
|
| 280 |
+
},
|
| 281 |
+
{
|
| 282 |
+
"name": "unsloth+lora+q4",
|
| 283 |
+
"info": "unsloth lora quantization bit 4",
|
| 284 |
+
"args": {
|
| 285 |
+
"train_type": "lora",
|
| 286 |
+
"tuner_backend": "unsloth",
|
| 287 |
+
"quantization_bit": 4,
|
| 288 |
+
"model": "LLM-Research/Meta-Llama-3-8B-Instruct"
|
| 289 |
+
}
|
| 290 |
+
},
|
| 291 |
+
{
|
| 292 |
+
"name": "unsloth+full",
|
| 293 |
+
"info": "unsloth full",
|
| 294 |
+
"args": {
|
| 295 |
+
"train_type": "full",
|
| 296 |
+
"tuner_backend": "unsloth",
|
| 297 |
+
"model_type": "LLM-Research/Meta-Llama-3-8B-Instruct"
|
| 298 |
+
}
|
| 299 |
+
}
|
| 300 |
+
]
|
| 301 |
+
}
|
scripts/benchmark/exp.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
import argparse
|
| 3 |
+
import os
|
| 4 |
+
import os.path
|
| 5 |
+
|
| 6 |
+
from exp_utils import ExpManager, find_all_config
|
| 7 |
+
|
| 8 |
+
from swift.utils import *
|
| 9 |
+
|
| 10 |
+
logger = get_logger()
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def parse_args():
|
| 14 |
+
parser = argparse.ArgumentParser(description='Simple args for swift experiments.')
|
| 15 |
+
parser.add_argument(
|
| 16 |
+
'--config',
|
| 17 |
+
type=str,
|
| 18 |
+
default=None,
|
| 19 |
+
required=True,
|
| 20 |
+
help='The experiment config file',
|
| 21 |
+
)
|
| 22 |
+
parser.add_argument(
|
| 23 |
+
'--save_dir',
|
| 24 |
+
type=str,
|
| 25 |
+
default='./experiment',
|
| 26 |
+
required=False,
|
| 27 |
+
help='The experiment output folder',
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
args = parser.parse_args()
|
| 31 |
+
return args
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def llm_exp():
|
| 35 |
+
args = parse_args()
|
| 36 |
+
config: str = args.config
|
| 37 |
+
config = config.split(',')
|
| 38 |
+
os.makedirs(args.save_dir, exist_ok=True)
|
| 39 |
+
all_configs = []
|
| 40 |
+
if not isinstance(config, list):
|
| 41 |
+
config = [config]
|
| 42 |
+
for dir_or_file in config:
|
| 43 |
+
all_configs.extend(find_all_config(dir_or_file))
|
| 44 |
+
args.config = all_configs
|
| 45 |
+
exp_manager = ExpManager()
|
| 46 |
+
exp_manager.begin(args)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
if __name__ == '__main__':
|
| 50 |
+
llm_exp()
|
scripts/benchmark/exp_utils.py
ADDED
|
@@ -0,0 +1,391 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import shutil
|
| 3 |
+
import subprocess
|
| 4 |
+
import time
|
| 5 |
+
from collections import deque
|
| 6 |
+
from copy import deepcopy
|
| 7 |
+
from dataclasses import asdict, dataclass, field
|
| 8 |
+
from typing import Any, Dict, List
|
| 9 |
+
|
| 10 |
+
import json
|
| 11 |
+
import torch
|
| 12 |
+
|
| 13 |
+
from swift.llm import ExportArguments
|
| 14 |
+
from swift.utils import find_free_port, get_device_count, get_logger
|
| 15 |
+
|
| 16 |
+
logger = get_logger()
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@dataclass
|
| 20 |
+
class Experiment:
|
| 21 |
+
|
| 22 |
+
name: str
|
| 23 |
+
|
| 24 |
+
cmd: str
|
| 25 |
+
|
| 26 |
+
group: str
|
| 27 |
+
|
| 28 |
+
requirements: Dict = field(default_factory=dict)
|
| 29 |
+
|
| 30 |
+
eval_requirements: Dict = field(default_factory=dict)
|
| 31 |
+
|
| 32 |
+
eval_dataset: List = field(default_factory=list)
|
| 33 |
+
|
| 34 |
+
args: Dict = field(default_factory=dict)
|
| 35 |
+
|
| 36 |
+
env: Dict = field(default_factory=dict)
|
| 37 |
+
|
| 38 |
+
record: Dict = field(default_factory=dict)
|
| 39 |
+
|
| 40 |
+
create_time: float = None
|
| 41 |
+
|
| 42 |
+
runtime: Dict = field(default_factory=dict)
|
| 43 |
+
|
| 44 |
+
input_args: Any = None
|
| 45 |
+
|
| 46 |
+
do_eval = False
|
| 47 |
+
|
| 48 |
+
def __init__(self,
|
| 49 |
+
name,
|
| 50 |
+
cmd,
|
| 51 |
+
group,
|
| 52 |
+
requirements=None,
|
| 53 |
+
eval_requirements=None,
|
| 54 |
+
eval_dataset=None,
|
| 55 |
+
args=None,
|
| 56 |
+
input_args=None,
|
| 57 |
+
**kwargs):
|
| 58 |
+
self.name = name
|
| 59 |
+
self.cmd = cmd
|
| 60 |
+
self.group = group
|
| 61 |
+
self.requirements = requirements or {}
|
| 62 |
+
self.args = args or {}
|
| 63 |
+
self.record = {}
|
| 64 |
+
self.env = {}
|
| 65 |
+
self.runtime = {}
|
| 66 |
+
self.input_args = input_args
|
| 67 |
+
self.eval_requirements = eval_requirements or {}
|
| 68 |
+
self.eval_dataset = eval_dataset or []
|
| 69 |
+
if self.cmd == 'eval':
|
| 70 |
+
self.do_eval = True
|
| 71 |
+
|
| 72 |
+
def load(self, _json):
|
| 73 |
+
self.name = _json['name']
|
| 74 |
+
self.cmd = _json['cmd']
|
| 75 |
+
self.requirements = _json['requirements']
|
| 76 |
+
self.args = _json['args']
|
| 77 |
+
self.record = _json['record']
|
| 78 |
+
self.env = _json['env']
|
| 79 |
+
self.create_time = _json['create_time']
|
| 80 |
+
|
| 81 |
+
@property
|
| 82 |
+
def priority(self):
|
| 83 |
+
return self.requirements.get('gpu', 0)
|
| 84 |
+
|
| 85 |
+
def to_dict(self):
|
| 86 |
+
_dict = asdict(self)
|
| 87 |
+
_dict.pop('runtime')
|
| 88 |
+
_dict.pop('input_args')
|
| 89 |
+
return _dict
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
class ExpManager:
|
| 93 |
+
|
| 94 |
+
RESULT_FILE = 'result.jsonl'
|
| 95 |
+
|
| 96 |
+
def __init__(self):
|
| 97 |
+
self.exps = []
|
| 98 |
+
|
| 99 |
+
def assert_gpu_not_overlap(self):
|
| 100 |
+
all_gpus = set()
|
| 101 |
+
for exp in self.exps:
|
| 102 |
+
gpus = exp.runtime['env']['CUDA_VISIBLE_DEVICES'].split(',')
|
| 103 |
+
if all_gpus & set(gpus):
|
| 104 |
+
raise ValueError(f'GPU overlap: {self.exps}!')
|
| 105 |
+
all_gpus.update(gpus)
|
| 106 |
+
|
| 107 |
+
def run(self, exp: Experiment):
|
| 108 |
+
if os.path.exists(os.path.join(exp.input_args.save_dir, exp.name + '.json')):
|
| 109 |
+
with open(os.path.join(exp.input_args.save_dir, exp.name + '.json'), 'r', encoding='utf-8') as f:
|
| 110 |
+
_json = json.load(f)
|
| 111 |
+
if exp.eval_dataset and 'eval_result' not in _json['record']:
|
| 112 |
+
if not exp.do_eval:
|
| 113 |
+
logger.info(f'Experiment {exp.name} need eval, load from file.')
|
| 114 |
+
exp.load(_json)
|
| 115 |
+
exp.do_eval = True
|
| 116 |
+
else:
|
| 117 |
+
logger.warn(f'Experiment {exp.name} already done, skip')
|
| 118 |
+
return
|
| 119 |
+
|
| 120 |
+
if exp.do_eval:
|
| 121 |
+
runtime = self._build_eval_cmd(exp)
|
| 122 |
+
exp.runtime = runtime
|
| 123 |
+
envs = deepcopy(runtime.get('env', {}))
|
| 124 |
+
envs.update(os.environ)
|
| 125 |
+
logger.info(f'Running cmd: {runtime["running_cmd"]}, env: {runtime.get("env", {})}')
|
| 126 |
+
os.makedirs('exp', exist_ok=True)
|
| 127 |
+
log_file = os.path.join('exp', f'{exp.name}.eval.log')
|
| 128 |
+
exp.handler = subprocess.Popen(runtime['running_cmd'] + f' > {log_file} 2>&1', env=envs, shell=True)
|
| 129 |
+
self.exps.append(exp)
|
| 130 |
+
self.assert_gpu_not_overlap()
|
| 131 |
+
return
|
| 132 |
+
|
| 133 |
+
if any([exp.name == e.name for e in self.exps]):
|
| 134 |
+
raise ValueError(f'Why exp name duplicate? {exp.name}')
|
| 135 |
+
elif exp.cmd == 'export' and any([exp.cmd == 'export' for exp in self.exps]): # noqa
|
| 136 |
+
raise AssertionError('Cannot run parallel export task.')
|
| 137 |
+
else:
|
| 138 |
+
exp.create_time = time.time()
|
| 139 |
+
runtime = self._build_cmd(exp)
|
| 140 |
+
exp.runtime = runtime
|
| 141 |
+
envs = deepcopy(runtime.get('env', {}))
|
| 142 |
+
envs.update(os.environ)
|
| 143 |
+
logger.info(f'Running cmd: {runtime["running_cmd"]}, env: {runtime.get("env", {})}')
|
| 144 |
+
os.makedirs('exp', exist_ok=True)
|
| 145 |
+
log_file = os.path.join('exp', f'{exp.name}.{exp.cmd}.log')
|
| 146 |
+
exp.handler = subprocess.Popen(runtime['running_cmd'] + f' > {log_file} 2>&1', env=envs, shell=True)
|
| 147 |
+
self.exps.append(exp)
|
| 148 |
+
self.assert_gpu_not_overlap()
|
| 149 |
+
|
| 150 |
+
def _build_eval_cmd(self, exp: Experiment):
|
| 151 |
+
gpu = exp.eval_requirements.get('gpu', None)
|
| 152 |
+
env = {}
|
| 153 |
+
allocated = []
|
| 154 |
+
if gpu:
|
| 155 |
+
allocated = self._find_free_gpu(int(gpu))
|
| 156 |
+
assert allocated, 'No free gpu for now!'
|
| 157 |
+
allocated = [str(gpu) for gpu in allocated]
|
| 158 |
+
env['CUDA_VISIBLE_DEVICES'] = ','.join(allocated)
|
| 159 |
+
|
| 160 |
+
best_model_checkpoint = exp.record.get('best_model_checkpoint')
|
| 161 |
+
eval_dataset = exp.eval_dataset
|
| 162 |
+
if best_model_checkpoint is not None:
|
| 163 |
+
if not os.path.exists(os.path.join(best_model_checkpoint, 'args.json')):
|
| 164 |
+
cmd = f'swift eval --ckpt_dir {best_model_checkpoint} ' \
|
| 165 |
+
+ f'--infer_backend pt --train_type full --eval_dataset {" ".join(eval_dataset)}'
|
| 166 |
+
else:
|
| 167 |
+
cmd = f'swift eval --model {exp.args.get("model")} --infer_backend pt ' \
|
| 168 |
+
f'--eval_dataset {" ".join(eval_dataset)}'
|
| 169 |
+
|
| 170 |
+
return {
|
| 171 |
+
'running_cmd': cmd,
|
| 172 |
+
'gpu': allocated,
|
| 173 |
+
'env': env,
|
| 174 |
+
}
|
| 175 |
+
|
| 176 |
+
def _build_cmd(self, exp: Experiment):
|
| 177 |
+
gpu = exp.requirements.get('gpu', None)
|
| 178 |
+
env = {}
|
| 179 |
+
allocated = []
|
| 180 |
+
if gpu:
|
| 181 |
+
allocated = self._find_free_gpu(int(gpu))
|
| 182 |
+
assert allocated, 'No free gpu for now!'
|
| 183 |
+
allocated = [str(gpu) for gpu in allocated]
|
| 184 |
+
env['CUDA_VISIBLE_DEVICES'] = ','.join(allocated)
|
| 185 |
+
if int(exp.requirements.get('ddp', 1)) > 1:
|
| 186 |
+
env['NPROC_PER_NODE'] = exp.requirements.get('ddp')
|
| 187 |
+
env['MASTER_PORT'] = str(find_free_port())
|
| 188 |
+
|
| 189 |
+
if exp.cmd == 'sft':
|
| 190 |
+
from swift.llm import TrainArguments
|
| 191 |
+
args = exp.args
|
| 192 |
+
sft_args = TrainArguments(**args)
|
| 193 |
+
args['output_dir'] = sft_args.output_dir
|
| 194 |
+
args['logging_dir'] = sft_args.logging_dir
|
| 195 |
+
args['add_version'] = False
|
| 196 |
+
os.makedirs(sft_args.output_dir, exist_ok=True)
|
| 197 |
+
os.makedirs(sft_args.logging_dir, exist_ok=True)
|
| 198 |
+
cmd = 'swift sft '
|
| 199 |
+
for key, value in args.items():
|
| 200 |
+
cmd += f' --{key} {value}'
|
| 201 |
+
elif exp.cmd == 'rlhf':
|
| 202 |
+
from swift.llm import RLHFArguments
|
| 203 |
+
args = exp.args
|
| 204 |
+
rlhf_args = RLHFArguments(**args)
|
| 205 |
+
args['output_dir'] = rlhf_args.output_dir
|
| 206 |
+
args['logging_dir'] = rlhf_args.logging_dir
|
| 207 |
+
args['add_version'] = False
|
| 208 |
+
os.makedirs(rlhf_args.output_dir, exist_ok=True)
|
| 209 |
+
os.makedirs(rlhf_args.logging_dir, exist_ok=True)
|
| 210 |
+
cmd = 'swift rlhf '
|
| 211 |
+
for key, value in args.items():
|
| 212 |
+
cmd += f' --{key} {value}'
|
| 213 |
+
elif exp.cmd == 'export':
|
| 214 |
+
args = exp.args
|
| 215 |
+
cmd = 'swift export '
|
| 216 |
+
for key, value in args.items():
|
| 217 |
+
cmd += f' --{key} {value}'
|
| 218 |
+
else:
|
| 219 |
+
raise ValueError(f'Unsupported cmd type: {exp.cmd}')
|
| 220 |
+
return {
|
| 221 |
+
'running_cmd': cmd,
|
| 222 |
+
'gpu': allocated,
|
| 223 |
+
'env': env,
|
| 224 |
+
'logging_dir': args.get('logging_dir'),
|
| 225 |
+
'output_dir': args.get('output_dir', args.get('ckpt_dir'))
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
def _find_free_gpu(self, n):
|
| 229 |
+
all_gpus = set()
|
| 230 |
+
for exp in self.exps:
|
| 231 |
+
all_gpus.update(exp.runtime.get('gpu', set()))
|
| 232 |
+
all_gpus = {int(g) for g in all_gpus}
|
| 233 |
+
free_gpu = set(range(get_device_count())) - all_gpus
|
| 234 |
+
if len(free_gpu) < n:
|
| 235 |
+
return None
|
| 236 |
+
return list(free_gpu)[:n]
|
| 237 |
+
|
| 238 |
+
def prepare_experiments(self, args: Any):
|
| 239 |
+
experiments = []
|
| 240 |
+
for config_file in args.config:
|
| 241 |
+
with open(config_file, 'r', encoding='utf-8') as f:
|
| 242 |
+
group = os.path.basename(config_file)
|
| 243 |
+
group = group[:-5]
|
| 244 |
+
content = json.load(f)
|
| 245 |
+
exps = content['experiment']
|
| 246 |
+
for exp in exps:
|
| 247 |
+
main_cfg = deepcopy(content)
|
| 248 |
+
name = exp['name']
|
| 249 |
+
cmd = main_cfg['cmd']
|
| 250 |
+
run_args = main_cfg['args']
|
| 251 |
+
env = main_cfg.get('env', {})
|
| 252 |
+
requirements = main_cfg.get('requirements', {})
|
| 253 |
+
eval_requirements = main_cfg.get('eval_requirements', {})
|
| 254 |
+
eval_dataset = main_cfg.get('eval_dataset', {})
|
| 255 |
+
if 'args' in exp:
|
| 256 |
+
run_args.update(exp['args'])
|
| 257 |
+
if 'requirements' in exp:
|
| 258 |
+
requirements.update(exp['requirements'])
|
| 259 |
+
if 'env' in exp:
|
| 260 |
+
env.update(exp['env'])
|
| 261 |
+
experiments.append(
|
| 262 |
+
Experiment(
|
| 263 |
+
group=group,
|
| 264 |
+
name=name,
|
| 265 |
+
cmd=cmd,
|
| 266 |
+
args=run_args,
|
| 267 |
+
env=env,
|
| 268 |
+
requirements=requirements,
|
| 269 |
+
eval_requirements=eval_requirements,
|
| 270 |
+
eval_dataset=eval_dataset,
|
| 271 |
+
input_args=args))
|
| 272 |
+
return experiments
|
| 273 |
+
|
| 274 |
+
@staticmethod
|
| 275 |
+
def _get_metric(exp: Experiment):
|
| 276 |
+
if exp.do_eval:
|
| 277 |
+
if os.path.isfile(os.path.join('exp', f'{exp.name}.eval.log')):
|
| 278 |
+
with open(os.path.join('exp', f'{exp.name}.eval.log'), 'r', encoding='utf-8') as f:
|
| 279 |
+
for line in f.readlines():
|
| 280 |
+
if 'Final report:' in line:
|
| 281 |
+
return json.loads(line.split('Final report:')[1].replace('\'', '"'))
|
| 282 |
+
elif exp.cmd == 'export':
|
| 283 |
+
exp_args = ExportArguments(**exp.args)
|
| 284 |
+
if exp_args.quant_bits > 0:
|
| 285 |
+
if exp_args.ckpt_dir is None:
|
| 286 |
+
path = f'{exp_args.model_type}-{exp_args.quant_method}-int{exp_args.quant_bits}'
|
| 287 |
+
else:
|
| 288 |
+
ckpt_dir, ckpt_name = os.path.split(exp_args.ckpt_dir)
|
| 289 |
+
path = os.path.join(ckpt_dir, f'{ckpt_name}-{exp_args.quant_method}-int{exp_args.quant_bits}')
|
| 290 |
+
else:
|
| 291 |
+
ckpt_dir, ckpt_name = os.path.split(exp_args.ckpt_dir)
|
| 292 |
+
path = os.path.join(ckpt_dir, f'{ckpt_name}-merged')
|
| 293 |
+
if os.path.exists(path):
|
| 294 |
+
shutil.rmtree(exp.name, ignore_errors=True)
|
| 295 |
+
os.makedirs(exp.name, exist_ok=True)
|
| 296 |
+
shutil.move(path, os.path.join(exp.name, path))
|
| 297 |
+
return {
|
| 298 |
+
'best_model_checkpoint': os.path.join(exp.name, path),
|
| 299 |
+
}
|
| 300 |
+
else:
|
| 301 |
+
logging_dir = exp.runtime.get('logging_dir')
|
| 302 |
+
logging_file = os.path.join(logging_dir, '..', 'logging.jsonl')
|
| 303 |
+
if os.path.isfile(logging_file):
|
| 304 |
+
with open(logging_file, 'r', encoding='utf-8') as f:
|
| 305 |
+
for line in f.readlines():
|
| 306 |
+
if 'model_info' in line:
|
| 307 |
+
return json.loads(line)
|
| 308 |
+
return None
|
| 309 |
+
|
| 310 |
+
@staticmethod
|
| 311 |
+
def write_record(exp: Experiment):
|
| 312 |
+
target_dir = exp.input_args.save_dir
|
| 313 |
+
file = os.path.join(target_dir, exp.name + '.json')
|
| 314 |
+
with open(file, 'w', encoding='utf-8') as f:
|
| 315 |
+
f.write(json.dumps(exp.to_dict()) + '\n')
|
| 316 |
+
|
| 317 |
+
def _poll(self):
|
| 318 |
+
while True:
|
| 319 |
+
time.sleep(5)
|
| 320 |
+
|
| 321 |
+
has_finished = False
|
| 322 |
+
for exp in self.exps:
|
| 323 |
+
rt = exp.handler.poll()
|
| 324 |
+
if rt is None:
|
| 325 |
+
continue
|
| 326 |
+
|
| 327 |
+
has_finished = True
|
| 328 |
+
if rt == 0:
|
| 329 |
+
if not exp.do_eval:
|
| 330 |
+
all_metric = self._get_metric(exp)
|
| 331 |
+
if all_metric:
|
| 332 |
+
exp.record.update(all_metric)
|
| 333 |
+
if exp.eval_dataset:
|
| 334 |
+
exp.do_eval = True
|
| 335 |
+
self.exp_queue.appendleft(exp)
|
| 336 |
+
self.write_record(exp)
|
| 337 |
+
else:
|
| 338 |
+
logger.error(f'Running {exp.name} task, but no result found')
|
| 339 |
+
else:
|
| 340 |
+
all_metric = self._get_metric(exp)
|
| 341 |
+
exp.record['eval_result'] = all_metric
|
| 342 |
+
if all_metric:
|
| 343 |
+
self.write_record(exp)
|
| 344 |
+
else:
|
| 345 |
+
logger.error(f'Running {exp.name} eval task, but no eval result found')
|
| 346 |
+
logger.info(f'Running {exp.name} finished with return code: {rt}')
|
| 347 |
+
|
| 348 |
+
if has_finished:
|
| 349 |
+
self.exps = [exp for exp in self.exps if exp.handler.poll() is None]
|
| 350 |
+
break
|
| 351 |
+
|
| 352 |
+
def begin(self, args: Any):
|
| 353 |
+
exps = self.prepare_experiments(args)
|
| 354 |
+
logger.info(f'all exps: {exps}')
|
| 355 |
+
exps.sort(key=lambda e: e.priority)
|
| 356 |
+
self.exp_queue = deque()
|
| 357 |
+
for exp in exps:
|
| 358 |
+
self.exp_queue.append(exp)
|
| 359 |
+
|
| 360 |
+
while len(self.exp_queue) or len(self.exps) > 0:
|
| 361 |
+
while len(self.exp_queue):
|
| 362 |
+
try:
|
| 363 |
+
logger.info(f'Running exp: {self.exp_queue[0].name}')
|
| 364 |
+
self.run(self.exp_queue[0])
|
| 365 |
+
except Exception as e:
|
| 366 |
+
if not isinstance(e, AssertionError):
|
| 367 |
+
logger.error(f'Adding exp {self.exp_queue[0].name} error because of:')
|
| 368 |
+
logger.error(e)
|
| 369 |
+
self.exp_queue.popleft()
|
| 370 |
+
else:
|
| 371 |
+
logger.info(f'Adding exp {self.exp_queue[0].name} error because of:', str(e))
|
| 372 |
+
if 'no free gpu' in str(e).lower():
|
| 373 |
+
break
|
| 374 |
+
else:
|
| 375 |
+
continue
|
| 376 |
+
else:
|
| 377 |
+
self.exp_queue.popleft()
|
| 378 |
+
self._poll()
|
| 379 |
+
logger.info(f'Run task finished because of exp queue: {self.exp_queue} and exps: {self.exps}')
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
def find_all_config(dir_or_file: str):
|
| 383 |
+
if os.path.isfile(dir_or_file):
|
| 384 |
+
return [dir_or_file]
|
| 385 |
+
else:
|
| 386 |
+
configs = []
|
| 387 |
+
for dirpath, dirnames, filenames in os.walk(dir_or_file):
|
| 388 |
+
for name in filenames:
|
| 389 |
+
if name.endswith('.json') and 'ipynb' not in dirpath:
|
| 390 |
+
configs.append(os.path.join(dirpath, name))
|
| 391 |
+
return configs
|
scripts/benchmark/generate_report.py
ADDED
|
@@ -0,0 +1,433 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
import dataclasses
|
| 3 |
+
import os
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from typing import Any, Dict, List
|
| 6 |
+
|
| 7 |
+
import json
|
| 8 |
+
import numpy as np
|
| 9 |
+
|
| 10 |
+
from swift.llm.template import split_str_parts_by
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@dataclass
|
| 14 |
+
class ModelOutput:
|
| 15 |
+
|
| 16 |
+
group: str = None
|
| 17 |
+
|
| 18 |
+
name: str = None
|
| 19 |
+
|
| 20 |
+
cmd: str = None
|
| 21 |
+
|
| 22 |
+
requirements: Dict[str, str] = dataclasses.field(default_factory=dict)
|
| 23 |
+
|
| 24 |
+
args: Dict[str, Any] = dataclasses.field(default_factory=dict)
|
| 25 |
+
|
| 26 |
+
memory: str = None
|
| 27 |
+
|
| 28 |
+
train_time: float = None
|
| 29 |
+
|
| 30 |
+
train_samples: int = None
|
| 31 |
+
|
| 32 |
+
train_samples_per_second: float = None
|
| 33 |
+
|
| 34 |
+
last_model_checkpoint: str = None
|
| 35 |
+
|
| 36 |
+
best_model_checkpoint: str = None
|
| 37 |
+
|
| 38 |
+
best_metric: Any = None
|
| 39 |
+
|
| 40 |
+
global_step: int = None
|
| 41 |
+
|
| 42 |
+
num_total_parameters: float = None
|
| 43 |
+
|
| 44 |
+
num_trainable_parameters: float = None
|
| 45 |
+
|
| 46 |
+
num_buffers: float = None
|
| 47 |
+
|
| 48 |
+
trainable_parameters_percentage: float = None
|
| 49 |
+
|
| 50 |
+
train_dataset_info: str = None
|
| 51 |
+
|
| 52 |
+
val_dataset_info: str = None
|
| 53 |
+
|
| 54 |
+
train_create_time: float = None
|
| 55 |
+
|
| 56 |
+
eval_tokens: int = None
|
| 57 |
+
|
| 58 |
+
eval_time: float = None
|
| 59 |
+
|
| 60 |
+
reports: Dict[str, Any] = None
|
| 61 |
+
|
| 62 |
+
train_loss: float = None
|
| 63 |
+
|
| 64 |
+
@property
|
| 65 |
+
def tuner_hyper_params(self):
|
| 66 |
+
hyper_params = ''
|
| 67 |
+
args = self.args
|
| 68 |
+
if 'sft_type' not in args:
|
| 69 |
+
return ''
|
| 70 |
+
if args['sft_type'] in ('lora', 'adalora', 'longlora'):
|
| 71 |
+
if 'lora_rank' in args:
|
| 72 |
+
hyper_params += f'rank={args["lora_rank"]}/' \
|
| 73 |
+
f'target={args["lora_target_modules"]}/' \
|
| 74 |
+
f'alpha={args["lora_alpha"]}/' \
|
| 75 |
+
f'lr_ratio={args.get("lora_lr_ratio", None)}/' \
|
| 76 |
+
f'use_rslora={args.get("use_rslora", False)}/' \
|
| 77 |
+
f'use_dora={args.get("use_dora", False)}'
|
| 78 |
+
else:
|
| 79 |
+
hyper_params = ''
|
| 80 |
+
if args['sft_type'] == 'full':
|
| 81 |
+
if 'use_galore' in args and args['use_galore'] == 'true':
|
| 82 |
+
hyper_params += f'galore_rank={args["galore_rank"]}/' \
|
| 83 |
+
f'galore_per_parameter={args["galore_optim_per_parameter"]}/' \
|
| 84 |
+
f'galore_with_embedding={args["galore_with_embedding"]}/'
|
| 85 |
+
if args['sft_type'] == 'llamapro':
|
| 86 |
+
hyper_params += f'num_blocks={args["llamapro_num_new_blocks"]}/'
|
| 87 |
+
if 'neftune_noise_alpha' in args and args['neftune_noise_alpha']:
|
| 88 |
+
hyper_params += f'neftune_noise_alpha={args["neftune_noise_alpha"]}/'
|
| 89 |
+
|
| 90 |
+
if hyper_params.endswith('/'):
|
| 91 |
+
hyper_params = hyper_params[:-1]
|
| 92 |
+
return hyper_params
|
| 93 |
+
|
| 94 |
+
@property
|
| 95 |
+
def hyper_parameters(self):
|
| 96 |
+
if 'learning_rate' not in self.args:
|
| 97 |
+
return ''
|
| 98 |
+
return f'lr={self.args["learning_rate"]}/' \
|
| 99 |
+
f'epoch={self.args["num_train_epochs"]}'
|
| 100 |
+
|
| 101 |
+
@property
|
| 102 |
+
def train_speed(self):
|
| 103 |
+
if self.train_samples_per_second:
|
| 104 |
+
return f'{self.train_samples_per_second:.2f}({self.train_samples} samples/{self.train_time:.2f} seconds)'
|
| 105 |
+
else:
|
| 106 |
+
return ''
|
| 107 |
+
|
| 108 |
+
@property
|
| 109 |
+
def infer_speed(self):
|
| 110 |
+
if self.eval_tokens:
|
| 111 |
+
return f'{self.eval_tokens / self.eval_time:.2f}({self.eval_tokens} tokens/{self.eval_time:.2f} seconds)'
|
| 112 |
+
return ''
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def generate_sft_report(outputs: List[ModelOutput]):
|
| 116 |
+
gsm8k_accs = []
|
| 117 |
+
arc_accs = []
|
| 118 |
+
ceval_accs = []
|
| 119 |
+
for output in outputs:
|
| 120 |
+
gsm8k_acc = None
|
| 121 |
+
arc_acc = None
|
| 122 |
+
ceval_acc = None
|
| 123 |
+
for report in (output.reports or []):
|
| 124 |
+
if report['name'] == 'gsm8k':
|
| 125 |
+
gsm8k_acc = report['score']
|
| 126 |
+
if report['name'] == 'arc':
|
| 127 |
+
arc_acc = report['score']
|
| 128 |
+
if report['name'] == 'ceval':
|
| 129 |
+
ceval_acc = report['score']
|
| 130 |
+
gsm8k_accs.append(gsm8k_acc)
|
| 131 |
+
arc_accs.append(arc_acc)
|
| 132 |
+
ceval_accs.append(ceval_acc)
|
| 133 |
+
|
| 134 |
+
tab = '| exp_name | model_type | dataset | ms-bench mix ratio | tuner | tuner_params | trainable params(M) | flash_attn | gradient_checkpointing | hypers | memory | train speed(samples/s) | infer speed(tokens/s) | train_loss | eval_loss | gsm8k weighted acc | arc weighted acc | ceval weighted acc |\n' \
|
| 135 |
+
'| -------- | ---------- | ------- | -------------------| ----- | ------------ | ------------------- | -----------| ---------------------- | ------ | ------ | ---------------------- | --------------------- | ---------- | --------- | ------------------ | ---------------- | ------------------ |\n' # noqa
|
| 136 |
+
min_best_metric = 999.
|
| 137 |
+
min_train_loss = 999.
|
| 138 |
+
if outputs:
|
| 139 |
+
min_best_metric = min([output.best_metric or 999. for output in outputs])
|
| 140 |
+
min_train_loss = min([output.train_loss or 999. for output in outputs])
|
| 141 |
+
|
| 142 |
+
max_gsm8k = 0.0
|
| 143 |
+
if gsm8k_accs:
|
| 144 |
+
max_gsm8k = max([gsm8k or 0. for gsm8k in gsm8k_accs])
|
| 145 |
+
|
| 146 |
+
max_arc = 0.0
|
| 147 |
+
if arc_accs:
|
| 148 |
+
max_arc = max([arc or 0. for arc in arc_accs])
|
| 149 |
+
|
| 150 |
+
max_ceval = 0.0
|
| 151 |
+
if ceval_accs:
|
| 152 |
+
max_ceval = max([ceval or 0. for ceval in ceval_accs])
|
| 153 |
+
|
| 154 |
+
for output, gsm8k_acc, arc_acc, ceval_acc in zip(outputs, gsm8k_accs, arc_accs, ceval_accs):
|
| 155 |
+
use_flash_attn = output.args.get('use_flash_attn', '')
|
| 156 |
+
use_gc = output.args.get('gradient_checkpointing', '')
|
| 157 |
+
memory = output.memory
|
| 158 |
+
train_speed = output.train_speed
|
| 159 |
+
infer_speed = output.infer_speed
|
| 160 |
+
|
| 161 |
+
is_best_metric = np.isclose(min_best_metric, output.best_metric or 999.0)
|
| 162 |
+
is_best_loss = np.isclose(min_train_loss, output.train_loss or 999.0)
|
| 163 |
+
is_best_gsm8k = np.isclose(max_gsm8k, gsm8k_acc or 0.0)
|
| 164 |
+
is_best_arc = np.isclose(max_arc, arc_acc or 0.0)
|
| 165 |
+
is_best_ceval = np.isclose(max_ceval, ceval_acc or 0.0)
|
| 166 |
+
|
| 167 |
+
if not is_best_metric:
|
| 168 |
+
best_metric = '' if not output.best_metric else f'{output.best_metric:.2f}'
|
| 169 |
+
else:
|
| 170 |
+
best_metric = '' if not output.best_metric else f'**{output.best_metric:.2f}**'
|
| 171 |
+
|
| 172 |
+
if not is_best_loss:
|
| 173 |
+
train_loss = '' if not output.train_loss else f'{output.train_loss:.2f}'
|
| 174 |
+
else:
|
| 175 |
+
train_loss = '' if not output.train_loss else f'**{output.train_loss:.2f}**'
|
| 176 |
+
|
| 177 |
+
if not is_best_gsm8k:
|
| 178 |
+
gsm8k_acc = '' if not gsm8k_acc else f'{gsm8k_acc:.3f}'
|
| 179 |
+
else:
|
| 180 |
+
gsm8k_acc = '' if not gsm8k_acc else f'**{gsm8k_acc:.3f}**'
|
| 181 |
+
|
| 182 |
+
if not is_best_arc:
|
| 183 |
+
arc_acc = '' if not arc_acc else f'{arc_acc:.3f}'
|
| 184 |
+
else:
|
| 185 |
+
arc_acc = '' if not arc_acc else f'**{arc_acc:.3f}**'
|
| 186 |
+
|
| 187 |
+
if not is_best_ceval:
|
| 188 |
+
ceval_acc = '' if not ceval_acc else f'{ceval_acc:.3f}'
|
| 189 |
+
else:
|
| 190 |
+
ceval_acc = '' if not ceval_acc else f'**{ceval_acc:.3f}**'
|
| 191 |
+
|
| 192 |
+
line = f'|{output.name}|' \
|
| 193 |
+
f'{output.args["model_type"]}|' \
|
| 194 |
+
f'{output.args.get("dataset")}|' \
|
| 195 |
+
f'{output.args.get("train_dataset_mix_ratio", 0.)}|' \
|
| 196 |
+
f'{output.args.get("sft_type")}|' \
|
| 197 |
+
f'{output.tuner_hyper_params}|' \
|
| 198 |
+
f'{output.num_trainable_parameters}({output.trainable_parameters_percentage})|' \
|
| 199 |
+
f'{use_flash_attn}|' \
|
| 200 |
+
f'{use_gc}|' \
|
| 201 |
+
f'{output.hyper_parameters}|' \
|
| 202 |
+
f'{memory}|' \
|
| 203 |
+
f'{train_speed}|' \
|
| 204 |
+
f'{infer_speed}|' \
|
| 205 |
+
f'{best_metric}|' \
|
| 206 |
+
f'{train_loss}|' \
|
| 207 |
+
f'{gsm8k_acc}|' \
|
| 208 |
+
f'{arc_acc}|' \
|
| 209 |
+
f'{ceval_acc}|\n'
|
| 210 |
+
tab += line
|
| 211 |
+
return tab
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def generate_export_report(outputs: List[ModelOutput]):
|
| 215 |
+
tab = '| exp_name | model_type | calibration dataset | quantization method | quantization bits | infer speed(tokens/s) | gsm8k weighted acc | arc weighted acc | ceval weighted acc |\n' \
|
| 216 |
+
'| -------- | ---------- | ------------------- | ------------------- | ----------------- | --------------------- | ------------------ | ---------------- | ------------------ |\n' # noqa
|
| 217 |
+
|
| 218 |
+
gsm8k_accs = []
|
| 219 |
+
arc_accs = []
|
| 220 |
+
ceval_accs = []
|
| 221 |
+
for output in outputs:
|
| 222 |
+
gsm8k_acc = None
|
| 223 |
+
arc_acc = None
|
| 224 |
+
ceval_acc = None
|
| 225 |
+
for report in (output.reports or []):
|
| 226 |
+
if report['name'] == 'gsm8k':
|
| 227 |
+
gsm8k_acc = report['score']
|
| 228 |
+
if report['name'] == 'arc':
|
| 229 |
+
arc_acc = report['score']
|
| 230 |
+
if report['name'] == 'ceval':
|
| 231 |
+
ceval_acc = report['score']
|
| 232 |
+
gsm8k_accs.append(gsm8k_acc)
|
| 233 |
+
arc_accs.append(arc_acc)
|
| 234 |
+
ceval_accs.append(ceval_acc)
|
| 235 |
+
|
| 236 |
+
max_gsm8k = 0.0
|
| 237 |
+
if gsm8k_accs:
|
| 238 |
+
max_gsm8k = max([gsm8k or 0. for gsm8k in gsm8k_accs])
|
| 239 |
+
|
| 240 |
+
max_arc = 0.0
|
| 241 |
+
if arc_accs:
|
| 242 |
+
max_arc = max([arc or 0. for arc in arc_accs])
|
| 243 |
+
|
| 244 |
+
max_ceval = 0.0
|
| 245 |
+
if ceval_accs:
|
| 246 |
+
max_ceval = max([ceval or 0. for ceval in ceval_accs])
|
| 247 |
+
|
| 248 |
+
for output, gsm8k_acc, arc_acc, ceval_acc in zip(outputs, gsm8k_accs, arc_accs, ceval_accs):
|
| 249 |
+
infer_speed = output.infer_speed
|
| 250 |
+
is_best_gsm8k = np.isclose(max_gsm8k, gsm8k_acc or 0.0)
|
| 251 |
+
is_best_arc = np.isclose(max_arc, arc_acc or 0.0)
|
| 252 |
+
is_best_ceval = np.isclose(max_ceval, ceval_acc or 0.0)
|
| 253 |
+
|
| 254 |
+
if not is_best_gsm8k:
|
| 255 |
+
gsm8k_acc = '' if not gsm8k_acc else f'{gsm8k_acc:.3f}'
|
| 256 |
+
else:
|
| 257 |
+
gsm8k_acc = '' if not gsm8k_acc else f'**{gsm8k_acc:.3f}**'
|
| 258 |
+
|
| 259 |
+
if not is_best_arc:
|
| 260 |
+
arc_acc = '' if not arc_acc else f'{arc_acc:.3f}'
|
| 261 |
+
else:
|
| 262 |
+
arc_acc = '' if not arc_acc else f'**{arc_acc:.3f}**'
|
| 263 |
+
|
| 264 |
+
if not is_best_ceval:
|
| 265 |
+
ceval_acc = '' if not ceval_acc else f'{ceval_acc:.3f}'
|
| 266 |
+
else:
|
| 267 |
+
ceval_acc = '' if not ceval_acc else f'**{ceval_acc:.3f}**'
|
| 268 |
+
|
| 269 |
+
if output.train_dataset_info:
|
| 270 |
+
dataset_info = f'{output.args["dataset"]}/{output.train_dataset_info}'
|
| 271 |
+
else:
|
| 272 |
+
dataset_info = f'{output.args["dataset"]}'
|
| 273 |
+
line = f'|{output.name}|' \
|
| 274 |
+
f'{output.args["model_type"]}|' \
|
| 275 |
+
f'{dataset_info}|' \
|
| 276 |
+
f'{output.args["quant_method"]}|' \
|
| 277 |
+
f'{output.args["quant_bits"]}|' \
|
| 278 |
+
f'{infer_speed}|' \
|
| 279 |
+
f'{gsm8k_acc}|' \
|
| 280 |
+
f'{arc_acc}|' \
|
| 281 |
+
f'{ceval_acc}|\n'
|
| 282 |
+
tab += line
|
| 283 |
+
return tab
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def parse_output(file):
|
| 287 |
+
with open(file, 'r', encoding='utf-8') as f:
|
| 288 |
+
content = json.load(f)
|
| 289 |
+
|
| 290 |
+
name = content['name']
|
| 291 |
+
group = content['group']
|
| 292 |
+
cmd = content['cmd']
|
| 293 |
+
requirements = content['requirements']
|
| 294 |
+
args = content['args']
|
| 295 |
+
create_time = float(content.get('create_time') or 0)
|
| 296 |
+
content = content['record']
|
| 297 |
+
if cmd == 'export':
|
| 298 |
+
best_model_checkpoint = content['best_model_checkpoint']
|
| 299 |
+
eval_tokens = 0
|
| 300 |
+
eval_time = 0.0
|
| 301 |
+
eval_result = None
|
| 302 |
+
if 'eval_result' in content:
|
| 303 |
+
eval_result = content['eval_result']
|
| 304 |
+
eval_tokens = eval_result['generation_info']['tokens']
|
| 305 |
+
eval_time = eval_result['generation_info']['time']
|
| 306 |
+
eval_result = eval_result['report']
|
| 307 |
+
return ModelOutput(
|
| 308 |
+
group=group,
|
| 309 |
+
name=name,
|
| 310 |
+
cmd=cmd,
|
| 311 |
+
requirements=requirements,
|
| 312 |
+
args=args,
|
| 313 |
+
best_model_checkpoint=best_model_checkpoint,
|
| 314 |
+
eval_time=eval_time,
|
| 315 |
+
eval_tokens=eval_tokens,
|
| 316 |
+
reports=eval_result,
|
| 317 |
+
)
|
| 318 |
+
else:
|
| 319 |
+
memory = None
|
| 320 |
+
train_time = None
|
| 321 |
+
train_samples = None
|
| 322 |
+
train_samples_per_second = None
|
| 323 |
+
last_model_checkpoint = None
|
| 324 |
+
best_model_checkpoint = None
|
| 325 |
+
best_metric = None
|
| 326 |
+
global_step = None
|
| 327 |
+
train_dataset_info = None
|
| 328 |
+
val_dataset_info = None
|
| 329 |
+
num_trainable_parameters = None
|
| 330 |
+
num_buffers = None
|
| 331 |
+
trainable_parameters_percentage = None
|
| 332 |
+
num_total_parameters = None
|
| 333 |
+
train_loss = None
|
| 334 |
+
if 'memory' in content:
|
| 335 |
+
memory = content['memory']
|
| 336 |
+
memory = '/'.join(memory.values())
|
| 337 |
+
if 'train_time' in content:
|
| 338 |
+
train_time = content['train_time']['train_runtime']
|
| 339 |
+
train_samples = content['train_time']['n_train_samples']
|
| 340 |
+
train_samples_per_second = content['train_time']['train_samples_per_second']
|
| 341 |
+
if 'last_model_checkpoint' in content:
|
| 342 |
+
last_model_checkpoint = content['last_model_checkpoint']
|
| 343 |
+
if 'best_model_checkpoint' in content:
|
| 344 |
+
best_model_checkpoint = content['best_model_checkpoint']
|
| 345 |
+
if 'best_metric' in content:
|
| 346 |
+
best_metric = content['best_metric']
|
| 347 |
+
if 'log_history' in content:
|
| 348 |
+
train_loss = content['log_history'][-1]['train_loss']
|
| 349 |
+
if 'global_step' in content:
|
| 350 |
+
global_step = content['global_step']
|
| 351 |
+
if 'dataset_info' in content:
|
| 352 |
+
train_dataset_info = content['dataset_info'].get('train_dataset')
|
| 353 |
+
val_dataset_info = content['dataset_info'].get('val_dataset')
|
| 354 |
+
if 'model_info' in content:
|
| 355 |
+
# model_info like: SwiftModel: 6758.4041M Params (19.9885M Trainable [0.2958%]), 16.7793M Buffers.
|
| 356 |
+
str_dict = split_str_parts_by(content['model_info'], [
|
| 357 |
+
'SwiftModel:', 'CausalLM:', 'Seq2SeqLM:', 'LMHeadModel:', 'M Params (', 'M Trainable [', ']), ',
|
| 358 |
+
'M Buffers.'
|
| 359 |
+
])
|
| 360 |
+
str_dict = {c['key']: c['content'] for c in str_dict}
|
| 361 |
+
if 'SwiftModel:' in str_dict:
|
| 362 |
+
num_total_parameters = float(str_dict['SwiftModel:'])
|
| 363 |
+
elif 'CausalLM:' in str_dict:
|
| 364 |
+
num_total_parameters = float(str_dict['CausalLM:'])
|
| 365 |
+
elif 'Seq2SeqLM:' in str_dict:
|
| 366 |
+
num_total_parameters = float(str_dict['Seq2SeqLM:'])
|
| 367 |
+
elif 'LMHeadModel:' in str_dict:
|
| 368 |
+
num_total_parameters = float(str_dict['LMHeadModel:'])
|
| 369 |
+
num_trainable_parameters = float(str_dict['M Params ('])
|
| 370 |
+
num_buffers = float(str_dict[']), '])
|
| 371 |
+
trainable_parameters_percentage = str_dict['M Trainable [']
|
| 372 |
+
|
| 373 |
+
eval_tokens = 0
|
| 374 |
+
eval_time = 0.0
|
| 375 |
+
eval_result = None
|
| 376 |
+
if 'eval_result' in content:
|
| 377 |
+
eval_result = content['eval_result']
|
| 378 |
+
eval_tokens = eval_result['generation_info']['tokens']
|
| 379 |
+
eval_time = eval_result['generation_info']['time']
|
| 380 |
+
eval_result = eval_result['report']
|
| 381 |
+
|
| 382 |
+
return ModelOutput(
|
| 383 |
+
group=group,
|
| 384 |
+
name=name,
|
| 385 |
+
cmd=cmd,
|
| 386 |
+
requirements=requirements,
|
| 387 |
+
args=args,
|
| 388 |
+
memory=memory,
|
| 389 |
+
train_time=train_time,
|
| 390 |
+
train_samples=train_samples,
|
| 391 |
+
train_samples_per_second=train_samples_per_second,
|
| 392 |
+
last_model_checkpoint=last_model_checkpoint,
|
| 393 |
+
best_model_checkpoint=best_model_checkpoint,
|
| 394 |
+
best_metric=best_metric,
|
| 395 |
+
global_step=global_step,
|
| 396 |
+
train_dataset_info=train_dataset_info,
|
| 397 |
+
val_dataset_info=val_dataset_info,
|
| 398 |
+
train_create_time=create_time,
|
| 399 |
+
num_total_parameters=num_total_parameters,
|
| 400 |
+
num_trainable_parameters=num_trainable_parameters,
|
| 401 |
+
num_buffers=num_buffers,
|
| 402 |
+
trainable_parameters_percentage=trainable_parameters_percentage,
|
| 403 |
+
eval_time=eval_time,
|
| 404 |
+
eval_tokens=eval_tokens,
|
| 405 |
+
reports=eval_result,
|
| 406 |
+
train_loss=train_loss,
|
| 407 |
+
)
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
def generate_reports():
|
| 411 |
+
outputs = []
|
| 412 |
+
for dirs, _, files in os.walk('./experiment'):
|
| 413 |
+
for file in files:
|
| 414 |
+
abs_file = os.path.join(dirs, file)
|
| 415 |
+
if not abs_file.endswith('.json') or 'ipynb' in abs_file:
|
| 416 |
+
continue
|
| 417 |
+
|
| 418 |
+
outputs.append(parse_output(abs_file))
|
| 419 |
+
|
| 420 |
+
all_groups = set([output.group for output in outputs])
|
| 421 |
+
for group in all_groups:
|
| 422 |
+
group_outputs = [output for output in outputs if output.group == group]
|
| 423 |
+
print(f'=================Printing the sft cmd result of exp {group}==================\n\n')
|
| 424 |
+
print(generate_sft_report([output for output in group_outputs if output.cmd in ('sft', 'eval')]))
|
| 425 |
+
# print(f'=================Printing the dpo result of exp {group}==================')
|
| 426 |
+
# print(generate_dpo_report([output for output in outputs if output.cmd == 'dpo']))
|
| 427 |
+
print(f'=================Printing the export cmd result of exp {group}==================\n\n')
|
| 428 |
+
print(generate_export_report([output for output in group_outputs if output.cmd == 'export']))
|
| 429 |
+
print('=================Printing done==================\n\n')
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
if __name__ == '__main__':
|
| 433 |
+
generate_reports()
|
scripts/utils/plot_loss.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
from swift.utils import plot_images
|
| 4 |
+
|
| 5 |
+
ckpt_dir = 'output/xxx/vx-xxx'
|
| 6 |
+
if __name__ == '__main__':
|
| 7 |
+
images_dir = os.path.join(ckpt_dir, 'images')
|
| 8 |
+
tb_dir = os.path.join(ckpt_dir, 'runs')
|
| 9 |
+
plot_images(images_dir, tb_dir, ['train/loss'], 0.9)
|
scripts/utils/run_dataset_info.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import re
|
| 3 |
+
|
| 4 |
+
import numpy as np
|
| 5 |
+
|
| 6 |
+
from swift.llm import DATASET_MAPPING, EncodePreprocessor, get_model_tokenizer, get_template, load_dataset
|
| 7 |
+
from swift.utils import stat_array
|
| 8 |
+
|
| 9 |
+
os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def get_cache_mapping(fpath):
|
| 13 |
+
with open(fpath, 'r', encoding='utf-8') as f:
|
| 14 |
+
text = f.read()
|
| 15 |
+
idx = text.find('| Dataset ID |')
|
| 16 |
+
text = text[idx:]
|
| 17 |
+
text_list = text.split('\n')[2:]
|
| 18 |
+
cache_mapping = {} # dataset_id -> (dataset_size, stat)
|
| 19 |
+
for text in text_list:
|
| 20 |
+
if not text:
|
| 21 |
+
continue
|
| 22 |
+
items = text.split('|')
|
| 23 |
+
key = items[1] if items[1] != '-' else items[6]
|
| 24 |
+
key = re.search(r'\[(.+?)\]', key).group(1)
|
| 25 |
+
stat = items[3:5]
|
| 26 |
+
if stat[0] == '-':
|
| 27 |
+
stat = ('huge dataset', '-')
|
| 28 |
+
cache_mapping[key] = stat
|
| 29 |
+
return cache_mapping
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def get_dataset_id(key):
|
| 33 |
+
for dataset_id in key:
|
| 34 |
+
if dataset_id is not None:
|
| 35 |
+
break
|
| 36 |
+
return dataset_id
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def run_dataset(key, template, cache_mapping):
|
| 40 |
+
ms_id, hf_id, _ = key
|
| 41 |
+
dataset_meta = DATASET_MAPPING[key]
|
| 42 |
+
tags = ', '.join(tag for tag in dataset_meta.tags) or '-'
|
| 43 |
+
dataset_id = ms_id or hf_id
|
| 44 |
+
use_hf = ms_id is None
|
| 45 |
+
if ms_id is not None:
|
| 46 |
+
ms_id = f'[{ms_id}](https://modelscope.cn/datasets/{ms_id})'
|
| 47 |
+
else:
|
| 48 |
+
ms_id = '-'
|
| 49 |
+
if hf_id is not None:
|
| 50 |
+
hf_id = f'[{hf_id}](https://huggingface.co/datasets/{hf_id})'
|
| 51 |
+
else:
|
| 52 |
+
hf_id = '-'
|
| 53 |
+
subsets = '<br>'.join(subset.name for subset in dataset_meta.subsets)
|
| 54 |
+
|
| 55 |
+
if dataset_meta.huge_dataset:
|
| 56 |
+
dataset_size = 'huge dataset'
|
| 57 |
+
stat_str = '-'
|
| 58 |
+
elif dataset_id in cache_mapping:
|
| 59 |
+
dataset_size, stat_str = cache_mapping[dataset_id]
|
| 60 |
+
else:
|
| 61 |
+
num_proc = 4
|
| 62 |
+
dataset, _ = load_dataset(f'{dataset_id}:all', strict=False, num_proc=num_proc, use_hf=use_hf)
|
| 63 |
+
dataset_size = len(dataset)
|
| 64 |
+
random_state = np.random.RandomState(42)
|
| 65 |
+
idx_list = random_state.choice(dataset_size, size=min(dataset_size, 100000), replace=False)
|
| 66 |
+
encoded_dataset = EncodePreprocessor(template)(dataset.select(idx_list), num_proc=num_proc)
|
| 67 |
+
|
| 68 |
+
input_ids = encoded_dataset['input_ids']
|
| 69 |
+
token_len = [len(tokens) for tokens in input_ids]
|
| 70 |
+
stat = stat_array(token_len)[0]
|
| 71 |
+
stat_str = f"{stat['mean']:.1f}±{stat['std']:.1f}, min={stat['min']}, max={stat['max']}"
|
| 72 |
+
|
| 73 |
+
return f'|{ms_id}|{subsets}|{dataset_size}|{stat_str}|{tags}|{hf_id}|'
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def write_dataset_info() -> None:
|
| 77 |
+
fpaths = ['docs/source/Instruction/支持的模型和数据集.md', 'docs/source_en/Instruction/Supported-models-and-datasets.md']
|
| 78 |
+
cache_mapping = get_cache_mapping(fpaths[0])
|
| 79 |
+
res_text_list = []
|
| 80 |
+
res_text_list.append('| Dataset ID | Subset Name | Dataset Size | Statistic (token) | Tags | HF Dataset ID |')
|
| 81 |
+
res_text_list.append('| ---------- | ----------- | -------------| ------------------| ---- | ------------- |')
|
| 82 |
+
|
| 83 |
+
all_keys = list(DATASET_MAPPING.keys())
|
| 84 |
+
all_keys = sorted(all_keys, key=lambda x: get_dataset_id(x))
|
| 85 |
+
_, tokenizer = get_model_tokenizer('Qwen/Qwen2.5-7B-Instruct', load_model=False)
|
| 86 |
+
template = get_template(tokenizer.model_meta.template, tokenizer)
|
| 87 |
+
try:
|
| 88 |
+
for i, key in enumerate(all_keys):
|
| 89 |
+
res = run_dataset(key, template, cache_mapping)
|
| 90 |
+
res_text_list.append(res)
|
| 91 |
+
print(res)
|
| 92 |
+
finally:
|
| 93 |
+
for fpath in fpaths:
|
| 94 |
+
with open(fpath, 'r', encoding='utf-8') as f:
|
| 95 |
+
text = f.read()
|
| 96 |
+
idx = text.find('| Dataset ID |')
|
| 97 |
+
|
| 98 |
+
new_text = '\n'.join(res_text_list)
|
| 99 |
+
text = text[:idx] + new_text + '\n'
|
| 100 |
+
with open(fpath, 'w', encoding='utf-8') as f:
|
| 101 |
+
f.write(text)
|
| 102 |
+
print(f'数据集总数: {len(all_keys)}')
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
if __name__ == '__main__':
|
| 106 |
+
write_dataset_info()
|
scripts/utils/run_model_info.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any, List
|
| 2 |
+
|
| 3 |
+
from swift.llm import MODEL_MAPPING, TEMPLATE_MAPPING, ModelType, TemplateType
|
| 4 |
+
from swift.utils import is_megatron_available
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def get_url_suffix(model_id):
|
| 8 |
+
if ':' in model_id:
|
| 9 |
+
return model_id.split(':')[0]
|
| 10 |
+
return model_id
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def get_cache_mapping(fpath):
|
| 14 |
+
with open(fpath, 'r', encoding='utf-8') as f:
|
| 15 |
+
text = f.read()
|
| 16 |
+
idx = text.find('| Model ID |')
|
| 17 |
+
text = text[idx:]
|
| 18 |
+
text_list = text.split('\n')[2:]
|
| 19 |
+
cache_mapping = {}
|
| 20 |
+
for text in text_list:
|
| 21 |
+
if not text:
|
| 22 |
+
continue
|
| 23 |
+
items = text.split('|')
|
| 24 |
+
if len(items) < 6:
|
| 25 |
+
break
|
| 26 |
+
cache_mapping[items[1]] = items[5]
|
| 27 |
+
return cache_mapping
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def get_model_info_table():
|
| 31 |
+
fpaths = ['docs/source/Instruction/支持的模型和数据集.md', 'docs/source_en/Instruction/Supported-models-and-datasets.md']
|
| 32 |
+
cache_mapping = get_cache_mapping(fpaths[0])
|
| 33 |
+
end_words = [['### 多模态大模型', '## 数据集'], ['### Multimodal large models', '## Datasets']]
|
| 34 |
+
result = [
|
| 35 |
+
'| Model ID | Model Type | Default Template | '
|
| 36 |
+
'Requires | Support Megatron | Tags | HF Model ID |\n'
|
| 37 |
+
'| -------- | -----------| ---------------- | '
|
| 38 |
+
'-------- | ---------------- | ---- | ----------- |\n'
|
| 39 |
+
] * 2
|
| 40 |
+
res_llm: List[Any] = []
|
| 41 |
+
res_mllm: List[Any] = []
|
| 42 |
+
mg_count = 0
|
| 43 |
+
for template in TemplateType.get_template_name_list():
|
| 44 |
+
assert template in TEMPLATE_MAPPING
|
| 45 |
+
|
| 46 |
+
for model_type in ModelType.get_model_name_list():
|
| 47 |
+
model_meta = MODEL_MAPPING[model_type]
|
| 48 |
+
template = model_meta.template
|
| 49 |
+
for group in model_meta.model_groups:
|
| 50 |
+
for model in group.models:
|
| 51 |
+
ms_model_id = model.ms_model_id
|
| 52 |
+
hf_model_id = model.hf_model_id
|
| 53 |
+
if ms_model_id:
|
| 54 |
+
ms_model_id = f'[{ms_model_id}](https://modelscope.cn/models/{get_url_suffix(ms_model_id)})'
|
| 55 |
+
else:
|
| 56 |
+
ms_model_id = '-'
|
| 57 |
+
if hf_model_id:
|
| 58 |
+
hf_model_id = f'[{hf_model_id}](https://huggingface.co/{get_url_suffix(hf_model_id)})'
|
| 59 |
+
else:
|
| 60 |
+
hf_model_id = '-'
|
| 61 |
+
tags = ', '.join(group.tags or model_meta.tags) or '-'
|
| 62 |
+
requires = ', '.join(group.requires or model_meta.requires) or '-'
|
| 63 |
+
if is_megatron_available():
|
| 64 |
+
from swift.megatron import model
|
| 65 |
+
support_megatron = getattr(model_meta, 'support_megatron', False)
|
| 66 |
+
for word in ['gptq', 'awq', 'bnb', 'aqlm', 'int', 'nf4', 'fp8']:
|
| 67 |
+
if word in ms_model_id.lower():
|
| 68 |
+
support_megatron = False
|
| 69 |
+
break
|
| 70 |
+
support_megatron = '✔' if support_megatron else '✘'
|
| 71 |
+
else:
|
| 72 |
+
support_megatron = cache_mapping.get(ms_model_id, '✘')
|
| 73 |
+
if support_megatron == '✔':
|
| 74 |
+
mg_count += 1
|
| 75 |
+
r = f'|{ms_model_id}|{model_type}|{template}|{requires}|{support_megatron}|{tags}|{hf_model_id}|\n'
|
| 76 |
+
if model_meta.is_multimodal:
|
| 77 |
+
res_mllm.append(r)
|
| 78 |
+
else:
|
| 79 |
+
res_llm.append(r)
|
| 80 |
+
print(f'LLM总数: {len(res_llm)}, MLLM总数: {len(res_mllm)}, Megatron支持模型: {mg_count}')
|
| 81 |
+
text = ['', ''] # llm, mllm
|
| 82 |
+
for i, res in enumerate([res_llm, res_mllm]):
|
| 83 |
+
for r in res:
|
| 84 |
+
text[i] += r
|
| 85 |
+
result[i] += text[i]
|
| 86 |
+
|
| 87 |
+
for i, fpath in enumerate(fpaths):
|
| 88 |
+
with open(fpath, 'r', encoding='utf-8') as f:
|
| 89 |
+
text = f.read()
|
| 90 |
+
llm_start_idx = text.find('| Model ID |')
|
| 91 |
+
mllm_start_idx = text[llm_start_idx + 1:].find('| Model ID |') + llm_start_idx + 1
|
| 92 |
+
llm_end_idx = text.find(end_words[i][0])
|
| 93 |
+
mllm_end_idx = text.find(end_words[i][1])
|
| 94 |
+
output = text[:llm_start_idx] + result[0] + '\n\n' + text[llm_end_idx:mllm_start_idx] + result[
|
| 95 |
+
1] + '\n\n' + text[mllm_end_idx:]
|
| 96 |
+
with open(fpath, 'w', encoding='utf-8') as f:
|
| 97 |
+
f.write(output)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
if __name__ == '__main__':
|
| 101 |
+
get_model_info_table()
|
scripts/utils/run_template.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from swift.llm import TemplateType
|
| 2 |
+
|
| 3 |
+
if __name__ == '__main__':
|
| 4 |
+
template_name_list = TemplateType.get_template_name_list()
|
| 5 |
+
tn_gen = ', '.join([tn for tn in template_name_list if 'generation' in tn])
|
| 6 |
+
tn_chat = ', '.join([tn for tn in template_name_list if 'generation' not in tn])
|
| 7 |
+
print(f'Text Generation: {tn_gen}')
|
| 8 |
+
print(f'Chat: {tn_chat}')
|
swift/__init__.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
from typing import TYPE_CHECKING
|
| 3 |
+
|
| 4 |
+
from .utils.import_utils import _LazyModule
|
| 5 |
+
|
| 6 |
+
if TYPE_CHECKING:
|
| 7 |
+
from .version import __version__, __release_datetime__
|
| 8 |
+
from .tuners import (Adapter, AdapterConfig, AdapterModule, SwiftModel, LoRA, LoRAConfig, SWIFT_MAPPING,
|
| 9 |
+
AdaLoraConfig, LoftQConfig, LoHaConfig, LoKrConfig, LoraConfig, OFTConfig, PeftConfig,
|
| 10 |
+
PeftModel, PeftModelForCausalLM, ResTuningConfig, SideConfig, PeftModelForSeq2SeqLM,
|
| 11 |
+
PeftModelForSequenceClassification, PeftModelForTokenClassification, PrefixTuningConfig,
|
| 12 |
+
PromptEncoderConfig, PromptLearningConfig, PromptTuningConfig, get_peft_config, get_peft_model,
|
| 13 |
+
get_peft_model_state_dict, Prompt, PromptConfig, PromptModule, SwiftConfig, SwiftOutput, Swift,
|
| 14 |
+
SwiftTuners, LongLoRAConfig, LongLoRA, LongLoRAModelType, SCETuning, SCETuningConfig)
|
| 15 |
+
from .trainers import (EvaluationStrategy, FSDPOption, HPSearchBackend, HubStrategy, IntervalStrategy,
|
| 16 |
+
SchedulerType, ShardedDDPOption, TrainingArguments, Seq2SeqTrainingArguments, Trainer,
|
| 17 |
+
Seq2SeqTrainer)
|
| 18 |
+
from .utils import get_logger
|
| 19 |
+
else:
|
| 20 |
+
_import_structure = {
|
| 21 |
+
'version': ['__release_datetime__', '__version__'],
|
| 22 |
+
'tuners': [
|
| 23 |
+
'Adapter', 'AdapterConfig', 'AdapterModule', 'SwiftModel', 'LoRA', 'LoRAConfig', 'SWIFT_MAPPING',
|
| 24 |
+
'LoraConfig', 'AdaLoraConfig', 'LoftQConfig', 'LoHaConfig', 'LoKrConfig', 'OFTConfig', 'PeftConfig',
|
| 25 |
+
'ResTuningConfig', 'SideConfig', 'PeftModel', 'PeftModelForCausalLM', 'PeftModelForSeq2SeqLM',
|
| 26 |
+
'PeftModelForSequenceClassification', 'PeftModelForTokenClassification', 'PrefixTuningConfig',
|
| 27 |
+
'PromptEncoderConfig', 'PromptLearningConfig', 'PromptTuningConfig', 'get_peft_config', 'get_peft_model',
|
| 28 |
+
'get_peft_model_state_dict', 'Prompt', 'PromptConfig', 'PromptModule', 'SwiftConfig', 'SwiftOutput',
|
| 29 |
+
'Swift', 'SwiftTuners', 'LongLoRAConfig', 'LongLoRA', 'LongLoRAModelType', 'SCETuning', 'SCETuningConfig'
|
| 30 |
+
],
|
| 31 |
+
'trainers': [
|
| 32 |
+
'EvaluationStrategy',
|
| 33 |
+
'FSDPOption',
|
| 34 |
+
'HPSearchBackend',
|
| 35 |
+
'HubStrategy',
|
| 36 |
+
'IntervalStrategy',
|
| 37 |
+
'SchedulerType',
|
| 38 |
+
'ShardedDDPOption',
|
| 39 |
+
'TrainingArguments',
|
| 40 |
+
'Seq2SeqTrainingArguments',
|
| 41 |
+
'Trainer',
|
| 42 |
+
'Seq2SeqTrainer',
|
| 43 |
+
],
|
| 44 |
+
'utils': ['get_logger']
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
import sys
|
| 48 |
+
|
| 49 |
+
sys.modules[__name__] = _LazyModule(
|
| 50 |
+
__name__,
|
| 51 |
+
globals()['__file__'],
|
| 52 |
+
_import_structure,
|
| 53 |
+
module_spec=__spec__,
|
| 54 |
+
extra_objects={},
|
| 55 |
+
)
|
swift/cli/__init__.py
ADDED
|
File without changes
|
swift/cli/_megatron/__init__.py
ADDED
|
File without changes
|
swift/cli/_megatron/main.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
from typing import Dict
|
| 3 |
+
|
| 4 |
+
from swift.utils import get_logger
|
| 5 |
+
from ..main import cli_main as swift_cli_main
|
| 6 |
+
|
| 7 |
+
logger = get_logger()
|
| 8 |
+
|
| 9 |
+
ROUTE_MAPPING: Dict[str, str] = {
|
| 10 |
+
'sft': 'swift.cli._megatron.sft',
|
| 11 |
+
'pt': 'swift.cli._megatron.pt',
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def cli_main():
|
| 16 |
+
return swift_cli_main(ROUTE_MAPPING)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
if __name__ == '__main__':
|
| 20 |
+
cli_main()
|
swift/cli/_megatron/pt.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from swift.megatron import megatron_pt_main
|
| 2 |
+
|
| 3 |
+
if __name__ == '__main__':
|
| 4 |
+
megatron_pt_main()
|
swift/cli/_megatron/sft.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from swift.megatron import megatron_sft_main
|
| 2 |
+
|
| 3 |
+
if __name__ == '__main__':
|
| 4 |
+
megatron_sft_main()
|
swift/cli/app.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from swift.llm import app_main
|
| 2 |
+
|
| 3 |
+
if __name__ == '__main__':
|
| 4 |
+
app_main()
|
swift/cli/deploy.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
from swift.llm import deploy_main
|
| 3 |
+
|
| 4 |
+
if __name__ == '__main__':
|
| 5 |
+
deploy_main()
|