novateur commited on
Commit
4d479b7
·
verified ·
1 Parent(s): c4532b4

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +2 -0
  2. docs/resources/web-ui-en.jpg +3 -0
  3. docs/resources/web-ui.jpg +3 -0
  4. examples/train/seq_cls/qwen2_5/deploy.sh +8 -0
  5. examples/train/seq_cls/qwen2_5/infer.sh +5 -0
  6. examples/train/seq_cls/qwen2_5/sft.sh +28 -0
  7. examples/train/streaming/train.sh +17 -0
  8. examples/train/think_model/qwen3_demo1.sh +30 -0
  9. examples/train/think_model/qwen3_demo2.sh +29 -0
  10. examples/train/tuners/adalora/train.sh +16 -0
  11. examples/train/tuners/adapter/train.sh +16 -0
  12. examples/train/tuners/boft/train.sh +16 -0
  13. examples/train/tuners/bone/train.sh +16 -0
  14. examples/train/tuners/dora/train.sh +19 -0
  15. examples/train/tuners/galore/train_galore.sh +18 -0
  16. examples/train/tuners/galore/train_qgalore.sh +20 -0
  17. examples/train/tuners/lisa/train.sh +17 -0
  18. examples/train/tuners/llamapro/train.sh +17 -0
  19. examples/train/tuners/longlora/train.sh +16 -0
  20. examples/train/tuners/lora-ga/train.sh +33 -0
  21. examples/train/tuners/lora/train.sh +18 -0
  22. examples/train/tuners/neftune/train.sh +19 -0
  23. examples/train/tuners/olora/train.sh +19 -0
  24. examples/train/tuners/pissa/train.sh +19 -0
  25. examples/train/tuners/qlora/train.sh +19 -0
  26. examples/train/tuners/reft/train.sh +17 -0
  27. examples/train/tuners/unsloth/train.sh +19 -0
  28. requirements/docs.txt +8 -0
  29. requirements/eval.txt +2 -0
  30. requirements/framework.txt +38 -0
  31. requirements/install_all.sh +12 -0
  32. requirements/seq_parallel.txt +1 -0
  33. requirements/swanlab.txt +1 -0
  34. requirements/tests.txt +6 -0
  35. scripts/benchmark/config/tuner.json +301 -0
  36. scripts/benchmark/exp.py +50 -0
  37. scripts/benchmark/exp_utils.py +391 -0
  38. scripts/benchmark/generate_report.py +433 -0
  39. scripts/utils/plot_loss.py +9 -0
  40. scripts/utils/run_dataset_info.py +106 -0
  41. scripts/utils/run_model_info.py +101 -0
  42. scripts/utils/run_template.py +8 -0
  43. swift/__init__.py +55 -0
  44. swift/cli/__init__.py +0 -0
  45. swift/cli/_megatron/__init__.py +0 -0
  46. swift/cli/_megatron/main.py +20 -0
  47. swift/cli/_megatron/pt.py +4 -0
  48. swift/cli/_megatron/sft.py +4 -0
  49. swift/cli/app.py +4 -0
  50. swift/cli/deploy.py +5 -0
.gitattributes CHANGED
@@ -43,3 +43,5 @@ docs/resources/grpo_countdown.png filter=lfs diff=lfs merge=lfs -text
43
  docs/resources/grpo_geoqa.png filter=lfs diff=lfs merge=lfs -text
44
  docs/resources/grpo_openr1_multimodal.png filter=lfs diff=lfs merge=lfs -text
45
  docs/resources/kto_data.png filter=lfs diff=lfs merge=lfs -text
 
 
 
43
  docs/resources/grpo_geoqa.png filter=lfs diff=lfs merge=lfs -text
44
  docs/resources/grpo_openr1_multimodal.png filter=lfs diff=lfs merge=lfs -text
45
  docs/resources/kto_data.png filter=lfs diff=lfs merge=lfs -text
46
+ docs/resources/web-ui.jpg filter=lfs diff=lfs merge=lfs -text
47
+ docs/resources/web-ui-en.jpg filter=lfs diff=lfs merge=lfs -text
docs/resources/web-ui-en.jpg ADDED

Git LFS Details

  • SHA256: 6ad958b680650b4f900f1a99560e825975f1a5807702c6fbbaef8e4f0e2a5efe
  • Pointer size: 131 Bytes
  • Size of remote file: 175 kB
docs/resources/web-ui.jpg ADDED

Git LFS Details

  • SHA256: 5e83bb4b4ecda9386286b99c6e83551f1dd1fdcdaf7be2efa3117208e3806000
  • Pointer size: 131 Bytes
  • Size of remote file: 182 kB
examples/train/seq_cls/qwen2_5/deploy.sh ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ CUDA_VISIBLE_DEVICES=0 \
2
+ swift deploy \
3
+ --adapters output/vx-xxx/checkpoint-xxx
4
+
5
+ # curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/json" -d '{
6
+ # "model": "Qwen2.5-0.5B",
7
+ # "messages": [{"role": "user", "content": "包装差,容易被调包。"}]
8
+ # }'
examples/train/seq_cls/qwen2_5/infer.sh ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ CUDA_VISIBLE_DEVICES=0 \
2
+ swift infer \
3
+ --adapters output/vx-xxx/checkpoint-xxx \
4
+ --load_data_args true \
5
+ --max_batch_size 16
examples/train/seq_cls/qwen2_5/sft.sh ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # If `num_labels` is provided, it will be considered a classification task,
2
+ # and AutoModelForSequenceClassification will be used to load the model.
3
+ # You can also specify `--model Qwen/Qwen2.5-0.5B-Instruct --use_chat_template true`.
4
+ CUDA_VISIBLE_DEVICES=0 \
5
+ swift sft \
6
+ --model Qwen/Qwen2.5-0.5B \
7
+ --train_type lora \
8
+ --dataset 'DAMO_NLP/jd:cls#2000' \
9
+ --torch_dtype bfloat16 \
10
+ --num_train_epochs 1 \
11
+ --per_device_train_batch_size 1 \
12
+ --per_device_eval_batch_size 1 \
13
+ --learning_rate 1e-4 \
14
+ --lora_rank 8 \
15
+ --lora_alpha 32 \
16
+ --target_modules all-linear \
17
+ --gradient_accumulation_steps 16 \
18
+ --eval_steps 50 \
19
+ --save_steps 50 \
20
+ --save_total_limit 2 \
21
+ --logging_steps 5 \
22
+ --max_length 2048 \
23
+ --output_dir output \
24
+ --warmup_ratio 0.05 \
25
+ --dataloader_num_workers 4 \
26
+ --num_labels 2 \
27
+ --task_type seq_cls \
28
+ --use_chat_template false
examples/train/streaming/train.sh ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ CUDA_VISIBLE_DEVICES=0 \
2
+ swift sft \
3
+ --model Qwen/Qwen2.5-7B-Instruct \
4
+ --train_type lora \
5
+ --dataset 'swift/self-cognition#1000' \
6
+ --streaming true \
7
+ --max_steps 1000 \
8
+ --learning_rate 1e-4 \
9
+ --lora_rank 8 \
10
+ --lora_alpha 32 \
11
+ --gradient_accumulation_steps 16 \
12
+ --eval_steps 100 \
13
+ --save_steps 100 \
14
+ --save_total_limit 2 \
15
+ --logging_steps 5 \
16
+ --model_author swift \
17
+ --model_name swift-robot
examples/train/think_model/qwen3_demo1.sh ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # use `--loss_scale ignore_empty_think`
2
+ # Avoid losing the think capability by ignoring the loss of empty `<think>\n\n</think>\n\n`
3
+ # This method is also applicable to the Deepseek-R1 series of models.
4
+ CUDA_VISIBLE_DEVICES=0 \
5
+ swift sft \
6
+ --model Qwen/Qwen3-8B \
7
+ --train_type lora \
8
+ --dataset 'swift/Qwen3-SFT-Mixin#2000' \
9
+ 'swift/self-cognition:empty_think#600' \
10
+ --torch_dtype bfloat16 \
11
+ --num_train_epochs 1 \
12
+ --per_device_train_batch_size 1 \
13
+ --per_device_eval_batch_size 1 \
14
+ --learning_rate 1e-4 \
15
+ --lora_rank 8 \
16
+ --lora_alpha 32 \
17
+ --target_modules all-linear \
18
+ --gradient_accumulation_steps 16 \
19
+ --eval_steps 50 \
20
+ --save_steps 50 \
21
+ --save_total_limit 2 \
22
+ --logging_steps 5 \
23
+ --max_length 2048 \
24
+ --output_dir output \
25
+ --warmup_ratio 0.05 \
26
+ --dataloader_num_workers 4 \
27
+ --use_liger_kernel true \
28
+ --loss_scale ignore_empty_think \
29
+ --model_author swift \
30
+ --model_name swift-robot
examples/train/think_model/qwen3_demo2.sh ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # use `swift/self-cognition:qwen3`
2
+ # Avoid losing the thinking capability by appending `/no_think` to the dataset query.
3
+ # https://github.com/modelscope/ms-swift/blob/77985c2ccdac8ed4037174ee222e79d1f1d5059d/swift/llm/dataset/dataset/llm.py#L835
4
+ CUDA_VISIBLE_DEVICES=0 \
5
+ swift sft \
6
+ --model Qwen/Qwen3-8B \
7
+ --train_type lora \
8
+ --dataset 'swift/Qwen3-SFT-Mixin#2000' \
9
+ 'swift/self-cognition:qwen3#600' \
10
+ --torch_dtype bfloat16 \
11
+ --num_train_epochs 1 \
12
+ --per_device_train_batch_size 1 \
13
+ --per_device_eval_batch_size 1 \
14
+ --learning_rate 1e-4 \
15
+ --lora_rank 8 \
16
+ --lora_alpha 32 \
17
+ --target_modules all-linear \
18
+ --gradient_accumulation_steps 16 \
19
+ --eval_steps 50 \
20
+ --save_steps 50 \
21
+ --save_total_limit 2 \
22
+ --logging_steps 5 \
23
+ --max_length 2048 \
24
+ --output_dir output \
25
+ --warmup_ratio 0.05 \
26
+ --dataloader_num_workers 4 \
27
+ --use_liger_kernel true \
28
+ --model_author swift \
29
+ --model_name swift-robot
examples/train/tuners/adalora/train.sh ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 17GiB
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2.5-7B-Instruct \
5
+ --train_type adalora \
6
+ --dataset 'swift/self-cognition#1000' \
7
+ --num_train_epochs 1 \
8
+ --per_device_train_batch_size 1 \
9
+ --learning_rate 1e-4 \
10
+ --gradient_accumulation_steps 16 \
11
+ --eval_steps 100 \
12
+ --save_steps 100 \
13
+ --save_total_limit 2 \
14
+ --logging_steps 5 \
15
+ --model_author swift \
16
+ --model_name swift-robot
examples/train/tuners/adapter/train.sh ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 17GiB
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2.5-7B-Instruct \
5
+ --train_type adapter \
6
+ --dataset 'swift/self-cognition#1000' \
7
+ --num_train_epochs 1 \
8
+ --per_device_train_batch_size 1 \
9
+ --learning_rate 1e-4 \
10
+ --gradient_accumulation_steps 16 \
11
+ --eval_steps 100 \
12
+ --save_steps 100 \
13
+ --save_total_limit 2 \
14
+ --logging_steps 5 \
15
+ --model_author swift \
16
+ --model_name swift-robot
examples/train/tuners/boft/train.sh ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 17GiB
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2.5-7B-Instruct \
5
+ --train_type boft \
6
+ --dataset 'swift/self-cognition#1000' \
7
+ --num_train_epochs 1 \
8
+ --per_device_train_batch_size 1 \
9
+ --learning_rate 1e-4 \
10
+ --gradient_accumulation_steps 16 \
11
+ --eval_steps 100 \
12
+ --save_steps 100 \
13
+ --save_total_limit 2 \
14
+ --logging_steps 5 \
15
+ --model_author swift \
16
+ --model_name swift-robot
examples/train/tuners/bone/train.sh ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 17.3GiB
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2.5-7B-Instruct \
5
+ --train_type bone \
6
+ --dataset 'swift/self-cognition#1000' \
7
+ --num_train_epochs 1 \
8
+ --per_device_train_batch_size 1 \
9
+ --learning_rate 1e-4 \
10
+ --gradient_accumulation_steps 16 \
11
+ --eval_steps 100 \
12
+ --save_steps 100 \
13
+ --save_total_limit 2 \
14
+ --logging_steps 5 \
15
+ --model_author swift \
16
+ --model_name swift-robot
examples/train/tuners/dora/train.sh ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 17.2GiB
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2.5-7B-Instruct \
5
+ --train_type lora \
6
+ --use_dora true \
7
+ --dataset 'swift/self-cognition#1000' \
8
+ --num_train_epochs 1 \
9
+ --per_device_train_batch_size 1 \
10
+ --learning_rate 1e-4 \
11
+ --lora_rank 8 \
12
+ --lora_alpha 32 \
13
+ --gradient_accumulation_steps 16 \
14
+ --eval_steps 100 \
15
+ --save_steps 100 \
16
+ --save_total_limit 2 \
17
+ --logging_steps 5 \
18
+ --model_author swift \
19
+ --model_name swift-robot
examples/train/tuners/galore/train_galore.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 38GiB
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2.5-7B-Instruct \
5
+ --train_type full \
6
+ --dataset 'swift/self-cognition#1000' \
7
+ --num_train_epochs 1 \
8
+ --per_device_train_batch_size 1 \
9
+ --learning_rate 1e-5 \
10
+ --gradient_accumulation_steps 16 \
11
+ --eval_steps 100 \
12
+ --save_steps 100 \
13
+ --save_total_limit 2 \
14
+ --logging_steps 5 \
15
+ --model_author swift \
16
+ --model_name swift-robot \
17
+ --use_galore true \
18
+ --galore_optim_per_parameter true
examples/train/tuners/galore/train_qgalore.sh ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 35GiB
2
+ # pip install bitsandbytes==0.40.0
3
+ CUDA_VISIBLE_DEVICES=0 \
4
+ swift sft \
5
+ --model Qwen/Qwen2.5-7B-Instruct \
6
+ --train_type full \
7
+ --torch_dtype bfloat16 \
8
+ --dataset 'lvjianjin/AdvertiseGen#1000' \
9
+ --num_train_epochs 1 \
10
+ --per_device_train_batch_size 1 \
11
+ --learning_rate 1e-5 \
12
+ --gradient_accumulation_steps 16 \
13
+ --eval_steps 100 \
14
+ --save_steps 100 \
15
+ --save_total_limit 2 \
16
+ --logging_steps 5 \
17
+ --model_author swift \
18
+ --model_name swift-robot \
19
+ --use_galore true \
20
+ --galore_quantization true
examples/train/tuners/lisa/train.sh ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 29GiB
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2.5-7B-Instruct \
5
+ --train_type full \
6
+ --dataset 'swift/self-cognition#1000' \
7
+ --lisa_activated_layers 2 \
8
+ --num_train_epochs 1 \
9
+ --per_device_train_batch_size 1 \
10
+ --learning_rate 1e-5 \
11
+ --gradient_accumulation_steps 16 \
12
+ --eval_steps 100 \
13
+ --save_steps 100 \
14
+ --save_total_limit 2 \
15
+ --logging_steps 5 \
16
+ --model_author swift \
17
+ --model_name swift-robot
examples/train/tuners/llamapro/train.sh ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 25.4GiB
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2.5-7B-Instruct \
5
+ --train_type llamapro \
6
+ --dataset 'swift/self-cognition#1000' \
7
+ --llamapro_num_new_blocks 4 \
8
+ --num_train_epochs 1 \
9
+ --per_device_train_batch_size 1 \
10
+ --learning_rate 1e-4 \
11
+ --gradient_accumulation_steps 16 \
12
+ --eval_steps 100 \
13
+ --save_steps 100 \
14
+ --save_total_limit 2 \
15
+ --logging_steps 5 \
16
+ --model_author swift \
17
+ --model_name swift-robot
examples/train/tuners/longlora/train.sh ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ CUDA_VISIBLE_DEVICES=0 \
2
+ swift sft \
3
+ --model LLM-Research/Meta-Llama-3.1-8B-Instruct \
4
+ --train_type longlora \
5
+ --dataset 'AI-ModelScope/LongAlpaca-12k#1000' \
6
+ --num_train_epochs 1 \
7
+ --learning_rate 1e-4 \
8
+ --attn_impl flash_attn \
9
+ --gradient_accumulation_steps 16 \
10
+ --lora_rank 8 \
11
+ --lora_alpha 32 \
12
+ --eval_steps 100 \
13
+ --save_steps 100 \
14
+ --max_length 10000 \
15
+ --save_total_limit 2 \
16
+ --logging_steps 5
examples/train/tuners/lora-ga/train.sh ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Train
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2-1.5B-Instruct \
5
+ --train_type lora \
6
+ --dataset 'swift/self-cognition#1000' \
7
+ --num_train_epochs 1 \
8
+ --per_device_train_batch_size 1 \
9
+ --learning_rate 1e-4 \
10
+ --lora_rank 8 \
11
+ --lora_alpha 32 \
12
+ --init_weights lora-ga \
13
+ --lora_ga_batch_size 2 \
14
+ --lora_ga_iters 2 \
15
+ --lora_ga_max_length 1024 \
16
+ --lora_ga_direction ArB2r \
17
+ --lora_ga_scale stable \
18
+ --lora_ga_stable_gamma 16 \
19
+ --gradient_accumulation_steps 16 \
20
+ --eval_steps 100 \
21
+ --save_steps 100 \
22
+ --save_total_limit 2 \
23
+ --logging_steps 5 \
24
+ --model_author swift \
25
+ --model_name swift-robot
26
+
27
+ # Infer
28
+ # swift infer \
29
+ # --model Qwen/Qwen2-1.5B-Instruct \
30
+ # --ckpt_dir ./output/Qwen2-1.5B-Instruct/v0-20241214-191235/checkpoint-62/converted/default \
31
+ # --infer_backend pt \
32
+ # --stream true \
33
+ # --max_new_tokens 2048
examples/train/tuners/lora/train.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 17.2GiB
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2.5-7B-Instruct \
5
+ --train_type lora \
6
+ --dataset 'swift/self-cognition#1000' \
7
+ --num_train_epochs 1 \
8
+ --per_device_train_batch_size 1 \
9
+ --learning_rate 1e-4 \
10
+ --lora_rank 8 \
11
+ --lora_alpha 32 \
12
+ --gradient_accumulation_steps 16 \
13
+ --eval_steps 100 \
14
+ --save_steps 100 \
15
+ --save_total_limit 2 \
16
+ --logging_steps 5 \
17
+ --model_author swift \
18
+ --model_name swift-robot
examples/train/tuners/neftune/train.sh ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 17GiB
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2.5-7B-Instruct \
5
+ --train_type lora \
6
+ --dataset 'swift/self-cognition#1000' \
7
+ --num_train_epochs 1 \
8
+ --per_device_train_batch_size 1 \
9
+ --neftune_noise_alpha 15 \
10
+ --learning_rate 1e-4 \
11
+ --lora_rank 8 \
12
+ --lora_alpha 32 \
13
+ --gradient_accumulation_steps 16 \
14
+ --eval_steps 100 \
15
+ --save_steps 100 \
16
+ --save_total_limit 2 \
17
+ --logging_steps 5 \
18
+ --model_author swift \
19
+ --model_name swift-robot
examples/train/tuners/olora/train.sh ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 17GiB
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2.5-7B-Instruct \
5
+ --train_type lora \
6
+ --dataset 'swift/self-cognition#1000' \
7
+ --num_train_epochs 1 \
8
+ --per_device_train_batch_size 1 \
9
+ --learning_rate 1e-4 \
10
+ --lora_rank 8 \
11
+ --lora_alpha 32 \
12
+ --init_lora_weights olora \
13
+ --gradient_accumulation_steps 16 \
14
+ --eval_steps 100 \
15
+ --save_steps 100 \
16
+ --save_total_limit 2 \
17
+ --logging_steps 5 \
18
+ --model_author swift \
19
+ --model_name swift-robot
examples/train/tuners/pissa/train.sh ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 17GiB
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2.5-7B-Instruct \
5
+ --train_type lora \
6
+ --dataset 'swift/self-cognition#1000' \
7
+ --num_train_epochs 1 \
8
+ --per_device_train_batch_size 1 \
9
+ --learning_rate 1e-4 \
10
+ --lora_rank 8 \
11
+ --lora_alpha 32 \
12
+ --init_lora_weights pissa \
13
+ --gradient_accumulation_steps 16 \
14
+ --eval_steps 100 \
15
+ --save_steps 100 \
16
+ --save_total_limit 2 \
17
+ --logging_steps 5 \
18
+ --model_author swift \
19
+ --model_name swift-robot
examples/train/tuners/qlora/train.sh ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ CUDA_VISIBLE_DEVICES=0 \
2
+ swift sft \
3
+ --model Qwen/Qwen2.5-7B-Instruct \
4
+ --train_type lora \
5
+ --dataset 'swift/self-cognition#1000' \
6
+ --num_train_epochs 1 \
7
+ --per_device_train_batch_size 1 \
8
+ --learning_rate 1e-4 \
9
+ --lora_rank 8 \
10
+ --lora_alpha 32 \
11
+ --gradient_accumulation_steps 16 \
12
+ --eval_steps 100 \
13
+ --save_steps 100 \
14
+ --save_total_limit 2 \
15
+ --logging_steps 5 \
16
+ --model_author swift \
17
+ --model_name swift-robot \
18
+ --quant_bits 4 \
19
+ --quant_method bnb
examples/train/tuners/reft/train.sh ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ CUDA_VISIBLE_DEVICES=0 \
2
+ swift sft \
3
+ --model Qwen/Qwen2.5-7B-Instruct \
4
+ --train_type reft \
5
+ --dataset 'swift/self-cognition#1000' \
6
+ --reft_intervention_type 'LoreftIntervention' \
7
+ --num_train_epochs 1 \
8
+ --per_device_train_batch_size 1 \
9
+ --learning_rate 1e-4 \
10
+ --gradient_checkpointing false \
11
+ --gradient_accumulation_steps 16 \
12
+ --eval_steps 100 \
13
+ --save_steps 100 \
14
+ --save_total_limit 2 \
15
+ --logging_steps 5 \
16
+ --model_author swift \
17
+ --model_name swift-robot
examples/train/tuners/unsloth/train.sh ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 17GiB
2
+ CUDA_VISIBLE_DEVICES=0 \
3
+ swift sft \
4
+ --model Qwen/Qwen2.5-7B-Instruct \
5
+ --tuner_backend unsloth \
6
+ --train_type lora \
7
+ --dataset 'swift/self-cognition#1000' \
8
+ --num_train_epochs 1 \
9
+ --per_device_train_batch_size 1 \
10
+ --learning_rate 1e-4 \
11
+ --lora_rank 8 \
12
+ --lora_alpha 32 \
13
+ --gradient_accumulation_steps 16 \
14
+ --eval_steps 100 \
15
+ --save_steps 100 \
16
+ --save_total_limit 2 \
17
+ --logging_steps 5 \
18
+ --model_author swift \
19
+ --model_name swift-robot
requirements/docs.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ docutils>=0.16.0
2
+ myst_parser
3
+ recommonmark
4
+ sphinx>=5.3.0
5
+ sphinx-book-theme
6
+ sphinx-copybutton
7
+ sphinx-rtd-theme
8
+ sphinx_markdown_tables
requirements/eval.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ evalscope[opencompass]
2
+ evalscope[vlmeval]
requirements/framework.txt ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ accelerate
2
+ addict
3
+ aiohttp
4
+ attrdict
5
+ binpacking
6
+ charset_normalizer
7
+ cpm_kernels
8
+ dacite
9
+ datasets>=3.0,<3.4
10
+ einops
11
+ fastapi
12
+ gradio>=3.40.0
13
+ importlib_metadata
14
+ jieba
15
+ matplotlib
16
+ modelscope>=1.23
17
+ nltk
18
+ numpy<2.0
19
+ openai
20
+ oss2
21
+ pandas
22
+ peft>=0.11,<0.16
23
+ pillow
24
+ requests
25
+ rouge
26
+ safetensors
27
+ scipy
28
+ sentencepiece
29
+ simplejson>=3.3.0
30
+ sortedcontainers>=1.5.9
31
+ tensorboard
32
+ tiktoken
33
+ tqdm
34
+ transformers>=4.33,<4.53
35
+ transformers_stream_generator
36
+ trl>=0.13,<0.18
37
+ uvicorn
38
+ zstandard
requirements/install_all.sh ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # please use python=3.10, cuda12.*
2
+ # sh requirements/install_all.sh
3
+ pip install "vllm>=0.5.1" -U
4
+ pip install "lmdeploy>=0.5" -U --no-deps
5
+ pip install autoawq -U --no-deps
6
+ pip install auto_gptq optimum bitsandbytes -U
7
+ pip install git+https://github.com/modelscope/ms-swift.git
8
+ pip install timm -U
9
+ pip install deepspeed -U
10
+ pip install qwen_vl_utils qwen_omni_utils decord librosa pyav icecream soundfile -U
11
+ pip install liger_kernel nvitop pre-commit -U
12
+ # flash-attn: https://github.com/Dao-AILab/flash-attention/releases
requirements/seq_parallel.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ xtuner
requirements/swanlab.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ swanlab
requirements/tests.txt ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ expecttest
2
+ flake8
3
+ isort>=4.3.21
4
+ modelscope
5
+ pre-commit
6
+ yapf==0.30.0 # use fix version to ensure consistent auto-styling
scripts/benchmark/config/tuner.json ADDED
@@ -0,0 +1,301 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cmd": "sft",
3
+ "requirements":{
4
+ "gpu": "1",
5
+ "ddp": "1"
6
+ },
7
+ "eval_requirements": {
8
+ "gpu": "1"
9
+ },
10
+ "eval_dataset": ["ceval", "gsm8k", "arc"],
11
+ "args": {
12
+ "model": "Qwen/Qwen-7B-Chat",
13
+ "dataset": "iic/ms_agent",
14
+ "per_device_train_batch_size": 1,
15
+ "max_length": 2048,
16
+ "loss_scale": "react",
17
+ "gradient_accumulation_steps": 16,
18
+ "learning_rate": 5e-5,
19
+ "attn_impl": "flash_attn",
20
+ "eval_steps": 2000,
21
+ "save_steps": 2000,
22
+ "num_train_epochs": 2,
23
+ "gradient_checkpointing": true,
24
+ "weight_decay": 0.01,
25
+ "warmup_ratio": 0.03,
26
+ "save_total_limit": 2,
27
+ "logging_steps": 10
28
+ },
29
+ "experiment": [
30
+ {
31
+ "name": "lora",
32
+ "args": {
33
+ "train_type": "lora",
34
+ "lora_rank": 8,
35
+ "lora_alpha": 32
36
+ }
37
+ },
38
+ {
39
+ "name": "lora+packing",
40
+ "args": {
41
+ "train_type": "lora",
42
+ "lora_rank": 8,
43
+ "lora_alpha": 32,
44
+ "packing": true,
45
+ "eval_steps": 200,
46
+ "save_steps": 200
47
+ }
48
+ },
49
+ {
50
+ "name": "lora+packing+ddp",
51
+ "requirements":{
52
+ "gpu": "2",
53
+ "ddp": "2"
54
+ },
55
+ "args": {
56
+ "train_type": "lora",
57
+ "lora_rank": 8,
58
+ "lora_alpha": 32,
59
+ "packing": true,
60
+ "eval_steps": 100,
61
+ "save_steps": 100
62
+ }
63
+ },
64
+ {
65
+ "name": "lora+packing+lazytokenize",
66
+ "args": {
67
+ "train_type": "lora",
68
+ "lora_rank": 8,
69
+ "lora_alpha": 32,
70
+ "packing": true,
71
+ "lazy_tokenize": true,
72
+ "eval_steps": 200,
73
+ "save_steps": 200
74
+ }
75
+ },
76
+ {
77
+ "name": "lora+",
78
+ "args": {
79
+ "train_type": "lora",
80
+ "lora_rank": 8,
81
+ "lora_alpha": 32,
82
+ "lorap_lr_ratio": 16.0
83
+ }
84
+ },
85
+ {
86
+ "name": "rslora",
87
+ "args": {
88
+ "train_type": "lora",
89
+ "lora_rank": 8,
90
+ "lora_alpha": 32,
91
+ "use_rslora": true
92
+ }
93
+ },
94
+ {
95
+ "name": "dora",
96
+ "args": {
97
+ "train_type": "lora",
98
+ "lora_rank": 8,
99
+ "lora_alpha": 32,
100
+ "use_dora": true
101
+ }
102
+ },
103
+ {
104
+ "name": "lora+neftune",
105
+ "args": {
106
+ "train_type": "lora",
107
+ "lora_rank": 8,
108
+ "lora_alpha": 32,
109
+ "neftune_noise_alpha": 15.0
110
+ }
111
+ },
112
+ {
113
+ "name": "llamapro",
114
+ "args": {
115
+ "train_type": "llamapro",
116
+ "llamapro_num_new_blocks": "4"
117
+ }
118
+ },
119
+ {
120
+ "name": "full",
121
+ "requirements":{
122
+ "gpu": "1",
123
+ "ddp": "1"
124
+ },
125
+ "args": {
126
+ "train_type": "full"
127
+ }
128
+ },
129
+ {
130
+ "name": "reft",
131
+ "requirements":{
132
+ "gpu": "1",
133
+ "ddp": "1"
134
+ },
135
+ "args": {
136
+ "train_type": "reft",
137
+ "gradient_checkpointing": "false",
138
+ "loss_scale": "default"
139
+ }
140
+ },
141
+ {
142
+ "name": "full+galore128+quantize",
143
+ "requirements":{
144
+ "gpu": "1",
145
+ "ddp": "1"
146
+ },
147
+ "args": {
148
+ "train_type": "full",
149
+ "use_galore": "true",
150
+ "galore_rank": "128",
151
+ "galore_update_proj_gap": "200",
152
+ "galore_optim_per_parameter": "false",
153
+ "galore_with_embedding": "false",
154
+ "galore_quantization": "true"
155
+ }
156
+ },
157
+ {
158
+ "name": "full+galore128+quantize+proj_quant",
159
+ "requirements":{
160
+ "gpu": "1",
161
+ "ddp": "1"
162
+ },
163
+ "args": {
164
+ "train_type": "full",
165
+ "use_galore": "true",
166
+ "galore_rank": "128",
167
+ "galore_update_proj_gap": "200",
168
+ "galore_optim_per_parameter": "false",
169
+ "galore_with_embedding": "false",
170
+ "galore_quantization": "true",
171
+ "galore_proj_quant": "true"
172
+ }
173
+ },
174
+ {
175
+ "name": "full+galore128",
176
+ "requirements":{
177
+ "gpu": "1",
178
+ "ddp": "1"
179
+ },
180
+ "args": {
181
+ "train_type": "full",
182
+ "use_galore": "true",
183
+ "galore_rank": "128",
184
+ "galore_update_proj_gap": "200",
185
+ "galore_optim_per_parameter": "false",
186
+ "galore_with_embedding": "false"
187
+ }
188
+ },
189
+ {
190
+ "name": "full+galore64",
191
+ "requirements":{
192
+ "gpu": "1",
193
+ "ddp": "1"
194
+ },
195
+ "args": {
196
+ "train_type": "full",
197
+ "use_galore": "true",
198
+ "galore_rank": "64",
199
+ "galore_update_proj_gap": "200",
200
+ "galore_optim_per_parameter": "false",
201
+ "galore_with_embedding": "false"
202
+ }
203
+ },
204
+ {
205
+ "name": "full+galore32",
206
+ "requirements":{
207
+ "gpu": "1",
208
+ "ddp": "1"
209
+ },
210
+ "args": {
211
+ "train_type": "full",
212
+ "use_galore": "true",
213
+ "galore_rank": "32",
214
+ "galore_update_proj_gap": "200",
215
+ "galore_optim_per_parameter": "false",
216
+ "galore_with_embedding": "false"
217
+ }
218
+ },
219
+ {
220
+ "name": "full+galore_emb",
221
+ "requirements":{
222
+ "gpu": "1",
223
+ "ddp": "1"
224
+ },
225
+ "args": {
226
+ "train_type": "full",
227
+ "use_galore": "true",
228
+ "galore_rank": "128",
229
+ "galore_update_proj_gap": "200",
230
+ "galore_optim_per_parameter": "false",
231
+ "galore_with_embedding": "true"
232
+ }
233
+ },
234
+ {
235
+ "name": "full+galore_perparam",
236
+ "requirements":{
237
+ "gpu": "1",
238
+ "ddp": "1"
239
+ },
240
+ "args": {
241
+ "train_type": "full",
242
+ "use_galore": "true",
243
+ "galore_rank": "128",
244
+ "galore_update_proj_gap": "200",
245
+ "galore_optim_per_parameter": "true",
246
+ "galore_with_embedding": "false"
247
+ }
248
+ },
249
+ {
250
+ "name": "adalora",
251
+ "args": {
252
+ "train_type": "adalora",
253
+ "lora_rank": 8,
254
+ "lora_alpha": 32
255
+ }
256
+ },
257
+ {
258
+ "name": "adapter",
259
+ "args": {
260
+ "train_type": "adapter"
261
+ }
262
+ },
263
+ {
264
+ "name": "full+lisa_2",
265
+ "info": "lisa 2layers + full",
266
+ "args": {
267
+ "train_type": "full",
268
+ "lisa_activated_layers": 2,
269
+ "lisa_step_interval": 20
270
+ }
271
+ },
272
+ {
273
+ "name": "full+lisa_4",
274
+ "info": "lisa 4layers + full",
275
+ "args": {
276
+ "train_type": "full",
277
+ "lisa_activated_layers": 4,
278
+ "lisa_step_interval": 20
279
+ }
280
+ },
281
+ {
282
+ "name": "unsloth+lora+q4",
283
+ "info": "unsloth lora quantization bit 4",
284
+ "args": {
285
+ "train_type": "lora",
286
+ "tuner_backend": "unsloth",
287
+ "quantization_bit": 4,
288
+ "model": "LLM-Research/Meta-Llama-3-8B-Instruct"
289
+ }
290
+ },
291
+ {
292
+ "name": "unsloth+full",
293
+ "info": "unsloth full",
294
+ "args": {
295
+ "train_type": "full",
296
+ "tuner_backend": "unsloth",
297
+ "model_type": "LLM-Research/Meta-Llama-3-8B-Instruct"
298
+ }
299
+ }
300
+ ]
301
+ }
scripts/benchmark/exp.py ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ import argparse
3
+ import os
4
+ import os.path
5
+
6
+ from exp_utils import ExpManager, find_all_config
7
+
8
+ from swift.utils import *
9
+
10
+ logger = get_logger()
11
+
12
+
13
+ def parse_args():
14
+ parser = argparse.ArgumentParser(description='Simple args for swift experiments.')
15
+ parser.add_argument(
16
+ '--config',
17
+ type=str,
18
+ default=None,
19
+ required=True,
20
+ help='The experiment config file',
21
+ )
22
+ parser.add_argument(
23
+ '--save_dir',
24
+ type=str,
25
+ default='./experiment',
26
+ required=False,
27
+ help='The experiment output folder',
28
+ )
29
+
30
+ args = parser.parse_args()
31
+ return args
32
+
33
+
34
+ def llm_exp():
35
+ args = parse_args()
36
+ config: str = args.config
37
+ config = config.split(',')
38
+ os.makedirs(args.save_dir, exist_ok=True)
39
+ all_configs = []
40
+ if not isinstance(config, list):
41
+ config = [config]
42
+ for dir_or_file in config:
43
+ all_configs.extend(find_all_config(dir_or_file))
44
+ args.config = all_configs
45
+ exp_manager = ExpManager()
46
+ exp_manager.begin(args)
47
+
48
+
49
+ if __name__ == '__main__':
50
+ llm_exp()
scripts/benchmark/exp_utils.py ADDED
@@ -0,0 +1,391 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import shutil
3
+ import subprocess
4
+ import time
5
+ from collections import deque
6
+ from copy import deepcopy
7
+ from dataclasses import asdict, dataclass, field
8
+ from typing import Any, Dict, List
9
+
10
+ import json
11
+ import torch
12
+
13
+ from swift.llm import ExportArguments
14
+ from swift.utils import find_free_port, get_device_count, get_logger
15
+
16
+ logger = get_logger()
17
+
18
+
19
+ @dataclass
20
+ class Experiment:
21
+
22
+ name: str
23
+
24
+ cmd: str
25
+
26
+ group: str
27
+
28
+ requirements: Dict = field(default_factory=dict)
29
+
30
+ eval_requirements: Dict = field(default_factory=dict)
31
+
32
+ eval_dataset: List = field(default_factory=list)
33
+
34
+ args: Dict = field(default_factory=dict)
35
+
36
+ env: Dict = field(default_factory=dict)
37
+
38
+ record: Dict = field(default_factory=dict)
39
+
40
+ create_time: float = None
41
+
42
+ runtime: Dict = field(default_factory=dict)
43
+
44
+ input_args: Any = None
45
+
46
+ do_eval = False
47
+
48
+ def __init__(self,
49
+ name,
50
+ cmd,
51
+ group,
52
+ requirements=None,
53
+ eval_requirements=None,
54
+ eval_dataset=None,
55
+ args=None,
56
+ input_args=None,
57
+ **kwargs):
58
+ self.name = name
59
+ self.cmd = cmd
60
+ self.group = group
61
+ self.requirements = requirements or {}
62
+ self.args = args or {}
63
+ self.record = {}
64
+ self.env = {}
65
+ self.runtime = {}
66
+ self.input_args = input_args
67
+ self.eval_requirements = eval_requirements or {}
68
+ self.eval_dataset = eval_dataset or []
69
+ if self.cmd == 'eval':
70
+ self.do_eval = True
71
+
72
+ def load(self, _json):
73
+ self.name = _json['name']
74
+ self.cmd = _json['cmd']
75
+ self.requirements = _json['requirements']
76
+ self.args = _json['args']
77
+ self.record = _json['record']
78
+ self.env = _json['env']
79
+ self.create_time = _json['create_time']
80
+
81
+ @property
82
+ def priority(self):
83
+ return self.requirements.get('gpu', 0)
84
+
85
+ def to_dict(self):
86
+ _dict = asdict(self)
87
+ _dict.pop('runtime')
88
+ _dict.pop('input_args')
89
+ return _dict
90
+
91
+
92
+ class ExpManager:
93
+
94
+ RESULT_FILE = 'result.jsonl'
95
+
96
+ def __init__(self):
97
+ self.exps = []
98
+
99
+ def assert_gpu_not_overlap(self):
100
+ all_gpus = set()
101
+ for exp in self.exps:
102
+ gpus = exp.runtime['env']['CUDA_VISIBLE_DEVICES'].split(',')
103
+ if all_gpus & set(gpus):
104
+ raise ValueError(f'GPU overlap: {self.exps}!')
105
+ all_gpus.update(gpus)
106
+
107
+ def run(self, exp: Experiment):
108
+ if os.path.exists(os.path.join(exp.input_args.save_dir, exp.name + '.json')):
109
+ with open(os.path.join(exp.input_args.save_dir, exp.name + '.json'), 'r', encoding='utf-8') as f:
110
+ _json = json.load(f)
111
+ if exp.eval_dataset and 'eval_result' not in _json['record']:
112
+ if not exp.do_eval:
113
+ logger.info(f'Experiment {exp.name} need eval, load from file.')
114
+ exp.load(_json)
115
+ exp.do_eval = True
116
+ else:
117
+ logger.warn(f'Experiment {exp.name} already done, skip')
118
+ return
119
+
120
+ if exp.do_eval:
121
+ runtime = self._build_eval_cmd(exp)
122
+ exp.runtime = runtime
123
+ envs = deepcopy(runtime.get('env', {}))
124
+ envs.update(os.environ)
125
+ logger.info(f'Running cmd: {runtime["running_cmd"]}, env: {runtime.get("env", {})}')
126
+ os.makedirs('exp', exist_ok=True)
127
+ log_file = os.path.join('exp', f'{exp.name}.eval.log')
128
+ exp.handler = subprocess.Popen(runtime['running_cmd'] + f' > {log_file} 2>&1', env=envs, shell=True)
129
+ self.exps.append(exp)
130
+ self.assert_gpu_not_overlap()
131
+ return
132
+
133
+ if any([exp.name == e.name for e in self.exps]):
134
+ raise ValueError(f'Why exp name duplicate? {exp.name}')
135
+ elif exp.cmd == 'export' and any([exp.cmd == 'export' for exp in self.exps]): # noqa
136
+ raise AssertionError('Cannot run parallel export task.')
137
+ else:
138
+ exp.create_time = time.time()
139
+ runtime = self._build_cmd(exp)
140
+ exp.runtime = runtime
141
+ envs = deepcopy(runtime.get('env', {}))
142
+ envs.update(os.environ)
143
+ logger.info(f'Running cmd: {runtime["running_cmd"]}, env: {runtime.get("env", {})}')
144
+ os.makedirs('exp', exist_ok=True)
145
+ log_file = os.path.join('exp', f'{exp.name}.{exp.cmd}.log')
146
+ exp.handler = subprocess.Popen(runtime['running_cmd'] + f' > {log_file} 2>&1', env=envs, shell=True)
147
+ self.exps.append(exp)
148
+ self.assert_gpu_not_overlap()
149
+
150
+ def _build_eval_cmd(self, exp: Experiment):
151
+ gpu = exp.eval_requirements.get('gpu', None)
152
+ env = {}
153
+ allocated = []
154
+ if gpu:
155
+ allocated = self._find_free_gpu(int(gpu))
156
+ assert allocated, 'No free gpu for now!'
157
+ allocated = [str(gpu) for gpu in allocated]
158
+ env['CUDA_VISIBLE_DEVICES'] = ','.join(allocated)
159
+
160
+ best_model_checkpoint = exp.record.get('best_model_checkpoint')
161
+ eval_dataset = exp.eval_dataset
162
+ if best_model_checkpoint is not None:
163
+ if not os.path.exists(os.path.join(best_model_checkpoint, 'args.json')):
164
+ cmd = f'swift eval --ckpt_dir {best_model_checkpoint} ' \
165
+ + f'--infer_backend pt --train_type full --eval_dataset {" ".join(eval_dataset)}'
166
+ else:
167
+ cmd = f'swift eval --model {exp.args.get("model")} --infer_backend pt ' \
168
+ f'--eval_dataset {" ".join(eval_dataset)}'
169
+
170
+ return {
171
+ 'running_cmd': cmd,
172
+ 'gpu': allocated,
173
+ 'env': env,
174
+ }
175
+
176
+ def _build_cmd(self, exp: Experiment):
177
+ gpu = exp.requirements.get('gpu', None)
178
+ env = {}
179
+ allocated = []
180
+ if gpu:
181
+ allocated = self._find_free_gpu(int(gpu))
182
+ assert allocated, 'No free gpu for now!'
183
+ allocated = [str(gpu) for gpu in allocated]
184
+ env['CUDA_VISIBLE_DEVICES'] = ','.join(allocated)
185
+ if int(exp.requirements.get('ddp', 1)) > 1:
186
+ env['NPROC_PER_NODE'] = exp.requirements.get('ddp')
187
+ env['MASTER_PORT'] = str(find_free_port())
188
+
189
+ if exp.cmd == 'sft':
190
+ from swift.llm import TrainArguments
191
+ args = exp.args
192
+ sft_args = TrainArguments(**args)
193
+ args['output_dir'] = sft_args.output_dir
194
+ args['logging_dir'] = sft_args.logging_dir
195
+ args['add_version'] = False
196
+ os.makedirs(sft_args.output_dir, exist_ok=True)
197
+ os.makedirs(sft_args.logging_dir, exist_ok=True)
198
+ cmd = 'swift sft '
199
+ for key, value in args.items():
200
+ cmd += f' --{key} {value}'
201
+ elif exp.cmd == 'rlhf':
202
+ from swift.llm import RLHFArguments
203
+ args = exp.args
204
+ rlhf_args = RLHFArguments(**args)
205
+ args['output_dir'] = rlhf_args.output_dir
206
+ args['logging_dir'] = rlhf_args.logging_dir
207
+ args['add_version'] = False
208
+ os.makedirs(rlhf_args.output_dir, exist_ok=True)
209
+ os.makedirs(rlhf_args.logging_dir, exist_ok=True)
210
+ cmd = 'swift rlhf '
211
+ for key, value in args.items():
212
+ cmd += f' --{key} {value}'
213
+ elif exp.cmd == 'export':
214
+ args = exp.args
215
+ cmd = 'swift export '
216
+ for key, value in args.items():
217
+ cmd += f' --{key} {value}'
218
+ else:
219
+ raise ValueError(f'Unsupported cmd type: {exp.cmd}')
220
+ return {
221
+ 'running_cmd': cmd,
222
+ 'gpu': allocated,
223
+ 'env': env,
224
+ 'logging_dir': args.get('logging_dir'),
225
+ 'output_dir': args.get('output_dir', args.get('ckpt_dir'))
226
+ }
227
+
228
+ def _find_free_gpu(self, n):
229
+ all_gpus = set()
230
+ for exp in self.exps:
231
+ all_gpus.update(exp.runtime.get('gpu', set()))
232
+ all_gpus = {int(g) for g in all_gpus}
233
+ free_gpu = set(range(get_device_count())) - all_gpus
234
+ if len(free_gpu) < n:
235
+ return None
236
+ return list(free_gpu)[:n]
237
+
238
+ def prepare_experiments(self, args: Any):
239
+ experiments = []
240
+ for config_file in args.config:
241
+ with open(config_file, 'r', encoding='utf-8') as f:
242
+ group = os.path.basename(config_file)
243
+ group = group[:-5]
244
+ content = json.load(f)
245
+ exps = content['experiment']
246
+ for exp in exps:
247
+ main_cfg = deepcopy(content)
248
+ name = exp['name']
249
+ cmd = main_cfg['cmd']
250
+ run_args = main_cfg['args']
251
+ env = main_cfg.get('env', {})
252
+ requirements = main_cfg.get('requirements', {})
253
+ eval_requirements = main_cfg.get('eval_requirements', {})
254
+ eval_dataset = main_cfg.get('eval_dataset', {})
255
+ if 'args' in exp:
256
+ run_args.update(exp['args'])
257
+ if 'requirements' in exp:
258
+ requirements.update(exp['requirements'])
259
+ if 'env' in exp:
260
+ env.update(exp['env'])
261
+ experiments.append(
262
+ Experiment(
263
+ group=group,
264
+ name=name,
265
+ cmd=cmd,
266
+ args=run_args,
267
+ env=env,
268
+ requirements=requirements,
269
+ eval_requirements=eval_requirements,
270
+ eval_dataset=eval_dataset,
271
+ input_args=args))
272
+ return experiments
273
+
274
+ @staticmethod
275
+ def _get_metric(exp: Experiment):
276
+ if exp.do_eval:
277
+ if os.path.isfile(os.path.join('exp', f'{exp.name}.eval.log')):
278
+ with open(os.path.join('exp', f'{exp.name}.eval.log'), 'r', encoding='utf-8') as f:
279
+ for line in f.readlines():
280
+ if 'Final report:' in line:
281
+ return json.loads(line.split('Final report:')[1].replace('\'', '"'))
282
+ elif exp.cmd == 'export':
283
+ exp_args = ExportArguments(**exp.args)
284
+ if exp_args.quant_bits > 0:
285
+ if exp_args.ckpt_dir is None:
286
+ path = f'{exp_args.model_type}-{exp_args.quant_method}-int{exp_args.quant_bits}'
287
+ else:
288
+ ckpt_dir, ckpt_name = os.path.split(exp_args.ckpt_dir)
289
+ path = os.path.join(ckpt_dir, f'{ckpt_name}-{exp_args.quant_method}-int{exp_args.quant_bits}')
290
+ else:
291
+ ckpt_dir, ckpt_name = os.path.split(exp_args.ckpt_dir)
292
+ path = os.path.join(ckpt_dir, f'{ckpt_name}-merged')
293
+ if os.path.exists(path):
294
+ shutil.rmtree(exp.name, ignore_errors=True)
295
+ os.makedirs(exp.name, exist_ok=True)
296
+ shutil.move(path, os.path.join(exp.name, path))
297
+ return {
298
+ 'best_model_checkpoint': os.path.join(exp.name, path),
299
+ }
300
+ else:
301
+ logging_dir = exp.runtime.get('logging_dir')
302
+ logging_file = os.path.join(logging_dir, '..', 'logging.jsonl')
303
+ if os.path.isfile(logging_file):
304
+ with open(logging_file, 'r', encoding='utf-8') as f:
305
+ for line in f.readlines():
306
+ if 'model_info' in line:
307
+ return json.loads(line)
308
+ return None
309
+
310
+ @staticmethod
311
+ def write_record(exp: Experiment):
312
+ target_dir = exp.input_args.save_dir
313
+ file = os.path.join(target_dir, exp.name + '.json')
314
+ with open(file, 'w', encoding='utf-8') as f:
315
+ f.write(json.dumps(exp.to_dict()) + '\n')
316
+
317
+ def _poll(self):
318
+ while True:
319
+ time.sleep(5)
320
+
321
+ has_finished = False
322
+ for exp in self.exps:
323
+ rt = exp.handler.poll()
324
+ if rt is None:
325
+ continue
326
+
327
+ has_finished = True
328
+ if rt == 0:
329
+ if not exp.do_eval:
330
+ all_metric = self._get_metric(exp)
331
+ if all_metric:
332
+ exp.record.update(all_metric)
333
+ if exp.eval_dataset:
334
+ exp.do_eval = True
335
+ self.exp_queue.appendleft(exp)
336
+ self.write_record(exp)
337
+ else:
338
+ logger.error(f'Running {exp.name} task, but no result found')
339
+ else:
340
+ all_metric = self._get_metric(exp)
341
+ exp.record['eval_result'] = all_metric
342
+ if all_metric:
343
+ self.write_record(exp)
344
+ else:
345
+ logger.error(f'Running {exp.name} eval task, but no eval result found')
346
+ logger.info(f'Running {exp.name} finished with return code: {rt}')
347
+
348
+ if has_finished:
349
+ self.exps = [exp for exp in self.exps if exp.handler.poll() is None]
350
+ break
351
+
352
+ def begin(self, args: Any):
353
+ exps = self.prepare_experiments(args)
354
+ logger.info(f'all exps: {exps}')
355
+ exps.sort(key=lambda e: e.priority)
356
+ self.exp_queue = deque()
357
+ for exp in exps:
358
+ self.exp_queue.append(exp)
359
+
360
+ while len(self.exp_queue) or len(self.exps) > 0:
361
+ while len(self.exp_queue):
362
+ try:
363
+ logger.info(f'Running exp: {self.exp_queue[0].name}')
364
+ self.run(self.exp_queue[0])
365
+ except Exception as e:
366
+ if not isinstance(e, AssertionError):
367
+ logger.error(f'Adding exp {self.exp_queue[0].name} error because of:')
368
+ logger.error(e)
369
+ self.exp_queue.popleft()
370
+ else:
371
+ logger.info(f'Adding exp {self.exp_queue[0].name} error because of:', str(e))
372
+ if 'no free gpu' in str(e).lower():
373
+ break
374
+ else:
375
+ continue
376
+ else:
377
+ self.exp_queue.popleft()
378
+ self._poll()
379
+ logger.info(f'Run task finished because of exp queue: {self.exp_queue} and exps: {self.exps}')
380
+
381
+
382
+ def find_all_config(dir_or_file: str):
383
+ if os.path.isfile(dir_or_file):
384
+ return [dir_or_file]
385
+ else:
386
+ configs = []
387
+ for dirpath, dirnames, filenames in os.walk(dir_or_file):
388
+ for name in filenames:
389
+ if name.endswith('.json') and 'ipynb' not in dirpath:
390
+ configs.append(os.path.join(dirpath, name))
391
+ return configs
scripts/benchmark/generate_report.py ADDED
@@ -0,0 +1,433 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ import dataclasses
3
+ import os
4
+ from dataclasses import dataclass
5
+ from typing import Any, Dict, List
6
+
7
+ import json
8
+ import numpy as np
9
+
10
+ from swift.llm.template import split_str_parts_by
11
+
12
+
13
+ @dataclass
14
+ class ModelOutput:
15
+
16
+ group: str = None
17
+
18
+ name: str = None
19
+
20
+ cmd: str = None
21
+
22
+ requirements: Dict[str, str] = dataclasses.field(default_factory=dict)
23
+
24
+ args: Dict[str, Any] = dataclasses.field(default_factory=dict)
25
+
26
+ memory: str = None
27
+
28
+ train_time: float = None
29
+
30
+ train_samples: int = None
31
+
32
+ train_samples_per_second: float = None
33
+
34
+ last_model_checkpoint: str = None
35
+
36
+ best_model_checkpoint: str = None
37
+
38
+ best_metric: Any = None
39
+
40
+ global_step: int = None
41
+
42
+ num_total_parameters: float = None
43
+
44
+ num_trainable_parameters: float = None
45
+
46
+ num_buffers: float = None
47
+
48
+ trainable_parameters_percentage: float = None
49
+
50
+ train_dataset_info: str = None
51
+
52
+ val_dataset_info: str = None
53
+
54
+ train_create_time: float = None
55
+
56
+ eval_tokens: int = None
57
+
58
+ eval_time: float = None
59
+
60
+ reports: Dict[str, Any] = None
61
+
62
+ train_loss: float = None
63
+
64
+ @property
65
+ def tuner_hyper_params(self):
66
+ hyper_params = ''
67
+ args = self.args
68
+ if 'sft_type' not in args:
69
+ return ''
70
+ if args['sft_type'] in ('lora', 'adalora', 'longlora'):
71
+ if 'lora_rank' in args:
72
+ hyper_params += f'rank={args["lora_rank"]}/' \
73
+ f'target={args["lora_target_modules"]}/' \
74
+ f'alpha={args["lora_alpha"]}/' \
75
+ f'lr_ratio={args.get("lora_lr_ratio", None)}/' \
76
+ f'use_rslora={args.get("use_rslora", False)}/' \
77
+ f'use_dora={args.get("use_dora", False)}'
78
+ else:
79
+ hyper_params = ''
80
+ if args['sft_type'] == 'full':
81
+ if 'use_galore' in args and args['use_galore'] == 'true':
82
+ hyper_params += f'galore_rank={args["galore_rank"]}/' \
83
+ f'galore_per_parameter={args["galore_optim_per_parameter"]}/' \
84
+ f'galore_with_embedding={args["galore_with_embedding"]}/'
85
+ if args['sft_type'] == 'llamapro':
86
+ hyper_params += f'num_blocks={args["llamapro_num_new_blocks"]}/'
87
+ if 'neftune_noise_alpha' in args and args['neftune_noise_alpha']:
88
+ hyper_params += f'neftune_noise_alpha={args["neftune_noise_alpha"]}/'
89
+
90
+ if hyper_params.endswith('/'):
91
+ hyper_params = hyper_params[:-1]
92
+ return hyper_params
93
+
94
+ @property
95
+ def hyper_parameters(self):
96
+ if 'learning_rate' not in self.args:
97
+ return ''
98
+ return f'lr={self.args["learning_rate"]}/' \
99
+ f'epoch={self.args["num_train_epochs"]}'
100
+
101
+ @property
102
+ def train_speed(self):
103
+ if self.train_samples_per_second:
104
+ return f'{self.train_samples_per_second:.2f}({self.train_samples} samples/{self.train_time:.2f} seconds)'
105
+ else:
106
+ return ''
107
+
108
+ @property
109
+ def infer_speed(self):
110
+ if self.eval_tokens:
111
+ return f'{self.eval_tokens / self.eval_time:.2f}({self.eval_tokens} tokens/{self.eval_time:.2f} seconds)'
112
+ return ''
113
+
114
+
115
+ def generate_sft_report(outputs: List[ModelOutput]):
116
+ gsm8k_accs = []
117
+ arc_accs = []
118
+ ceval_accs = []
119
+ for output in outputs:
120
+ gsm8k_acc = None
121
+ arc_acc = None
122
+ ceval_acc = None
123
+ for report in (output.reports or []):
124
+ if report['name'] == 'gsm8k':
125
+ gsm8k_acc = report['score']
126
+ if report['name'] == 'arc':
127
+ arc_acc = report['score']
128
+ if report['name'] == 'ceval':
129
+ ceval_acc = report['score']
130
+ gsm8k_accs.append(gsm8k_acc)
131
+ arc_accs.append(arc_acc)
132
+ ceval_accs.append(ceval_acc)
133
+
134
+ tab = '| exp_name | model_type | dataset | ms-bench mix ratio | tuner | tuner_params | trainable params(M) | flash_attn | gradient_checkpointing | hypers | memory | train speed(samples/s) | infer speed(tokens/s) | train_loss | eval_loss | gsm8k weighted acc | arc weighted acc | ceval weighted acc |\n' \
135
+ '| -------- | ---------- | ------- | -------------------| ----- | ------------ | ------------------- | -----------| ---------------------- | ------ | ------ | ---------------------- | --------------------- | ---------- | --------- | ------------------ | ---------------- | ------------------ |\n' # noqa
136
+ min_best_metric = 999.
137
+ min_train_loss = 999.
138
+ if outputs:
139
+ min_best_metric = min([output.best_metric or 999. for output in outputs])
140
+ min_train_loss = min([output.train_loss or 999. for output in outputs])
141
+
142
+ max_gsm8k = 0.0
143
+ if gsm8k_accs:
144
+ max_gsm8k = max([gsm8k or 0. for gsm8k in gsm8k_accs])
145
+
146
+ max_arc = 0.0
147
+ if arc_accs:
148
+ max_arc = max([arc or 0. for arc in arc_accs])
149
+
150
+ max_ceval = 0.0
151
+ if ceval_accs:
152
+ max_ceval = max([ceval or 0. for ceval in ceval_accs])
153
+
154
+ for output, gsm8k_acc, arc_acc, ceval_acc in zip(outputs, gsm8k_accs, arc_accs, ceval_accs):
155
+ use_flash_attn = output.args.get('use_flash_attn', '')
156
+ use_gc = output.args.get('gradient_checkpointing', '')
157
+ memory = output.memory
158
+ train_speed = output.train_speed
159
+ infer_speed = output.infer_speed
160
+
161
+ is_best_metric = np.isclose(min_best_metric, output.best_metric or 999.0)
162
+ is_best_loss = np.isclose(min_train_loss, output.train_loss or 999.0)
163
+ is_best_gsm8k = np.isclose(max_gsm8k, gsm8k_acc or 0.0)
164
+ is_best_arc = np.isclose(max_arc, arc_acc or 0.0)
165
+ is_best_ceval = np.isclose(max_ceval, ceval_acc or 0.0)
166
+
167
+ if not is_best_metric:
168
+ best_metric = '' if not output.best_metric else f'{output.best_metric:.2f}'
169
+ else:
170
+ best_metric = '' if not output.best_metric else f'**{output.best_metric:.2f}**'
171
+
172
+ if not is_best_loss:
173
+ train_loss = '' if not output.train_loss else f'{output.train_loss:.2f}'
174
+ else:
175
+ train_loss = '' if not output.train_loss else f'**{output.train_loss:.2f}**'
176
+
177
+ if not is_best_gsm8k:
178
+ gsm8k_acc = '' if not gsm8k_acc else f'{gsm8k_acc:.3f}'
179
+ else:
180
+ gsm8k_acc = '' if not gsm8k_acc else f'**{gsm8k_acc:.3f}**'
181
+
182
+ if not is_best_arc:
183
+ arc_acc = '' if not arc_acc else f'{arc_acc:.3f}'
184
+ else:
185
+ arc_acc = '' if not arc_acc else f'**{arc_acc:.3f}**'
186
+
187
+ if not is_best_ceval:
188
+ ceval_acc = '' if not ceval_acc else f'{ceval_acc:.3f}'
189
+ else:
190
+ ceval_acc = '' if not ceval_acc else f'**{ceval_acc:.3f}**'
191
+
192
+ line = f'|{output.name}|' \
193
+ f'{output.args["model_type"]}|' \
194
+ f'{output.args.get("dataset")}|' \
195
+ f'{output.args.get("train_dataset_mix_ratio", 0.)}|' \
196
+ f'{output.args.get("sft_type")}|' \
197
+ f'{output.tuner_hyper_params}|' \
198
+ f'{output.num_trainable_parameters}({output.trainable_parameters_percentage})|' \
199
+ f'{use_flash_attn}|' \
200
+ f'{use_gc}|' \
201
+ f'{output.hyper_parameters}|' \
202
+ f'{memory}|' \
203
+ f'{train_speed}|' \
204
+ f'{infer_speed}|' \
205
+ f'{best_metric}|' \
206
+ f'{train_loss}|' \
207
+ f'{gsm8k_acc}|' \
208
+ f'{arc_acc}|' \
209
+ f'{ceval_acc}|\n'
210
+ tab += line
211
+ return tab
212
+
213
+
214
+ def generate_export_report(outputs: List[ModelOutput]):
215
+ tab = '| exp_name | model_type | calibration dataset | quantization method | quantization bits | infer speed(tokens/s) | gsm8k weighted acc | arc weighted acc | ceval weighted acc |\n' \
216
+ '| -------- | ---------- | ------------------- | ------------------- | ----------------- | --------------------- | ------------------ | ---------------- | ------------------ |\n' # noqa
217
+
218
+ gsm8k_accs = []
219
+ arc_accs = []
220
+ ceval_accs = []
221
+ for output in outputs:
222
+ gsm8k_acc = None
223
+ arc_acc = None
224
+ ceval_acc = None
225
+ for report in (output.reports or []):
226
+ if report['name'] == 'gsm8k':
227
+ gsm8k_acc = report['score']
228
+ if report['name'] == 'arc':
229
+ arc_acc = report['score']
230
+ if report['name'] == 'ceval':
231
+ ceval_acc = report['score']
232
+ gsm8k_accs.append(gsm8k_acc)
233
+ arc_accs.append(arc_acc)
234
+ ceval_accs.append(ceval_acc)
235
+
236
+ max_gsm8k = 0.0
237
+ if gsm8k_accs:
238
+ max_gsm8k = max([gsm8k or 0. for gsm8k in gsm8k_accs])
239
+
240
+ max_arc = 0.0
241
+ if arc_accs:
242
+ max_arc = max([arc or 0. for arc in arc_accs])
243
+
244
+ max_ceval = 0.0
245
+ if ceval_accs:
246
+ max_ceval = max([ceval or 0. for ceval in ceval_accs])
247
+
248
+ for output, gsm8k_acc, arc_acc, ceval_acc in zip(outputs, gsm8k_accs, arc_accs, ceval_accs):
249
+ infer_speed = output.infer_speed
250
+ is_best_gsm8k = np.isclose(max_gsm8k, gsm8k_acc or 0.0)
251
+ is_best_arc = np.isclose(max_arc, arc_acc or 0.0)
252
+ is_best_ceval = np.isclose(max_ceval, ceval_acc or 0.0)
253
+
254
+ if not is_best_gsm8k:
255
+ gsm8k_acc = '' if not gsm8k_acc else f'{gsm8k_acc:.3f}'
256
+ else:
257
+ gsm8k_acc = '' if not gsm8k_acc else f'**{gsm8k_acc:.3f}**'
258
+
259
+ if not is_best_arc:
260
+ arc_acc = '' if not arc_acc else f'{arc_acc:.3f}'
261
+ else:
262
+ arc_acc = '' if not arc_acc else f'**{arc_acc:.3f}**'
263
+
264
+ if not is_best_ceval:
265
+ ceval_acc = '' if not ceval_acc else f'{ceval_acc:.3f}'
266
+ else:
267
+ ceval_acc = '' if not ceval_acc else f'**{ceval_acc:.3f}**'
268
+
269
+ if output.train_dataset_info:
270
+ dataset_info = f'{output.args["dataset"]}/{output.train_dataset_info}'
271
+ else:
272
+ dataset_info = f'{output.args["dataset"]}'
273
+ line = f'|{output.name}|' \
274
+ f'{output.args["model_type"]}|' \
275
+ f'{dataset_info}|' \
276
+ f'{output.args["quant_method"]}|' \
277
+ f'{output.args["quant_bits"]}|' \
278
+ f'{infer_speed}|' \
279
+ f'{gsm8k_acc}|' \
280
+ f'{arc_acc}|' \
281
+ f'{ceval_acc}|\n'
282
+ tab += line
283
+ return tab
284
+
285
+
286
+ def parse_output(file):
287
+ with open(file, 'r', encoding='utf-8') as f:
288
+ content = json.load(f)
289
+
290
+ name = content['name']
291
+ group = content['group']
292
+ cmd = content['cmd']
293
+ requirements = content['requirements']
294
+ args = content['args']
295
+ create_time = float(content.get('create_time') or 0)
296
+ content = content['record']
297
+ if cmd == 'export':
298
+ best_model_checkpoint = content['best_model_checkpoint']
299
+ eval_tokens = 0
300
+ eval_time = 0.0
301
+ eval_result = None
302
+ if 'eval_result' in content:
303
+ eval_result = content['eval_result']
304
+ eval_tokens = eval_result['generation_info']['tokens']
305
+ eval_time = eval_result['generation_info']['time']
306
+ eval_result = eval_result['report']
307
+ return ModelOutput(
308
+ group=group,
309
+ name=name,
310
+ cmd=cmd,
311
+ requirements=requirements,
312
+ args=args,
313
+ best_model_checkpoint=best_model_checkpoint,
314
+ eval_time=eval_time,
315
+ eval_tokens=eval_tokens,
316
+ reports=eval_result,
317
+ )
318
+ else:
319
+ memory = None
320
+ train_time = None
321
+ train_samples = None
322
+ train_samples_per_second = None
323
+ last_model_checkpoint = None
324
+ best_model_checkpoint = None
325
+ best_metric = None
326
+ global_step = None
327
+ train_dataset_info = None
328
+ val_dataset_info = None
329
+ num_trainable_parameters = None
330
+ num_buffers = None
331
+ trainable_parameters_percentage = None
332
+ num_total_parameters = None
333
+ train_loss = None
334
+ if 'memory' in content:
335
+ memory = content['memory']
336
+ memory = '/'.join(memory.values())
337
+ if 'train_time' in content:
338
+ train_time = content['train_time']['train_runtime']
339
+ train_samples = content['train_time']['n_train_samples']
340
+ train_samples_per_second = content['train_time']['train_samples_per_second']
341
+ if 'last_model_checkpoint' in content:
342
+ last_model_checkpoint = content['last_model_checkpoint']
343
+ if 'best_model_checkpoint' in content:
344
+ best_model_checkpoint = content['best_model_checkpoint']
345
+ if 'best_metric' in content:
346
+ best_metric = content['best_metric']
347
+ if 'log_history' in content:
348
+ train_loss = content['log_history'][-1]['train_loss']
349
+ if 'global_step' in content:
350
+ global_step = content['global_step']
351
+ if 'dataset_info' in content:
352
+ train_dataset_info = content['dataset_info'].get('train_dataset')
353
+ val_dataset_info = content['dataset_info'].get('val_dataset')
354
+ if 'model_info' in content:
355
+ # model_info like: SwiftModel: 6758.4041M Params (19.9885M Trainable [0.2958%]), 16.7793M Buffers.
356
+ str_dict = split_str_parts_by(content['model_info'], [
357
+ 'SwiftModel:', 'CausalLM:', 'Seq2SeqLM:', 'LMHeadModel:', 'M Params (', 'M Trainable [', ']), ',
358
+ 'M Buffers.'
359
+ ])
360
+ str_dict = {c['key']: c['content'] for c in str_dict}
361
+ if 'SwiftModel:' in str_dict:
362
+ num_total_parameters = float(str_dict['SwiftModel:'])
363
+ elif 'CausalLM:' in str_dict:
364
+ num_total_parameters = float(str_dict['CausalLM:'])
365
+ elif 'Seq2SeqLM:' in str_dict:
366
+ num_total_parameters = float(str_dict['Seq2SeqLM:'])
367
+ elif 'LMHeadModel:' in str_dict:
368
+ num_total_parameters = float(str_dict['LMHeadModel:'])
369
+ num_trainable_parameters = float(str_dict['M Params ('])
370
+ num_buffers = float(str_dict[']), '])
371
+ trainable_parameters_percentage = str_dict['M Trainable [']
372
+
373
+ eval_tokens = 0
374
+ eval_time = 0.0
375
+ eval_result = None
376
+ if 'eval_result' in content:
377
+ eval_result = content['eval_result']
378
+ eval_tokens = eval_result['generation_info']['tokens']
379
+ eval_time = eval_result['generation_info']['time']
380
+ eval_result = eval_result['report']
381
+
382
+ return ModelOutput(
383
+ group=group,
384
+ name=name,
385
+ cmd=cmd,
386
+ requirements=requirements,
387
+ args=args,
388
+ memory=memory,
389
+ train_time=train_time,
390
+ train_samples=train_samples,
391
+ train_samples_per_second=train_samples_per_second,
392
+ last_model_checkpoint=last_model_checkpoint,
393
+ best_model_checkpoint=best_model_checkpoint,
394
+ best_metric=best_metric,
395
+ global_step=global_step,
396
+ train_dataset_info=train_dataset_info,
397
+ val_dataset_info=val_dataset_info,
398
+ train_create_time=create_time,
399
+ num_total_parameters=num_total_parameters,
400
+ num_trainable_parameters=num_trainable_parameters,
401
+ num_buffers=num_buffers,
402
+ trainable_parameters_percentage=trainable_parameters_percentage,
403
+ eval_time=eval_time,
404
+ eval_tokens=eval_tokens,
405
+ reports=eval_result,
406
+ train_loss=train_loss,
407
+ )
408
+
409
+
410
+ def generate_reports():
411
+ outputs = []
412
+ for dirs, _, files in os.walk('./experiment'):
413
+ for file in files:
414
+ abs_file = os.path.join(dirs, file)
415
+ if not abs_file.endswith('.json') or 'ipynb' in abs_file:
416
+ continue
417
+
418
+ outputs.append(parse_output(abs_file))
419
+
420
+ all_groups = set([output.group for output in outputs])
421
+ for group in all_groups:
422
+ group_outputs = [output for output in outputs if output.group == group]
423
+ print(f'=================Printing the sft cmd result of exp {group}==================\n\n')
424
+ print(generate_sft_report([output for output in group_outputs if output.cmd in ('sft', 'eval')]))
425
+ # print(f'=================Printing the dpo result of exp {group}==================')
426
+ # print(generate_dpo_report([output for output in outputs if output.cmd == 'dpo']))
427
+ print(f'=================Printing the export cmd result of exp {group}==================\n\n')
428
+ print(generate_export_report([output for output in group_outputs if output.cmd == 'export']))
429
+ print('=================Printing done==================\n\n')
430
+
431
+
432
+ if __name__ == '__main__':
433
+ generate_reports()
scripts/utils/plot_loss.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ from swift.utils import plot_images
4
+
5
+ ckpt_dir = 'output/xxx/vx-xxx'
6
+ if __name__ == '__main__':
7
+ images_dir = os.path.join(ckpt_dir, 'images')
8
+ tb_dir = os.path.join(ckpt_dir, 'runs')
9
+ plot_images(images_dir, tb_dir, ['train/loss'], 0.9)
scripts/utils/run_dataset_info.py ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import re
3
+
4
+ import numpy as np
5
+
6
+ from swift.llm import DATASET_MAPPING, EncodePreprocessor, get_model_tokenizer, get_template, load_dataset
7
+ from swift.utils import stat_array
8
+
9
+ os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'
10
+
11
+
12
+ def get_cache_mapping(fpath):
13
+ with open(fpath, 'r', encoding='utf-8') as f:
14
+ text = f.read()
15
+ idx = text.find('| Dataset ID |')
16
+ text = text[idx:]
17
+ text_list = text.split('\n')[2:]
18
+ cache_mapping = {} # dataset_id -> (dataset_size, stat)
19
+ for text in text_list:
20
+ if not text:
21
+ continue
22
+ items = text.split('|')
23
+ key = items[1] if items[1] != '-' else items[6]
24
+ key = re.search(r'\[(.+?)\]', key).group(1)
25
+ stat = items[3:5]
26
+ if stat[0] == '-':
27
+ stat = ('huge dataset', '-')
28
+ cache_mapping[key] = stat
29
+ return cache_mapping
30
+
31
+
32
+ def get_dataset_id(key):
33
+ for dataset_id in key:
34
+ if dataset_id is not None:
35
+ break
36
+ return dataset_id
37
+
38
+
39
+ def run_dataset(key, template, cache_mapping):
40
+ ms_id, hf_id, _ = key
41
+ dataset_meta = DATASET_MAPPING[key]
42
+ tags = ', '.join(tag for tag in dataset_meta.tags) or '-'
43
+ dataset_id = ms_id or hf_id
44
+ use_hf = ms_id is None
45
+ if ms_id is not None:
46
+ ms_id = f'[{ms_id}](https://modelscope.cn/datasets/{ms_id})'
47
+ else:
48
+ ms_id = '-'
49
+ if hf_id is not None:
50
+ hf_id = f'[{hf_id}](https://huggingface.co/datasets/{hf_id})'
51
+ else:
52
+ hf_id = '-'
53
+ subsets = '<br>'.join(subset.name for subset in dataset_meta.subsets)
54
+
55
+ if dataset_meta.huge_dataset:
56
+ dataset_size = 'huge dataset'
57
+ stat_str = '-'
58
+ elif dataset_id in cache_mapping:
59
+ dataset_size, stat_str = cache_mapping[dataset_id]
60
+ else:
61
+ num_proc = 4
62
+ dataset, _ = load_dataset(f'{dataset_id}:all', strict=False, num_proc=num_proc, use_hf=use_hf)
63
+ dataset_size = len(dataset)
64
+ random_state = np.random.RandomState(42)
65
+ idx_list = random_state.choice(dataset_size, size=min(dataset_size, 100000), replace=False)
66
+ encoded_dataset = EncodePreprocessor(template)(dataset.select(idx_list), num_proc=num_proc)
67
+
68
+ input_ids = encoded_dataset['input_ids']
69
+ token_len = [len(tokens) for tokens in input_ids]
70
+ stat = stat_array(token_len)[0]
71
+ stat_str = f"{stat['mean']:.1f}±{stat['std']:.1f}, min={stat['min']}, max={stat['max']}"
72
+
73
+ return f'|{ms_id}|{subsets}|{dataset_size}|{stat_str}|{tags}|{hf_id}|'
74
+
75
+
76
+ def write_dataset_info() -> None:
77
+ fpaths = ['docs/source/Instruction/支持的模型和数据集.md', 'docs/source_en/Instruction/Supported-models-and-datasets.md']
78
+ cache_mapping = get_cache_mapping(fpaths[0])
79
+ res_text_list = []
80
+ res_text_list.append('| Dataset ID | Subset Name | Dataset Size | Statistic (token) | Tags | HF Dataset ID |')
81
+ res_text_list.append('| ---------- | ----------- | -------------| ------------------| ---- | ------------- |')
82
+
83
+ all_keys = list(DATASET_MAPPING.keys())
84
+ all_keys = sorted(all_keys, key=lambda x: get_dataset_id(x))
85
+ _, tokenizer = get_model_tokenizer('Qwen/Qwen2.5-7B-Instruct', load_model=False)
86
+ template = get_template(tokenizer.model_meta.template, tokenizer)
87
+ try:
88
+ for i, key in enumerate(all_keys):
89
+ res = run_dataset(key, template, cache_mapping)
90
+ res_text_list.append(res)
91
+ print(res)
92
+ finally:
93
+ for fpath in fpaths:
94
+ with open(fpath, 'r', encoding='utf-8') as f:
95
+ text = f.read()
96
+ idx = text.find('| Dataset ID |')
97
+
98
+ new_text = '\n'.join(res_text_list)
99
+ text = text[:idx] + new_text + '\n'
100
+ with open(fpath, 'w', encoding='utf-8') as f:
101
+ f.write(text)
102
+ print(f'数据集总数: {len(all_keys)}')
103
+
104
+
105
+ if __name__ == '__main__':
106
+ write_dataset_info()
scripts/utils/run_model_info.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any, List
2
+
3
+ from swift.llm import MODEL_MAPPING, TEMPLATE_MAPPING, ModelType, TemplateType
4
+ from swift.utils import is_megatron_available
5
+
6
+
7
+ def get_url_suffix(model_id):
8
+ if ':' in model_id:
9
+ return model_id.split(':')[0]
10
+ return model_id
11
+
12
+
13
+ def get_cache_mapping(fpath):
14
+ with open(fpath, 'r', encoding='utf-8') as f:
15
+ text = f.read()
16
+ idx = text.find('| Model ID |')
17
+ text = text[idx:]
18
+ text_list = text.split('\n')[2:]
19
+ cache_mapping = {}
20
+ for text in text_list:
21
+ if not text:
22
+ continue
23
+ items = text.split('|')
24
+ if len(items) < 6:
25
+ break
26
+ cache_mapping[items[1]] = items[5]
27
+ return cache_mapping
28
+
29
+
30
+ def get_model_info_table():
31
+ fpaths = ['docs/source/Instruction/支持的模型和数据集.md', 'docs/source_en/Instruction/Supported-models-and-datasets.md']
32
+ cache_mapping = get_cache_mapping(fpaths[0])
33
+ end_words = [['### 多模态大模型', '## 数据集'], ['### Multimodal large models', '## Datasets']]
34
+ result = [
35
+ '| Model ID | Model Type | Default Template | '
36
+ 'Requires | Support Megatron | Tags | HF Model ID |\n'
37
+ '| -------- | -----------| ---------------- | '
38
+ '-------- | ---------------- | ---- | ----------- |\n'
39
+ ] * 2
40
+ res_llm: List[Any] = []
41
+ res_mllm: List[Any] = []
42
+ mg_count = 0
43
+ for template in TemplateType.get_template_name_list():
44
+ assert template in TEMPLATE_MAPPING
45
+
46
+ for model_type in ModelType.get_model_name_list():
47
+ model_meta = MODEL_MAPPING[model_type]
48
+ template = model_meta.template
49
+ for group in model_meta.model_groups:
50
+ for model in group.models:
51
+ ms_model_id = model.ms_model_id
52
+ hf_model_id = model.hf_model_id
53
+ if ms_model_id:
54
+ ms_model_id = f'[{ms_model_id}](https://modelscope.cn/models/{get_url_suffix(ms_model_id)})'
55
+ else:
56
+ ms_model_id = '-'
57
+ if hf_model_id:
58
+ hf_model_id = f'[{hf_model_id}](https://huggingface.co/{get_url_suffix(hf_model_id)})'
59
+ else:
60
+ hf_model_id = '-'
61
+ tags = ', '.join(group.tags or model_meta.tags) or '-'
62
+ requires = ', '.join(group.requires or model_meta.requires) or '-'
63
+ if is_megatron_available():
64
+ from swift.megatron import model
65
+ support_megatron = getattr(model_meta, 'support_megatron', False)
66
+ for word in ['gptq', 'awq', 'bnb', 'aqlm', 'int', 'nf4', 'fp8']:
67
+ if word in ms_model_id.lower():
68
+ support_megatron = False
69
+ break
70
+ support_megatron = '&#x2714;' if support_megatron else '&#x2718;'
71
+ else:
72
+ support_megatron = cache_mapping.get(ms_model_id, '&#x2718;')
73
+ if support_megatron == '&#x2714;':
74
+ mg_count += 1
75
+ r = f'|{ms_model_id}|{model_type}|{template}|{requires}|{support_megatron}|{tags}|{hf_model_id}|\n'
76
+ if model_meta.is_multimodal:
77
+ res_mllm.append(r)
78
+ else:
79
+ res_llm.append(r)
80
+ print(f'LLM总数: {len(res_llm)}, MLLM总数: {len(res_mllm)}, Megatron支持模型: {mg_count}')
81
+ text = ['', ''] # llm, mllm
82
+ for i, res in enumerate([res_llm, res_mllm]):
83
+ for r in res:
84
+ text[i] += r
85
+ result[i] += text[i]
86
+
87
+ for i, fpath in enumerate(fpaths):
88
+ with open(fpath, 'r', encoding='utf-8') as f:
89
+ text = f.read()
90
+ llm_start_idx = text.find('| Model ID |')
91
+ mllm_start_idx = text[llm_start_idx + 1:].find('| Model ID |') + llm_start_idx + 1
92
+ llm_end_idx = text.find(end_words[i][0])
93
+ mllm_end_idx = text.find(end_words[i][1])
94
+ output = text[:llm_start_idx] + result[0] + '\n\n' + text[llm_end_idx:mllm_start_idx] + result[
95
+ 1] + '\n\n' + text[mllm_end_idx:]
96
+ with open(fpath, 'w', encoding='utf-8') as f:
97
+ f.write(output)
98
+
99
+
100
+ if __name__ == '__main__':
101
+ get_model_info_table()
scripts/utils/run_template.py ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ from swift.llm import TemplateType
2
+
3
+ if __name__ == '__main__':
4
+ template_name_list = TemplateType.get_template_name_list()
5
+ tn_gen = ', '.join([tn for tn in template_name_list if 'generation' in tn])
6
+ tn_chat = ', '.join([tn for tn in template_name_list if 'generation' not in tn])
7
+ print(f'Text Generation: {tn_gen}')
8
+ print(f'Chat: {tn_chat}')
swift/__init__.py ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ from typing import TYPE_CHECKING
3
+
4
+ from .utils.import_utils import _LazyModule
5
+
6
+ if TYPE_CHECKING:
7
+ from .version import __version__, __release_datetime__
8
+ from .tuners import (Adapter, AdapterConfig, AdapterModule, SwiftModel, LoRA, LoRAConfig, SWIFT_MAPPING,
9
+ AdaLoraConfig, LoftQConfig, LoHaConfig, LoKrConfig, LoraConfig, OFTConfig, PeftConfig,
10
+ PeftModel, PeftModelForCausalLM, ResTuningConfig, SideConfig, PeftModelForSeq2SeqLM,
11
+ PeftModelForSequenceClassification, PeftModelForTokenClassification, PrefixTuningConfig,
12
+ PromptEncoderConfig, PromptLearningConfig, PromptTuningConfig, get_peft_config, get_peft_model,
13
+ get_peft_model_state_dict, Prompt, PromptConfig, PromptModule, SwiftConfig, SwiftOutput, Swift,
14
+ SwiftTuners, LongLoRAConfig, LongLoRA, LongLoRAModelType, SCETuning, SCETuningConfig)
15
+ from .trainers import (EvaluationStrategy, FSDPOption, HPSearchBackend, HubStrategy, IntervalStrategy,
16
+ SchedulerType, ShardedDDPOption, TrainingArguments, Seq2SeqTrainingArguments, Trainer,
17
+ Seq2SeqTrainer)
18
+ from .utils import get_logger
19
+ else:
20
+ _import_structure = {
21
+ 'version': ['__release_datetime__', '__version__'],
22
+ 'tuners': [
23
+ 'Adapter', 'AdapterConfig', 'AdapterModule', 'SwiftModel', 'LoRA', 'LoRAConfig', 'SWIFT_MAPPING',
24
+ 'LoraConfig', 'AdaLoraConfig', 'LoftQConfig', 'LoHaConfig', 'LoKrConfig', 'OFTConfig', 'PeftConfig',
25
+ 'ResTuningConfig', 'SideConfig', 'PeftModel', 'PeftModelForCausalLM', 'PeftModelForSeq2SeqLM',
26
+ 'PeftModelForSequenceClassification', 'PeftModelForTokenClassification', 'PrefixTuningConfig',
27
+ 'PromptEncoderConfig', 'PromptLearningConfig', 'PromptTuningConfig', 'get_peft_config', 'get_peft_model',
28
+ 'get_peft_model_state_dict', 'Prompt', 'PromptConfig', 'PromptModule', 'SwiftConfig', 'SwiftOutput',
29
+ 'Swift', 'SwiftTuners', 'LongLoRAConfig', 'LongLoRA', 'LongLoRAModelType', 'SCETuning', 'SCETuningConfig'
30
+ ],
31
+ 'trainers': [
32
+ 'EvaluationStrategy',
33
+ 'FSDPOption',
34
+ 'HPSearchBackend',
35
+ 'HubStrategy',
36
+ 'IntervalStrategy',
37
+ 'SchedulerType',
38
+ 'ShardedDDPOption',
39
+ 'TrainingArguments',
40
+ 'Seq2SeqTrainingArguments',
41
+ 'Trainer',
42
+ 'Seq2SeqTrainer',
43
+ ],
44
+ 'utils': ['get_logger']
45
+ }
46
+
47
+ import sys
48
+
49
+ sys.modules[__name__] = _LazyModule(
50
+ __name__,
51
+ globals()['__file__'],
52
+ _import_structure,
53
+ module_spec=__spec__,
54
+ extra_objects={},
55
+ )
swift/cli/__init__.py ADDED
File without changes
swift/cli/_megatron/__init__.py ADDED
File without changes
swift/cli/_megatron/main.py ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ from typing import Dict
3
+
4
+ from swift.utils import get_logger
5
+ from ..main import cli_main as swift_cli_main
6
+
7
+ logger = get_logger()
8
+
9
+ ROUTE_MAPPING: Dict[str, str] = {
10
+ 'sft': 'swift.cli._megatron.sft',
11
+ 'pt': 'swift.cli._megatron.pt',
12
+ }
13
+
14
+
15
+ def cli_main():
16
+ return swift_cli_main(ROUTE_MAPPING)
17
+
18
+
19
+ if __name__ == '__main__':
20
+ cli_main()
swift/cli/_megatron/pt.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ from swift.megatron import megatron_pt_main
2
+
3
+ if __name__ == '__main__':
4
+ megatron_pt_main()
swift/cli/_megatron/sft.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ from swift.megatron import megatron_sft_main
2
+
3
+ if __name__ == '__main__':
4
+ megatron_sft_main()
swift/cli/app.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ from swift.llm import app_main
2
+
3
+ if __name__ == '__main__':
4
+ app_main()
swift/cli/deploy.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ from swift.llm import deploy_main
3
+
4
+ if __name__ == '__main__':
5
+ deploy_main()