base-100k / 04.pretrain.sh

final pretrained checkpoint at step 100000

3b70275 verified 3 months ago

1.92 kB

#!/usr/bin/env bash
set -e

# --- initialize conda ---
source /root/miniconda3/etc/profile.d/conda.sh

# --- activate env ---
conda activate bpe_v2

# select GPU
work_dir=/root/NaN/dna-tokenizer
data_dir=${work_dir}/dna-tokenizer-data
cache_dir=${data_dir}/cache

# runnum=len2_2048
runnum=base_5120

mkdir -p ${work_dir}/pretrain/models/model_${runnum}
output_dir=${work_dir}/pretrain/models/model_${runnum}

# export CUDA_VISIBLE_DEVICES=0
# export WORLD_SIZE=1
# export RANK=0
# export LOCAL_RANK=0
# export MASTER_ADDR=127.0.0.1
# export MASTER_PORT=29500

cp 04.pretrain.sh ${output_dir}
torchrun --nproc_per_node=8  run_mlm.py \
					--output_dir ${output_dir} \
					--model_type bert \
					--tokenizer_name ${work_dir}/baseline_bpe/vocab_5120/5120_tokenizer.json \
					--config_name ${work_dir}/BPE_merge/config_5120.json \
					--project_name token_eval_base_5120 \
					--do_train True \
					--model_max_length 512 \
					--max_seq_length 512 \
					--line_by_line True \
					--pad_to_max_length True \
					--train_file ${data_dir}/baseline_bpe_5120_allchr_all_tokenized_train_chrOnly.tsv \
					--validation_file ${data_dir}/baseline_bpe_5120_allchr_all_tokenized_val_chrOnly.tsv \
					--cache_dir ${cache_dir} \
					--use_fast_tokenizer True \
					--do_eval True \
					--gradient_accumulation_steps 1 \
					--per_device_train_batch_size 96 \
					--per_device_eval_batch_size 96 \
					--save_steps 1000 \
					--save_total_limit 10 \
					--max_steps 100000 \
					--logging_steps 1000 \
					--learning_rate 4e-5 \
					--adam_epsilon 1e-6 \
					--weight_decay 0.01 \
					--adam_beta1 0.9 \
					--adam_beta2 0.98 \
					--mlm_probability 0.15 \
					--warmup_steps 10000 \
					--seed 42 \
					--preprocessing_num_workers 8 \
					--overwrite_output_dir True
                    # --resume_from_checkpoint /root/NaN/dna-tokenizer/pretrain/models/model_base_2048/checkpoint-37000

	#!/usr/bin/env bash
	set -e

	# --- initialize conda ---
	source /root/miniconda3/etc/profile.d/conda.sh

	# --- activate env ---
	conda activate bpe_v2

	# select GPU
	work_dir=/root/NaN/dna-tokenizer
	data_dir=${work_dir}/dna-tokenizer-data
	cache_dir=${data_dir}/cache

	# runnum=len2_2048
	runnum=base_5120

	mkdir -p ${work_dir}/pretrain/models/model_${runnum}
	output_dir=${work_dir}/pretrain/models/model_${runnum}

	# export CUDA_VISIBLE_DEVICES=0
	# export WORLD_SIZE=1
	# export RANK=0
	# export LOCAL_RANK=0
	# export MASTER_ADDR=127.0.0.1
	# export MASTER_PORT=29500

	cp 04.pretrain.sh ${output_dir}
	torchrun --nproc_per_node=8 run_mlm.py \
	--output_dir ${output_dir} \
	--model_type bert \
	--tokenizer_name ${work_dir}/baseline_bpe/vocab_5120/5120_tokenizer.json \
	--config_name ${work_dir}/BPE_merge/config_5120.json \
	--project_name token_eval_base_5120 \
	--do_train True \
	--model_max_length 512 \
	--max_seq_length 512 \
	--line_by_line True \
	--pad_to_max_length True \
	--train_file ${data_dir}/baseline_bpe_5120_allchr_all_tokenized_train_chrOnly.tsv \
	--validation_file ${data_dir}/baseline_bpe_5120_allchr_all_tokenized_val_chrOnly.tsv \
	--cache_dir ${cache_dir} \
	--use_fast_tokenizer True \
	--do_eval True \
	--gradient_accumulation_steps 1 \
	--per_device_train_batch_size 96 \
	--per_device_eval_batch_size 96 \
	--save_steps 1000 \
	--save_total_limit 10 \
	--max_steps 100000 \
	--logging_steps 1000 \
	--learning_rate 4e-5 \
	--adam_epsilon 1e-6 \
	--weight_decay 0.01 \
	--adam_beta1 0.9 \
	--adam_beta2 0.98 \
	--mlm_probability 0.15 \
	--warmup_steps 10000 \
	--seed 42 \
	--preprocessing_num_workers 8 \
	--overwrite_output_dir True
	# --resume_from_checkpoint /root/NaN/dna-tokenizer/pretrain/models/model_base_2048/checkpoint-37000