feat(dsv3): add dsv3 fast pretrain into paddleformers

paddleformers/examples/deepseek_v3/config/config.json

@@ -0,0 +1,76 @@
+{
+  "architectures": [
+    "DeepseekV3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_deepseek.DeepseekV3Config",
+    "AutoModel": "modeling_deepseek.DeepseekV3Model",
+    "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM"
+  },
+  "aux_loss_alpha": 0.001,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "ep_size": 1,
+  "first_k_dense_replace": 3,
+  "hidden_act": "silu",
+  "hidden_size": 7168,
+  "initializer_range": 0.02,
+  "intermediate_size": 18432,
+  "kv_lora_rank": 512,
+  "max_position_embeddings": 163840,
+  "model_type": "deepseek_v3",
+  "moe_intermediate_size": 2048,
+  "moe_layer_freq": 1,
+  "n_group": 8,
+  "n_routed_experts": 8,
+  "n_shared_experts": 1,
+  "norm_topk_prob": true,
+  "num_attention_heads": 128,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 15,
+  "num_key_value_heads": 128,
+  "num_nextn_predict_layers": 1,
+  "pretraining_tp": 1,
+  "q_lora_rank": 1536,
+  "qk_nope_head_dim": 128,
+  "qk_rope_head_dim": 64,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "beta_fast": 32,
+    "beta_slow": 1,
+    "factor": 40,
+    "mscale": 1.0,
+    "mscale_all_dim": 1.0,
+    "original_max_position_embeddings": 4096,
+    "type": "yarn"
+  },
+  "rope_theta": 10000,
+  "routed_scaling_factor": 2.5,
+  "scoring_func": "sigmoid",
+  "seq_aux": true,
+  "tie_word_embeddings": false,
+  "topk_group": 4,
+  "topk_method": "noaux_tc",
+  "dtype": "bfloat16",
+  "transformers_version": "4.33.1",
+  "use_cache": true,
+  "v_head_dim": 128,
+  "vocab_size": 129280,
+  "using_flex_token": true,
+  "using_fake_gate": true,
+  "use_fused_rms_norm": true,
+  "fuse_attention_ffn": true,
+  "use_fused_rope": true,
+  "token_drop_steps": 0,
+  "recompute_fwd_gate_up": true,
+  "adaptive_remained_O1_recompute_ratio": 0.3,
+  "using_post_norm_recompute": true,
+  "is_split_group_gemm": false,
+  "use_dualpipev": true,
+  "send_mtp_embed": true,
+  "offline_quant_expert_weight": false,
+  "clear_origin_weight_when_offline_quant": false
+}
+

paddleformers/examples/deepseek_v3/config/pretrain_argument.json

@@ -0,0 +1,53 @@
+{
+    "model_name_or_path": "./config/",
+    "tokenizer_name_or_path": "deepseek-ai/DeepSeek-V3",
+    "input_dir": "./data",
+    "output_dir": "./checkpoints/pretrain_ckpts",
+    "per_device_train_batch_size": 1,
+    "gradient_accumulation_steps": 24,
+    "per_device_eval_batch_size": 1,
+    "tensor_parallel_degree": 1,
+    "pipeline_parallel_degree": 4,
+    "pipeline_parallel_config": "use_dualpipev",
+    "sharding_parallel_degree": 2,
+    "sharding_parallel_config": "split_param",
+    "sharding_comm_buffer_size_MB": 2048,
+    "expert_parallel_degree": 2,
+    "sharding": "stage1",
+    "virtual_pp_degree": 1,
+    "sequence_parallel": 0,
+    "use_flash_attention": true,
+    "max_seq_length": 4097,
+    "learning_rate": 3e-05,
+    "min_learning_rate": 3e-06,
+    "warmup_steps": 30,
+    "logging_steps": 1,
+    "max_steps": 200,
+    "save_steps": 5000,
+    "eval_steps": 1000,
+    "weight_decay": 0.01,
+    "bf16": true,
+    "fp16_opt_level": "O2",
+    "warmup_ratio": 0.01,
+    "max_grad_norm": 1.0,
+    "amp_master_grad": 1,
+    "dataloader_num_workers": 8,
+    "continue_training": 0,
+    "do_train": true,
+    "do_eval": true,
+    "do_predict": false,
+    "disable_tqdm": true,
+    "recompute": false,
+    "distributed_dataloader": 1,
+    "unified_checkpoint": true,
+    "save_total_limit": 2,
+    "skip_profile_timer": false,
+    "use_fused_rms_norm": true,
+    "fuse_attention_ffn": true,
+    "use_fused_rope": true,
+    "save_sharded_model": false,
+    "load_sharded_model": false,
+    "use_expert_parallel": true,
+    "unified_checkpoint_config": "skip_save_model_weight",
+    "offload_optim": true
+  }

paddleformers/examples/deepseek_v3/run.sh

@@ -0,0 +1,21 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# llama 模型数据下载
+# mkdir -p data
+# wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
+# wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
+
+rm -rf output
+nohup sh script/train_gpu.sh config/pretrain_argument.json > run.log 2>&1 &