!921 sync commit to master

Merge pull request !921 from Lin/update-master

Author: Lin

README.md

@@ -90,10 +90,17 @@ processor:
 # ==== dataset config ====
 train_dataset: &train_dataset
   data_loader:
-    type: MindDataset
-    dataset_dir: ""
+    type: ADGenDataLoader
+    dataset_dir: "/path/to/AdvertiseGen"
     shuffle: True
+    phase: "train"
+  tokenizer:
+    type: ChatGLMTokenizer
+    vocab_file: "/path/to/ice_text.model"
   input_columns: ["input_ids", "label", "position_ids", "attention_mask"]
+  max_source_length: 64
+  max_target_length: 64
+  ignore_pad_token_for_loss: True
   num_parallel_workers: 8
   python_multiprocessing: False
   drop_remainder: True
@@ -104,14 +111,21 @@ train_dataset: &train_dataset
   seed: 0
 
 train_dataset_task:
-  type: CausalLanguageModelDataset
+  type: KeyWordGenDataset
   dataset_config: *train_dataset
 
 eval_dataset: &eval_dataset
   data_loader:
-    type: MindDataset
-    dataset_dir: ""
-    shuffle: True
+    type: ADGenDataLoader
+    dataset_dir: "/path/to/AdvertiseGen"
+    shuffle: False
+    phase: "eval"
+  tokenizer:
+    type: ChatGLMTokenizer
+    vocab_file: "/path/to/ice_text.model"
+  max_source_length: 256
+  max_target_length: 256
+  ignore_pad_token_for_loss: True
   input_columns: ["input_ids", "label"]
   num_parallel_workers: 8
   python_multiprocessing: False
@@ -123,7 +137,7 @@ eval_dataset: &eval_dataset
   seed: 0
 
 eval_dataset_task:
-  type: CausalLanguageModelDataset
+  type: KeyWordGenDataset
   dataset_config: *eval_dataset
 
 # ==== runner config ====

configs/glm/run_glm_6b_finetune.yaml

@@ -53,7 +53,7 @@ model:
     max_decode_length: 2048  # The maximum length of the generated words.
     is_enhanced_encoder: True
     is_npu_acceleration: True   # sample in npu to acceleration
-    checkpoint_name_or_path: "glm_6b"
+    checkpoint_name_or_path: "glm_6b_chat"
     top_k: 1
     top_p: 1    # top_p not support for npu_acceleration yet 
     repetition_penalty: 1
@@ -86,10 +86,17 @@ processor:
 # ==== dataset config ====
 train_dataset: &train_dataset
   data_loader:
-    type: MindDataset
-    dataset_dir: ""
+    type: ADGenDataLoader
+    dataset_dir: "/path/to/AdvertiseGen"
     shuffle: True
+    phase: "train"
+  tokenizer:
+    type: ChatGLMTokenizer
+    vocab_file: "/path/to/ice_text.model"
   input_columns: ["input_ids", "label", "position_ids", "attention_mask"]
+  max_source_length: 64
+  max_target_length: 64
+  ignore_pad_token_for_loss: True
   num_parallel_workers: 8
   python_multiprocessing: False
   drop_remainder: True
@@ -100,14 +107,21 @@ train_dataset: &train_dataset
   seed: 0
 
 train_dataset_task:
-  type: CausalLanguageModelDataset
+  type: KeyWordGenDataset
   dataset_config: *train_dataset
 
 eval_dataset: &eval_dataset
   data_loader:
-    type: MindDataset
-    dataset_dir: ""
-    shuffle: True
+    type: ADGenDataLoader
+    dataset_dir: "/path/to/AdvertiseGen"
+    shuffle: False
+    phase: "eval"
+  tokenizer:
+    type: ChatGLMTokenizer
+    vocab_file: "/path/to/ice_text.model"
+  max_source_length: 256
+  max_target_length: 256
+  ignore_pad_token_for_loss: True
   input_columns: ["input_ids", "label"]
   num_parallel_workers: 8
   python_multiprocessing: False
@@ -119,7 +133,7 @@ eval_dataset: &eval_dataset
   seed: 0
 
 eval_dataset_task:
-  type: CausalLanguageModelDataset
+  type: KeyWordGenDataset
   dataset_config: *eval_dataset
 
 # ==== runner config ====

configs/glm/run_glm_6b_infer.yaml

@@ -52,7 +52,7 @@ model:
     max_decode_length: 2048  # The maximum length of the generated words.
     is_enhanced_encoder: True
     is_npu_acceleration: False
-    checkpoint_name_or_path: "glm_6b"
+    checkpoint_name_or_path: "glm_6b_lora"
     top_k: 1
     top_p: 1
     repetition_penalty: 1
@@ -96,10 +96,17 @@ processor:
 # ==== dataset config ====
 train_dataset: &train_dataset
   data_loader:
-    type: MindDataset
-    dataset_dir: ""
+    type: ADGenDataLoader
+    dataset_dir: "/path/to/AdvertiseGen"
     shuffle: True
+    phase: "train"
+  tokenizer:
+    type: ChatGLMTokenizer
+    vocab_file: "/path/to/ice_text.model"
   input_columns: ["input_ids", "label", "position_ids", "attention_mask"]
+  max_source_length: 64
+  max_target_length: 64
+  ignore_pad_token_for_loss: True
   num_parallel_workers: 8
   python_multiprocessing: False
   drop_remainder: True
@@ -110,14 +117,21 @@ train_dataset: &train_dataset
   seed: 0
 
 train_dataset_task:
-  type: CausalLanguageModelDataset
+  type: KeyWordGenDataset
   dataset_config: *train_dataset
 
 eval_dataset: &eval_dataset
   data_loader:
-    type: MindDataset
-    dataset_dir: ""
-    shuffle: True
+    type: ADGenDataLoader
+    dataset_dir: "/path/to/AdvertiseGen"
+    shuffle: False
+    phase: "eval"
+  tokenizer:
+    type: ChatGLMTokenizer
+    vocab_file: "/path/to/ice_text.model"
+  max_source_length: 256
+  max_target_length: 256
+  ignore_pad_token_for_loss: True
   input_columns: ["input_ids", "label"]
   num_parallel_workers: 8
   python_multiprocessing: False
@@ -129,7 +143,7 @@ eval_dataset: &eval_dataset
   seed: 0
 
 eval_dataset_task:
-  type: CausalLanguageModelDataset
+  type: KeyWordGenDataset
   dataset_config: *eval_dataset
 
 # ==== runner config ====
@@ -169,7 +183,7 @@ lr_scale: False
 # parallel config
 use_parallel: False
 parallel:
-  parallel_mode: 1 # 0-dataset, 1-semi, 2-auto, 3-hybrid
+  parallel_mode: 0 # 0-dataset, 1-semi, 2-auto, 3-hybrid
   gradients_mean: False
   loss_repeated_mean: True
   enable_alltoall: False
@@ -179,7 +193,7 @@ parallel:
   strategy_ckpt_save_file: "./ckpt_strategy.ckpt"
 parallel_config:
   data_parallel: 1
-  model_parallel: 4
+  model_parallel: 1
   pipeline_stage: 1
   expert_parallel: 1
   optimizer_shard: False  # optimizer shard

configs/glm/run_glm_6b_lora.yaml

@@ -53,7 +53,7 @@ model:
     max_decode_length: 2048  # The maximum length of the generated words.
     is_enhanced_encoder: True
     is_npu_acceleration: True   # sample in npu to acceleration
-    checkpoint_name_or_path: "glm_6b"
+    checkpoint_name_or_path: "glm_6b_lora_chat"
     top_k: 1
     top_p: 1    # top_p not support for npu_acceleration yet 
     repetition_penalty: 1

configs/glm/run_glm_6b_lora_infer.yaml

@@ -25,12 +25,12 @@ optimizer:
   beta1: 0.9
   beta2: 0.95
   eps: 1.e-8
-  learning_rate: 2.e-5
+  learning_rate: 1.e-4
 
 # lr sechdule
 lr_schedule:
   type: CosineWithWarmUpLR
-  learning_rate: 2.e-5
+  learning_rate: 1.e-4
   warmup_ratio: 0.03
   total_steps: -1 # -1 means it will load the total steps of the dataset
 
@@ -166,7 +166,7 @@ model:
         # configurition of lora
         in_channels: 4096 
         out_channels: 4096
-        lora_rank: 8
+        lora_rank: 16
         lora_alpha: 16
         lora_dropout: 0.05
 
@@ -193,6 +193,9 @@ runner_wrapper:
     scale_window: 1000
   use_clip_grad: True
 
+eval_callbacks:
+  - type: ObsMonitor
+
 auto_tune: False
 filepath_prefix: './autotune'
 autotune_per_step: 10