Zhwang/bart

.vscode/settings.json

@@ -67,6 +67,23 @@
         "unordered_set": "cpp",
         "future": "cpp",
         "cfenv": "cpp",
-        "typeindex": "cpp"
+        "typeindex": "cpp",
+        "__bit_reference": "cpp",
+        "__config": "cpp",
+        "__debug": "cpp",
+        "__errc": "cpp",
+        "__hash_table": "cpp",
+        "__locale": "cpp",
+        "__mutex_base": "cpp",
+        "__node_handle": "cpp",
+        "__split_buffer": "cpp",
+        "__threading_support": "cpp",
+        "__tree": "cpp",
+        "__verbose_abort": "cpp",
+        "charconv": "cpp",
+        "ios": "cpp",
+        "locale": "cpp",
+        "variant": "cpp",
+        "__memory": "cpp"
     }
-}
+}

CMakeLists.txt

@@ -322,6 +322,7 @@ add_library(transformer-shared SHARED
   $<TARGET_OBJECTS:BertLayerWeight>
   $<TARGET_OBJECTS:BertTritonBackend>
   $<TARGET_OBJECTS:BertWeight>
+  $<TARGET_OBJECTS:BartTritonBackend>
   $<TARGET_OBJECTS:Deberta>
   $<TARGET_OBJECTS:DebertaLayerWeight>
   $<TARGET_OBJECTS:DebertaTritonBackend>

examples/cpp/CMakeLists.txt

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+add_subdirectory(bart)
 add_subdirectory(bert)
 add_subdirectory(bert_int8)
 add_subdirectory(decoding)

examples/cpp/bart/CMakeLists.txt

@@ -0,0 +1,18 @@
+# Copyright (c) 2019-2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_executable(bart_triton_example bart_triton_example.cc)
+target_link_libraries(bart_triton_example PUBLIC -lcublas -lcublasLt -lcudart -lpthread
+                      BartTritonBackend TransformerTritonBackend custom_ar_comm
+                      gpt_example_utils word_list mpi_utils nccl_utils nvtx_utils)

examples/cpp/bart/bad_words.csv

@@ -0,0 +1,2 @@
+7768,3908
+1,2

examples/cpp/bart/check_with_huggingface.py

@@ -0,0 +1,16 @@
+import transformers
+
+from transformers import LlamaForCausalLM, LlamaTokenizer
+
+tokenizer = LlamaTokenizer.from_pretrained('/data/llama-7b-hf')
+
+prompt = "Hey, are you consciours? Can you talk to me?"
+inputs = tokenizer(prompt, return_tensors='pt')
+model = LlamaForCausalLM.from_pretrained("/data/llama-7b-hf")
+hf_config = vars(model.config)
+print(hf_config)
+generated_ids = model.forward(inputs.input_ids, output_hidden_states=True)
+print(generated_ids)
+
+tokens = [0,18637,29892,526,366,1136,455,2470,29973,1815,366,5193,304,592,29973,18637,29892,526,366,1136,455,2470,29973,1815,366,5193,304,592,29973,18637,29892,526,366,1136,455,2470,29973,1815,366,5193,304,592,29973,18637,29892,526,366]
+print(tokenizer.decode(tokens))

examples/cpp/bart/config.ini

@@ -0,0 +1,47 @@
+[ft_instance_hyperparameter]
+data_type=fp16
+enable_custom_all_reduce=0
+
+tensor_para_size=1
+pipeline_para_size=1
+
+model_name=decoder
+model_dir=/notebooks/bart-ft/1/1-gpu/
+
+[request]
+beam_width=1 # beam width for beam search
+top_k=1 ; k value for top k sampling
+top_p=0.0 ; p value for top p sampling
+temperature=1.0 ; Use for sampling
+repetition_penalty=1.0 ; Use for sampling
+presence_penalty=0.0  ; Only one of repetition_penalty and presence_penalty are allowed.
+len_penalty=0.0
+beam_search_diversity_rate=0.0
+request_batch_size=8 # determine by the request
+request_output_len=32 # determine by the request
+
+[encoder]
+num_heads = 12
+d_kv = 64
+d_model = 768
+d_ff = 3072
+num_layers = 6
+vocab_size = 50265
+max_pos_seq_len = 1024
+model_name = bart
+feed_forward_proj = gelu
+weight_data_type = fp32
+
+[decoder]
+num_heads = 12
+d_kv = 64
+d_model = 768
+d_ff = 3072
+num_layers = 6
+vocab_size = 50265
+max_pos_seq_len = 1024
+decoder_start_token_id = 2
+eos_token_id = 2
+start_id = 1
+end_id = 1
+weight_data_type = fp32

examples/cpp/bart/start_ids.csv

@@ -0,0 +1,2 @@
+0, 4154, 1231, 15674, 345, 1534, 440, 50264, 11, 1854, 2
+0, 4154, 1231, 15674, 345, 1534, 440, 50264, 11, 1854, 2

examples/cpp/bart/stop_words.csv

@@ -0,0 +1,2 @@
+287, 4346, 12
+3, -1, -1

examples/cpp/multi_gpu_gpt/gpt_example_utils.cc

@@ -91,6 +91,9 @@ int read_start_ids(size_t            batch_size,
             v_start_lengths->push_back(tmp_start_lengths[i]);
         }
     }
+    for (int i : *v_start_lengths) {
+        printf("v_start_lengths %d\n", i);
+    }
     return batch_size;
 }
 

examples/pytorch/bart/translate_example.py

@@ -213,7 +213,7 @@ def translate(args_dict):
                                      config.decoder_start_token_id, config.eos_token_id, config.vocab_size,
                                      tensor_para_size=tensor_para_size, pipeline_para_size=pipeline_para_size, 
                                      bart_with_bias=bart_with_bias, mbart=is_mbart,
-                                     position_embedding_type=position_embedding_type, 
+                                     position_embedding_type=position_embedding_type,
                                      activation_type=activation_type, layernorm_type=layernorm_type)
 
         ft_bart = FTBart(ft_encoder, ft_decoding)
@@ -375,4 +375,4 @@ def translate(args_dict):
     args = parser.parse_args()
     log_format = "%(asctime)s %(name)s [%(levelname)s] %(message)s"
     logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format=log_format)
-    translate(vars(args))
+    translate(vars(args))