From 2201642cb16a5454a045b2f6deef36820bd1ae19 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 19 Jul 2023 09:36:19 -0700 Subject: [PATCH 01/60] abc --- CMakeLists.txt | 2 +- src/libfastertransformer.cc | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e9c6c5c..d539453 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,7 +110,7 @@ if (EXISTS ${FT_DIR}) else() FetchContent_Declare( repo-ft - GIT_REPOSITORY https://github.com/NVIDIA/FasterTransformer.git + GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git GIT_TAG main GIT_SHALLOW ON ) diff --git a/src/libfastertransformer.cc b/src/libfastertransformer.cc index a870aa0..f2bbc0e 100644 --- a/src/libfastertransformer.cc +++ b/src/libfastertransformer.cc @@ -49,6 +49,7 @@ // FT's libraries have dependency with triton's lib #include "src/fastertransformer/triton_backend/bert/BertTritonModel.h" +#include "src/fastertransformer/triton_backend/deberta/DebertaTritonModel.h" #include "src/fastertransformer/triton_backend/gptj/GptJTritonModel.h" #include "src/fastertransformer/triton_backend/gptj/GptJTritonModelInstance.h" #include "src/fastertransformer/triton_backend/gptneox/GptNeoXTritonModel.h" @@ -327,6 +328,22 @@ std::shared_ptr ModelState::ModelFactory( } else if (data_type == "bf16") { ft_model = std::make_shared>( tp, pp, custom_ar, model_dir, int8_mode, is_sparse, remove_padding); +#endif + } + } else if (model_type == "deberta") { + const int is_sparse = param_get_bool(param,"is_sparse", false); + const int remove_padding = param_get_bool(param,"is_remove_padding", false); + + if (data_type == "fp16") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, is_sparse, remove_padding); + } else if (data_type == "fp32") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, is_sparse, remove_padding); +#ifdef ENABLE_BF16 + } else if (data_type == "bf16") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, is_sparse, remove_padding); #endif } } else { From 5b7db76901768a2e073aef2484deaa8747979d4c Mon Sep 17 00:00:00 2001 From: Francisco Kattan <2194202+fjkattan@users.noreply.github.com> Date: Fri, 21 Jul 2023 19:40:42 -0700 Subject: [PATCH 02/60] Add files via upload --- LEGAL.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 LEGAL.md diff --git a/LEGAL.md b/LEGAL.md new file mode 100644 index 0000000..e52a5f1 --- /dev/null +++ b/LEGAL.md @@ -0,0 +1 @@ +This application is not part of the Snowflake Service and is governed by the terms in LICENSE, unless expressly agreed to in writing. You use this application at your own risk, and Snowflake has no obligation to support your use of this application. From 968920ad849522e401ef321252723a7b0847fa68 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Fri, 18 Aug 2023 23:59:17 -0700 Subject: [PATCH 03/60] commit --- src/libfastertransformer.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/libfastertransformer.cc b/src/libfastertransformer.cc index a870aa0..a3b459f 100644 --- a/src/libfastertransformer.cc +++ b/src/libfastertransformer.cc @@ -53,6 +53,7 @@ #include "src/fastertransformer/triton_backend/gptj/GptJTritonModelInstance.h" #include "src/fastertransformer/triton_backend/gptneox/GptNeoXTritonModel.h" #include "src/fastertransformer/triton_backend/gptneox/GptNeoXTritonModelInstance.h" +#include "src/fastertransformer/triton_backend/llama/LlamaTritonModel.h" #include "src/fastertransformer/triton_backend/multi_gpu_gpt/ParallelGptTritonModel.h" #include "src/fastertransformer/triton_backend/multi_gpu_gpt/ParallelGptTritonModelInstance.h" #include "src/fastertransformer/triton_backend/t5/T5TritonModel.h" @@ -327,6 +328,21 @@ std::shared_ptr ModelState::ModelFactory( } else if (data_type == "bf16") { ft_model = std::make_shared>( tp, pp, custom_ar, model_dir, int8_mode, is_sparse, remove_padding); +#endif + } + } else if (model_type == "llama") { + const int int8_mode = param_get_int(param, "int8_mode"); + + if (data_type == "fp16") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, int8_mode); + } else if (data_type == "fp32") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, int8_mode); +#ifdef ENABLE_BF16 + } else if (data_type == "bf16") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, int8_mode); #endif } } else { From 1d8b7fd88df97cd9d2ee0f1642c57418a2a691f9 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sat, 19 Aug 2023 00:00:31 -0700 Subject: [PATCH 04/60] commit --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e9c6c5c..bd0fe84 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,8 +110,8 @@ if (EXISTS ${FT_DIR}) else() FetchContent_Declare( repo-ft - GIT_REPOSITORY https://github.com/NVIDIA/FasterTransformer.git - GIT_TAG main + GIT_REPOSITORY https://github.com/sfc-gh-zhwang/FasterTransformer + GIT_TAG e770ddf2bc66217034b6e9e3b0c3256ebf1c1b40 GIT_SHALLOW ON ) endif() From 123933b93a97488e6bf8ad28d15670ed18bb7637 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sat, 19 Aug 2023 22:24:08 -0700 Subject: [PATCH 05/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bd0fe84..afb42dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/sfc-gh-zhwang/FasterTransformer - GIT_TAG e770ddf2bc66217034b6e9e3b0c3256ebf1c1b40 + GIT_TAG d7ac0faef8b31ad207dd80a57deb65405992f7f9 GIT_SHALLOW ON ) endif() From ab152e2d3c797d18d4c22f1673be215df2826758 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 21 Aug 2023 13:26:33 -0700 Subject: [PATCH 06/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index afb42dd..7d8ba3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/sfc-gh-zhwang/FasterTransformer - GIT_TAG d7ac0faef8b31ad207dd80a57deb65405992f7f9 + GIT_TAG 7555256b6d861345886e09120083cae82c61a1fc GIT_SHALLOW ON ) endif() From 5db164bc56df46825c28353233f43d4a398a60fb Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 21 Aug 2023 13:47:52 -0700 Subject: [PATCH 07/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d8ba3e..8aba851 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/sfc-gh-zhwang/FasterTransformer - GIT_TAG 7555256b6d861345886e09120083cae82c61a1fc + GIT_TAG 0cda56c9830ebba575477457258e1046c121fdea GIT_SHALLOW ON ) endif() From 27b9a3660076fb6621a84bbec3e08318c7322782 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 23 Aug 2023 20:17:42 -0700 Subject: [PATCH 08/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8aba851..b88d2ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/sfc-gh-zhwang/FasterTransformer - GIT_TAG 0cda56c9830ebba575477457258e1046c121fdea + GIT_TAG 5db164bc56df46825c28353233f43d4a398a60fb GIT_SHALLOW ON ) endif() From 314cc9d0435fd58387dcbe8e75fa32ddc1f90ef7 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 23 Aug 2023 20:19:19 -0700 Subject: [PATCH 09/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b88d2ee..411b120 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/sfc-gh-zhwang/FasterTransformer - GIT_TAG 5db164bc56df46825c28353233f43d4a398a60fb + GIT_TAG 5a28f0e460ac005545c893acd265329c884d24e8 GIT_SHALLOW ON ) endif() From ec8ba9476923b6b14acc4e3734f36227aed5f864 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 23 Aug 2023 20:22:49 -0700 Subject: [PATCH 10/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 411b120..139b1ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/sfc-gh-zhwang/FasterTransformer - GIT_TAG 5a28f0e460ac005545c893acd265329c884d24e8 + GIT_TAG f6e340308d71ec0ddb2a9e009d3df568505a1612 GIT_SHALLOW ON ) endif() From 63cb0b67a192df6bca0a853480cb3bf20db775ab Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 23 Aug 2023 23:47:08 -0700 Subject: [PATCH 11/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 139b1ff..0ef201e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/sfc-gh-zhwang/FasterTransformer - GIT_TAG f6e340308d71ec0ddb2a9e009d3df568505a1612 + GIT_TAG 2d332b3a3766a56df18c9b981841ead3236415ba GIT_SHALLOW ON ) endif() From 25fbeb902e1c53939af7a7f7df471d08fdb48c8d Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 24 Aug 2023 14:43:11 -0700 Subject: [PATCH 12/60] commit --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ef201e..f7b8f3f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,8 +110,8 @@ if (EXISTS ${FT_DIR}) else() FetchContent_Declare( repo-ft - GIT_REPOSITORY https://github.com/sfc-gh-zhwang/FasterTransformer - GIT_TAG 2d332b3a3766a56df18c9b981841ead3236415ba + GIT_REPOSITORY https://github.com/neevaco/FasterTransformer + GIT_TAG 9c7b9934db47ba6d8034e3c54294288a165f520a GIT_SHALLOW ON ) endif() From 35b938849c522058079e84bfa89da78f32e2f8a0 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 24 Aug 2023 14:54:47 -0700 Subject: [PATCH 13/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f7b8f3f..b03ee1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer - GIT_TAG 9c7b9934db47ba6d8034e3c54294288a165f520a + GIT_TAG 86c24253bae05e507f784cd28e5726dbb86cdea9 GIT_SHALLOW ON ) endif() From d259efc5c299238cb6b854761b50fff3f243dc71 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 24 Aug 2023 15:04:37 -0700 Subject: [PATCH 14/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b03ee1f..572c8df 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer - GIT_TAG 86c24253bae05e507f784cd28e5726dbb86cdea9 + GIT_TAG f1e61e6429f7342ab5aa42b56e9feddf708c26dc GIT_SHALLOW ON ) endif() From e0562b113af64c1a149489a7a8aef2ef14882f76 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 24 Aug 2023 17:36:57 -0700 Subject: [PATCH 15/60] commit --- src/libfastertransformer.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/libfastertransformer.cc b/src/libfastertransformer.cc index a3b459f..f478c22 100644 --- a/src/libfastertransformer.cc +++ b/src/libfastertransformer.cc @@ -49,6 +49,7 @@ // FT's libraries have dependency with triton's lib #include "src/fastertransformer/triton_backend/bert/BertTritonModel.h" +#include "src/fastertransformer/triton_backend/deberta/DebertaTritonModel.h" #include "src/fastertransformer/triton_backend/gptj/GptJTritonModel.h" #include "src/fastertransformer/triton_backend/gptj/GptJTritonModelInstance.h" #include "src/fastertransformer/triton_backend/gptneox/GptNeoXTritonModel.h" @@ -343,6 +344,22 @@ std::shared_ptr ModelState::ModelFactory( } else if (data_type == "bf16") { ft_model = std::make_shared>( tp, pp, custom_ar, model_dir, int8_mode); +#endif + } + } else if (model_type == "deberta") { + const int is_sparse = param_get_bool(param,"is_sparse", false); + const int remove_padding = param_get_bool(param,"is_remove_padding", false); + + if (data_type == "fp16") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, is_sparse, remove_padding); + } else if (data_type == "fp32") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, is_sparse, remove_padding); +#ifdef ENABLE_BF16 + } else if (data_type == "bf16") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, is_sparse, remove_padding); #endif } } else { From 2d60bbec0dd8e8aa9b2c6257129dbb1e5c5507d0 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 28 Aug 2023 14:15:51 -0700 Subject: [PATCH 16/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 572c8df..23919e4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer - GIT_TAG f1e61e6429f7342ab5aa42b56e9feddf708c26dc + GIT_TAG affa1ef1c175d03db8ff5b14824cc58dd2c52c2b GIT_SHALLOW ON ) endif() From 31babb05d8052699b340b6d42f4571f818bc2c9b Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 28 Aug 2023 14:19:20 -0700 Subject: [PATCH 17/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 23919e4..9ff166f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,7 +110,7 @@ if (EXISTS ${FT_DIR}) else() FetchContent_Declare( repo-ft - GIT_REPOSITORY https://github.com/neevaco/FasterTransformer + GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git GIT_TAG affa1ef1c175d03db8ff5b14824cc58dd2c52c2b GIT_SHALLOW ON ) From 08625c3aafb375f1da80c84174062208cbf6ae97 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 29 Aug 2023 23:17:44 -0700 Subject: [PATCH 18/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ff166f..06a780b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG affa1ef1c175d03db8ff5b14824cc58dd2c52c2b + GIT_TAG 6dad842e87f72bd074867d1c11ddca0121862e85 GIT_SHALLOW ON ) endif() From ffd06a78527ef910cdb5be86453731455f31c71e Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 30 Aug 2023 15:40:28 -0700 Subject: [PATCH 19/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 06a780b..96e733a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 6dad842e87f72bd074867d1c11ddca0121862e85 + GIT_TAG 031c8c433cb84c091eeb1839c18c089d15de381d GIT_SHALLOW ON ) endif() From 58b54ecab0b0a4447c61cab28d11f2caf54100f0 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 30 Aug 2023 16:15:51 -0700 Subject: [PATCH 20/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 96e733a..7fe767f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 031c8c433cb84c091eeb1839c18c089d15de381d + GIT_TAG 7bb372317da21dc7a898cb0e6e0ce7c11b0b38ec GIT_SHALLOW ON ) endif() From f0da91f0aefeab5635e5c65e3adba2065baa509a Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 6 Sep 2023 13:46:47 -0700 Subject: [PATCH 21/60] update FasterTransformer commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eb64ecf..ab328b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 7bb372317da21dc7a898cb0e6e0ce7c11b0b38ec + GIT_TAG 5b9ff0fc656566a54682469f777f2aa44181b5fa GIT_SHALLOW ON ) endif() From c0d26f6a34defcca88b221ce31976328c9b2ae64 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 11 Sep 2023 01:18:25 -0700 Subject: [PATCH 22/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ab328b8..a0aaa1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 5b9ff0fc656566a54682469f777f2aa44181b5fa + GIT_TAG cf5fc55fc00e81684f228f7eb1ad6ed3ec80313e GIT_SHALLOW ON ) endif() From 52ec31274561361f88de978422623c836e73223c Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 12 Sep 2023 12:54:58 -0700 Subject: [PATCH 23/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a0aaa1f..f86ee8a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG cf5fc55fc00e81684f228f7eb1ad6ed3ec80313e + GIT_TAG 23b37c7b158d05f6206119b1ef831c8e63cc1eb9 GIT_SHALLOW ON ) endif() From 314735723e1a2345c71763f193a609ec82744530 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sat, 23 Sep 2023 20:44:21 -0700 Subject: [PATCH 24/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f86ee8a..4f628ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 23b37c7b158d05f6206119b1ef831c8e63cc1eb9 + GIT_TAG 8895fb20887b5b7ed3435739426316f560ac6f6d GIT_SHALLOW ON ) endif() From 8de4960e738b31da2a8203b3162a8e46d47b0244 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sat, 23 Sep 2023 20:45:30 -0700 Subject: [PATCH 25/60] commit --- src/libfastertransformer.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/libfastertransformer.cc b/src/libfastertransformer.cc index f478c22..b9d2f8e 100644 --- a/src/libfastertransformer.cc +++ b/src/libfastertransformer.cc @@ -344,6 +344,22 @@ std::shared_ptr ModelState::ModelFactory( } else if (data_type == "bf16") { ft_model = std::make_shared>( tp, pp, custom_ar, model_dir, int8_mode); +#endif + } + } else if (model_type == "bart") { + const int is_sparse = param_get_bool(param,"is_sparse", false); + const int remove_padding = param_get_bool(param,"is_remove_padding", false); + + if (data_type == "fp16") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, is_sparse, remove_padding); + } else if (data_type == "fp32") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, is_sparse, remove_padding); +#ifdef ENABLE_BF16 + } else if (data_type == "bf16") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, is_sparse, remove_padding); #endif } } else if (model_type == "deberta") { From bde5c23a9d23310ee6ca22a5525f762f590b91fd Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sat, 23 Sep 2023 20:46:03 -0700 Subject: [PATCH 26/60] commit --- src/libfastertransformer.cc | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/libfastertransformer.cc b/src/libfastertransformer.cc index b9d2f8e..e4b1a1c 100644 --- a/src/libfastertransformer.cc +++ b/src/libfastertransformer.cc @@ -347,19 +347,16 @@ std::shared_ptr ModelState::ModelFactory( #endif } } else if (model_type == "bart") { - const int is_sparse = param_get_bool(param,"is_sparse", false); - const int remove_padding = param_get_bool(param,"is_remove_padding", false); - if (data_type == "fp16") { - ft_model = std::make_shared>( - tp, pp, custom_ar, model_dir, is_sparse, remove_padding); + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, 0); } else if (data_type == "fp32") { - ft_model = std::make_shared>( - tp, pp, custom_ar, model_dir, is_sparse, remove_padding); + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, 0); #ifdef ENABLE_BF16 } else if (data_type == "bf16") { - ft_model = std::make_shared>( - tp, pp, custom_ar, model_dir, is_sparse, remove_padding); + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, 0); #endif } } else if (model_type == "deberta") { From 7768d469d20ab7a8b63fff3e39aeb7328b1446b9 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sat, 23 Sep 2023 20:46:30 -0700 Subject: [PATCH 27/60] commit --- src/libfastertransformer.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfastertransformer.cc b/src/libfastertransformer.cc index e4b1a1c..55184cc 100644 --- a/src/libfastertransformer.cc +++ b/src/libfastertransformer.cc @@ -49,6 +49,7 @@ // FT's libraries have dependency with triton's lib #include "src/fastertransformer/triton_backend/bert/BertTritonModel.h" +#include "src/fastertransformer/triton_backend/bart/BartTritonModel.h" #include "src/fastertransformer/triton_backend/deberta/DebertaTritonModel.h" #include "src/fastertransformer/triton_backend/gptj/GptJTritonModel.h" #include "src/fastertransformer/triton_backend/gptj/GptJTritonModelInstance.h" From 065a0831f3b1b7f973548787430e48eef8c3667d Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sun, 24 Sep 2023 12:02:39 -0700 Subject: [PATCH 28/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f628ee..22da5b6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 8895fb20887b5b7ed3435739426316f560ac6f6d + GIT_TAG 179061956cb1045b01fa4286d51019fabf87d49d GIT_SHALLOW ON ) endif() From 49772345409597d489663a067bab3bae13ba150e Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sun, 24 Sep 2023 14:44:40 -0700 Subject: [PATCH 29/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 22da5b6..b02afb5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 179061956cb1045b01fa4286d51019fabf87d49d + GIT_TAG b997e177ff37d099597de5f884a41d1a59623588 GIT_SHALLOW ON ) endif() From 1931ae85b841839b24f96a7d5920bb2194be90ab Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sun, 24 Sep 2023 15:40:18 -0700 Subject: [PATCH 30/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b02afb5..bc4470e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG b997e177ff37d099597de5f884a41d1a59623588 + GIT_TAG a456d0452ed5b913983bdb9cccdc3fb4b202e113 GIT_SHALLOW ON ) endif() From 7473999a2007651d4ff0a36254efd6e790275064 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 25 Sep 2023 15:13:35 -0700 Subject: [PATCH 31/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bc4470e..4675ee5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG a456d0452ed5b913983bdb9cccdc3fb4b202e113 + GIT_TAG 6b12e674bb8add01f1dbfafa94ce65c9be9a466e GIT_SHALLOW ON ) endif() From 636c79f9d8bf8505ea9633c561e602c6c01b026f Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 25 Sep 2023 19:09:13 -0700 Subject: [PATCH 32/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4675ee5..e71f2ab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 6b12e674bb8add01f1dbfafa94ce65c9be9a466e + GIT_TAG 4eabe3be17493aa6d29f4b800afe71bccce85699 GIT_SHALLOW ON ) endif() From d6c01f6feaada4c742b9363a934dc677b09b2fa5 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 25 Sep 2023 19:35:08 -0700 Subject: [PATCH 33/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e71f2ab..d45dd0c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 4eabe3be17493aa6d29f4b800afe71bccce85699 + GIT_TAG 218c92e2e78261d79c2f8e7bd00e35e7f64776f5 GIT_SHALLOW ON ) endif() From ed99f64b3f5c6b1409a7b85d4ead159d5fcec3f6 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 26 Sep 2023 08:50:21 -0700 Subject: [PATCH 34/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d45dd0c..d959442 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 218c92e2e78261d79c2f8e7bd00e35e7f64776f5 + GIT_TAG 1ef6939fdf1c0ff566de0b7fa7ccd0fe338984a7 GIT_SHALLOW ON ) endif() From 4d0a407ee38c60bb7284d5a6bd9fa107b6fea0c9 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 26 Sep 2023 13:44:49 -0700 Subject: [PATCH 35/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d959442..c623447 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 1ef6939fdf1c0ff566de0b7fa7ccd0fe338984a7 + GIT_TAG a776ecacfb8ef054366ff66de02b491ee93317af GIT_SHALLOW ON ) endif() From 55afffb08071b84e79b0394571d6a10c85ea5469 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 28 Sep 2023 10:32:38 -0700 Subject: [PATCH 36/60] commit --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c623447..d919ce8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,7 +115,6 @@ else() repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git GIT_TAG a776ecacfb8ef054366ff66de02b491ee93317af - GIT_SHALLOW ON ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From 9e66f9adc6dfd04f4cacec3998518e05603e577c Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 28 Sep 2023 20:28:11 -0700 Subject: [PATCH 37/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d919ce8..62ce8cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG a776ecacfb8ef054366ff66de02b491ee93317af + GIT_TAG e3b7ef5aec942b50aac01d2fa2f4d066ab82da22 ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From a190564923c52e0e80c720d93c95b85cf8a560b8 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 28 Sep 2023 23:29:37 -0700 Subject: [PATCH 38/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 62ce8cf..7d09949 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG e3b7ef5aec942b50aac01d2fa2f4d066ab82da22 + GIT_TAG 71472e5cf22475f94eb32fcd6d5742fd821f9305 ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From d67080639a55c0671a9bc6cdcce19d406aa0ca66 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Fri, 29 Sep 2023 10:24:37 -0700 Subject: [PATCH 39/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d09949..6c814f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 71472e5cf22475f94eb32fcd6d5742fd821f9305 + GIT_TAG e4e4e5187664ff7ddede308e0a0670d0463fa660 ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From 33729f044c905ba9c4682ba3b71e92cf4d48cfa4 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 5 Oct 2023 13:57:43 -0700 Subject: [PATCH 40/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c814f6..80bd1aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG e4e4e5187664ff7ddede308e0a0670d0463fa660 + GIT_TAG e0b124a77c847f431775b0fe68fab8dc207503cc ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From a33ad4eae53126117dc7fe7dc0afeb61de2ebc0d Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sat, 7 Oct 2023 19:08:56 -0700 Subject: [PATCH 41/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 80bd1aa..c5a4d4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG e0b124a77c847f431775b0fe68fab8dc207503cc + GIT_TAG 63002c8323862b27984fcf488f208f4c943c5db1 ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From 03bebc896563cc10cc96973da1f3ede0a866b8a5 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sat, 7 Oct 2023 23:21:27 -0700 Subject: [PATCH 42/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c5a4d4b..b1ede18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 63002c8323862b27984fcf488f208f4c943c5db1 + GIT_TAG e2bc80686859eb4bf83c66ee49beee8d29e8aafd ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From 9a788317365240a1e8322d91ed328157af82139a Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 12 Oct 2023 10:54:57 -0700 Subject: [PATCH 43/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b1ede18..2a90d37 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG e2bc80686859eb4bf83c66ee49beee8d29e8aafd + GIT_TAG d1f088243f98ea967b44658ba8b50dd05da1a2eb ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From 48051df599c3f9e7c64586adb987fd7bda7e4962 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 12 Oct 2023 22:12:07 -0700 Subject: [PATCH 44/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a90d37..b511b1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG d1f088243f98ea967b44658ba8b50dd05da1a2eb + GIT_TAG 5ed1f245e7f06d9cb72220ee9c9a254918b251ef ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From 75508a7e590fb7c387d3796c94622a12ea0345a4 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 16 Oct 2023 13:23:51 -0700 Subject: [PATCH 45/60] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b511b1f..61ea721 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 5ed1f245e7f06d9cb72220ee9c9a254918b251ef + GIT_TAG 336e487a7d42932e819af5a405c6fe0b2d4b7053 ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From fb2e9d9ac4bc9a3fb0cc7c39b51e5299fccc7a2e Mon Sep 17 00:00:00 2001 From: Yahia Bsat Date: Thu, 19 Oct 2023 17:59:45 -0700 Subject: [PATCH 46/60] add langid --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index bfbfba0..9eaebb1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -42,7 +42,8 @@ RUN apt-get update && \ RUN pip3 install --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cu118 torch==2.0.1+cu118 && \ pip3 install --no-cache-dir --extra-index-url https://pypi.ngc.nvidia.com regex fire tritonclient[all] && \ pip3 install --no-cache-dir accelerate transformers huggingface_hub tokenizers SentencePiece sacrebleu datasets tqdm omegaconf rouge_score && \ - pip3 install --no-cache-dir cmake==3.24.3 + pip3 install --no-cache-dir cmake==3.24.3 && \ + pip3 install --no-cache-dir langid==1.1.6 # backend build ADD . /workspace/build/fastertransformer_backend From dcfde00c95d996585d181f8e10d8e9de247224b0 Mon Sep 17 00:00:00 2001 From: Flex Wang Date: Tue, 24 Oct 2023 15:58:29 -0700 Subject: [PATCH 47/60] commit (#31) --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..2428d96 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +@sfc-gh-zhwang @sfc-gh-hykim From d2e15ad798235ee4f74f5b723a37bda17ac69814 Mon Sep 17 00:00:00 2001 From: Flex Wang Date: Thu, 26 Oct 2023 14:10:29 -0700 Subject: [PATCH 48/60] Zhwang/codeowner (#32) * commit * commit --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2428d96..4fbf6af 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -@sfc-gh-zhwang @sfc-gh-hykim +* @sfc-gh-zhwang @sfc-gh-hykim \ No newline at end of file From 801c149642a6f8fc3412e8d0533262d5409f045f Mon Sep 17 00:00:00 2001 From: Flex Wang Date: Mon, 30 Oct 2023 11:03:34 -0700 Subject: [PATCH 49/60] commit (#33) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 61ea721..a7f4aa4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 336e487a7d42932e819af5a405c6fe0b2d4b7053 + GIT_TAG 8e10a85d2859e385c11604d049e3d575a275a2d7 ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From d9ab9133fa43895b7558d555009f52a03b874f8e Mon Sep 17 00:00:00 2001 From: Flex Wang Date: Mon, 30 Oct 2023 22:24:17 -0700 Subject: [PATCH 50/60] commit (#34) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a7f4aa4..a9339d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 8e10a85d2859e385c11604d049e3d575a275a2d7 + GIT_TAG 1f8a87581e38f6f4290d827385ea1fb7b6ad8fe3 ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From df3d9f578dab588facd243f39938220c7531c6f1 Mon Sep 17 00:00:00 2001 From: Yahia Bsat Date: Fri, 10 Nov 2023 16:41:25 -0800 Subject: [PATCH 51/60] update --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a9339d6..075db2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 1f8a87581e38f6f4290d827385ea1fb7b6ad8fe3 + GIT_TAG 71ff0c5e6814aba1b03cfdef499ad52d5202aaed ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From 85139a9d05fa85527b0fd0f33ce60d2b56d1d90d Mon Sep 17 00:00:00 2001 From: Yahia Bsat Date: Mon, 13 Nov 2023 16:46:37 -0800 Subject: [PATCH 52/60] update --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 075db2c..7fba4fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 71ff0c5e6814aba1b03cfdef499ad52d5202aaed + GIT_TAG ca36cf846b45593b0c295b00ffc63fb3797ca73f ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From 106e65897906b04acf4c54776ddc0170e1152502 Mon Sep 17 00:00:00 2001 From: Yahia Bsat Date: Fri, 17 Nov 2023 18:55:27 -0800 Subject: [PATCH 53/60] update tag --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7fba4fe..14dc813 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG ca36cf846b45593b0c295b00ffc63fb3797ca73f + GIT_TAG e7d2d16fd1adba3fd797ca68f3f0136d9fa95ada ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From ede71f750f2cf4d14a690f04b8c6a351f5de964b Mon Sep 17 00:00:00 2001 From: Yahia Bsat Date: Tue, 16 Jan 2024 18:47:49 -0800 Subject: [PATCH 54/60] m2m --- src/libfastertransformer.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libfastertransformer.cc b/src/libfastertransformer.cc index 06024dd..06ec7a0 100644 --- a/src/libfastertransformer.cc +++ b/src/libfastertransformer.cc @@ -50,6 +50,7 @@ // FT's libraries have dependency with triton's lib #include "src/fastertransformer/triton_backend/bert/BertTritonModel.h" #include "src/fastertransformer/triton_backend/bart/BartTritonModel.h" +#include "src/fastertransformer/triton_backend/m2m/M2MTritonModel.h" #include "src/fastertransformer/triton_backend/deberta/DebertaTritonModel.h" #include "src/fastertransformer/triton_backend/gptj/GptJTritonModel.h" #include "src/fastertransformer/triton_backend/gptj/GptJTritonModelInstance.h" @@ -358,6 +359,19 @@ std::shared_ptr ModelState::ModelFactory( } else if (data_type == "bf16") { ft_model = std::make_shared>( tp, pp, custom_ar, model_dir, 0); +#endif + } + } else if (model_type == "m2m") { + if (data_type == "fp16") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, 0); + } else if (data_type == "fp32") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, 0); +#ifdef ENABLE_BF16 + } else if (data_type == "bf16") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, 0); #endif } } else if (model_type == "deberta") { From 341ad6cae00bdc73fa071d81d21d9896fd1f26fc Mon Sep 17 00:00:00 2001 From: Yahia Bsat Date: Wed, 17 Jan 2024 08:10:58 -0800 Subject: [PATCH 55/60] tag --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 14dc813..8c3d456 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG e7d2d16fd1adba3fd797ca68f3f0136d9fa95ada + GIT_TAG f3ca6ae2b0a041dbd44636f4919b59d93fb0908b ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From eb0f589fed610559db8df4532954f8216a1c0da5 Mon Sep 17 00:00:00 2001 From: Yahia Bsat Date: Wed, 17 Jan 2024 12:03:05 -0800 Subject: [PATCH 56/60] update --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c3d456..c65b596 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG f3ca6ae2b0a041dbd44636f4919b59d93fb0908b + GIT_TAG b6b21406449ab19f00d1d5f97338065037b5f8e3 ) endif() FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft) From d6baf7c2c68d74b77598d59e19ff00659a28c1ba Mon Sep 17 00:00:00 2001 From: Yahia Bsat Date: Thu, 8 Feb 2024 09:34:49 -0800 Subject: [PATCH 57/60] d --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9eaebb1..9a0c413 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -43,7 +43,8 @@ RUN pip3 install --no-cache-dir --extra-index-url https://download.pytorch.org/w pip3 install --no-cache-dir --extra-index-url https://pypi.ngc.nvidia.com regex fire tritonclient[all] && \ pip3 install --no-cache-dir accelerate transformers huggingface_hub tokenizers SentencePiece sacrebleu datasets tqdm omegaconf rouge_score && \ pip3 install --no-cache-dir cmake==3.24.3 && \ - pip3 install --no-cache-dir langid==1.1.6 + pip3 install --no-cache-dir langid==1.1.6 && \ + pip3 install --no-cache-dir lingua-language-detector==2.0.2 # backend build ADD . /workspace/build/fastertransformer_backend From 312992c83cfb3277e3de0594540c2a83c325c7ee Mon Sep 17 00:00:00 2001 From: Domenic Bove Date: Wed, 2 Oct 2024 09:01:46 -0700 Subject: [PATCH 58/60] going to latest triton version to resolve cves --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9a0c413..5db1afc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG TRITON_VERSION=23.05 +ARG TRITON_VERSION=24.09 ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 FROM ${BASE_IMAGE} From 84642ae34907c02eb1ecc10093a6589ea579178a Mon Sep 17 00:00:00 2001 From: Domenic Bove Date: Wed, 2 Oct 2024 12:26:50 -0700 Subject: [PATCH 59/60] build failing w latest triton image... lets try just removing the cve packages --- docker/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5db1afc..52267a4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG TRITON_VERSION=24.09 +ARG TRITON_VERSION=23.05 ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 FROM ${BASE_IMAGE} @@ -68,6 +68,11 @@ RUN CUDAFLAGS="-include stdio.h" cmake \ rm /workspace/build/fastertransformer_backend/build/bin/*_example -rf && \ rm /workspace/build/fastertransformer_backend/build/lib/lib*Backend.so -rf +# Removing git because of CVEs, no longer needed after build +RUN apt-get remove git git-man -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + ENV NCCL_LAUNCH_MODE=GROUP ENV WORKSPACE /workspace WORKDIR /workspace From ff4dc040796247b48d95f4aa682fbe80143086f8 Mon Sep 17 00:00:00 2001 From: Domenic Bove Date: Thu, 3 Oct 2024 13:03:35 -0700 Subject: [PATCH 60/60] had to change to purge for grype scan to not list the vulns --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 52267a4..2768efc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -69,7 +69,7 @@ RUN CUDAFLAGS="-include stdio.h" cmake \ rm /workspace/build/fastertransformer_backend/build/lib/lib*Backend.so -rf # Removing git because of CVEs, no longer needed after build -RUN apt-get remove git git-man -y && \ +RUN apt-get purge git git-man -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/*