From 2201642cb16a5454a045b2f6deef36820bd1ae19 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 19 Jul 2023 09:36:19 -0700 Subject: [PATCH 01/47] abc --- CMakeLists.txt | 2 +- src/libfastertransformer.cc | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e9c6c5c..d539453 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,7 +110,7 @@ if (EXISTS ${FT_DIR}) else() FetchContent_Declare( repo-ft - GIT_REPOSITORY https://github.com/NVIDIA/FasterTransformer.git + GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git GIT_TAG main GIT_SHALLOW ON ) diff --git a/src/libfastertransformer.cc b/src/libfastertransformer.cc index a870aa0..f2bbc0e 100644 --- a/src/libfastertransformer.cc +++ b/src/libfastertransformer.cc @@ -49,6 +49,7 @@ // FT's libraries have dependency with triton's lib #include "src/fastertransformer/triton_backend/bert/BertTritonModel.h" +#include "src/fastertransformer/triton_backend/deberta/DebertaTritonModel.h" #include "src/fastertransformer/triton_backend/gptj/GptJTritonModel.h" #include "src/fastertransformer/triton_backend/gptj/GptJTritonModelInstance.h" #include "src/fastertransformer/triton_backend/gptneox/GptNeoXTritonModel.h" @@ -327,6 +328,22 @@ std::shared_ptr ModelState::ModelFactory( } else if (data_type == "bf16") { ft_model = std::make_shared>( tp, pp, custom_ar, model_dir, int8_mode, is_sparse, remove_padding); +#endif + } + } else if (model_type == "deberta") { + const int is_sparse = param_get_bool(param,"is_sparse", false); + const int remove_padding = param_get_bool(param,"is_remove_padding", false); + + if (data_type == "fp16") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, is_sparse, remove_padding); + } else if (data_type == "fp32") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, is_sparse, remove_padding); +#ifdef ENABLE_BF16 + } else if (data_type == "bf16") { + ft_model = std::make_shared>( + tp, pp, custom_ar, model_dir, is_sparse, remove_padding); #endif } } else { From 4debf6d94ba1a49deda87c66607d4f5b80d2145b Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 9 Aug 2023 22:00:50 -0700 Subject: [PATCH 02/47] commit --- CMakeLists.txt | 2 +- src/libfastertransformer.cc | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d539453..3e6be64 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG main + GIT_TAG zhwang/llama GIT_SHALLOW ON ) endif() diff --git a/src/libfastertransformer.cc b/src/libfastertransformer.cc index f2bbc0e..aff267b 100644 --- a/src/libfastertransformer.cc +++ b/src/libfastertransformer.cc @@ -54,6 +54,8 @@ #include "src/fastertransformer/triton_backend/gptj/GptJTritonModelInstance.h" #include "src/fastertransformer/triton_backend/gptneox/GptNeoXTritonModel.h" #include "src/fastertransformer/triton_backend/gptneox/GptNeoXTritonModelInstance.h" +#include "src/fastertransformer/triton_backend/llama/LlamaTritonModel.h" +#include "src/fastertransformer/triton_backend/llama/LlamaTritonModelInstance.h" #include "src/fastertransformer/triton_backend/multi_gpu_gpt/ParallelGptTritonModel.h" #include "src/fastertransformer/triton_backend/multi_gpu_gpt/ParallelGptTritonModelInstance.h" #include "src/fastertransformer/triton_backend/t5/T5TritonModel.h" @@ -345,6 +347,14 @@ std::shared_ptr ModelState::ModelFactory( ft_model = std::make_shared>( tp, pp, custom_ar, model_dir, is_sparse, remove_padding); #endif + } else if (model_type == "Llama") { + const int is_sparse = param_get_bool(param,"is_sparse", false); + const int remove_padding = param_get_bool(param,"is_remove_padding", false); + + if (data_type == "fp16") { + ft_model = std::make_shared>(tp, pp, custom_ar, model_dir); + } else if (data_type == "fp32") { + ft_model = std::make_shared>(tp, pp, custom_ar, model_dir); } } else { THROW_IF_BACKEND_MODEL_ERROR(TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNSUPPORTED, From 05303f97aee9d32362edd33aec4695bb92b2f7fd Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 10 Aug 2023 16:43:04 -0700 Subject: [PATCH 03/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e6be64..50bd389 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG zhwang/llama + GIT_TAG 87609f8cb0ae3ac20f5d749c14e9106df21fbe9b GIT_SHALLOW ON ) endif() From 8925434e5821bc37668829df287edff0b0374886 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 10 Aug 2023 21:15:33 -0700 Subject: [PATCH 04/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 50bd389..25f149e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 87609f8cb0ae3ac20f5d749c14e9106df21fbe9b + GIT_TAG dbf67f7990c77377724362d2e6c361d5c64aed41 GIT_SHALLOW ON ) endif() From 110490a27d3e18bcf42e8cef881d100643d22a9a Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 10 Aug 2023 22:13:32 -0700 Subject: [PATCH 05/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 25f149e..175bd19 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG dbf67f7990c77377724362d2e6c361d5c64aed41 + GIT_TAG 7f40dfb9c577079fbc2cbf079329a29ef6a7580e GIT_SHALLOW ON ) endif() From c8f29763f966bb04a851bfefc8c6df3981591599 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Thu, 10 Aug 2023 22:59:53 -0700 Subject: [PATCH 06/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 175bd19..7ea0600 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 7f40dfb9c577079fbc2cbf079329a29ef6a7580e + GIT_TAG 791ae3965a23d3f72ca0e5b09fa414fc3e231675 GIT_SHALLOW ON ) endif() From 255ca7976421c88cae979b9faac6156adedbfb0e Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Fri, 11 Aug 2023 12:44:21 -0700 Subject: [PATCH 07/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ea0600..f98aa59 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 791ae3965a23d3f72ca0e5b09fa414fc3e231675 + GIT_TAG 35a83c36036434d821d3ac123974c292654ab941 GIT_SHALLOW ON ) endif() From c804c16bfd2387901d0dfea47dfa99e170c1aa0f Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Fri, 11 Aug 2023 14:37:54 -0700 Subject: [PATCH 08/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f98aa59..0c37fe0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 35a83c36036434d821d3ac123974c292654ab941 + GIT_TAG a6291dcdd94f66aa05bc232a2019074fd2804760 GIT_SHALLOW ON ) endif() From fc215576f72a000ae090add107379c63b5233ab4 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Fri, 11 Aug 2023 15:57:03 -0700 Subject: [PATCH 09/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c37fe0..da5dae9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG a6291dcdd94f66aa05bc232a2019074fd2804760 + GIT_TAG 28f4b386be0aed24da7c03807c011ac883393f1e GIT_SHALLOW ON ) endif() From c74dfca74ac3c61f092afbb6e49923b58b463312 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Fri, 11 Aug 2023 17:21:07 -0700 Subject: [PATCH 10/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index da5dae9..45c030b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 28f4b386be0aed24da7c03807c011ac883393f1e + GIT_TAG 78d55bb59abac1c6a179212fed3b33463f073dc9 GIT_SHALLOW ON ) endif() From 2a035952a98d3e13c2b5a56033c1ec65b8a0c8dd Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Fri, 11 Aug 2023 18:42:10 -0700 Subject: [PATCH 11/47] commit --- src/libfastertransformer.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libfastertransformer.cc b/src/libfastertransformer.cc index aff267b..4b7074f 100644 --- a/src/libfastertransformer.cc +++ b/src/libfastertransformer.cc @@ -347,7 +347,8 @@ std::shared_ptr ModelState::ModelFactory( ft_model = std::make_shared>( tp, pp, custom_ar, model_dir, is_sparse, remove_padding); #endif - } else if (model_type == "Llama") { + } + } else if (model_type == "Llama") { const int is_sparse = param_get_bool(param,"is_sparse", false); const int remove_padding = param_get_bool(param,"is_remove_padding", false); From 91d18a4cbe8f3722d2e07697aa499efa63a89624 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sun, 13 Aug 2023 13:55:43 -0700 Subject: [PATCH 12/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 45c030b..d1e710f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 78d55bb59abac1c6a179212fed3b33463f073dc9 + GIT_TAG 184d3c7bf7fd83b1fa41c73cf3c21879c5eaf562 GIT_SHALLOW ON ) endif() From 4c6cd41c75edc385e42e994802ddc28edc27b381 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sun, 13 Aug 2023 19:39:06 -0700 Subject: [PATCH 13/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d1e710f..bc6c39d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 184d3c7bf7fd83b1fa41c73cf3c21879c5eaf562 + GIT_TAG 776b431152089ead0e256552acd7036087e9dc3b GIT_SHALLOW ON ) endif() From d42ee281e19a3cc50699cb1244b33e16511f7ebf Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sun, 13 Aug 2023 19:59:08 -0700 Subject: [PATCH 14/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bc6c39d..bb4319a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 776b431152089ead0e256552acd7036087e9dc3b + GIT_TAG 8633a0945f916445a5aa75cbce3e25af8b50b6ef GIT_SHALLOW ON ) endif() From f8005eb34f79331088cc4f4225b9a5112c634d96 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sun, 13 Aug 2023 20:09:03 -0700 Subject: [PATCH 15/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bb4319a..59ee307 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 8633a0945f916445a5aa75cbce3e25af8b50b6ef + GIT_TAG f4059382af2380ea88308d310349d2b0db6bc9df GIT_SHALLOW ON ) endif() From 1bc93f0bc255f44913e83736bccf39c4047b0b50 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sun, 13 Aug 2023 20:21:39 -0700 Subject: [PATCH 16/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59ee307..b05028d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG f4059382af2380ea88308d310349d2b0db6bc9df + GIT_TAG 3b80acf062aa0fb4d3a41ff69676031fb6ec3614 GIT_SHALLOW ON ) endif() From 0208d271a99691d9e6bca932366537c4d3813278 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sun, 13 Aug 2023 21:15:45 -0700 Subject: [PATCH 17/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b05028d..c11ceda 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 3b80acf062aa0fb4d3a41ff69676031fb6ec3614 + GIT_TAG a00767ff07a849118f4924e99878237c678489af GIT_SHALLOW ON ) endif() From c9c9cee84eb4016ca7f84ab9db502df4b0db9f5f Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sun, 13 Aug 2023 22:48:49 -0700 Subject: [PATCH 18/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c11ceda..7a7c3eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG a00767ff07a849118f4924e99878237c678489af + GIT_TAG 3b5becacc15980c53aadd809ce87821e4f835110 GIT_SHALLOW ON ) endif() From 178595627d5081d93e488765cb93dd6e2cc7ea3c Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Sun, 13 Aug 2023 23:52:20 -0700 Subject: [PATCH 19/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a7c3eb..f82b7c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 3b5becacc15980c53aadd809ce87821e4f835110 + GIT_TAG 404bd9e9950fbf2532b7a5c37859a9082840094b GIT_SHALLOW ON ) endif() From d2bfc1656ae92b2f56c119f4f0ada1df8bbae13a Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 14 Aug 2023 10:34:07 -0700 Subject: [PATCH 20/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f82b7c0..d343a6e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 404bd9e9950fbf2532b7a5c37859a9082840094b + GIT_TAG ffbd8c24ddbb3de273699cae314dff95e573d0c4 GIT_SHALLOW ON ) endif() From d6ae2e2945b3932dd422d510171ffa53a9e48efb Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 14 Aug 2023 13:58:04 -0700 Subject: [PATCH 21/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d343a6e..bdfb1ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG ffbd8c24ddbb3de273699cae314dff95e573d0c4 + GIT_TAG b39129b4644c3161e6d91a7aa77e06a0087e99da GIT_SHALLOW ON ) endif() From 6e29c25eac8ea1a98181401efb57a5a8b4a760fa Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 14 Aug 2023 14:54:40 -0700 Subject: [PATCH 22/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bdfb1ee..0f584fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG b39129b4644c3161e6d91a7aa77e06a0087e99da + GIT_TAG 06b27b7cd3bfc93a2349575a6a600b25a7019f97 GIT_SHALLOW ON ) endif() From f84bb7f8ef3f8ecc4d096e867579dac395d55a98 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 14 Aug 2023 16:41:09 -0700 Subject: [PATCH 23/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f584fb..53e5881 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 06b27b7cd3bfc93a2349575a6a600b25a7019f97 + GIT_TAG e4beb126e093ce38df2339efbe2766ce8bbaec47 GIT_SHALLOW ON ) endif() From ab960ad309ae10a46416797df2f694aa212f003b Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 14 Aug 2023 18:39:37 -0700 Subject: [PATCH 24/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 53e5881..45c0c62 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG e4beb126e093ce38df2339efbe2766ce8bbaec47 + GIT_TAG ab887b70371932b3b6625d7a8192bb314d04f270 GIT_SHALLOW ON ) endif() From 714c8a60a990a630b715c37219dd0e5a8eb950dc Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 14 Aug 2023 19:52:44 -0700 Subject: [PATCH 25/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 45c0c62..c512794 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG ab887b70371932b3b6625d7a8192bb314d04f270 + GIT_TAG 50af94f206dd801964e43ef74f279ae537961836 GIT_SHALLOW ON ) endif() From 30e610a4b5135d176e8257858e2c7be9df034fab Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 14 Aug 2023 20:08:03 -0700 Subject: [PATCH 26/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c512794..5aa6f71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 50af94f206dd801964e43ef74f279ae537961836 + GIT_TAG a734b2704d4fb55695e1994a2403a0d11c5e0581 GIT_SHALLOW ON ) endif() From 66b1b5196d3f39bcbb9788f811345c25b8631751 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 14 Aug 2023 21:01:48 -0700 Subject: [PATCH 27/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5aa6f71..5349f28 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG a734b2704d4fb55695e1994a2403a0d11c5e0581 + GIT_TAG 0da489eb98eb63e8f86d3658b672a4be79ec9c75 GIT_SHALLOW ON ) endif() From e568f163c23d9c305746c9d33458b3994a69abeb Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 14 Aug 2023 21:48:18 -0700 Subject: [PATCH 28/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5349f28..5b1cb70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 0da489eb98eb63e8f86d3658b672a4be79ec9c75 + GIT_TAG 35ef77202a868f1a9bc9b4acc25c6bd1aeef5fe7 GIT_SHALLOW ON ) endif() From 02702e4d0afc368a6fe4885997134740fdf9c971 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 14 Aug 2023 22:47:40 -0700 Subject: [PATCH 29/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b1cb70..c278341 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 35ef77202a868f1a9bc9b4acc25c6bd1aeef5fe7 + GIT_TAG 98ac11acdf320e452018008d6d146b86b5bfe976 GIT_SHALLOW ON ) endif() From f3f92158644ed7139fd2229e46774a603bbc9ef6 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Mon, 14 Aug 2023 23:39:13 -0700 Subject: [PATCH 30/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c278341..2b9858a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 98ac11acdf320e452018008d6d146b86b5bfe976 + GIT_TAG 8cef8d73b0bed1517a4787ceb3bc75dab0b0e763 GIT_SHALLOW ON ) endif() From 7c5f2048fe5bca7ee5d1378987e393311408ad49 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 15 Aug 2023 00:54:36 -0700 Subject: [PATCH 31/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b9858a..ab27cdc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 8cef8d73b0bed1517a4787ceb3bc75dab0b0e763 + GIT_TAG 65606d3b4d1a60ee9354365f1747672310e6b2a3 GIT_SHALLOW ON ) endif() From e0a69a5b5816986f5b847d1c768c4d009b1ad6ce Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 15 Aug 2023 11:08:14 -0700 Subject: [PATCH 32/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ab27cdc..3693020 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 65606d3b4d1a60ee9354365f1747672310e6b2a3 + GIT_TAG 566067dc6e0023c35cdf86f6274c7251eb2100a6 GIT_SHALLOW ON ) endif() From 4882622839b408e7d49a951ab303da319970affc Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 15 Aug 2023 12:29:24 -0700 Subject: [PATCH 33/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3693020..9bfa053 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 566067dc6e0023c35cdf86f6274c7251eb2100a6 + GIT_TAG 722f6a5675cce71d6ab5e559e3dd63297c799980 GIT_SHALLOW ON ) endif() From 11ea0510f27978a9562bc29318f9fe43efdb9d1f Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 15 Aug 2023 13:15:10 -0700 Subject: [PATCH 34/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9bfa053..f5d4187 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 722f6a5675cce71d6ab5e559e3dd63297c799980 + GIT_TAG cc181921b9919510c0d75ce1abe10ec8ab1a9acf GIT_SHALLOW ON ) endif() From 0a011345c747f2b92a3756517a428d64aaacac36 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 15 Aug 2023 14:59:51 -0700 Subject: [PATCH 35/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f5d4187..6841956 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG cc181921b9919510c0d75ce1abe10ec8ab1a9acf + GIT_TAG bacdc3dc1b2fe7fa69d79037a633933f013c9701 GIT_SHALLOW ON ) endif() From b63fb3a035838e66468085bdc60aa0fd4527f7ba Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 15 Aug 2023 15:04:40 -0700 Subject: [PATCH 36/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6841956..fce1beb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG bacdc3dc1b2fe7fa69d79037a633933f013c9701 + GIT_TAG 1cd3b513c9dcef8c57eece733326521a8e6ab0c8 GIT_SHALLOW ON ) endif() From 46dabdf808bc56ef2472a29dbbc7218403f969c6 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 15 Aug 2023 21:16:18 -0700 Subject: [PATCH 37/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fce1beb..aae6a35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 1cd3b513c9dcef8c57eece733326521a8e6ab0c8 + GIT_TAG faee7c8c82b9a2c06930674476d66b5def9c1057 GIT_SHALLOW ON ) endif() From c3d1a9c533fdca8f93a363a358f3c2c02c89429b Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 15 Aug 2023 21:20:55 -0700 Subject: [PATCH 38/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index aae6a35..027415f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG faee7c8c82b9a2c06930674476d66b5def9c1057 + GIT_TAG 0a7bf2d9913bbef4bd00201ff04d6c1e776499f1 GIT_SHALLOW ON ) endif() From bc97cad00edb4c27dd1d48a9587eb3fb1a9ed576 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 15 Aug 2023 21:49:37 -0700 Subject: [PATCH 39/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 027415f..51cd5a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 0a7bf2d9913bbef4bd00201ff04d6c1e776499f1 + GIT_TAG 1ff58c678a293d7a57a3d23b9da5f850d4d9bfbf GIT_SHALLOW ON ) endif() From 3b21384222eeb359d1e25b874638c26f179b683b Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 15 Aug 2023 22:07:02 -0700 Subject: [PATCH 40/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 51cd5a3..4a4f968 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 1ff58c678a293d7a57a3d23b9da5f850d4d9bfbf + GIT_TAG e127037694abf3b938c18cdd355bda0f94f777d7 GIT_SHALLOW ON ) endif() From 2ddb1eddcaf886b1d0013b139b2634b7021c3bf7 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 15 Aug 2023 23:29:16 -0700 Subject: [PATCH 41/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a4f968..b654b9e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG e127037694abf3b938c18cdd355bda0f94f777d7 + GIT_TAG b7b186f1be69f93a112968153dfe4d35757e3108 GIT_SHALLOW ON ) endif() From c1212719762ffd48f2efd090c7657fb40d152d11 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 16 Aug 2023 09:03:27 -0700 Subject: [PATCH 42/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b654b9e..a362ef5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG b7b186f1be69f93a112968153dfe4d35757e3108 + GIT_TAG 6254f0c6bb28b601a41caf2315f71340a6425bf6 GIT_SHALLOW ON ) endif() From faf12ff3834367fb3fd31af0162cee3a2ecd2317 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 16 Aug 2023 17:36:44 -0700 Subject: [PATCH 43/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a362ef5..39b71f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 6254f0c6bb28b601a41caf2315f71340a6425bf6 + GIT_TAG 027263550fdc67bd2a9c1718b83053af72b1cdd9 GIT_SHALLOW ON ) endif() From a57e70a0ef8307395ef558c2ee86f5cae3175708 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 16 Aug 2023 18:40:08 -0700 Subject: [PATCH 44/47] commit --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 39b71f5..1572362 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ else() FetchContent_Declare( repo-ft GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG 027263550fdc67bd2a9c1718b83053af72b1cdd9 + GIT_TAG c14c7e62427bf4ecba80816f08d2b41f6b9e7052 GIT_SHALLOW ON ) endif() From 1237419a3b326678d8dea718a2dfa37287378bd2 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 16 Aug 2023 19:43:14 -0700 Subject: [PATCH 45/47] commit --- src/libfastertransformer.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libfastertransformer.cc b/src/libfastertransformer.cc index 4b7074f..7574f22 100644 --- a/src/libfastertransformer.cc +++ b/src/libfastertransformer.cc @@ -65,6 +65,7 @@ #include "src/fastertransformer/triton_backend/transformer_triton_backend.hpp" #include "src/fastertransformer/utils/Tensor.h" #include "src/fastertransformer/utils/cuda_bf16_wrapper.h" +#include "src/fastertransformer/utils/instance_comm.h" #include "src/fastertransformer/utils/mpi_utils.h" #include "src/fastertransformer/utils/nccl_utils.h" @@ -724,6 +725,8 @@ class ModelInstanceState : public BackendModelInstance { std::vector> ft_model_instance_; + std::unique_ptr instance_comm_; + // inter-node broadcast buffer std::vector bcast_buffers; @@ -849,6 +852,8 @@ ModelInstanceState::ModelInstanceState( t.join(); } + instance_comm_ = shared_ft_model->createInstanceComm(tp_pp_size_); + LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Model instance is created on GPU ") + model_instance_gpu_ids).c_str()); @@ -1446,7 +1451,7 @@ ModelInstanceState::Execute( .c_str()); threads.push_back(std::thread( ThreadForward, &ft_model_instance_[instance_local_id], &input_tensors, - &output_tensors_list[instance_local_id], &exception_ptr[instance_local_id], gid, + &output_tensors_list[instance_local_id], instance_comm_.get(), &exception_ptr[instance_local_id], gid, is_decoupled_ && gid == model_instance_device_id_start_, context)); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, From e78f75be588fb6d7e7d3015304bfe81af9a1bca6 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Wed, 16 Aug 2023 20:28:04 -0700 Subject: [PATCH 46/47] commit --- src/libfastertransformer.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libfastertransformer.cc b/src/libfastertransformer.cc index 7574f22..520e1da 100644 --- a/src/libfastertransformer.cc +++ b/src/libfastertransformer.cc @@ -1279,6 +1279,7 @@ ThreadForward( std::unique_ptr* ft_model_instance, std::shared_ptr>* input_tensors, std::shared_ptr>* output_tensors, + ft::AbstractInstanceComm* instance_comm, std::exception_ptr* exception_ptr, const int device_id, const int use_stream_cb, stream_callback_ctx_t* context) @@ -1292,7 +1293,7 @@ ThreadForward( if (use_stream_cb) { (*ft_model_instance)->registerCallback(streaming_callback, (void*)context); } - *output_tensors = (*ft_model_instance)->forward(*input_tensors); + *output_tensors = (*ft_model_instance)->forward(*input_tensors, instance_comm); if (use_stream_cb) { (*ft_model_instance)->unRegisterCallback(); } From 379221c1ac7b8d5b7c7de6d2b783c232ab4f9206 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Fri, 18 Aug 2023 23:04:59 -0700 Subject: [PATCH 47/47] commit --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1572362..833e298 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,8 +110,8 @@ if (EXISTS ${FT_DIR}) else() FetchContent_Declare( repo-ft - GIT_REPOSITORY https://github.com/neevaco/FasterTransformer.git - GIT_TAG c14c7e62427bf4ecba80816f08d2b41f6b9e7052 + GIT_REPOSITORY https://github.com/sfc-gh-zhwang/FasterTransformer + GIT_TAG e770ddf2bc66217034b6e9e3b0c3256ebf1c1b40 GIT_SHALLOW ON ) endif()