From d0d1c6ec86b9a192a4e9887b4fd7c3efcc8f040a Mon Sep 17 00:00:00 2001 From: Sahar Fatima Date: Wed, 26 Apr 2023 12:13:16 +0530 Subject: [PATCH 1/5] Ontrib op changes --- onnxruntime/contrib_ops/openvino/inverse.cc | 93 +++++++++++++++++++ .../openvino/openvino_contrib_kernels.cc | 40 ++++++++ .../openvino/openvino_contrib_kernels.h | 15 +++ .../openvino/openvino_execution_provider.cc | 24 +++++ .../openvino/openvino_execution_provider.h | 2 + .../openvino/openvino_provider_factory.cc | 6 ++ .../openvino/ov_versions/data_ops.cc | 5 +- 7 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 onnxruntime/contrib_ops/openvino/inverse.cc create mode 100644 onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc create mode 100644 onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h diff --git a/onnxruntime/contrib_ops/openvino/inverse.cc b/onnxruntime/contrib_ops/openvino/inverse.cc new file mode 100644 index 0000000000000..7037f38a66a06 --- /dev/null +++ b/onnxruntime/contrib_ops/openvino/inverse.cc @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/common/narrow.h" +#include "core/framework/op_kernel.h" +#include "core/platform/threadpool.h" +#include "core/util/math_cpuonly.h" +#include "Eigen/src/Core/Map.h" +#include "Eigen/LU" +#include + +namespace onnxruntime { +namespace contrib { +namespace openvino { +class Inverse final : public OpKernel { + public: + explicit Inverse(const OpKernelInfo& info) : OpKernel(info) {} + Status Compute(OpKernelContext* ctx) const override; + + private: + template + struct ComputeImpl; +}; + +ONNX_OPERATOR_KERNEL_EX( + Inverse, + kMSDomain, + 1, + kOpenVINOExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", BuildKernelDefConstraints()), + Inverse); + +template +using MatrixT = Eigen::Matrix; + +template +struct Inverse::ComputeImpl { + void operator()(const Tensor* input, Tensor* output, + int64_t batch_num, int64_t rows, int64_t cols) const { + auto batch_offset = batch_num * rows * cols; + const auto* input_data = input->Data() + batch_offset; + auto* output_data = output->MutableData() + batch_offset; + + Eigen::Map> input_matrix(input_data, narrow(rows), narrow(cols)); + Eigen::Map> output_matrix(output_data, narrow(rows), narrow(cols)); + output_matrix = input_matrix.inverse(); + } +}; + +template <> +struct Inverse::ComputeImpl { + void operator()(const Tensor* input, Tensor* output, + int64_t batch_num, int64_t rows, int64_t cols) const { + auto batch_offset = batch_num * rows * cols; + // Direct cast to half as it just as MLFloat16 containes only uint16_t + const auto* input_data = reinterpret_cast(input->Data() + batch_offset); + auto* output_data = reinterpret_cast(output->MutableData() + batch_offset); + + Eigen::Map> input_matrix(input_data, narrow(rows), narrow(cols)); + Eigen::Map> output_matrix(output_data, narrow(rows), narrow(cols)); + output_matrix = input_matrix.inverse(); + } +}; + +Status Inverse::Compute(OpKernelContext* ctx) const { + const auto& input = ctx->Input(0); + const auto elem_type = input->GetElementType(); + const auto& input_shape = input->Shape(); + const auto num_dim = input_shape.NumDimensions(); + auto* output = ctx->Output(0, input_shape); + + int64_t num_batches = 1; + const int64_t rows = input_shape.GetDims()[num_dim - 2]; + const int64_t cols = input_shape.GetDims()[num_dim - 1]; + if (num_dim > 2) { + num_batches = input_shape.SizeToDimension(num_dim - 2); + } + + std::function fn = [elem_type, input, output, rows, cols](ptrdiff_t batch_num) { + utils::MLTypeCallDispatcher t_disp(elem_type); + t_disp.Invoke(input, output, batch_num, rows, cols); + }; + + concurrency::ThreadPool::TryBatchParallelFor(ctx->GetOperatorThreadPool(), narrow(num_batches), std::move(fn), 0); + + return Status::OK(); +} + +} // namespace openvino +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc new file mode 100644 index 0000000000000..9c1b062618e1a --- /dev/null +++ b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "contrib_ops/openvino/openvino_contrib_kernels.h" + + +namespace onnxruntime { +namespace contrib { +namespace openvino { + +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kOpenVINOExecutionProvider, kMSDomain, 1, float, Inverse); +//class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kOpenVINOExecutionProvider, kMSDomain, 1, float, GreedySearch); + +template <> +KernelCreateInfo BuildKernelCreateInfo() { + KernelCreateInfo info; + return info; +} + +Status RegisterOpenVINOContribKernels(KernelRegistry& kernel_registry) { + static const BuildKernelCreateInfoFn function_table[] = { + BuildKernelCreateInfo, // default entry to avoid the list become empty after ops-reducing + // add more kernels here + BuildKernelCreateInfo, + // BuildKernelCreateInfo, + }; + + for (auto& function_table_entry : function_table) { + KernelCreateInfo info = function_table_entry(); + if (info.kernel_def != nullptr) { // filter disabled entries where type is void + ORT_RETURN_IF_ERROR(kernel_registry.Register(std::move(info))); + } + } + + return Status::OK(); +} + +} // namespace openvino +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h new file mode 100644 index 0000000000000..97582d4b63687 --- /dev/null +++ b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/op_kernel.h" +#include "core/framework/kernel_registry.h" + +namespace onnxruntime { +namespace contrib { +namespace openvino { +Status RegisterOpenVINOContribKernels(KernelRegistry& kernel_registry); +} // namespace OpenVINO +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 05eec6b1013bd..2bf24236722c6 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -6,6 +6,7 @@ #include "contexts.h" #include "backend_manager.h" #include "ov_versions/capabilities.h" +#include "contrib_ops/openvino/openvino_contrib_kernels.h" #define MEMCPY_S(dest, src, destsz, srcsz) memcpy(dest, src, std::min(destsz, srcsz)) @@ -188,4 +189,27 @@ common::Status OpenVINOExecutionProvider::Compile( return Status::OK(); } +static std::shared_ptr s_kernel_registry; + +void InitializeRegistry() { + s_kernel_registry = KernelRegistry::Create(); + ORT_THROW_IF_ERROR(openvino::RegisterOpenVINOKernels(*s_kernel_registry)); +} + +void DeleteRegistry() { + s_kernel_registry.reset(); +} + +std::shared_ptr OpenVINOExecutionProvider::GetKernelRegistry() const { + return s_kernel_registry; +} + } // namespace onnxruntime + +// namespace openvino +namespace openvino { +static Status RegisterOpenVINOKernels(KernelRegistry& kernel_registry) { + ORT_RETURN_IF_ERROR(::onnxruntime::contrib::openvino::RegisterOpenVINOContribKernels(kernel_registry)); +} +} + diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 5eae9c78c91d4..c9a838cd3a1ab 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -176,6 +176,8 @@ class OpenVINOExecutionProvider : public IExecutionProvider { const void* GetExecutionHandle() const noexcept override { return nullptr; } + + std::shared_ptr GetKernelRegistry() const override; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index d118b37f8ab6f..7c69bd6a2214a 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -7,6 +7,10 @@ #include "openvino_provider_factory_creator.h" namespace onnxruntime { + +void InitializeRegistry(); +void DeleteRegistry(); + struct OpenVINOProviderFactory : IExecutionProviderFactory { OpenVINOProviderFactory(const char* device_type, bool enable_vpu_fast_compile, const char* device_id, size_t num_of_threads, @@ -72,10 +76,12 @@ struct OpenVINO_Provider : Provider { } void Initialize() override { + InitializeRegistry(); } void Shutdown() override { openvino_ep::BackendManager::ReleaseGlobalContext(); + DeleteRegistry(); } } g_provider; diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc index 88db95c1fbe92..6b793fe1709f0 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc @@ -57,13 +57,14 @@ std::set ops_supported_only_in_model = { "Slice", "Split", "Tile", - "TopK"}; + "TopK", + "BeamSearch"}; // Ops which are supported as functions (as composite ops) std::set ops_supported_as_function = { "LessOrEqual", "GreaterOrEqual", -}; + "BeamSearch"}; std::vector supported_op_mode = { {"Abs", V_2020_4, {"CPU", "GPU"}}, From e0aa86848c8fba6cf873ca5453dd6408700d5bcb Mon Sep 17 00:00:00 2001 From: Sahar Fatima Date: Thu, 27 Apr 2023 09:16:36 +0530 Subject: [PATCH 2/5] OnnxRuntime --- cmake/onnxruntime_providers.cmake | 11 +++ .../cpu/transformers/subgraph_base.cc | 3 +- .../cuda/bert/add_bias_transpose.cu | 3 +- .../transformers/generation_device_helper.h | 2 +- .../contrib_ops/openvino/beam_search.cc | 36 +++++++ .../contrib_ops/openvino/beam_search.h | 26 ++++++ onnxruntime/contrib_ops/openvino/inverse.cc | 93 ------------------- .../openvino/openvino_contrib_kernels.cc | 15 ++- .../openvino/openvino_contrib_kernels.h | 5 +- .../openvino/openvino_execution_provider.cc | 31 ++++--- .../openvino/openvino_execution_provider.h | 1 - 11 files changed, 106 insertions(+), 120 deletions(-) create mode 100644 onnxruntime/contrib_ops/openvino/beam_search.cc create mode 100644 onnxruntime/contrib_ops/openvino/beam_search.h delete mode 100644 onnxruntime/contrib_ops/openvino/inverse.cc diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index caae2aacfd582..a2bbf6de7fd3e 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -15,8 +15,10 @@ function(substitute_op_reduction_srcs all_srcs) set(original_srcs "${ONNXRUNTIME_ROOT}/contrib_ops/cpu/cpu_contrib_kernels.cc" "${ONNXRUNTIME_ROOT}/contrib_ops/cuda/cuda_contrib_kernels.cc" + "${ONNXRUNTIME_ROOT}/contrib_ops/openvino/openvino_contrib_kernels.cc" "${ONNXRUNTIME_ROOT}/core/providers/cpu/cpu_execution_provider.cc" "${ONNXRUNTIME_ROOT}/core/providers/cuda/cuda_execution_provider.cc" + "${ONNXRUNTIME_ROOT}/core/providers/openvino/openvino_execution_provider.cc" "${ONNXRUNTIME_ROOT}/core/providers/op_kernel_type_control_overrides.inc" "${ORTTRAINING_SOURCE_DIR}/training_ops/cpu/cpu_training_kernels.cc" "${ORTTRAINING_SOURCE_DIR}/training_ops/cuda/cuda_training_kernels.cc" @@ -93,6 +95,11 @@ file(GLOB_RECURSE onnxruntime_rocm_contrib_ops_cu_srcs CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/contrib_ops/rocm/*.cuh" ) +file(GLOB_RECURSE onnxruntime_openvino_contrib_ops_cc_srcs CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/contrib_ops/openvino/*.h" + "${ONNXRUNTIME_ROOT}/contrib_ops/openvino/*.cc" +) + file(GLOB onnxruntime_providers_common_srcs CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/core/providers/*.h" "${ONNXRUNTIME_ROOT}/core/providers/*.cc" @@ -824,6 +831,7 @@ if (onnxruntime_USE_OPENVINO) unset(CMAKE_MAP_IMPORTED_CONFIG_RELWITHDEBINFO) endif() + if ((DEFINED ENV{OPENCL_LIBS}) AND (DEFINED ENV{OPENCL_INCS})) add_definitions(-DIO_BUFFER_ENABLED=1) list(APPEND OPENVINO_LIB_LIST $ENV{OPENCL_LIBS} ${OV_20_LIBS} ${InferenceEngine_LIBRARIES} ${NGRAPH_LIBRARIES} ngraph::onnx_importer ${PYTHON_LIBRARIES}) @@ -832,6 +840,9 @@ if (onnxruntime_USE_OPENVINO) endif() source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_openvino_cc_srcs}) + source_group(TREE ${ONNXRUNTIME_ROOT} FILES ${onnxruntime_openvino_contrib_ops_cc_srcs}) + list(APPEND onnxruntime_providers_openvino_cc_srcs ${onnxruntime_openvino_contrib_ops_cc_srcs}) + onnxruntime_add_shared_library_module(onnxruntime_providers_openvino ${onnxruntime_providers_openvino_cc_srcs} "${ONNXRUNTIME_ROOT}/core/dll/onnxruntime.rc") onnxruntime_add_include_to_target(onnxruntime_providers_openvino onnxruntime_common onnx) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/openvino DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) diff --git a/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc b/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc index 5f0d1db87bef3..af67aeb0b0c86 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc +++ b/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc @@ -128,8 +128,9 @@ const IExecutionProvider* Subgraph::GetProvider() const { const IExecutionProvider* cpu_provider = providers.Get(onnxruntime::kCpuExecutionProvider); const IExecutionProvider* cuda_provider = providers.Get(onnxruntime::kCudaExecutionProvider); const IExecutionProvider* rocm_provider = providers.Get(onnxruntime::kRocmExecutionProvider); + const IExecutionProvider* openvino_provider = providers.Get(onnxruntime::kOpenVINOExecutionProvider); const IExecutionProvider* gpu_provider = cuda_provider ? cuda_provider : rocm_provider; - const IExecutionProvider* provider = gpu_provider ? gpu_provider : cpu_provider; + const IExecutionProvider* provider = openvino_provider ? openvino_provider : cpu_provider; return provider; } diff --git a/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.cu b/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.cu index 8782201a64e6b..5af43a77fc90b 100644 --- a/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.cu +++ b/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.cu @@ -2,7 +2,8 @@ // Licensed under the MIT License. #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/cu_inc/common.cuh" -#include "contrib_ops/cuda/bert/add_bias_transpose.h" +#include " +/cuda/bert/add_bias_transpose.h" #include "contrib_ops/cuda/bert/rotary_embedding_util.h" using namespace onnxruntime::cuda; diff --git a/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.h b/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.h index 5ed956f9a2ecd..ca030e8321d20 100644 --- a/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.h +++ b/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.h @@ -44,7 +44,7 @@ Status AddToFeeds(const IExecutionProvider* execution_provider, std::initializer_list inputs, std::vector& feeds, IAllocatorUniquePtr& buffer); - +. template void InitBeamState(transformers::IBeamSearchState* beam_state, gsl::span& sequence_lengths, diff --git a/onnxruntime/contrib_ops/openvino/beam_search.cc b/onnxruntime/contrib_ops/openvino/beam_search.cc new file mode 100644 index 0000000000000..8cdbfb3765b2c --- /dev/null +++ b/onnxruntime/contrib_ops/openvino/beam_search.cc @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/openvino/openvino_execution_provider.h" +#include "contrib_ops/openvino/beam_search.h" + +namespace onnxruntime { +namespace contrib { +namespace openvino_ep { + +ONNX_OPERATOR_KERNEL_EX( + BeamSearch, + kMSDomain, + 1, + kOpenVINOExecutionProvider, + (*KernelDefBuilder::Create()) + .TypeConstraint("T", DataTypeImpl::GetTensorType()), + transformers::BeamSearch BeamSearch); + +BeamSearch::BeamSearch(const OpKernelInfo& info) + : onnxruntime::contrib::transformers::BeamSearch(info) { + +} + +Status BeamSearch::ComputeInternal(OpKernelContext* context) const { + return onnxruntime::contrib::transformers::BeamSearch::Compute(context); +} + +Status BeamSearch::Compute(OpKernelContext* context) const { + auto s = ComputeInternal(context); + return s; +} + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/openvino/beam_search.h b/onnxruntime/contrib_ops/openvino/beam_search.h new file mode 100644 index 0000000000000..9b71273e487dc --- /dev/null +++ b/onnxruntime/contrib_ops/openvino/beam_search.h @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "contrib_ops/cpu/transformers/beam_search.h" + +namespace onnxruntime { +class SessionState; + +namespace contrib { +namespace openvino_ep { + +class BeamSearch final : public onnxruntime::contrib::transformers::BeamSearch { + public: + BeamSearch(const OpKernelInfo& info); + + Status Compute(OpKernelContext* context) const override; + + private: + Status ComputeInternal(OpKernelContext* context) const; +}; + +} // namespace cuda. +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/openvino/inverse.cc b/onnxruntime/contrib_ops/openvino/inverse.cc deleted file mode 100644 index 7037f38a66a06..0000000000000 --- a/onnxruntime/contrib_ops/openvino/inverse.cc +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/common/common.h" -#include "core/common/narrow.h" -#include "core/framework/op_kernel.h" -#include "core/platform/threadpool.h" -#include "core/util/math_cpuonly.h" -#include "Eigen/src/Core/Map.h" -#include "Eigen/LU" -#include - -namespace onnxruntime { -namespace contrib { -namespace openvino { -class Inverse final : public OpKernel { - public: - explicit Inverse(const OpKernelInfo& info) : OpKernel(info) {} - Status Compute(OpKernelContext* ctx) const override; - - private: - template - struct ComputeImpl; -}; - -ONNX_OPERATOR_KERNEL_EX( - Inverse, - kMSDomain, - 1, - kOpenVINOExecutionProvider, - KernelDefBuilder() - .TypeConstraint("T", BuildKernelDefConstraints()), - Inverse); - -template -using MatrixT = Eigen::Matrix; - -template -struct Inverse::ComputeImpl { - void operator()(const Tensor* input, Tensor* output, - int64_t batch_num, int64_t rows, int64_t cols) const { - auto batch_offset = batch_num * rows * cols; - const auto* input_data = input->Data() + batch_offset; - auto* output_data = output->MutableData() + batch_offset; - - Eigen::Map> input_matrix(input_data, narrow(rows), narrow(cols)); - Eigen::Map> output_matrix(output_data, narrow(rows), narrow(cols)); - output_matrix = input_matrix.inverse(); - } -}; - -template <> -struct Inverse::ComputeImpl { - void operator()(const Tensor* input, Tensor* output, - int64_t batch_num, int64_t rows, int64_t cols) const { - auto batch_offset = batch_num * rows * cols; - // Direct cast to half as it just as MLFloat16 containes only uint16_t - const auto* input_data = reinterpret_cast(input->Data() + batch_offset); - auto* output_data = reinterpret_cast(output->MutableData() + batch_offset); - - Eigen::Map> input_matrix(input_data, narrow(rows), narrow(cols)); - Eigen::Map> output_matrix(output_data, narrow(rows), narrow(cols)); - output_matrix = input_matrix.inverse(); - } -}; - -Status Inverse::Compute(OpKernelContext* ctx) const { - const auto& input = ctx->Input(0); - const auto elem_type = input->GetElementType(); - const auto& input_shape = input->Shape(); - const auto num_dim = input_shape.NumDimensions(); - auto* output = ctx->Output(0, input_shape); - - int64_t num_batches = 1; - const int64_t rows = input_shape.GetDims()[num_dim - 2]; - const int64_t cols = input_shape.GetDims()[num_dim - 1]; - if (num_dim > 2) { - num_batches = input_shape.SizeToDimension(num_dim - 2); - } - - std::function fn = [elem_type, input, output, rows, cols](ptrdiff_t batch_num) { - utils::MLTypeCallDispatcher t_disp(elem_type); - t_disp.Invoke(input, output, batch_num, rows, cols); - }; - - concurrency::ThreadPool::TryBatchParallelFor(ctx->GetOperatorThreadPool(), narrow(num_batches), std::move(fn), 0); - - return Status::OK(); -} - -} // namespace openvino -} // namespace contrib -} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc index 9c1b062618e1a..05854ff1c2d2c 100644 --- a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc @@ -2,27 +2,25 @@ // Licensed under the MIT License. #include "contrib_ops/openvino/openvino_contrib_kernels.h" - +#include "core/providers/shared_library/provider_api.h" namespace onnxruntime { namespace contrib { -namespace openvino { +namespace openvino_ep { -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kOpenVINOExecutionProvider, kMSDomain, 1, float, Inverse); -//class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kOpenVINOExecutionProvider, kMSDomain, 1, float, GreedySearch); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kOpenVINOExecutionProvider, kMSDomain, 1, float, BeamSearch); -template <> +/* template <> KernelCreateInfo BuildKernelCreateInfo() { KernelCreateInfo info; return info; -} +}*/ Status RegisterOpenVINOContribKernels(KernelRegistry& kernel_registry) { static const BuildKernelCreateInfoFn function_table[] = { BuildKernelCreateInfo, // default entry to avoid the list become empty after ops-reducing // add more kernels here - BuildKernelCreateInfo, - // BuildKernelCreateInfo, + BuildKernelCreateInfo, }; for (auto& function_table_entry : function_table) { @@ -38,3 +36,4 @@ Status RegisterOpenVINOContribKernels(KernelRegistry& kernel_registry) { } // namespace openvino } // namespace contrib } // namespace onnxruntime + diff --git a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h index 97582d4b63687..a17e0cbb13cfb 100644 --- a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h +++ b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h @@ -3,12 +3,9 @@ #pragma once -#include "core/framework/op_kernel.h" -#include "core/framework/kernel_registry.h" - namespace onnxruntime { namespace contrib { -namespace openvino { +namespace openvino_ep { Status RegisterOpenVINOContribKernels(KernelRegistry& kernel_registry); } // namespace OpenVINO } // namespace contrib diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 2bf24236722c6..cde2c0048c99e 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -2,6 +2,8 @@ // Licensed under the MIT License #include "core/providers/shared_library/provider_api.h" +#include "backend_manager.h" +#define ORT_API_MANUAL_INIT #include "openvino_execution_provider.h" #include "contexts.h" #include "backend_manager.h" @@ -10,6 +12,8 @@ #define MEMCPY_S(dest, src, destsz, srcsz) memcpy(dest, src, std::min(destsz, srcsz)) +using namespace onnxruntime::common; + namespace onnxruntime { OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProviderInfo& info) @@ -189,27 +193,32 @@ common::Status OpenVINOExecutionProvider::Compile( return Status::OK(); } -static std::shared_ptr s_kernel_registry; +namespace openvino_ep { +static Status RegisterOpenVINOKernels(KernelRegistry& kernel_registry) { + ORT_RETURN_IF_ERROR(::onnxruntime::contrib::openvino_ep::RegisterOpenVINOContribKernels(kernel_registry)); + return Status::OK(); +} +} // namespace openvino_ep + +static std::shared_ptr o_kernel_registry; void InitializeRegistry() { - s_kernel_registry = KernelRegistry::Create(); - ORT_THROW_IF_ERROR(openvino::RegisterOpenVINOKernels(*s_kernel_registry)); + o_kernel_registry = KernelRegistry::Create(); + ORT_THROW_IF_ERROR(openvino_ep::RegisterOpenVINOKernels(*o_kernel_registry)); } void DeleteRegistry() { - s_kernel_registry.reset(); + o_kernel_registry.reset(); } std::shared_ptr OpenVINOExecutionProvider::GetKernelRegistry() const { - return s_kernel_registry; + return o_kernel_registry; } + } // namespace onnxruntime -// namespace openvino -namespace openvino { -static Status RegisterOpenVINOKernels(KernelRegistry& kernel_registry) { - ORT_RETURN_IF_ERROR(::onnxruntime::contrib::openvino::RegisterOpenVINOContribKernels(kernel_registry)); -} -} + + + diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index c9a838cd3a1ab..e02ee3e6af9d9 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -3,7 +3,6 @@ #pragma once -#include "backend_manager.h" #include #include #include From 75353d5f0cfce77e5a074bf774dfc337219cb682 Mon Sep 17 00:00:00 2001 From: Sahar Fatima Date: Fri, 28 Apr 2023 11:59:48 +0530 Subject: [PATCH 3/5] Bug fixes --- .../onnxruntime/core/framework/op_kernel.h | 8 +++++++ .../contrib_ops/openvino/beam_search.cc | 21 +++++++++++-------- .../openvino/openvino_contrib_kernels.cc | 11 ++++++---- .../openvino/openvino_contrib_kernels.h | 2 ++ .../openvino/openvino_execution_provider.h | 6 ++++++ 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/include/onnxruntime/core/framework/op_kernel.h b/include/onnxruntime/core/framework/op_kernel.h index ed712cf00e096..baa1a04809857 100644 --- a/include/onnxruntime/core/framework/op_kernel.h +++ b/include/onnxruntime/core/framework/op_kernel.h @@ -199,6 +199,14 @@ KernelCreateInfo BuildKernelCreateInfo(); } // namespace snpe } // namespace contrib +namespace contrib { +namespace openvino_ep { +template +KernelCreateInfo BuildKernelCreateInfo(); +} // namespace openvino_ep +} // namespace contrib + + using BuildKernelCreateInfoFn = KernelCreateInfo (*)(); // Naming convention for operator kernel classes diff --git a/onnxruntime/contrib_ops/openvino/beam_search.cc b/onnxruntime/contrib_ops/openvino/beam_search.cc index 8cdbfb3765b2c..2193172f41feb 100644 --- a/onnxruntime/contrib_ops/openvino/beam_search.cc +++ b/onnxruntime/contrib_ops/openvino/beam_search.cc @@ -1,21 +1,24 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/providers/openvino/openvino_execution_provider.h" #include "contrib_ops/openvino/beam_search.h" namespace onnxruntime { namespace contrib { namespace openvino_ep { -ONNX_OPERATOR_KERNEL_EX( - BeamSearch, - kMSDomain, - 1, - kOpenVINOExecutionProvider, - (*KernelDefBuilder::Create()) - .TypeConstraint("T", DataTypeImpl::GetTensorType()), - transformers::BeamSearch BeamSearch); +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + BeamSearch, \ + kMSDomain, \ + 1, \ + T, \ + kOpenVINOExecutionProvider, \ + (*KernelDefBuilder::Create()) \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + BeamSearch); + +REGISTER_KERNEL_TYPED(float) BeamSearch::BeamSearch(const OpKernelInfo& info) : onnxruntime::contrib::transformers::BeamSearch(info) { diff --git a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc index 05854ff1c2d2c..8be116385c9fa 100644 --- a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc @@ -3,6 +3,9 @@ #include "contrib_ops/openvino/openvino_contrib_kernels.h" #include "core/providers/shared_library/provider_api.h" +#include "core/common/status.h" + +using namespace onnxruntime::common; namespace onnxruntime { namespace contrib { @@ -10,23 +13,23 @@ namespace openvino_ep { class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kOpenVINOExecutionProvider, kMSDomain, 1, float, BeamSearch); -/* template <> +template <> KernelCreateInfo BuildKernelCreateInfo() { KernelCreateInfo info; return info; -}*/ +} Status RegisterOpenVINOContribKernels(KernelRegistry& kernel_registry) { static const BuildKernelCreateInfoFn function_table[] = { BuildKernelCreateInfo, // default entry to avoid the list become empty after ops-reducing - // add more kernels here - BuildKernelCreateInfo, + BuildKernelCreateInfo }; for (auto& function_table_entry : function_table) { KernelCreateInfo info = function_table_entry(); if (info.kernel_def != nullptr) { // filter disabled entries where type is void ORT_RETURN_IF_ERROR(kernel_registry.Register(std::move(info))); + //return kernel_registry.Register(std::move(info)); } } diff --git a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h index a17e0cbb13cfb..c4897637595ef 100644 --- a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h +++ b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h @@ -2,8 +2,10 @@ // Licensed under the MIT License. #pragma once +#include "core/common/status.h" namespace onnxruntime { + class KernelRegistry; namespace contrib { namespace openvino_ep { Status RegisterOpenVINOContribKernels(KernelRegistry& kernel_registry); diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index e02ee3e6af9d9..1b2b18cc57fbb 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -6,6 +6,12 @@ #include #include #include +#include +#include +#include "core/providers/shared_library/provider_api.h" +#include "contexts.h" +#include "backend_manager.h" +#include "ov_versions/capabilities.h" namespace onnxruntime { From 79146f7f0efe9ae74675180dc5d869fb52db1f99 Mon Sep 17 00:00:00 2001 From: Sahar Fatima Date: Fri, 28 Apr 2023 22:19:11 +0530 Subject: [PATCH 4/5] Bug Fix --- cmake/onnxruntime_providers.cmake | 10 +++++----- .../contrib_ops/cpu/transformers/subgraph_base.cc | 4 ++-- onnxruntime/contrib_ops/openvino/beam_search.cc | 1 + .../contrib_ops/openvino/openvino_contrib_kernels.h | 1 - 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index a2bbf6de7fd3e..521ac7ff2b48f 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -806,12 +806,12 @@ if (onnxruntime_USE_OPENVINO) # include_directories("${CMAKE_CURRENT_BINARY_DIR}/onnx") file(GLOB_RECURSE onnxruntime_providers_openvino_cc_srcs CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/core/providers/shared_library/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/shared_library/*.cc" "${ONNXRUNTIME_ROOT}/core/providers/openvino/*.h" "${ONNXRUNTIME_ROOT}/core/providers/openvino/*.cc" "${ONNXRUNTIME_ROOT}/core/providers/openvino/*.hpp" "${ONNXRUNTIME_ROOT}/core/providers/openvino/*.cpp" - "${ONNXRUNTIME_ROOT}/core/providers/shared_library/*.h" - "${ONNXRUNTIME_ROOT}/core/providers/shared_library/*.cc" ) if (WIN32) @@ -840,11 +840,11 @@ if (onnxruntime_USE_OPENVINO) endif() source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_openvino_cc_srcs}) - source_group(TREE ${ONNXRUNTIME_ROOT} FILES ${onnxruntime_openvino_contrib_ops_cc_srcs}) - list(APPEND onnxruntime_providers_openvino_cc_srcs ${onnxruntime_openvino_contrib_ops_cc_srcs}) + source_group(TREE ${ONNXRUNTIME_ROOT} FILES ${onnxruntime_openvino_contrib_ops_cc_srcs}) + list(APPEND onnxruntime_providers_openvino_cc_srcs ${onnxruntime_openvino_contrib_ops_cc_srcs}) onnxruntime_add_shared_library_module(onnxruntime_providers_openvino ${onnxruntime_providers_openvino_cc_srcs} "${ONNXRUNTIME_ROOT}/core/dll/onnxruntime.rc") - onnxruntime_add_include_to_target(onnxruntime_providers_openvino onnxruntime_common onnx) + onnxruntime_add_include_to_target(onnxruntime_providers_openvino onnxruntime_common onnx_framework onnx onnx_proto) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/openvino DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers_openvino PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(onnxruntime_providers_openvino PROPERTIES FOLDER "ONNXRuntime") diff --git a/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc b/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc index af67aeb0b0c86..8c969d97938b8 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc +++ b/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc @@ -128,9 +128,9 @@ const IExecutionProvider* Subgraph::GetProvider() const { const IExecutionProvider* cpu_provider = providers.Get(onnxruntime::kCpuExecutionProvider); const IExecutionProvider* cuda_provider = providers.Get(onnxruntime::kCudaExecutionProvider); const IExecutionProvider* rocm_provider = providers.Get(onnxruntime::kRocmExecutionProvider); - const IExecutionProvider* openvino_provider = providers.Get(onnxruntime::kOpenVINOExecutionProvider); + //const IExecutionProvider* openvino_provider = providers.Get(onnxruntime::kOpenVINOExecutionProvider); const IExecutionProvider* gpu_provider = cuda_provider ? cuda_provider : rocm_provider; - const IExecutionProvider* provider = openvino_provider ? openvino_provider : cpu_provider; + const IExecutionProvider* provider = gpu_provider ? gpu_provider : cpu_provider; return provider; } diff --git a/onnxruntime/contrib_ops/openvino/beam_search.cc b/onnxruntime/contrib_ops/openvino/beam_search.cc index 2193172f41feb..168a4a9da58ba 100644 --- a/onnxruntime/contrib_ops/openvino/beam_search.cc +++ b/onnxruntime/contrib_ops/openvino/beam_search.cc @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "core/providers/shared_library/provider_api.h" #include "contrib_ops/openvino/beam_search.h" namespace onnxruntime { diff --git a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h index c4897637595ef..22ef036318ffa 100644 --- a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h +++ b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h @@ -5,7 +5,6 @@ #include "core/common/status.h" namespace onnxruntime { - class KernelRegistry; namespace contrib { namespace openvino_ep { Status RegisterOpenVINOContribKernels(KernelRegistry& kernel_registry); From d882e54d139aa9c1dbc7d71cbc6c24ed389765a9 Mon Sep 17 00:00:00 2001 From: Sahar Fatima Date: Tue, 2 May 2023 16:22:37 +0530 Subject: [PATCH 5/5] Compilation fix --- .../cpu/transformers/beam_search.h | 2 ++ .../contrib_ops/openvino/beam_search.cc | 1 - .../contrib_ops/openvino/beam_search.h | 5 ++++- .../openvino/openvino_contrib_kernels.cc | 2 -- .../openvino/openvino_contrib_kernels.h | 1 + .../core/providers/cpu/cpu_provider_shared.cc | 19 ++++++++++++++++++- .../core/providers/cpu/cpu_provider_shared.h | 13 ++++++++++++- .../provider_bridge_provider.cc | 15 +++++++++++++++ 8 files changed, 52 insertions(+), 6 deletions(-) diff --git a/onnxruntime/contrib_ops/cpu/transformers/beam_search.h b/onnxruntime/contrib_ops/cpu/transformers/beam_search.h index 63b0418a12d59..e2e4b10d4a90f 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/beam_search.h +++ b/onnxruntime/contrib_ops/cpu/transformers/beam_search.h @@ -4,8 +4,10 @@ #pragma once #include +#ifndef SHARED_PROVIDER #include "core/common/common.h" #include "core/framework/op_kernel.h" +#endif #include "core/providers/cpu/controlflow/utils.h" #include "contrib_ops/cpu/transformers/beam_search_parameters.h" #include "contrib_ops/cpu/transformers/subgraph_gpt.h" diff --git a/onnxruntime/contrib_ops/openvino/beam_search.cc b/onnxruntime/contrib_ops/openvino/beam_search.cc index 168a4a9da58ba..2193172f41feb 100644 --- a/onnxruntime/contrib_ops/openvino/beam_search.cc +++ b/onnxruntime/contrib_ops/openvino/beam_search.cc @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/providers/shared_library/provider_api.h" #include "contrib_ops/openvino/beam_search.h" namespace onnxruntime { diff --git a/onnxruntime/contrib_ops/openvino/beam_search.h b/onnxruntime/contrib_ops/openvino/beam_search.h index 9b71273e487dc..c4d94a6c31327 100644 --- a/onnxruntime/contrib_ops/openvino/beam_search.h +++ b/onnxruntime/contrib_ops/openvino/beam_search.h @@ -3,12 +3,15 @@ #pragma once +#include "core/providers/shared_library/provider_api.h" #include "contrib_ops/cpu/transformers/beam_search.h" namespace onnxruntime { + class SessionState; namespace contrib { + namespace openvino_ep { class BeamSearch final : public onnxruntime::contrib::transformers::BeamSearch { @@ -21,6 +24,6 @@ class BeamSearch final : public onnxruntime::contrib::transformers::BeamSearch { Status ComputeInternal(OpKernelContext* context) const; }; -} // namespace cuda. +} // namespace openvino } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc index 8be116385c9fa..44a9d6f70d610 100644 --- a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.cc @@ -2,8 +2,6 @@ // Licensed under the MIT License. #include "contrib_ops/openvino/openvino_contrib_kernels.h" -#include "core/providers/shared_library/provider_api.h" -#include "core/common/status.h" using namespace onnxruntime::common; diff --git a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h index 22ef036318ffa..c06deac45514e 100644 --- a/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h +++ b/onnxruntime/contrib_ops/openvino/openvino_contrib_kernels.h @@ -3,6 +3,7 @@ #pragma once #include "core/common/status.h" +#include "core/providers/shared_library/provider_api.h" namespace onnxruntime { namespace contrib { diff --git a/onnxruntime/core/providers/cpu/cpu_provider_shared.cc b/onnxruntime/core/providers/cpu/cpu_provider_shared.cc index d3d252cf7026e..04147264a10ef 100644 --- a/onnxruntime/core/providers/cpu/cpu_provider_shared.cc +++ b/onnxruntime/core/providers/cpu/cpu_provider_shared.cc @@ -75,6 +75,23 @@ struct ProviderHostCPUImpl : ProviderHostCPU { Status NonMaxSuppressionBase__PrepareCompute(OpKernelContext* ctx, PrepareContext& pc) override { return NonMaxSuppressionBase::PrepareCompute(ctx, pc); } Status NonMaxSuppressionBase__GetThresholdsFromInputs(const PrepareContext& pc, int64_t& max_output_boxes_per_class, float& iou_threshold, float& score_threshold) override { return NonMaxSuppressionBase::GetThresholdsFromInputs(pc, max_output_boxes_per_class, iou_threshold, score_threshold); } +#if defined(USE_OPENVINO) + void BeamSearch__Init(contrib::transformers::BeamSearch* p, const OpKernelInfo& info) override { + p->contrib::transformers::BeamSearch::Init(info); + } + + Status BeamSearch__Compute(const contrib::transformers::BeamSearch* p, OpKernelContext* ctx) override { + return p->contrib::transformers::BeamSearch::Compute(ctx); + } + + Status BeamSearch__SetupSubgraphExecutionInfo(contrib::transformers::BeamSearch* p, const SessionState& session_state, + const std::string& attribute_name, + const SessionState& subgraph_session_state) override { + return p->contrib::transformers::BeamSearch::SetupSubgraphExecutionInfo(session_state, attribute_name, + subgraph_session_state); + } +#endif + #if defined(USE_CUDA) || defined(USE_ROCM) // From cpu/tensor/size.h (direct) Status Size__Compute(const Size* p, OpKernelContext* context) override { return p->Size::Compute(context); } @@ -218,7 +235,7 @@ struct ProviderHostCPUImpl : ProviderHostCPU { int sequence_length, int& past_sequence_length) override { return p->contrib::AttentionBase::GetPresent(context, past, batch_size, head_size, sequence_length, past_sequence_length); - } + }. void BeamSearch__Init(contrib::transformers::BeamSearch* p, const OpKernelInfo& info) override { p->contrib::transformers::BeamSearch::Init(info); diff --git a/onnxruntime/core/providers/cpu/cpu_provider_shared.h b/onnxruntime/core/providers/cpu/cpu_provider_shared.h index 1fafc646c8aaa..43fd5655963dd 100644 --- a/onnxruntime/core/providers/cpu/cpu_provider_shared.h +++ b/onnxruntime/core/providers/cpu/cpu_provider_shared.h @@ -30,7 +30,18 @@ struct ProviderHostCPU { virtual Status NonMaxSuppressionBase__PrepareCompute(OpKernelContext* ctx, PrepareContext& pc) = 0; virtual Status NonMaxSuppressionBase__GetThresholdsFromInputs(const PrepareContext& pc, int64_t& max_output_boxes_per_class, float& iou_threshold, float& score_threshold) = 0; -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_OPENVINO) + // BeamSearch + virtual void BeamSearch__Init(contrib::transformers::BeamSearch* p, const OpKernelInfo& info) = 0; + virtual Status BeamSearch__Compute(const contrib::transformers::BeamSearch* p, OpKernelContext* ctx) = 0; + virtual Status BeamSearch__SetupSubgraphExecutionInfo(contrib::transformers::BeamSearch* p, + const SessionState& session_state, + const std::string& attribute_name, + const SessionState& subgraph_session_state) = 0; + +#endif + +#if defined(USE_CUDA) || defined(USE_ROCM) // From cpu/tensor/size.h virtual Status Size__Compute(const Size* p, OpKernelContext* context) = 0; diff --git a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc index 2e4d5b6bfa500..aa99b0c580fe2 100644 --- a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc +++ b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc @@ -490,6 +490,21 @@ Status NonMaxSuppressionBase::GetThresholdsFromInputs(const PrepareContext& pc, Status GatherBase::PrepareForCompute(OpKernelContext* context, GatherBase::Prepare& p) const { return g_host_cpu.GatherBase__PrepareForCompute(this, context, reinterpret_cast(p)); } Status UnsqueezeBase::PrepareCompute(OpKernelContext* ctx, UnsqueezeBase::Prepare& p) const { return g_host_cpu.UnsqueezeBase__PrepareCompute(this, ctx, reinterpret_cast(p)); } +#if defined(USE_OPENVINO) +namespace contrib { +namespace transformers { +void BeamSearch::Init(const OpKernelInfo& info) { g_host_cpu.BeamSearch__Init(this, info); } + +Status BeamSearch::Compute(OpKernelContext* ctx) const { return g_host_cpu.BeamSearch__Compute(this, ctx); } + +Status BeamSearch::SetupSubgraphExecutionInfo(const SessionState& session_state, const std::string& attribute_name, + const SessionState& subgraph_session_state) { + return g_host_cpu.BeamSearch__SetupSubgraphExecutionInfo(this, session_state, attribute_name, subgraph_session_state); +} +} // namespace transformers +} // namespace contrib +#endif + #if defined(USE_CUDA) || defined(USE_ROCM) bool TileOp::IsTileMemcpy(const TensorShape& input_shape, const int64_t* repeats, size_t rank, bool& is_batched_memcpy, size_t& num_of_elements_per_batch, size_t& num_of_copies_per_batch, size_t& num_of_batch_copies) { return g_host_cpu.TileOp__IsTileMemcpy(input_shape, repeats, rank, is_batched_memcpy, num_of_elements_per_batch, num_of_copies_per_batch, num_of_batch_copies);