1From 548f98e1129d33e31314c4cfa8446584f4e48b29 Mon Sep 17 00:00:00 2001 2From: chengfeng27 <chengfeng27@huawei.com> 3Date: Thu, 1 Aug 2024 16:57:50 +0800 4Subject: [PATCH] allocator get/set crash in multi thread 5 6--- 7 .../core/mindrt/src/thread/core_affinity.cc | 2 +- 8 .../core/mindrt/src/thread/threadpool.cc | 2 +- 9 mindspore/lite/BUILD.gn | 1 + 10 mindspore/lite/src/common/context_util.cc | 3 +- 11 mindspore/lite/src/litert/c_api/context_c.cc | 11 +- 12 mindspore/lite/src/litert/c_api/tensor_c.cc | 16 ++- 13 .../delegate/nnrt/extension_options_parser.cc | 6 +- 14 .../delegate/nnrt/extension_options_parser.h | 4 +- 15 .../delegate/nnrt/hiai_foundation_wrapper.cc | 1 + 16 .../litert/delegate/nnrt/nnrt_allocator.cc | 61 ++++++++-- 17 .../src/litert/delegate/nnrt/nnrt_allocator.h | 4 + 18 .../src/litert/delegate/nnrt/nnrt_delegate.cc | 43 ++++--- 19 .../litert/delegate/nnrt/nnrt_model_kernel.cc | 112 +++++++----------- 20 .../litert/delegate/nnrt/nnrt_model_kernel.h | 15 +-- 21 .../src/litert/delegate/nnrt/nnrt_utils.cc | 110 +++++++++++++++++ 22 .../src/litert/delegate/nnrt/nnrt_utils.h | 29 +++++ 23 mindspore/lite/src/litert/infer_manager.cc | 5 +- 24 mindspore/lite/src/tensor.cc | 16 ++- 25 18 files changed, 320 insertions(+), 121 deletions(-) 26 create mode 100644 mindspore/lite/src/litert/delegate/nnrt/nnrt_utils.cc 27 create mode 100644 mindspore/lite/src/litert/delegate/nnrt/nnrt_utils.h 28 29diff --git a/mindspore/core/mindrt/src/thread/core_affinity.cc b/mindspore/core/mindrt/src/thread/core_affinity.cc 30index a3478dff..6886f743 100644 31--- a/mindspore/core/mindrt/src/thread/core_affinity.cc 32+++ b/mindspore/core/mindrt/src/thread/core_affinity.cc 33@@ -349,7 +349,7 @@ int CoreAffinity::SetAffinity(const pthread_t &thread_id, cpu_set_t *cpu_set) { 34 THREAD_INFO("thread: %d, mask: %lu", pthread_gettid_np(thread_id), cpu_set->__bits[0]); 35 int ret = sched_setaffinity(pthread_gettid_np(thread_id), sizeof(cpu_set_t), cpu_set); 36 if (ret != THREAD_OK) { 37- THREAD_ERROR("bind thread %d to cpu failed. ERROR %{public}d", pthread_gettid_np(thread_id), ret); 38+ THREAD_ERROR("bind thread %d to cpu failed. ERROR %d", pthread_gettid_np(thread_id), ret); 39 return THREAD_ERROR; 40 } 41 #endif 42diff --git a/mindspore/core/mindrt/src/thread/threadpool.cc b/mindspore/core/mindrt/src/thread/threadpool.cc 43index f166a104..e3856c26 100644 44--- a/mindspore/core/mindrt/src/thread/threadpool.cc 45+++ b/mindspore/core/mindrt/src/thread/threadpool.cc 46@@ -72,7 +72,7 @@ void Worker::SetAffinity() { 47 THREAD_INFO("thread: %d, mask: %lu", gettid(), mask_.__bits[0]); 48 int ret = sched_setaffinity(gettid(), sizeof(cpu_set_t), &mask_); 49 if (ret != THREAD_OK) { 50- THREAD_ERROR("bind thread %d to cpu failed. ERROR %{public}d", gettid(), errno); 51+ THREAD_ERROR("bind thread %d to cpu failed. ERROR %d", gettid(), errno); 52 } 53 return; 54 #else 55diff --git a/mindspore/lite/BUILD.gn b/mindspore/lite/BUILD.gn 56index 124c84c9..acee9733 100644 57--- a/mindspore/lite/BUILD.gn 58+++ b/mindspore/lite/BUILD.gn 59@@ -447,6 +447,7 @@ ohos_shared_library("mindspore_lib") { 60 "src/litert/delegate/nnrt/nnrt_model_kernel.cc", 61 "src/litert/delegate/nnrt/nnrt_allocator.cc", 62 "src/litert/delegate/nnrt/extension_options_parser.cc", 63+ "src/litert/delegate/nnrt/nnrt_utils.cc", 64 ] 65 include_dirs += [ 66 "src/delegate/nnrt/include", 67diff --git a/mindspore/lite/src/common/context_util.cc b/mindspore/lite/src/common/context_util.cc 68index 0fa4ebd0..dae3a7cc 100644 69--- a/mindspore/lite/src/common/context_util.cc 70+++ b/mindspore/lite/src/common/context_util.cc 71@@ -185,8 +185,7 @@ bool DeviceTypePriority(const InnerContext *context, int device_type1, int devic 72 if (context == nullptr) { 73 return false; 74 } 75- std::vector<DeviceContext> device_infos = context->device_list_; 76- for (DeviceContext device_info : device_infos) { 77+ for (const DeviceContext& device_info : context->device_list_) { 78 if (device_info.device_type_ == device_type1) { 79 return true; 80 } 81diff --git a/mindspore/lite/src/litert/c_api/context_c.cc b/mindspore/lite/src/litert/c_api/context_c.cc 82index 6b6a50d5..2fe3b055 100644 83--- a/mindspore/lite/src/litert/c_api/context_c.cc 84+++ b/mindspore/lite/src/litert/c_api/context_c.cc 85@@ -25,6 +25,10 @@ 86 #include "interfaces/kits/c/neural_network_runtime/neural_network_runtime.h" 87 #endif 88 89+namespace { 90+const auto kNpuNamePrefixLen = 4; 91+} 92+ 93 // ================ Context ================ 94 OH_AI_ContextHandle OH_AI_ContextCreate() { 95 auto impl = new (std::nothrow) mindspore::Context(); 96@@ -308,7 +312,6 @@ NNRTDeviceDesc *OH_AI_GetAllNNRTDeviceDescs(size_t *num) { 97 auto ret_load = mindspore::lite::LoadHiaiFLibraryFromPath(&hiai_handle_); 98 if (!ret_load || hiai_handle_ == nullptr) { 99 MS_LOG(ERROR) << "Load HiAI_Foundation so failed."; 100- return nullptr; 101 } 102 #endif 103 *num = 0; 104@@ -420,7 +423,7 @@ OH_AI_DeviceInfoHandle OH_AI_CreateNNRTDeviceInfoByType(OH_AI_NNRTDeviceType typ 105 106 OH_AI_DeviceInfoHandle handle = nullptr; 107 for (size_t i = 0; i < num; i++) { 108- if (desc[i].device_type == type) { 109+ if (desc[i].device_type == type && strncmp(desc[i].device_name, "NPU_", kNpuNamePrefixLen) == 0) { 110 handle = OH_AI_DeviceInfoCreate(OH_AI_DEVICETYPE_NNRT); 111 OH_AI_DeviceInfoSetDeviceId(handle, desc[i].device_id); 112 break; 113@@ -514,6 +517,10 @@ OH_AI_API OH_AI_Status OH_AI_DeviceInfoAddExtension(OH_AI_DeviceInfoHandle devic 114 MS_LOG(ERROR) << "device info is null"; 115 return OH_AI_STATUS_LITE_NULLPTR; 116 } 117+ if (name == nullptr || value == nullptr || value_size < 0) { 118+ MS_LOG(ERROR) << "name/value/value_size is not valid"; 119+ return OH_AI_STATUS_LITE_NULLPTR; 120+ } 121 if (OH_AI_DeviceInfoGetDeviceType(device_info) != OH_AI_DEVICETYPE_NNRT) { 122 MS_LOG(ERROR) << "Add extension to non-NNRT device is not allowable, ignored"; 123 return OH_AI_STATUS_LITE_ERROR; 124diff --git a/mindspore/lite/src/litert/c_api/tensor_c.cc b/mindspore/lite/src/litert/c_api/tensor_c.cc 125index fc3814dd..7d6c7930 100644 126--- a/mindspore/lite/src/litert/c_api/tensor_c.cc 127+++ b/mindspore/lite/src/litert/c_api/tensor_c.cc 128@@ -19,9 +19,13 @@ 129 #include "src/tensor.h" 130 #include "src/litert/cxx_api/tensor/tensor_impl.h" 131 132+// allocator_table contains mapping of raw_ptr to weak_ptr of Allocator, allocator_table_mutex is used in multi-thread 133+// scene when user build multiple models, to avoid read/write unordered_map conflicts crash. 134 static std::unordered_map<void *, std::weak_ptr<mindspore::Allocator>> allocator_table; 135+static std::mutex allocator_table_mutex; 136 137 void CleanAllocatorTable() { 138+ std::lock_guard<std::mutex> lock(allocator_table_mutex); 139 allocator_table.clear(); 140 } 141 142@@ -222,21 +226,29 @@ OH_AI_Status OH_AI_TensorSetAllocator(OH_AI_TensorHandle tensor, void *allocator 143 return OH_AI_STATUS_LITE_NULLPTR; 144 } 145 auto impl = static_cast<mindspore::MSTensor *>(tensor); 146+ std::lock_guard<std::mutex> lock(allocator_table_mutex); 147 if (allocator_table.count(allocator) == 0) { 148 MS_LOG(ERROR) << "the input allocator does not belong to framework"; 149 return OH_AI_STATUS_LITE_PARAM_INVALID; 150 } 151 std::static_pointer_cast<mindspore::LiteTensorImpl>(impl->impl())->set_own_data(true); 152- impl->SetAllocator(allocator_table[allocator].lock()); 153+ auto allocator_ptr = allocator_table[allocator].lock(); 154+ if (allocator_ptr != nullptr) { 155+ impl->SetAllocator(allocator_ptr); 156+ } else { 157+ MS_LOG(ERROR) << "get allocator shared ptr failed."; 158+ return OH_AI_STATUS_LITE_NULLPTR; 159+ } 160 return OH_AI_STATUS_SUCCESS; 161 } 162 163-void *OH_AI_TensorGetAllocator(const OH_AI_TensorHandle tensor) { 164+void *OH_AI_TensorGetAllocator(OH_AI_TensorHandle tensor) { 165 if (tensor == nullptr) { 166 MS_LOG(ERROR) << "param is nullptr."; 167 return nullptr; 168 } 169 auto impl = static_cast<mindspore::MSTensor *>(tensor); 170+ std::lock_guard<std::mutex> lock(allocator_table_mutex); 171 allocator_table[impl->allocator().get()] = impl->allocator(); 172 return impl->allocator().get(); 173 } 174diff --git a/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.cc b/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.cc 175index 98343898..e35cc2a5 100644 176--- a/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.cc 177+++ b/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.cc 178@@ -38,7 +38,7 @@ int ExtensionOptionsParser::Parse(const std::vector<Extension> &extensions, Exte 179 DoParseCachePath(extensions, ¶m->cache_path_); 180 DoParseCacheVersion(extensions, ¶m->cache_version_); 181 DoParseBondMode(extensions, ¶m->band_mode); 182- DoParseQuantConfig(extensions, ¶m->quant_config, ¶m->quant_config_size); 183+ DoParseQuantConfig(extensions, ¶m->quant_config, ¶m->quant_config_size, ¶m->is_optional_quant_setted); 184 return RET_OK; 185 } 186 187@@ -76,7 +76,8 @@ void ExtensionOptionsParser::DoParseBondMode(const std::vector<Extension> &exten 188 } 189 } 190 191-void ExtensionOptionsParser::DoParseQuantConfig(const std::vector<Extension> &extensions, void **quant_config, size_t *num) { 192+void ExtensionOptionsParser::DoParseQuantConfig(const std::vector<Extension> &extensions, 193+ void **quant_config, size_t *num, bool *quant_setted) { 194 MS_CHECK_TRUE_RET_VOID(quant_config != nullptr); 195 MS_CHECK_TRUE_RET_VOID(num != nullptr); 196 auto iter_config = std::find_if(extensions.begin(), extensions.end(), [](const Extension &extension) { 197@@ -85,6 +86,7 @@ void ExtensionOptionsParser::DoParseQuantConfig(const std::vector<Extension> &ex 198 if (iter_config != extensions.end()) { 199 *quant_config = static_cast<void *>(const_cast<uint8_t *>(iter_config->value.data())); 200 *num = iter_config->value.size(); 201+ *quant_setted = true; 202 } 203 } 204 } // mindspore::lite::nnrt 205\ No newline at end of file 206diff --git a/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.h b/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.h 207index 792805a4..f24682ce 100644 208--- a/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.h 209+++ b/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.h 210@@ -28,6 +28,7 @@ struct ExtensionOptions { 211 mindspore::lite::HiAI_BandMode band_mode{HIAI_BANDMODE_UNSET}; 212 void *quant_config; 213 size_t quant_config_size = 0; 214+ bool is_optional_quant_setted = false; 215 }; 216 217 class ExtensionOptionsParser { 218@@ -36,7 +37,8 @@ public: 219 220 private: 221 static void DoParseBondMode(const std::vector<Extension> &extensions, mindspore::lite::HiAI_BandMode *band_mode); 222- static void DoParseQuantConfig(const std::vector<Extension> &extensions, void **quant_config, size_t *num); 223+ static void DoParseQuantConfig(const std::vector<Extension> &extensions, void **quant_config, size_t *num, 224+ bool *quant_setted); 225 static void DoParseCachePath(const std::vector<Extension> &extensions, std::string *cache_path); 226 static void DoParseCacheVersion(const std::vector<Extension> &extensions, uint32_t *cache_version); 227 }; 228diff --git a/mindspore/lite/src/litert/delegate/nnrt/hiai_foundation_wrapper.cc b/mindspore/lite/src/litert/delegate/nnrt/hiai_foundation_wrapper.cc 229index e7a52827..a155b761 100644 230--- a/mindspore/lite/src/litert/delegate/nnrt/hiai_foundation_wrapper.cc 231+++ b/mindspore/lite/src/litert/delegate/nnrt/hiai_foundation_wrapper.cc 232@@ -39,6 +39,7 @@ bool LoadHiaiFLibraryFromPath(void **handle_ptr) { 233 234 *handle_ptr = dlopen(HIAI_F_LIB, RTLD_NOW | RTLD_LOCAL); 235 if (*handle_ptr == nullptr) { 236+ MS_LOG(WARNING) << "dlopen failed, error: " << dlerror(); 237 return false; 238 } 239 240diff --git a/mindspore/lite/src/litert/delegate/nnrt/nnrt_allocator.cc b/mindspore/lite/src/litert/delegate/nnrt/nnrt_allocator.cc 241index b38fff62..4910343f 100644 242--- a/mindspore/lite/src/litert/delegate/nnrt/nnrt_allocator.cc 243+++ b/mindspore/lite/src/litert/delegate/nnrt/nnrt_allocator.cc 244@@ -20,6 +20,7 @@ 245 #include <map> 246 #include <mutex> 247 #include "src/litert/delegate/nnrt/nnrt_allocator.h" 248+#include "src/litert/delegate/nnrt/nnrt_utils.h" 249 #include "src/common/log.h" 250 251 namespace mindspore { 252@@ -43,7 +44,42 @@ NNRTAllocator::~NNRTAllocator() { 253 free_list_.clear(); 254 } 255 256-void *NNRTAllocator::Malloc(size_t size) { 257+NN_TensorDesc *NNRTAllocator::CreateNNRtTensorDesc(const std::vector<int> &shape, const TypeId data_type, 258+ const Format format, const std::string &name) { 259+ auto tensor_desc = OH_NNTensorDesc_Create(); 260+ if (tensor_desc == nullptr) { 261+ MS_LOG(ERROR) << "OH_NNTensorDesc_Create failed, i = " << index_; 262+ return nullptr; 263+ } 264+ OH_NN_ReturnCode ret = OH_NNTensorDesc_SetShape(tensor_desc, shape.data(), shape.size()); 265+ if (ret != OH_NN_SUCCESS) { 266+ MS_LOG(ERROR) << "OH_NNTensorDesc_SetShape failed, i = " << index_ << ", shape: " << shape; 267+ OH_NNTensorDesc_Destroy(&tensor_desc); 268+ return nullptr; 269+ } 270+ ret = OH_NNTensorDesc_SetDataType(tensor_desc, CastToNNRtDataType(data_type)); 271+ if (ret != OH_NN_SUCCESS) { 272+ MS_LOG(ERROR) << "OH_NNTensorDesc_SetDataType failed, i = " << index_ << ", data_type: " << data_type; 273+ OH_NNTensorDesc_Destroy(&tensor_desc); 274+ return nullptr; 275+ } 276+ ret = OH_NNTensorDesc_SetFormat(tensor_desc, CastToNNRtFormat(format)); 277+ if (ret != OH_NN_SUCCESS) { 278+ MS_LOG(ERROR) << "OH_NNTensorDesc_SetFormat failed, i = " << index_ << ", format: " << format; 279+ OH_NNTensorDesc_Destroy(&tensor_desc); 280+ return nullptr; 281+ } 282+ ret = OH_NNTensorDesc_SetName(tensor_desc, name.c_str()); 283+ if (ret != OH_NN_SUCCESS) { 284+ MS_LOG(ERROR) << "OH_NNTensorDesc_SetName failed, i = " << index_ << ", name: " << name; 285+ OH_NNTensorDesc_Destroy(&tensor_desc); 286+ return nullptr; 287+ } 288+ return tensor_desc; 289+} 290+ 291+void *NNRTAllocator::MallocByDesc(size_t size, const std::vector<int> &shape, const TypeId data_type, 292+ const Format format, const std::string &name) { 293 std::lock_guard<std::mutex> locker(mutex_); 294 auto iter = free_list_.lower_bound(size); 295 if (iter != free_list_.end()) { 296@@ -60,17 +96,13 @@ void *NNRTAllocator::Malloc(size_t size) { 297 return nullptr; 298 } 299 membuf->ref_count_ = 0; 300- if (memory_category_ == NNRT_INPUT) { 301- membuf->tensor_desc_ = OH_NNExecutor_CreateInputTensorDesc(executor_, index_); 302- } else { 303- membuf->tensor_desc_ = OH_NNExecutor_CreateOutputTensorDesc(executor_, index_); 304- } 305+ membuf->tensor_desc_ = CreateNNRtTensorDesc(shape, data_type, format, name); 306 if (membuf->tensor_desc_ == nullptr) { 307- MS_LOG(ERROR) << "OH_NNExecutor_CreateInput/OutputTensorDesc failed, i = " << index_; 308+ MS_LOG(ERROR) << "create NN_TensorDesc failed."; 309 delete membuf; 310 return nullptr; 311 } 312- membuf->tensor_ = OH_NNTensor_CreateWithSize(device_id_, membuf->tensor_desc_, size); 313+ membuf->tensor_ = OH_NNTensor_Create(device_id_, membuf->tensor_desc_); 314 if (membuf->tensor_ == nullptr) { 315 MS_LOG(ERROR) << "OH_NNTensor_CreateWithSize failed, i = " << index_; 316 OH_NNTensorDesc_Destroy(&membuf->tensor_desc_); 317@@ -91,6 +123,11 @@ void *NNRTAllocator::Malloc(size_t size) { 318 return membuf->data; 319 } 320 321+void *NNRTAllocator::Malloc(size_t size) { 322+ MS_LOG(ERROR) << "NNRt Allocator is not support malloc by size."; 323+ return nullptr; 324+} 325+ 326 void NNRTAllocator::Free(void *ptr) { 327 if (ptr == nullptr) { 328 return; 329@@ -143,8 +180,8 @@ int NNRTAllocator::DecRefCount(void *ptr, int ref_count) { 330 auto iter = allocated_list_.find(ptr); 331 if (iter != allocated_list_.end()) { 332 auto membuf = iter->second; 333- auto ref = std::atomic_fetch_sub(&membuf->ref_count_, ref_count); 334- return ref; 335+ std::atomic_fetch_sub(&membuf->ref_count_, ref_count); 336+ return membuf->ref_count_; 337 } 338 return -1; 339 } 340@@ -157,8 +194,8 @@ int NNRTAllocator::IncRefCount(void *ptr, int ref_count) { 341 auto iter = allocated_list_.find(ptr); 342 if (iter != allocated_list_.end()) { 343 auto membuf = iter->second; 344- auto ref = std::atomic_fetch_add(&membuf->ref_count_, ref_count); 345- return ref; 346+ std::atomic_fetch_add(&membuf->ref_count_, ref_count); 347+ return membuf->ref_count_; 348 } 349 return -1; 350 } 351diff --git a/mindspore/lite/src/litert/delegate/nnrt/nnrt_allocator.h b/mindspore/lite/src/litert/delegate/nnrt/nnrt_allocator.h 352index 52e6def7..ef27f307 100644 353--- a/mindspore/lite/src/litert/delegate/nnrt/nnrt_allocator.h 354+++ b/mindspore/lite/src/litert/delegate/nnrt/nnrt_allocator.h 355@@ -40,6 +40,10 @@ class NNRTAllocator : public Allocator { 356 ~NNRTAllocator() override; 357 358 void *Malloc(size_t size) override; 359+ void *MallocByDesc(size_t size, const std::vector<int> &shape, const TypeId data_type, const Format format, 360+ const std::string &name); 361+ NN_TensorDesc *CreateNNRtTensorDesc(const std::vector<int> &shape, const TypeId data_type, const Format format, 362+ const std::string &name); 363 void Free(void *ptr) override; 364 int RefCount(void *ptr) override; 365 int SetRefCount(void *ptr, int ref_count) override; 366diff --git a/mindspore/lite/src/litert/delegate/nnrt/nnrt_delegate.cc b/mindspore/lite/src/litert/delegate/nnrt/nnrt_delegate.cc 367index 17abd0ed..a49e7449 100644 368--- a/mindspore/lite/src/litert/delegate/nnrt/nnrt_delegate.cc 369+++ b/mindspore/lite/src/litert/delegate/nnrt/nnrt_delegate.cc 370@@ -33,8 +33,7 @@ Status NNRTDelegate::Init() { 371 #ifdef SUPPORT_NNRT_METAGRAPH 372 auto ret = mindspore::lite::LoadHiaiFLibraryFromPath(&hiai_handle_); 373 if (!ret || hiai_handle_ == nullptr) { 374- MS_LOG(ERROR) << "Load HiAI_Foundation so failed."; 375- return kLiteError; 376+ MS_LOG(WARNING) << "Load HiAI_Foundation so failed."; 377 } 378 #endif 379 return kSuccess; 380@@ -194,7 +193,7 @@ Status NNRTDelegate::BuildOfflineModel(DelegateModel<schema::Primitive> *model) 381 } 382 OH_NNCompilation_Destroy(&nn_compilation); 383 384- auto nnrt_model_kernel = new (std::nothrow) NNRTModelKernel(nn_executor, nnrt_device_info_.device_id_, model->inputs(), model->outputs()); 385+ auto nnrt_model_kernel = new (std::nothrow) NNRTModelKernel(nn_executor, nnrt_device_info_, model->inputs(), model->outputs()); 386 if (nnrt_model_kernel == nullptr) { 387 OH_NNExecutor_Destroy(&nn_executor); 388 MS_LOG(ERROR) << "new NNRTModelKernel failed"; 389@@ -233,7 +232,7 @@ Status NNRTDelegate::CreateFullModelKernel(DelegateModel<schema::Primitive> *mod 390 } 391 OH_NNCompilation_Destroy(&nn_compilation); 392 393- auto nnrt_model_kernel = new (std::nothrow)NNRTModelKernel(nn_executor, nnrt_device_info_.device_id_, model->inputs(), model->outputs()); 394+ auto nnrt_model_kernel = new (std::nothrow) NNRTModelKernel(nn_executor, nnrt_device_info_, model->inputs(), model->outputs()); 395 if (nnrt_model_kernel == nullptr) { 396 OH_NNExecutor_Destroy(&nn_executor); 397 MS_LOG(ERROR) << "new NNRTModelKernel failed"; 398@@ -547,20 +546,30 @@ Status NNRTDelegate::InitNNCompilation(OH_NNCompilation *nn_compilation) const { 399 } 400 401 #ifdef SUPPORT_NNRT_METAGRAPH 402- ret_code = mindspore::lite::HMS_HiAIOptions_SetBandMode(nn_compilation, extension_options_.band_mode); 403- if ((ret_code != OH_NN_SUCCESS) && (ret_code != OH_NN_OPERATION_FORBIDDEN)) { 404- MS_LOG(ERROR) << "NNCompilation set BandMode failed, ret: " << ret_code; 405- return kLiteError; 406- } 407+ if (hiai_handle_ != nullptr) { 408+ if (extension_options_.band_mode != mindspore::lite::HIAI_BANDMODE_UNSET) { 409+ ret_code = mindspore::lite::HMS_HiAIOptions_SetBandMode(nn_compilation, extension_options_.band_mode); 410+ if ((ret_code != OH_NN_SUCCESS) && (ret_code != OH_NN_OPERATION_FORBIDDEN)) { 411+ MS_LOG(ERROR) << "NNCompilation set BandMode failed, ret: " << ret_code; 412+ return kLiteError; 413+ } 414+ } 415 416- if (extension_options_.quant_config != nullptr && extension_options_.quant_config_size != 0) { 417- ret_code = mindspore::lite::HMS_HiAIOptions_SetQuantConfig(nn_compilation, 418- extension_options_.quant_config, 419- extension_options_.quant_config_size); 420- if ((ret_code != OH_NN_SUCCESS) && (ret_code != OH_NN_OPERATION_FORBIDDEN)) { 421- MS_LOG(ERROR) << "NNCompilation set QuantConfig failed, ret: " << ret_code; 422- return kLiteError; 423+ if (extension_options_.is_optional_quant_setted) { 424+ if (extension_options_.quant_config == nullptr || extension_options_.quant_config_size <= 0) { 425+ MS_LOG(ERROR) << "NNCompilation set QuantConfig faild, input quant config is invalid, please make sure buffer " 426+ << "is not null and size > 0."; 427+ return kLiteError; 428+ } 429+ ret_code = mindspore::lite::HMS_HiAIOptions_SetQuantConfig(nn_compilation, extension_options_.quant_config, 430+ extension_options_.quant_config_size); 431+ if ((ret_code != OH_NN_SUCCESS) && (ret_code != OH_NN_OPERATION_FORBIDDEN)) { 432+ MS_LOG(ERROR) << "NNCompilation set QuantConfig failed, ret: " << ret_code; 433+ return kLiteError; 434+ } 435 } 436+ } else { 437+ MS_LOG(WARNING) << "hiai_foundation is nullptr."; 438 } 439 #endif 440 441@@ -664,7 +673,7 @@ Status NNRTDelegate::CreateNNRTSubgraphKernels(DelegateModel<schema::Primitive> 442 continue ; 443 } 444 445- auto nnrt_model_kernel = new (std::nothrow)NNRTModelKernel(nn_executor, nnrt_device_info_.device_id_, in_tensors, out_tensors); 446+ auto nnrt_model_kernel = new (std::nothrow) NNRTModelKernel(nn_executor, nnrt_device_info_, in_tensors, out_tensors); 447 if (nnrt_model_kernel == nullptr) { 448 MS_LOG(ERROR) << "new NNRTModelKernel failed"; 449 return kLiteError; 450diff --git a/mindspore/lite/src/litert/delegate/nnrt/nnrt_model_kernel.cc b/mindspore/lite/src/litert/delegate/nnrt/nnrt_model_kernel.cc 451index 2a66d133..1411020b 100644 452--- a/mindspore/lite/src/litert/delegate/nnrt/nnrt_model_kernel.cc 453+++ b/mindspore/lite/src/litert/delegate/nnrt/nnrt_model_kernel.cc 454@@ -17,9 +17,15 @@ 455 #include "nnrt_model_kernel.h" 456 #include "nnrt_allocator.h" 457 #include "litert/cxx_api/tensor/tensor_impl.h" 458-int mindspore::NNRTModelKernel::Prepare() { 459+ 460+namespace mindspore{ 461+namespace { 462+constexpr auto kDynamicDims = "DynamicDims"; 463+} 464+ 465+int NNRTModelKernel::Prepare() { 466 for (size_t i = 0; i < inputs_.size(); i++) { 467- auto nnrt_allocator = std::make_shared<lite::NNRTAllocator>(oh_nn_executor, i, device_id_, lite::NNRT_INPUT); 468+ auto nnrt_allocator = std::make_shared<lite::NNRTAllocator>(oh_nn_executor_, i, nnrt_device_info_.device_id_, lite::NNRT_INPUT); 469 if (nnrt_allocator == nullptr) { 470 MS_LOG(ERROR) << "Create NNRTAllocator failed"; 471 return lite::RET_NULL_PTR; 472@@ -27,7 +33,7 @@ int mindspore::NNRTModelKernel::Prepare() { 473 inputs_[i].SetAllocator(nnrt_allocator); 474 } 475 for (size_t i = 0; i < outputs_.size(); i++) { 476- auto nnrt_allocator = std::make_shared<lite::NNRTAllocator>(oh_nn_executor, i, device_id_, lite::NNRT_OUTPUT); 477+ auto nnrt_allocator = std::make_shared<lite::NNRTAllocator>(oh_nn_executor_, i, nnrt_device_info_.device_id_, lite::NNRT_OUTPUT); 478 if (nnrt_allocator == nullptr) { 479 MS_LOG(ERROR) << "Create NNRTAllocator failed"; 480 return lite::RET_NULL_PTR; 481@@ -37,7 +43,19 @@ int mindspore::NNRTModelKernel::Prepare() { 482 return lite::RET_OK; 483 } 484 485-int mindspore::NNRTModelKernel::Execute() { 486+int NNRTModelKernel::ReSize() { 487+ const auto &extensions = nnrt_device_info_.extensions_; 488+ auto iter = std::find_if(extensions.begin(), extensions.end(), [](const lite::Extension &extension) { 489+ return extension.name == kDynamicDims; 490+ }); 491+ if (iter != extensions.end() && !iter->value.empty()) { 492+ return lite::RET_OK; 493+ } 494+ MS_LOG(ERROR) << "NNRT only support the resize function when DynamicDims is enabled."; 495+ return lite::RET_ERROR; 496+} 497+ 498+int NNRTModelKernel::Execute() { 499 MS_CHECK_TRUE_RET(this->outputs().empty() != true, lite::RET_ERROR); 500 zero_copy_ = IS_NNRT_ALLOCATOR(this->outputs()[Index0].allocator()); 501 502@@ -61,7 +79,7 @@ int mindspore::NNRTModelKernel::Execute() { 503 } 504 MS_LOG(INFO) << "Running NNRtModel Kernel..."; 505 OH_NN_ReturnCode ret_code; 506- ret_code = OH_NNExecutor_RunSync(oh_nn_executor, nn_input_tensors_.data(), nn_input_tensors_.size(), 507+ ret_code = OH_NNExecutor_RunSync(oh_nn_executor_, nn_input_tensors_.data(), nn_input_tensors_.size(), 508 nn_output_tensors_.data(), nn_output_tensors_.size()); 509 510 if (ret_code != OH_NN_SUCCESS) { 511@@ -73,67 +91,11 @@ int mindspore::NNRTModelKernel::Execute() { 512 return lite::RET_OK; 513 } 514 515-OH_NN_DataType mindspore::NNRTModelKernel::ConvertDataType(mindspore::DataType data_type) { 516- OH_NN_DataType oh_data_type; 517- switch (data_type) { 518- case DataType::kTypeUnknown: 519- case DataType::kObjectTypeString: 520- case DataType::kObjectTypeList: 521- case DataType::kObjectTypeTuple: 522- case DataType::kObjectTypeTensorType: 523- case DataType::kNumberTypeBegin: 524- case DataType::kNumberTypeEnd: 525- case DataType::kInvalidType: 526- oh_data_type = OH_NN_UNKNOWN; 527- break; 528- case DataType::kNumberTypeBool: 529- oh_data_type = OH_NN_BOOL; 530- break; 531- case DataType::kNumberTypeInt8: 532- oh_data_type = OH_NN_INT8; 533- break; 534- case DataType::kNumberTypeInt16: 535- oh_data_type = OH_NN_INT16; 536- break; 537- case DataType::kNumberTypeInt32: 538- oh_data_type = OH_NN_INT32; 539- break; 540- case DataType::kNumberTypeInt64: 541- oh_data_type = OH_NN_INT64; 542- break; 543- case DataType::kNumberTypeUInt8: 544- oh_data_type = OH_NN_UINT8; 545- break; 546- case DataType::kNumberTypeUInt16: 547- oh_data_type = OH_NN_UINT16; 548- break; 549- case DataType::kNumberTypeUInt32: 550- oh_data_type = OH_NN_UINT32; 551- break; 552- case DataType::kNumberTypeUInt64: 553- oh_data_type = OH_NN_UINT64; 554- break; 555- case DataType::kNumberTypeFloat16: 556- oh_data_type = OH_NN_FLOAT16; 557- break; 558- case DataType::kNumberTypeFloat32: 559- oh_data_type = OH_NN_FLOAT32; 560- break; 561- case DataType::kNumberTypeFloat64: 562- oh_data_type = OH_NN_FLOAT64; 563- break; 564- default: { 565- oh_data_type = OH_NN_UNKNOWN; 566- } 567- } 568- return oh_data_type; 569-} 570- 571-int mindspore::NNRTModelKernel::SetInputs() { 572+int NNRTModelKernel::SetInputs() { 573 if (!zero_copy_) { 574 OH_NN_ReturnCode ret{OH_NN_FAILED}; 575 size_t nn_input_count = 0; 576- ret = OH_NNExecutor_GetInputCount(oh_nn_executor, &nn_input_count); 577+ ret = OH_NNExecutor_GetInputCount(oh_nn_executor_, &nn_input_count); 578 if (ret != OH_NN_SUCCESS) { 579 MS_LOG(ERROR) << "OH_NNExecutor_GetInputCount failed."; 580 return lite::RET_ERROR; 581@@ -143,13 +105,13 @@ int mindspore::NNRTModelKernel::SetInputs() { 582 return lite::RET_ERROR; 583 } 584 for (size_t i = 0; i < nn_input_count; i++) { 585- NN_TensorDesc *tensor_desc_tmp = OH_NNExecutor_CreateInputTensorDesc(oh_nn_executor, i); 586+ NN_TensorDesc *tensor_desc_tmp = OH_NNExecutor_CreateInputTensorDesc(oh_nn_executor_, i); 587 if (tensor_desc_tmp == nullptr) { 588 MS_LOG(ERROR) << "OH_NNExecutor_CreateInputTensorDesc failed, i = " << i; 589 return lite::RET_ERROR; 590 } 591 nn_input_tensor_descs_.emplace_back(tensor_desc_tmp); 592- NN_Tensor *tensor_tmp = OH_NNTensor_Create(device_id_, tensor_desc_tmp); 593+ NN_Tensor *tensor_tmp = OH_NNTensor_Create(nnrt_device_info_.device_id_, tensor_desc_tmp); 594 if (tensor_tmp == nullptr) { 595 MS_LOG(ERROR) << "OH_NNTensor_Create input failed, i = " << i; 596 return lite::RET_ERROR; 597@@ -166,6 +128,10 @@ int mindspore::NNRTModelKernel::SetInputs() { 598 } 599 } else { 600 for (size_t i = 0; i < inputs_.size(); i++) { 601+ if (inputs_[i].allocator() == nullptr) { 602+ MS_LOG(ERROR) << "NNRTAllocator is nullptr, i = " << i; 603+ return lite::RET_ERROR; 604+ } 605 void *data = inputs_[i].MutableData(); 606 NN_Tensor *tensor_tmp = reinterpret_cast<lite::NNRTAllocator *>(inputs_[i].allocator().get())->GetNNTensor(data); 607 if (tensor_tmp == nullptr) { 608@@ -178,11 +144,11 @@ int mindspore::NNRTModelKernel::SetInputs() { 609 return lite::RET_OK; 610 } 611 612-int mindspore::NNRTModelKernel::SetOutputs() { 613+int NNRTModelKernel::SetOutputs() { 614 if (!zero_copy_) { 615 OH_NN_ReturnCode ret{OH_NN_FAILED}; 616 size_t nn_output_count = 0; 617- ret = OH_NNExecutor_GetOutputCount(oh_nn_executor, &nn_output_count); 618+ ret = OH_NNExecutor_GetOutputCount(oh_nn_executor_, &nn_output_count); 619 if (ret != OH_NN_SUCCESS) { 620 MS_LOG(ERROR) << "OH_NNExecutor_GetOutputCount failed."; 621 return lite::RET_ERROR; 622@@ -192,13 +158,13 @@ int mindspore::NNRTModelKernel::SetOutputs() { 623 return lite::RET_ERROR; 624 } 625 for (size_t i = 0; i < nn_output_count; i++) { 626- NN_TensorDesc *tensor_desc_tmp = OH_NNExecutor_CreateOutputTensorDesc(oh_nn_executor, i); 627+ NN_TensorDesc *tensor_desc_tmp = OH_NNExecutor_CreateOutputTensorDesc(oh_nn_executor_, i); 628 if (tensor_desc_tmp == nullptr) { 629 MS_LOG(ERROR) << "OH_NNExecutor_CreateOutputTensorDesc failed, i = " << i; 630 return lite::RET_ERROR; 631 } 632 nn_output_tensor_descs_.emplace_back(tensor_desc_tmp); 633- NN_Tensor *tensor_tmp = OH_NNTensor_Create(device_id_, tensor_desc_tmp); 634+ NN_Tensor *tensor_tmp = OH_NNTensor_Create(nnrt_device_info_.device_id_, tensor_desc_tmp); 635 if (tensor_tmp == nullptr) { 636 MS_LOG(ERROR) << "OH_NNTensor_Create output failed, i = " << i; 637 return lite::RET_ERROR; 638@@ -210,6 +176,10 @@ int mindspore::NNRTModelKernel::SetOutputs() { 639 } 640 } else { 641 for (size_t i = 0; i < outputs_.size(); i++) { 642+ if (outputs_[i].allocator() == nullptr) { 643+ MS_LOG(ERROR) << "NNRTAllocator is nullptr, i = " << i; 644+ return lite::RET_ERROR; 645+ } 646 void *data = outputs_[i].MutableData(); 647 NN_Tensor *tensor_tmp = reinterpret_cast<lite::NNRTAllocator *>(outputs_[i].allocator().get())->GetNNTensor(data); 648 if (tensor_tmp == nullptr) { 649@@ -222,7 +192,7 @@ int mindspore::NNRTModelKernel::SetOutputs() { 650 return lite::RET_OK; 651 } 652 653-void mindspore::NNRTModelKernel::FreeNNTensor() { 654+void NNRTModelKernel::FreeNNTensor() { 655 for (size_t i = 0; i < nn_input_tensors_.size(); i++) { 656 OH_NNTensor_Destroy(&nn_input_tensors_[i]); 657 OH_NNTensorDesc_Destroy(&nn_input_tensor_descs_[i]); 658@@ -232,3 +202,5 @@ void mindspore::NNRTModelKernel::FreeNNTensor() { 659 OH_NNTensorDesc_Destroy(&nn_output_tensor_descs_[i]); 660 } 661 } 662+ 663+} // namespace mindspore 664diff --git a/mindspore/lite/src/litert/delegate/nnrt/nnrt_model_kernel.h b/mindspore/lite/src/litert/delegate/nnrt/nnrt_model_kernel.h 665index 40800a2a..7590d036 100644 666--- a/mindspore/lite/src/litert/delegate/nnrt/nnrt_model_kernel.h 667+++ b/mindspore/lite/src/litert/delegate/nnrt/nnrt_model_kernel.h 668@@ -22,6 +22,7 @@ 669 #include "include/api/kernel.h" 670 #include "interfaces/kits/c/neural_network_runtime/neural_network_runtime.h" 671 #include "src/common/log_adapter.h" 672+#include "src/litert/inner_context.h" 673 #include "include/errorcode.h" 674 675 namespace mindspore { 676@@ -31,16 +32,12 @@ class NNRTModelKernel : public kernel::Kernel { 677 * Because nnr can't run single op, but the whole model. So we decide to make the whole model into one kernel. 678 * */ 679 public: 680- NNRTModelKernel(OH_NNExecutor *oh_nn_executor, size_t device_id, const std::vector<mindspore::MSTensor> &inputs, 681+ NNRTModelKernel(OH_NNExecutor *oh_nn_executor, lite::NNRtDeviceInfo nnrt_device_info, const std::vector<mindspore::MSTensor> &inputs, 682 const std::vector<mindspore::MSTensor> &outputs) 683- : kernel::Kernel(inputs, outputs, nullptr, nullptr), device_id_(device_id), oh_nn_executor(oh_nn_executor) {} 684+ : kernel::Kernel(inputs, outputs, nullptr, nullptr), oh_nn_executor_(oh_nn_executor), nnrt_device_info_(nnrt_device_info) {} 685 int Prepare() override; 686 int Execute() override; 687- int ReSize() override { 688- MS_LOG(ERROR) << "NNRT does not support the resize function temporarily."; 689- return lite::RET_ERROR; 690- }; 691- OH_NN_DataType ConvertDataType(mindspore::DataType data_type); 692+ int ReSize() override; 693 int SetInputs(); 694 int SetOutputs(); 695 void FreeNNTensor(); 696@@ -52,8 +49,8 @@ class NNRTModelKernel : public kernel::Kernel { 697 } 698 699 protected: 700- size_t device_id_; 701- OH_NNExecutor *oh_nn_executor = nullptr; 702+ OH_NNExecutor *oh_nn_executor_ = nullptr; 703+ lite::NNRtDeviceInfo nnrt_device_info_; 704 std::vector<NN_Tensor *> nn_input_tensors_; 705 std::vector<NN_TensorDesc *> nn_input_tensor_descs_; 706 std::vector<NN_Tensor *> nn_output_tensors_; 707diff --git a/mindspore/lite/src/litert/delegate/nnrt/nnrt_utils.cc b/mindspore/lite/src/litert/delegate/nnrt/nnrt_utils.cc 708new file mode 100644 709index 00000000..049857bb 710--- /dev/null 711+++ b/mindspore/lite/src/litert/delegate/nnrt/nnrt_utils.cc 712@@ -0,0 +1,110 @@ 713+/** 714+ * Copyright 2024 Huawei Technologies Co., Ltd 715+ * 716+ * Licensed under the Apache License, Version 2.0 (the "License"); 717+ * you may not use this file except in compliance with the License. 718+ * You may obtain a copy of the License at 719+ * 720+ * http://www.apache.org/licenses/LICENSE-2.0 721+ * 722+ * Unless required by applicable law or agreed to in writing, software 723+ * distributed under the License is distributed on an "AS IS" BASIS, 724+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 725+ * See the License for the specific language governing permissions and 726+ * limitations under the License. 727+ */ 728+ 729+#include "src/litert/delegate/nnrt/nnrt_utils.h" 730+#include <unordered_map> 731+ 732+namespace mindspore::lite { 733+OH_NN_Format CastToNNRtFormat(Format format) { 734+ const std::unordered_map<Format, OH_NN_Format> kFormatMap = { 735+ {Format::NCHW, OH_NN_FORMAT_NCHW}, 736+ {Format::NHWC, OH_NN_FORMAT_NHWC}, 737+ }; 738+ auto iter = kFormatMap.find(format); 739+ if (iter == kFormatMap.end()) { 740+ return OH_NN_FORMAT_NONE; 741+ } 742+ return iter->second; 743+} 744+ 745+OH_NN_DataType CastToNNRtDataType(TypeId data_type) { 746+ OH_NN_DataType oh_data_type; 747+ switch (data_type) { 748+ case TypeId::kMetaTypeBegin: 749+ case TypeId::kMetaTypeType: 750+ case TypeId::kMetaTypeAny: 751+ case TypeId::kMetaTypeObject: 752+ case TypeId::kMetaTypeTypeType: 753+ case TypeId::kMetaTypeProblem: 754+ case TypeId::kMetaTypeExternal: 755+ case TypeId::kMetaTypeNone: 756+ case TypeId::kMetaTypeNull: 757+ case TypeId::kMetaTypeEllipsis: 758+ case TypeId::kMetaTypeEnd: 759+ case TypeId::kObjectTypeNumber: 760+ case TypeId::kObjectTypeString: 761+ case TypeId::kObjectTypeList: 762+ case TypeId::kObjectTypeTuple: 763+ case TypeId::kObjectTypeSlice: 764+ case TypeId::kObjectTypeKeyword: 765+ case TypeId::kObjectTypeTensorType: 766+ case TypeId::kObjectTypeRowTensorType: 767+ case TypeId::kObjectTypeCOOTensorType: 768+ case TypeId::kObjectTypeUndeterminedType: 769+ case TypeId::kObjectTypeClass: 770+ case TypeId::kObjectTypeDictionary: 771+ case TypeId::kObjectTypeFunction: 772+ case TypeId::kObjectTypeJTagged: 773+ case TypeId::kObjectTypeSymbolicKeyType: 774+ case TypeId::kObjectTypeEnvType: 775+ case TypeId::kObjectTypeRefKey: 776+ case TypeId::kObjectTypeRef: 777+ case TypeId::kObjectTypeEnd: 778+ oh_data_type = OH_NN_UNKNOWN; 779+ break; 780+ case TypeId::kNumberTypeBool: 781+ oh_data_type = OH_NN_BOOL; 782+ break; 783+ case TypeId::kNumberTypeInt8: 784+ oh_data_type = OH_NN_INT8; 785+ break; 786+ case TypeId::kNumberTypeInt16: 787+ oh_data_type = OH_NN_INT16; 788+ break; 789+ case TypeId::kNumberTypeInt32: 790+ oh_data_type = OH_NN_INT32; 791+ break; 792+ case TypeId::kNumberTypeInt64: 793+ oh_data_type = OH_NN_INT64; 794+ break; 795+ case TypeId::kNumberTypeUInt8: 796+ oh_data_type = OH_NN_UINT8; 797+ break; 798+ case TypeId::kNumberTypeUInt16: 799+ oh_data_type = OH_NN_UINT16; 800+ break; 801+ case TypeId::kNumberTypeUInt32: 802+ oh_data_type = OH_NN_UINT32; 803+ break; 804+ case TypeId::kNumberTypeUInt64: 805+ oh_data_type = OH_NN_UINT64; 806+ break; 807+ case TypeId::kNumberTypeFloat16: 808+ oh_data_type = OH_NN_FLOAT16; 809+ break; 810+ case TypeId::kNumberTypeFloat32: 811+ oh_data_type = OH_NN_FLOAT32; 812+ break; 813+ case TypeId::kNumberTypeFloat64: 814+ oh_data_type = OH_NN_FLOAT64; 815+ break; 816+ default: { 817+ oh_data_type = OH_NN_UNKNOWN; 818+ } 819+ } 820+ return oh_data_type; 821+} 822+} // namespace mindspore::lite 823diff --git a/mindspore/lite/src/litert/delegate/nnrt/nnrt_utils.h b/mindspore/lite/src/litert/delegate/nnrt/nnrt_utils.h 824new file mode 100644 825index 00000000..f8055686 826--- /dev/null 827+++ b/mindspore/lite/src/litert/delegate/nnrt/nnrt_utils.h 828@@ -0,0 +1,29 @@ 829+/** 830+ * Copyright 2024 Huawei Technologies Co., Ltd 831+ * 832+ * Licensed under the Apache License, Version 2.0 (the "License"); 833+ * you may not use this file except in compliance with the License. 834+ * You may obtain a copy of the License at 835+ * 836+ * http://www.apache.org/licenses/LICENSE-2.0 837+ * 838+ * Unless required by applicable law or agreed to in writing, software 839+ * distributed under the License is distributed on an "AS IS" BASIS, 840+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 841+ * See the License for the specific language governing permissions and 842+ * limitations under the License. 843+ */ 844+ 845+#ifndef MINDSPORE_LITE_SRC_RUNTIME_DELEGATE_NNRT_UTILS_H 846+#define MINDSPORE_LITE_SRC_RUNTIME_DELEGATE_NNRT_UTILS_H 847+ 848+#include "include/api/format.h" 849+#include "ir/dtype/type_id.h" 850+#include "interfaces/kits/c/neural_network_runtime/neural_network_runtime.h" 851+ 852+namespace mindspore::lite { 853+OH_NN_Format CastToNNRtFormat(Format format); 854+OH_NN_DataType CastToNNRtDataType(TypeId data_type); 855+} // namespace mindspore::lite 856+ 857+#endif // MINDSPORE_LITE_SRC_RUNTIME_DELEGATE_NNRT_UTILS_H 858diff --git a/mindspore/lite/src/litert/infer_manager.cc b/mindspore/lite/src/litert/infer_manager.cc 859index 908ab122..3a5f8832 100644 860--- a/mindspore/lite/src/litert/infer_manager.cc 861+++ b/mindspore/lite/src/litert/infer_manager.cc 862@@ -208,7 +208,10 @@ int KernelInferShape(const std::vector<lite::Tensor *> &inputs, const std::vecto 863 return tensor_ret; 864 } 865 } else { 866- if (out_tensors.at(i)->data_ != nullptr) { 867+ // During the online phase of shape operator fusion, the output data is computed in advance during the infer shape 868+ // stage. Therefore, the output data is not nullptr and is constant. 869+ if (parameter->type_ == static_cast<int>(PrimType::PrimType_Inner_ShapeFusion) && 870+ out_tensors.at(i)->data_ != nullptr) { 871 outputs.at(i)->set_own_data(true); 872 outputs.at(i)->set_category(CONST_TENSOR); 873 } 874diff --git a/mindspore/lite/src/tensor.cc b/mindspore/lite/src/tensor.cc 875index a7bb1899..9d9a1491 100644 876--- a/mindspore/lite/src/tensor.cc 877+++ b/mindspore/lite/src/tensor.cc 878@@ -18,6 +18,9 @@ 879 #include <vector> 880 #include <string> 881 #include <utility> 882+#ifdef SUPPORT_NNRT 883+#include "src/litert/delegate/nnrt/nnrt_allocator.h" 884+#endif 885 #include "schema/ops_types_generated.h" 886 #include "securec/include/securec.h" 887 #include "include/errorcode.h" 888@@ -427,7 +430,18 @@ int Tensor::MallocData(const AllocatorPtr allocator) { 889 if (allocator_ == nullptr) { 890 this->tensor_c_.data_ = malloc(data_size); 891 } else { 892- this->tensor_c_.data_ = allocator_->Malloc(data_size); 893+#ifdef SUPPORT_NNRT 894+ if (IS_NNRT_ALLOCATOR(allocator_)) { 895+ this->tensor_c_.data_ = dynamic_cast<NNRTAllocator *>(allocator_.get())->MallocByDesc(data_size, this->shape(), 896+ this->data_type(), 897+ this->format(), 898+ this->tensor_name()); 899+ } else { 900+#endif 901+ this->tensor_c_.data_ = allocator_->Malloc(data_size); 902+#ifdef SUPPORT_NNRT 903+ } 904+#endif 905 allocator_->SetRefCount(this->tensor_c_.data_, 1); 906 } 907 if (this->tensor_c_.data_ == nullptr) { 908-- 9092.17.1 910 911