1 /* 2 * Copyright (c) Meta Platforms, Inc. and affiliates. 3 * All rights reserved. 4 * 5 * This source code is licensed under the BSD-style license found in the 6 * LICENSE file in the root directory of this source tree. 7 */ 8 9 #include <executorch/backends/xnnpack/runtime/XNNCompiler.h> 10 #include <executorch/runtime/backend/interface.h> 11 #include <executorch/runtime/core/error.h> 12 #include <executorch/runtime/core/evalue.h> 13 #include <executorch/runtime/platform/profiler.h> 14 15 #include <memory> 16 #include <mutex> 17 18 #pragma clang diagnostic ignored "-Wglobal-constructors" 19 20 namespace executorch { 21 namespace backends { 22 23 using executorch::runtime::ArrayRef; 24 using executorch::runtime::Backend; 25 using executorch::runtime::BackendExecutionContext; 26 using executorch::runtime::BackendInitContext; 27 using executorch::runtime::CompileSpec; 28 using executorch::runtime::DelegateHandle; 29 using executorch::runtime::Error; 30 using executorch::runtime::EValue; 31 using executorch::runtime::FreeableBuffer; 32 using executorch::runtime::Result; 33 34 class XnnpackBackend final : public ::executorch::runtime::BackendInterface { 35 public: 36 ~XnnpackBackend() = default; 37 XnnpackBackend()38 XnnpackBackend() { 39 // Initialize XNNPACK 40 xnn_status status = xnn_initialize(/*allocator=*/nullptr); 41 if (status != xnn_status_success) { 42 ET_LOG( 43 Error, 44 "Failed to initialize, XNNPACK status: 0x%x", 45 (unsigned int)status); 46 return; 47 } 48 49 #ifdef ENABLE_XNNPACK_SHARED_WORKSPACE 50 // Create a workspace for the XNNExecutor to use. This workspace will be 51 // shared across all delegate instances. 52 ET_LOG(Debug, "Creating XNN workspace"); 53 xnn_workspace_t workspace = nullptr; 54 status = xnn_create_workspace(&workspace); 55 if (status != xnn_status_success) { 56 ET_LOG( 57 Error, 58 "Failed to create XNN workspace, XNNPACK status: 0x%x", 59 (unsigned int)status); 60 workspace = nullptr; 61 return; 62 } 63 workspace_.reset(workspace); 64 ET_LOG(Debug, "Created XNN workspace: %p", workspace_.get()); 65 #endif // ENABLE_XNNPACK_SHARED_WORKSPACE 66 } 67 is_available() const68 bool is_available() const override { 69 return xnn_status_success == xnn_initialize(/*allocator=*/nullptr); 70 } 71 init(BackendInitContext & context,FreeableBuffer * processed,ArrayRef<CompileSpec> compile_specs) const72 Result<DelegateHandle*> init( 73 BackendInitContext& context, 74 FreeableBuffer* processed, 75 ArrayRef<CompileSpec> compile_specs) const override { 76 auto executor = ET_ALLOCATE_INSTANCE_OR_RETURN_ERROR( 77 context.get_runtime_allocator(), xnnpack::delegate::XNNExecutor); 78 79 // Executor has been allocated but not constructed, ensure that runtime_ is 80 // nullptr by constructing it in place here. NOTE: Since we use placement 81 // new and since this type is not trivially destructible, we must call the 82 // destructor manually in destroy(). 83 new (executor) xnnpack::delegate::XNNExecutor; 84 Error err = xnnpack::delegate::XNNCompiler::compileModel( 85 processed->data(), 86 processed->size(), 87 executor, 88 context.get_runtime_allocator(), 89 workspace_.get()); 90 // This backend does not need its processed data after compiling the model. 91 processed->Free(); 92 93 if (err != Error::Ok) { 94 // destroy() won't be called on this handle, so we need to clean it up 95 // now. 96 executor->~XNNExecutor(); 97 98 ET_LOG( 99 Error, "XNNCompiler::compileModel failed: 0x%x", (unsigned int)err); 100 return err; 101 } 102 return executor; 103 } 104 execute(BackendExecutionContext & context,DelegateHandle * handle,EValue ** args) const105 Error execute( 106 BackendExecutionContext& context, 107 DelegateHandle* handle, 108 EValue** args) const override { 109 auto executor = static_cast<xnnpack::delegate::XNNExecutor*>(handle); 110 111 #ifdef ENABLE_XNNPACK_SHARED_WORKSPACE 112 const std::lock_guard<std::mutex> lock(workspace_mutex_); 113 #endif 114 115 // Prepare Inputs/Outputs and Propagate Input Shapes 116 Error err = executor->prepare_args(args); 117 if (err != Error::Ok) { 118 return err; 119 } 120 121 err = executor->forward(context); 122 123 if (err != Error::Ok) { 124 return err; 125 } 126 127 // Resize outputs and recast pointers if necessary 128 err = executor->resize_outputs(args); 129 130 return err; 131 } 132 destroy(DelegateHandle * handle) const133 void destroy(DelegateHandle* handle) const override { 134 if (handle != nullptr) { 135 #ifdef ENABLE_XNNPACK_SHARED_WORKSPACE 136 // This is needed to serialize access to xnn_delete_runtime which is not 137 // thread safe. This can heppen when multiple threads call destroy() on 138 // the same backend instance. 139 const std::lock_guard<std::mutex> lock(workspace_mutex_); 140 #endif 141 auto executor = static_cast<xnnpack::delegate::XNNExecutor*>(handle); 142 #ifdef ENABLE_XNNPACK_PROFILING 143 executor->print_avg_op_timings(); 144 #endif 145 // XNNExecutor is not trivially destructible. Since this was constructed 146 // manually in init(), we must destroy it manually here. 147 executor->~XNNExecutor(); 148 } 149 } 150 151 private: 152 // This is a global workspace for all delegate instances. 153 mutable std::mutex workspace_mutex_; 154 std::unique_ptr<xnn_workspace, decltype(&xnn_release_workspace)> workspace_{ 155 nullptr, 156 &xnn_release_workspace}; 157 }; 158 159 namespace { 160 auto cls = XnnpackBackend(); 161 Backend backend{"XnnpackBackend", &cls}; 162 static auto success_with_compiler = register_backend(backend); 163 } // namespace 164 165 } // namespace backends 166 } // namespace executorch 167