1 //
2 // Copyright © 2021 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5 #include "TestBlocks.hpp"
6 #include "TestStrategy.hpp"
7
8 #include <IMemoryOptimizerStrategy.hpp>
9 #include <MemoryOptimizerStrategyLibrary.hpp>
10 #include <strategies/StrategyValidator.hpp>
11
12
13 #include <cxxopts.hpp>
14
15 #include <iostream>
16 #include <algorithm>
17 #include <iomanip>
18
19 std::vector<TestBlock> testBlocks
20 {
21 {"fsrcnn", fsrcnn},
22 {"inceptionv4", inceptionv4},
23 {"deeplabv3", deeplabv3},
24 {"deepspeechv1", deepspeechv1},
25 {"mobilebert", mobilebert},
26 {"ssd_mobilenetv2", ssd_mobilenetv2},
27 {"resnetv2", resnetv2},
28 {"yolov3",yolov3}
29 };
30
PrintModels()31 void PrintModels()
32 {
33 std::cout << "Available models:\n";
34 for (const auto& model : testBlocks)
35 {
36 std::cout << model.m_Name << "\n";
37 }
38 std::cout << "\n";
39 }
40
GetMinPossibleMemorySize(const std::vector<armnn::MemBlock> & blocks)41 size_t GetMinPossibleMemorySize(const std::vector<armnn::MemBlock>& blocks)
42 {
43 unsigned int maxLifetime = 0;
44 for (auto& block: blocks)
45 {
46 maxLifetime = std::max(maxLifetime, block.m_EndOfLife);
47 }
48 maxLifetime++;
49
50 std::vector<size_t> lifetimes(maxLifetime);
51 for (const auto& block : blocks)
52 {
53 for (auto lifetime = block.m_StartOfLife; lifetime <= block.m_EndOfLife; ++lifetime)
54 {
55 lifetimes[lifetime] += block.m_MemSize;
56 }
57 }
58 return *std::max_element(lifetimes.begin(), lifetimes.end());
59 }
60
RunBenchmark(armnn::IMemoryOptimizerStrategy * strategy,std::vector<TestBlock> * models)61 void RunBenchmark(armnn::IMemoryOptimizerStrategy* strategy, std::vector<TestBlock>* models)
62 {
63 using Clock = std::chrono::high_resolution_clock;
64 float avgEfficiency = 0;
65 std::chrono::duration<double, std::milli> avgDuration{};
66 std::cout << "\nMemory Strategy: " << strategy->GetName()<< "\n";
67 std::cout << "===============================================\n";
68 for (auto& model : *models)
69 {
70 auto now = Clock::now();
71 const std::vector<armnn::MemBin> result = strategy->Optimize(model.m_Blocks);
72 auto duration = std::chrono::duration<double, std::milli>(Clock::now() - now);
73
74 avgDuration += duration;
75 size_t memoryUsage = 0;
76 for (auto bin : result)
77 {
78 memoryUsage += bin.m_MemSize;
79 }
80 size_t minSize = GetMinPossibleMemorySize(model.m_Blocks);
81
82 float efficiency = static_cast<float>(minSize) / static_cast<float>(memoryUsage);
83 efficiency*=100;
84 avgEfficiency += efficiency;
85 std::cout << "\nModel: " << model.m_Name << "\n";
86
87 std::cout << "Strategy execution time: " << std::setprecision(4) << duration.count() << " milliseconds\n";
88
89 std::cout << "Memory usage: " << memoryUsage/1024 << " kb\n";
90
91 std::cout << "Minimum possible usage: " << minSize/1024 << " kb\n";
92
93 std::cout << "Memory efficiency: " << std::setprecision(3) << efficiency << "%\n";
94 }
95
96 avgDuration/= static_cast<double>(models->size());
97 avgEfficiency/= static_cast<float>(models->size());
98
99 std::cout << "\n===============================================\n";
100 std::cout << "Average memory duration: " << std::setprecision(4) << avgDuration.count() << " milliseconds\n";
101 std::cout << "Average memory efficiency: " << std::setprecision(3) << avgEfficiency << "%\n";
102 }
103
104 struct BenchmarkOptions
105 {
106 std::string m_StrategyName;
107 std::string m_ModelName;
108 bool m_UseDefaultStrategy = false;
109 bool m_Validate = false;
110 };
111
ParseOptions(int argc,char * argv[])112 BenchmarkOptions ParseOptions(int argc, char* argv[])
113 {
114 cxxopts::Options options("Memory Benchmark", "Tests memory optimization strategies on different models");
115
116 options.add_options()
117 ("s, strategy", "Strategy name, do not specify to use default strategy", cxxopts::value<std::string>())
118 ("m, model", "Model name", cxxopts::value<std::string>())
119 ("v, validate", "Validate strategy", cxxopts::value<bool>()->default_value("false")->implicit_value("true"))
120 ("h,help", "Display usage information");
121
122 auto result = options.parse(argc, argv);
123 if (result.count("help"))
124 {
125 std::cout << options.help() << std::endl;
126 PrintModels();
127
128 std::cout << "\nAvailable strategies:\n";
129
130 for (const auto& s :armnn::GetMemoryOptimizerStrategyNames())
131 {
132 std::cout << s << "\n";
133 }
134 exit(EXIT_SUCCESS);
135 }
136
137 BenchmarkOptions benchmarkOptions;
138
139 if(result.count("strategy"))
140 {
141 benchmarkOptions.m_StrategyName = result["strategy"].as<std::string>();
142 }
143 else
144 {
145 std::cout << "No Strategy given, using default strategy";
146
147 benchmarkOptions.m_UseDefaultStrategy = true;
148 }
149
150 if(result.count("model"))
151 {
152 benchmarkOptions.m_ModelName = result["model"].as<std::string>();
153 }
154
155 benchmarkOptions.m_Validate = result["validate"].as<bool>();
156
157 return benchmarkOptions;
158 }
159
main(int argc,char * argv[])160 int main(int argc, char* argv[])
161 {
162 BenchmarkOptions benchmarkOptions = ParseOptions(argc, argv);
163
164 std::shared_ptr<armnn::IMemoryOptimizerStrategy> strategy;
165
166 if (benchmarkOptions.m_UseDefaultStrategy)
167 {
168 strategy = std::make_shared<armnn::TestStrategy>();
169 }
170 else
171 {
172 strategy = armnn::GetMemoryOptimizerStrategy(benchmarkOptions.m_StrategyName);
173
174 if (!strategy)
175 {
176 std::cout << "Strategy name not found\n";
177 return 0;
178 }
179 }
180
181 std::vector<TestBlock> model;
182 std::vector<TestBlock>* modelsToTest = &testBlocks;
183 if (benchmarkOptions.m_ModelName.size() != 0)
184 {
185 auto it = std::find_if(testBlocks.cbegin(), testBlocks.cend(), [&](const TestBlock testBlock)
186 {
187 return testBlock.m_Name == benchmarkOptions.m_ModelName;
188 });
189
190 if (it == testBlocks.end())
191 {
192 std::cout << "Model name not found\n";
193 return 0;
194 }
195 else
196 {
197 model.push_back(*it);
198 modelsToTest = &model;
199 }
200 }
201
202 if (benchmarkOptions.m_Validate)
203 {
204 armnn::StrategyValidator strategyValidator;
205
206 strategyValidator.SetStrategy(strategy);
207
208 RunBenchmark(&strategyValidator, modelsToTest);
209 }
210 else
211 {
212 RunBenchmark(strategy.get(), modelsToTest);
213 }
214
215 }