1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef SRC_TRACE_PROCESSOR_PERFETTO_SQL_ENGINE_PERFETTO_SQL_ENGINE_H_
18 #define SRC_TRACE_PROCESSOR_PERFETTO_SQL_ENGINE_PERFETTO_SQL_ENGINE_H_
19
20 #include <cstddef>
21 #include <cstdint>
22 #include <memory>
23 #include <string>
24 #include <string_view>
25 #include <utility>
26 #include <vector>
27
28 #include "perfetto/base/logging.h"
29 #include "perfetto/base/status.h"
30 #include "perfetto/ext/base/flat_hash_map.h"
31 #include "perfetto/ext/base/status_or.h"
32 #include "perfetto/trace_processor/basic_types.h"
33 #include "src/trace_processor/containers/string_pool.h"
34 #include "src/trace_processor/db/runtime_table.h"
35 #include "src/trace_processor/db/table.h"
36 #include "src/trace_processor/perfetto_sql/engine/runtime_table_function.h"
37 #include "src/trace_processor/perfetto_sql/intrinsics/functions/sql_function.h"
38 #include "src/trace_processor/perfetto_sql/intrinsics/table_functions/static_table_function.h"
39 #include "src/trace_processor/perfetto_sql/parser/function_util.h"
40 #include "src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.h"
41 #include "src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h"
42 #include "src/trace_processor/sqlite/bindings/sqlite_result.h"
43 #include "src/trace_processor/sqlite/bindings/sqlite_window_function.h"
44 #include "src/trace_processor/sqlite/db_sqlite_table.h"
45 #include "src/trace_processor/sqlite/sql_source.h"
46 #include "src/trace_processor/sqlite/sqlite_engine.h"
47 #include "src/trace_processor/sqlite/sqlite_utils.h"
48 #include "src/trace_processor/util/sql_argument.h"
49 #include "src/trace_processor/util/sql_modules.h"
50
51 namespace perfetto::trace_processor {
52
53 // Intermediary class which translates high-level concepts and algorithms used
54 // in trace processor into lower-level concepts and functions can be understood
55 // by and executed against SQLite.
56 class PerfettoSqlEngine {
57 public:
58 struct ExecutionStats {
59 uint32_t column_count = 0;
60 uint32_t statement_count = 0;
61 uint32_t statement_count_with_output = 0;
62 };
63 struct ExecutionResult {
64 SqliteEngine::PreparedStatement stmt;
65 ExecutionStats stats;
66 };
67
68 PerfettoSqlEngine(StringPool* pool, bool enable_extra_checks);
69
70 // Executes all the statements in |sql| and returns a |ExecutionResult|
71 // object. The metadata will reference all the statements executed and the
72 // |ScopedStmt| be empty.
73 //
74 // Returns an error if the execution of any statement failed or if there was
75 // no valid SQL to run.
76 base::StatusOr<ExecutionStats> Execute(SqlSource sql);
77
78 // Executes all the statements in |sql| fully until the final statement and
79 // returns a |ExecutionResult| object containing a |ScopedStmt| for the final
80 // statement (which has been stepped once) and metadata about all statements
81 // executed.
82 //
83 // Returns an error if the execution of any statement failed or if there was
84 // no valid SQL to run.
85 base::StatusOr<ExecutionResult> ExecuteUntilLastStatement(SqlSource sql);
86
87 // Prepares a single SQLite statement in |sql| and returns a
88 // |PreparedStatement| object.
89 //
90 // Returns an error if the preparation of the statement failed or if there was
91 // no valid SQL to run.
92 base::StatusOr<SqliteEngine::PreparedStatement> PrepareSqliteStatement(
93 SqlSource sql);
94
95 // Registers a trace processor C++ function to be runnable from SQL.
96 //
97 // The format of the function is given by the |SqlFunction|.
98 //
99 // |name|: name of the function in SQL.
100 // |argc|: number of arguments for this function. This can be -1 if
101 // the number of arguments is variable.
102 // |ctx|: context object for the function (see SqlFunction::Run);
103 // this object *must* outlive the function so should likely
104 // be either static or scoped to the lifetime of
105 // TraceProcessor.
106 // |deterministic|: whether this function has deterministic output given the
107 // same set of arguments.
108 template <typename Function = SqlFunction>
109 base::Status RegisterStaticFunction(const char* name,
110 int argc,
111 typename Function::Context* ctx,
112 bool deterministic = true);
113
114 // Registers a trace processor C++ function to be runnable from SQL.
115 //
116 // This function is the same as the above except allows a unique_ptr to be
117 // passed for the context; this allows for SQLite to manage the lifetime of
118 // this pointer instead of the essentially static requirement of the context
119 // pointer above.
120 template <typename Function>
121 base::Status RegisterStaticFunction(
122 const char* name,
123 int argc,
124 std::unique_ptr<typename Function::Context> ctx,
125 bool deterministic = true);
126
127 // Registers a trace processor C++ function to be runnable from SQL.
128 //
129 // The format of the function is given by the |SqliteFunction|.
130 //
131 // |ctx|: context object for the function; this object *must*
132 // outlive the function so should likely be either static or
133 // scoped to the lifetime of TraceProcessor.
134 // |deterministic|: whether this function has deterministic output given the
135 // same set of arguments.
136 template <typename Function>
137 base::Status RegisterSqliteFunction(typename Function::UserDataContext* ctx,
138 bool deterministic = true);
139 template <typename Function>
140 base::Status RegisterSqliteFunction(
141 std::unique_ptr<typename Function::UserDataContext> ctx,
142 bool deterministic = true);
143
144 // Registers a trace processor C++ aggregate function to be runnable from SQL.
145 //
146 // The format of the function is given by the |SqliteAggregateFunction|.
147 //
148 // |ctx|: context object for the function; this object *must*
149 // outlive the function so should likely be either static or
150 // scoped to the lifetime of TraceProcessor.
151 // |deterministic|: whether this function has deterministic output given the
152 // same set of arguments.
153 template <typename Function>
154 base::Status RegisterSqliteAggregateFunction(
155 typename Function::UserDataContext* ctx,
156 bool deterministic = true);
157
158 // Registers a trace processor C++ window function to be runnable from SQL.
159 //
160 // The format of the function is given by the |SqliteWindowFunction|.
161 //
162 // |name|: name of the function in SQL.
163 // |argc|: number of arguments for this function. This can be -1 if
164 // the number of arguments is variable.
165 // |ctx|: context object for the function; this object *must*
166 // outlive the function so should likely be either static or
167 // scoped to the lifetime of TraceProcessor.
168 // |deterministic|: whether this function has deterministic output given the
169 // same set of arguments.
170 template <typename Function = SqliteWindowFunction>
171 base::Status RegisterSqliteWindowFunction(const char* name,
172 int argc,
173 typename Function::Context* ctx,
174 bool deterministic = true);
175
176 // Registers a function with the prototype |prototype| which returns a value
177 // of |return_type| and is implemented by executing the SQL statement |sql|.
178 base::Status RegisterRuntimeFunction(bool replace,
179 const FunctionPrototype& prototype,
180 const std::string& return_type,
181 SqlSource sql);
182
183 // Enables memoization for the given SQL function.
184 base::Status EnableSqlFunctionMemoization(const std::string& name);
185
186 // Registers a trace processor C++ table with SQLite with an SQL name of
187 // |name|.
188 void RegisterStaticTable(Table*,
189 const std::string& name,
190 Table::Schema schema);
191
192 // Registers a trace processor C++ table function with SQLite.
193 void RegisterStaticTableFunction(std::unique_ptr<StaticTableFunction> fn);
194
sqlite_engine()195 SqliteEngine* sqlite_engine() { return engine_.get(); }
196
197 // Makes new SQL package available to include.
RegisterPackage(const std::string & name,sql_modules::RegisteredPackage package)198 void RegisterPackage(const std::string& name,
199 sql_modules::RegisteredPackage package) {
200 packages_.Erase(name);
201 packages_.Insert(name, std::move(package));
202 }
203
204 // Fetches registered SQL package.
FindPackage(const std::string & name)205 sql_modules::RegisteredPackage* FindPackage(const std::string& name) {
206 return packages_.Find(name);
207 }
208
209 // Returns the number of objects (tables, views, functions etc) registered
210 // with SQLite.
SqliteRegisteredObjectCount()211 uint64_t SqliteRegisteredObjectCount() {
212 // This query will return all the tables, views, indexes and table functions
213 // SQLite knows about.
214 constexpr char kAllTablesQuery[] =
215 "SELECT COUNT() FROM (SELECT * FROM sqlite_master "
216 "UNION ALL SELECT * FROM sqlite_temp_master)";
217 auto stmt = ExecuteUntilLastStatement(
218 SqlSource::FromTraceProcessorImplementation(kAllTablesQuery));
219 PERFETTO_CHECK(stmt.ok());
220 uint32_t query_count =
221 static_cast<uint32_t>(sqlite3_column_int(stmt->stmt.sqlite_stmt(), 0));
222 PERFETTO_CHECK(!stmt->stmt.Step());
223 PERFETTO_CHECK(stmt->stmt.status().ok());
224
225 // The missing objects from the above query are static functions, runtime
226 // functions and macros. Add those in now.
227 return query_count + static_function_count_ +
228 static_window_function_count_ + static_aggregate_function_count_ +
229 runtime_function_count_ + macros_.size();
230 }
231
232 // Find table (Static or Runtime) registered with engine with provided name.
GetTableOrNull(std::string_view name)233 const Table* GetTableOrNull(std::string_view name) const {
234 if (auto maybe_runtime = GetRuntimeTableOrNull(name); maybe_runtime) {
235 return maybe_runtime;
236 }
237 return GetStaticTableOrNull(name);
238 }
239
240 // Find RuntimeTable registered with engine with provided name.
241 const RuntimeTable* GetRuntimeTableOrNull(std::string_view) const;
242
243 // Find static table registered with engine with provided name.
244 const Table* GetStaticTableOrNull(std::string_view) const;
245
246 // Find table (Static or Runtime) registered with engine with provided name.
GetMutableTableOrNull(std::string_view name)247 Table* GetMutableTableOrNull(std::string_view name) {
248 if (auto maybe_runtime = GetMutableRuntimeTableOrNull(name);
249 maybe_runtime) {
250 return maybe_runtime;
251 }
252 return GetMutableStaticTableOrNull(name);
253 }
254
255 // Find RuntimeTable registered with engine with provided name.
256 RuntimeTable* GetMutableRuntimeTableOrNull(std::string_view);
257
258 // Find static table registered with engine with provided name.
259 Table* GetMutableStaticTableOrNull(std::string_view);
260
261 private:
262 base::Status ExecuteCreateFunction(const PerfettoSqlParser::CreateFunction&);
263
264 base::Status ExecuteInclude(const PerfettoSqlParser::Include&,
265 const PerfettoSqlParser& parser);
266
267 // Creates a runtime table and registers it with SQLite.
268 base::Status ExecuteCreateTable(
269 const PerfettoSqlParser::CreateTable& create_table);
270
271 base::Status ExecuteCreateView(const PerfettoSqlParser::CreateView&);
272
273 base::Status ExecuteCreateMacro(const PerfettoSqlParser::CreateMacro&);
274
275 base::Status ExecuteCreateIndex(const PerfettoSqlParser::CreateIndex&);
276
277 base::Status ExecuteDropIndex(const PerfettoSqlParser::DropIndex&);
278
279 enum class CreateTableType {
280 kCreateTable,
281 // For now, bytes columns are not supported in CREATE PERFETTO TABLE,
282 // but supported in CREATE PERFETTO VIEW, so we skip them when validating
283 // views.
284 kValidateOnly
285 };
286 // |effective_schema| should have been normalised and its column order
287 // should match |column_names|.
288 base::StatusOr<std::unique_ptr<RuntimeTable>> CreateTableImpl(
289 const char* tag,
290 const std::string& name,
291 SqliteEngine::PreparedStatement source,
292 const std::vector<std::string>& column_names,
293 const std::vector<sql_argument::ArgumentDefinition>& effective_schema,
294 CreateTableType type);
295
296 template <typename Function>
297 base::Status RegisterFunctionWithSqlite(
298 const char* name,
299 int argc,
300 std::unique_ptr<typename Function::Context> ctx,
301 bool deterministic = true);
302
303 // Get the column names from a statement.
304 // |tag| is used in the error message if the statement is invalid.
305 base::StatusOr<std::vector<std::string>> GetColumnNamesFromSelectStatement(
306 const SqliteEngine::PreparedStatement& stmt,
307 const char* tag) const;
308
309 // Validates that the column names in |column_names| match the |schema|.
310 // Given that PerfettoSQL supports an arbitrary order of columns in the
311 // schema, this function also normalises the schema by reordering the schema
312 // columns to match the order of columns in the query. |tag| is used in the
313 // error message if the statement is invalid.
314 base::StatusOr<std::vector<sql_argument::ArgumentDefinition>>
315 ValidateAndGetEffectiveSchema(
316 const std::vector<std::string>& column_names,
317 const std::vector<sql_argument::ArgumentDefinition>& schema,
318 const char* tag) const;
319
320 // Given a package and a key, include the correct file(s) from the package.
321 // The key can contain a wildcard to include all files in the module with the
322 // matching prefix.
323 base::Status IncludePackageImpl(sql_modules::RegisteredPackage&,
324 const std::string& key,
325 const PerfettoSqlParser&);
326
327 // Include a given module.
328 base::Status IncludeModuleImpl(sql_modules::RegisteredPackage::ModuleFile&,
329 const std::string& key,
330 const PerfettoSqlParser&);
331
332 StringPool* pool_ = nullptr;
333 // If true, engine will perform additional consistency checks when e.g.
334 // creating tables and views.
335 const bool enable_extra_checks_;
336
337 uint64_t static_function_count_ = 0;
338 uint64_t static_aggregate_function_count_ = 0;
339 uint64_t static_window_function_count_ = 0;
340 uint64_t runtime_function_count_ = 0;
341
342 RuntimeTableFunctionModule::Context* runtime_table_fn_context_ = nullptr;
343 DbSqliteModule::Context* runtime_table_context_ = nullptr;
344 DbSqliteModule::Context* static_table_context_ = nullptr;
345 DbSqliteModule::Context* static_table_fn_context_ = nullptr;
346 base::FlatHashMap<std::string, sql_modules::RegisteredPackage> packages_;
347 base::FlatHashMap<std::string, PerfettoSqlPreprocessor::Macro> macros_;
348 std::unique_ptr<SqliteEngine> engine_;
349 };
350
351 // The rest of this file is just implementation details which we need
352 // in the header file because it is templated code. We separate it out
353 // like this to keep the API people actually care about easy to read.
354
355 namespace perfetto_sql_internal {
356
357 // RAII type to call Function::Cleanup when destroyed.
358 template <typename Function>
359 struct ScopedCleanup {
360 typename Function::Context* ctx;
~ScopedCleanupScopedCleanup361 ~ScopedCleanup() { Function::Cleanup(ctx); }
362 };
363
364 template <typename Function>
WrapSqlFunction(sqlite3_context * ctx,int argc,sqlite3_value ** argv)365 void WrapSqlFunction(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
366 using Context = typename Function::Context;
367 auto* ud = static_cast<Context*>(sqlite3_user_data(ctx));
368
369 ScopedCleanup<Function> scoped_cleanup{ud};
370 SqlValue value{};
371 SqlFunction::Destructors destructors{};
372 base::Status status =
373 Function::Run(ud, static_cast<size_t>(argc), argv, value, destructors);
374 if (!status.ok()) {
375 sqlite::result::Error(ctx, status.c_message());
376 return;
377 }
378
379 if (Function::kVoidReturn) {
380 if (!value.is_null()) {
381 sqlite::result::Error(ctx, "void SQL function returned value");
382 return;
383 }
384
385 // If the function doesn't want to return anything, set the "VOID"
386 // pointer type to a non-null value. Note that because of the weird
387 // way |sqlite3_value_pointer| works, we need to set some value even
388 // if we don't actually read it - just set it to a pointer to an empty
389 // string for this reason.
390 static char kVoidValue[] = "";
391 sqlite::result::StaticPointer(ctx, kVoidValue, "VOID");
392 } else {
393 sqlite::utils::ReportSqlValue(ctx, value, destructors.string_destructor,
394 destructors.bytes_destructor);
395 }
396
397 status = Function::VerifyPostConditions(ud);
398 if (!status.ok()) {
399 sqlite::result::Error(ctx, status.c_message());
400 return;
401 }
402 }
403
404 } // namespace perfetto_sql_internal
405
406 template <typename Function>
RegisterStaticFunction(const char * name,int argc,typename Function::Context * ctx,bool deterministic)407 base::Status PerfettoSqlEngine::RegisterStaticFunction(
408 const char* name,
409 int argc,
410 typename Function::Context* ctx,
411 bool deterministic) {
412 // Metric proto builder functions can be reregistered: don't double count when
413 // this happens.
414 if (!engine_->GetFunctionContext(name, argc)) {
415 static_function_count_++;
416 }
417 return engine_->RegisterFunction(
418 name, argc, perfetto_sql_internal::WrapSqlFunction<Function>, ctx,
419 nullptr, deterministic);
420 }
421
422 template <typename Function>
RegisterSqliteFunction(typename Function::UserDataContext * ctx,bool deterministic)423 base::Status PerfettoSqlEngine::RegisterSqliteFunction(
424 typename Function::UserDataContext* ctx,
425 bool deterministic) {
426 static_function_count_++;
427 return engine_->RegisterFunction(Function::kName, Function::kArgCount,
428 Function::Step, ctx, nullptr, deterministic);
429 }
430
431 template <typename Function>
RegisterSqliteFunction(std::unique_ptr<typename Function::UserDataContext> ctx,bool deterministic)432 base::Status PerfettoSqlEngine::RegisterSqliteFunction(
433 std::unique_ptr<typename Function::UserDataContext> ctx,
434 bool deterministic) {
435 static_function_count_++;
436 return engine_->RegisterFunction(
437 Function::kName, Function::kArgCount, Function::Step, ctx.release(),
438 [](void* ptr) {
439 std::unique_ptr<typename Function::UserDataContext>(
440 static_cast<typename Function::UserDataContext*>(ptr));
441 },
442 deterministic);
443 }
444
445 template <typename Function>
RegisterSqliteAggregateFunction(typename Function::UserDataContext * ctx,bool deterministic)446 base::Status PerfettoSqlEngine::RegisterSqliteAggregateFunction(
447 typename Function::UserDataContext* ctx,
448 bool deterministic) {
449 static_aggregate_function_count_++;
450 return engine_->RegisterAggregateFunction(
451 Function::kName, Function::kArgCount, Function::Step, Function::Final,
452 ctx, nullptr, deterministic);
453 }
454
455 template <typename Function>
RegisterSqliteWindowFunction(const char * name,int argc,typename Function::Context * ctx,bool deterministic)456 base::Status PerfettoSqlEngine::RegisterSqliteWindowFunction(
457 const char* name,
458 int argc,
459 typename Function::Context* ctx,
460 bool deterministic) {
461 static_window_function_count_++;
462 return engine_->RegisterWindowFunction(
463 name, argc, Function::Step, Function::Inverse, Function::Value,
464 Function::Final, ctx, nullptr, deterministic);
465 }
466
467 template <typename Function>
RegisterStaticFunction(const char * name,int argc,std::unique_ptr<typename Function::Context> ctx,bool deterministic)468 base::Status PerfettoSqlEngine::RegisterStaticFunction(
469 const char* name,
470 int argc,
471 std::unique_ptr<typename Function::Context> ctx,
472 bool deterministic) {
473 // Metric proto builder functions can be reregistered: don't double count when
474 // this happens.
475 if (!engine_->GetFunctionContext(name, argc)) {
476 static_function_count_++;
477 }
478 return RegisterFunctionWithSqlite<Function>(name, argc, std::move(ctx),
479 deterministic);
480 }
481
482 template <typename Function>
RegisterFunctionWithSqlite(const char * name,int argc,std::unique_ptr<typename Function::Context> ctx,bool deterministic)483 base::Status PerfettoSqlEngine::RegisterFunctionWithSqlite(
484 const char* name,
485 int argc,
486 std::unique_ptr<typename Function::Context> ctx,
487 bool deterministic) {
488 auto ctx_destructor = [](void* ptr) {
489 delete static_cast<typename Function::Context*>(ptr);
490 };
491 return engine_->RegisterFunction(
492 name, argc, perfetto_sql_internal::WrapSqlFunction<Function>,
493 ctx.release(), ctx_destructor, deterministic);
494 }
495
496 } // namespace perfetto::trace_processor
497
498 #endif // SRC_TRACE_PROCESSOR_PERFETTO_SQL_ENGINE_PERFETTO_SQL_ENGINE_H_
499