/*------------------------------------------------------------------------- * drawElements Quality Program Test Executor * ------------------------------------------ * * Copyright 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * *//*! * \file * \brief Test log compare utility. *//*--------------------------------------------------------------------*/ #include "xeTestLogParser.hpp" #include "xeTestResultParser.hpp" #include "deFilePath.hpp" #include "deString.h" #include "deThread.hpp" #include "deCommandLine.hpp" #include #include #include #include #include #include #include #include using std::map; using std::set; using std::string; using std::vector; enum OutputMode { OUTPUTMODE_ALL = 0, OUTPUTMODE_DIFF, OUTPUTMODE_LAST }; enum OutputFormat { OUTPUTFORMAT_TEXT = 0, OUTPUTFORMAT_CSV, OUTPUTFORMAT_LAST }; enum OutputValue { OUTPUTVALUE_STATUS_CODE = 0, OUTPUTVALUE_STATUS_DETAILS, OUTPUTVALUE_LAST }; namespace opt { DE_DECLARE_COMMAND_LINE_OPT(OutMode, OutputMode); DE_DECLARE_COMMAND_LINE_OPT(OutFormat, OutputFormat); DE_DECLARE_COMMAND_LINE_OPT(OutValue, OutputValue); static void registerOptions(de::cmdline::Parser &parser) { using de::cmdline::NamedValue; using de::cmdline::Option; static const NamedValue s_outputModes[] = {{"all", OUTPUTMODE_ALL}, {"diff", OUTPUTMODE_DIFF}}; static const NamedValue s_outputFormats[] = {{"text", OUTPUTFORMAT_TEXT}, {"csv", OUTPUTFORMAT_CSV}}; static const NamedValue s_outputValues[] = {{"code", OUTPUTVALUE_STATUS_CODE}, {"details", OUTPUTVALUE_STATUS_DETAILS}}; parser << Option("f", "format", "Output format", s_outputFormats, "csv") << Option("m", "mode", "Output mode", s_outputModes, "all") << Option("v", "value", "Value to extract", s_outputValues, "code"); } } // namespace opt struct CommandLine { CommandLine(void) : outMode(OUTPUTMODE_ALL), outFormat(OUTPUTFORMAT_CSV), outValue(OUTPUTVALUE_STATUS_CODE) { } OutputMode outMode; OutputFormat outFormat; OutputValue outValue; vector filenames; }; struct ShortBatchResult { vector resultHeaders; map resultMap; }; class ShortResultHandler : public xe::TestLogHandler { public: ShortResultHandler(ShortBatchResult &result) : m_result(result) { } void setSessionInfo(const xe::SessionInfo &) { // Ignored. } xe::TestCaseResultPtr startTestCaseResult(const char *casePath) { return xe::TestCaseResultPtr(new xe::TestCaseResultData(casePath)); } void testCaseResultUpdated(const xe::TestCaseResultPtr &) { // Ignored. } void testCaseResultComplete(const xe::TestCaseResultPtr &caseData) { xe::TestCaseResultHeader header; int caseNdx = (int)m_result.resultHeaders.size(); header.casePath = caseData->getTestCasePath(); header.caseType = xe::TESTCASETYPE_SELF_VALIDATE; header.statusCode = caseData->getStatusCode(); header.statusDetails = caseData->getStatusDetails(); if (header.statusCode == xe::TESTSTATUSCODE_LAST) { xe::TestCaseResult fullResult; xe::parseTestCaseResultFromData(&m_testResultParser, &fullResult, *caseData.get()); header = xe::TestCaseResultHeader(fullResult); } // Insert into result list & map. m_result.resultHeaders.push_back(header); m_result.resultMap[header.casePath] = caseNdx; } private: ShortBatchResult &m_result; xe::TestResultParser m_testResultParser; }; static void readLogFile(ShortBatchResult &batchResult, const char *filename) { std::ifstream in(filename, std::ifstream::binary | std::ifstream::in); ShortResultHandler resultHandler(batchResult); xe::TestLogParser parser(&resultHandler); uint8_t buf[1024]; int numRead = 0; for (;;) { in.read((char *)&buf[0], DE_LENGTH_OF_ARRAY(buf)); numRead = (int)in.gcount(); if (numRead <= 0) break; parser.parse(&buf[0], numRead); } in.close(); } class LogFileReader : public de::Thread { public: LogFileReader(ShortBatchResult &batchResult, const char *filename) : m_batchResult(batchResult) , m_filename(filename) { } void run(void) { readLogFile(m_batchResult, m_filename.c_str()); } private: ShortBatchResult &m_batchResult; std::string m_filename; }; static void computeCaseList(vector &cases, const vector &batchResults) { // \todo [2012-07-10 pyry] Do proper case ordering (eg. handle missing cases nicely). set addedCases; for (vector::const_iterator batchIter = batchResults.begin(); batchIter != batchResults.end(); batchIter++) { for (vector::const_iterator caseIter = batchIter->resultHeaders.begin(); caseIter != batchIter->resultHeaders.end(); caseIter++) { if (addedCases.find(caseIter->casePath) == addedCases.end()) { cases.push_back(caseIter->casePath); addedCases.insert(caseIter->casePath); } } } } static void getTestResultHeaders(vector &headers, const vector &batchResults, const char *casePath) { headers.resize(batchResults.size()); for (int ndx = 0; ndx < (int)batchResults.size(); ndx++) { const ShortBatchResult &batchResult = batchResults[ndx]; map::const_iterator resultPos = batchResult.resultMap.find(casePath); if (resultPos != batchResult.resultMap.end()) headers[ndx] = batchResult.resultHeaders[resultPos->second]; else { headers[ndx].casePath = casePath; headers[ndx].caseType = xe::TESTCASETYPE_SELF_VALIDATE; headers[ndx].statusCode = xe::TESTSTATUSCODE_LAST; } } } static const char *getStatusCodeName(xe::TestStatusCode code) { if (code == xe::TESTSTATUSCODE_LAST) return "Missing"; else return xe::getTestStatusCodeName(code); } static bool runCompare(const CommandLine &cmdLine, std::ostream &dst) { vector results; vector batchNames; bool compareOk = true; XE_CHECK(!cmdLine.filenames.empty()); try { // Read in batch results results.resize(cmdLine.filenames.size()); { std::vector> readers; for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++) { readers.push_back( de::SharedPtr(new LogFileReader(results[ndx], cmdLine.filenames[ndx].c_str()))); readers.back()->start(); } for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++) { readers[ndx]->join(); // Use file name as batch name. batchNames.push_back(de::FilePath(cmdLine.filenames[ndx].c_str()).getBaseName()); } } // Compute unified case list. vector caseList; computeCaseList(caseList, results); // Stats. int numCases = (int)caseList.size(); int numEqual = 0; if (cmdLine.outFormat == OUTPUTFORMAT_CSV) { dst << "TestCasePath"; for (vector::const_iterator nameIter = batchNames.begin(); nameIter != batchNames.end(); nameIter++) dst << "," << *nameIter; dst << "\n"; } // Compare cases. for (vector::const_iterator caseIter = caseList.begin(); caseIter != caseList.end(); caseIter++) { const string &caseName = *caseIter; vector headers; bool allEqual = true; getTestResultHeaders(headers, results, caseName.c_str()); for (vector::const_iterator iter = headers.begin() + 1; iter != headers.end(); iter++) { if (iter->statusCode != headers[0].statusCode) { allEqual = false; break; } } if (allEqual) numEqual += 1; if (cmdLine.outMode == OUTPUTMODE_ALL || !allEqual) { if (cmdLine.outFormat == OUTPUTFORMAT_TEXT) { dst << caseName << "\n"; for (int ndx = 0; ndx < (int)headers.size(); ndx++) dst << " " << batchNames[ndx] << ": " << getStatusCodeName(headers[ndx].statusCode) << " (" << headers[ndx].statusDetails << ")\n"; dst << "\n"; } else if (cmdLine.outFormat == OUTPUTFORMAT_CSV) { dst << caseName; for (vector::const_iterator iter = headers.begin(); iter != headers.end(); iter++) dst << "," << (cmdLine.outValue == OUTPUTVALUE_STATUS_CODE ? getStatusCodeName(iter->statusCode) : iter->statusDetails.c_str()); dst << "\n"; } } } compareOk = numEqual == numCases; if (cmdLine.outFormat == OUTPUTFORMAT_TEXT) { dst << " " << numEqual << " / " << numCases << " test case results match.\n"; dst << " Comparison " << (compareOk ? "passed" : "FAILED") << "!\n"; } } catch (const std::exception &e) { printf("%s\n", e.what()); compareOk = false; } return compareOk; } static bool parseCommandLine(CommandLine &cmdLine, int argc, const char *const *argv) { de::cmdline::Parser parser; de::cmdline::CommandLine opts; XE_CHECK(argc >= 1); opt::registerOptions(parser); if (!parser.parse(argc - 1, &argv[1], &opts, std::cerr) || opts.getArgs().empty()) { std::cout << argv[0] << ": [options] [filenames]\n"; parser.help(std::cout); return false; } cmdLine.outFormat = opts.getOption(); cmdLine.outMode = opts.getOption(); cmdLine.outValue = opts.getOption(); cmdLine.filenames = opts.getArgs(); return true; } int main(int argc, const char *const *argv) { CommandLine cmdLine; if (!parseCommandLine(cmdLine, argc, argv)) return -1; try { bool compareOk = runCompare(cmdLine, std::cout); return compareOk ? 0 : -1; } catch (const std::exception &e) { printf("FATAL ERROR: %s\n", e.what()); return -1; } }