| // |
| // Copyright 2019 The ANGLE Project Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| // |
| // TestSuite: |
| // Basic implementation of a test harness in ANGLE. |
| |
| #include "TestSuite.h" |
| |
| #include "common/debug.h" |
| #include "common/platform.h" |
| #include "common/system_utils.h" |
| #include "util/Timer.h" |
| |
| #include <time.h> |
| #include <fstream> |
| |
| #include <gtest/gtest.h> |
| #include <rapidjson/document.h> |
| #include <rapidjson/filewritestream.h> |
| #include <rapidjson/istreamwrapper.h> |
| #include <rapidjson/prettywriter.h> |
| |
| // We directly call into a function to register the parameterized tests. This saves spinning up |
| // a subprocess with a new gtest filter. |
| #include <gtest/../../src/gtest-internal-inl.h> |
| |
| namespace js = rapidjson; |
| |
| namespace angle |
| { |
| namespace |
| { |
| constexpr char kTestTimeoutArg[] = "--test-timeout="; |
| constexpr char kFilterFileArg[] = "--filter-file="; |
| constexpr char kResultFileArg[] = "--results-file="; |
| #if defined(NDEBUG) |
| constexpr int kDefaultTestTimeout = 20; |
| #else |
| constexpr int kDefaultTestTimeout = 60; |
| #endif |
| #if defined(NDEBUG) |
| constexpr int kDefaultBatchTimeout = 240; |
| #else |
| constexpr int kDefaultBatchTimeout = 600; |
| #endif |
| constexpr int kDefaultBatchSize = 1000; |
| |
| const char *ParseFlagValue(const char *flag, const char *argument) |
| { |
| if (strstr(argument, flag) == argument) |
| { |
| return argument + strlen(flag); |
| } |
| |
| return nullptr; |
| } |
| |
| bool ParseIntArg(const char *flag, const char *argument, int *valueOut) |
| { |
| const char *value = ParseFlagValue(flag, argument); |
| if (!value) |
| { |
| return false; |
| } |
| |
| char *end = nullptr; |
| const long longValue = strtol(value, &end, 10); |
| |
| if (*end != '\0') |
| { |
| printf("Error parsing integer flag value.\n"); |
| exit(1); |
| } |
| |
| if (longValue == LONG_MAX || longValue == LONG_MIN || static_cast<int>(longValue) != longValue) |
| { |
| printf("Overflow when parsing integer flag value.\n"); |
| exit(1); |
| } |
| |
| *valueOut = static_cast<int>(longValue); |
| return true; |
| } |
| |
| bool ParseFlag(const char *expected, const char *actual, bool *flagOut) |
| { |
| if (strcmp(expected, actual) == 0) |
| { |
| *flagOut = true; |
| return true; |
| } |
| return false; |
| } |
| |
| bool ParseStringArg(const char *flag, const char *argument, std::string *valueOut) |
| { |
| const char *value = ParseFlagValue(flag, argument); |
| if (!value) |
| { |
| return false; |
| } |
| |
| *valueOut = value; |
| return true; |
| } |
| |
| void DeleteArg(int *argc, char **argv, int argIndex) |
| { |
| // Shift the remainder of the argv list left by one. Note that argv has (*argc + 1) elements, |
| // the last one always being NULL. The following loop moves the trailing NULL element as well. |
| for (int index = argIndex; index < *argc; ++index) |
| { |
| argv[index] = argv[index + 1]; |
| } |
| (*argc)--; |
| } |
| |
| void AddArg(int *argc, char **argv, const char *arg) |
| { |
| // This unsafe const_cast is necessary to work around gtest limitations. |
| argv[*argc] = const_cast<char *>(arg); |
| argv[*argc + 1] = nullptr; |
| (*argc)++; |
| } |
| |
| const char *ResultTypeToString(TestResultType type) |
| { |
| switch (type) |
| { |
| case TestResultType::Crash: |
| return "CRASH"; |
| case TestResultType::Fail: |
| return "FAIL"; |
| case TestResultType::Pass: |
| return "PASS"; |
| case TestResultType::Skip: |
| return "SKIP"; |
| case TestResultType::Timeout: |
| return "TIMEOUT"; |
| case TestResultType::Unknown: |
| return "UNKNOWN"; |
| } |
| } |
| |
| TestResultType GetResultTypeFromString(const std::string &str) |
| { |
| if (str == "CRASH") |
| return TestResultType::Crash; |
| if (str == "FAIL") |
| return TestResultType::Fail; |
| if (str == "PASS") |
| return TestResultType::Pass; |
| if (str == "SKIP") |
| return TestResultType::Skip; |
| if (str == "TIMEOUT") |
| return TestResultType::Timeout; |
| return TestResultType::Unknown; |
| } |
| |
| js::Value ResultTypeToJSString(TestResultType type, js::Document::AllocatorType *allocator) |
| { |
| js::Value jsName; |
| jsName.SetString(ResultTypeToString(type), *allocator); |
| return jsName; |
| } |
| |
| // Writes out a TestResults to the Chromium JSON Test Results format. |
| // https://chromium.googlesource.com/chromium/src.git/+/master/docs/testing/json_test_results_format.md |
| void WriteTestResults(bool interrupted, |
| const TestResults &testResults, |
| const std::string &outputFile, |
| const char *testSuiteName) |
| { |
| time_t ltime; |
| time(<ime); |
| struct tm *timeinfo = gmtime(<ime); |
| ltime = mktime(timeinfo); |
| |
| uint64_t secondsSinceEpoch = static_cast<uint64_t>(ltime); |
| |
| js::Document doc; |
| doc.SetObject(); |
| |
| js::Document::AllocatorType &allocator = doc.GetAllocator(); |
| |
| doc.AddMember("interrupted", interrupted, allocator); |
| doc.AddMember("path_delimiter", ".", allocator); |
| doc.AddMember("version", 3, allocator); |
| doc.AddMember("seconds_since_epoch", secondsSinceEpoch, allocator); |
| |
| js::Value testSuite; |
| testSuite.SetObject(); |
| |
| std::map<TestResultType, uint32_t> counts; |
| |
| for (const auto &resultIter : testResults.results) |
| { |
| const TestIdentifier &id = resultIter.first; |
| const TestResult &result = resultIter.second; |
| |
| js::Value jsResult; |
| jsResult.SetObject(); |
| |
| counts[result.type]++; |
| |
| jsResult.AddMember("expected", "PASS", allocator); |
| jsResult.AddMember("actual", ResultTypeToJSString(result.type, &allocator), allocator); |
| |
| js::Value times; |
| times.SetArray(); |
| times.PushBack(result.elapsedTimeSeconds, allocator); |
| |
| jsResult.AddMember("times", times, allocator); |
| |
| char testName[500]; |
| id.sprintfName(testName); |
| js::Value jsName; |
| jsName.SetString(testName, allocator); |
| |
| testSuite.AddMember(jsName, jsResult, allocator); |
| } |
| |
| js::Value numFailuresByType; |
| numFailuresByType.SetObject(); |
| |
| for (const auto &countIter : counts) |
| { |
| TestResultType type = countIter.first; |
| uint32_t count = countIter.second; |
| |
| js::Value jsCount(count); |
| numFailuresByType.AddMember(ResultTypeToJSString(type, &allocator), jsCount, allocator); |
| } |
| |
| doc.AddMember("num_failures_by_type", numFailuresByType, allocator); |
| |
| js::Value tests; |
| tests.SetObject(); |
| tests.AddMember(js::StringRef(testSuiteName), testSuite, allocator); |
| |
| doc.AddMember("tests", tests, allocator); |
| |
| printf("Writing test results to %s\n", outputFile.c_str()); |
| |
| FILE *fp = fopen(outputFile.c_str(), "w"); |
| |
| constexpr size_t kBufferSize = 0xFFFF; |
| std::vector<char> writeBuffer(kBufferSize); |
| js::FileWriteStream os(fp, writeBuffer.data(), kBufferSize); |
| js::PrettyWriter<js::FileWriteStream> writer(os); |
| doc.Accept(writer); |
| |
| fclose(fp); |
| } |
| |
| void UpdateCurrentTestResult(const testing::TestResult &resultIn, TestResults *resultsOut) |
| { |
| TestResult &resultOut = resultsOut->results[resultsOut->currentTest]; |
| |
| // Note: Crashes and Timeouts are detected by the crash handler and a watchdog thread. |
| if (resultIn.Skipped()) |
| { |
| resultOut.type = TestResultType::Skip; |
| } |
| else if (resultIn.Failed()) |
| { |
| resultOut.type = TestResultType::Fail; |
| } |
| else |
| { |
| resultOut.type = TestResultType::Pass; |
| } |
| |
| resultOut.elapsedTimeSeconds = resultsOut->currentTestTimer.getElapsedTime(); |
| } |
| |
| TestIdentifier GetTestIdentifier(const testing::TestInfo &testInfo) |
| { |
| return {testInfo.test_suite_name(), testInfo.name()}; |
| } |
| |
| class TestEventListener : public testing::EmptyTestEventListener |
| { |
| public: |
| // Note: TestResults is owned by the TestSuite. It should outlive TestEventListener. |
| TestEventListener(const std::string &outputFile, |
| const char *testSuiteName, |
| TestResults *testResults) |
| : mResultsFile(outputFile), mTestSuiteName(testSuiteName), mTestResults(testResults) |
| {} |
| |
| void OnTestStart(const testing::TestInfo &testInfo) override |
| { |
| std::lock_guard<std::mutex> guard(mTestResults->currentTestMutex); |
| mTestResults->currentTest = GetTestIdentifier(testInfo); |
| mTestResults->currentTestTimer.start(); |
| } |
| |
| void OnTestEnd(const testing::TestInfo &testInfo) override |
| { |
| std::lock_guard<std::mutex> guard(mTestResults->currentTestMutex); |
| mTestResults->currentTestTimer.stop(); |
| const testing::TestResult &resultIn = *testInfo.result(); |
| UpdateCurrentTestResult(resultIn, mTestResults); |
| mTestResults->currentTest = TestIdentifier(); |
| } |
| |
| void OnTestProgramEnd(const testing::UnitTest &testProgramInfo) override |
| { |
| std::lock_guard<std::mutex> guard(mTestResults->currentTestMutex); |
| mTestResults->allDone = true; |
| WriteTestResults(false, *mTestResults, mResultsFile, mTestSuiteName); |
| } |
| |
| private: |
| std::string mResultsFile; |
| const char *mTestSuiteName; |
| TestResults *mTestResults; |
| }; |
| |
| bool IsTestDisabled(const testing::TestInfo &testInfo) |
| { |
| return ::strstr(testInfo.name(), "DISABLED_") == testInfo.name(); |
| } |
| |
| using TestIdentifierFilter = std::function<bool(const TestIdentifier &id)>; |
| |
| std::vector<TestIdentifier> FilterTests(std::map<TestIdentifier, FileLine> *fileLinesOut, |
| TestIdentifierFilter filter, |
| bool alsoRunDisabledTests) |
| { |
| std::vector<TestIdentifier> tests; |
| |
| const testing::UnitTest &testProgramInfo = *testing::UnitTest::GetInstance(); |
| for (int suiteIndex = 0; suiteIndex < testProgramInfo.total_test_suite_count(); ++suiteIndex) |
| { |
| const testing::TestSuite &testSuite = *testProgramInfo.GetTestSuite(suiteIndex); |
| for (int testIndex = 0; testIndex < testSuite.total_test_count(); ++testIndex) |
| { |
| const testing::TestInfo &testInfo = *testSuite.GetTestInfo(testIndex); |
| TestIdentifier id = GetTestIdentifier(testInfo); |
| if (filter(id) && (!IsTestDisabled(testInfo) || alsoRunDisabledTests)) |
| { |
| tests.emplace_back(id); |
| |
| if (fileLinesOut) |
| { |
| (*fileLinesOut)[id] = {testInfo.file(), testInfo.line()}; |
| } |
| } |
| } |
| } |
| |
| return tests; |
| } |
| |
| std::vector<TestIdentifier> GetFilteredTests(std::map<TestIdentifier, FileLine> *fileLinesOut, |
| bool alsoRunDisabledTests) |
| { |
| TestIdentifierFilter gtestIDFilter = [](const TestIdentifier &id) { |
| return testing::internal::UnitTestOptions::FilterMatchesTest(id.testSuiteName, id.testName); |
| }; |
| |
| return FilterTests(fileLinesOut, gtestIDFilter, alsoRunDisabledTests); |
| } |
| |
| std::vector<TestIdentifier> GetShardTests(int shardIndex, |
| int shardCount, |
| std::map<TestIdentifier, FileLine> *fileLinesOut, |
| bool alsoRunDisabledTests) |
| { |
| std::vector<TestIdentifier> allTests = GetFilteredTests(fileLinesOut, alsoRunDisabledTests); |
| std::vector<TestIdentifier> shardTests; |
| |
| for (int testIndex = shardIndex; testIndex < static_cast<int>(allTests.size()); |
| testIndex += shardCount) |
| { |
| shardTests.emplace_back(allTests[testIndex]); |
| } |
| |
| return shardTests; |
| } |
| |
| std::string GetTestFilter(const std::vector<TestIdentifier> &tests) |
| { |
| std::stringstream filterStream; |
| |
| filterStream << "--gtest_filter="; |
| |
| for (size_t testIndex = 0; testIndex < tests.size(); ++testIndex) |
| { |
| if (testIndex != 0) |
| { |
| filterStream << ":"; |
| } |
| |
| filterStream << tests[testIndex]; |
| } |
| |
| return filterStream.str(); |
| } |
| |
| std::string ParseTestSuiteName(const char *executable) |
| { |
| const char *baseNameStart = strrchr(executable, GetPathSeparator()); |
| if (!baseNameStart) |
| { |
| baseNameStart = executable; |
| } |
| else |
| { |
| baseNameStart++; |
| } |
| |
| const char *suffix = GetExecutableExtension(); |
| size_t suffixLen = strlen(suffix); |
| if (suffixLen == 0) |
| { |
| return baseNameStart; |
| } |
| |
| const char *baseNameSuffix = strstr(baseNameStart, suffix); |
| ASSERT(baseNameSuffix == (baseNameStart + strlen(baseNameStart) - suffixLen)); |
| return std::string(baseNameStart, baseNameSuffix); |
| } |
| |
| bool GetTestResultsFromJSON(const js::Document &document, TestResults *resultsOut) |
| { |
| if (!document.HasMember("tests") || !document["tests"].IsObject()) |
| { |
| return false; |
| } |
| |
| const js::Value::ConstObject &tests = document["tests"].GetObject(); |
| if (tests.MemberCount() != 1) |
| { |
| return false; |
| } |
| |
| const js::Value::Member &suite = *tests.MemberBegin(); |
| if (!suite.value.IsObject()) |
| { |
| return false; |
| } |
| |
| const js::Value::ConstObject &actual = suite.value.GetObject(); |
| |
| for (auto iter = actual.MemberBegin(); iter != actual.MemberEnd(); ++iter) |
| { |
| // Get test identifier. |
| const js::Value &name = iter->name; |
| if (!name.IsString()) |
| { |
| return false; |
| } |
| |
| TestIdentifier id; |
| if (!TestIdentifier::ParseFromString(name.GetString(), &id)) |
| { |
| return false; |
| } |
| |
| // Get test result. |
| const js::Value &value = iter->value; |
| if (!value.IsObject()) |
| { |
| return false; |
| } |
| |
| const js::Value::ConstObject &obj = value.GetObject(); |
| if (!obj.HasMember("expected") || !obj.HasMember("actual")) |
| { |
| return false; |
| } |
| |
| const js::Value &expected = obj["expected"]; |
| const js::Value &actual = obj["actual"]; |
| |
| if (!expected.IsString() || !actual.IsString()) |
| { |
| return false; |
| } |
| |
| const std::string expectedStr = expected.GetString(); |
| const std::string actualStr = actual.GetString(); |
| |
| if (expectedStr != "PASS") |
| { |
| return false; |
| } |
| |
| TestResultType resultType = GetResultTypeFromString(actualStr); |
| if (resultType == TestResultType::Unknown) |
| { |
| return false; |
| } |
| |
| double elapsedTimeSeconds = 0.0; |
| if (obj.HasMember("times")) |
| { |
| const js::Value × = obj["times"]; |
| if (!times.IsArray()) |
| { |
| return false; |
| } |
| |
| const js::Value::ConstArray ×Array = times.GetArray(); |
| if (timesArray.Size() != 1 || !timesArray[0].IsDouble()) |
| { |
| return false; |
| } |
| |
| elapsedTimeSeconds = timesArray[0].GetDouble(); |
| } |
| |
| TestResult &result = resultsOut->results[id]; |
| result.elapsedTimeSeconds = elapsedTimeSeconds; |
| result.type = resultType; |
| } |
| |
| return true; |
| } |
| |
| bool MergeTestResults(const TestResults &input, TestResults *output) |
| { |
| for (const auto &resultsIter : input.results) |
| { |
| const TestIdentifier &id = resultsIter.first; |
| const TestResult &inputResult = resultsIter.second; |
| TestResult &outputResult = output->results[id]; |
| |
| // This should probably handle situations where a test is run more than once. |
| if (inputResult.type != TestResultType::Skip) |
| { |
| if (outputResult.type != TestResultType::Skip) |
| { |
| printf("Warning: duplicate entry for %s.%s.\n", id.testSuiteName.c_str(), |
| id.testName.c_str()); |
| return false; |
| } |
| |
| outputResult.elapsedTimeSeconds = inputResult.elapsedTimeSeconds; |
| outputResult.type = inputResult.type; |
| } |
| } |
| |
| return true; |
| } |
| |
| void PrintTestOutputSnippet(const TestIdentifier &id, |
| const TestResult &result, |
| const std::string &fullOutput) |
| { |
| std::stringstream nameStream; |
| nameStream << id; |
| std::string fullName = nameStream.str(); |
| |
| size_t runPos = fullOutput.find(std::string("[ RUN ] ") + fullName); |
| if (runPos == std::string::npos) |
| { |
| printf("Cannot locate test output snippet.\n"); |
| return; |
| } |
| |
| size_t endPos = fullOutput.find(std::string("[ FAILED ] ") + fullName, runPos); |
| // Only clip the snippet to the "OK" message if the test really |
| // succeeded. It still might have e.g. crashed after printing it. |
| if (endPos == std::string::npos && result.type == TestResultType::Pass) |
| { |
| endPos = fullOutput.find(std::string("[ OK ] ") + fullName, runPos); |
| } |
| if (endPos != std::string::npos) |
| { |
| size_t newline_pos = fullOutput.find("\n", endPos); |
| if (newline_pos != std::string::npos) |
| endPos = newline_pos + 1; |
| } |
| |
| std::cout << "\n"; |
| if (endPos != std::string::npos) |
| { |
| std::cout << fullOutput.substr(runPos, endPos - runPos); |
| } |
| else |
| { |
| std::cout << fullOutput.substr(runPos); |
| } |
| std::cout << "\n"; |
| } |
| } // namespace |
| |
| TestIdentifier::TestIdentifier() = default; |
| |
| TestIdentifier::TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn) |
| : testSuiteName(suiteNameIn), testName(nameIn) |
| {} |
| |
| TestIdentifier::TestIdentifier(const TestIdentifier &other) = default; |
| |
| TestIdentifier::~TestIdentifier() = default; |
| |
| TestIdentifier &TestIdentifier::operator=(const TestIdentifier &other) = default; |
| |
| void TestIdentifier::sprintfName(char *outBuffer) const |
| { |
| sprintf(outBuffer, "%s.%s", testSuiteName.c_str(), testName.c_str()); |
| } |
| |
| // static |
| bool TestIdentifier::ParseFromString(const std::string &str, TestIdentifier *idOut) |
| { |
| size_t separator = str.find("."); |
| if (separator == std::string::npos) |
| { |
| return false; |
| } |
| |
| idOut->testSuiteName = str.substr(0, separator); |
| idOut->testName = str.substr(separator + 1, str.length() - separator - 1); |
| return true; |
| } |
| |
| TestResults::TestResults() = default; |
| |
| TestResults::~TestResults() = default; |
| |
| ProcessInfo::ProcessInfo() = default; |
| |
| ProcessInfo &ProcessInfo::operator=(ProcessInfo &&rhs) |
| { |
| process = std::move(rhs.process); |
| testsInBatch = std::move(rhs.testsInBatch); |
| resultsFileName = std::move(rhs.resultsFileName); |
| filterFileName = std::move(rhs.filterFileName); |
| commandLine = std::move(rhs.commandLine); |
| return *this; |
| } |
| |
| ProcessInfo::~ProcessInfo() = default; |
| |
| ProcessInfo::ProcessInfo(ProcessInfo &&other) |
| { |
| *this = std::move(other); |
| } |
| |
| TestSuite::TestSuite(int *argc, char **argv) |
| : mShardCount(-1), |
| mShardIndex(-1), |
| mBotMode(false), |
| mBatchSize(kDefaultBatchSize), |
| mCurrentResultCount(0), |
| mTotalResultCount(0), |
| mMaxProcesses(NumberOfProcessors()), |
| mTestTimeout(kDefaultTestTimeout), |
| mBatchTimeout(kDefaultBatchTimeout) |
| { |
| bool hasFilter = false; |
| bool alsoRunDisabledTests = false; |
| |
| #if defined(ANGLE_PLATFORM_WINDOWS) |
| testing::GTEST_FLAG(catch_exceptions) = false; |
| #endif |
| |
| // Note that the crash callback must be owned and not use global constructors. |
| mCrashCallback = [this]() { onCrashOrTimeout(TestResultType::Crash); }; |
| InitCrashHandler(&mCrashCallback); |
| |
| if (*argc <= 0) |
| { |
| printf("Missing test arguments.\n"); |
| exit(1); |
| } |
| |
| mTestExecutableName = argv[0]; |
| mTestSuiteName = ParseTestSuiteName(mTestExecutableName.c_str()); |
| |
| for (int argIndex = 1; argIndex < *argc;) |
| { |
| if (parseSingleArg(argv[argIndex])) |
| { |
| DeleteArg(argc, argv, argIndex); |
| continue; |
| } |
| |
| if (ParseFlagValue("--gtest_filter=", argv[argIndex])) |
| { |
| hasFilter = true; |
| } |
| else |
| { |
| // Don't include disabled tests in test lists unless the user asks for them. |
| if (strcmp("--gtest_also_run_disabled_tests", argv[argIndex]) == 0) |
| { |
| alsoRunDisabledTests = true; |
| } |
| |
| mGoogleTestCommandLineArgs.push_back(argv[argIndex]); |
| } |
| ++argIndex; |
| } |
| |
| if ((mShardIndex >= 0) != (mShardCount > 1)) |
| { |
| printf("Shard index and shard count must be specified together.\n"); |
| exit(1); |
| } |
| |
| if (!mFilterFile.empty()) |
| { |
| if (hasFilter) |
| { |
| printf("Cannot use gtest_filter in conjunction with a filter file.\n"); |
| exit(1); |
| } |
| |
| uint32_t fileSize = 0; |
| if (!GetFileSize(mFilterFile.c_str(), &fileSize)) |
| { |
| printf("Error getting filter file size: %s\n", mFilterFile.c_str()); |
| exit(1); |
| } |
| |
| std::vector<char> fileContents(fileSize + 1, 0); |
| if (!ReadEntireFileToString(mFilterFile.c_str(), fileContents.data(), fileSize)) |
| { |
| printf("Error loading filter file: %s\n", mFilterFile.c_str()); |
| exit(1); |
| } |
| mFilterString.assign(fileContents.data()); |
| |
| if (mFilterString.substr(0, strlen("--gtest_filter=")) != std::string("--gtest_filter=")) |
| { |
| printf("Filter file must start with \"--gtest_filter=\"."); |
| exit(1); |
| } |
| |
| // Note that we only add a filter string if we previously deleted a shader filter file |
| // argument. So we will have space for the new filter string in argv. |
| AddArg(argc, argv, mFilterString.c_str()); |
| } |
| |
| if (mShardCount > 0) |
| { |
| // Call into gtest internals to force parameterized test name registration. |
| testing::internal::UnitTestImpl *impl = testing::internal::GetUnitTestImpl(); |
| impl->RegisterParameterizedTests(); |
| |
| // Initialize internal GoogleTest filter arguments so we can call "FilterMatchesTest". |
| testing::internal::ParseGoogleTestFlagsOnly(argc, argv); |
| |
| mTestQueue = GetShardTests(mShardIndex, mShardCount, &mTestFileLines, alsoRunDisabledTests); |
| mFilterString = GetTestFilter(mTestQueue); |
| |
| // Note that we only add a filter string if we previously deleted a shader index/count |
| // argument. So we will have space for the new filter string in argv. |
| AddArg(argc, argv, mFilterString.c_str()); |
| |
| // Force-re-initialize GoogleTest flags to load the shard filter. |
| testing::internal::ParseGoogleTestFlagsOnly(argc, argv); |
| } |
| |
| testing::InitGoogleTest(argc, argv); |
| |
| if (mShardCount <= 0) |
| { |
| mTestQueue = GetFilteredTests(&mTestFileLines, alsoRunDisabledTests); |
| } |
| |
| mTotalResultCount = mTestQueue.size(); |
| |
| if ((mBotMode || !mResultsDirectory.empty()) && mResultsFile.empty()) |
| { |
| // Create a default output file in bot mode. |
| mResultsFile = "output.json"; |
| } |
| |
| if (!mResultsDirectory.empty()) |
| { |
| std::stringstream resultFileName; |
| resultFileName << mResultsDirectory << GetPathSeparator() << mResultsFile; |
| mResultsFile = resultFileName.str(); |
| } |
| |
| if (!mResultsFile.empty()) |
| { |
| testing::TestEventListeners &listeners = testing::UnitTest::GetInstance()->listeners(); |
| listeners.Append( |
| new TestEventListener(mResultsFile, mTestSuiteName.c_str(), &mTestResults)); |
| |
| std::vector<TestIdentifier> testList = GetFilteredTests(nullptr, alsoRunDisabledTests); |
| |
| for (const TestIdentifier &id : testList) |
| { |
| mTestResults.results[id].type = TestResultType::Skip; |
| } |
| } |
| } |
| |
| TestSuite::~TestSuite() |
| { |
| if (mWatchdogThread.joinable()) |
| { |
| mWatchdogThread.detach(); |
| } |
| TerminateCrashHandler(); |
| } |
| |
| bool TestSuite::parseSingleArg(const char *argument) |
| { |
| return (ParseIntArg("--shard-count=", argument, &mShardCount) || |
| ParseIntArg("--shard-index=", argument, &mShardIndex) || |
| ParseIntArg("--batch-size=", argument, &mBatchSize) || |
| ParseIntArg("--max-processes=", argument, &mMaxProcesses) || |
| ParseIntArg(kTestTimeoutArg, argument, &mTestTimeout) || |
| ParseIntArg("--batch-timeout=", argument, &mBatchTimeout) || |
| ParseStringArg("--results-directory=", argument, &mResultsDirectory) || |
| ParseStringArg(kResultFileArg, argument, &mResultsFile) || |
| ParseStringArg(kFilterFileArg, argument, &mFilterFile) || |
| ParseFlag("--bot-mode", argument, &mBotMode)); |
| } |
| |
| void TestSuite::onCrashOrTimeout(TestResultType crashOrTimeout) |
| { |
| if (mTestResults.currentTest.valid()) |
| { |
| TestResult &result = mTestResults.results[mTestResults.currentTest]; |
| result.type = crashOrTimeout; |
| result.elapsedTimeSeconds = mTestResults.currentTestTimer.getElapsedTime(); |
| } |
| |
| if (mResultsFile.empty()) |
| { |
| printf("No results file specified.\n"); |
| return; |
| } |
| |
| WriteTestResults(true, mTestResults, mResultsFile, mTestSuiteName.c_str()); |
| } |
| |
| bool TestSuite::launchChildTestProcess(const std::vector<TestIdentifier> &testsInBatch) |
| { |
| constexpr uint32_t kMaxPath = 1000; |
| |
| // Create a temporary file to store the test list |
| ProcessInfo processInfo; |
| |
| char filterBuffer[kMaxPath] = {}; |
| if (!CreateTemporaryFile(filterBuffer, kMaxPath)) |
| { |
| std::cerr << "Error creating temporary file for test list.\n"; |
| return false; |
| } |
| processInfo.filterFileName.assign(filterBuffer); |
| |
| std::string filterString = GetTestFilter(testsInBatch); |
| |
| FILE *fp = fopen(processInfo.filterFileName.c_str(), "w"); |
| if (!fp) |
| { |
| std::cerr << "Error opening temporary file for test list.\n"; |
| return false; |
| } |
| fprintf(fp, "%s", filterString.c_str()); |
| fclose(fp); |
| |
| std::string filterFileArg = kFilterFileArg + processInfo.filterFileName; |
| |
| // Create a temporary file to store the test output. |
| char resultsBuffer[kMaxPath] = {}; |
| if (!CreateTemporaryFile(resultsBuffer, kMaxPath)) |
| { |
| std::cerr << "Error creating temporary file for test list.\n"; |
| return false; |
| } |
| processInfo.resultsFileName.assign(resultsBuffer); |
| |
| std::string resultsFileArg = kResultFileArg + processInfo.resultsFileName; |
| |
| // Construct commandline for child process. |
| std::vector<const char *> args; |
| |
| args.push_back(mTestExecutableName.c_str()); |
| args.push_back(filterFileArg.c_str()); |
| args.push_back(resultsFileArg.c_str()); |
| |
| for (const std::string &arg : mGoogleTestCommandLineArgs) |
| { |
| args.push_back(arg.c_str()); |
| } |
| |
| std::string timeoutStr; |
| if (mTestTimeout != kDefaultTestTimeout) |
| { |
| std::stringstream timeoutStream; |
| timeoutStream << kTestTimeoutArg << mTestTimeout; |
| timeoutStr = timeoutStream.str(); |
| args.push_back(timeoutStr.c_str()); |
| } |
| |
| // Launch child process and wait for completion. |
| processInfo.process = LaunchProcess(args, true, true); |
| |
| if (!processInfo.process->started()) |
| { |
| std::cerr << "Error launching child process.\n"; |
| return false; |
| } |
| |
| std::stringstream commandLineStr; |
| for (const char *arg : args) |
| { |
| commandLineStr << arg << " "; |
| } |
| |
| processInfo.commandLine = commandLineStr.str(); |
| processInfo.testsInBatch = testsInBatch; |
| mCurrentProcesses.emplace_back(std::move(processInfo)); |
| return true; |
| } |
| |
| bool TestSuite::finishProcess(ProcessInfo *processInfo) |
| { |
| // Get test results and merge into master list. |
| TestResults batchResults; |
| |
| if (!GetTestResultsFromFile(processInfo->resultsFileName.c_str(), &batchResults)) |
| { |
| std::cerr << "Error reading test results from child process.\n"; |
| return false; |
| } |
| |
| if (!MergeTestResults(batchResults, &mTestResults)) |
| { |
| std::cerr << "Error merging batch test results.\n"; |
| return false; |
| } |
| |
| // Process results and print unexpected errors. |
| for (const auto &resultIter : batchResults.results) |
| { |
| const TestIdentifier &id = resultIter.first; |
| const TestResult &result = resultIter.second; |
| |
| // Skip results aren't procesed since they're added back to the test queue below. |
| if (result.type == TestResultType::Skip) |
| { |
| continue; |
| } |
| |
| mCurrentResultCount++; |
| printf("[%d/%d] %s.%s", mCurrentResultCount, mTotalResultCount, id.testSuiteName.c_str(), |
| id.testName.c_str()); |
| |
| if (result.type == TestResultType::Pass) |
| { |
| printf(" (%g ms)\n", result.elapsedTimeSeconds * 1000.0); |
| } |
| else |
| { |
| printf(" (%s)\n", ResultTypeToString(result.type)); |
| |
| const std::string &batchStdout = processInfo->process->getStdout(); |
| PrintTestOutputSnippet(id, result, batchStdout); |
| } |
| } |
| |
| // On unexpected exit, re-queue any unfinished tests. |
| if (processInfo->process->getExitCode() != 0) |
| { |
| for (const auto &resultIter : batchResults.results) |
| { |
| const TestIdentifier &id = resultIter.first; |
| const TestResult &result = resultIter.second; |
| |
| if (result.type == TestResultType::Skip) |
| { |
| mTestQueue.emplace_back(id); |
| } |
| } |
| } |
| |
| // Clean up any dirty temporary files. |
| for (const std::string &tempFile : {processInfo->filterFileName, processInfo->resultsFileName}) |
| { |
| // Note: we should be aware that this cleanup won't happen if the harness itself crashes. |
| // If this situation comes up in the future we should add crash cleanup to the harness. |
| if (!angle::DeleteFile(tempFile.c_str())) |
| { |
| std::cerr << "Warning: Error cleaning up temp file: " << tempFile << "\n"; |
| } |
| } |
| |
| processInfo->process.reset(); |
| return true; |
| } |
| |
| int TestSuite::run() |
| { |
| // Run tests serially. |
| if (!mBotMode) |
| { |
| startWatchdog(); |
| return RUN_ALL_TESTS(); |
| } |
| |
| constexpr double kIdleMessageTimeout = 5.0; |
| |
| Timer messageTimer; |
| messageTimer.start(); |
| |
| while (!mTestQueue.empty() || !mCurrentProcesses.empty()) |
| { |
| bool progress = false; |
| |
| // Spawn a process if needed and possible. |
| while (static_cast<int>(mCurrentProcesses.size()) < mMaxProcesses && !mTestQueue.empty()) |
| { |
| int numTests = std::min<int>(mTestQueue.size(), mBatchSize); |
| |
| std::vector<TestIdentifier> testsInBatch; |
| testsInBatch.assign(mTestQueue.begin(), mTestQueue.begin() + numTests); |
| mTestQueue.erase(mTestQueue.begin(), mTestQueue.begin() + numTests); |
| |
| if (!launchChildTestProcess(testsInBatch)) |
| { |
| return 1; |
| } |
| |
| progress = true; |
| } |
| |
| // Check for process completion. |
| for (auto processIter = mCurrentProcesses.begin(); processIter != mCurrentProcesses.end();) |
| { |
| ProcessInfo &processInfo = *processIter; |
| if (processInfo.process->finished()) |
| { |
| if (!finishProcess(&processInfo)) |
| { |
| return 1; |
| } |
| processIter = mCurrentProcesses.erase(processIter); |
| progress = true; |
| } |
| else if (processInfo.process->getElapsedTimeSeconds() > mBatchTimeout) |
| { |
| // Terminate the process and record timeouts for the batch. |
| // Because we can't determine which sub-test caused a timeout, record the whole |
| // batch as a timeout failure. Can be improved by using socket message passing. |
| if (!processInfo.process->kill()) |
| { |
| return 1; |
| } |
| for (const TestIdentifier &testIdentifier : processInfo.testsInBatch) |
| { |
| // Because the whole batch failed we can't know how long each test took. |
| mTestResults.results[testIdentifier].type = TestResultType::Timeout; |
| } |
| |
| processIter = mCurrentProcesses.erase(processIter); |
| progress = true; |
| } |
| else |
| { |
| processIter++; |
| } |
| } |
| |
| if (!progress && messageTimer.getElapsedTime() > kIdleMessageTimeout) |
| { |
| for (const ProcessInfo &processInfo : mCurrentProcesses) |
| { |
| double processTime = processInfo.process->getElapsedTimeSeconds(); |
| if (processTime > kIdleMessageTimeout) |
| { |
| printf("Running for %d seconds: %s\n", static_cast<int>(processTime), |
| processInfo.commandLine.c_str()); |
| } |
| } |
| |
| messageTimer.start(); |
| } |
| |
| // Sleep briefly and continue. |
| angle::Sleep(10); |
| } |
| |
| // Dump combined results. |
| WriteTestResults(true, mTestResults, mResultsFile, mTestSuiteName.c_str()); |
| |
| return printFailuresAndReturnCount() == 0; |
| } |
| |
| int TestSuite::printFailuresAndReturnCount() const |
| { |
| std::vector<std::string> failures; |
| |
| for (const auto &resultIter : mTestResults.results) |
| { |
| const TestIdentifier &id = resultIter.first; |
| const TestResult &result = resultIter.second; |
| |
| if (result.type != TestResultType::Pass) |
| { |
| const FileLine &fileLine = mTestFileLines.find(id)->second; |
| |
| std::stringstream failureMessage; |
| failureMessage << id << " (" << fileLine.file << ":" << fileLine.line << ") (" |
| << ResultTypeToString(result.type) << ")"; |
| failures.emplace_back(failureMessage.str()); |
| } |
| } |
| |
| if (failures.empty()) |
| return 0; |
| |
| printf("%zu test%s failed:\n", failures.size(), failures.size() > 1 ? "s" : ""); |
| for (const std::string &failure : failures) |
| { |
| printf(" %s\n", failure.c_str()); |
| } |
| |
| return static_cast<int>(failures.size()); |
| } |
| |
| void TestSuite::startWatchdog() |
| { |
| auto watchdogMain = [this]() { |
| do |
| { |
| { |
| std::lock_guard<std::mutex> guard(mTestResults.currentTestMutex); |
| if (mTestResults.currentTestTimer.getElapsedTime() > |
| static_cast<double>(mTestTimeout)) |
| { |
| onCrashOrTimeout(TestResultType::Timeout); |
| exit(2); |
| } |
| |
| if (mTestResults.allDone) |
| return; |
| } |
| |
| angle::Sleep(1000); |
| } while (true); |
| }; |
| mWatchdogThread = std::thread(watchdogMain); |
| } |
| |
| bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut) |
| { |
| std::ifstream ifs(fileName); |
| if (!ifs.is_open()) |
| { |
| std::cerr << "Error opening " << fileName << "\n"; |
| return false; |
| } |
| |
| js::IStreamWrapper ifsWrapper(ifs); |
| js::Document document; |
| document.ParseStream(ifsWrapper); |
| |
| if (document.HasParseError()) |
| { |
| std::cerr << "Parse error reading JSON document: " << document.GetParseError() << "\n"; |
| return false; |
| } |
| |
| if (!GetTestResultsFromJSON(document, resultsOut)) |
| { |
| std::cerr << "Error getting test results from JSON.\n"; |
| return false; |
| } |
| |
| return true; |
| } |
| |
| const char *TestResultTypeToString(TestResultType type) |
| { |
| switch (type) |
| { |
| case TestResultType::Crash: |
| return "Crash"; |
| case TestResultType::Fail: |
| return "Fail"; |
| case TestResultType::Skip: |
| return "Skip"; |
| case TestResultType::Pass: |
| return "Pass"; |
| case TestResultType::Timeout: |
| return "Timeout"; |
| case TestResultType::Unknown: |
| return "Unknown"; |
| } |
| } |
| } // namespace angle |