blob: e68c74e440b0d560fb565107d389140e9fa83db3 [file] [log] [blame]
//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#include "GPUTestExpectationsParser.h"
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include "common/angleutils.h"
#include "common/debug.h"
#include "common/string_utils.h"
#include "GPUTestConfig.h"
namespace angle
{
namespace
{
enum LineParserStage
{
kLineParserBegin = 0,
kLineParserBugID,
kLineParserConfigs,
kLineParserColon,
kLineParserTestName,
kLineParserEqual,
kLineParserExpectations,
};
enum Token
{
// os
kConfigWinXP = 0,
kConfigWinVista,
kConfigWin7,
kConfigWin8,
kConfigWin10,
kConfigWin,
kConfigMacLeopard,
kConfigMacSnowLeopard,
kConfigMacLion,
kConfigMacMountainLion,
kConfigMacMavericks,
kConfigMacYosemite,
kConfigMacElCapitan,
kConfigMacSierra,
kConfigMacHighSierra,
kConfigMacMojave,
kConfigMac,
kConfigLinux,
kConfigChromeOS,
kConfigAndroid,
// gpu vendor
kConfigNVIDIA,
kConfigAMD,
kConfigIntel,
kConfigVMWare,
// build type
kConfigRelease,
kConfigDebug,
// ANGLE renderer
kConfigD3D9,
kConfigD3D11,
kConfigGLDesktop,
kConfigGLES,
kConfigVulkan,
kConfigSwiftShader,
// Android devices
kConfigNexus5X,
kConfigPixel2,
// GPU devices
kConfigNVIDIAQuadroP400,
// expectation
kExpectationPass,
kExpectationFail,
kExpectationFlaky,
kExpectationTimeout,
kExpectationSkip,
// separator
kSeparatorColon,
kSeparatorEqual,
kNumberOfExactMatchTokens,
// others
kTokenComment,
kTokenWord,
kNumberOfTokens,
};
enum ErrorType
{
kErrorFileIO = 0,
kErrorIllegalEntry,
kErrorInvalidEntry,
kErrorEntryWithExpectationConflicts,
kErrorEntriesOverlap,
kNumberOfErrors,
};
struct TokenInfo
{
const char *name;
GPUTestConfig::Condition condition;
GPUTestExpectationsParser::GPUTestExpectation expectation;
};
const TokenInfo kTokenData[kNumberOfTokens] = {
{"xp", GPUTestConfig::kConditionWinXP},
{"vista", GPUTestConfig::kConditionWinVista},
{"win7", GPUTestConfig::kConditionWin7},
{"win8", GPUTestConfig::kConditionWin8},
{"win10", GPUTestConfig::kConditionWin10},
{"win", GPUTestConfig::kConditionWin},
{"leopard", GPUTestConfig::kConditionMacLeopard},
{"snowleopard", GPUTestConfig::kConditionMacSnowLeopard},
{"lion", GPUTestConfig::kConditionMacLion},
{"mountainlion", GPUTestConfig::kConditionMacMountainLion},
{"mavericks", GPUTestConfig::kConditionMacMavericks},
{"yosemite", GPUTestConfig::kConditionMacYosemite},
{"elcapitan", GPUTestConfig::kConditionMacElCapitan},
{"sierra", GPUTestConfig::kConditionMacSierra},
{"highsierra", GPUTestConfig::kConditionMacHighSierra},
{"mojave", GPUTestConfig::kConditionMacMojave},
{"mac", GPUTestConfig::kConditionMac},
{"linux", GPUTestConfig::kConditionLinux},
{"chromeos"}, // (https://anglebug.com/3363) ChromeOS not supported yet
{"android", GPUTestConfig::kConditionAndroid},
{"nvidia", GPUTestConfig::kConditionNVIDIA},
{"amd", GPUTestConfig::kConditionAMD},
{"intel", GPUTestConfig::kConditionIntel},
{"vmware", GPUTestConfig::kConditionVMWare},
{"release", GPUTestConfig::kConditionRelease},
{"debug", GPUTestConfig::kConditionDebug},
{"d3d9", GPUTestConfig::kConditionD3D9},
{"d3d11", GPUTestConfig::kConditionD3D11},
{"opengl", GPUTestConfig::kConditionGLDesktop},
{"gles", GPUTestConfig::kConditionGLES},
{"vulkan", GPUTestConfig::kConditionVulkan},
{"swiftshader", GPUTestConfig::kConditionSwiftShader},
{"nexus5x", GPUTestConfig::kConditionNexus5X},
{"pixel2", GPUTestConfig::kConditionPixel2},
{"quadrop400", GPUTestConfig::kConditionNVIDIAQuadroP400},
{"pass", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestPass},
{"fail", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestFail},
{"flaky", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestFlaky},
{"timeout", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestTimeout},
{"skip", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestSkip},
{":"}, // kSeparatorColon
{"="}, // kSeparatorEqual
{}, // kNumberOfExactMatchTokens
{}, // kTokenComment
{}, // kTokenWord
};
const char *kErrorMessage[kNumberOfErrors] = {
"file IO failed",
"entry with wrong format",
"entry invalid, likely unimplemented modifiers",
"entry with expectation modifier conflicts",
"two entries' configs overlap",
};
inline bool StartsWithASCII(const std::string &str, const std::string &search, bool caseSensitive)
{
ASSERT(!caseSensitive);
return str.compare(0, search.length(), search) == 0;
}
template <class Char>
inline Char ToLowerASCII(Char c)
{
return (c >= 'A' && c <= 'Z') ? (c + ('a' - 'A')) : c;
}
template <typename Iter>
inline bool DoLowerCaseEqualsASCII(Iter a_begin, Iter a_end, const char *b)
{
for (Iter it = a_begin; it != a_end; ++it, ++b)
{
if (!*b || ToLowerASCII(*it) != *b)
return false;
}
return *b == 0;
}
inline bool LowerCaseEqualsASCII(const std::string &a, const char *b)
{
return DoLowerCaseEqualsASCII(a.begin(), a.end(), b);
}
inline Token ParseToken(const std::string &word)
{
if (StartsWithASCII(word, "//", false))
return kTokenComment;
for (int32_t i = 0; i < kNumberOfExactMatchTokens; ++i)
{
if (LowerCaseEqualsASCII(word, kTokenData[i].name))
return static_cast<Token>(i);
}
return kTokenWord;
}
// reference name can have *.
inline bool NamesMatching(const char *ref, const char *testName)
{
// Find the first * in ref.
const char *firstWildcard = strchr(ref, '*');
// If there are no wildcards, match the strings precisely.
if (firstWildcard == nullptr)
{
return strcmp(ref, testName) == 0;
}
// Otherwise, match up to the wildcard first.
size_t preWildcardLen = firstWildcard - ref;
if (strncmp(ref, testName, preWildcardLen) != 0)
{
return false;
}
const char *postWildcardRef = ref + preWildcardLen + 1;
// As a small optimization, if the wildcard is the last character in ref, accept the match
// already.
if (postWildcardRef[0] == '\0')
{
return true;
}
// Try to match the wildcard with a number of characters.
for (size_t matchSize = 0; testName[matchSize] != '\0'; ++matchSize)
{
if (NamesMatching(postWildcardRef, testName + matchSize))
{
return true;
}
}
return false;
}
} // anonymous namespace
GPUTestExpectationsParser::GPUTestExpectationsParser()
{
// Some sanity check.
ASSERT((static_cast<unsigned int>(kNumberOfTokens)) ==
(sizeof(kTokenData) / sizeof(kTokenData[0])));
ASSERT((static_cast<unsigned int>(kNumberOfErrors)) ==
(sizeof(kErrorMessage) / sizeof(kErrorMessage[0])));
}
GPUTestExpectationsParser::~GPUTestExpectationsParser() = default;
bool GPUTestExpectationsParser::loadTestExpectations(const GPUTestConfig &config,
const std::string &data)
{
mEntries.clear();
mErrorMessages.clear();
std::vector<std::string> lines = SplitString(data, "\n", TRIM_WHITESPACE, SPLIT_WANT_ALL);
bool rt = true;
for (size_t i = 0; i < lines.size(); ++i)
{
if (!parseLine(config, lines[i], i + 1))
rt = false;
}
if (detectConflictsBetweenEntries())
{
mEntries.clear();
rt = false;
}
return rt;
}
bool GPUTestExpectationsParser::loadTestExpectationsFromFile(const GPUTestConfig &config,
const std::string &path)
{
mEntries.clear();
mErrorMessages.clear();
std::string data;
if (!ReadFileToString(path, &data))
{
mErrorMessages.push_back(kErrorMessage[kErrorFileIO]);
return false;
}
return loadTestExpectations(config, data);
}
int32_t GPUTestExpectationsParser::getTestExpectation(const std::string &testName)
{
size_t maxExpectationLen = 0;
GPUTestExpectationEntry *foundEntry = nullptr;
for (size_t i = 0; i < mEntries.size(); ++i)
{
if (NamesMatching(mEntries[i].testName.c_str(), testName.c_str()))
{
size_t expectationLen = mEntries[i].testName.length();
// The longest/most specific matching expectation overrides any others.
if (expectationLen > maxExpectationLen)
{
maxExpectationLen = expectationLen;
foundEntry = &mEntries[i];
}
}
}
if (foundEntry != nullptr)
{
foundEntry->used = true;
return foundEntry->testExpectation;
}
return kGpuTestPass;
}
const std::vector<std::string> &GPUTestExpectationsParser::getErrorMessages() const
{
return mErrorMessages;
}
std::vector<std::string> GPUTestExpectationsParser::getUnusedExpectationsMessages() const
{
std::vector<std::string> messages;
std::vector<GPUTestExpectationsParser::GPUTestExpectationEntry> unusedExpectations =
getUnusedExpectations();
for (size_t i = 0; i < unusedExpectations.size(); ++i)
{
std::string message =
"Line " + ToString(unusedExpectations[i].lineNumber) + ": expectation was unused.";
messages.push_back(message);
}
return messages;
}
bool GPUTestExpectationsParser::parseLine(const GPUTestConfig &config,
const std::string &lineData,
size_t lineNumber)
{
std::vector<std::string> tokens =
SplitString(lineData, kWhitespaceASCII, KEEP_WHITESPACE, SPLIT_WANT_NONEMPTY);
int32_t stage = kLineParserBegin;
GPUTestExpectationEntry entry;
entry.lineNumber = lineNumber;
entry.used = false;
bool skipLine = false;
for (size_t i = 0; i < tokens.size() && !skipLine; ++i)
{
Token token = ParseToken(tokens[i]);
switch (token)
{
case kTokenComment:
skipLine = true;
break;
case kConfigWinXP:
case kConfigWinVista:
case kConfigWin7:
case kConfigWin8:
case kConfigWin10:
case kConfigWin:
case kConfigMacLeopard:
case kConfigMacSnowLeopard:
case kConfigMacLion:
case kConfigMacMountainLion:
case kConfigMacMavericks:
case kConfigMacYosemite:
case kConfigMacElCapitan:
case kConfigMacSierra:
case kConfigMacHighSierra:
case kConfigMacMojave:
case kConfigMac:
case kConfigLinux:
case kConfigChromeOS:
case kConfigAndroid:
case kConfigNVIDIA:
case kConfigAMD:
case kConfigIntel:
case kConfigVMWare:
case kConfigRelease:
case kConfigDebug:
case kConfigD3D9:
case kConfigD3D11:
case kConfigGLDesktop:
case kConfigGLES:
case kConfigVulkan:
case kConfigSwiftShader:
case kConfigNexus5X:
case kConfigPixel2:
case kConfigNVIDIAQuadroP400:
// MODIFIERS, check each condition and add accordingly.
if (stage != kLineParserConfigs && stage != kLineParserBugID)
{
pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
return false;
}
{
bool err = false;
if (!checkTokenCondition(config, err, token, lineNumber))
{
skipLine = true; // Move to the next line without adding this one.
}
if (err)
{
return false;
}
}
if (stage == kLineParserBugID)
{
stage++;
}
break;
case kSeparatorColon:
// :
// If there are no modifiers, move straight to separator colon
if (stage == kLineParserBugID)
{
stage++;
}
if (stage != kLineParserConfigs)
{
pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
return false;
}
stage++;
break;
case kSeparatorEqual:
// =
if (stage != kLineParserTestName)
{
pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
return false;
}
stage++;
break;
case kTokenWord:
// BUG_ID or TEST_NAME
if (stage == kLineParserBegin)
{
// Bug ID is not used for anything; ignore it.
}
else if (stage == kLineParserColon)
{
entry.testName = tokens[i];
}
else
{
pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
return false;
}
stage++;
break;
case kExpectationPass:
case kExpectationFail:
case kExpectationFlaky:
case kExpectationTimeout:
case kExpectationSkip:
// TEST_EXPECTATIONS
if (stage != kLineParserEqual && stage != kLineParserExpectations)
{
pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
return false;
}
if (entry.testExpectation != 0)
{
pushErrorMessage(kErrorMessage[kErrorEntryWithExpectationConflicts],
lineNumber);
return false;
}
entry.testExpectation = kTokenData[token].expectation;
if (stage == kLineParserEqual)
stage++;
break;
default:
ASSERT(false);
break;
}
}
if (stage == kLineParserBegin || skipLine)
{
// The whole line is empty or all comments, or has been skipped to to a condition token.
return true;
}
if (stage == kLineParserExpectations)
{
mEntries.push_back(entry);
return true;
}
pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
return false;
}
bool GPUTestExpectationsParser::checkTokenCondition(const GPUTestConfig &config,
bool &err,
int32_t token,
size_t lineNumber)
{
if (token >= kNumberOfTokens)
{
pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
err = true;
return false;
}
if (kTokenData[token].condition == GPUTestConfig::kConditionNone ||
kTokenData[token].condition >= GPUTestConfig::kNumberOfConditions)
{
pushErrorMessage(kErrorMessage[kErrorInvalidEntry], lineNumber);
// error on any unsupported conditions
err = true;
return false;
}
err = false;
return config.getConditions()[kTokenData[token].condition];
}
bool GPUTestExpectationsParser::detectConflictsBetweenEntries()
{
bool rt = false;
for (size_t i = 0; i < mEntries.size(); ++i)
{
for (size_t j = i + 1; j < mEntries.size(); ++j)
{
if (mEntries[i].testName == mEntries[j].testName)
{
pushErrorMessage(kErrorMessage[kErrorEntriesOverlap], mEntries[i].lineNumber,
mEntries[j].lineNumber);
rt = true;
}
}
}
return rt;
}
std::vector<GPUTestExpectationsParser::GPUTestExpectationEntry>
GPUTestExpectationsParser::getUnusedExpectations() const
{
std::vector<GPUTestExpectationsParser::GPUTestExpectationEntry> unusedExpectations;
for (size_t i = 0; i < mEntries.size(); ++i)
{
if (!mEntries[i].used)
{
unusedExpectations.push_back(mEntries[i]);
}
}
return unusedExpectations;
}
void GPUTestExpectationsParser::pushErrorMessage(const std::string &message, size_t lineNumber)
{
mErrorMessages.push_back("Line " + ToString(lineNumber) + " : " + message.c_str());
}
void GPUTestExpectationsParser::pushErrorMessage(const std::string &message,
size_t entry1LineNumber,
size_t entry2LineNumber)
{
mErrorMessages.push_back("Line " + ToString(entry1LineNumber) + " and " +
ToString(entry2LineNumber) + " : " + message.c_str());
}
GPUTestExpectationsParser::GPUTestExpectationEntry::GPUTestExpectationEntry()
: testExpectation(0), lineNumber(0)
{}
} // namespace angle