dkanus/Acedia
Archived
1
0
Fork 0
Code Issues 1 Pull Requests Releases Wiki Activity

Refactor Testing subsystem

Browse Source
This commit is contained in:
Anton Tarasenko 2020-07-18 02:16:41 +07:00
parent 960e787de7
commit 3849fd5c9d
6 changed files with 1330 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

294
sources/Core/Testing/IssueSummary.uc Normal file
View File

@ -0,0 +1,294 @@
/**
* Class for storing and processing the information about how well testing
* against a certain issue went.
* Copyright 2020 Anton Tarasenko
*------------------------------------------------------------------------------
* This file is part of Acedia.
*
* Acedia is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3 of the License, or
* (at your option) any later version.
*
* Acedia is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Acedia. If not, see <https://www.gnu.org/licenses/>.
*/
class IssueSummary extends AcediaObject;
// Each issue is uniquely identified by these values.
var private class<TestCase> ownerCase;
var private string context;
var private string description;
// Records, in chronological order, results of the tests that were
// run to test this issue.
var private array<byte> successRecords;
private final function byte BoolToByte(bool boolToConvert)
{
if (boolToConvert) return 1;
return 0;
}
/**
* Sets `TestCase`, context and description for the issue,
* tracked in this summary.
*
* Can only be successfully called once, but will fail if passed a `none`
* class reference to `TestCase`.
*
* @param targetCase `TestCase`, in which issue,
* relevant to this summary, is defined.
* @param targetContext Context, in which this issue,
* relevant to this summary, is defined.
* @param targetDescription Description of the issue relevant to
* this summary.
* @return `true` if `TestCase`, context and description were successfully set,
* `false` otherwise.
*/
public final function bool SetIssue(
class<TestCase> targetCase,
string targetContext,
string targetDescription
)
{
if (ownerCase != none) return false;
if (initCase == none) return false;
ownerCase = targetCase;
context = targetContext;
description = targetDescription;
return true;
}
/**
* Returns context for the issue in question.
*
* `TestCase` can be important for both displaying information about testing to
* the user and distinguishing between two different issues with the same
* description and context.
* @see `TestCase` for more information.
*
* @return Test case that tested for relevant issue.
*/
public final function class<TestCase> GetTestCase()
{
return ownerCase;
}
/**
* Returns context for the issue in question.
*
* Context can be important for both displaying information about testing to
* the user and distinguishing between two different issues with
* the same description and in the same `TestCase`.
* @see `TestCase` for more information.
*
* @return Context for relevant issue.
*/
public final function string GetContext()
{
if (ownerCase == none) return "";
return context;
}
/**
* Returns description for the issue in question.
*
* Description of an issue is the main way to distinguish between
* different possibly arising problems.
* Two different issues can have the same description if they are defined
* in different `TestCase`s and/or in different context.
* @see `TestCase` for more information.
*
* @return Description for the issue in question.
*/
public final function string GetDescription()
{
if (ownerCase == none) return "";
return description;
}
/**
* Adds result of another test (success or not) to the records of this summary.
*
* @param success `true` if test was successful and had passed,
* `false` otherwise.
*/
public final function AddTestResult(bool success)
{
successRecords[successRecords.length] = BoolToByte(success);
}
/**
* Returns total amount of test results recorded in caller summary.
* Never a negative value.
*
* @return Amount of tests that were run.
*/
public final function int GetTotalTestsAmount()
{
return successRecords.length;
}
/**
* Returns total amount of recorded successful test results in caller summary.
* Never a negative value.
*
* @return Amount of recorded successfully performed tests for
* the relevant issue.
*/
public final function int GetSuccessfulTestsAmount()
{
local int i;
local int counter;
counter = 0;
for (i = 0; i < successRecords.length; i += 1)
{
if (successRecords[i] > 0) {
counter += 1;
}
}
return counter;
}
/**
* Returns total amount of recorded failed test results in caller summary.
* Never a negative value.
*
* @return Amount of recorded failed tests for the relevant issue.
*/
public final function int GetFailedTestsAmount()
{
return GetTotalTestsAmount() - GetSuccessfulTestsAmount();
}
/**
* Returns total success rate ("amount of successes" / "total amount of tests")
* of recorded test results for relevant issue
* (value between 0 and 1, including boundaries).
*
* If there are no test results recorded - returns `-1`.
*
* @return Success rate of recorded test results for the relevant issue
* Returns values outside [0; 1] segment (specifically, negative values)
* iff no test results at all were recorded.
*/
public final function float GetSuccessRate()
{
local int totalTestsAmount;
totalTestsAmount = GetTotalTestsAmount();
if (totalTestsAmount <= 0) {
return -1;
}
return GetSuccessfulTestsAmount() / totalTestsAmount;
}
/**
* Checks whether all tests recorded in this summary have passed.
*
* @return `true` if all tests for relevant issue have passed,
* `false` otherwise.
*/
public final function bool HasPassedAllTests()
{
return (GetFailedTestsAmount() <= 0);
}
/**
* Returns boolean array of test results: each element recording whether test
* was a success (`>0`) or a failure (`0`).
*
* All results in the array are in a chronological order of arrival.
*
* @return Returns copy of boolean array of recorded test results.
*/
public final function array<byte> GetTestRecords()
{
return successRecords;
}
/**
* Returns index numbers (starting from 1, not 0) of tests that ended in
* a success, while performed for the same test case, context and issue.
* So if tests went: [success, success, failure, success, failure],
* method will return: [1, 2, 4].
*
* All results in the array are in a chronological order of arrival.
*
* @return index numbers of successful tests.
*/
public final function array<int> GetSuccessfulTests()
{
local int i;
local array<int> result;
for (i = 0; i < successRecords.length; i += 1)
{
if (successRecords[i] > 0) {
result[result.length] = i + 1;
}
}
return result;
}
/**
* Returns index numbers (starting from 1, not 0) of tests that ended in
* a failure, while performed for the same test case, context and issue.
* So if tests went: [success, success, failure, success, failure],
* method will return: [3, 5].
*
* All results in the array are in a chronological order of arrival.
*
* @return index numbers of successful tests.
*/
public final function array<int> GetFailedTests()
{
local int i;
local array<int> result;
for (i = 0; i < successRecords.length; i += 1)
{
if (successRecords[i] == 0) {
result[result.length] = i + 1;
}
}
return result;
}
/**
* Returns a formatted text representation of the caller `IssueSummary`
* in a following format:
* "{$text_default <issue_description>} {$text_subtle [<failed_test_numbers>]}"
*
* @return Formatted string with text representation of the
* caller `IssueSummary`.
*/
public final function string ToString()
{
local int i;
local string result;
local array<int> failedTests;
result = "{$text_default" @ GetDescription() $ "}";
if (GetFailedTestsAmount() <= 0) {
return result;
}
result @= "{$text_subtle [";
failedTests = GetFailedTests();
for (i = 0; i < failedTests.length; i += 1)
{
if (i < failedTests.length - 1) {
result $= string(failedTests[i]) $ ", ";
}
else {
result $= string(failedTests[i]);
}
}
return (result $ "]");
}
defaultproperties
{
}

66
sources/Core/Testing/Service/TestingEvents.uc Normal file
View File

@ -0,0 +1,66 @@
/**
* Event generator for events related to testing.
* Copyright 2020 Anton Tarasenko
*------------------------------------------------------------------------------
* This file is part of Acedia.
*
* Acedia is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3 of the License, or
* (at your option) any later version.
*
* Acedia is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Acedia. If not, see <https://www.gnu.org/licenses/>.
*/
class TestingEvents extends Events
abstract;
static function CallTestingBegan(array< class<TestCase> > testQueue)
{
local int i;
local array< class<Listener> > listeners;
listeners = GetListeners();
for (i = 0; i < listeners.length; i += 1)
{
class<TestingListenerBase>(listeners[i])
.static.TestingBegan(testQueue);
}
}
static function CallCaseTested(
class<TestCase> testedCase,
TestCaseSummary result)
{
local int i;
local array< class<Listener> > listeners;
listeners = GetListeners();
for (i = 0; i < listeners.length; i += 1)
{
class<TestingListenerBase>(listeners[i])
.static.CaseTested(testedCase, result);
}
}
static function CallTestingEnded(
array< class<TestCase> > testQueue,
array<TestCaseSummary> results)
{
local int i;
local array< class<Listener> > listeners;
listeners = GetListeners();
for (i = 0; i < listeners.length; i += 1)
{
class<TestingListenerBase>(listeners[i])
.static.TestingEnded(testQueue, results);
}
}
defaultproperties
{
relatedListener = class'TestingListenerBase'
}

34
sources/Core/Testing/Service/TestingListenerBase.uc Normal file
View File

@ -0,0 +1,34 @@
/**
* Listener for events related to testing.
* Copyright 2020 Anton Tarasenko
*------------------------------------------------------------------------------
* This file is part of Acedia.
*
* Acedia is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3 of the License, or
* (at your option) any later version.
*
* Acedia is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Acedia. If not, see <https://www.gnu.org/licenses/>.
*/
class TestingListenerBase extends Listener
abstract;
static function TestingBegan(array< class<TestCase> > testQueue) {}
static function CaseTested(class<TestCase> testQueue, TestCaseSummary result) {}
static function TestingEnded(
array< class<TestCase> > testedCase,
array<TestCaseSummary> results) {}
defaultproperties
{
relatedEvents = class'TestingEvents'
}

253
sources/Core/Testing/Service/TestingService.uc Normal file
View File

@ -0,0 +1,253 @@
/**
* This service allows to separate running separate `TestCase`s in separate
* ticks, which helps to avoid hang ups or false infinite loop detection.
* Copyright 2020 Anton Tarasenko
*------------------------------------------------------------------------------
* This file is part of Acedia.
*
* Acedia is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3 of the License, or
* (at your option) any later version.
*
* Acedia is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Acedia. If not, see <https://www.gnu.org/licenses/>.
*/
class TestingService extends Service
config(AcediaSystem);
// All test cases, loaded from all available packages.
// Always use `default` copy of this array.
var private array< class<TestCase> > registeredTestCases;
// Will be `true` if we have yet more tests to run
// (either during current or following ticks)
var private bool runningTests;
// Queue with all test cases for the current/next testing
var private array< class<TestCase> > testCasesToRun;
// Track which test case we need to execute during next tick
var private int nextTestCase;
// Record test results during the last test run here.
// After testing has finished - copy them into it's default value
// `default.summarizedResults` to be available even after `TestingService`
// shuts down.
var private array<TestCaseSummary> summarizedResults;
// Configuration variables that tell Acedia what tests to run
// (and whether to run any at all) on start up.
var public config const bool runTestsOnStartUp;
var public config const bool filterTestsByName;
var public config const bool filterTestsByGroup;
var public config const string requiredName;
var public config const string requiredGroup;
// Shortcut to `TestingEvents`, so that we don't have to write
// class'TestingEvents' every time.
var const class<TestingEvents> events;
/**
* Registers another `TestCase` class for later testing.
*
* @return `true` if registration was successful.
*/
public final static function bool RegisterTestCase(class<TestCase> newTestCase)
{
local int i;
if (newTestCase == none) return false;
for (i = 0; i < default.registeredTestCases.length; i += 1)
{
if (default.registeredTestCases[i] == newTestCase) {
return false;
}
// Warn if there are test cases with the same name and group
if ( !(default.registeredTestCases[i].static.GetGroup()
~= newTestCase.static.GetGroup())) {
continue;
}
if ( !(default.registeredTestCases[i].static.GetName()
~= newTestCase.static.GetName())) {
continue;
}
default._.logger.Warning("Two different test cases with name \""
$ newTestCase.static.GetName() $ "\" in the same group \""
$ newTestCase.static.GetGroup() $ "\"have been registered:"
@ "\"" $ string(newTestCase) $ "\" and \""
$ string(default.registeredTestCases[i])
$ "\". This can lead to issues and it is not something you can fix,"
@ "- contact developers of the relevant packages.");
}
default.registeredTestCases[default.registeredTestCases.length] =
newTestCase;
return true;
}
/**
* Checks whether service is still in the process of running tests.
*
* @return `true` if there are still some tests that are scheduled, but
* were not yet ran and `false` otherwise.
*/
public final static function bool IsRunningTests()
{
local TestingService myInstance;
myInstance = TestingService(class'TestingService'.static.GetInstance());
if (myInstance == none) return false;
return myInstance.runningTests;
}
/**
* Returns the results of the last tests run.
*
* If no tests were run - returns an empty array.
*
* @return Results of the last tests run.
*/
public final static function array<TestCaseSummary> GetLastResults()
{
return default.summarizedResults;
}
/**
* Adds all tests to the testing queue.
*
* To actually run them use `Run()`.
* To only run certain tests, - filter them by `FilterByName()`
* and `FilterByGroup()`
*
* Will do nothing if service is already in the process of testing
* (`IsRunningTests() == true`).
*
* @return Caller `TestService` to allow for method chaining.
*/
public final function TestingService PrepareTests()
{
if (runningTests) {
return self;
}
testCasesToRun = default.registeredTestCases;
return self;
}
/**
* Filters tests in current queue to only those that have a specific name.
* Should be used after `PrepareTests()` call, but before `Run()`.
*
* Will do nothing if service is already in the process of testing
* (`IsRunningTests() == true`).
*
* @return Caller `TestService` to allow for method chaining.
*/
public final function TestingService FilterByName(string caseName)
{
local int i;
local array< class<TestCase> > preFiltered;
if (runningTests) {
return self;
}
preFiltered = testCasesToRun;
testCasesToRun.length = 0;
for (i = 0; i < preFiltered.length; i += 1)
{
if (preFiltered[i].static.GetName() ~= caseName) {
testCasesToRun[testCasesToRun.length] = preFiltered[i];
}
}
return self;
}
/**
* Filters tests in current queue to only those that belong to
* a specific group. Should be used after `PrepareTests()` call,
* but before `Run()`.
*
* Will do nothing if service is already in the process of testing
* (`IsRunningTests() == true`).
*
* @return Caller `TestService` to allow for method chaining.
*/
public final function TestingService FilterByGroup(string caseGroup)
{
local int i;
local array< class<TestCase> > preFiltered;
if (runningTests) {
return self;
}
preFiltered = testCasesToRun;
testCasesToRun.length = 0;
for (i = 0; i < preFiltered.length; i += 1)
{
if (preFiltered[i].static.GetGroup() ~= caseGroup) {
testCasesToRun[testCasesToRun.length] = preFiltered[i];
}
}
return self;
}
/**
* Makes `TestingService` run all tests in a current queue.
*
* Queue musty be build before hand: start with `PrepareTests()` call and
* optionally use `FilterByName()` / `FilterByGroup()` before
* `Run()` method call.
*
* @return `false` if service is already performing the testing
* and `true` otherwise. Note that `TestingService` might be inactive even
* after `Run()` call that returns `true`, if the testing queue was empty.
*/
public final function bool Run()
{
if (runningTests) {
return false;
}
nextTestCase = 0;
runningTests = true;
summarizedResults.length = 0;
events.static.CallTestingBegan(testCasesToRun);
if (testCasesToRun.length <= 0) {
runningTests = false;
events.static.CallTestingEnded(testCasesToRun, summarizedResults);
}
return true;
}
private final function DoTestingStep()
{
local TestCaseSummary newResult;
if (nextTestCase >= testCasesToRun.length)
{
runningTests = false;
default.summarizedResults = summarizedResults;
events.static.CallTestingEnded(testCasesToRun, summarizedResults);
return;
}
testCasesToRun[nextTestCase].static.PerformTests();
newResult = testCasesToRun[nextTestCase].static.GetSummary();
events.static.CallCaseTested(testCasesToRun[nextTestCase], newResult);
summarizedResults[summarizedResults.length] = newResult;
nextTestCase += 1;
}
event Tick(float delta)
{
// This will destroy us on the next tick after we were
// either created or finished performing tests
if (!runningTests) {
Destroy();
return;
}
DoTestingStep();
}
defaultproperties
{
runTestsOnStartUp = false
events = class'TestingEvents'
}

323
sources/Core/Testing/TestCase.uc
View File

@ -1,13 +1,8 @@
/** /**
* Base class aimed to contain sets of unit tests for various components of * Base class aimed to contain sets of tests for various components of
* Acedia and it's features. * Acedia and it's features.
* Currently provides bare-bones testing functions that check boolean * Neither this class, nor it's children aren't supposed to
* variables for true/false and objects for whether they're `none` or not. * be instantiated.
* Tests:
* ~ can be grouped by their "context",
* describing what they are testing;
* ~ test (or several tests) can be assigned an error message,
* describing what exactly went wrong.
* Copyright 2020 Anton Tarasenko * Copyright 2020 Anton Tarasenko
*------------------------------------------------------------------------------ *------------------------------------------------------------------------------
* This file is part of Acedia. * This file is part of Acedia.
@ -30,202 +25,165 @@ class TestCase extends AcediaObject
// Name by which this set of unit tests can be referred to. // Name by which this set of unit tests can be referred to.
var protected const string caseName; var protected const string caseName;
// Name of group to which this set of unit tests belong.
var protected const string caseGroup;
// Information about how well testing went for a particular context,
// i.e. subsets of tests for a particular functionality.
struct ContextSummary
{
// Text, human-readable description of the purpose of
// tests in this context.
var string description;
// `false` if at least one test failed, `true` otherwise.
var bool passed;
// How many test were performed.
var int testsPerformed;
// How many tests failed.
var int testsFailed;
// Error messages generated by failed tests.
var array<string> errors;
};
// Collection of summaries for all contexts defined by the user so far.
struct Summary
{
var bool passed;
var array<ContextSummary> contextSummaries;
};
// Has function for defining context (`Context()`) been called.
var private bool userDefinedContext;
// Were all tests performed? // Were all tests performed?
var private bool finishedTests; var private bool finishedTests;
// Context under which we are currently performing our tests.
var private string currentContext;
// Error message that will be generated if some test will fail now. // Error message that will be generated if some test will fail now.
var private string currentErrorMessage; var private string currentIssue;
// Store complete summary here. // Summary where we are recording results of all our tests.
var private Summary currentSummary; var private TestCaseSummary currentSummary;
// For quick access store current context's summary here and update it in
// `currentSummary` once done.
var private ContextSummary activeContextSummary;
// Call this function to define a context for subsequent test /**
// (until another call). * Sets context for any tests that will follow this call (but before the next
public final static function Context(string description) * `Context()` call).
*
* Context is supposed to be a short description about what
* exactly you are testing. When reporting failed tests, - failures will be
* grouped up by a context.
*
* Changing current context will also reset current issue, to set it up
* use `Issue()` method.
*
* @param context Context for the following tests.
*/
public final static function Context(string context)
{ {
if ( default.userDefinedContext default.currentContext = context;
|| default.activeContextSummary.testsPerformed > 0) default.currentIssue = ""; // Reset issue.
{
UpdateContextSummary(default.activeContextSummary);
}
default.userDefinedContext = true;
default.activeContextSummary = GetContextSummary(description);
default.currentErrorMessage = "";
} }
// Call this function to define an error message for tests that // Call this function to define an error message for tests that
// would fail after it. // would fail after it.
// Message is reset by another call of `Issue()` or // Message is reset by another call of `Issue()` or
// by changing the context via `Context()`. // by changing the context via `Context()`.
public final static function Issue(string errorMessage) /**
* Changes an issue that any following tests (but before the next `Issue()` or
* `Context()` call) will test for.
*
* Issue is the message that will be displayed to the user if any relevant
* tests have failed.
*
* NOTE: Current issue will be reset by any `Context()` call.
*
* @param issue Issue that following tests will test for.
*/
public final static function Issue(string issue)
{ {
default.currentErrorMessage = errorMessage; default.currentIssue = issue;
} }
// All tests to be performed can be placed in this function, // Following functions provide simple test primitives
// along with appropriate calls to `Context()` and `Issue()`.
// For an example see class `TEST_JSON`.
protected static function TESTS(){}
// Following functions provide simple test primitives, /**
* This call will record either one success or one failure for the caller
* `TestCase` class, depending on passed `bool` argument.
*
* @param result Your test's result as a `bool` value: `true` will record a
* success and `false` a failure.
*/
public final static function TEST_ExpectTrue(bool result) public final static function TEST_ExpectTrue(bool result)
{ {
RecordTestResult(result, default.currentErrorMessage); RecordTestResult(result);
} }
/**
* This call will record either one success or one failure for the caller
* `TestCase` class, depending on passed `bool` argument.
*
* @param result Your test's result as a `bool` value: `false` will result in
* recording a success and `true` in a failure.
*/
public final static function TEST_ExpectFalse(bool result) public final static function TEST_ExpectFalse(bool result)
{ {
RecordTestResult(!result, default.currentErrorMessage); RecordTestResult(!result);
} }
/**
* This call will record either one success or one failure for the caller
* `TestCase` class, depending on passed `Object` argument.
*
* @param result Your test's result as an `Object` value: `none` will result
* in recording success and any non-`none` value in failure.
*/
public final static function TEST_ExpectNone(Object object) public final static function TEST_ExpectNone(Object object)
{ {
RecordTestResult(object == none, default.currentErrorMessage); RecordTestResult(object == none);
} }
/**
* This call will record either one success or one failure for the caller
* `TestCase` class, depending on passed `Object` argument.
*
* @param result Your test's result as an `Object` value: any non-`none`
* value will result in recording success and `none` in failure.
*/
public final static function TEST_ExpectNotNone(Object object) public final static function TEST_ExpectNotNone(Object object)
{ {
RecordTestResult(object != none, default.currentErrorMessage); RecordTestResult(object != none);
} }
// Returns the summary of how testing went. // Records (in current context summary) that another test was performed and
public final static function Summary GetSummary() // succeeded/failed, along with given error message.
private final static function RecordTestResult(bool isSuccessful)
{ {
if (default.finishedTests) return;
if (default.currentSummary == none) return;
default.currentSummary.AddTestResult( default.currentContext,
default.currentIssue,
isSuccessful);
}
/**
* Once testing has finished returns compiled results as a
* `TestCaseSummary` object.
*
* @return `TestCaseSummary` with compiled results if the testing has finished
* and `none` otherwise.
*/
public final static function TestCaseSummary GetSummary()
{
if (!default.finishedTests) {
return none;
}
return default.currentSummary; return default.currentSummary;
} }
// Name by which this set of unit tests can be referred to. /**
* Checks whether this `TestCase` has already finished running all it's tests.
* Finished testing means a prepared `TestCaseSummary` is available
* (by `GetSummary()` method).
*
* @return `true` if this test case already did the testing
* and `false` otherwise.
*/
public final static function bool HasFinishedTesting()
{
return default.finishedTests;
}
/**
* Returns name of this `TestCase`.
*
* @return Name of this `TestCase`.
*/
public final static function string GetName() public final static function string GetName()
{ {
return default.caseName; return default.caseName;
} }
// Creates brand new summary for context with a given description, /**
// marked as "passed" and zero tests done. * Returns group name of this `TestCase`.
private final static function ContextSummary NewContextSummary *
( * @return Group name of this `TestCase`.
string description */
) public final static function string GetGroup()
{ {
local ContextSummary newSummary; return default.caseGroup;
newSummary.passed = true;
newSummary.description = description;
newSummary.testsPerformed = 0;
newSummary.testsFailed = 0;
newSummary.errors.length = 0;
return newSummary;
}
// Returns index of summary with given description
// in our records (`currentSummary`).
// Return `-1` if there is no context with such description.
private final static function int GetContextSummaryIndex(string description)
{
local int i;
for (i = 0; i < default.currentSummary.contextSummaries.length; i += 1)
{
if ( default.currentSummary.contextSummaries[i].description
~= description)
{
return i;
}
}
return -1;
}
// Returns index summary with given description
// in our records.
// Return new context summary if there is no context with such description.
private final static function ContextSummary GetContextSummary
(
string description
)
{
local int index;
if (default.activeContextSummary.description ~= description)
{
return default.activeContextSummary;
}
index = GetContextSummaryIndex(description);
if (index < 0)
{
return NewContextSummary(description);
}
return default.currentSummary.contextSummaries[index];
}
// Rewrites summary (with the same name as a given summary)
// in `currentSummary` records.
// If there's no such record - adds a new one.
private final static function UpdateContextSummary
(
ContextSummary relevantSummary
)
{
local int index;
index = GetContextSummaryIndex(relevantSummary.description);
if (index < 0)
{
index = default.currentSummary.contextSummaries.length;
}
default.currentSummary.contextSummaries[index] = relevantSummary;
}
// Records (in current context summary) that another test was performed and
// succeeded/failed, along with given error message.
private final static function RecordTestResult
(
bool isSuccessful,
string errorMessage
)
{
local int i;
local int errorsAmount;
if (default.finishedTests) return;
default.activeContextSummary.testsPerformed += 1;
if (isSuccessful) return;
default.currentSummary.passed = false;
default.activeContextSummary.passed = false;
default.activeContextSummary.testsFailed += 1;
errorsAmount = default.activeContextSummary.errors.length;
for (i = 0; i < errorsAmount; i += 1)
{
if (default.activeContextSummary.errors[i] ~= errorMessage)
{
return;
}
}
default.activeContextSummary.errors[errorsAmount] = errorMessage;
} }
// Calling this function will perform unit tests defined in `TESTS()` // Calling this function will perform unit tests defined in `TESTS()`
@ -233,31 +191,36 @@ private final static function RecordTestResult
// obtainable through `GetSummary()` function. // obtainable through `GetSummary()` function.
// Returns `true` if all tests have successfully passed // Returns `true` if all tests have successfully passed
// and `false` otherwise. // and `false` otherwise.
/**
* Performs all tests for this `TestCase`.
* Guaranteed to be done after this finishes.
*
* @return `true` if all tests have finished successfully
* and `false` otherwise.
*/
public final static function bool PerformTests() public final static function bool PerformTests()
{ {
default.finishedTests = false; default.finishedTests = false;
default.userDefinedContext = false; _().memory.Free(default.currentSummary);
default.currentSummary.passed = true; default.currentSummary = new class'TestCaseSummary';
default.currentSummary.contextSummaries.length = 0; default.currentSummary.Initialize(default.class);
default.activeContextSummary = NewContextSummary("");
TESTS(); TESTS();
UpdateContextSummary(default.activeContextSummary);
default.finishedTests = true; default.finishedTests = true;
return default.currentSummary.passed; return default.currentSummary.HasPassedAllTests();
} }
// TODO: Support for testing in stages to avoid infinite loop crashes. /**
// TODO: Add support for test scening: grabbing pawns, placing them, waiting. * Any tests that your `TestCase` class needs to perform should be put in
// TODO: Expand scening support: triggering functions on client, moving. * this function.
// TODO: Expand scening support: zed spawning, aggro setting. * To separate tests into groups it's recommended (as a style
// TODO: Expand scening support: function calls (i.e. for CashToss), * consideration) to put them in separate function calls and give these
// testing `FixDoshSpam` feature. * functions names starting with "Test_". They can have further folded
// TODO: Expand scening support: lag detection. * functions with prefix "SubTest_", which can contain "SubSubTest_", etc..
// TODO: Expand scening support: test `FixZedTime`. */
// TODO: Expand scening support: aiming shooting, detecting damage. protected static function TESTS(){}
// TODO: Expand scening support: testing `FixFFHack`.
// TODO: Testing infinite nade (partially), ammo selling, dualies cost.
defaultproperties defaultproperties
{ {
caseName = "" caseName = ""
caseGroup = ""
} }

540
sources/Core/Testing/TestCaseSummary.uc Normal file
View File

@ -0,0 +1,540 @@
/**
* Class for storing and processing the information about how well testing
* for a certain `TestCase` went. That information is stored as
* a collection of `IssueSummary`s, that can be accessed all at once
* or by their context.
* `TestCaseSummary` must be initialized for some `TestCase` before it can
* be used for anything (unlike `IssueSummary`).
* Copyright 2020 Anton Tarasenko
*------------------------------------------------------------------------------
* This file is part of Acedia.
*
* Acedia is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3 of the License, or
* (at your option) any later version.
*
* Acedia is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Acedia. If not, see <https://www.gnu.org/licenses/>.
*/
class TestCaseSummary extends AcediaObject;
// Case for which this summary was initialized.
// `none` if it was not.
var private class<TestCase> ownerCase;
/**
*
* We will store issue summaries for different contexts separately.
* INVARIANT: any function that adds records to `contextRecords`
* must guarantee that:
* 1. No two distinct records will have the same `context`;
* 2. All the `IssueSummary`s in `issueSummaries` array have different
* issue descriptions.
* Comparisons of `string`s for two above conditions are case-insensitive.
*/
struct ContextRecord
{
var string context;
var array<IssueSummary> issueSummaries;
};
var private array<ContextRecord> contextRecords;
// String literals used for displaying array of test case summaries
var private const string indent;
var private const string reportHeader;
var private const string reportSuccessfulEnding;
var private const string reportUnsuccessfulEnding;
/**
* Initializes caller summary for given `TestCase` class.
* Can only be successfully done once, but will fail if
* passed a `none` reference.
*
* @param targetCase `TestCase` class for which this summary will be
* recording test results.
* @return `true` if initialization was successful and `false otherwise
* (either summary already initialized or passed reference is `none`).
*/
public final function bool Initialize(class<TestCase> targetCase)
{
if (ownerCase != none) return false;
if (targetCase == none) return false;
ownerCase = targetCase;
return true;
}
/**
* Returns index of a context record with a given description
* (`context`) in `contextRecords`.
* Creates one if missing. Never fails.
*
* @param context Context that desired record must match.
* @return Index of the context record that matches `context`.
* Returned index is always valid.
*/
private final function int TouchContext(string context)
{
local int i;
local ContextRecord newRecord;
// Try to find existing record with given context description
for (i = 0; i < contextRecords.length; i += 1)
{
if (context ~= contextRecords[i].context) {
return i;
}
}
// If there is none - make a new one
newRecord.context = context;
contextRecords[contextRecords.length] = newRecord;
return (contextRecords.length - 1);
}
/**
* Finds indices of a context record and an `IssueSummary` in
* a nested array that have matching `context`
* and `issueDescription`.
* Creates records and/or `IssueSummary` if missing. Never fails.
*
* @param context Context description that
* desired record must match.
* @param issueDescription Issue description that
* desired `IssueSummary`must match.
* @param recordIndex Index of the context record that matches
* `context` description will be recorded here.
* Returned value is always valid. Passed value is discarded.
* @param recordIndex Index of the `IssueSummary` that matches
* `issueDescription` description will be recorded here.
* Returned value is always valid. Passed value is discarded.
*/
private final function TouchIssue(
string context,
string issueDescription,
out int recordIndex,
out int issueIndex
)
{
local int i;
local array<IssueSummary> issueSummaries;
recordIndex = TouchContext(context);
issueSummaries = contextRecords[recordIndex].issueSummaries;
// Try to find existing issue summary with a given description
for (i = 0; i < issueSummaries.length; i += 1)
{
if (issueSummaries[i] == none) continue;
if (issueDescription ~= issueSummaries[i].GetDescription())
{
issueIndex = i;
return;
}
}
// If there is none - add a new one
issueIndex = issueSummaries.length;
issueSummaries[issueIndex] = new class'IssueSummary';
issueSummaries[issueIndex].SetIssue(ownerCase, context, issueDescription);
contextRecords[recordIndex].issueSummaries = issueSummaries;
}
/**
* Checks if caller summary was correctly initialized.
*
* @return `true` if summary was correctly initialized and `false` otherwise.
*/
public final function bool IsInitialized()
{
return (ownerCase != none);
}
/**
* Adds result of another test (success or not) to the records of this summary.
*
* @param context Context under which test was performed.
* @param issueDescription Description of issue,
* for which test was performed.
* @param success `true` if test was successful and had passed,
* `false` otherwise.
*/
public final function AddTestResult(
string context,
string issueDescription,
bool success
)
{
local int recordIndex, issueIndex;
TouchIssue(context, issueDescription, recordIndex, issueIndex);
contextRecords[recordIndex]
.issueSummaries[issueIndex]
.AddTestResult(success);
}
/**
* Returns all contexts, for which caller summary has any records of tests
* being performed.
*
* To check if particular context exists you can use `DoesContextExists()`.
*
* @return Array of `string`s, each representing one of the contexts,
* used in tests.
* Guarantees no duplicates (equality without accounting for case).
*/
public final function array<string> GetContexts()
{
local int i;
local array<string> result;
for (i = 0; i < contextRecords.length; i += 1) {
result[result.length] = contextRecords[i].context;
}
return result;
}
/**
* Checks if given context has any records about performing tests
* (whether they ended in success or a failure) under it.
*
* To get an array of all existing contexts use `GetContexts()`.
*
* @param context A context to check for existing in records.
* @return `true` if there was a record about a test being performed under
* a given context and `false` otherwise.
*/
public final function bool DoesContextExists(string context)
{
local int i;
for (i = 0; i < contextRecords.length; i += 1)
{
if (contextRecords[i].context ~= context) {
return true;
}
}
return false;
}
/**
* `IssueSummary`s for every issue that was tested and recorded in
* the caller `TestCaseSummary`.
*
* @return Array of `IssueSummary`s for every tested and recorded issue.
*/
public final function array<IssueSummary> GetIssueSummaries()
{
local int i, j;
local array<IssueSummary> recordedSummaries;
local array<IssueSummary> result;
for (i = 0; i < contextRecords.length; i += 1)
{
recordedSummaries = contextRecords[i].issueSummaries;
for (j = 0; j < recordedSummaries.length; j += 1) {
result[result.length] = recordedSummaries[j];
}
}
return result;
}
/**
* Returns `IssueSummary`s for every issue that was tested under
* a given context and recorded in caller `TestCaseSummary`.
*
* @param context Context under which issues of interest were tested.
* @return Array of `IssueSummary`s for every issue that was tested under
* given context.
*/
public final function array<IssueSummary> GetIssueSummariesForContext(
string context
)
{
local int i;
local array<IssueSummary> emptyResult;
for (i = 0; i < contextRecords.length; i += 1)
{
if (contextRecords[i].context ~= context) {
return contextRecords[i].issueSummaries;
}
}
return emptyResult;
}
// Counts total amount of tests performed under the contexts
// corresponding to `contextRecords[recordIndex]` record.
private final function int GetTotalTestsAmountForRecord(int recordIndex)
{
local int i;
local int result;
local array<IssueSummary> issueSummaries;
issueSummaries = contextRecords[recordIndex].issueSummaries;
result = 0;
for (i = 0; i < issueSummaries.length; i += 1)
{
if (issueSummaries[i] == none) continue;
result += issueSummaries[i].GetTotalTestsAmount();
}
return result;
}
/**
* Total amount of performed tests, recorded in caller `TestCaseSummary`.
*
* If you are interested in amount of test under a specific context, -
* use `GetTotalTestsAmountForContext()` instead.
*
* @return Total amount of performed tests.
*/
public final function int GetTotalTestsAmount()
{
local int i;
local int result;
for (i = 0; i < contextRecords.length; i += 1)
{
result += GetTotalTestsAmountForRecord(i);
}
return result;
}
/**
* Total amount of tests, performed under a context `context` and
* recorded in caller `TestCaseSummary`.
*
* If you are interested in total amount of test under all contexts, -
* use `GetTotalTestsAmount()` instead.
*
* @param context Context for which method must count amount of
* performed tests.
* @return Total amount of tests, performed under given context.
* If given context does not exist in records, - returns `-1`.
*/
public final function int GetTotalTestsAmountForContext(string context)
{
local int i;
for (i = 0; i < contextRecords.length; i += 1)
{
if (context ~= contextRecords[i].context) {
return GetTotalTestsAmountForRecord(i);
}
}
return -1;
}
// Counts total amount of successful tests performed under the contexts
// corresponding to `contextRecords[recordIndex]` record.
private final function int GetSuccessfulTestsAmountForRecord(int recordIndex)
{
local int i;
local int result;
local array<IssueSummary> issueSummaries;
issueSummaries = contextRecords[recordIndex].issueSummaries;
result = 0;
for (i = 0; i < issueSummaries.length; i += 1)
{
if (issueSummaries[i] == none) continue;
result += issueSummaries[i].GetSuccessfulTestsAmount();
}
return result;
}
/**
* Total amount of successfully performed tests,
* recorded in caller `TestCaseSummary`.
*
* If you are interested in amount of successful test under a specific context,
* - use `GetSuccessfulTestsAmountForContext()` instead.
*
* @return Total amount of successfully performed tests.
*/
public final function int GetSuccessfulTestsAmount()
{
local int i;
local int result;
for (i = 0; i < contextRecords.length; i += 1)
{
result += GetSuccessfulTestsAmountForRecord(i);
}
return result;
}
/**
* Total amount of tests, performed under a context `context` and
* recorded in caller `TestCaseSummary`.
*
* If you are interested in total amount of successful test under all contexts,
* - use `GetSuccessfulTestsAmount()` instead.
*
* @param context Context for which we method must count amount of
* successful tests.
* @return Total amount of successful tests, performed under given context.
* If given context does not exist in records, - returns `-1`.
*/
public final function int GetSuccessfulTestsAmountForContext(string context)
{
local int i;
for (i = 0; i < contextRecords.length; i += 1)
{
if (context ~= contextRecords[i].context) {
return GetSuccessfulTestsAmountForRecord(i);
}
}
return -1;
}
// Counts total amount of tests, failed under the contexts
// corresponding to `contextRecords[recordIndex]` record.
private final function int GetFailedTestsAmountForRecord(int recordIndex)
{
local int i;
local int result;
local array<IssueSummary> issueSummaries;
issueSummaries = contextRecords[recordIndex].issueSummaries;
result = 0;
for (i = 0; i < issueSummaries.length; i += 1)
{
if (issueSummaries[i] == none) continue;
result += issueSummaries[i].GetFailedTestsAmount();
}
return result;
}
/**
* Total amount of failed tests, recorded in caller `TestCaseSummary`.
*
* If you are interested in amount of failed test under a specific context, -
* use `GetFailedTestsAmountForContext()` instead.
*
* @return Total amount of failed tests.
*/
public final function int GetFailedTestsAmount()
{
local int i;
local int result;
for (i = 0; i < contextRecords.length; i += 1)
{
result += GetFailedTestsAmountForRecord(i);
}
return result;
}
/**
* Total amount of failed tests, performed under a context `context` and
* recorded in caller `TestCaseSummary`.
*
* If you are interested in total amount of failed test under all contexts, -
* use `GetFailedTestsAmount()` instead.
*
* @param context Context for which method must count amount of
* failed tests.
* @return Total amount of failed tests, performed under given context.
* If given context does not exist in records, - returns `-1`.
*/
public final function int GetFailedTestsAmountForContext(string context)
{
local int i;
for (i = 0; i < contextRecords.length; i += 1)
{
if (context ~= contextRecords[i].context) {
return GetFailedTestsAmountForRecord(i);
}
}
return -1;
}
/**
* Checks whether all tests recorded in this summary have passed.
*
* @return `true` if all tests have passed, `false` otherwise.
*/
public final function bool HasPassedAllTests()
{
return (GetFailedTestsAmount() <= 0);
}
/**
* Checks whether all tests, performed under given context and
* recorded in this summary, have passed.
*
* @return `true` if all tests under given context have passed,
* `false` otherwise.
* If given context does not exists - it did not fail any tests.
*/
public final function bool HasPassedAllTestsForContext(string context)
{
return (GetFailedTestsAmountForContext(context) <= 0);
}
/**
* Generates a text summary for a set of results, given as array of
* `TestCaseSummary`s (exactly how results are returned by `TestingService`).
*
* @param summaries `TestCase` summaries (obtained as a result of testing)
* that we want to display.
* @return Test representation of `summaries` as an array of
* formatted strings, where each string corresponds to it's own line.
*/
public final static function array<string> GenerateStringSummary(
array<TestCaseSummary> summaries)
{
local int i;
local bool allTestsPassed;
local array<string> result;
allTestsPassed = true;
result[0] = default.reportHeader;
for (i = 0; i < summaries.length; i += 1)
{
if (summaries[i] == none) continue;
summaries[i].AppendCaseSummary(result);
allTestsPassed = allTestsPassed && summaries[i].HasPassedAllTests();
}
if (allTestsPassed) {
result[result.length] = default.reportSuccessfulEnding;
}
else {
result[result.length] = default.reportUnsuccessfulEnding;
}
return result;
}
// Add text representation of caller `TestCase` to the existing array `result`.
private final function AppendCaseSummary(out array<string> result)
{
local int i, j;
local array<string> contexts;
local string testCaseAnnouncement;
local array<IssueSummary> issues;
if (ownerCase == none) return;
// Announce case
testCaseAnnouncement = "{$text_default Test case {$text_emphasis";
if (ownerCase.static.GetGroup() != "") {
testCaseAnnouncement @= "[" $ ownerCase.static.GetGroup() $ "]";
}
testCaseAnnouncement @= ownerCase.static.GetName() $ "}:}";
if (GetFailedTestsAmount() > 0) {
testCaseAnnouncement @= "{$text_failure failed}!";
}
else {
testCaseAnnouncement @= "{$text_ok passed}!";
}
result[result.length] = testCaseAnnouncement;
// Report failed tests
contexts = GetContexts();
for (i = 0;i < contexts.length; i += 1)
{
if (GetFailedTestsAmountForContext(contexts[i]) <= 0) continue;
result[result.length] = "{$text_warning " $ contexts[i] $ "}";
issues = GetIssueSummariesForContext(contexts[i]);
for (j = 0; j < issues.length; j += 1)
{
if (issues[j] == none) continue;
if (issues[j].GetFailedTestsAmount() <= 0) continue;
result[result.length] = indent $ issues[j].ToString();
}
}
}
defaultproperties
{
indent = " "
reportHeader = "{$text_default ############################## {$text_emphasis Test summary} ###############################}"
reportSuccessfulEnding = "{$text_default ########################### {$text_ok All tests have passed!} ############################}"
reportUnsuccessfulEnding = "{$text_default ########################## {$text_failure Some tests have failed :(} ###########################}"
}