mirror of
https://github.com/microsoft/terminal.git
synced 2025-12-10 18:43:54 -06:00
ddae2a1d49
As VS 2022 doesn't seem to store files with UTF-8 BOM as often anymore, we've been getting more and more pull requests which seemingly randomly change files. This cleans the situation up by removing the BOM from all files that have one. Additionally, `Host.Tests.Feature.rc` was converted from UTF-16 to UTF-8.
670 lines
30 KiB
C#
670 lines
30 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Runtime.Serialization;
|
|
using System.Runtime.Serialization.Json;
|
|
using System.Text;
|
|
using System.Xml.Linq;
|
|
|
|
namespace HelixTestHelpers
|
|
{
|
|
public class TestResult
|
|
{
|
|
public TestResult()
|
|
{
|
|
Screenshots = new List<string>();
|
|
RerunResults = new List<TestResult>();
|
|
}
|
|
|
|
public string Name { get; set; }
|
|
public string SourceWttFile { get; set; }
|
|
public bool Passed { get; set; }
|
|
public bool CleanupPassed { get; set; }
|
|
public TimeSpan ExecutionTime { get; set; }
|
|
public string Details { get; set; }
|
|
|
|
public List<string> Screenshots { get; private set; }
|
|
public List<TestResult> RerunResults { get; private set; }
|
|
|
|
// Returns true if the test pass rate is sufficient to avoid being counted as a failure.
|
|
public bool PassedOrUnreliable(int requiredNumberOfPasses)
|
|
{
|
|
if(Passed)
|
|
{
|
|
return true;
|
|
}
|
|
else
|
|
{
|
|
if(RerunResults.Count == 1)
|
|
{
|
|
return RerunResults[0].Passed;
|
|
}
|
|
else
|
|
{
|
|
return RerunResults.Where(r => r.Passed).Count() >= requiredNumberOfPasses;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
//
|
|
// Azure DevOps doesn't currently provide a way to directly report sub-results for tests that failed at least once
|
|
// that were run multiple times. To get around that limitation, we'll mark the test as "Skip" since
|
|
// that's the only non-pass/fail result we can return, and will then report the information about the
|
|
// runs in the "reason" category for the skipped test. In order to save space, we'll make the following
|
|
// optimizations for size:
|
|
//
|
|
// 1. Serialize as JSON, which is more compact than XML;
|
|
// 2. Don't serialize values that we don't need;
|
|
// 3. Store the URL prefix and suffix for the blob storage URL only once instead of
|
|
// storing every log and screenshot URL in its entirety; and
|
|
// 4. Store a list of unique error messages and then index into that instead of
|
|
// storing every error message in its entirety.
|
|
//
|
|
// #4 is motivated by the fact that if a test fails multiple times, it probably failed for the same reason
|
|
// each time, in which case we'd just be repeating ourselves if we stored every error message each time.
|
|
//
|
|
// TODO (https://github.com/dotnet/arcade/issues/2773): Once we're able to directly report things in a
|
|
// more granular fashion than just a binary pass/fail result, we should do that.
|
|
//
|
|
[DataContract]
|
|
internal class JsonSerializableTestResults
|
|
{
|
|
[DataMember]
|
|
internal string blobPrefix;
|
|
|
|
[DataMember]
|
|
internal string blobSuffix;
|
|
|
|
[DataMember]
|
|
internal string[] errors;
|
|
|
|
[DataMember]
|
|
internal JsonSerializableTestResult[] results;
|
|
}
|
|
|
|
[DataContract]
|
|
internal class JsonSerializableTestResult
|
|
{
|
|
[DataMember]
|
|
internal string outcome;
|
|
|
|
[DataMember]
|
|
internal int duration;
|
|
|
|
[DataMember(EmitDefaultValue = false)]
|
|
internal string log;
|
|
|
|
[DataMember(EmitDefaultValue = false)]
|
|
internal string[] screenshots;
|
|
|
|
[DataMember(EmitDefaultValue = false)]
|
|
internal int errorIndex;
|
|
}
|
|
|
|
public class TestPass
|
|
{
|
|
public TimeSpan TestPassExecutionTime { get; set; }
|
|
public List<TestResult> TestResults { get; set; }
|
|
|
|
public static TestPass ParseTestWttFile(string fileName, bool cleanupFailuresAreRegressions, bool truncateTestNames)
|
|
{
|
|
using (var stream = File.OpenRead(fileName))
|
|
{
|
|
var doc = XDocument.Load(stream);
|
|
var testResults = new List<TestResult>();
|
|
var testExecutionTimeMap = new Dictionary<string, List<double>>();
|
|
|
|
TestResult currentResult = null;
|
|
long frequency = 0;
|
|
long startTime = 0;
|
|
long stopTime = 0;
|
|
bool inTestCleanup = false;
|
|
|
|
bool shouldLogToTestDetails = false;
|
|
|
|
long testPassStartTime = 0;
|
|
long testPassStopTime = 0;
|
|
|
|
Func<XElement, bool> isScopeData = (elt) =>
|
|
{
|
|
return
|
|
elt.Element("Data") != null &&
|
|
elt.Element("Data").Element("WexContext") != null &&
|
|
(
|
|
elt.Element("Data").Element("WexContext").Value == "Cleanup" ||
|
|
elt.Element("Data").Element("WexContext").Value == "TestScope" ||
|
|
elt.Element("Data").Element("WexContext").Value == "TestScope" ||
|
|
elt.Element("Data").Element("WexContext").Value == "ClassScope" ||
|
|
elt.Element("Data").Element("WexContext").Value == "ModuleScope"
|
|
);
|
|
};
|
|
|
|
Func<XElement, bool> isModuleOrClassScopeStart = (elt) =>
|
|
{
|
|
return
|
|
elt.Name == "Msg" &&
|
|
elt.Element("Data") != null &&
|
|
elt.Element("Data").Element("StartGroup") != null &&
|
|
elt.Element("Data").Element("WexContext") != null &&
|
|
(elt.Element("Data").Element("WexContext").Value == "ClassScope" ||
|
|
elt.Element("Data").Element("WexContext").Value == "ModuleScope");
|
|
};
|
|
|
|
Func<XElement, bool> isModuleScopeEnd = (elt) =>
|
|
{
|
|
return
|
|
elt.Name == "Msg" &&
|
|
elt.Element("Data") != null &&
|
|
elt.Element("Data").Element("EndGroup") != null &&
|
|
elt.Element("Data").Element("WexContext") != null &&
|
|
elt.Element("Data").Element("WexContext").Value == "ModuleScope";
|
|
};
|
|
|
|
Func<XElement, bool> isClassScopeEnd = (elt) =>
|
|
{
|
|
return
|
|
elt.Name == "Msg" &&
|
|
elt.Element("Data") != null &&
|
|
elt.Element("Data").Element("EndGroup") != null &&
|
|
elt.Element("Data").Element("WexContext") != null &&
|
|
elt.Element("Data").Element("WexContext").Value == "ClassScope";
|
|
};
|
|
|
|
int testsExecuting = 0;
|
|
foreach (XElement element in doc.Root.Elements())
|
|
{
|
|
// Capturing the frequency data to record accurate
|
|
// timing data.
|
|
if (element.Name == "RTI")
|
|
{
|
|
frequency = Int64.Parse(element.Attribute("Frequency").Value);
|
|
}
|
|
|
|
// It's possible for a test to launch another test. If that happens, we won't modify the
|
|
// current result. Instead, we'll continue operating like normal and expect that we get two
|
|
// EndTests nodes before our next StartTests. We'll check that we've actually got a stop time
|
|
// before creating a new result. This will result in the two results being squashed
|
|
// into one result of the outer test that ran the inner one.
|
|
if (element.Name == "StartTest")
|
|
{
|
|
testsExecuting++;
|
|
if (testsExecuting == 1)
|
|
{
|
|
string testName = element.Attribute("Title").Value;
|
|
|
|
if (truncateTestNames)
|
|
{
|
|
const string xamlNativePrefix = "Windows::UI::Xaml::Tests::";
|
|
const string xamlManagedPrefix = "Windows.UI.Xaml.Tests.";
|
|
if (testName.StartsWith(xamlNativePrefix))
|
|
{
|
|
testName = testName.Substring(xamlNativePrefix.Length);
|
|
}
|
|
else if (testName.StartsWith(xamlManagedPrefix))
|
|
{
|
|
testName = testName.Substring(xamlManagedPrefix.Length);
|
|
}
|
|
}
|
|
|
|
currentResult = new TestResult() { Name = testName, SourceWttFile = fileName, Passed = true, CleanupPassed = true };
|
|
testResults.Add(currentResult);
|
|
startTime = Int64.Parse(element.Descendants("WexTraceInfo").First().Attribute("TimeStamp").Value);
|
|
inTestCleanup = false;
|
|
shouldLogToTestDetails = true;
|
|
stopTime = 0;
|
|
}
|
|
}
|
|
else if (currentResult != null && element.Name == "EndTest")
|
|
{
|
|
testsExecuting--;
|
|
|
|
// If any inner test fails, we'll still fail the outer
|
|
currentResult.Passed &= element.Attribute("Result").Value == "Pass";
|
|
|
|
// Only gather execution data if this is the outer test we ran initially
|
|
if (testsExecuting == 0)
|
|
{
|
|
stopTime = Int64.Parse(element.Descendants("WexTraceInfo").First().Attribute("TimeStamp").Value);
|
|
if (!testExecutionTimeMap.Keys.Contains(currentResult.Name))
|
|
testExecutionTimeMap[currentResult.Name] = new List<double>();
|
|
testExecutionTimeMap[currentResult.Name].Add((double)(stopTime - startTime) / frequency);
|
|
currentResult.ExecutionTime = TimeSpan.FromSeconds(testExecutionTimeMap[currentResult.Name].Average());
|
|
|
|
startTime = 0;
|
|
inTestCleanup = true;
|
|
}
|
|
}
|
|
else if (currentResult != null &&
|
|
(isModuleOrClassScopeStart(element) || isModuleScopeEnd(element) || isClassScopeEnd(element)))
|
|
{
|
|
shouldLogToTestDetails = false;
|
|
inTestCleanup = false;
|
|
}
|
|
|
|
// Log-appending methods.
|
|
if (currentResult != null && element.Name == "Error")
|
|
{
|
|
if (shouldLogToTestDetails)
|
|
{
|
|
currentResult.Details += "\r\n[Error]: " + element.Attribute("UserText").Value;
|
|
if (element.Attribute("File") != null && element.Attribute("File").Value != "")
|
|
{
|
|
currentResult.Details += (" [File " + element.Attribute("File").Value);
|
|
if (element.Attribute("Line") != null)
|
|
currentResult.Details += " Line: " + element.Attribute("Line").Value;
|
|
currentResult.Details += "]";
|
|
}
|
|
}
|
|
|
|
|
|
// The test cleanup errors will often come after the test claimed to have
|
|
// 'passed'. We treat them as errors as well.
|
|
if (inTestCleanup)
|
|
{
|
|
currentResult.CleanupPassed = false;
|
|
currentResult.Passed = false;
|
|
// In stress mode runs, this test will run n times before cleanup is run. If the cleanup
|
|
// fails, we want to fail every test.
|
|
if (cleanupFailuresAreRegressions)
|
|
{
|
|
foreach (var result in testResults.Where(res => res.Name == currentResult.Name))
|
|
{
|
|
result.Passed = false;
|
|
result.CleanupPassed = false;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (currentResult != null && element.Name == "Warn")
|
|
{
|
|
if (shouldLogToTestDetails)
|
|
{
|
|
currentResult.Details += "\r\n[Warn]: " + element.Attribute("UserText").Value;
|
|
}
|
|
|
|
if (element.Attribute("File") != null && element.Attribute("File").Value != "")
|
|
{
|
|
currentResult.Details += (" [File " + element.Attribute("File").Value);
|
|
if (element.Attribute("Line") != null)
|
|
currentResult.Details += " Line: " + element.Attribute("Line").Value;
|
|
currentResult.Details += "]";
|
|
}
|
|
}
|
|
|
|
if (currentResult != null && element.Name == "Msg")
|
|
{
|
|
var dataElement = element.Element("Data");
|
|
if (dataElement != null)
|
|
{
|
|
var supportingInfo = dataElement.Element("SupportingInfo");
|
|
if (supportingInfo != null)
|
|
{
|
|
var screenshots = supportingInfo.Elements("Item")
|
|
.Where(item => GetAttributeValue(item, "Name") == "Screenshot")
|
|
.Select(item => GetAttributeValue(item, "Value"));
|
|
|
|
foreach(var screenshot in screenshots)
|
|
{
|
|
string fileNameSuffix = string.Empty;
|
|
|
|
if (fileName.Contains("_rerun_multiple"))
|
|
{
|
|
fileNameSuffix = "_rerun_multiple";
|
|
}
|
|
else if (fileName.Contains("_rerun"))
|
|
{
|
|
fileNameSuffix = "_rerun";
|
|
}
|
|
|
|
currentResult.Screenshots.Add(screenshot.Replace(".jpg", fileNameSuffix + ".jpg"));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
testPassStartTime = Int64.Parse(doc.Root.Descendants("WexTraceInfo").First().Attribute("TimeStamp").Value);
|
|
testPassStopTime = Int64.Parse(doc.Root.Descendants("WexTraceInfo").Last().Attribute("TimeStamp").Value);
|
|
|
|
var testPassTime = TimeSpan.FromSeconds((double)(testPassStopTime - testPassStartTime) / frequency);
|
|
|
|
foreach (TestResult testResult in testResults)
|
|
{
|
|
if (testResult.Details != null)
|
|
{
|
|
testResult.Details = testResult.Details.Trim();
|
|
}
|
|
}
|
|
|
|
var testpass = new TestPass
|
|
{
|
|
TestPassExecutionTime = testPassTime,
|
|
TestResults = testResults
|
|
};
|
|
|
|
return testpass;
|
|
}
|
|
}
|
|
|
|
public static TestPass ParseTestWttFileWithReruns(string fileName, string singleRerunFileName, string multipleRerunFileName, bool cleanupFailuresAreRegressions, bool truncateTestNames)
|
|
{
|
|
TestPass testPass = ParseTestWttFile(fileName, cleanupFailuresAreRegressions, truncateTestNames);
|
|
TestPass singleRerunTestPass = File.Exists(singleRerunFileName) ? ParseTestWttFile(singleRerunFileName, cleanupFailuresAreRegressions, truncateTestNames) : null;
|
|
TestPass multipleRerunTestPass = File.Exists(multipleRerunFileName) ? ParseTestWttFile(multipleRerunFileName, cleanupFailuresAreRegressions, truncateTestNames) : null;
|
|
|
|
List<TestResult> rerunTestResults = new List<TestResult>();
|
|
|
|
if (singleRerunTestPass != null)
|
|
{
|
|
rerunTestResults.AddRange(singleRerunTestPass.TestResults);
|
|
}
|
|
|
|
if (multipleRerunTestPass != null)
|
|
{
|
|
rerunTestResults.AddRange(multipleRerunTestPass.TestResults);
|
|
}
|
|
|
|
// For each failed test result, we'll check to see whether the test passed at least once upon rerun.
|
|
// If so, we'll set PassedOnRerun to true to flag the fact that this is an unreliable test
|
|
// rather than a genuine test failure.
|
|
foreach (TestResult failedTestResult in testPass.TestResults.Where(r => !r.Passed))
|
|
{
|
|
failedTestResult.RerunResults.AddRange(rerunTestResults.Where(r => r.Name == failedTestResult.Name));
|
|
}
|
|
|
|
return testPass;
|
|
}
|
|
|
|
private static string GetAttributeValue(XElement element, string attributeName)
|
|
{
|
|
if(element.Attribute(attributeName) != null)
|
|
{
|
|
return element.Attribute(attributeName).Value;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public static class FailedTestDetector
|
|
{
|
|
public static void OutputFailedTestQuery(string wttInputPath)
|
|
{
|
|
var testPass = TestPass.ParseTestWttFile(wttInputPath, cleanupFailuresAreRegressions: true, truncateTestNames: false);
|
|
|
|
List<string> failedTestNames = new List<string>();
|
|
|
|
foreach (var result in testPass.TestResults)
|
|
{
|
|
if (!result.Passed)
|
|
{
|
|
failedTestNames.Add(result.Name);
|
|
}
|
|
}
|
|
|
|
if (failedTestNames.Count > 0)
|
|
{
|
|
string failedTestSelectQuery = "(@Name='";
|
|
|
|
for (int i = 0; i < failedTestNames.Count; i++)
|
|
{
|
|
failedTestSelectQuery += failedTestNames[i];
|
|
|
|
if (i < failedTestNames.Count - 1)
|
|
{
|
|
failedTestSelectQuery += "' or @Name='";
|
|
}
|
|
}
|
|
|
|
failedTestSelectQuery += "')";
|
|
|
|
Console.WriteLine(failedTestSelectQuery);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("");
|
|
}
|
|
}
|
|
}
|
|
|
|
public class TestResultParser
|
|
{
|
|
private string testNamePrefix;
|
|
private string helixResultsContainerUri;
|
|
private string helixResultsContainerRsas;
|
|
|
|
public TestResultParser(string testNamePrefix, string helixResultsContainerUri, string helixResultsContainerRsas)
|
|
{
|
|
this.testNamePrefix = testNamePrefix;
|
|
this.helixResultsContainerUri = helixResultsContainerUri;
|
|
this.helixResultsContainerRsas = helixResultsContainerRsas;
|
|
}
|
|
|
|
public Dictionary<string, string> GetSubResultsJsonByMethodName(string wttInputPath, string wttSingleRerunInputPath, string wttMultipleRerunInputPath)
|
|
{
|
|
Dictionary<string, string> subResultsJsonByMethod = new Dictionary<string, string>();
|
|
TestPass testPass = TestPass.ParseTestWttFileWithReruns(wttInputPath, wttSingleRerunInputPath, wttMultipleRerunInputPath, cleanupFailuresAreRegressions: true, truncateTestNames: false);
|
|
|
|
foreach (var result in testPass.TestResults)
|
|
{
|
|
var methodName = result.Name.Substring(result.Name.LastIndexOf('.') + 1);
|
|
|
|
if (!result.Passed)
|
|
{
|
|
// If a test failed, we'll have rerun it multiple times. We'll record the results of each run
|
|
// formatted as JSON.
|
|
JsonSerializableTestResults serializableResults = new JsonSerializableTestResults();
|
|
serializableResults.blobPrefix = helixResultsContainerUri;
|
|
serializableResults.blobSuffix = helixResultsContainerRsas;
|
|
|
|
List<string> errorList = new List<string>();
|
|
errorList.Add(result.Details);
|
|
|
|
foreach (TestResult rerunResult in result.RerunResults)
|
|
{
|
|
errorList.Add(rerunResult.Details);
|
|
}
|
|
|
|
serializableResults.errors = errorList.Distinct().Where(s => s != null).ToArray();
|
|
|
|
var reason = new XElement("reason");
|
|
List<JsonSerializableTestResult> serializableResultList = new List<JsonSerializableTestResult>();
|
|
serializableResultList.Add(ConvertToSerializableResult(result, serializableResults.errors));
|
|
|
|
foreach (TestResult rerunResult in result.RerunResults)
|
|
{
|
|
serializableResultList.Add(ConvertToSerializableResult(rerunResult, serializableResults.errors));
|
|
}
|
|
|
|
serializableResults.results = serializableResultList.ToArray();
|
|
|
|
using (MemoryStream stream = new MemoryStream())
|
|
{
|
|
DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(JsonSerializableTestResults));
|
|
serializer.WriteObject(stream, serializableResults);
|
|
stream.Position = 0;
|
|
|
|
using (StreamReader streamReader = new StreamReader(stream))
|
|
{
|
|
subResultsJsonByMethod.Add(methodName, streamReader.ReadToEnd());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return subResultsJsonByMethod;
|
|
}
|
|
|
|
public void ConvertWttLogToXUnitLog(string wttInputPath, string wttSingleRerunInputPath, string wttMultipleRerunInputPath, string xunitOutputPath, int requiredPassRateThreshold)
|
|
{
|
|
TestPass testPass = TestPass.ParseTestWttFileWithReruns(wttInputPath, wttSingleRerunInputPath, wttMultipleRerunInputPath, cleanupFailuresAreRegressions: true, truncateTestNames: false);
|
|
var results = testPass.TestResults;
|
|
|
|
int resultCount = results.Count;
|
|
int passedCount = results.Where(r => r.Passed).Count();
|
|
|
|
// Since we re-run tests on failure, we'll mark every test that failed at least once as "skipped" rather than "failed".
|
|
// If the test failed sufficiently often enough for it to count as a failed test (determined by a property on the
|
|
// Azure DevOps job), we'll later mark it as failed during test results processing.
|
|
|
|
int failedCount = results.Where(r => !r.PassedOrUnreliable(requiredPassRateThreshold)).Count();
|
|
int skippedCount = results.Where(r => !r.Passed && r.PassedOrUnreliable(requiredPassRateThreshold)).Count();
|
|
|
|
var root = new XElement("assemblies");
|
|
|
|
var assembly = new XElement("assembly");
|
|
assembly.SetAttributeValue("name", "MUXControls.Test.dll");
|
|
assembly.SetAttributeValue("test-framework", "TAEF");
|
|
assembly.SetAttributeValue("run-date", DateTime.Now.ToString("yyyy-MM-dd"));
|
|
|
|
// This doesn't need to be completely accurate since it's not exposed anywhere.
|
|
// If we need accurate an start time we can probably calculate it from the te.wtl file, but for
|
|
// now this is fine.
|
|
assembly.SetAttributeValue("run-time", (DateTime.Now - testPass.TestPassExecutionTime).ToString("hh:mm:ss"));
|
|
|
|
assembly.SetAttributeValue("total", resultCount);
|
|
assembly.SetAttributeValue("passed", passedCount);
|
|
assembly.SetAttributeValue("failed", failedCount);
|
|
assembly.SetAttributeValue("skipped", skippedCount);
|
|
|
|
assembly.SetAttributeValue("time", (int)testPass.TestPassExecutionTime.TotalSeconds);
|
|
assembly.SetAttributeValue("errors", 0);
|
|
root.Add(assembly);
|
|
|
|
var collection = new XElement("collection");
|
|
collection.SetAttributeValue("total", resultCount);
|
|
collection.SetAttributeValue("passed", passedCount);
|
|
collection.SetAttributeValue("failed", failedCount);
|
|
collection.SetAttributeValue("skipped", skippedCount);
|
|
collection.SetAttributeValue("name", "Test collection");
|
|
collection.SetAttributeValue("time", (int)testPass.TestPassExecutionTime.TotalSeconds);
|
|
assembly.Add(collection);
|
|
|
|
foreach (var result in results)
|
|
{
|
|
var test = new XElement("test");
|
|
test.SetAttributeValue("name", testNamePrefix + "." + result.Name);
|
|
|
|
var className = GetTestClassName(result.Name);
|
|
var methodName = GetTestMethodName(result.Name);
|
|
test.SetAttributeValue("type", className);
|
|
test.SetAttributeValue("method", methodName);
|
|
|
|
test.SetAttributeValue("time", result.ExecutionTime.TotalSeconds);
|
|
|
|
string resultString = string.Empty;
|
|
|
|
if (result.Passed)
|
|
{
|
|
resultString = "Pass";
|
|
}
|
|
else if(result.PassedOrUnreliable(requiredPassRateThreshold))
|
|
{
|
|
resultString = "Skip";
|
|
}
|
|
else
|
|
{
|
|
resultString = "Fail";
|
|
}
|
|
|
|
|
|
test.SetAttributeValue("result", resultString);
|
|
|
|
if (!result.Passed)
|
|
{
|
|
// If a test failed, we'll have rerun it multiple times.
|
|
// We'll save the subresults to a JSON text file that we'll upload to the helix results container -
|
|
// this allows it to be as long as we want, whereas the reason field in Azure DevOps has a 4000 character limit.
|
|
string subResultsFileName = methodName + "_subresults.json";
|
|
string subResultsFilePath = Path.Combine(Path.GetDirectoryName(wttInputPath), subResultsFileName);
|
|
|
|
if (result.PassedOrUnreliable(requiredPassRateThreshold))
|
|
{
|
|
var reason = new XElement("reason");
|
|
reason.Add(new XCData(GetUploadedFileUrl(subResultsFileName, helixResultsContainerUri, helixResultsContainerRsas)));
|
|
test.Add(reason);
|
|
}
|
|
else
|
|
{
|
|
var failure = new XElement("failure");
|
|
var message = new XElement("message");
|
|
message.Add(new XCData(GetUploadedFileUrl(subResultsFileName, helixResultsContainerUri, helixResultsContainerRsas)));
|
|
failure.Add(message);
|
|
test.Add(failure);
|
|
}
|
|
}
|
|
collection.Add(test);
|
|
}
|
|
|
|
File.WriteAllText(xunitOutputPath, root.ToString());
|
|
}
|
|
|
|
private JsonSerializableTestResult ConvertToSerializableResult(TestResult rerunResult, string[] uniqueErrors)
|
|
{
|
|
var serializableResult = new JsonSerializableTestResult();
|
|
|
|
serializableResult.outcome = rerunResult.Passed ? "Passed" : "Failed";
|
|
serializableResult.duration = (int)Math.Round(rerunResult.ExecutionTime.TotalMilliseconds);
|
|
|
|
if (!rerunResult.Passed)
|
|
{
|
|
serializableResult.log = Path.GetFileName(rerunResult.SourceWttFile);
|
|
|
|
if (rerunResult.Screenshots.Any())
|
|
{
|
|
List<string> screenshots = new List<string>();
|
|
|
|
foreach (var screenshot in rerunResult.Screenshots)
|
|
{
|
|
screenshots.Add(Path.GetFileName(screenshot));
|
|
}
|
|
|
|
serializableResult.screenshots = screenshots.ToArray();
|
|
}
|
|
|
|
// To conserve space, we'll log the index of the error to index in a list of unique errors rather than
|
|
// jotting down every single error in its entirety. We'll add one to the result so we can avoid
|
|
// serializing this property when it has the default value of 0.
|
|
serializableResult.errorIndex = Array.IndexOf(uniqueErrors, rerunResult.Details) + 1;
|
|
}
|
|
|
|
return serializableResult;
|
|
}
|
|
|
|
private string GetUploadedFileUrl(string filePath, string helixResultsContainerUri, string helixResultsContainerRsas)
|
|
{
|
|
var filename = Path.GetFileName(filePath);
|
|
return string.Format("{0}/{1}{2}", helixResultsContainerUri, filename, helixResultsContainerRsas);
|
|
}
|
|
|
|
private string GetTestNameSeparator(string testname)
|
|
{
|
|
var separatorString = ".";
|
|
if (!testname.Contains(separatorString))
|
|
{
|
|
separatorString = "::";
|
|
}
|
|
return separatorString;
|
|
}
|
|
|
|
private string GetTestMethodName(string fullyQualifiedName)
|
|
{
|
|
var separatorString = GetTestNameSeparator(fullyQualifiedName);
|
|
var methodName = fullyQualifiedName.Substring(fullyQualifiedName.LastIndexOf(separatorString) + separatorString.Length);
|
|
|
|
return methodName;
|
|
}
|
|
|
|
private string GetTestClassName(string fullyQualifiedName)
|
|
{
|
|
var separatorString = GetTestNameSeparator(fullyQualifiedName);
|
|
var className = fullyQualifiedName.Substring(0, fullyQualifiedName.LastIndexOf(separatorString));
|
|
|
|
return className;
|
|
}
|
|
}
|
|
}
|