using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.Drawing.Drawing2D;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using DelftTools.Utils.IO;
using log4net.Appender;
using log4net.Config;
using log4net.Core;
using Newtonsoft.Json;
using NUnit.Framework;
namespace DelftTools.TestUtils
{
public class TestHelper
{
private static string solutionRoot;
private static int assertInTestMethod;
private static string lastTestName;
private static Color[] colors;
public static string SolutionRoot
{
get
{
if (solutionRoot == null)
{
solutionRoot = GetSolutionRoot();
}
return solutionRoot;
}
}
public static string TestDataDirectory
{
get
{
string testDataPath = SolutionRoot + @"\test-data\";
return Path.GetDirectoryName(testDataPath);
}
}
public static string GetCurrentMethodName()
{
MethodBase callingMethod = new StackFrame(1, false).GetMethod();
return callingMethod.DeclaringType.Name + "." + callingMethod.Name;
}
///
/// Returns full path to the file or directory in "test-data"
///
///
///
public static string GetTestDataPath(TestDataPath path)
{
return Path.Combine(TestDataDirectory, path.Path);
}
///
/// Returns full path to the file or directory in "test-data"
///
///
///
///
public static string GetTestDataPath(TestDataPath testDataPath, string path)
{
return Path.Combine(Path.Combine(TestDataDirectory, testDataPath.Path), path);
}
public static string GetTestProjectDirectory()
{
var stackFrames = new StackTrace().GetFrames();
if (stackFrames == null)
{
throw new Exception("Could not get stacktrace.");
}
var testMethod = stackFrames.FirstOrDefault(f => f.GetMethod().GetCustomAttributes(typeof(TestAttribute), true).Any() ||
f.GetMethod().GetCustomAttributes(typeof(SetUpAttribute), true).Any() ||
f.GetMethod().GetCustomAttributes(typeof(TestFixtureSetUpAttribute), true).Any());
if (testMethod == null)
{
throw new Exception("Could not determine the test method.");
}
var testClassType = testMethod.GetMethod().DeclaringType;
if (testClassType == null)
{
throw new Exception("Could not find test class type.");
}
return Path.GetDirectoryName((new Uri(testClassType.Assembly.CodeBase)).AbsolutePath);
}
///
/// Gets the test-data directory for the current test project.
///
public static string GetDataDir()
{
var testProjectDirectory = GetTestProjectDirectory();
var rootedTestProjectFolderPath = Path.GetFullPath(Path.Combine(testProjectDirectory, "..", ".."));
// Skip Debug/Release and bin folders
var rootedTestDirectory = Path.Combine(SolutionRoot, "test");
var relativeTestProjectFolderPath = FileUtils.GetRelativePath(rootedTestDirectory, rootedTestProjectFolderPath);
return Path.GetFullPath(Path.Combine(SolutionRoot, "test-data", relativeTestProjectFolderPath) + Path.DirectorySeparatorChar);
}
///
/// Get's the path in test-data tree section
///
///
///
public static string GetTestFilePath(string filename)
{
var path = Path.Combine(GetDataDir(), filename);
var uri = new UriBuilder(path);
path = Uri.UnescapeDataString(uri.Path);
if (File.Exists(path))
{
return path;
}
// file not found..exception
throw new FileNotFoundException(String.Format("File not found: {0}", path), path);
}
///
///
///
///
///
/// Take HDD speed into account, makes sure that test timing is divided by MACHINE_HDD_PERFORMANCE_RANK environmental variable.
///
public static double AssertIsFasterThan(float maxMilliseconds, Action action, bool rankHddAccess = false)
{
return AssertIsFasterThan(maxMilliseconds, null, action, rankHddAccess);
}
///
///
///
///
///
///
/// Take HDD speed into account, makes sure that test timing is divided by MACHINE_HDD_PERFORMANCE_RANK environmental variable.
///
public static double AssertIsFasterThan(float maxMilliseconds, string message, Action action, bool rankHddAccess)
{
var stopwatch = new Stopwatch();
double actualMillisecond = default(double);
stopwatch.Start();
action();
stopwatch.Stop();
actualMillisecond = Math.Abs(actualMillisecond - default(double)) > 1e-5
? Math.Min(stopwatch.ElapsedMilliseconds, actualMillisecond)
: stopwatch.ElapsedMilliseconds;
stopwatch.Reset();
string testName = GetCurrentTestClassMethodName();
if (testName == lastTestName) // check if there are more than one assert in a single test
{
assertInTestMethod++;
testName += assertInTestMethod;
}
else
{
lastTestName = testName;
assertInTestMethod = 1; // reset
}
float machinePerformanceRank = GetMachinePerformanceRank();
float machineHddPerformanceRank = GetMachineHddPerformanceRank();
var reportDirectory = GetSolutionRoot() + Path.DirectorySeparatorChar + "target/";
FileUtils.CreateDirectoryIfNotExists(reportDirectory);
var path = reportDirectory + "performance-times.html";
WriteTimesToLogFile(maxMilliseconds, (int) actualMillisecond, machinePerformanceRank,
machineHddPerformanceRank, rankHddAccess, testName, false, path);
path = reportDirectory + "performance-times-charts.html";
WriteTimesToLogFile(maxMilliseconds, (int) actualMillisecond, machinePerformanceRank,
machineHddPerformanceRank, rankHddAccess, testName, true, path);
float rank = machineHddPerformanceRank;
if (rankHddAccess) // when test relies a lot on HDD - multiply rank by hdd speed factor
{
rank *= machineHddPerformanceRank;
}
var userMessage = String.IsNullOrEmpty(message) ? "" : message + ". ";
if (rank != 1.0f)
{
Assert.IsTrue(rank*actualMillisecond < maxMilliseconds, userMessage + "Maximum of {0} milliseconds exceeded. Actual was {1}, machine performance weighted actual was {2}",
maxMilliseconds, actualMillisecond, actualMillisecond*rank);
Console.WriteLine(userMessage + String.Format("Test took {1} milliseconds (machine performance weighted {2}). Maximum was {0}",
maxMilliseconds, actualMillisecond, actualMillisecond*rank));
}
else
{
Assert.IsTrue(actualMillisecond < maxMilliseconds, userMessage + "Maximum of {0} milliseconds exceeded. Actual was {1}", maxMilliseconds,
actualMillisecond);
Console.WriteLine(userMessage + String.Format("Test took {1} milliseconds. Maximum was {0}", maxMilliseconds, actualMillisecond));
}
return actualMillisecond;
}
///
/// Checks if the given messages occurs in the log.
///
/// Action to be performed while recording the log
/// The message that should occur in the log
/// Optional: assert that log has this number of messages.
public static void AssertLogMessageIsGenerated(Action action, string message, int? expectedLogMessageCount = null)
{
AssertLogMessagesAreGenerated(action, new[]
{
message
}, expectedLogMessageCount);
}
///
/// Method used to check if a collection of messages have occured in the log.
/// This function allowed checking log messages being generated like
/// without having to rerun the action for every single message. Fails the test when any of checks fail.
///
/// Action to be performed while recording the log
/// The collection of messages that should occur in the log
/// Optional: assert that log has this number of messages.
///
public static void AssertLogMessagesAreGenerated(Action action, IEnumerable messages, int? expectedLogMessageCount = null)
{
var renderedMessages = GetAllRenderedMessages(action);
AssertExpectedMessagesInRenderedMessages(messages, renderedMessages);
if (expectedLogMessageCount != null)
{
Assert.AreEqual((int) expectedLogMessageCount, renderedMessages.Count());
}
}
///
/// Method use to perform any type of assertion on the generated log while performing
/// a particular action.
///
/// Action to be performed while recording the log.
/// The assertion logic performed on the generated log-messages.
public static void AssertLogMessages(Action action, Action> assertLogMessages)
{
var renderedMessages = GetAllRenderedMessages(action);
assertLogMessages(renderedMessages);
}
///
/// Method used to check if a specific message is not in the log.
/// This function allowed checking log messages being generated like
/// without having to rerun the action for every single message.
///
/// The action to be performed that should not generate
/// The log message that should not occur in the log
/// Optional: assert that log has this number of messages.
///
[Obsolete("Please keep in mind that asserting unwanted messages are 'very easy to pass' asserts and can pass by a random change of 1 character.")]
public static void AssertLogMessageIsNotGenerated(Action action, string unwantedMessage, int? expectedLogMessageCount = null)
{
AssertLogMessagesAreNotGenerated(action, new[]
{
unwantedMessage
}, expectedLogMessageCount);
}
///
/// Method used to check if a collection of messages have not occurred in the log.
/// This function allowed checking log messages being generated like
/// without having to rerun the action for every single message. Fails the test when any of checks fail.
///
/// The action to be performed that should not generate any of the messages in
/// The log messages that should not occur in the log
/// Optional: assert that log has this number of messages.
///
[Obsolete("Please keep in mind that asserting unwanted messages are 'very easy to pass' asserts and can pass by a random change of 1 character.")]
public static void AssertLogMessagesAreNotGenerated(Action action, IEnumerable unwantedMessages, int? expectedLogMessageCount = null)
{
var renderedMessages = GetAllRenderedMessages(action);
AssertUnwantedMessagesNotInRenderedMessages(unwantedMessages, renderedMessages);
if (expectedLogMessageCount != null)
{
Assert.AreEqual((int) expectedLogMessageCount, renderedMessages.Count());
}
}
///
/// Checks if all expected messages occur in the log,
/// while none of the unwanted messages should occur in the log.
///
/// Action to be performed while recording the log
/// Collection of expected messages that should be in the log
/// Collection of messages of which none should be in the log
/// Optional: assert that log has this number of messages.
[Obsolete("Please keep in mind that asserting unwanted messages are 'very easy to pass' asserts and can pass by a random change of 1 character.")]
public static void AssertLogExpectedAndUnwantedMessages(Action action, IEnumerable expectedMessages, IEnumerable unwantedMessages, int? expectedLogMessageCount = null)
{
var renderedMessages = GetAllRenderedMessages(action);
AssertUnwantedMessagesNotInRenderedMessages(unwantedMessages, renderedMessages);
AssertExpectedMessagesInRenderedMessages(expectedMessages, renderedMessages);
if (expectedLogMessageCount != null)
{
Assert.AreEqual((int) expectedLogMessageCount, renderedMessages.Count());
}
}
///
/// Checks the number of messages in the log.
///
/// Action to be performed while recording the log
/// The expected number of messages
public static void AssertLogMessagesCount(Action action, int count)
{
var renderedMessages = GetAllRenderedMessages(action);
Assert.AreEqual(count, renderedMessages.Count());
}
private static string GetCurrentTestClassMethodName()
{
var stackTrace = new StackTrace(false);
for (int i = 1; i < stackTrace.FrameCount; i++)
{
StackFrame stackFrame = stackTrace.GetFrame(i);
if (stackFrame.GetMethod().GetCustomAttributes(true).OfType().Count() != 0)
{
MethodBase method = stackFrame.GetMethod();
return method.DeclaringType.Name + "." + method.Name;
}
}
return "";
}
private static string GetSolutionRoot()
{
const string solutionName = "DeltaShell.sln";
//get the current directory and scope up
//TODO find a faster safer method
string curDir = ".";
while (Directory.Exists(curDir) && !File.Exists(curDir + @"\" + solutionName))
{
curDir += "/../";
}
if (!File.Exists(Path.Combine(curDir, solutionName)))
{
throw new InvalidOperationException("Solution file not found.");
}
return Path.GetFullPath(curDir);
}
private static float GetMachineHddPerformanceRank()
{
string rank = Environment.GetEnvironmentVariable("MACHINE_HDD_PERFORMANCE_RANK");
if (!String.IsNullOrEmpty(rank))
{
return Single.Parse(rank);
}
return 1.0f;
}
private static float GetMachinePerformanceRank()
{
string rank = Environment.GetEnvironmentVariable("MACHINE_PERFORMANCE_RANK");
if (!String.IsNullOrEmpty(rank))
{
return Single.Parse(rank);
}
return 1.0f;
}
private static void WriteTimesToLogFile(float maxMilliseconds, float actualMilliseconds, float machineRank, float machineHddRank, bool useHddAccessRank, string testName, bool includeCharts, string path)
{
if (!File.Exists(path))
{
if (!includeCharts)
{
File.AppendAllText(path, "View with charts
");
}
if (machineRank != 1.0f)
{
File.AppendAllText(path, "Machine performance rank (multiplier):" + machineRank + "
");
File.AppendAllText(path, "Time is in milliseconds
");
File.AppendAllText(path, String.Format("\n| Time | Name | {0}MaxTime | ActualTime | RankedActualTime | Percentage |
", includeCharts ? "Chart | " : ""));
}
else
{
File.AppendAllText(path, "Time is in milliseconds
");
File.AppendAllText(path, String.Format("\n| Time | Name | {0}MaxTime | ActualTime | Percentage |
", includeCharts ? "Chart | " : ""));
}
}
string contents;
float rank = machineRank*(useHddAccessRank ? machineHddRank : 1.0f);
var chartContent = includeCharts ? String.Format(" | ", testName) : "";
float fraction;
if (machineRank != 1.0f)
{
contents = String.Format(CultureInfo.InvariantCulture,
"| {0} | {1} | {2}{3:G5} | {4:G5} | {5:G5} | ",
DateTime.Now, testName, chartContent, maxMilliseconds, actualMilliseconds, actualMilliseconds*rank);
fraction = (maxMilliseconds - actualMilliseconds*rank)/maxMilliseconds;
}
else
{
contents = String.Format(CultureInfo.InvariantCulture,
"
| {0} | {1} | {2}{3:G5} | {4:G5} | ", DateTime.Now, testName, chartContent,
maxMilliseconds, actualMilliseconds);
fraction = (maxMilliseconds - actualMilliseconds)/maxMilliseconds;
}
string color = ColorTranslator.ToHtml(GetPerformanceColor(fraction));
contents += String.Format(CultureInfo.InvariantCulture, "{1:G5}% | ", color, (100 - fraction*100));
contents += "
\n";
File.AppendAllText(path, contents);
// update test reports in JSON files on build server (tests statistics)
// TODO: find way to write it somewhere so that it will be shared between build agents
int buildNumber = 0;
string s = Environment.GetEnvironmentVariable("BUILD_NUMBER");
if (!String.IsNullOrEmpty(s))
{
buildNumber = Int32.Parse(s); // defined on build server
}
// generate JSON files locally
string testHistoryDirectoryPath = GetSolutionRoot() + "/target/performace-test-reports";
FileUtils.CreateDirectoryIfNotExists(testHistoryDirectoryPath);
string testHistoryFilePath = testHistoryDirectoryPath + Path.DirectorySeparatorChar + testName + ".json";
var testInfos = new List();
if (File.Exists(testHistoryFilePath))
{
testInfos = JsonConvert.DeserializeObject>(File.ReadAllText(testHistoryFilePath));
}
if (buildNumber == 0)
{
if (testInfos == null || testInfos.Count == 0)
{
testInfos = new List();
}
else
{
var maxBuildNumber = testInfos.Select(i => i.BuildNumber).Max();
// reset build numbers if they were not set
foreach (var testInfo in testInfos)
{
if (testInfo.BuildNumber == 0)
{
testInfo.BuildNumber = buildNumber;
buildNumber++;
}
}
buildNumber = maxBuildNumber + 1;
}
}
int maxTestInfoCount = 100; // max number of tests locally
while (testInfos.Count >= maxTestInfoCount)
{
testInfos.RemoveAt(0);
}
testInfos.Add(new TestRunInfo
{
TestName = testName,
Actual = actualMilliseconds,
ActualWeighted =
(int) (actualMilliseconds*machineRank*(useHddAccessRank ? machineHddRank : 1.0)),
Max = maxMilliseconds,
MachineHddRank = machineHddRank,
MachineRank = machineRank,
Time = DateTime.Now.ToString(CultureInfo.InvariantCulture),
BuildNumber = buildNumber,
UseMachineHddRank = useHddAccessRank
});
CreateChart(testInfos, testHistoryFilePath + ".html");
string json = JsonConvert.SerializeObject(testInfos, Formatting.Indented);
File.WriteAllText(testHistoryFilePath, json);
}
private static void CreateChart(List testInfos, string filePath)
{
var content = File.ReadAllText(TestDataDirectory + "/Common/DelftTools.TestUtils/chart.template.html");
var seriesPassed = "";
var seriesFailed = "";
var seriesThreshold = "";
for (var i = 0; i < testInfos.Count; i++)
{
var info = testInfos[i];
if (info.Actual > info.Max)
{
seriesFailed += "[" + info.BuildNumber + ", " + info.ActualWeighted + "], ";
}
else
{
seriesPassed += "[" + info.BuildNumber + ", " + info.ActualWeighted + "], ";
}
seriesThreshold += "[" + info.BuildNumber + ", " + info.Max + "], ";
}
content = content.Replace("$$SERIES_PASSED$$", seriesPassed);
content = content.Replace("$$SERIES_FAILED$$", seriesFailed);
content = content.Replace("$$SERIES_THRESHOLD$$", seriesThreshold);
File.WriteAllText(filePath, content);
}
private static Color GetPerformanceColor(double fraction)
{
if (fraction < 0)
{
return Color.Red;
}
if (colors == null)
{
var bitmap = new Bitmap(101, 1);
Graphics graphics = Graphics.FromImage(bitmap);
var rectangle = new Rectangle(0, 0, 101, 1);
var brush = new LinearGradientBrush(rectangle, Color.Green, Color.Yellow, 0.0f);
graphics.FillRectangle(brush, rectangle);
colors = new Color[101];
for (int i = 0; i < 101; i++)
{
colors[i] = bitmap.GetPixel(i, 0);
}
}
// 25% is the best result GREEN, less or greater than goes to yellow
var localValue = fraction >= 0.25 ? Math.Min(1, (fraction - 0.25)/0.75) : Math.Max(0, (0.25 - fraction)/0.25);
return colors[(int) (localValue*100.0)];
}
///
/// Checks if all messages from occur in
///
/// The collection of expected messages
/// The collection of messages in the log
private static void AssertExpectedMessagesInRenderedMessages(IEnumerable messages, IEnumerable renderedMessages)
{
foreach (string message in messages)
{
if (!renderedMessages.Contains(message))
{
Assert.Fail("Message \"{0}\" not found in messages of log4net", message);
}
}
}
///
/// Checks if none of the messages from occurs in
///
/// The collection of unwanted messages
/// The collection of log messages in the log
private static void AssertUnwantedMessagesNotInRenderedMessages(IEnumerable messages, IEnumerable renderedMessages)
{
foreach (var renderedMessage in renderedMessages)
{
if (messages.Contains(renderedMessage))
{
Assert.Fail("Message \"{0}\" found in messages of log4net", renderedMessage);
}
}
}
private static IEnumerable GetAllRenderedMessages(Action action)
{
var memoryAppender = new MemoryAppender();
BasicConfigurator.Configure(memoryAppender);
LogHelper.SetLoggingLevel(Level.All);
action();
var renderedMessages = memoryAppender.GetEvents().Select(le => le.RenderedMessage).ToList();
memoryAppender.Close();
LogHelper.ResetLogging();
return renderedMessages;
}
#region Nested type: TestRunInfo
internal class TestRunInfo
{
public string Time;
public string TestName;
public float Actual; // millis
public float ActualWeighted; // millis
public float Max; // millis
public int BuildNumber;
public float MachineHddRank;
public float MachineRank;
public bool UseMachineHddRank;
}
#endregion
}
}