using System;
using System.Collections;
using System.Diagnostics;
using Unity.PerformanceTesting.Data;
using Unity.PerformanceTesting.Runtime;
using Unity.PerformanceTesting.Statistics;
using UnityEngine;
using Debug = UnityEngine.Debug;
using Object = UnityEngine.Object;
namespace Unity.PerformanceTesting.Measurements
{
///
/// Allows measuring of frame times.
///
public class FramesMeasurement
{
private const int k_MinTestTimeMs = 500;
private const int k_MinWarmupTimeMs = 80;
private const int k_ProbingMultiplier = 4;
internal const int k_MinIterations = 7;
internal const int k_MaxDynamicMeasurements = 1000;
private const double k_DefaultMaxRelativeError = 0.02;
private const ConfidenceLevel k_DefaultConfidenceLevel = ConfidenceLevel.L99;
private const OutlierMode k_DefaultOutlierMode = OutlierMode.Remove;
private SampleGroup[] m_ProfilerSampleGroups;
private SampleGroup m_SampleGroup = new SampleGroup("FrameTime");
private int m_DesiredFrameCount;
internal bool m_DynamicMeasurementCount;
private double m_MaxRelativeError = k_DefaultMaxRelativeError;
private ConfidenceLevel m_ConfidenceLevel = k_DefaultConfidenceLevel;
private OutlierMode m_OutlierMode = k_DefaultOutlierMode;
private int m_Executions;
private int m_Warmup = -1;
private bool m_RecordFrametime = true;
///
/// Records provided profiler markers once per frame.
///
/// Profiler marker names as in profiler window.
///
public FramesMeasurement ProfilerMarkers(params string[] profilerMarkerNames)
{
m_ProfilerSampleGroups = Utils.CreateSampleGroupsFromMarkerNames(profilerMarkerNames);
return this;
}
///
/// Records provided profiler markers once per frame.
///
/// List of SampleGroups where a name matches the profiler marker and desired SampleUnit
///
public FramesMeasurement ProfilerMarkers(params SampleGroup[] sampleGroups)
{
m_ProfilerSampleGroups = sampleGroups;
return this;
}
///
/// Overrides the name of default sample group "Time".
///
/// Name of the sample group.
///
public FramesMeasurement SampleGroup(string name)
{
m_SampleGroup.Name = name;
return this;
}
///
/// Overrides the default sample group "Time"
///
/// Sample group to use.
///
public FramesMeasurement SampleGroup(SampleGroup sampleGroup)
{
m_SampleGroup = sampleGroup;
return this;
}
///
/// Count of measurements to take.
///
/// Count of measurements.
///
public FramesMeasurement MeasurementCount(int count)
{
m_Executions = count;
return this;
}
///
/// Dynamically find a suitable measurement count based on the margin of error of the samples.
/// The measurements will stop once a certain amount of samples (specified by a confidence interval)
/// falls within an acceptable error range from the result (defined by a relative error of the mean).
/// A default margin of error range of 2% and a default confidence interval of 99% will be used.
///
/// Outlier mode allows to include or exclude outliers when evaluating the stop criterion.
///
public FramesMeasurement DynamicMeasurementCount(OutlierMode outlierMode = k_DefaultOutlierMode)
{
m_DynamicMeasurementCount = true;
m_OutlierMode = outlierMode;
return this;
}
///
/// Dynamically find a suitable measurement count based on the margin of error of the samples.
/// The measurements will stop once a certain amount of samples (specified by a confidence interval)
/// falls within an acceptable error range from the result (defined by a relative error of the mean).
///
/// The maximum relative error of the mean that the margin of error must fall into.
/// The confidence interval which will be used to calculate the margin of error.
/// Outlier mode allows to include or exclude outliers when evaluating the stop criterion.
///
public FramesMeasurement DynamicMeasurementCount(double maxRelativeError, ConfidenceLevel confidenceLevel = k_DefaultConfidenceLevel,
OutlierMode outlierMode = k_DefaultOutlierMode)
{
m_MaxRelativeError = maxRelativeError;
m_ConfidenceLevel = confidenceLevel;
m_DynamicMeasurementCount = true;
m_OutlierMode = outlierMode;
return this;
}
///
/// Count of warmup executions.
///
/// Count of warmup executions.
///
public FramesMeasurement WarmupCount(int count)
{
m_Warmup = count;
return this;
}
///
/// Specifies frame times should not be recorded.
///
///
public FramesMeasurement DontRecordFrametime()
{
m_RecordFrametime = false;
return this;
}
///
/// Switches frame time measurement to asynchronous scope measurement.
///
/// Sample group name.
///
public ScopedFrameTimeMeasurement Scope(string name = "Time")
{
return new ScopedFrameTimeMeasurement(name);
}
///
/// Switches frame time measurement to asynchronous scope measurement.
///
/// Sample group to save measurements.
///
public ScopedFrameTimeMeasurement Scope(SampleGroup sampleGroup)
{
return new ScopedFrameTimeMeasurement(sampleGroup);
}
///
/// Executes the frame time measurement with given parameters. When MeasurementCount is not provided, a probing method will run to determine desired measurement counts.
///
/// IEnumerator to yield until finish.
public IEnumerator Run()
{
ValidateCorrectDynamicMeasurementCountUsage();
if (!ValidateMeasurementAndWarmupCount()) yield break;
SettingsOverride();
yield return m_Warmup > -1 ? WaitFor(m_Warmup) : GetDesiredIterationCount();
using (Measure.ProfilerMarkers(m_ProfilerSampleGroups))
{
if (m_DynamicMeasurementCount)
{
yield return RunDynamicMeasurementCount();
}
else
{
yield return RunFixedMeasurementCount();
}
// WaitForEndOfFrame coroutine is not invoked on the editor in batch mode
// This may lead to unexpected behavior and is better to avoid
// https://docs.unity3d.com/ScriptReference/WaitForEndOfFrame.html
if (!Application.isBatchMode && Application.isPlaying)
{
yield return new WaitForEndOfFrame();
}
}
}
private IEnumerator RunDynamicMeasurementCount()
{
while (true)
{
using (Measure.Scope(m_SampleGroup))
{
yield return null;
}
if (SampleCountFulfillsRequirements())
break;
}
}
private IEnumerator RunFixedMeasurementCount()
{
m_DesiredFrameCount = m_Executions > 0 ? m_Executions : m_DesiredFrameCount;
for (var i = 0; i < m_DesiredFrameCount; i++)
{
if (m_RecordFrametime)
{
using (Measure.Scope(m_SampleGroup))
{
yield return null;
}
}
else
{
yield return null;
}
}
}
private bool ValidateMeasurementAndWarmupCount()
{
if (m_DynamicMeasurementCount || m_Executions != 0 || m_Warmup < 0) return true;
Debug.LogError("Provide execution count or remove warmup count from frames measurement.");
return false;
}
private void ValidateCorrectDynamicMeasurementCountUsage()
{
if (!m_DynamicMeasurementCount)
return;
if (m_Executions > 0)
{
m_DynamicMeasurementCount = false;
Debug.LogWarning("DynamicMeasurementCount will be ignored because MeasurementCount was specified.");
return;
}
if (!m_RecordFrametime)
{
m_DynamicMeasurementCount = false;
Debug.LogWarning("DynamicMeasurementCount will be ignored because FrameTime measurement was disabled.");
}
}
private bool SampleCountFulfillsRequirements()
{
var samples = m_SampleGroup.Samples;
var sampleCount = samples.Count;
var statistics = MeasurementsStatistics.Calculate(samples, m_OutlierMode, m_ConfidenceLevel);
var actualError = statistics.MarginOfError;
var maxError = m_MaxRelativeError * statistics.Mean;
if (sampleCount >= k_MinIterations && actualError < maxError)
return true;
if (sampleCount >= k_MaxDynamicMeasurements)
return true;
return false;
}
///
/// Overrides measurement count based on performance run settings
///
private void SettingsOverride()
{
var count = RunSettings.Instance.MeasurementCount;
if (count < 0) { return; }
m_Executions = count;
m_Warmup = m_Warmup < 1 ? 0 : count;
m_DynamicMeasurementCount = false;
}
private IEnumerator GetDesiredIterationCount()
{
var executionTime = 0.0D;
var iterations = 1;
while (executionTime < k_MinWarmupTimeMs)
{
var sw = Stopwatch.GetTimestamp();
yield return WaitFor(iterations);
executionTime = TimeSpan.FromTicks(Stopwatch.GetTimestamp() - sw).TotalMilliseconds;
if (iterations == 1 && executionTime > 40)
{
m_DesiredFrameCount = k_MinIterations;
yield break;
}
if (iterations == 64)
{
m_DesiredFrameCount = 120;
yield break;
}
if (executionTime < k_MinWarmupTimeMs)
{
iterations *= k_ProbingMultiplier;
}
}
m_DesiredFrameCount = (int)(k_MinTestTimeMs * iterations / executionTime);
}
private IEnumerator WaitFor(int iterations)
{
for (var i = 0; i < iterations; i++)
{
yield return null;
}
}
///
/// Provides a way to measure frame time within a scope.
///
public struct ScopedFrameTimeMeasurement : IDisposable
{
private readonly FrameTimeMeasurement m_Test;
///
/// Initializes a scoped frame time measurement.
///
/// Sample group used to store measurements.
public ScopedFrameTimeMeasurement(SampleGroup sampleGroup)
{
var go = new GameObject("Recorder");
if (Application.isPlaying) Object.DontDestroyOnLoad(go);
m_Test = go.AddComponent();
m_Test.SampleGroup = sampleGroup;
PerformanceTest.Disposables.Add(this);
}
///
/// Initializes a scoped frame time measurement.
///
/// Sample group name used to store measurements.
public ScopedFrameTimeMeasurement(string name): this(new SampleGroup(name))
{
}
///
/// Stops scoped frame time measurement and adds it to provided sample group.
///
public void Dispose()
{
PerformanceTest.Disposables.Remove(this);
Object.DestroyImmediate(m_Test.gameObject);
}
}
}
}