using System; using System.Collections; using System.Diagnostics; using Unity.PerformanceTesting.Data; using Unity.PerformanceTesting.Runtime; using Unity.PerformanceTesting.Statistics; using UnityEngine; using Debug = UnityEngine.Debug; using Object = UnityEngine.Object; namespace Unity.PerformanceTesting.Measurements { /// /// Allows measuring of frame times. /// public class FramesMeasurement { private const int k_MinTestTimeMs = 500; private const int k_MinWarmupTimeMs = 80; private const int k_ProbingMultiplier = 4; internal const int k_MinIterations = 7; internal const int k_MaxDynamicMeasurements = 1000; private const double k_DefaultMaxRelativeError = 0.02; private const ConfidenceLevel k_DefaultConfidenceLevel = ConfidenceLevel.L99; private const OutlierMode k_DefaultOutlierMode = OutlierMode.Remove; private SampleGroup[] m_ProfilerSampleGroups; private SampleGroup m_SampleGroup = new SampleGroup("FrameTime"); private int m_DesiredFrameCount; internal bool m_DynamicMeasurementCount; private double m_MaxRelativeError = k_DefaultMaxRelativeError; private ConfidenceLevel m_ConfidenceLevel = k_DefaultConfidenceLevel; private OutlierMode m_OutlierMode = k_DefaultOutlierMode; private int m_Executions; private int m_Warmup = -1; private bool m_RecordFrametime = true; /// /// Records provided profiler markers once per frame. /// /// Profiler marker names as in profiler window. /// public FramesMeasurement ProfilerMarkers(params string[] profilerMarkerNames) { m_ProfilerSampleGroups = Utils.CreateSampleGroupsFromMarkerNames(profilerMarkerNames); return this; } /// /// Records provided profiler markers once per frame. /// /// List of SampleGroups where a name matches the profiler marker and desired SampleUnit /// public FramesMeasurement ProfilerMarkers(params SampleGroup[] sampleGroups) { m_ProfilerSampleGroups = sampleGroups; return this; } /// /// Overrides the name of default sample group "Time". /// /// Name of the sample group. /// public FramesMeasurement SampleGroup(string name) { m_SampleGroup.Name = name; return this; } /// /// Overrides the default sample group "Time" /// /// Sample group to use. /// public FramesMeasurement SampleGroup(SampleGroup sampleGroup) { m_SampleGroup = sampleGroup; return this; } /// /// Count of measurements to take. /// /// Count of measurements. /// public FramesMeasurement MeasurementCount(int count) { m_Executions = count; return this; } /// /// Dynamically find a suitable measurement count based on the margin of error of the samples. /// The measurements will stop once a certain amount of samples (specified by a confidence interval) /// falls within an acceptable error range from the result (defined by a relative error of the mean). /// A default margin of error range of 2% and a default confidence interval of 99% will be used. /// /// Outlier mode allows to include or exclude outliers when evaluating the stop criterion. /// public FramesMeasurement DynamicMeasurementCount(OutlierMode outlierMode = k_DefaultOutlierMode) { m_DynamicMeasurementCount = true; m_OutlierMode = outlierMode; return this; } /// /// Dynamically find a suitable measurement count based on the margin of error of the samples. /// The measurements will stop once a certain amount of samples (specified by a confidence interval) /// falls within an acceptable error range from the result (defined by a relative error of the mean). /// /// The maximum relative error of the mean that the margin of error must fall into. /// The confidence interval which will be used to calculate the margin of error. /// Outlier mode allows to include or exclude outliers when evaluating the stop criterion. /// public FramesMeasurement DynamicMeasurementCount(double maxRelativeError, ConfidenceLevel confidenceLevel = k_DefaultConfidenceLevel, OutlierMode outlierMode = k_DefaultOutlierMode) { m_MaxRelativeError = maxRelativeError; m_ConfidenceLevel = confidenceLevel; m_DynamicMeasurementCount = true; m_OutlierMode = outlierMode; return this; } /// /// Count of warmup executions. /// /// Count of warmup executions. /// public FramesMeasurement WarmupCount(int count) { m_Warmup = count; return this; } /// /// Specifies frame times should not be recorded. /// /// public FramesMeasurement DontRecordFrametime() { m_RecordFrametime = false; return this; } /// /// Switches frame time measurement to asynchronous scope measurement. /// /// Sample group name. /// public ScopedFrameTimeMeasurement Scope(string name = "Time") { return new ScopedFrameTimeMeasurement(name); } /// /// Switches frame time measurement to asynchronous scope measurement. /// /// Sample group to save measurements. /// public ScopedFrameTimeMeasurement Scope(SampleGroup sampleGroup) { return new ScopedFrameTimeMeasurement(sampleGroup); } /// /// Executes the frame time measurement with given parameters. When MeasurementCount is not provided, a probing method will run to determine desired measurement counts. /// /// IEnumerator to yield until finish. public IEnumerator Run() { ValidateCorrectDynamicMeasurementCountUsage(); if (!ValidateMeasurementAndWarmupCount()) yield break; SettingsOverride(); yield return m_Warmup > -1 ? WaitFor(m_Warmup) : GetDesiredIterationCount(); using (Measure.ProfilerMarkers(m_ProfilerSampleGroups)) { if (m_DynamicMeasurementCount) { yield return RunDynamicMeasurementCount(); } else { yield return RunFixedMeasurementCount(); } // WaitForEndOfFrame coroutine is not invoked on the editor in batch mode // This may lead to unexpected behavior and is better to avoid // https://docs.unity3d.com/ScriptReference/WaitForEndOfFrame.html if (!Application.isBatchMode && Application.isPlaying) { yield return new WaitForEndOfFrame(); } } } private IEnumerator RunDynamicMeasurementCount() { while (true) { using (Measure.Scope(m_SampleGroup)) { yield return null; } if (SampleCountFulfillsRequirements()) break; } } private IEnumerator RunFixedMeasurementCount() { m_DesiredFrameCount = m_Executions > 0 ? m_Executions : m_DesiredFrameCount; for (var i = 0; i < m_DesiredFrameCount; i++) { if (m_RecordFrametime) { using (Measure.Scope(m_SampleGroup)) { yield return null; } } else { yield return null; } } } private bool ValidateMeasurementAndWarmupCount() { if (m_DynamicMeasurementCount || m_Executions != 0 || m_Warmup < 0) return true; Debug.LogError("Provide execution count or remove warmup count from frames measurement."); return false; } private void ValidateCorrectDynamicMeasurementCountUsage() { if (!m_DynamicMeasurementCount) return; if (m_Executions > 0) { m_DynamicMeasurementCount = false; Debug.LogWarning("DynamicMeasurementCount will be ignored because MeasurementCount was specified."); return; } if (!m_RecordFrametime) { m_DynamicMeasurementCount = false; Debug.LogWarning("DynamicMeasurementCount will be ignored because FrameTime measurement was disabled."); } } private bool SampleCountFulfillsRequirements() { var samples = m_SampleGroup.Samples; var sampleCount = samples.Count; var statistics = MeasurementsStatistics.Calculate(samples, m_OutlierMode, m_ConfidenceLevel); var actualError = statistics.MarginOfError; var maxError = m_MaxRelativeError * statistics.Mean; if (sampleCount >= k_MinIterations && actualError < maxError) return true; if (sampleCount >= k_MaxDynamicMeasurements) return true; return false; } /// /// Overrides measurement count based on performance run settings /// private void SettingsOverride() { var count = RunSettings.Instance.MeasurementCount; if (count < 0) { return; } m_Executions = count; m_Warmup = m_Warmup < 1 ? 0 : count; m_DynamicMeasurementCount = false; } private IEnumerator GetDesiredIterationCount() { var executionTime = 0.0D; var iterations = 1; while (executionTime < k_MinWarmupTimeMs) { var sw = Stopwatch.GetTimestamp(); yield return WaitFor(iterations); executionTime = TimeSpan.FromTicks(Stopwatch.GetTimestamp() - sw).TotalMilliseconds; if (iterations == 1 && executionTime > 40) { m_DesiredFrameCount = k_MinIterations; yield break; } if (iterations == 64) { m_DesiredFrameCount = 120; yield break; } if (executionTime < k_MinWarmupTimeMs) { iterations *= k_ProbingMultiplier; } } m_DesiredFrameCount = (int)(k_MinTestTimeMs * iterations / executionTime); } private IEnumerator WaitFor(int iterations) { for (var i = 0; i < iterations; i++) { yield return null; } } /// /// Provides a way to measure frame time within a scope. /// public struct ScopedFrameTimeMeasurement : IDisposable { private readonly FrameTimeMeasurement m_Test; /// /// Initializes a scoped frame time measurement. /// /// Sample group used to store measurements. public ScopedFrameTimeMeasurement(SampleGroup sampleGroup) { var go = new GameObject("Recorder"); if (Application.isPlaying) Object.DontDestroyOnLoad(go); m_Test = go.AddComponent(); m_Test.SampleGroup = sampleGroup; PerformanceTest.Disposables.Add(this); } /// /// Initializes a scoped frame time measurement. /// /// Sample group name used to store measurements. public ScopedFrameTimeMeasurement(string name): this(new SampleGroup(name)) { } /// /// Stops scoped frame time measurement and adds it to provided sample group. /// public void Dispose() { PerformanceTest.Disposables.Remove(this); Object.DestroyImmediate(m_Test.gameObject); } } } }