From: Jean-Philippe Bruyère Date: Fri, 22 Jan 2021 10:28:08 +0000 (+0100) Subject: perftests improvments, truncated mean, unittests, start/end stages X-Git-Tag: v0.9.5-beta~92 X-Git-Url: https://git.osiis.dedyn.io/?a=commitdiff_plain;h=e6e2f8aec4c20bc04094cc5ff265fdb42fd0c8f2;p=jp%2Fcrow.git perftests improvments, truncated mean, unittests, start/end stages --- diff --git a/Samples/PerfTests/PerfTests.csproj b/Samples/PerfTests/PerfTests.csproj index 52161ee4..3d437294 100644 --- a/Samples/PerfTests/PerfTests.csproj +++ b/Samples/PerfTests/PerfTests.csproj @@ -20,9 +20,7 @@ Icons\%(Filename)%(Extension) --> - - - + diff --git a/Samples/PerfTests/Program.cs b/Samples/PerfTests/Program.cs index a50003f6..dcc412ed 100644 --- a/Samples/PerfTests/Program.cs +++ b/Samples/PerfTests/Program.cs @@ -8,6 +8,7 @@ using System.IO; using Crow; using System.Reflection; using System.Runtime.InteropServices; +using System.Buffers; namespace PerfTests { @@ -31,11 +32,9 @@ namespace PerfTests System.Runtime.Loader.AssemblyLoadContext.Default.ResolvingUnmanagedDll+=resolveUnmanaged; } #endif - readonly int count = 1, updateCycles = 0; - readonly bool miliseconds = false; - readonly bool resetItors = false; + readonly int count = 1, updateCycles = 0; readonly bool screenOutput = false; - readonly string inDirectory = "Interfaces";//directory to test + readonly string inDirectory = null;//directory to test readonly string outFilePath; @@ -44,20 +43,38 @@ namespace PerfTests bool logToDisk => writer != null; + enum Stage + { + ITor = 0, + Instantiation, + Add, + Datasource, + FirstUpdate, + Update, + Delete + } + Stage StartStage = Stage.ITor; + Stage EndStage = Stage.Update; + void writeHeader (TextWriter txtWriter) { txtWriter.WriteLine ($"Crow perf test ({clientRectangle.Width} X {clientRectangle.Height}), output to screen = { screenOutput }"); - txtWriter.WriteLine ($"repeat = {count}, update cycles = {updateCycles} reset Instantiators = {resetItors}"); + txtWriter.WriteLine ($"repeat = {count}, {StartStage} -> {EndStage}, update cycles = {updateCycles}"); txtWriter.WriteLine ($"git:{ThisAssembly.Git.Commit} {ThisAssembly.Git.Branch} {ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Minor}.{ThisAssembly.Git.SemVer.Patch} {ThisAssembly.Git.SemVer.Label}"); } void printHelp () { Console.WriteLine ("Usage: PerfTests.exe [options]\n"); Console.WriteLine ("-o,--output:\n\tWrite results to output directory, if omited, results are printed to screen."); - Console.WriteLine ("-i,--input:\n\tInput directory to search recursively for '.crow' file to test."); + Console.WriteLine ("-i,--input:\n\tInput directory to search recursively for '.crow' file to test. If ommitted, builtin unit tests are performs"); Console.WriteLine ("-w,--width:\n\toutput surface width, not displayed on screen."); Console.WriteLine ("-h,--height:\n\toutput surface height, not displayed on screen."); Console.WriteLine ("-c,--count:\n\trepeat each test 'c' times."); - Console.WriteLine ("-m,--millisec:\n\tenable measure time in milisecond, if omitted measure in ticks."); + + Console.WriteLine ("-b,--begin:\n\tStarting stage for measures, may be the stage name or stage index"); + foreach (Stage s in Enum.GetValues(typeof(Stage))) { + Console.WriteLine ($"\t\t\t- [{(int)s}] {s}"); + } + Console.WriteLine ("-e,--end:\n\tEnding stage for measures, may be the stage name or stage index"); Console.WriteLine ("-r,--reset:\n\tenable clear iterators after each test file."); Console.WriteLine ("-u,--update:\n\tmeasure 'n' update cycle with DateTime.Now string notified."); Console.WriteLine ("-s,--screen:\n\tenable output to screen."); @@ -93,13 +110,15 @@ namespace PerfTests case "--height": clientRectangle.Height = int.Parse (args[i++]); break; - case "-m": - case "--millisec": - miliseconds = true; + case "-b": + case "--begin": + if (!Enum.TryParse (args[i++], out StartStage)) + StartStage = (Stage)int.Parse (args[i++]); break; - case "-r": - case "--reset": - resetItors = true; + case "-e": + case "--end": + if (!Enum.TryParse (args[i++], out EndStage)) + EndStage = (Stage)int.Parse (args[i++]); break; case "-u": case "--update": @@ -114,6 +133,11 @@ namespace PerfTests throw new Exception (); } } + if (EndStage < StartStage) { + Console.WriteLine ($"Ending stage (){EndStage} is before Starting stage ({StartStage})"); + throw new Exception (); + } + } catch (Exception) { printHelp (); throw; @@ -122,7 +146,7 @@ namespace PerfTests if (string.IsNullOrEmpty (outDir)) { writeHeader (Console.Out); Console.WriteLine (new string ('-', 100)); - Console.WriteLine ($"{" Path",-50}| Min | Mean | Max | Alloc(kb)| Lost(Kb) |"); + Console.WriteLine ($"{" Path",-50}| Time(ms) | AllocKB | MemKB | σ time | σ Alloc| σ Mem |"); } else { string dirName = Path.IsPathRooted (outDir) ? outDir : Path.Combine (Directory.GetCurrentDirectory (), outDir); @@ -134,26 +158,16 @@ namespace PerfTests outStream = new FileStream (outFilePath, FileMode.Create); writer = new StreamWriter (outStream); writeHeader (writer); - writer.WriteLine ("Path;Min;Mean;Max;Allocated;LostMem"); + writer.WriteLine ("Path;MinEllapsed;MaxEllapsed;MeanEllapsed;MedianEllapsed;sigmaEllapsed;MinMem;MaxMem;MeanMem;MedianMem;sigmaMem;MinAlloc;MaxAlloc;MeanAlloc;MedianAlloc;sigmaAlloc"); } if (screenOutput) initSurface (); else surf = new Crow.Cairo.ImageSurface (Crow.Cairo.Format.Argb32, ClientRectangle.Width, ClientRectangle.Height); - Init (); - } - public void PerformTests () { - string dirName = Path.IsPathRooted (inDirectory) ? inDirectory : - Path.Combine (Directory.GetCurrentDirectory (), inDirectory); - if (!Directory.Exists (dirName)) - throw new FileNotFoundException ("Input directory not found: " + dirName); - - testDir (dirName); + loadStyling (); - if (logToDisk) - Console.WriteLine ($"\ntest log written to: {outFilePath}"); } protected override void Dispose (bool disposing) { @@ -165,67 +179,248 @@ namespace PerfTests } } - long Test (string path, out long min, out long max) - { - min = long.MaxValue; - max = long.MinValue; + struct Measures + { + public double Elapsed; + public double AllocatedKB; + public double UsedKB; + } + struct Result + { + public double MinElapsed; + public double MaxElapsed; + public double MeanElapsed; + public double MedianElapsed; + public double sigmaElapsed; + + public double MinMem; + public double MaxMem; + public double MeanMem; + public double MedianMem; + public double sigmaMem; - long total = 0; + public double MinAlloc; + public double MaxAlloc; + public double MeanAlloc; + public double MedianAlloc; + public double sigmaAlloc; - Stopwatch sw = new Stopwatch (); + public override string ToString () => + $"{MinElapsed};{MaxElapsed};{MeanElapsed};{MedianElapsed};{sigmaElapsed};{MinMem};{MaxMem};{MeanMem};{MedianMem};{sigmaMem};{MinAlloc};{MaxAlloc};{MeanAlloc};{MedianAlloc};{sigmaAlloc}"; + } + + void getMemUsage (out long allocations, out long memory ) { + GC.Collect (); + GC.WaitForPendingFinalizers (); + allocations = GC.GetTotalAllocatedBytes (true); + memory = GC.GetTotalMemory (true); + //memory = GC.GetGCMemoryInfo ().HeapSizeBytes; + } + + Stopwatch chrono = new Stopwatch (); + double freq = 0.001 * Stopwatch.Frequency; + long totMemBefore = 0, allocBefore = 0, allocAfter = 0, totMemAfter = 0; + double trimMinElapsed = double.MaxValue, extMinElapsed = double.MaxValue; + double trimMaxElapsed = double.MinValue, extMaxElapsed = double.MinValue; + double totElapsed = 0; + double trimMinMem = double.MaxValue, extMinMem = double.MaxValue; + double trimMaxMem = double.MinValue, extMaxMem = double.MinValue; + double totMem = 0; + double trimMinAlloc = double.MaxValue, extMinAlloc = double.MaxValue; + double trimMaxAlloc = double.MinValue, extMaxAlloc = double.MinValue; + double totAlloc = 0; + + Measures[] measures = null; + Result result = default; + + void Test (string iml, bool isImlFragment = false) + { for (int i = 0; i < count; i++) { - if (updateCycles == 0) { - sw.Restart ();//dont measure load time when measuring updates - Load (path); - }else - Load (path).DataSource = this; - - Update (); - while (LayoutingQueue.Count > 0) - Update (); - - if (updateCycles > 0) { - sw.Restart (); - for (int j = 0; j < updateCycles; j++) { - NotifyValueChanged ("elapsed", sw.ElapsedTicks); - Update (); - while (LayoutingQueue.Count > 0) - Update (); - } + + if (StartStage == Stage.ITor) { + getMemUsage (out allocBefore, out totMemBefore); + chrono.Restart (); } - sw.Stop (); - - ClearInterface (); - if (resetItors) { - //this.Styling.Clear (); - //this.StylingConstants.Clear (); - this.Instantiators.Clear (); - this.Templates.Clear (); - this.DefaultTemplates.Clear (); - this.DefaultValuesLoader.Clear (); - GC.Collect (); - } - - if (miliseconds) { - if (sw.ElapsedMilliseconds < min) - min = sw.ElapsedMilliseconds; - if (sw.ElapsedMilliseconds > max) - max = sw.ElapsedMilliseconds; - total += sw.ElapsedMilliseconds; - } else { - if (sw.ElapsedTicks < min) - min = sw.ElapsedTicks; - if (sw.ElapsedTicks > max) - max = sw.ElapsedTicks; - total += sw.ElapsedTicks; + Crow.IML.Instantiator iTor = isImlFragment ? + Crow.IML.Instantiator.CreateFromImlFragment (this, iml) : new Crow.IML.Instantiator (this, iml); + + if (EndStage == Stage.ITor) { + chrono.Stop (); + getMemUsage (out allocAfter, out totMemAfter); + } else { + if (StartStage == Stage.Instantiation) { + getMemUsage (out allocBefore, out totMemBefore); + chrono.Restart (); + } + + Widget w = iTor.CreateInstance (); + + if (EndStage == Stage.Instantiation) { + chrono.Stop (); + getMemUsage (out allocAfter, out totMemAfter); + } else { + if (StartStage == Stage.Add) { + getMemUsage (out allocBefore, out totMemBefore); + chrono.Restart (); + } + + AddWidget (w); + + if (EndStage == Stage.Add) { + chrono.Stop (); + getMemUsage (out allocAfter, out totMemAfter); + } else { + if (StartStage == Stage.Datasource) { + getMemUsage (out allocBefore, out totMemBefore); + chrono.Restart (); + } + + w.DataSource = this; + + if (EndStage == Stage.Datasource) { + chrono.Stop (); + getMemUsage (out allocAfter, out totMemAfter); + } else { + if (StartStage == Stage.FirstUpdate) { + getMemUsage (out allocBefore, out totMemBefore); + chrono.Restart (); + } + + Update (); + while (LayoutingQueue.Count > 0) + Update (); + + if (EndStage == Stage.FirstUpdate) { + chrono.Stop (); + getMemUsage (out allocAfter, out totMemAfter); + } else { + if (StartStage == Stage.Update) { + getMemUsage (out allocBefore, out totMemBefore); + chrono.Restart (); + } + + for (int j = 0; j < updateCycles; j++) { + NotifyValueChanged ("elapsed", chrono.ElapsedTicks); + Update (); + while (LayoutingQueue.Count > 0) + Update (); + } + + if (EndStage == Stage.Update) { + chrono.Stop (); + getMemUsage (out allocAfter, out totMemAfter); + } else if (StartStage == Stage.Delete) { + getMemUsage (out allocBefore, out totMemBefore); + chrono.Restart (); + } + } + } + } + + DeleteWidget (w); + w = null; + + if (EndStage == Stage.Delete) { + chrono.Stop (); + iTor = null; + getMemUsage (out allocAfter, out totMemAfter); + } + } + + w?.Dispose (); } + LayoutingQueue.Clear (); + ClippingQueue.Clear (); + /*this.Instantiators.Clear (); + this.Templates.Clear (); + this.DefaultTemplates.Clear ();*/ + this.DefaultValuesLoader.Clear (); + + measures[i].AllocatedKB = (double)(allocAfter - allocBefore) / 1024.0; + measures[i].UsedKB = (double)(totMemAfter - totMemBefore) / 1024.0; + measures[i].Elapsed = (double)chrono.ElapsedTicks / freq; } - return total / count; + trimMinElapsed = double.MaxValue; extMinElapsed = double.MaxValue; + trimMaxElapsed = double.MinValue; extMaxElapsed = double.MinValue; + totElapsed = 0; + trimMinMem = double.MaxValue; extMinMem = double.MaxValue; + trimMaxMem = double.MinValue; extMaxMem = double.MinValue; + totMem = 0; + trimMinAlloc = double.MaxValue; extMinAlloc = double.MaxValue; + trimMaxAlloc = double.MinValue; extMaxAlloc = double.MinValue; + totAlloc = 0; + + for (int i = 0; i < count; i++) { + if (measures[i].Elapsed < extMinElapsed) + extMinElapsed = measures[i].Elapsed; + if (measures[i].Elapsed > extMaxElapsed) + extMaxElapsed = measures[i].Elapsed; + totElapsed += measures[i].Elapsed; + + if (measures[i].UsedKB < extMinMem) + extMinMem = measures[i].UsedKB; + if (measures[i].UsedKB > extMaxMem) + extMaxMem = measures[i].UsedKB; + totMem += measures[i].UsedKB; + + if (measures[i].AllocatedKB< extMinAlloc) + extMinAlloc = measures[i].AllocatedKB; + if (measures[i].AllocatedKB > extMaxAlloc) + extMaxAlloc = measures[i].AllocatedKB; + totAlloc += measures[i].AllocatedKB; + } + + for (int i = 0; i < count; i++) { + if (measures[i].Elapsed < trimMinElapsed && measures[i].Elapsed != extMinElapsed) + trimMinElapsed = measures[i].Elapsed; + if (measures[i].Elapsed > trimMaxElapsed && measures[i].Elapsed != extMaxElapsed) + trimMaxElapsed = measures[i].Elapsed; + + if (measures[i].UsedKB < trimMinMem && measures[i].UsedKB != extMinMem) + trimMinMem = measures[i].UsedKB; + if (measures[i].UsedKB > trimMaxMem && measures[i].UsedKB != extMaxMem) + trimMaxMem = measures[i].UsedKB; + + if (measures[i].AllocatedKB < trimMinAlloc && measures[i].AllocatedKB != extMinAlloc) + trimMinAlloc = measures[i].AllocatedKB; + if (measures[i].AllocatedKB > trimMaxAlloc && measures[i].AllocatedKB != extMaxAlloc) + trimMaxAlloc = measures[i].AllocatedKB; + } + + result.MinElapsed = trimMinElapsed; + result.MaxElapsed = trimMaxElapsed; + result.MeanElapsed = totElapsed / count; + result.MedianElapsed = (totElapsed - extMaxElapsed - extMinElapsed) / (count - 2); + + result.MinMem = trimMinMem; + result.MaxMem = trimMaxMem; + result.MeanMem = totMem / count; + result.MedianMem = (totMem - extMaxMem - extMinMem) / (count - 2); + + result.MinAlloc = trimMinAlloc; + result.MaxAlloc = trimMaxAlloc; + result.MeanAlloc = totAlloc / count; + result.MedianAlloc = (totAlloc - extMaxAlloc - extMinAlloc) / (count - 2); + + for (int i = 0; i < count; i++) { + if (measures[i].Elapsed != extMinElapsed && measures[i].Elapsed != extMaxElapsed) + result.sigmaElapsed += Math.Pow (measures[i].Elapsed - result.MedianElapsed, 2); + + if (measures[i].UsedKB != extMinMem && measures[i].UsedKB!= extMaxMem) + result.sigmaMem += Math.Pow (measures[i].UsedKB - result.MedianMem, 2); + + if (measures[i].AllocatedKB != extMinAlloc && measures[i].AllocatedKB != extMaxAlloc) + result.sigmaAlloc += Math.Pow (measures[i].AllocatedKB - result.MedianAlloc, 2); + } + + result.sigmaElapsed = Math.Sqrt (result.sigmaElapsed / (count - 2)); + result.sigmaMem = Math.Sqrt (result.sigmaMem / (count - 2)); + result.sigmaAlloc = Math.Sqrt (result.sigmaAlloc / (count - 2)); } void testDir (string dirPath, int level = 0) { @@ -240,21 +435,14 @@ namespace PerfTests foreach (string f in Directory.GetFiles (dirPath, "*.crow")) { label = $"{new string (' ', level * 4)}{ Path.GetFileName (f)}"; try { - long totMemBefore = GC.GetTotalMemory (true); - long allocBefore = GC.GetTotalAllocatedBytes (true); - long mean = Test (f, out long min, out long max); - long allocAfter = GC.GetTotalAllocatedBytes (true); - long totMemAfter = GC.GetTotalMemory (true); - double allocated = (double)(allocAfter - allocBefore) / 1024.0; - double lostMem = (double)(totMemAfter - totMemBefore) / 1024.0; + + Test (f); + if (logToDisk) { - writer.WriteLine ($"{f};{min};{mean};{max};{allocAfter - allocBefore};{totMemAfter - totMemBefore}"); + writer.WriteLine ($"{f};{result}"); Console.Write ("."); } else { - if (miliseconds) - Console.WriteLine ($"{label,-50}|{min,10}|{mean,10}|{max,10}| {allocated,8:0.0} | {lostMem,8:0.0} |"); - else - Console.WriteLine ($"{label,-50}|{0.001 * min,10:0.00}|{0.001 * mean,10:0.00}|{0.001 * max,10:0.00}| {allocated,8:0.0} | {lostMem,8:0.0} |"); + Console.WriteLine ($"{label,-50}|{result.MedianElapsed,10:0.000}|{result.MedianAlloc,10:0.000}|{result.MedianMem,8:0.000}|{result.sigmaElapsed,10:0.000}|{result.sigmaAlloc,8:0.000}|{result.sigmaMem,8:0.000}|"); } } catch (Exception ex) { if (logToDisk) { @@ -268,11 +456,64 @@ namespace PerfTests } } } - + + static string w = @""; + static string[] unitTests = + { + @"", + @"", + @"", + @"