diff --git a/src/BenchmarkDotNet/Code/CodeGenerator.cs b/src/BenchmarkDotNet/Code/CodeGenerator.cs index 005564b77c..183ce5b71a 100644 --- a/src/BenchmarkDotNet/Code/CodeGenerator.cs +++ b/src/BenchmarkDotNet/Code/CodeGenerator.cs @@ -45,7 +45,6 @@ internal static string Generate(BuildPartition buildPartition) .Replace("$ID$", buildInfo.Id.ToString()) .Replace("$OperationsPerInvoke$", provider.OperationsPerInvoke) .Replace("$WorkloadTypeName$", provider.WorkloadTypeName) - .Replace("$WorkloadMethodDelegate$", provider.WorkloadMethodDelegate(passArguments)) .Replace("$WorkloadMethodReturnType$", provider.WorkloadMethodReturnTypeName) .Replace("$WorkloadMethodReturnTypeModifiers$", provider.WorkloadMethodReturnTypeModifiers) .Replace("$OverheadMethodReturnTypeName$", provider.OverheadMethodReturnTypeName) diff --git a/src/BenchmarkDotNet/Code/DeclarationsProvider.cs b/src/BenchmarkDotNet/Code/DeclarationsProvider.cs index 7528e8ed62..ed6e641066 100644 --- a/src/BenchmarkDotNet/Code/DeclarationsProvider.cs +++ b/src/BenchmarkDotNet/Code/DeclarationsProvider.cs @@ -36,8 +36,6 @@ internal abstract class DeclarationsProvider public virtual string WorkloadMethodReturnTypeName => WorkloadMethodReturnType.GetCorrectCSharpTypeName(); - public virtual string WorkloadMethodDelegate(string passArguments) => Descriptor.WorkloadMethod.Name; - public virtual string WorkloadMethodReturnTypeModifiers => null; public virtual string GetWorkloadMethodCall(string passArguments) => $"{Descriptor.WorkloadMethod.Name}({passArguments})"; @@ -149,9 +147,6 @@ internal class TaskDeclarationsProvider : VoidDeclarationsProvider { public TaskDeclarationsProvider(Descriptor descriptor) : base(descriptor) { } - public override string WorkloadMethodDelegate(string passArguments) - => $"({passArguments}) => {{ BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments})); }}"; - public override string GetWorkloadMethodCall(string passArguments) => $"BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments}))"; protected override Type WorkloadMethodReturnType => typeof(void); @@ -166,9 +161,6 @@ public GenericTaskDeclarationsProvider(Descriptor descriptor) : base(descriptor) protected override Type WorkloadMethodReturnType => Descriptor.WorkloadMethod.ReturnType.GetTypeInfo().GetGenericArguments().Single(); - public override string WorkloadMethodDelegate(string passArguments) - => $"({passArguments}) => {{ return BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments})); }}"; - public override string GetWorkloadMethodCall(string passArguments) => $"BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments}))"; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/Reflection.Emit/IlGeneratorEmitOpExtensions.cs b/src/BenchmarkDotNet/Helpers/Reflection.Emit/IlGeneratorEmitOpExtensions.cs index 50a3392f44..8b405300d2 100644 --- a/src/BenchmarkDotNet/Helpers/Reflection.Emit/IlGeneratorEmitOpExtensions.cs +++ b/src/BenchmarkDotNet/Helpers/Reflection.Emit/IlGeneratorEmitOpExtensions.cs @@ -116,6 +116,22 @@ public static void EmitLdarg(this ILGenerator ilBuilder, ParameterInfo argument) } } + public static void EmitStarg(this ILGenerator ilBuilder, ParameterInfo argument) + { + var position = argument.Position; + if (!((MethodBase) argument.Member).IsStatic) + position++; + + if (position < 255) + { + ilBuilder.Emit(OpCodes.Starg_S, (byte) position); + } + else + { + ilBuilder.Emit(OpCodes.Starg, checked((short) position)); + } + } + public static void EmitLdindStind(this ILGenerator ilBuilder, Type resultType) { if (!resultType.IsByRef) diff --git a/src/BenchmarkDotNet/Helpers/Reflection.Emit/IlGeneratorStatementExtensions.cs b/src/BenchmarkDotNet/Helpers/Reflection.Emit/IlGeneratorStatementExtensions.cs index 6d601e6495..72f32a3a2e 100644 --- a/src/BenchmarkDotNet/Helpers/Reflection.Emit/IlGeneratorStatementExtensions.cs +++ b/src/BenchmarkDotNet/Helpers/Reflection.Emit/IlGeneratorStatementExtensions.cs @@ -84,64 +84,55 @@ public static void EmitSetDelegateToThisField( } } - public static void EmitLoopBeginFromLocToArg( + public static void EmitLoopBeginFromArgToZero( this ILGenerator ilBuilder, Label loopStartLabel, - Label loopHeadLabel, - LocalBuilder indexLocal, - ParameterInfo toArg) + Label loopHeadLabel) { - // loop counter stored as loc0, loop max passed as arg1 + // invokeCount passed as arg /* - // for (long i = 0L; i < invokeCount; i++) - IL_0000: ldc.i4.0 - IL_0001: conv.i8 - IL_0002: stloc.0 + // while (--invokeCount >= 0) */ - ilBuilder.Emit(OpCodes.Ldc_I4_0); - ilBuilder.Emit(OpCodes.Conv_I8); - ilBuilder.EmitStloc(indexLocal); - // IL_0003: br.s IL_0036 // loop head: IL_0036 // we use long jump + // IL_0000: br.s IL_000e // loop head: IL_000e // we use long jump ilBuilder.Emit(OpCodes.Br, loopHeadLabel); - // loop start (head: IL_0036) + // loop start (head: IL_000e) ilBuilder.MarkLabel(loopStartLabel); } - public static void EmitLoopEndFromLocToArg( + public static void EmitLoopEndFromArgToZero( this ILGenerator ilBuilder, Label loopStartLabel, Label loopHeadLabel, - LocalBuilder indexLocal, - ParameterInfo toArg) + ParameterInfo arg) { - // loop counter stored as loc0, loop max passed as arg1 + // invokeCount passed as arg /* - // for (long i = 0L; i < invokeCount; i++) - IL_0031: ldloc.0 - IL_0032: ldc.i4.1 - IL_0033: conv.i8 - IL_0034: add - IL_0035: stloc.0 - */ - ilBuilder.EmitLdloc(indexLocal); - ilBuilder.Emit(OpCodes.Ldc_I4_1); - ilBuilder.Emit(OpCodes.Conv_I8); - ilBuilder.Emit(OpCodes.Add); - ilBuilder.EmitStloc(indexLocal); - - /* - // for (long i = 0L; i < invokeCount; i++) - IL_0036: ldloc.0 // loop head: IL_0036 - IL_0037: ldarg.1 - IL_0038: blt.s IL_0005 // we use long jump + // while (--invokeCount >= 0) + IL_0008: ldarg.1 + IL_0009: ldc.i4.1 + IL_000a: conv.i8 + IL_000b: sub + IL_000c: dup + IL_000d: starg.s invokeCount + IL_000f: ldc.i4.0 + IL_0010: conv.i8 + IL_0011: bge.s IL_0002 // end loop */ + + // loop head ilBuilder.MarkLabel(loopHeadLabel); - ilBuilder.EmitLdloc(indexLocal); - ilBuilder.EmitLdarg(toArg); - ilBuilder.Emit(OpCodes.Blt, loopStartLabel); + ilBuilder.EmitLdarg(arg); + ilBuilder.Emit(OpCodes.Ldc_I4_1); + ilBuilder.Emit(OpCodes.Conv_I8); + ilBuilder.Emit(OpCodes.Sub); + ilBuilder.Emit(OpCodes.Dup); + ilBuilder.EmitStarg(arg); + ilBuilder.Emit(OpCodes.Ldc_I4_0); + ilBuilder.Emit(OpCodes.Conv_I8); + ilBuilder.Emit(OpCodes.Bge, loopStartLabel); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/Reflection.Emit/MethodBuilderExtensions.cs b/src/BenchmarkDotNet/Helpers/Reflection.Emit/MethodBuilderExtensions.cs index c09a7d19c2..73a703193d 100644 --- a/src/BenchmarkDotNet/Helpers/Reflection.Emit/MethodBuilderExtensions.cs +++ b/src/BenchmarkDotNet/Helpers/Reflection.Emit/MethodBuilderExtensions.cs @@ -38,13 +38,5 @@ public static MethodBuilder SetNoOptimizationImplementationFlag(this MethodBuild return methodBuilder; } - - public static MethodBuilder SetAggressiveOptimizationImplementationFlag(this MethodBuilder methodBuilder) - { - methodBuilder.SetImplementationFlags( - methodBuilder.GetMethodImplementationFlags() | CodeGenHelper.AggressiveOptimizationOptionForEmit); - - return methodBuilder; - } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/Reflection.Emit/ModuleBuilderExtensions.cs b/src/BenchmarkDotNet/Helpers/Reflection.Emit/ModuleBuilderExtensions.cs deleted file mode 100644 index c879bc1f46..0000000000 --- a/src/BenchmarkDotNet/Helpers/Reflection.Emit/ModuleBuilderExtensions.cs +++ /dev/null @@ -1,60 +0,0 @@ -using System; -using System.Linq; -using System.Reflection; -using System.Reflection.Emit; - -namespace BenchmarkDotNet.Helpers.Reflection.Emit -{ - internal static class ModuleBuilderExtensions - { - public static Type EmitCustomDelegate( - this ModuleBuilder moduleBuilder, - string delegateTypeName, - ParameterInfo returnType, - ParameterInfo[] parameters) - { - // TODO: begin/end invoke ? - var delegatePatternType = typeof(Action); - - var typeBuilder = moduleBuilder.DefineType( - delegateTypeName, - delegatePatternType.Attributes, - delegatePatternType.BaseType); - - var ctorPattern = delegatePatternType.GetConstructors().Single(); - var ctorBuilder = typeBuilder.DefineConstructor( - ctorPattern.Attributes, - ctorPattern.CallingConvention, - ctorPattern.GetParameterTypes()); - - foreach (var parameterInfo in ctorPattern.GetParameters()) - { - ctorBuilder.DefineParameter(parameterInfo.Position + 1, parameterInfo.Attributes, parameterInfo.Name); - } - - ctorBuilder.SetImplementationFlags(ctorPattern.GetMethodImplementationFlags()); - - var invokePatternMethod = TypeBuilderExtensions.GetDelegateInvokeMethod(delegatePatternType); - - var invokeBuilder = typeBuilder.DefineMethod( - invokePatternMethod.Name, - invokePatternMethod.Attributes, - invokePatternMethod.CallingConvention, - returnType.ParameterType, - parameters.Select(p => p.ParameterType).ToArray()); - foreach (var parameterInfo in parameters) - { - invokeBuilder.DefineParameter(parameterInfo.Position + 1, parameterInfo.Attributes, parameterInfo.Name); - } - invokeBuilder.DefineParameter(0, returnType.Attributes, ""); - - invokeBuilder.SetImplementationFlags(invokePatternMethod.GetMethodImplementationFlags()); - -#if NETFRAMEWORK - return typeBuilder.CreateType(); -#else - return typeBuilder.CreateTypeInfo(); -#endif - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/CodeGen.cs b/src/BenchmarkDotNet/Portability/CodeGen.cs index 0db9306b43..cf55606a70 100644 --- a/src/BenchmarkDotNet/Portability/CodeGen.cs +++ b/src/BenchmarkDotNet/Portability/CodeGen.cs @@ -7,6 +7,5 @@ public static class CodeGenHelper { // AggressiveOptimization is not available in netstandard2.0, so just use the value casted to enum. public const MethodImplOptions AggressiveOptimizationOption = (MethodImplOptions) 512; - public const MethodImplAttributes AggressiveOptimizationOptionForEmit = (MethodImplAttributes) 512; } } diff --git a/src/BenchmarkDotNet/Templates/BenchmarkType.txt b/src/BenchmarkDotNet/Templates/BenchmarkType.txt index f17737d646..560aace8cb 100644 --- a/src/BenchmarkDotNet/Templates/BenchmarkType.txt +++ b/src/BenchmarkDotNet/Templates/BenchmarkType.txt @@ -1,5 +1,5 @@ // the type name must be in sync with WindowsDisassembler.BuildArguments - public unsafe class Runnable_$ID$ : global::$WorkloadTypeName$ + public unsafe sealed class Runnable_$ID$ : global::$WorkloadTypeName$ { public static void Run(BenchmarkDotNet.Engines.IHost host, System.String benchmarkName) { @@ -51,18 +51,12 @@ } } - public delegate $OverheadMethodReturnTypeName$ OverheadDelegate($ArgumentsDefinition$); - - public delegate $WorkloadMethodReturnTypeModifiers$ $WorkloadMethodReturnType$ WorkloadDelegate($ArgumentsDefinition$); - public Runnable_$ID$() { globalSetupAction = $GlobalSetupMethodName$; globalCleanupAction = $GlobalCleanupMethodName$; iterationSetupAction = $IterationSetupMethodName$; iterationCleanupAction = $IterationCleanupMethodName$; - overheadDelegate = __Overhead; - workloadDelegate = $WorkloadMethodDelegate$; $InitializeArgumentFields$ } @@ -70,8 +64,6 @@ private System.Action globalCleanupAction; private System.Action iterationSetupAction; private System.Action iterationCleanupAction; - private BenchmarkDotNet.Autogenerated.Runnable_$ID$.OverheadDelegate overheadDelegate; - private BenchmarkDotNet.Autogenerated.Runnable_$ID$.WorkloadDelegate workloadDelegate; $DeclareArgumentFields$ // this method is used only for the disassembly diagnoser purposes @@ -112,44 +104,45 @@ #if RETURNS_CONSUMABLE_$ID$ private BenchmarkDotNet.Engines.Consumer consumer = new BenchmarkDotNet.Engines.Consumer(); - - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + + // Prevent inlining the method invoke. + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void OverheadActionUnroll(System.Int64 invokeCount) { $LoadArguments$ - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - consumer.Consume(overheadDelegate($PassArguments$));@Unroll@ + consumer.Consume(__Overhead($PassArguments$));@Unroll@ } } - - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void OverheadActionNoUnroll(System.Int64 invokeCount) { $LoadArguments$ - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - consumer.Consume(overheadDelegate($PassArguments$)); + consumer.Consume(__Overhead($PassArguments$)); } } - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void WorkloadActionUnroll(System.Int64 invokeCount) { $LoadArguments$ - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - consumer.Consume(workloadDelegate($PassArguments$)$ConsumeField$);@Unroll@ + consumer.Consume($WorkloadMethodCall$$ConsumeField$);@Unroll@ } } - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void WorkloadActionNoUnroll(System.Int64 invokeCount) { $LoadArguments$ - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - consumer.Consume(workloadDelegate($PassArguments$)$ConsumeField$); + consumer.Consume($WorkloadMethodCall$$ConsumeField$); } } @@ -167,50 +160,50 @@ #elif RETURNS_NON_CONSUMABLE_STRUCT_$ID$ - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void OverheadActionUnroll(System.Int64 invokeCount) { $LoadArguments$ $OverheadMethodReturnTypeName$ result = default($OverheadMethodReturnTypeName$); - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - result = overheadDelegate($PassArguments$);@Unroll@ + result = __Overhead($PassArguments$);@Unroll@ } BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxing(result); } - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void OverheadActionNoUnroll(System.Int64 invokeCount) { $LoadArguments$ $OverheadMethodReturnTypeName$ result = default($OverheadMethodReturnTypeName$); - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - result = overheadDelegate($PassArguments$); + result = __Overhead($PassArguments$); } BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxing(result); } - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void WorkloadActionUnroll(System.Int64 invokeCount) { $LoadArguments$ $WorkloadMethodReturnType$ result = default($WorkloadMethodReturnType$); - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - result = workloadDelegate($PassArguments$);@Unroll@ + result = $WorkloadMethodCall$;@Unroll@ } NonGenericKeepAliveWithoutBoxing(result); } - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void WorkloadActionNoUnroll(System.Int64 invokeCount) { $LoadArguments$ $WorkloadMethodReturnType$ result = default($WorkloadMethodReturnType$); - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - result = workloadDelegate($PassArguments$); + result = $WorkloadMethodCall$; } NonGenericKeepAliveWithoutBoxing(result); } @@ -234,52 +227,52 @@ #elif RETURNS_BYREF_$ID$ - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void OverheadActionUnroll(System.Int64 invokeCount) { $LoadArguments$ $OverheadMethodReturnTypeName$ value = default($OverheadMethodReturnTypeName$); - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - value = overheadDelegate($PassArguments$);@Unroll@ + value = __Overhead($PassArguments$);@Unroll@ } BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxing(value); } - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void OverheadActionNoUnroll(System.Int64 invokeCount) { $LoadArguments$ $OverheadMethodReturnTypeName$ value = default($OverheadMethodReturnTypeName$); - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - value = overheadDelegate($PassArguments$); + value = __Overhead($PassArguments$); } BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxing(value); } private $WorkloadMethodReturnType$ workloadDefaultValueHolder = default($WorkloadMethodReturnType$); - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void WorkloadActionUnroll(System.Int64 invokeCount) { $LoadArguments$ ref $WorkloadMethodReturnType$ alias = ref workloadDefaultValueHolder; - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - alias = workloadDelegate($PassArguments$);@Unroll@ + alias = $WorkloadMethodCall$;@Unroll@ } BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxing(ref alias); } - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void WorkloadActionNoUnroll(System.Int64 invokeCount) { $LoadArguments$ ref $WorkloadMethodReturnType$ alias = ref workloadDefaultValueHolder; - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - alias = workloadDelegate($PassArguments$); + alias = $WorkloadMethodCall$; } BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxing(ref alias); } @@ -297,52 +290,52 @@ } #elif RETURNS_BYREF_READONLY_$ID$ - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void OverheadActionUnroll(System.Int64 invokeCount) { $LoadArguments$ $OverheadMethodReturnTypeName$ value = default($OverheadMethodReturnTypeName$); - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - value = overheadDelegate($PassArguments$);@Unroll@ + value = __Overhead($PassArguments$);@Unroll@ } BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxing(value); } - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void OverheadActionNoUnroll(System.Int64 invokeCount) { $LoadArguments$ $OverheadMethodReturnTypeName$ value = default($OverheadMethodReturnTypeName$); - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - value = overheadDelegate($PassArguments$); + value = __Overhead($PassArguments$); } BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxing(value); } private $WorkloadMethodReturnType$ workloadDefaultValueHolder = default($WorkloadMethodReturnType$); - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void WorkloadActionUnroll(System.Int64 invokeCount) { $LoadArguments$ ref $WorkloadMethodReturnType$ alias = ref workloadDefaultValueHolder; - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - alias = workloadDelegate($PassArguments$);@Unroll@ + alias = $WorkloadMethodCall$;@Unroll@ } BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxingReadonly(alias); } - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void WorkloadActionNoUnroll(System.Int64 invokeCount) { $LoadArguments$ ref $WorkloadMethodReturnType$ alias = ref workloadDefaultValueHolder; - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - alias = workloadDelegate($PassArguments$); + alias = $WorkloadMethodCall$; } BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxingReadonly(alias); } @@ -360,43 +353,43 @@ } #elif RETURNS_VOID_$ID$ - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void OverheadActionUnroll(System.Int64 invokeCount) { $LoadArguments$ - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - overheadDelegate($PassArguments$);@Unroll@ + __Overhead($PassArguments$);@Unroll@ } } - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void OverheadActionNoUnroll(System.Int64 invokeCount) { $LoadArguments$ - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - overheadDelegate($PassArguments$); + __Overhead($PassArguments$); } } - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void WorkloadActionUnroll(System.Int64 invokeCount) { $LoadArguments$ - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - workloadDelegate($PassArguments$);@Unroll@ + $WorkloadMethodCall$;@Unroll@ } } - [System.Runtime.CompilerServices.MethodImpl(BenchmarkDotNet.Portability.CodeGenHelper.AggressiveOptimizationOption)] + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private void WorkloadActionNoUnroll(System.Int64 invokeCount) { $LoadArguments$ - for (System.Int64 i = 0; i < invokeCount; i++) + while (--invokeCount >= 0) { - workloadDelegate($PassArguments$); + $WorkloadMethodCall$; } } diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs index 0e9d386a24..3afd5c239c 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs @@ -170,7 +170,7 @@ private static TypeBuilder DefineRunnableTypeBuilder( BenchmarkBuildInfo benchmark, ModuleBuilder moduleBuilder) { - // .class public auto ansi beforefieldinit BenchmarkDotNet.Autogenerated.Runnable_0 + // .class public auto ansi sealed beforefieldinit BenchmarkDotNet.Autogenerated.Runnable_0 // extends [BenchmarkDotNet]BenchmarkDotNet.Samples.SampleBenchmark var benchmarkDescriptor = benchmark.BenchmarkCase.Descriptor; @@ -184,7 +184,7 @@ private static TypeBuilder DefineRunnableTypeBuilder( } var result = moduleBuilder.DefineType( GetRunnableTypeName(benchmark), - workloadTypeAttributes, + workloadTypeAttributes | TypeAttributes.Sealed, workloadType); return result; @@ -242,8 +242,6 @@ private static void EmitNoArgsMethodCallPopReturn( private int jobUnrollFactor; private int dummyUnrollFactor; - private Type overheadDelegateType; - private Type workloadDelegateType; private TypeBuilder runnableBuilder; private ConsumableTypeInfo consumableInfo; private ConsumeEmitter consumeEmitter; @@ -256,8 +254,6 @@ private static void EmitNoArgsMethodCallPopReturn( private FieldBuilder globalCleanupActionField; private FieldBuilder iterationSetupActionField; private FieldBuilder iterationCleanupActionField; - private FieldBuilder overheadDelegateField; - private FieldBuilder workloadDelegateField; private FieldBuilder notElevenField; private FieldBuilder dummyVarField; @@ -267,7 +263,6 @@ private static void EmitNoArgsMethodCallPopReturn( private MethodBuilder dummy1Method; private MethodBuilder dummy2Method; private MethodBuilder dummy3Method; - private MethodInfo workloadImplementationMethod; private MethodBuilder overheadImplementationMethod; private MethodBuilder overheadActionUnrollMethod; private MethodBuilder overheadActionNoUnrollMethod; @@ -325,7 +320,6 @@ private Type EmitRunnableCore(BenchmarkBuildInfo newBenchmark) overheadActionNoUnrollMethod = EmitOverheadAction(OverheadActionNoUnrollMethodName, 1); // Workload impl - workloadImplementationMethod = EmitWorkloadImplementation(WorkloadImplementationMethodName); workloadActionUnrollMethod = EmitWorkloadAction(WorkloadActionUnrollMethodName, jobUnrollFactor); workloadActionNoUnrollMethod = EmitWorkloadAction(WorkloadActionNoUnrollMethodName, 1); @@ -367,10 +361,8 @@ private void InitForEmitRunnable(BenchmarkBuildInfo newBenchmark) iterationSetupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.IterationSetupMethod?.ReturnType); iterationCleanupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.IterationCleanupMethod?.ReturnType); - // Init types + // Init type runnableBuilder = DefineRunnableTypeBuilder(benchmark, moduleBuilder); - overheadDelegateType = EmitOverheadDelegateType(); - workloadDelegateType = EmitWorkloadDelegateType(); } private static ConsumableTypeInfo GetConsumableTypeInfo(Type methodReturnType) @@ -378,52 +370,6 @@ private static ConsumableTypeInfo GetConsumableTypeInfo(Type methodReturnType) return methodReturnType == null ? null : new ConsumableTypeInfo(methodReturnType); } - private Type EmitOverheadDelegateType() - { - // .class public auto ansi sealed BenchmarkDotNet.Autogenerated.Runnable_0OverheadDelegate - // extends[mscorlib]System.MulticastDelegate; - var overheadReturnType = EmitParameterInfo.CreateReturnParameter(consumableInfo.OverheadMethodReturnType); - - // replace arg names - var overheadParameters = Descriptor.WorkloadMethod.GetParameters() - .Select(p => - (ParameterInfo)new EmitParameterInfo( - p.Position, - ArgParamPrefix + p.Position, - p.ParameterType, - p.Attributes, - null)) - .ToArray(); - - return moduleBuilder.EmitCustomDelegate( - GetRunnableTypeName(benchmark) + OverheadDelegateTypeSuffix, - overheadReturnType, - overheadParameters); - } - - private Type EmitWorkloadDelegateType() - { - // .class public auto ansi sealed BenchmarkDotNet.Autogenerated.Runnable_0WorkloadDelegate - // extends [mscorlib]System.MulticastDelegate - var workloadReturnType = EmitParameterInfo.CreateReturnParameter(consumableInfo.WorkloadMethodReturnType); - - // Replace arg names - var workloadParameters = Descriptor.WorkloadMethod.GetParameters() - .Select(p => - (ParameterInfo)new EmitParameterInfo( - p.Position, - ArgParamPrefix + p.Position, - p.ParameterType, - p.Attributes, - null)) - .ToArray(); - - return moduleBuilder.EmitCustomDelegate( - GetRunnableTypeName(benchmark) + WorkloadDelegateTypeSuffix, - workloadReturnType, - workloadParameters); - } - private void DefineFields() { globalSetupActionField = @@ -434,10 +380,6 @@ private void DefineFields() runnableBuilder.DefineField(IterationSetupActionFieldName, typeof(Action), FieldAttributes.Private); iterationCleanupActionField = runnableBuilder.DefineField(IterationCleanupActionFieldName, typeof(Action), FieldAttributes.Private); - overheadDelegateField = - runnableBuilder.DefineField(OverheadDelegateFieldName, overheadDelegateType, FieldAttributes.Private); - workloadDelegateField = - runnableBuilder.DefineField(WorkloadDelegateFieldName, workloadDelegateType, FieldAttributes.Private); // Define arg fields foreach (var parameter in Descriptor.WorkloadMethod.GetParameters()) @@ -553,15 +495,25 @@ private MethodBuilder EmitDummyMethod(string methodName, int unrollFactor) private MethodBuilder EmitOverheadImplementation(string methodName) { - var overheadInvokeMethod = TypeBuilderExtensions.GetDelegateInvokeMethod(overheadDelegateType); - //.method private hidebysig // instance int32 __Overhead(int64 arg0) cil managed + + // Replace arg names + var parameters = Descriptor.WorkloadMethod.GetParameters() + .Select(p => + (ParameterInfo) new EmitParameterInfo( + p.Position, + ArgParamPrefix + p.Position, + p.ParameterType, + p.Attributes, + null)) + .ToArray(); + var methodBuilder = runnableBuilder.DefineNonVirtualInstanceMethod( methodName, MethodAttributes.Private, - overheadInvokeMethod.ReturnParameter, - overheadInvokeMethod.GetParameters()); + EmitParameterInfo.CreateReturnParameter(consumableInfo.OverheadMethodReturnType), + parameters); var ilBuilder = methodBuilder.GetILGenerator(); var returnType = methodBuilder.ReturnType; @@ -578,54 +530,6 @@ private MethodBuilder EmitOverheadImplementation(string methodName) return methodBuilder; } - private MethodInfo EmitWorkloadImplementation(string methodName) - { - // Shortcut: DO NOT emit method if the result type is not awaitable - if (!consumableInfo.IsAwaitable) - return Descriptor.WorkloadMethod; - - var workloadInvokeMethod = TypeBuilderExtensions.GetDelegateInvokeMethod(workloadDelegateType); - - //.method private hidebysig - // instance int32 __Workload(int64 arg0) cil managed - var args = workloadInvokeMethod.GetParameters(); - var methodBuilder = runnableBuilder.DefineNonVirtualInstanceMethod( - methodName, - MethodAttributes.Private, - workloadInvokeMethod.ReturnParameter, - args); - args = methodBuilder.GetEmitParameters(args); - - var ilBuilder = methodBuilder.GetILGenerator(); - - /* - IL_0026: ldarg.0 - IL_0027: ldloc.0 - IL_0028: ldloc.1 - IL_0029: ldloc.2 - IL_002a: ldloc.3 - IL_002b: call instance class [System.Private.CoreLib]System.Threading.Tasks.Task`1 BenchmarkDotNet.Helpers.Runnable_0::WorkloadMethod(string, string, string, string) - */ - if (!Descriptor.WorkloadMethod.IsStatic) - ilBuilder.Emit(OpCodes.Ldarg_0); - ilBuilder.EmitLdargs(args); - ilBuilder.Emit(OpCodes.Call, Descriptor.WorkloadMethod); - - /* - // BenchmarkDotNet.Helpers.AwaitHelper.GetResult(...); - IL_000e: call !!0 BenchmarkDotNet.Helpers.AwaitHelper::GetResult(valuetype [System.Runtime]System.Threading.Tasks.ValueTask`1) - */ - - ilBuilder.Emit(OpCodes.Call, consumableInfo.GetResultMethod); - - /* - IL_0014: ret - */ - ilBuilder.Emit(OpCodes.Ret); - - return methodBuilder; - } - private MethodBuilder EmitOverheadAction(string methodName, int unrollFactor) { return EmitActionImpl(methodName, RunnableActionKind.Overhead, unrollFactor); @@ -636,43 +540,70 @@ private MethodBuilder EmitWorkloadAction(string methodName, int unrollFactor) return EmitActionImpl(methodName, RunnableActionKind.Workload, unrollFactor); } + private void EmitCallOverhead(ILGenerator ilBuilder, IReadOnlyList argLocals) + { + ilBuilder.Emit(OpCodes.Ldarg_0); + ilBuilder.EmitLdLocals(argLocals); + ilBuilder.Emit(OpCodes.Call, overheadImplementationMethod); + } + + private void EmitCallWorkload(ILGenerator ilBuilder, IReadOnlyList argLocals) + { + MethodInfo invokeMethod = Descriptor.WorkloadMethod; + if (!invokeMethod.IsStatic) + { + ilBuilder.Emit(OpCodes.Ldarg_0); + } + ilBuilder.EmitLdLocals(argLocals); + ilBuilder.Emit(OpCodes.Call, invokeMethod); + + if (consumableInfo.IsAwaitable) + { + /* + // BenchmarkDotNet.Helpers.AwaitHelper.GetResult(...); + IL_000e: call !!0 BenchmarkDotNet.Helpers.AwaitHelper::GetResult(valuetype [System.Runtime]System.Threading.Tasks.ValueTask`1) + */ + ilBuilder.Emit(OpCodes.Call, consumableInfo.GetResultMethod); + } + } + private MethodBuilder EmitActionImpl(string methodName, RunnableActionKind actionKind, int unrollFactor) { - FieldInfo actionDelegateField; - MethodInfo actionInvokeMethod; + Action> callMethodEmitter; + MethodInfo invokeMethod; switch (actionKind) { case RunnableActionKind.Overhead: - actionDelegateField = overheadDelegateField; - actionInvokeMethod = TypeBuilderExtensions.GetDelegateInvokeMethod(overheadDelegateType); + callMethodEmitter = EmitCallOverhead; + invokeMethod = overheadImplementationMethod; break; case RunnableActionKind.Workload: - actionDelegateField = workloadDelegateField; - actionInvokeMethod = TypeBuilderExtensions.GetDelegateInvokeMethod(workloadDelegateType); + callMethodEmitter = EmitCallWorkload; + invokeMethod = Descriptor.WorkloadMethod; break; default: throw new ArgumentOutOfRangeException(nameof(actionKind), actionKind, null); } // .method private hidebysig - // instance void OverheadActionUnroll(int64 invokeCount) cil managed aggressiveoptimization - var toArg = new EmitParameterInfo(0, InvokeCountParamName, typeof(long)); + // instance void OverheadActionUnroll(int64 invokeCount) cil managed noinlining nooptimization + var invokeCountArg = new EmitParameterInfo(0, InvokeCountParamName, typeof(long)); var actionMethodBuilder = runnableBuilder.DefineNonVirtualInstanceMethod( methodName, MethodAttributes.Private, EmitParameterInfo.CreateReturnVoidParameter(), - toArg) - .SetAggressiveOptimizationImplementationFlag(); - toArg.SetMember(actionMethodBuilder); + invokeCountArg) + .SetNoOptimizationImplementationFlag() + .SetNoInliningImplementationFlag(); + invokeCountArg.SetMember(actionMethodBuilder); // Emit impl var ilBuilder = actionMethodBuilder.GetILGenerator(); - consumeEmitter.BeginEmitAction(actionMethodBuilder, ilBuilder, actionInvokeMethod, actionKind); + consumeEmitter.BeginEmitAction(actionMethodBuilder, ilBuilder, invokeMethod, actionKind); // init locals var argLocals = EmitDeclareArgLocals(ilBuilder); consumeEmitter.DeclareActionLocals(ilBuilder); - var indexLocal = ilBuilder.DeclareLocal(typeof(long)); // load fields EmitLoadArgFieldsToLocals(ilBuilder, argLocals); @@ -681,7 +612,7 @@ private MethodBuilder EmitActionImpl(string methodName, RunnableActionKind actio // loop var loopStartLabel = ilBuilder.DefineLabel(); var loopHeadLabel = ilBuilder.DefineLabel(); - ilBuilder.EmitLoopBeginFromLocToArg(loopStartLabel, loopHeadLabel, indexLocal, toArg); + ilBuilder.EmitLoopBeginFromArgToZero(loopStartLabel, loopHeadLabel); { /* // overheadDelegate(); @@ -702,14 +633,12 @@ private MethodBuilder EmitActionImpl(string methodName, RunnableActionKind actio { consumeEmitter.EmitActionBeforeCall(ilBuilder); - ilBuilder.Emit(OpCodes.Ldarg_0); - ilBuilder.Emit(OpCodes.Ldfld, actionDelegateField); - ilBuilder.EmitInstanceCallThisValueOnStack(null, actionInvokeMethod, argLocals); + callMethodEmitter(ilBuilder, argLocals); consumeEmitter.EmitActionAfterCall(ilBuilder); } } - ilBuilder.EmitLoopEndFromLocToArg(loopStartLabel, loopHeadLabel, indexLocal, toArg); + ilBuilder.EmitLoopEndFromArgToZero(loopStartLabel, loopHeadLabel, invokeCountArg); consumeEmitter.EmitActionAfterLoop(ilBuilder); consumeEmitter.CompleteEmitAction(ilBuilder); @@ -993,12 +922,6 @@ private void EmitCtorBody() ilBuilder.EmitSetDelegateToThisField(globalCleanupActionField, globalCleanupMethod); ilBuilder.EmitSetDelegateToThisField(iterationSetupActionField, iterationSetupMethod); ilBuilder.EmitSetDelegateToThisField(iterationCleanupActionField, iterationCleanupMethod); - ilBuilder.EmitSetDelegateToThisField(overheadDelegateField, overheadImplementationMethod); - - if (workloadImplementationMethod == null) - ilBuilder.EmitSetDelegateToThisField(workloadDelegateField, Descriptor.WorkloadMethod); - else - ilBuilder.EmitSetDelegateToThisField(workloadDelegateField, workloadImplementationMethod); ilBuilder.EmitCtorReturn(ctorMethod); } diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableConstants.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableConstants.cs index c6e8cd8ae1..312d95651f 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableConstants.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableConstants.cs @@ -20,8 +20,6 @@ public class RunnableConstants public const string GlobalCleanupActionFieldName = "globalCleanupAction"; public const string IterationSetupActionFieldName = "iterationSetupAction"; public const string IterationCleanupActionFieldName = "iterationCleanupAction"; - public const string WorkloadDelegateFieldName = "workloadDelegate"; - public const string OverheadDelegateFieldName = "overheadDelegate"; public const string NotElevenFieldName = "NotEleven"; public const string DummyVarFieldName = "dummyVar"; diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs index 29bf8275dd..1473c1be0f 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -13,25 +13,26 @@ using BenchmarkDotNet.Reports; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Perfolizer; using Perfolizer.Horology; +using Perfolizer.Mathematics.Common; +using Perfolizer.Mathematics.SignificanceTesting; +using Perfolizer.Mathematics.SignificanceTesting.MannWhitney; using Perfolizer.Metrology; using Xunit; using Xunit.Abstractions; namespace BenchmarkDotNet.IntegrationTests.ManualRunning { - public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor + public class ExpectedBenchmarkResultsTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) { // NativeAot takes a long time to build, so not including it in these tests. // We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains. private static readonly TimeInterval FallbackCpuResolutionValue = TimeInterval.FromNanoseconds(0.2d); - public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { } - private static IEnumerable EmptyBenchmarkTypes() => - new[] - { + [ typeof(EmptyVoid), typeof(EmptyByte), typeof(EmptySByte), @@ -46,7 +47,7 @@ private static IEnumerable EmptyBenchmarkTypes() => typeof(EmptyUIntPtr), typeof(EmptyVoidPointer), typeof(EmptyClass) - }; + ]; public static IEnumerable InProcessData() { @@ -60,8 +61,8 @@ public static IEnumerable CoreData() { foreach (var type in EmptyBenchmarkTypes()) { - yield return new object[] { type, RuntimeMoniker.Net70 }; - yield return new object[] { type, RuntimeMoniker.Mono70 }; + yield return new object[] { type, RuntimeMoniker.Net80 }; + yield return new object[] { type, RuntimeMoniker.Mono80 }; } } @@ -76,17 +77,19 @@ public static IEnumerable FrameworkData() [Theory] [MemberData(nameof(InProcessData))] - public void EmptyBenchmarksReportZeroTimeAndAllocated_InProcess(Type benchmarkType) + public void EmptyBenchmarkReportsZeroTimeAndAllocated_InProcess(Type benchmarkType) { AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty() .AddJob(Job.Default .WithToolchain(InProcessEmitToolchain.Instance) - )); + // IL Emit has incorrect overhead measurement. https://github.com/dotnet/runtime/issues/89685 + // We multiply the threshold to account for it. + ), multiplyThresholdBy: RuntimeInformation.IsNetCore ? 3 : 1); } [TheoryEnvSpecific("To not repeat tests in both Full .NET Framework and Core", EnvRequirement.DotNetCoreOnly)] [MemberData(nameof(CoreData))] - public void EmptyBenchmarksReportZeroTimeAndAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker) + public void EmptyBenchmarkReportsZeroTimeAndAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker) { AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty() .AddJob(Job.Default @@ -96,7 +99,7 @@ public void EmptyBenchmarksReportZeroTimeAndAllocated_Core(Type benchmarkType, R [TheoryEnvSpecific("Can only run Full .NET Framework and Mono tests from Framework host", EnvRequirement.FullFrameworkOnly)] [MemberData(nameof(FrameworkData))] - public void EmptyBenchmarksReportZeroTimeAndAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker) + public void EmptyBenchmarkReportsZeroTimeAndAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker) { AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty() .AddJob(Job.Default @@ -104,7 +107,7 @@ public void EmptyBenchmarksReportZeroTimeAndAllocated_Framework(Type benchmarkTy )); } - private void AssertZeroResults(Type benchmarkType, IConfig config) + private void AssertZeroResults(Type benchmarkType, IConfig config, int multiplyThresholdBy = 1) { var summary = CanExecute(benchmarkType, config .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond)) @@ -112,7 +115,7 @@ private void AssertZeroResults(Type benchmarkType, IConfig config) ); var cpuResolution = CpuDetector.Cpu?.MaxFrequency()?.ToResolution() ?? FallbackCpuResolutionValue; - var threshold = new NumberValue(cpuResolution.Nanoseconds).ToThreshold(); + var threshold = new NumberValue(cpuResolution.Nanoseconds * multiplyThresholdBy).ToThreshold(); foreach (var report in summary.Reports) { @@ -126,87 +129,137 @@ private void AssertZeroResults(Type benchmarkType, IConfig config) } } - [Fact] - public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_InProcess() + private static IEnumerable NonEmptyBenchmarkTypes() => + [ + typeof(DifferentSizedStructs), + typeof(ActualWork) + ]; + + public static IEnumerable NonEmptyInProcessData() + { + foreach (var type in NonEmptyBenchmarkTypes()) + { + yield return new object[] { type }; + } + } + + public static IEnumerable NonEmptyCoreData() + { + foreach (var type in NonEmptyBenchmarkTypes()) + { + yield return new object[] { type, RuntimeMoniker.Net80 }; + yield return new object[] { type, RuntimeMoniker.Mono80 }; + } + } + + public static IEnumerable NonEmptyFrameworkData() { - AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty() + foreach (var type in NonEmptyBenchmarkTypes()) + { + yield return new object[] { type, RuntimeMoniker.Net462 }; + yield return new object[] { type, RuntimeMoniker.Mono }; + } + } + + [Theory] + [MemberData(nameof(NonEmptyInProcessData))] + public void NonEmptyBenchmarkReportsNonZeroTimeAndZeroAllocated_InProcess(Type benchmarkType) + { + AssertNonZeroResults(benchmarkType, ManualConfig.CreateEmpty() .AddJob(Job.Default .WithToolchain(InProcessEmitToolchain.Instance) - )); + // InProcess overhead measurements are incorrect, so we adjust the results to account for it. https://github.com/dotnet/runtime/issues/89685 + ), subtractOverheadByClocks: RuntimeInformation.IsNetCore ? 3 : 1); } [TheoryEnvSpecific("To not repeat tests in both Full .NET Framework and Core", EnvRequirement.DotNetCoreOnly)] - [InlineData(RuntimeMoniker.Net70)] - [InlineData(RuntimeMoniker.Mono70)] - public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Core(RuntimeMoniker runtimeMoniker) + [MemberData(nameof(NonEmptyCoreData))] + public void NonEmptyBenchmarkReportsNonZeroTimeAndZeroAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker) { - AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty() + AssertNonZeroResults(benchmarkType, ManualConfig.CreateEmpty() .AddJob(Job.Default .WithRuntime(runtimeMoniker.GetRuntime()) )); } - [TheoryEnvSpecific("Can only run Full .NET Framework and Mono tests from Framework host", EnvRequirement.FullFrameworkOnly)] - [InlineData(RuntimeMoniker.Net462)] - [InlineData(RuntimeMoniker.Mono)] - public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Framework(RuntimeMoniker runtimeMoniker) + [TheoryEnvSpecific("Can only run Mono tests from Framework host", EnvRequirement.FullFrameworkOnly)] + [MemberData(nameof(NonEmptyFrameworkData))] + public void NonEmptyBenchmarkReportsNonZeroTimeAndZeroAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker) { - AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty() + AssertNonZeroResults(benchmarkType, ManualConfig.CreateEmpty() .AddJob(Job.Default .WithRuntime(runtimeMoniker.GetRuntime()) )); } - private void AssertDifferentSizedStructsResults(IConfig config) + private void AssertNonZeroResults(Type benchmarkType, IConfig config, int subtractOverheadByClocks = 0) { - var summary = CanExecute(config + var summary = CanExecute(benchmarkType, config .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond)) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); var cpuResolution = CpuDetector.Cpu?.MaxFrequency()?.ToResolution() ?? FallbackCpuResolutionValue; - var threshold = (cpuResolution / 2).ToThreshold(); + // Modern cpus can execute multiple instructions per clock cycle, + // resulting in measurements greater than 0 but less than 1 clock cycle. + // (example: Intel Core i9-9880H CPU 2.30GHz reports 0.2852 ns for `_field++;`) + var threshold = new NumberValue(cpuResolution.Nanoseconds / 4).ToThreshold(); + // InProcess overhead measurements are incorrect, so we adjust the results to account for it. https://github.com/dotnet/runtime/issues/89685 + var overheadSubtraction = cpuResolution.Nanoseconds * subtractOverheadByClocks; foreach (var report in summary.Reports) { var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().Sample; - var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().Sample; + var overheadMeasurements = new Sample(report.AllMeasurements + .Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)) + .GetStatistics().OriginalValues + .Select(x => x - overheadSubtraction).ToArray()); - bool isZero = ZeroMeasurementHelper.AreIndistinguishable(workloadMeasurements, overheadMeasurements, threshold); - Assert.False(isZero, $"Actual time was 0."); + var comparisonResult = new SimpleEquivalenceTest(MannWhitneyTest.Instance).Perform(workloadMeasurements, overheadMeasurements, threshold, SignificanceLevel.P1E5); + Assert.True(comparisonResult == ComparisonResult.Greater, "Workload measurements are not greater than overhead."); Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); } } } +} - public struct Struct16 - { - public long l1, l2; - } +// Types outside of namespace so it's easier to read in the test explorer. +#pragma warning disable CA1050 // Declare types in namespaces +public struct Struct16 +{ + public long l1, l2; +} - public struct Struct32 - { - public long l1, l2, l3, l4; - } +public struct Struct32 +{ + public long l1, l2, l3, l4; +} - public struct Struct64 - { - public long l1, l2, l3, l4, l5, l6, l7, l8; - } +public struct Struct64 +{ + public long l1, l2, l3, l4, l5, l6, l7, l8; +} - public struct Struct128 - { - public long l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16; - } +public struct Struct128 +{ + public long l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16; +} - public class DifferentSizedStructs - { - [Benchmark] public Struct16 Struct16() => default; - [Benchmark] public Struct32 Struct32() => default; - [Benchmark] public Struct64 Struct64() => default; - [Benchmark] public Struct128 Struct128() => default; - } +public class DifferentSizedStructs +{ + [Benchmark] public Struct16 Struct16() => default; + [Benchmark] public Struct32 Struct32() => default; + [Benchmark] public Struct64 Struct64() => default; + [Benchmark] public Struct128 Struct128() => default; +} + +public class ActualWork +{ + public int _field; + + [Benchmark] + public void IncrementField() => _field++; } public class EmptyVoid diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests/NaiveRunnableEmitDiff.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests/NaiveRunnableEmitDiff.cs index 65cd9c752e..75438fc0ce 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests/NaiveRunnableEmitDiff.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests/NaiveRunnableEmitDiff.cs @@ -31,7 +31,8 @@ public class NaiveRunnableEmitDiff { { OpCodes.Br_S, OpCodes.Br }, { OpCodes.Blt_S, OpCodes.Blt }, - { OpCodes.Bne_Un_S, OpCodes.Bne_Un } + { OpCodes.Bne_Un_S, OpCodes.Bne_Un }, + { OpCodes.Bge_S, OpCodes.Bge } }; public static void RunDiff(string roslynAssemblyPath, string emittedAssemblyPath, ILogger logger)