Skip to content

Commit

Permalink
Merge pull request #1520 from Aaronontheweb/nbench-integration
Browse files Browse the repository at this point in the history
added initial performance specs using NBench
  • Loading branch information
rogeralsing committed Dec 9, 2015
2 parents 6b722e4 + 1f8ac8c commit 3f54642
Show file tree
Hide file tree
Showing 16 changed files with 399 additions and 9 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ bld/
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*

# NBench results
[Pp]erf[Rr]esult*/

#NUNIT
*.VisualState.xml
TestResult.xml
Expand Down
1 change: 1 addition & 0 deletions build.cmd
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ src\.nuget\NuGet.exe install FAKE -ConfigFile src\.nuget\Nuget.Config -OutputDir

src\.nuget\NuGet.exe install xunit.runner.console -ConfigFile src\.nuget\Nuget.Config -OutputDirectory src\packages\FAKE -ExcludeVersion -Version 2.0.0
src\.nuget\NuGet.exe install nunit.runners -ConfigFile src\.nuget\Nuget.Config -OutputDirectory src\packages\FAKE -ExcludeVersion -Version 2.6.4
src\.nuget\NuGet.exe install NBench.Runner -OutputDirectory src\packages -ExcludeVersion

if not exist src\packages\SourceLink.Fake\tools\SourceLink.fsx (
src\.nuget\nuget.exe install SourceLink.Fake -ConfigFile src\.nuget\Nuget.Config -OutputDirectory src\packages -ExcludeVersion
Expand Down
39 changes: 39 additions & 0 deletions build.fsx
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ printfn "Assembly version: %s\nNuget version; %s\n" release.AssemblyVersion rele

let binDir = "bin"
let testOutput = FullName "TestResults"
let perfOutput = FullName "PerfResults"

let nugetDir = binDir @@ "nuget"
let workingDir = binDir @@ "build"
Expand Down Expand Up @@ -276,6 +277,40 @@ Target "MultiNodeTests" <| fun _ ->

multiNodeTestAssemblies |> Seq.iter (runMultiNodeSpec)

//--------------------------------------------------------------------------------
// NBench targets
//--------------------------------------------------------------------------------
Target "NBench" <| fun _ ->
let testSearchPath =
let assemblyFilter = getBuildParamOrDefault "spec-assembly" String.Empty
sprintf "src/**/bin/Release/*%s*.Tests.Performance.dll" assemblyFilter

mkdir perfOutput
let nbenchTestPath = findToolInSubPath "NBench.Runner.exe" "src/packges/NBench.Runner*"
let nbenchTestAssemblies = !! testSearchPath
printfn "Using NBench.Runner: %s" nbenchTestPath

let runNBench assembly =
let spec = getBuildParam "spec"

let args = new StringBuilder()
|> append assembly
|> append (sprintf "output-directory=\"%s\"" perfOutput)
|> toText

let result = ExecProcess(fun info ->
info.FileName <- nbenchTestPath
info.WorkingDirectory <- (Path.GetDirectoryName (FullName nbenchTestPath))
info.Arguments <- args) (System.TimeSpan.FromMinutes 15.0) (* Reasonably long-running task. *)
if result <> 0 then failwithf "NBench.Runner failed. %s %s" nbenchTestPath args

nbenchTestAssemblies |> Seq.iter (runNBench)

//--------------------------------------------------------------------------------
// Clean NBench output
Target "CleanPerf" <| fun _ ->
DeleteDir perfOutput


//--------------------------------------------------------------------------------
// Nuget targets
Expand Down Expand Up @@ -561,6 +596,9 @@ Target "HelpMultiNodeTests" <| fun _ ->
"CleanTests" ==> "RunTests"
"CleanTests" ==> "MultiNodeTests"

// NBench dependencies
"CleanPerf" ==> "NBench"

// nuget dependencies
"CleanNuget" ==> "CreateNuget"
"CleanNuget" ==> "BuildRelease" ==> "Nuget"
Expand All @@ -572,6 +610,7 @@ Target "All" DoNothing
"BuildRelease" ==> "All"
"RunTests" ==> "All"
"MultiNodeTests" ==> "All"
"NBench" ==> "All"
"Nuget" ==> "All"

Target "AllTests" DoNothing //used for Mono builds, due to Mono 4.0 bug with FAKE / NuGet https://github.com/fsharp/fsharp/issues/427
Expand Down
3 changes: 3 additions & 0 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ mono $SCRIPT_PATH/src/.nuget/NuGet.exe install FAKE -OutputDirectory $SCRIPT_PAT
mono $SCRIPT_PATH/src/.nuget/NuGet.exe install xunit.runners -OutputDirectory $SCRIPT_PATH/src/packages/FAKE -ExcludeVersion -Version 2.0.0
mono $SCRIPT_PATH/src/.nuget/NuGet.exe install nunit.runners -OutputDirectory $SCRIPT_PATH/src/packages/FAKE -ExcludeVersion -Version 2.6.4

mono $SCRIPT_PATH/src/.nuget/NuGet.exe install NBench.Runner -OutputDirectory $SCRIPT_PATH/src/packages -ExcludeVersion


if ! [ -e $SCRIPT_PATH/src/packages/SourceLink.Fake/tools/SourceLink.fsx ] ; then
mono $SCRIPT_PATH/src/.nuget/NuGet.exe install SourceLink.Fake -OutputDirectory $SCRIPT_PATH/src/packages -ExcludeVersion

Expand Down
13 changes: 12 additions & 1 deletion src/Akka.sln
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 14
VisualStudioVersion = 14.0.24606.1
VisualStudioVersion = 14.0.23107.0
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Examples", "Examples", "{69279534-1DBA-4115-BF8B-03F77FC8125E}"
EndProject
Expand Down Expand Up @@ -235,6 +235,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Akka.Serialization.WireTest
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Akka.Logger.log4net", "contrib\loggers\Akka.Logger.log4net\Akka.Logger.log4net.csproj", "{DAEC9503-CA75-433F-83DE-E28965D94D56}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Akka.Tests.Performance", "core\Akka.Tests.Performance\Akka.Tests.Performance.csproj", "{B17DEDCB-D2AA-472D-82A0-D242B711F488}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug Mono|Any CPU = Debug Mono|Any CPU
Expand Down Expand Up @@ -864,6 +866,14 @@ Global
{DAEC9503-CA75-433F-83DE-E28965D94D56}.Release Mono|Any CPU.Build.0 = Release|Any CPU
{DAEC9503-CA75-433F-83DE-E28965D94D56}.Release|Any CPU.ActiveCfg = Release|Any CPU
{DAEC9503-CA75-433F-83DE-E28965D94D56}.Release|Any CPU.Build.0 = Release|Any CPU
{B17DEDCB-D2AA-472D-82A0-D242B711F488}.Debug Mono|Any CPU.ActiveCfg = Debug|Any CPU
{B17DEDCB-D2AA-472D-82A0-D242B711F488}.Debug Mono|Any CPU.Build.0 = Debug|Any CPU
{B17DEDCB-D2AA-472D-82A0-D242B711F488}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B17DEDCB-D2AA-472D-82A0-D242B711F488}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B17DEDCB-D2AA-472D-82A0-D242B711F488}.Release Mono|Any CPU.ActiveCfg = Release|Any CPU
{B17DEDCB-D2AA-472D-82A0-D242B711F488}.Release Mono|Any CPU.Build.0 = Release|Any CPU
{B17DEDCB-D2AA-472D-82A0-D242B711F488}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B17DEDCB-D2AA-472D-82A0-D242B711F488}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -968,5 +978,6 @@ Global
{CAA97041-CFC0-4081-9BD2-8B139E62A611} = {0E55F1F8-E212-43D7-A0C0-ACEA9794B0D7}
{402FA351-D6C6-40FD-8868-F07156035919} = {0E55F1F8-E212-43D7-A0C0-ACEA9794B0D7}
{DAEC9503-CA75-433F-83DE-E28965D94D56} = {FFEC736B-EDA3-433C-8564-7C14676601A1}
{B17DEDCB-D2AA-472D-82A0-D242B711F488} = {01167D3C-49C4-4CDE-9787-C176D139ACDD}
EndGlobalSection
EndGlobal
4 changes: 2 additions & 2 deletions src/SharedAssemblyInfo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,5 @@
[assembly: AssemblyCompanyAttribute("Akka.NET Team")]
[assembly: AssemblyCopyrightAttribute("Copyright © 2013-2015 Akka.NET Team")]
[assembly: AssemblyTrademarkAttribute("")]
[assembly: AssemblyVersionAttribute("1.0.5.0")]
[assembly: AssemblyFileVersionAttribute("1.0.5.0")]
[assembly: AssemblyVersionAttribute("1.0.6.0")]
[assembly: AssemblyFileVersionAttribute("1.0.6.0")]
6 changes: 3 additions & 3 deletions src/core/Akka.FSharp/Properties/AssemblyInfo.fs
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ open System.Runtime.InteropServices
[<assembly: AssemblyCompanyAttribute("Akka.NET Team")>]
[<assembly: ComVisibleAttribute(false)>]
[<assembly: CLSCompliantAttribute(true)>]
[<assembly: AssemblyVersionAttribute("1.0.5.0")>]
[<assembly: AssemblyFileVersionAttribute("1.0.5.0")>]
[<assembly: AssemblyVersionAttribute("1.0.6.0")>]
[<assembly: AssemblyFileVersionAttribute("1.0.6.0")>]
do ()

module internal AssemblyVersionInformation =
let [<Literal>] Version = "1.0.5.0"
let [<Literal>] Version = "1.0.6.0"
3 changes: 0 additions & 3 deletions src/core/Akka.Remote/Akka.Remote.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,6 @@
<Optimize>false</Optimize>
</PropertyGroup>
<ItemGroup>
<Content Include="CHANGES.txt">
<Paket>True</Paket>
</Content>
<Content Include="licenses\license.txt">
<Paket>True</Paket>
</Content>
Expand Down
82 changes: 82 additions & 0 deletions src/core/Akka.Tests.Performance/Actor/ActorMemoryFootprintSpec.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
using System;
using Akka.Actor;
using Akka.Util.Internal;
using NBench;

namespace Akka.Tests.Performance.Actor
{
/// <summary>
/// Tests the default memory footprint of an Akka.NET actor
/// </summary>
public class ActorMemoryFootprintSpec
{
class MemoryUntypedActor : UntypedActor {
protected override void OnReceive(object message)
{
if (message is string) return;
if (message is int) return;
if (message is bool) return;
Unhandled(message);
}
}

class MemoryReceiveActor : ReceiveActor
{
public MemoryReceiveActor()
{
Receive<string>(s => { });
Receive<int>(i => { });
Receive<bool>(b => { });
}
}

private static readonly AtomicCounter Counter = new AtomicCounter(0);
private ActorSystem _system;
private Counter _createActorThroughput;

private const string CreateThroughputCounter = "ActorCreateThroughput";
private const int ActorCreateNumber = 10000;

private static readonly Props UntypedActorProps = Props.Create(() => new MemoryUntypedActor());
private static readonly Props ReceiveActorProps = Props.Create(() => new MemoryReceiveActor());

[PerfSetup]
public void Setup(BenchmarkContext context)
{
_system = ActorSystem.Create("ActorMemoryFootprintSpec" + Counter.GetAndIncrement());
_createActorThroughput = context.GetCounter(CreateThroughputCounter);
}

[PerfBenchmark(Description = "Measures the amount of memory used by 10,000 UntypedActors", RunMode = RunMode.Iterations, NumberOfIterations = 13, TestMode = TestMode.Measurement)]
[MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
[CounterMeasurement(CreateThroughputCounter)]
public void UntypedActorMemoryFootprint(BenchmarkContext context)
{
for (var i = 0; i < ActorCreateNumber; i++)
{
_system.ActorOf(UntypedActorProps);
_createActorThroughput.Increment();
}
}

[PerfBenchmark(Description = "Measures the amount of memory used by 10,000 ReceiveActors", RunMode = RunMode.Iterations, NumberOfIterations = 13, TestMode = TestMode.Measurement)]
[MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
[CounterMeasurement(CreateThroughputCounter)]
public void ReceiveActorMemoryFootprint(BenchmarkContext context)
{
for (var i = 0; i < ActorCreateNumber; i++)
{
_system.ActorOf(ReceiveActorProps);
_createActorThroughput.Increment();
}
}

[PerfCleanup]
public void Teardown(BenchmarkContext context)
{
_system.Shutdown();
_system.AwaitTermination(TimeSpan.FromSeconds(2.0d));
_system = null;
}
}
}
67 changes: 67 additions & 0 deletions src/core/Akka.Tests.Performance/Akka.Tests.Performance.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{B17DEDCB-D2AA-472D-82A0-D242B711F488}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Akka.Tests.Performance</RootNamespace>
<AssemblyName>Akka.Tests.Performance</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="NBench, Version=0.1.3.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\..\packages\NBench.0.1.3\lib\net45\NBench.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="Microsoft.CSharp" />
</ItemGroup>
<ItemGroup>
<Compile Include="Actor\ActorMemoryFootprintSpec.cs" />
<Compile Include="Dispatch\MailboxMemoryFootprintSpec.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Akka.TestKit\Akka.TestKit.csproj">
<Project>{0d3cbad0-bbdb-43e5-afc4-ed1d3ecdc224}</Project>
<Name>Akka.TestKit</Name>
</ProjectReference>
<ProjectReference Include="..\Akka\Akka.csproj">
<Project>{5deddf90-37f0-48d3-a0b0-a5cbd8a7e377}</Project>
<Name>Akka</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
using Akka.Dispatch;

namespace Akka.Tests.Performance.Dispatch
{
public class ConcurrentQueueMailboxThroughputSpec : MailboxThroughputSpecBase
{
protected override Mailbox CreateMailbox()
{
return new ConcurrentQueueMailbox();
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
using System.Collections.Generic;
using Akka.Dispatch;
using NBench;

namespace Akka.Tests.Performance.Dispatch
{
public class MailboxMemoryFootprintSpec
{
private Counter _createMailboxThroughput;

private const string CreateThroughputCounter = "MailboxCreateThroughput";
private const int MailboxCreateNumber = 10000;

private List<Mailbox> _mailboxes;

[PerfSetup]
public void Setup(BenchmarkContext context)
{
_createMailboxThroughput = context.GetCounter(CreateThroughputCounter);
_mailboxes = new List<Mailbox>(MailboxCreateNumber);
}

[PerfBenchmark(Description = "Measures the amount of memory used by 10,000 ConcurrentQueueMailboxes", RunMode = RunMode.Iterations, NumberOfIterations = 13, TestMode = TestMode.Measurement)]
[MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
[CounterMeasurement(CreateThroughputCounter)]
public void ConcurrentQueueMailbox()
{
for (var i = 0; i < MailboxCreateNumber; i++)
{
_mailboxes.Add(new ConcurrentQueueMailbox());
_createMailboxThroughput.Increment();
}
}

[PerfBenchmark(Description = "Measures the amount of memory used by 10,000 UnboundedDequeBasedMailboxes", RunMode = RunMode.Iterations, NumberOfIterations = 13, TestMode = TestMode.Measurement)]
[MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
[CounterMeasurement(CreateThroughputCounter)]
public void UnboundedDequeBasedMailbox()
{
for (var i = 0; i < MailboxCreateNumber; i++)
{
_mailboxes.Add(new UnboundedDequeBasedMailbox());
_createMailboxThroughput.Increment();
}
}

[PerfBenchmark(Description = "Measures the amount of memory used by 10,000 BoundedDequeBasedMailboxes", RunMode = RunMode.Iterations, NumberOfIterations = 13, TestMode = TestMode.Measurement)]
[MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
[CounterMeasurement(CreateThroughputCounter)]
public void BoundedDequeBasedMailbox()
{
for (var i = 0; i < MailboxCreateNumber; i++)
{
_mailboxes.Add(new BoundedDequeBasedMailbox());
_createMailboxThroughput.Increment();
}
}

[PerfCleanup]
public void Teardown()
{
_mailboxes.Clear();
_mailboxes = null;
}
}
}
Loading

0 comments on commit 3f54642

Please sign in to comment.