From c3a3240f26ed8ad327bc38763596887fa58cfd99 Mon Sep 17 00:00:00 2001 From: to11mtm <12536917+to11mtm@users.noreply.github.com> Date: Thu, 30 Jan 2025 17:55:21 -0500 Subject: [PATCH 1/5] wip ByteString ReadOnlySequence --- src/core/Akka/Util/ByteString.cs | 65 +++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/src/core/Akka/Util/ByteString.cs b/src/core/Akka/Util/ByteString.cs index 46436cfc747..f1e195f3d8e 100644 --- a/src/core/Akka/Util/ByteString.cs +++ b/src/core/Akka/Util/ByteString.cs @@ -6,6 +6,7 @@ //----------------------------------------------------------------------- using System; +using System.Buffers; using System.Collections; using System.Collections.Generic; using System.Diagnostics; @@ -19,6 +20,43 @@ namespace Akka.IO // TODO: Move to Akka.Util namespace - this will require changes as name clashes with ProtoBuf class using ByteBuffer = ArraySegment; + public class ByteStringReadOnlySequenceSegment : ReadOnlySequenceSegment + { + public ByteStringReadOnlySequenceSegment(ReadOnlyMemory memory, long runningIndex) + { + Memory = memory; + RunningIndex = runningIndex; + } + public static (ByteStringReadOnlySequenceSegment first, ByteStringReadOnlySequenceSegment last) Create(ByteString bs) + { + var bArr = bs.Buffers; + var toMake = bArr.Count; + var first = new ByteStringReadOnlySequenceSegment(bArr[0],0); + var last = new ByteStringReadOnlySequenceSegment(bArr[toMake-1], bs.Count-bArr[toMake-1].Count); + if (toMake == 2) + { + first.Next = last; + } + else + { + var prior = first; + for (int i = 1; i < toMake-1; i++) + { + var item = bArr[i]; + var curr = new ByteStringReadOnlySequenceSegment(item, prior.RunningIndex+prior.Memory.Length); + prior.Next = curr; + prior = curr; + } + + prior.Next = last; + } + //var first = new ByteStringReadOnlySequenceSegment(bs, 0, 0); + //var last = new ByteStringReadOnlySequenceSegment(bs, bs.Count, bs.Buffers.Count - 1); + //first.Next = last; + return (first, last); + } + } + /// /// A rope-like immutable data structure containing bytes. /// The goal of this structure is to reduce copying of arrays @@ -28,6 +66,7 @@ namespace Akka.IO [DebuggerDisplay("(Count = {_count}, Buffers = {_buffers})")] public sealed class ByteString : IEquatable, IEnumerable { + #region creation methods /// @@ -246,7 +285,7 @@ public static ByteString FromString(string str, Encoding encoding) private readonly int _count; private readonly ByteBuffer[] _buffers; - + private ByteString(ByteBuffer[] buffers, int count) { _buffers = buffers; @@ -528,6 +567,30 @@ public ReadOnlySpan ToReadOnlySpan() return new ReadOnlySpan(ToArray()); } + + /// + /// Returns a ReadOnlySequence over the contents of this ByteString. + /// This is not a copying operation and zero-alloc when ByteString is compact + /// Otherwise N s will be allocated + /// Where N is the number of arrays currently internally held by the ByteString + /// + /// A ReadOnlySpan over the byte data. + public ReadOnlySequence ToReadOnlySequence() + { + if (_count == 0) + { + return ReadOnlySequence.Empty; + } + else if (_buffers.Length == 1) + { + return new ReadOnlySequence(_buffers[0]); + } + else + { + var (first, last) = ByteStringReadOnlySequenceSegment.Create(this); + return new ReadOnlySequence(first,0,last,last.Memory.Length); + } + } /// /// Appends at the tail From c280650f7eaf94254996bc375ad90df69dd883c1 Mon Sep 17 00:00:00 2001 From: to11mtm <12536917+to11mtm@users.noreply.github.com> Date: Thu, 30 Jan 2025 18:40:09 -0500 Subject: [PATCH 2/5] Some cleanup aand basic tests --- src/core/Akka.Tests/Util/ByteStringSpec.cs | 57 +++++++++++++++ src/core/Akka/Util/ByteString.cs | 80 +++++++++++----------- 2 files changed, 98 insertions(+), 39 deletions(-) diff --git a/src/core/Akka.Tests/Util/ByteStringSpec.cs b/src/core/Akka.Tests/Util/ByteStringSpec.cs index 053cf6bbd43..76a1a80b053 100644 --- a/src/core/Akka.Tests/Util/ByteStringSpec.cs +++ b/src/core/Akka.Tests/Util/ByteStringSpec.cs @@ -6,6 +6,7 @@ //----------------------------------------------------------------------- using System; +using System.Buffers; using System.Linq; using System.Runtime.InteropServices; using System.Text; @@ -97,6 +98,62 @@ public void A_ByteString_ToReadOnlySpan_compacted_must_have_identical_memory_ref var concatSpan4 = compacted.ToReadOnlySpan(); (concatSpan3 == concatSpan4).Should().BeTrue(); } + + [Fact] + public void A_ByteString_ToReadOnlySequence_must_have_correct_size() + { + Prop.ForAll((ByteString a, ByteString b) => + { + a.ToReadOnlySequence().Length.Should().Be(a.Count); + b.ToReadOnlySequence().Length.Should().Be(b.Count); + var concat = a + b; + var spanConcat = concat.ToReadOnlySequence(); + return spanConcat.Length == concat.Count; + }).QuickCheckThrowOnFailure(); + } + + [Fact] + public void A_ByteString_ToReadOnlySequence_compacted_must_have_identical_contents_to_original() + { + var bytesA = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + var bytesB = new byte[] { 10, 11, 12, 13, 14, 15, 16, 17, 18 }; + var a = ByteString.FromBytes(bytesA); + var b = ByteString.FromBytes(bytesB); + + var concat = a + b; + + var concatSpan = concat.ToReadOnlySequence(); + var concatSpan2 = concat.ToReadOnlySequence(); + // not compact, returns a new span + (concatSpan.ToArray().SequenceEqual(concatSpan2.ToArray())).Should().BeTrue(); + + // compact, will return same span + var compacted = concat.Compact(); + var concatSpan3 = compacted.ToReadOnlySequence(); + (concatSpan.ToArray().SequenceEqual(concatSpan3.ToArray())).Should().BeTrue(); + } + + [Fact] + public void A_ByteString_ToReadOnlySequence_compacted_must_have_identical_contents_to_original_with_many_Segments() + { + var bytesA = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + var bytesB = new byte[] { 10, 11, 12, 13, 14, 15, 16, 17, 18 }; + var bytesC = new byte[] { 19, 20, 21, 22, 23, 24, 25, 26, 27 }; + var a = ByteString.FromBytes(bytesA); + var b = ByteString.FromBytes(bytesB); + var c = ByteString.FromBytes(bytesC); + var concat = a + b +c; + + var concatSpan = concat.ToReadOnlySequence(); + var concatSpan2 = concat.ToReadOnlySequence(); + // not compact, returns a new span + (concatSpan.ToArray().SequenceEqual(concatSpan2.ToArray())).Should().BeTrue(); + + // compact, will return same span + var compacted = concat.Compact(); + var concatSpan3 = compacted.ToReadOnlySequence(); + (concatSpan.ToArray().SequenceEqual(concatSpan3.ToArray())).Should().BeTrue(); + } [Fact] public void A_ByteString_must_have_correct_size_when_slicing_from_index() diff --git a/src/core/Akka/Util/ByteString.cs b/src/core/Akka/Util/ByteString.cs index f1e195f3d8e..bd1905cc47e 100644 --- a/src/core/Akka/Util/ByteString.cs +++ b/src/core/Akka/Util/ByteString.cs @@ -20,43 +20,6 @@ namespace Akka.IO // TODO: Move to Akka.Util namespace - this will require changes as name clashes with ProtoBuf class using ByteBuffer = ArraySegment; - public class ByteStringReadOnlySequenceSegment : ReadOnlySequenceSegment - { - public ByteStringReadOnlySequenceSegment(ReadOnlyMemory memory, long runningIndex) - { - Memory = memory; - RunningIndex = runningIndex; - } - public static (ByteStringReadOnlySequenceSegment first, ByteStringReadOnlySequenceSegment last) Create(ByteString bs) - { - var bArr = bs.Buffers; - var toMake = bArr.Count; - var first = new ByteStringReadOnlySequenceSegment(bArr[0],0); - var last = new ByteStringReadOnlySequenceSegment(bArr[toMake-1], bs.Count-bArr[toMake-1].Count); - if (toMake == 2) - { - first.Next = last; - } - else - { - var prior = first; - for (int i = 1; i < toMake-1; i++) - { - var item = bArr[i]; - var curr = new ByteStringReadOnlySequenceSegment(item, prior.RunningIndex+prior.Memory.Length); - prior.Next = curr; - prior = curr; - } - - prior.Next = last; - } - //var first = new ByteStringReadOnlySequenceSegment(bs, 0, 0); - //var last = new ByteStringReadOnlySequenceSegment(bs, bs.Count, bs.Buffers.Count - 1); - //first.Next = last; - return (first, last); - } - } - /// /// A rope-like immutable data structure containing bytes. /// The goal of this structure is to reduce copying of arrays @@ -66,6 +29,45 @@ public static (ByteStringReadOnlySequenceSegment first, ByteStringReadOnlySequen [DebuggerDisplay("(Count = {_count}, Buffers = {_buffers})")] public sealed class ByteString : IEquatable, IEnumerable { + public class ByteStringReadOnlySequenceSegment : ReadOnlySequenceSegment + { + public ByteStringReadOnlySequenceSegment(ReadOnlyMemory memory, long runningIndex) + { + Memory = memory; + RunningIndex = runningIndex; + } + + /// + /// This is here because + /// has predefined properties with Protected Setters. + /// + public static ReadOnlySequence CreateSequence(ByteString bs) + { + var bArr = bs._buffers; + var first = new ByteStringReadOnlySequenceSegment(bArr[0],0); + ByteBuffer item = default; + ByteStringReadOnlySequenceSegment last = first; + if (bArr.Length == 2) + { + item = bArr[1]; + last = new ByteStringReadOnlySequenceSegment(item, bs.Count-item.Count); + first.Next = last; + } + else + { + var prior = first; + for (int i = 1; i < bArr.Length; i++) + { + item = bArr[i]; + var curr = new ByteStringReadOnlySequenceSegment(item, prior.RunningIndex+prior.Memory.Length); + prior.Next = curr; + prior = curr; + } + last = prior; + } + return new ReadOnlySequence(first,0,last,last.Memory.Length); + } + } #region creation methods @@ -583,12 +585,12 @@ public ReadOnlySequence ToReadOnlySequence() } else if (_buffers.Length == 1) { + //Happy path, we can just pass ArraySegment here and avoid alloc. return new ReadOnlySequence(_buffers[0]); } else { - var (first, last) = ByteStringReadOnlySequenceSegment.Create(this); - return new ReadOnlySequence(first,0,last,last.Memory.Length); + return ByteStringReadOnlySequenceSegment.CreateSequence(this); } } From ecf4b3f3ae212bf70623d3461a7ed63607265472 Mon Sep 17 00:00:00 2001 From: to11mtm <12536917+to11mtm@users.noreply.github.com> Date: Sat, 1 Feb 2025 13:17:55 -0500 Subject: [PATCH 3/5] ApiDocs, xmlcdocs --- .../CoreAPISpec.ApproveCore.DotNet.verified.txt | 6 ++++++ .../CoreAPISpec.ApproveCore.Net.verified.txt | 6 ++++++ src/core/Akka.Tests/Util/ByteStringSpec.cs | 2 ++ src/core/Akka/Util/ByteString.cs | 16 ++++++++++++---- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.DotNet.verified.txt b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.DotNet.verified.txt index c8dcb164155..33b27611567 100644 --- a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.DotNet.verified.txt +++ b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.DotNet.verified.txt @@ -3796,6 +3796,7 @@ namespace Akka.IO public Akka.IO.ByteString Slice(int index) { } public Akka.IO.ByteString Slice(int index, int count) { } public byte[] ToArray() { } + public System.Buffers.ReadOnlySequence ToReadOnlySequence() { } public System.ReadOnlySpan ToReadOnlySpan() { } public override string ToString() { } public string ToString(System.Text.Encoding encoding) { } @@ -3806,6 +3807,11 @@ namespace Akka.IO public static Akka.IO.ByteString op_Explicit(byte[] bytes) { } public static byte[] op_Explicit(Akka.IO.ByteString byteString) { } public static bool !=(Akka.IO.ByteString x, Akka.IO.ByteString y) { } + [System.Diagnostics.DebuggerDisplayAttribute("(RunningIndex = {RunningIndex}, Length = {Memory.Length})}")] + public sealed class ByteStringReadOnlySequenceSegment : System.Buffers.ReadOnlySequenceSegment + { + public static System.Buffers.ReadOnlySequence CreateSequence(Akka.IO.ByteString bs) { } + } } [Akka.Annotations.InternalApiAttribute()] public class ConnectException : System.Exception diff --git a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.Net.verified.txt b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.Net.verified.txt index d1553938cc4..ca84abb831f 100644 --- a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.Net.verified.txt +++ b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.Net.verified.txt @@ -3786,6 +3786,7 @@ namespace Akka.IO public Akka.IO.ByteString Slice(int index) { } public Akka.IO.ByteString Slice(int index, int count) { } public byte[] ToArray() { } + public System.Buffers.ReadOnlySequence ToReadOnlySequence() { } public System.ReadOnlySpan ToReadOnlySpan() { } public override string ToString() { } public string ToString(System.Text.Encoding encoding) { } @@ -3796,6 +3797,11 @@ namespace Akka.IO public static Akka.IO.ByteString op_Explicit(byte[] bytes) { } public static byte[] op_Explicit(Akka.IO.ByteString byteString) { } public static bool !=(Akka.IO.ByteString x, Akka.IO.ByteString y) { } + [System.Diagnostics.DebuggerDisplayAttribute("(RunningIndex = {RunningIndex}, Length = {Memory.Length})}")] + public sealed class ByteStringReadOnlySequenceSegment : System.Buffers.ReadOnlySequenceSegment + { + public static System.Buffers.ReadOnlySequence CreateSequence(Akka.IO.ByteString bs) { } + } } [Akka.Annotations.InternalApiAttribute()] public class ConnectException : System.Exception diff --git a/src/core/Akka.Tests/Util/ByteStringSpec.cs b/src/core/Akka.Tests/Util/ByteStringSpec.cs index 76a1a80b053..cc127b784ab 100644 --- a/src/core/Akka.Tests/Util/ByteStringSpec.cs +++ b/src/core/Akka.Tests/Util/ByteStringSpec.cs @@ -153,6 +153,8 @@ public void A_ByteString_ToReadOnlySequence_compacted_must_have_identical_conten var compacted = concat.Compact(); var concatSpan3 = compacted.ToReadOnlySequence(); (concatSpan.ToArray().SequenceEqual(concatSpan3.ToArray())).Should().BeTrue(); + + (concatSpan.ToArray().SequenceEqual(compacted.ToArray())).Should().BeTrue(); } [Fact] diff --git a/src/core/Akka/Util/ByteString.cs b/src/core/Akka/Util/ByteString.cs index bd1905cc47e..3cbbbe0b382 100644 --- a/src/core/Akka/Util/ByteString.cs +++ b/src/core/Akka/Util/ByteString.cs @@ -29,9 +29,14 @@ namespace Akka.IO [DebuggerDisplay("(Count = {_count}, Buffers = {_buffers})")] public sealed class ByteString : IEquatable, IEnumerable { - public class ByteStringReadOnlySequenceSegment : ReadOnlySequenceSegment + /// + /// A to encapsulate + /// Parts of a in a + /// + [DebuggerDisplay("(RunningIndex = {RunningIndex}, Length = {Memory.Length})}")] + public sealed class ByteStringReadOnlySequenceSegment : ReadOnlySequenceSegment { - public ByteStringReadOnlySequenceSegment(ReadOnlyMemory memory, long runningIndex) + private ByteStringReadOnlySequenceSegment(ReadOnlyMemory memory, long runningIndex) { Memory = memory; RunningIndex = runningIndex; @@ -571,12 +576,15 @@ public ReadOnlySpan ToReadOnlySpan() } /// - /// Returns a ReadOnlySequence over the contents of this ByteString. + /// Returns a of over the contents of this ByteString. /// This is not a copying operation and zero-alloc when ByteString is compact /// Otherwise N s will be allocated /// Where N is the number of arrays currently internally held by the ByteString /// - /// A ReadOnlySpan over the byte data. + /// + /// A of + /// over the data in the + /// public ReadOnlySequence ToReadOnlySequence() { if (_count == 0) From 516d8074f7f8ce14de67b63bf4c15638fda87135 Mon Sep 17 00:00:00 2001 From: to11mtm <12536917+to11mtm@users.noreply.github.com> Date: Tue, 4 Feb 2025 19:03:09 -0500 Subject: [PATCH 4/5] Simplify creation --- src/core/Akka/Util/ByteString.cs | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/src/core/Akka/Util/ByteString.cs b/src/core/Akka/Util/ByteString.cs index 3cbbbe0b382..cfb8a9b5d20 100644 --- a/src/core/Akka/Util/ByteString.cs +++ b/src/core/Akka/Util/ByteString.cs @@ -41,7 +41,7 @@ private ByteStringReadOnlySequenceSegment(ReadOnlyMemory memory, long runn Memory = memory; RunningIndex = runningIndex; } - + /// /// This is here because /// has predefined properties with Protected Setters. @@ -49,28 +49,19 @@ private ByteStringReadOnlySequenceSegment(ReadOnlyMemory memory, long runn public static ReadOnlySequence CreateSequence(ByteString bs) { var bArr = bs._buffers; - var first = new ByteStringReadOnlySequenceSegment(bArr[0],0); - ByteBuffer item = default; + var first = new ByteStringReadOnlySequenceSegment(bArr[0], 0); ByteStringReadOnlySequenceSegment last = first; - if (bArr.Length == 2) - { - item = bArr[1]; - last = new ByteStringReadOnlySequenceSegment(item, bs.Count-item.Count); - first.Next = last; - } - else + var prior = first; + for (int i = 1; i < bArr.Length; i++) { - var prior = first; - for (int i = 1; i < bArr.Length; i++) - { - item = bArr[i]; - var curr = new ByteStringReadOnlySequenceSegment(item, prior.RunningIndex+prior.Memory.Length); - prior.Next = curr; - prior = curr; - } - last = prior; + var item = bArr[i]; + var curr = new ByteStringReadOnlySequenceSegment(item, prior.RunningIndex + prior.Memory.Length); + prior.Next = curr; + prior = curr; } - return new ReadOnlySequence(first,0,last,last.Memory.Length); + + last = prior; + return new ReadOnlySequence(first, 0, last, last.Memory.Length); } } From cf63b0b509ab80c486d6e21a83bc2ca3019f6c1e Mon Sep 17 00:00:00 2001 From: to11mtm <12536917+to11mtm@users.noreply.github.com> Date: Tue, 4 Feb 2025 19:06:21 -0500 Subject: [PATCH 5/5] more cleanup --- src/core/Akka/Util/ByteString.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core/Akka/Util/ByteString.cs b/src/core/Akka/Util/ByteString.cs index cfb8a9b5d20..7cd76fccfee 100644 --- a/src/core/Akka/Util/ByteString.cs +++ b/src/core/Akka/Util/ByteString.cs @@ -50,7 +50,6 @@ public static ReadOnlySequence CreateSequence(ByteString bs) { var bArr = bs._buffers; var first = new ByteStringReadOnlySequenceSegment(bArr[0], 0); - ByteStringReadOnlySequenceSegment last = first; var prior = first; for (int i = 1; i < bArr.Length; i++) { @@ -60,8 +59,7 @@ public static ReadOnlySequence CreateSequence(ByteString bs) prior = curr; } - last = prior; - return new ReadOnlySequence(first, 0, last, last.Memory.Length); + return new ReadOnlySequence(first, 0, prior, prior.Memory.Length); } } @@ -283,7 +281,7 @@ public static ByteString FromString(string str, Encoding encoding) private readonly int _count; private readonly ByteBuffer[] _buffers; - + private ByteString(ByteBuffer[] buffers, int count) { _buffers = buffers;