Change ByteString to use memory and support unsafe create without copy
diff --git a/csharp/src/Google.Protobuf/ByteString.cs b/csharp/src/Google.Protobuf/ByteString.cs
index 45b3885..2619be5 100644
--- a/csharp/src/Google.Protobuf/ByteString.cs
+++ b/csharp/src/Google.Protobuf/ByteString.cs
@@ -34,6 +34,7 @@
using System.Collections;
using System.Collections.Generic;
using System.IO;
+using System.Runtime.InteropServices;
using System.Security;
using System.Text;
#if !NET35
@@ -49,40 +50,26 @@
/// <summary>
/// Immutable array of bytes.
/// </summary>
+ [SecuritySafeCritical]
public sealed class ByteString : IEnumerable<byte>, IEquatable<ByteString>
{
private static readonly ByteString empty = new ByteString(new byte[0]);
- private readonly byte[] bytes;
+ private readonly ReadOnlyMemory<byte> bytes;
/// <summary>
- /// Unsafe operations that can cause IO Failure and/or other catastrophic side-effects.
+ /// Internal use only. Ensure that the provided memory is not mutated and belongs to this instance.
/// </summary>
- internal static class Unsafe
- {
- /// <summary>
- /// Constructs a new ByteString from the given byte array. The array is
- /// *not* copied, and must not be modified after this constructor is called.
- /// </summary>
- internal static ByteString FromBytes(byte[] bytes)
- {
- return new ByteString(bytes);
- }
- }
-
- /// <summary>
- /// Internal use only. Ensure that the provided array is not mutated and belongs to this instance.
- /// </summary>
- internal static ByteString AttachBytes(byte[] bytes)
+ internal static ByteString AttachBytes(ReadOnlyMemory<byte> bytes)
{
return new ByteString(bytes);
}
/// <summary>
- /// Constructs a new ByteString from the given byte array. The array is
+ /// Constructs a new ByteString from the given memory. The memory is
/// *not* copied, and must not be modified after this constructor is called.
/// </summary>
- private ByteString(byte[] bytes)
+ private ByteString(ReadOnlyMemory<byte> bytes)
{
this.bytes = bytes;
}
@@ -117,11 +104,7 @@
/// </summary>
public ReadOnlySpan<byte> Span
{
- [SecuritySafeCritical]
- get
- {
- return new ReadOnlySpan<byte>(bytes);
- }
+ get { return bytes.Span; }
}
/// <summary>
@@ -130,11 +113,7 @@
/// </summary>
public ReadOnlyMemory<byte> Memory
{
- [SecuritySafeCritical]
- get
- {
- return new ReadOnlyMemory<byte>(bytes);
- }
+ get { return bytes; }
}
/// <summary>
@@ -144,7 +123,7 @@
/// <returns>A byte array with the same data as this <c>ByteString</c>.</returns>
public byte[] ToByteArray()
{
- return (byte[]) bytes.Clone();
+ return bytes.ToArray();
}
/// <summary>
@@ -153,7 +132,16 @@
/// <returns>A base64 representation of this <c>ByteString</c>.</returns>
public string ToBase64()
{
- return Convert.ToBase64String(bytes);
+ if (MemoryMarshal.TryGetArray(bytes, out ArraySegment<byte> segment))
+ {
+ // Fast path. ByteString was created with an array, so pass the underlying array.
+ return Convert.ToBase64String(segment.Array, segment.Offset, segment.Count);
+ }
+ else
+ {
+ // Slow path. BytesString is not an array. Convert memory and pass result to ToBase64String.
+ return Convert.ToBase64String(bytes.ToArray());
+ }
}
/// <summary>
@@ -197,21 +185,10 @@
/// <param name="stream">The stream to copy into a ByteString.</param>
/// <param name="cancellationToken">The cancellation token to use when reading from the stream, if any.</param>
/// <returns>A ByteString with content read from the given stream.</returns>
- public async static Task<ByteString> FromStreamAsync(Stream stream, CancellationToken cancellationToken = default(CancellationToken))
+ public static Task<ByteString> FromStreamAsync(Stream stream, CancellationToken cancellationToken = default(CancellationToken))
{
ProtoPreconditions.CheckNotNull(stream, nameof(stream));
- int capacity = stream.CanSeek ? checked((int) (stream.Length - stream.Position)) : 0;
- var memoryStream = new MemoryStream(capacity);
- // We have to specify the buffer size here, as there's no overload accepting the cancellation token
- // alone. But it's documented to use 81920 by default if not specified.
- await stream.CopyToAsync(memoryStream, 81920, cancellationToken);
-#if NETSTANDARD1_1 || NETSTANDARD2_0
- byte[] bytes = memoryStream.ToArray();
-#else
- // Avoid an extra copy if we can.
- byte[] bytes = memoryStream.Length == memoryStream.Capacity ? memoryStream.GetBuffer() : memoryStream.ToArray();
-#endif
- return AttachBytes(bytes);
+ return ByteStringAsync.FromStreamAsyncCore(stream, cancellationToken);
}
#endif
@@ -242,7 +219,6 @@
/// are copied, so further modifications to the span will not
/// be reflected in the returned <see cref="ByteString" />.
/// </summary>
- [SecuritySafeCritical]
public static ByteString CopyFrom(ReadOnlySpan<byte> bytes)
{
return new ByteString(bytes.ToArray());
@@ -270,7 +246,7 @@
/// </summary>
public byte this[int index]
{
- get { return bytes[index]; }
+ get { return bytes.Span[index]; }
}
/// <summary>
@@ -284,7 +260,18 @@
/// <returns>The result of decoding the binary data with the given decoding.</returns>
public string ToString(Encoding encoding)
{
- return encoding.GetString(bytes, 0, bytes.Length);
+ if (MemoryMarshal.TryGetArray(bytes, out ArraySegment<byte> segment))
+ {
+ // Fast path. ByteString was created with an array.
+ return encoding.GetString(segment.Array, segment.Offset, segment.Count);
+ }
+ else
+ {
+ // Slow path. BytesString is not an array. Convert memory and pass result to GetString.
+ // TODO: Consider using GetString overload that takes a pointer.
+ byte[] array = bytes.ToArray();
+ return encoding.GetString(array, 0, array.Length);
+ }
}
/// <summary>
@@ -304,9 +291,10 @@
/// Returns an iterator over the bytes in this <see cref="ByteString"/>.
/// </summary>
/// <returns>An iterator over the bytes in this object.</returns>
+ [SecuritySafeCritical]
public IEnumerator<byte> GetEnumerator()
{
- return ((IEnumerable<byte>) bytes).GetEnumerator();
+ return MemoryMarshal.ToEnumerable(bytes).GetEnumerator();
}
/// <summary>
@@ -324,7 +312,17 @@
public CodedInputStream CreateCodedInput()
{
// We trust CodedInputStream not to reveal the provided byte array or modify it
- return new CodedInputStream(bytes);
+ if (MemoryMarshal.TryGetArray(bytes, out ArraySegment<byte> segment) && segment.Count == bytes.Length)
+ {
+ // Fast path. ByteString was created with a complete array.
+ return new CodedInputStream(segment.Array);
+ }
+ else
+ {
+ // Slow path. BytesString is not an array, or is a slice of an array.
+ // Convert memory and pass result to WriteRawBytes.
+ return new CodedInputStream(bytes.ToArray());
+ }
}
/// <summary>
@@ -343,18 +341,8 @@
{
return false;
}
- if (lhs.bytes.Length != rhs.bytes.Length)
- {
- return false;
- }
- for (int i = 0; i < lhs.Length; i++)
- {
- if (rhs.bytes[i] != lhs.bytes[i])
- {
- return false;
- }
- }
- return true;
+
+ return lhs.bytes.Span.SequenceEqual(rhs.bytes.Span);
}
/// <summary>
@@ -373,6 +361,7 @@
/// </summary>
/// <param name="obj">The object to compare this with.</param>
/// <returns><c>true</c> if <paramref name="obj"/> refers to an equal <see cref="ByteString"/>; <c>false</c> otherwise.</returns>
+ [SecuritySafeCritical]
public override bool Equals(object obj)
{
return this == (obj as ByteString);
@@ -383,12 +372,15 @@
/// will return the same hash code.
/// </summary>
/// <returns>A hash code for this object.</returns>
+ [SecuritySafeCritical]
public override int GetHashCode()
{
+ ReadOnlySpan<byte> b = bytes.Span;
+
int ret = 23;
- foreach (byte b in bytes)
+ for (int i = 0; i < b.Length; i++)
{
- ret = (ret * 31) + b;
+ ret = (ret * 31) + b[i];
}
return ret;
}
@@ -404,19 +396,11 @@
}
/// <summary>
- /// Used internally by CodedOutputStream to avoid creating a copy for the write
- /// </summary>
- internal void WriteRawBytesTo(CodedOutputStream outputStream)
- {
- outputStream.WriteRawBytes(bytes, 0, bytes.Length);
- }
-
- /// <summary>
/// Copies the entire byte array to the destination array provided at the offset specified.
/// </summary>
public void CopyTo(byte[] array, int position)
{
- ByteArray.Copy(bytes, 0, array, position, bytes.Length);
+ bytes.CopyTo(array.AsMemory(position));
}
/// <summary>
@@ -424,7 +408,17 @@
/// </summary>
public void WriteTo(Stream outputStream)
{
- outputStream.Write(bytes, 0, bytes.Length);
+ if (MemoryMarshal.TryGetArray(bytes, out ArraySegment<byte> segment))
+ {
+ // Fast path. ByteString was created with an array, so pass the underlying array.
+ outputStream.Write(segment.Array, segment.Offset, segment.Count);
+ }
+ else
+ {
+ // Slow path. BytesString is not an array. Convert memory and pass result to WriteRawBytes.
+ var array = bytes.ToArray();
+ outputStream.Write(array, 0, array.Length);
+ }
}
}
}
\ No newline at end of file
diff --git a/csharp/src/Google.Protobuf/ByteStringAsync.cs b/csharp/src/Google.Protobuf/ByteStringAsync.cs
new file mode 100644
index 0000000..8bf8add
--- /dev/null
+++ b/csharp/src/Google.Protobuf/ByteStringAsync.cs
@@ -0,0 +1,64 @@
+#region Copyright notice and license
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#endregion
+
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Google.Protobuf
+{
+ /// <summary>
+ /// SecuritySafeCritical attribute can not be placed on types with async methods.
+ /// This class has ByteString's async methods so it can be marked with SecuritySafeCritical.
+ /// </summary>
+ internal static class ByteStringAsync
+ {
+#if !NET35
+ internal static async Task<ByteString> FromStreamAsyncCore(Stream stream, CancellationToken cancellationToken)
+ {
+ int capacity = stream.CanSeek ? checked((int)(stream.Length - stream.Position)) : 0;
+ var memoryStream = new MemoryStream(capacity);
+ // We have to specify the buffer size here, as there's no overload accepting the cancellation token
+ // alone. But it's documented to use 81920 by default if not specified.
+ await stream.CopyToAsync(memoryStream, 81920, cancellationToken);
+#if NETSTANDARD1_1 || NETSTANDARD2_0
+ byte[] bytes = memoryStream.ToArray();
+#else
+ // Avoid an extra copy if we can.
+ byte[] bytes = memoryStream.Length == memoryStream.Capacity ? memoryStream.GetBuffer() : memoryStream.ToArray();
+#endif
+ return ByteString.AttachBytes(bytes);
+ }
+#endif
+ }
+}
\ No newline at end of file
diff --git a/csharp/src/Google.Protobuf/UnsafeByteOperations.cs b/csharp/src/Google.Protobuf/UnsafeByteOperations.cs
new file mode 100644
index 0000000..5ef25d4
--- /dev/null
+++ b/csharp/src/Google.Protobuf/UnsafeByteOperations.cs
@@ -0,0 +1,81 @@
+#region Copyright notice and license
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#endregion
+
+using System;
+using System.Security;
+
+namespace Google.Protobuf
+{
+ /// <summary>
+ /// Provides a number of unsafe byte operations to be used by advanced applications with high performance
+ /// requirements. These methods are referred to as "unsafe" due to the fact that they potentially expose
+ /// the backing buffer of a <see cref="ByteString"/> to the application.
+ /// </summary>
+ /// <remarks>
+ /// <para>
+ /// The methods in this class should only be called if it is guaranteed that the buffer backing the
+ /// <see cref="ByteString"/> will never change! Mutation of a <see cref="ByteString"/> can lead to unexpected
+ /// and undesirable consequences in your application, and will likely be difficult to debug. Proceed with caution!
+ /// </para>
+ /// <para>
+ /// This can have a number of significant side affects that have spooky-action-at-a-distance-like behavior. In
+ /// particular, if the bytes value changes out from under a Protocol Buffer:
+ /// </para>
+ /// <list type="bullet">
+ /// <item>
+ /// <description>serialization may throw</description>
+ /// </item>
+ /// <item>
+ /// <description>serialization may succeed but the wrong bytes may be written out</description>
+ /// </item>
+ /// <item>
+ /// <description>messages are no longer threadsafe</description>
+ /// </item>
+ /// <item>
+ /// <description>hashCode may be incorrect</description>
+ /// </item>
+ /// </list>
+ /// </remarks>
+ [SecuritySafeCritical]
+ public static class UnsafeByteOperations
+ {
+ /// <summary>
+ /// Constructs a new <see cref="ByteString" /> from the given bytes. The bytes are not copied,
+ /// and must not be modified while the <see cref="ByteString" /> is in use.
+ /// This API is experimental and subject to change.
+ /// </summary>
+ public static ByteString UnsafeWrap(ReadOnlyMemory<byte> bytes)
+ {
+ return ByteString.AttachBytes(bytes);
+ }
+ }
+}
