Skip to content

Commit 8e84391

Browse files
TwoTenPvPTwoTenPvP
authored
Merge pull request #41 from GabrielTofvesson/bit-serializer
Optimized byte serialization
2 parents 902fb77 + 435db1f commit 8e84391

File tree

2 files changed

+91
-89
lines changed

2 files changed

+91
-89
lines changed

‎MLAPI/NetworkingManagerComponents/Binary/BitReader.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ public byte ReadByte()
3535
bitCount += 8;
3636
return result;
3737
}
38+
public void SkipPadded() => bitCount += (8 - (bitCount % 8)) % 8;
3839
public ushort ReadUShort() => (ushort)ReadULong();
3940
public uint ReadUInt() => (uint)ReadULong();
4041
public sbyte ReadSByte() => (sbyte)ZigZagDecode(ReadByte(), 1);

‎MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs‎

Lines changed: 90 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.Diagnostics;
34
using System.Reflection;
5+
using System.Runtime.InteropServices;
46
using System.Text;
57
using UnityEngine;
68

@@ -52,7 +54,6 @@ static BitWriter()
5254

5355
private List<object> collect = null;
5456
private bool tempAlloc = false;
55-
private int collectCount = 0;
5657

5758
/// <summary>
5859
/// Allocates a new binary collector.
@@ -73,10 +74,11 @@ public BitWriter()
7374

7475
private void Push<T>(T b)
7576
{
76-
if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType()))
77+
if (b == null) collect.Add(b);
78+
else if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType()))
7779
collect.Add(b is string ? Encoding.UTF8.GetBytes(b as string) : b as object);
78-
//else
79-
// Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored");
80+
else
81+
Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored");
8082
}
8183

8284

@@ -87,11 +89,12 @@ private void Push<T>(T b)
8789
public void WriteUShort(ushort s) => Push(s);
8890
public void WriteUInt(uint i) => Push(i);
8991
public void WriteULong(ulong l) => Push(l);
90-
public void WriteSByte(sbyte b) => Push(b);
91-
public void WriteShort(short s) => Push(s);
92-
public void WriteInt(int i) => Push(i);
93-
public void WriteLong(long l) => Push(l);
92+
public void WriteSByte(sbyte b) => Push(ZigZagEncode(b, 8));
93+
public void WriteShort(short s) => Push(ZigZagEncode(s, 8));
94+
public void WriteInt(int i) => Push(ZigZagEncode(i, 8));
95+
public void WriteLong(long l) => Push(ZigZagEncode(l, 8));
9496
public void WriteString(string s) => Push(s);
97+
public void WriteAlignBits() => Push<object>(null);
9598
public void WriteFloatArray(float[] f, bool known = false) => PushArray(f, known);
9699
public void WriteDoubleArray(double[] d, bool known = false) => PushArray(d, known);
97100
public void WriteByteArray(byte[] b, bool known = false) => PushArray(b, known);
@@ -103,10 +106,12 @@ private void Push<T>(T b)
103106
public void WriteIntArray(int[] i, bool known = false) => PushArray(i, known);
104107
public void WriteLongArray(long[] l, bool known = false) => PushArray(l, known);
105108

106-
private void PushArray<T>(T[] t, bool knownSize = false)
109+
public void PushArray<T>(T[] t, bool knownSize = false)
107110
{
108-
if (!knownSize) Push(t);
109-
else foreach (T t1 in t) Push(t1);
111+
if (!knownSize) Push((uint)t.Length);
112+
bool signed = IsSigned(t.GetType().GetElementType());
113+
int size = Marshal.SizeOf(t.GetType().GetElementType());
114+
foreach (T t1 in t) Push(signed ? (object)ZigZagEncode(t1 as long? ?? t1 as int? ?? t1 as short? ?? t1 as sbyte? ?? 0, size) : (object)t1);
110115
}
111116

112117
public long Finalize(ref byte[] buffer)
@@ -117,52 +122,59 @@ public long Finalize(ref byte[] buffer)
117122
return 0;
118123
}
119124
long bitCount = 0;
120-
for (int i = 0; i < collectCount; ++i) bitCount += GetBitCount(collect[i]);
125+
for (int i = 0; i < collect.Count; ++i) bitCount += collect[i] == null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]);
121126

122-
if(buffer.Length < ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)))
127+
if (buffer.Length < ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)))
123128
{
124129
Debug.LogWarning("MLAPI: The buffer size is not large enough");
125130
return 0;
126131
}
127132
long bitOffset = 0;
133+
bool isAligned = true;
128134
foreach (var item in collect)
129-
Serialize(item, buffer, ref bitOffset);
135+
if (item == null)
136+
{
137+
bitOffset += (8 - (bitOffset % 8)) % 8;
138+
isAligned = true;
139+
}
140+
else Serialize(item, buffer, ref bitOffset, ref isAligned);
130141

131142
return (bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1);
132143
}
133144

134145
public long GetFinalizeSize()
135146
{
136147
long bitCount = 0;
137-
for (int i = 0; i < collectCount; ++i) bitCount += GetBitCount(collect[i]);
148+
for (int i = 0; i < collect.Count; ++i) bitCount += collect[i]==null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]);
138149
return ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1));
139150
}
140151

141-
private static void Serialize<T>(T t, byte[] writeTo, ref long bitOffset)
152+
private static void Serialize<T>(T t, byte[] writeTo, ref long bitOffset, ref bool isAligned)
142153
{
143154
Type type = t.GetType();
144155
bool size = false;
145156
if (type.IsArray)
146157
{
147158
var array = t as Array;
148-
Serialize((uint)array.Length, writeTo, ref bitOffset);
159+
Serialize((uint)array.Length, writeTo, ref bitOffset, ref isAligned);
149160
foreach (var element in array)
150-
Serialize(element, writeTo, ref bitOffset);
161+
Serialize(element, writeTo, ref bitOffset, ref isAligned);
151162
}
152163
else if (IsSupportedType(type))
153164
{
154-
long offset = GetBitAllocation(type);
165+
long offset = t is bool ? 1 : BytesToRead(t) * 8;
155166
if (type == typeof(bool))
156167
{
157168
WriteBit(writeTo, t as bool? ?? false, bitOffset);
158169
bitOffset += offset;
170+
isAligned = bitOffset % 8 == 0;
159171
}
160172
else if (type == typeof(decimal))
161173
{
162-
WriteDynamic(writeTo, (int)dec_lo.GetValue(t), 4, bitOffset);
163-
WriteDynamic(writeTo, (int)dec_mid.GetValue(t), 4, bitOffset + 32);
164-
WriteDynamic(writeTo, (int)dec_hi.GetValue(t), 4, bitOffset + 64);
165-
WriteDynamic(writeTo, (int)dec_flags.GetValue(t), 4, bitOffset + 96);
174+
WriteDynamic(writeTo, (int)dec_lo.GetValue(t), 4, bitOffset, isAligned);
175+
WriteDynamic(writeTo, (int)dec_mid.GetValue(t), 4, bitOffset + 32, isAligned);
176+
WriteDynamic(writeTo, (int)dec_hi.GetValue(t), 4, bitOffset + 64, isAligned);
177+
WriteDynamic(writeTo, (int)dec_flags.GetValue(t), 4, bitOffset + 96, isAligned);
166178
bitOffset += offset;
167179
}
168180
else if ((size = type == typeof(float)) || type == typeof(double))
@@ -181,71 +193,75 @@ private static void Serialize<T>(T t, byte[] writeTo, ref long bitOffset)
181193

182194
// Since floating point flag bits are seemingly the highest bytes of the floating point values
183195
// and even very small values have them, we swap the endianness in the hopes of reducing the size
184-
if(size) Serialize(BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0)), writeTo, ref bitOffset);
185-
else Serialize(BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)), writeTo, ref bitOffset);
196+
if(size) Serialize(BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0)), writeTo, ref bitOffset, ref isAligned);
197+
else Serialize(BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)), writeTo, ref bitOffset, ref isAligned);
186198
}
187-
//bitOffset += offset;
188199
}
189200
else
190201
{
191-
bool signed = IsSigned(t.GetType());
202+
//bool signed = IsSigned(t.GetType());
192203
ulong value;
193-
if (signed)
204+
/*if (signed)
194205
{
195206
Type t1 = t.GetType();
196207
if (t1 == typeof(sbyte)) value = (byte)ZigZagEncode(t as sbyte? ?? 0, 1);
197208
else if (t1 == typeof(short)) value = (ushort)ZigZagEncode(t as short? ?? 0, 2);
198209
else if (t1 == typeof(int)) value = (uint)ZigZagEncode(t as int? ?? 0, 4);
199-
else /*if (t1 == typeof(long))*/ value = (ulong)ZigZagEncode(t as long? ?? 0, 8);
210+
else /*if (t1 == typeof(long)) value = (ulong)ZigZagEncode(t as long? ?? 0, 8);
211+
}
212+
else*/
213+
if (t is byte)
214+
{
215+
WriteByte(writeTo, t as byte? ?? 0, bitOffset, isAligned);
216+
return;
200217
}
201-
else if (t is byte) value = t as byte? ?? 0;
202218
else if (t is ushort) value = t as ushort? ?? 0;
203219
else if (t is uint) value = t as uint? ?? 0;
204220
else /*if (t is ulong)*/ value = t as ulong? ?? 0;
205221

206-
if (value <= 240) WriteByte(writeTo, (byte)value, bitOffset);
222+
if (value <= 240) WriteByte(writeTo, (byte)value, bitOffset, isAligned);
207223
else if (value <= 2287)
208224
{
209-
WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset);
210-
WriteByte(writeTo, (value - 240) % 256, bitOffset + 8);
225+
WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset, isAligned);
226+
WriteByte(writeTo, (value - 240) % 256, bitOffset + 8, isAligned);
211227
}
212228
else if (value <= 67823)
213229
{
214-
WriteByte(writeTo, 249, bitOffset);
215-
WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8);
216-
WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16);
230+
WriteByte(writeTo, 249, bitOffset, isAligned);
231+
WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8, isAligned);
232+
WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16, isAligned);
217233
}
218234
else
219235
{
220-
WriteByte(writeTo, value & 255, bitOffset + 8);
221-
WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16);
222-
WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24);
236+
WriteByte(writeTo, value & 255, bitOffset + 8, isAligned);
237+
WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16, isAligned);
238+
WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24, isAligned);
223239
if (value > 16777215)
224240
{
225-
WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32);
241+
WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32, isAligned);
226242
if (value > 4294967295)
227243
{
228-
WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40);
244+
WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40, isAligned);
229245
if (value > 1099511627775)
230246
{
231-
WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48);
247+
WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48, isAligned);
232248
if (value > 281474976710655)
233249
{
234-
WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56);
250+
WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56, isAligned);
235251
if (value > 72057594037927935)
236252
{
237-
WriteByte(writeTo, 255, bitOffset);
238-
WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64);
253+
WriteByte(writeTo, 255, bitOffset, isAligned);
254+
WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64, isAligned);
239255
}
240-
else WriteByte(writeTo, 254, bitOffset);
256+
else WriteByte(writeTo, 254, bitOffset, isAligned);
241257
}
242-
else WriteByte(writeTo, 253, bitOffset);
258+
else WriteByte(writeTo, 253, bitOffset, isAligned);
243259
}
244-
else WriteByte(writeTo, 252, bitOffset);
260+
else WriteByte(writeTo, 252, bitOffset, isAligned);
245261
}
246-
else WriteByte(writeTo, 251, bitOffset);
262+
else WriteByte(writeTo, 251, bitOffset, isAligned);
247263
}
248-
else WriteByte(writeTo, 250, bitOffset);
264+
else WriteByte(writeTo, 250, bitOffset, isAligned);
249265
}
250266
bitOffset += BytesToRead(value) * 8;
251267
}
@@ -255,7 +271,7 @@ private static void Serialize<T>(T t, byte[] writeTo, ref long bitOffset)
255271
private static byte Read7BitRange(byte higher, byte lower, int bottomBits) => (byte)((higher << bottomBits) & (lower & (0xFF << (8-bottomBits))));
256272
private static byte ReadNBits(byte from, int offset, int count) => (byte)(from & ((0xFF >> (8-count)) << offset));
257273

258-
private static bool IsSigned(Type t) => Convert.ToBoolean(t.GetField("MinValue").GetValue(null));
274+
private static bool IsSigned(Type t) => t == typeof(sbyte) || t == typeof(short) || t == typeof(int) || t == typeof(long);
259275

260276
private static Type GetUnsignedType(Type t) =>
261277
t == typeof(sbyte) ? typeof(byte) :
@@ -274,13 +290,16 @@ private static long GetBitCount<T>(T t)
274290
{
275291
Type elementType = type.GetElementType();
276292

277-
count += 16; // Int16 array size. Arrays shouldn't be syncing more than 65k elements
278-
foreach (var element in t as Array)
279-
count += GetBitCount(element);
293+
count += BytesToRead((t as Array).Length) * 8; // Int16 array size. Arrays shouldn't be syncing more than 65k elements
294+
295+
if (elementType == typeof(bool)) count += (t as Array).Length;
296+
else
297+
foreach (var element in t as Array)
298+
count += GetBitCount(element);
280299
}
281300
else if (IsSupportedType(type))
282301
{
283-
long ba = GetBitAllocation(type);
302+
long ba = t is bool ? 1 : BytesToRead(t)*8;
284303
if (ba == 0) count += Encoding.UTF8.GetByteCount(t as string);
285304
else if (t is bool || t is decimal) count += ba;
286305
else count += BytesToRead(t) * 8;
@@ -292,33 +311,32 @@ private static long GetBitCount<T>(T t)
292311

293312
private static void WriteBit(byte[] b, bool bit, long index)
294313
=> b[index / 8] = (byte)((b[index / 8] & ~(1 << (int)(index % 8))) | (bit ? 1 << (int)(index % 8) : 0));
295-
private static void WriteByte(byte[] b, ulong value, long index) => WriteByte(b, (byte)value, index);
296-
private static void WriteByte(byte[] b, byte value, long index)
314+
private static void WriteByte(byte[] b, ulong value, long index, bool isAligned) => WriteByte(b, (byte)value, index, isAligned);
315+
private static void WriteByte(byte[] b, byte value, long index, bool isAligned)
297316
{
298-
int byteIndex = (int)(index / 8);
299-
int shift = (int)(index % 8);
300-
byte upper_mask = (byte)(0xFF << shift);
301-
byte lower_mask = (byte)~upper_mask;
317+
if (isAligned) b[index / 8] = value;
318+
else
319+
{
320+
int byteIndex = (int)(index / 8);
321+
int shift = (int)(index % 8);
322+
byte upper_mask = (byte)(0xFF << shift);
302323

303-
b[byteIndex] = (byte)((b[byteIndex] & lower_mask) | (value << shift));
304-
if(shift != 0 && byteIndex + 1 < b.Length)
324+
b[byteIndex] = (byte)((b[byteIndex] & (byte)~upper_mask) | (value << shift));
305325
b[byteIndex + 1] = (byte)((b[byteIndex + 1] & upper_mask) | (value >> (8 - shift)));
326+
}
306327
}
307-
private static void WriteBits(byte[] b, byte value, int bits, int offset, long index)
308-
{
309-
for (int i = 0; i < bits; ++i)
310-
WriteBit(b, (value & (1 << (i + offset))) != 0, index + i);
311-
}
312-
private static void WriteDynamic(byte[] b, int value, int byteCount, long index)
328+
private static void WriteDynamic(byte[] b, int value, int byteCount, long index, bool isAligned)
313329
{
314330
for (int i = 0; i < byteCount; ++i)
315-
WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i));
331+
WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i), isAligned);
316332
}
317333

318334
private static int BytesToRead(object i)
319335
{
336+
if (i is byte) return 1;
320337
bool size;
321338
ulong integer;
339+
if (i is decimal) return BytesToRead((int)dec_flags.GetValue(i)) + BytesToRead((int)dec_lo.GetValue(i)) + BytesToRead((int)dec_mid.GetValue(i)) + BytesToRead((int)dec_hi.GetValue(i));
322340
if ((size = i is float) || i is double)
323341
{
324342
int bytes = size ? 4 : 8;
@@ -337,7 +355,7 @@ private static int BytesToRead(object i)
337355
else integer = BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0));
338356
}
339357
}
340-
else integer = i as ulong? ?? 0;
358+
else integer = i as ulong? ?? i as uint? ?? i as ushort? ?? i as byte? ?? 0;
341359
return
342360
integer <= 240 ? 1 :
343361
integer <= 2287 ? 2 :
@@ -352,24 +370,7 @@ private static int BytesToRead(object i)
352370

353371
// Supported datatypes for serialization
354372
private static bool IsSupportedType(Type t) => supportedTypes.Contains(t);
355-
356-
// Specifies how many bits will be written
357-
private static long GetBitAllocation(Type t) =>
358-
t == typeof(bool) ? 1 :
359-
t == typeof(byte) ? 8 :
360-
t == typeof(sbyte) ? 8 :
361-
t == typeof(short) ? 16 :
362-
t == typeof(char) ? 16 :
363-
t == typeof(ushort) ? 16 :
364-
t == typeof(int) ? 32 :
365-
t == typeof(uint) ? 32 :
366-
t == typeof(long) ? 64 :
367-
t == typeof(ulong) ? 64 :
368-
t == typeof(float) ? 32 :
369-
t == typeof(double) ? 64 :
370-
t == typeof(decimal) ? 128 :
371-
0; // Unknown type
372-
373+
373374
// Creates a weak reference to the allocated collector so that reuse may be possible
374375
public void Dispose()
375376
{

0 commit comments

Comments
 (0)