fast and memory-efficient way of converting byte[] to arrays of primitives

Question

I am passed a byte array which is then converted to arrays of primitives using System.Buffer.BlockCopy. Essentially my code looks like this:

    void Convert(byte[] b)
    {
        int[] i1 = new int[100];    // real arrays much larger
        double[] d1 = new double[100];
        int iPos=0, iSize;

        iSize = i1.Length * sizeof(int);
        System.Buffer.BlockCopy(b, iPos, i1, 0, iSize);
        iPos += iSize;

        iSize = d1.Length * sizeof(double);
        System.Buffer.BlockCopy(b, iPos, d1, 0, iSize);
        iPos += iSize;

        //etc: lots of arrays

        b=null;         
    }

This is quite performant, but it's memory usage is obviously 2x the size of my byte array until b is freed.

Is there any way to directly cast sections of the byte array to my primitive arrays? One that does not involve copying the data (therefore doesn't double the memory usage), and presumably is even faster?

Answer 1

This is an attempt with unioned structure of arrays (unlike the array of struct in other postings) because this should be faster.

[StructLayout(LayoutKind.Explicit)]
public unsafe struct ByteBuffer
{        
    public const int IntSize=10;
    public const int DoubleSize=5;
    public const int IntBytes=sizeof(int)*IntSize;
    public const int DoubleBytes=sizeof(double)*DoubleSize;

    // Int array is unioned with byte array
    [FieldOffset(0)]
    fixed int int_array[IntSize];
    [FieldOffset(0)]
    fixed byte int_array_bytes[IntBytes];

    // Double array us unioned with byte array
    [FieldOffset(IntBytes)]
    fixed double double_array[DoubleSize];
    [FieldOffset(IntBytes)]
    fixed byte double_array_bytes[DoubleBytes];

    // Take array of bytes and distribute it 
    // by byte to each array
    public ByteBuffer(byte[] b)
    {
        fixed(byte* ptr=int_array_bytes)
        {
            for(int i=0; i<IntBytes; i++)
            {
                ptr[i]=b[i];
            }
        }
        fixed(byte* ptr=double_array_bytes)
        {

            for(int i=0; i<DoubleBytes; i++)
            {
                ptr[i]=b[IntBytes+i];
            }
        }
    }
    // Convert unmanaged array to managed array
    public int[] ToIntArray()
    {
        int[] result=new int[IntSize];
        fixed(int* ptr=int_array)
        {
            for(int i=0; i<IntSize; i++)
            {
                result[i]=ptr[i];
            }
        }
        return result;
    }
    // Convert unmanaged array to managed array
    public double[] ToDoubleArray()
    {
        double[] result=new double[DoubleSize];
        fixed(double* ptr=double_array)
        {
            for(int i=0; i<DoubleSize; i++)
            {
                result[i]=ptr[i];
            }
        }
        return result;
    }
}

class Program
{
    static void Main(string[] args)
    {
        // Load up with test data
        byte[] data=new byte[..];
        // I tested with 10 ints and 5 doubles encoded into bytes

        //Now to test the Fast conversion
        ByteBuffer bb=new ByteBuffer(data);
        int[] data1=bb.ToIntArray();
        double[] data2=bb.ToDoubleArray();
    }
}

Answer 2

It is possible using unsafe code. One solution using a union (what my not be applicable in your situation)

namespace TestApplication
{
    using System;
    using System.Runtime.InteropServices;

    internal static class Program
    {
        private static unsafe void Main()
        {           
            var x = new ByteDoubleUnion();

            x.bytes[0] =  24;
            x.bytes[1] =  45;
            x.bytes[2] =  68;
            x.bytes[3] =  84;
            x.bytes[4] = 251;
            x.bytes[5] =  33;
            x.bytes[6] =   9;
            x.bytes[7] =  64;

            // Prints pi.
            Console.WriteLine(x.doubleValue);

            Console.ReadLine();
        }
    }

    [StructLayout(LayoutKind.Explicit)]
    internal unsafe struct ByteDoubleUnion
    {
        [FieldOffset(0)]
        internal Double doubleValue;

        [FieldOffset(0)]
        internal fixed Byte bytes[8];
    }
}

and one solution just casting pointers.

namespace TestApplication
{
    using System;

    internal static class Program
    {
        private static unsafe void Main()
        {           
            var bytes = new Byte[] { 24, 45, 68, 84, 251, 33, 9, 64 };

            fixed (Byte* p = &bytes[0])
            {
                // Prints pi, too.
                Console.WriteLine(*((Double*)p));
            }

            Console.ReadLine();
        }
    }
}

Answer 3

You can use unsafe code (I don't if you are allowed to use it). But you can try something like this (no need to use extra arrays, just the array of bytes):

    unsafe public void Convert(void* b)
    {
        int i;

        double* asDouble = (double*)b;
        double sum1 = 0.0;
        for (i = 0; i < 100; i++, asDouble++)
            sum1 += *asDouble;

        int* asInt = (int*)asDouble;
        int sum2 = 0;
        for (i = 0; i < 100; i++, asInt++)
            sum2 += *asInt;
    }

    public unsafe void SomeThing()
    {
        byte[] rawbytes = new byte[44000];

        // Fill the "rawbytes" array somehow

        fixed (byte* ptr = rawbytes)
        {
            Convert(ptr);
        }
    }