DmitriyH
DmitriyH

Reputation: 450

"xpress" content-encoding decoding tool

Is there any binary tool or implementation source code example, which can decode and encode content in "xpress" content-encoding?

The only thing I could find, is the [MS-XCA]: Xpress Compression Algorithm verbal description.

It is required to understand the content of server responses like the following:

HTTP/1.1 200 OK
Cache-Control: private
Content-Type: application/soap+xml; charset=utf-8
Content-Encoding: xpress
Vary: Accept-Encoding
Server: Microsoft-IIS/10.0
X-Powered-By: ASP.NET
Date: Wed, 13 Sep 2017 10:56:16 GMT
Content-Length: 1003

<!-- Response in xpress encoding -->

Upvotes: 2

Views: 448

Answers (1)

Mr. Doge
Mr. Doge

Reputation: 886

A more accurate description than [MS-XCA] can be found here: [MS-WUSP]: Windows Update Services: Client-Server Protocol

The Block Header is described:
original(4 bytes), compressed(4 bytes)
compressed data follows...

only LZ77 and DIRECT2 are described, no Huffman for instance

from your link I got: [MS-WUSP].pdf

Plain LZ77 Decompression Algorithm Details: this seems to be the one used in the encoding, since the decompression works

Pseudocode: does not specify output buffer size, uses Little-Endian

BufferedFlags = 0
BufferedFlagCount = 0
InputPosition = 0
OutputPosition = 0
LastLengthHalfByte = 0
Loop until break instruction or error
    If BufferedFlagCount == 0
        BufferedFlags = read 4 bytes at InputPosition
        InputPosition += 4
        BufferedFlagCount = 32
    BufferedFlagCount = BufferedFlagCount - 1
    If (BufferedFlags & (1 << BufferedFlagCount)) == 0
        Copy 1 byte from InputPosition to OutputPosition.  Advance both.
    Else
        If InputPosition == InputBufferSize
            Decompression is complete.  Return with success.
        MatchBytes = read 2 bytes from InputPosition
        InputPosition += 2
        MatchLength = MatchBytes mod 8
        MatchOffset = (MatchBytes / 8) + 1
        If MatchLength == 7
            If LastLengthHalfByte == 0
                MatchLength = read 1 byte from InputPosition
                MatchLength = MatchLength mod 16
                LastLengthHalfByte = InputPosition
                InputPosition += 1
            Else
                MatchLength = read 1 byte from LastLengthHalfByte position
                MatchLength = MatchLength / 16
                LastLengthHalfByte = 0
            If MatchLength == 15
                MatchLength = read 1 byte from InputPosition
                InputPosition += 1
                If MatchLength == 255
                    MatchLength = read 2 bytes from InputPosition
                    InputPosition += 2
                    If MatchLength == 0
                        MatchLength = read 4 bytes from InputPosition
                        InputPosition += 4 bytes
                    If MatchLength < 15 + 7
                       Return error.
                    MatchLength -= (15 + 7)
                MatchLength += 15
            MatchLength += 7
        MatchLength += 3
        For i = 0 to MatchLength - 1
            Copy 1 byte from OutputBuffer[OutputPosition - MatchOffset]
            OutputPosition += 1

Xpress_LZ77_DecompressRaw same as the Pseudocode, used to decode the example
Xpress_LZ77_Decompress reads output buffer size from the block header, this is format used in the network response
The example to decode from https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/72da4f8d-2ba3-437d-b772-2e4173713a0b is included: ffffff1f61626317000fff2601

import { readFileSync } from "fs";

function Uint8Array_from_hex(hex) {
    return new Uint8Array(hex.match(/[0-9a-fA-F]{2}/g).map(v=>parseInt(v,16)))
}

const textDecoder = new TextDecoder()

{//scoped-block
    //example: raw decompression
    const inBuf = Uint8Array_from_hex("ffffff1f61626317000fff2601")
    const outBuf = Xpress_LZ77_DecompressRaw(inBuf)
    console.log(textDecoder.decode(outBuf))
}
{//scoped-block
    //File decompression <originalSize,compressedSize>[compressed]*
    const outBuf = Xpress_LZ77_Decompress(readFileSync(String.raw`Untitled1`))
    console.log(textDecoder.decode(outBuf))
}

/**
 * @param {Uint8Array} inBuf
 */
function Xpress_LZ77_Decompress(inBuf) {
    //ensure Uint8Array
    inBuf.buffer && (inBuf = new Uint8Array(inBuf.buffer, inBuf.byteOffset, inBuf.byteLength))

    const InputBufferSize = inBuf.byteLength
    const inView = new DataView(inBuf.buffer, inBuf.byteOffset, inBuf.byteLength)
    let totalOutputLength = 0
    let InputPosition = 0
    while (InputPosition < InputBufferSize) {
        const original = inView.getUint32(InputPosition, true)
        InputPosition += 4
        if (original > 65535) {
            throw new Error("Uncompressed size of each block MUST NOT be greater than 65535 bytes.")
        }
        const compressed = inView.getUint32(InputPosition, true)
        InputPosition += 4
        if (compressed > 65535) {
            throw new Error("The compressed size of each block MUST NOT be greater than 65535 bytes.")
        }
        totalOutputLength += original
        InputPosition += compressed
    }
    const outBuf = new Uint8Array(totalOutputLength)
    InputPosition = 0
    let OutputPosition = 0
    while (InputPosition < InputBufferSize) {
        const original = inView.getUint32(InputPosition, true)
        InputPosition += 4
        const compressed = inView.getUint32(InputPosition, true)
        InputPosition += 4
        OutputPosition += Xpress_LZ77_DecompressRaw(inBuf.subarray(InputPosition, InputPosition + compressed), outBuf.subarray(OutputPosition, OutputPosition + original)).byteLength
        InputPosition += compressed
    }
    return outBuf
}
/**
 * @param {Uint8Array} inBuf
 * @param {Uint8Array} outBuf
 */
function Xpress_LZ77_DecompressRaw(inBuf, outBuf = new Uint8Array(1024**3)) {
    const InputBufferSize = inBuf.byteLength
    const inView = new DataView(inBuf.buffer, inBuf.byteOffset, inBuf.byteLength)
    const outView = new DataView(outBuf.buffer, outBuf.byteOffset, outBuf.byteLength)
    let BufferedFlags = 0
    let BufferedFlagCount = 0
    let InputPosition = 0
    let OutputPosition = 0
    let LastLengthHalfByte = 0
    while (1) {
        if (BufferedFlagCount === 0) {
            BufferedFlags = inView.getUint32(InputPosition, true)
            InputPosition += 4
            BufferedFlagCount = 32
        }
        BufferedFlagCount = BufferedFlagCount - 1
        if ((BufferedFlags & (1 << BufferedFlagCount)) === 0) {
            // Copy 1 byte from InputPosition to OutputPosition. Advance both.
            outView.setUint8(OutputPosition, inView.getUint8(InputPosition))
            InputPosition += 1
            OutputPosition += 1
        } else {
            if (InputPosition === InputBufferSize) {
                // Decompression is complete. Return with success.
                return outBuf.subarray(0, OutputPosition)
            }
            const MatchBytes = inView.getUint16(InputPosition, true)
            InputPosition += 2
            let MatchLength = MatchBytes & 0b111
            const MatchOffset = (MatchBytes >>> 3) + 1
            if (MatchLength === 7) {

                if (LastLengthHalfByte === 0) {
                    MatchLength = inView.getUint8(InputPosition)
                    MatchLength = MatchLength & 0b1111
                    LastLengthHalfByte = InputPosition
                    InputPosition += 1
                } else {
                    MatchLength = inView.getUint8(LastLengthHalfByte)
                    MatchLength = MatchLength >>> 4
                    LastLengthHalfByte = 0
                }
                if (MatchLength === 15) {
                    MatchLength = inView.getUint8(InputPosition)
                    InputPosition += 1
                    if (MatchLength === 255) {
                        MatchLength = inView.getUint16(InputPosition, true)
                        InputPosition += 2
                        if (MatchLength === 0) {
                            MatchLength = inView.getUint32(InputPosition, true)
                            InputPosition += 4
                        }
                        if (MatchLength < 15 + 7) {
                            throw new Error("Invalid match length")
                        }
                        MatchLength -= (15 + 7)
                    }
                    MatchLength += 15
                }
                MatchLength += 7
            }
            MatchLength += 3
            for (let i = 0; i < MatchLength; i++) {
                outView.setUint8(OutputPosition, outView.getUint8(OutputPosition - MatchOffset))
                OutputPosition += 1
            }
        }
    }
}

Upvotes: 0

Related Questions