How to compress and decompress a file using lz4?

Question

I want to compress and decompress a file using lz4 algorithm in Go. Is there any package available to do this? I searched and found a package called https://github.com/pierrec/lz4

I am new Go and I cannot figure out how to use this package to compress and decompress a file.

I need to use this package to compress a file to binary format and decompress the binary file to original file using Go.

Answer 1

I am also new to Go and have struggled a bit using github.com/pierrec/lz4.

What I was misunderstanding is that calling Close() on NewWriter is not optional and failing to do so will lead to incorrect results. (I spent a lot of time banging my head against the wall for thinking this was optional and just a best-practice, as it is in closing file handlers, network connections, etc)

I wrote two wrapper versions for compressing/decompressing.

First, a generic reader/writer approach (similar to the example on the README, but without pipes) [playground]:

func compress(r io.Reader, w io.Writer) error {
    zw := lz4.NewWriter(w)
    _, err := io.Copy(zw, r)
    if err != nil {
        return err
    }
    // Closing is *very* important
    return zw.Close()
}

func decompress(r io.Reader, w io.Writer) error {
    zr := lz4.NewReader(r)
    _, err := io.Copy(w, zr)
    return err
}

If your data size is small and you don't need to/want to mess with buffers and just want to have uncompressed bytes in, compressed bytes out, (in a more "functional" fashion) this second version may be more convenient [playground]:

func compress(in []byte) ([]byte, error) {
    r := bytes.NewReader(in)
    w := &bytes.Buffer{}
    zw := lz4.NewWriter(w)
    _, err := io.Copy(zw, r)
    if err != nil {
        return nil, err
    }
    // Closing is *very* important
    if err := zw.Close(); err != nil {
        return nil, err
    }
    return w.Bytes(), nil
}

func decompress(in []byte) ([]byte, error) {
    r := bytes.NewReader(in)
    w := &bytes.Buffer{}
    zr := lz4.NewReader(r)
    _, err := io.Copy(w, zr)
    if err != nil {
        return nil, err
    }
    return w.Bytes(), nil
}

Answer 2

The result What i expected is from the below code. I got this [ https://www.google.com/url?q=https%3A%2F%2Fgithub.com%2Fpierrec%2Flz4%2Fblob%2Fmaster%2Flz4c%2Fmain.go&sa=D&sntz=1&usg=AFQjCNFIT2O1Grs0vu4Gh8Af96GSBaa9EA ] from this file. File is given as input in the command line argument and its compressed/Decompressed Successfully.

package main
import (
    //  "bytes"

    "flag"
    "fmt"
    "io"
    "log"
    "os"
    "path"
    "runtime"
    "strings"

    "github.com/pierrec/lz4"
)

func main() {
    // Process command line arguments
    var (
        blockMaxSizeDefault = 4 << 20
        flagStdout          = flag.Bool("c", false, "output to stdout")
        flagDecompress      = flag.Bool("d", false, "decompress flag")
        flagBlockMaxSize    = flag.Int("B", blockMaxSizeDefault, "block max size [64Kb,256Kb,1Mb,4Mb]")
        flagBlockDependency = flag.Bool("BD", false, "enable block dependency")
        flagBlockChecksum   = flag.Bool("BX", false, "enable block checksum")
        flagStreamChecksum  = flag.Bool("Sx", false, "disable stream checksum")
        flagHighCompression = flag.Bool("9", false, "enabled high compression")
    )
    flag.Usage = func() {
        fmt.Fprintf(os.Stderr, "Usage:\n\t%s [arg] [input]...\n\tNo input means [de]compress stdin to stdout\n\n", os.Args[0])
        flag.PrintDefaults()
    }
    flag.Parse()
    fmt.Println("output to stdout ", *flagStdout)
    fmt.Println("Decompress", *flagDecompress)
    // Use all CPUs
    runtime.GOMAXPROCS(runtime.NumCPU())

    zr := lz4.NewReader(nil)
    zw := lz4.NewWriter(nil)
    zh := lz4.Header{
        BlockDependency: *flagBlockDependency,
        BlockChecksum:   *flagBlockChecksum,
        BlockMaxSize:    *flagBlockMaxSize,
        NoChecksum:      *flagStreamChecksum,
        HighCompression: *flagHighCompression,
    }

    worker := func(in io.Reader, out io.Writer) {
        if *flagDecompress {
            fmt.Println("\n Decompressing the data")
            zr.Reset(in)
            if _, err := io.Copy(out, zr); err != nil {
                log.Fatalf("Error while decompressing input: %v", err)
            }
        } else {
            zw.Reset(out)
            zw.Header = zh
            if _, err := io.Copy(zw, in); err != nil {
                log.Fatalf("Error while compressing input: %v", err)
            }
        }
    }

    // No input means [de]compress stdin to stdout
    if len(flag.Args()) == 0 {
        worker(os.Stdin, os.Stdout)
        os.Exit(0)
    }

    // Compress or decompress all input files
    for _, inputFileName := range flag.Args() {
        outputFileName := path.Clean(inputFileName)

        if !*flagStdout {
            if *flagDecompress {
                outputFileName = strings.TrimSuffix(outputFileName, lz4.Extension)
                if outputFileName == inputFileName {
                    log.Fatalf("Invalid output file name: same as input: %s", inputFileName)
                }
            } else {
                outputFileName += lz4.Extension
            }
        }

        inputFile, err := os.Open(inputFileName)
        if err != nil {
            log.Fatalf("Error while opening input: %v", err)
        }

        outputFile := os.Stdout
        if !*flagStdout {
            outputFile, err = os.Create(outputFileName)
            if err != nil {
                log.Fatalf("Error while opening output: %v", err)
            }
        }
        worker(inputFile, outputFile)

        inputFile.Close()
        if !*flagStdout {
            outputFile.Close()
        }
    }
}

Sample Input

go run compress.go -9=true sample.txt

Answer 3

Using the bufio package you can (de)compress files without slurping their entire contents into your memory all at once.

In effect this allows you to (de)compress files larger than the memory available to the system, which may or may not be relevant to your specific circumstances.

If this is relevant, you can find a working example here:

package main

import (
    "bufio"
    "io"
    "os"

    "github.com/pierrec/lz4"
)

// Compress a file, then decompress it again!
func main() {
    compress("./compress-me.txt", "./compressed.txt")
    decompress("./compressed.txt", "./decompressed.txt")
}

func compress(inputFile, outputFile string) {
    // open input file
    fin, err := os.Open(inputFile)
    if err != nil {
        panic(err)
    }
    defer func() {
        if err := fin.Close(); err != nil {
            panic(err)
        }
    }()
    // make a read buffer
    r := bufio.NewReader(fin)

    // open output file
    fout, err := os.Create(outputFile)
    if err != nil {
        panic(err)
    }
    defer func() {
        if err := fout.Close(); err != nil {
            panic(err)
        }
    }()
    // make an lz4 write buffer
    w := lz4.NewWriter(fout)

    // make a buffer to keep chunks that are read
    buf := make([]byte, 1024)
    for {
        // read a chunk
        n, err := r.Read(buf)
        if err != nil && err != io.EOF {
            panic(err)
        }
        if n == 0 {
            break
        }

        // write a chunk
        if _, err := w.Write(buf[:n]); err != nil {
            panic(err)
        }
    }

    if err = w.Flush(); err != nil {
        panic(err)
    }
}

func decompress(inputFile, outputFile string) {
    // open input file
    fin, err := os.Open(inputFile)
    if err != nil {
        panic(err)
    }
    defer func() {
        if err := fin.Close(); err != nil {
            panic(err)
        }
    }()

    // make an lz4 read buffer
    r := lz4.NewReader(fin)

    // open output file
    fout, err := os.Create(outputFile)
    if err != nil {
        panic(err)
    }
    defer func() {
        if err := fout.Close(); err != nil {
            panic(err)
        }
    }()

    // make a write buffer
    w := bufio.NewWriter(fout)

    // make a buffer to keep chunks that are read
    buf := make([]byte, 1024)
    for {
        // read a chunk
        n, err := r.Read(buf)
        if err != nil && err != io.EOF {
            panic(err)
        }
        if n == 0 {
            break
        }

        // write a chunk
        if _, err := w.Write(buf[:n]); err != nil {
            panic(err)
        }
    }

    if err = w.Flush(); err != nil {
        panic(err)
    }
}

Answer 4

I think blow example should direct you to correct direction. It is the simplest example of how to compress and decompress using github.com/pierrec/lz4 package.

//compress project main.go
package main

import "fmt"
import "github.com/pierrec/lz4"

var fileContent = `CompressBlock compresses the source buffer starting at soffet into the destination one.
This is the fast version of LZ4 compression and also the default one.
The size of the compressed data is returned. If it is 0 and no error, then the data is incompressible.
An error is returned if the destination buffer is too small.`

func main() {
    toCompress := []byte(fileContent)
    compressed := make([]byte, len(toCompress))

    //compress
    l, err := lz4.CompressBlock(toCompress, compressed, 0)
    if err != nil {
        panic(err)
    }
    fmt.Println("compressed Data:", string(compressed[:l]))

    //decompress
    decompressed := make([]byte, len(toCompress))
    l, err = lz4.UncompressBlock(compressed[:l], decompressed, 0)
    if err != nil {
        panic(err)
    }
    fmt.Println("\ndecompressed Data:", string(decompressed[:l]))
}