Java LZ4 compression using Input/Output streams

Question

I'm using jpountz LZ4 to try and compress files and I want to read in and output files using Java file input and output streams. I've tried to find a solution online but theres nothing, I found a previous stackoverflow question on how to implement LZ4 correctly and I've taken that and tried to modify it to use the streams, but I'm not sure if this is correct or if it's even working.

When running the compression on a text file it outputs a file which has some characters missing or replaced with symbols

ðHello world Heðo world Hello ðrld Hello worlðHello worl

but when running it with a image file it throws an out of bounds error. I've also been unable to get decompression to work as it throws a Error decoding offset 3 of input buffer.

Here is my code any help would be appreciated thanks

public void LZ4Compress(InputStream in, OutputStream out){
    int noBytesRead = 0;        //number of bytes read from input
    int noBytesProcessed = 0;   //number of bytes processed
    try {
        while ((noBytesRead = in.read(inputBuffer)) >= 0) {
            noBytesProcessed = inputBuffer.length;
            decompressedLength = inputBuffer.length;
            outputBuffer = compress(inputBuffer, decompressedLength);
            out.write(outputBuffer, 0, noBytesRead);
        }
        out.flush();
        in.close();
        out.close();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}

public void LZ4decompress(InputStream in, OutputStream out){
    int noBytesRead = 0;        //number of bytes read from input
    try {
        while((noBytesRead = in.read(inputBuffer)) >= 0){
            noBytesProcessed = inputBuffer.length;
            outputBuffer = decompress(inputBuffer);
            out.write(outputBuffer, 0, noBytesRead);

        }
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}

public static byte[] compress(byte[] src, int srcLen) {
    decompressedLength = srcLen;
    int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
    byte[] compressed = new byte[maxCompressedLength];
    int compressLen = compressor.compress(src, 0, decompressedLength, compressed, 0, maxCompressedLength);
    byte[] finalCompressedArray = Arrays.copyOf(compressed, compressLen);
    return finalCompressedArray;
}

private static LZ4SafeDecompressor decompressor = factory.safeDecompressor();

public static byte[] decompress(byte[] finalCompressedArray) {
    byte[] restored = new byte[finalCompressedArray.length];
    restored = decompressor.decompress(finalCompressedArray, finalCompressedArray.length);
    return restored;
}

Answer 1

Just for those who are looking for the Java utility code to tar & compress files/directories using lz4. Let me know if you find any problems with it.

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Path;
import java.nio.file.Paths;

import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import net.jpountz.lz4.LZ4FrameOutputStream;

public class TarLz4Util {
    private static final Logger logger = LoggerFactory.getLogger(TarLz4Util .class);

    private final static int BUFFER_SIZE = 1024 * 1024 * 10;

    public static void compress(String sourcePath, String targetPath) throws IOException {
        compress(Paths.get(sourcePath), Paths.get(sourcePath));
    }

    /**
     * compress with lz4
     *
     */
    private static void compress(Path sourcePath, Path targetPath) throws IOException {
        //Files ->  Tar  -> LZ4 
        try (LZ4FrameOutputStream lz4Out = new LZ4FrameOutputStream(
                new FileOutputStream(targetPath.toFile()))) {
            TarArchiveOutputStream tarOut = new TarArchiveOutputStream(lz4Out);
            tarOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
            tar(sourcePath.toFile(), tarOut);
        }
    }

    /**
     * tar your files/directories recursively
     *
     */
    private static void tar(File sourceFile, TarArchiveOutputStream tarOut) throws IOException {
        if (!sourceFile.exists()) {
            logger.warn("source file/directory does not exist!, path={}", sourceFile.getAbsolutePath());
            return;
        }

        TarArchiveEntry entry = new TarArchiveEntry(sourceFile);
        tarOut.putArchiveEntry(entry);
        if (sourceFile.isFile()) {
            writeFile(new BufferedInputStream(new FileInputStream(sourceFile)), tarOut);
            tarOut.closeArchiveEntry();
        } else {
            for (File file : sourceFile.listFiles()) {
                tar(file, tarOut);
            }
        }
    }

    /**
     * copy file
     *
     */
    private static long writeFile(InputStream in, OutputStream out) throws IOException {
        int read = 0;
        long count = 0;
        byte[] buf = new byte[BUFFER_SIZE];
        while ((read = in.read(buf)) != -1) {
            out.write(buf, 0, read);
            count += read;
        }
        return count;
    }

    // a demo
    public static void main(String[] args) throws Exception {
        String srcDir = "d:\\test";
        String tarFile = "test.tar.lz4";
       
        compress(Paths.get(srctDir), Paths.get(tarFile));
    }
}

Answer 2

So I solved my problem by using LZ4block input/output streams

public static void LZ4compress(String filename, String lz4file){
    byte[] buf = new byte[2048];
    try {
        String outFilename = lz4file;
        LZ4BlockOutputStream out = new LZ4BlockOutputStream(new FileOutputStream(outFilename), 32*1024*1024);
        FileInputStream in = new FileInputStream(filename);
        int len;
        while((len = in.read(buf)) > 0){
            out.write(buf, 0, len);
        }
        in.close();
        out.close();
    } catch (IOException e) {

    }
}

public static void LZ4Uncompress(String lz4file, String filename){
    byte[] buf = new byte[2048];
    try {
        String outFilename = filename;
        LZ4BlockInputStream in = new LZ4BlockInputStream(new FileInputStream(lz4file));
        FileOutputStream out = new FileOutputStream(outFilename);
        int len;
        while((len = in.read(buf)) > 0){
            out.write(buf, 0, len);
        }
        in.close();
        out.close();
    } catch (IOException e) {

    }
}

Answer 3

Looking only at code, I would say you are going wrong here:

 outputBuffer = compress(inputBuffer, decompressedLength);
 out.write(outputBuffer, 0, noBytesRead);

You have already trimmed outputBuffer in compress. Try:

out.write(outputBuffer);