Reputation: 21
I am trying to write code to test the speed of lz4 compress and decompress However, when I add the decompress function in the code, the error "corrupted size vs. prev_size" starts to show up.
In my mind, I think the problems should be in the calloc function of src. Because the program stops at free(src) when running debug.
Her is my code.
(the lz4.h and lz4.c is on https://github.com/lz4/lz4/tree/355f60952938e5bd4d45118af720d4b8fb0c8a09
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "lz4.h"
int splitFile(char* fileIn, size_t maxSize);
#include "lz4.c"
int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
int LZ4_compressBound(int inputSize);
int main()
{
clock_t start_lz4_c,end_lz4_c,start_lz4_d,end_lz4_d;
double compress_time,large_time,decompress_time,compress_speed,decompress_speed,small_time_c,small_average_time_c,small_time_d,small_average_time_d;
int64_t num_f=0;
int64_t i;
char * temp_src=0;
int srcSize= 0;
int dstCapacity=16384;
int lz4_compressed_data_size=0,lz4_decompressed_data_size=0;
int size_after_comp=0;
int size_before_comp=0;
int size_of_file=0;
char* compressed_data = malloc((size_t)dstCapacity);
char buff[200];
FILE * fc;
FILE * ft;
//split file
num_f = splitFile("/home/ziruo/research/1stpro/test.txt",16384);
printf("num_f=%ld\n",num_f);
//read the length for file
ft = fopen("/home/ziruo/research/1stpro/test.txt","r");
fseek(ft,0,SEEK_END);
size_of_file = ftell(ft);
printf("file len = %d\n",size_of_file);
fclose(ft);
//main loop
for( i = 1; i <= num_f; i++)
{
char* src=calloc(16384,1);
char* regen_buffer = calloc(srcSize,1);
//read in
sprintf(buff,"/home/ziruo/research/1stpro/test.txt.%03ld",I);
if (compressed_data == NULL)
{
printf("faild to generae storage\n");
}
fc = fopen(buff,"r");
fread(src,16384,1,fc);
srcSize=(int)(strlen(src) + 1);
fclose(fc);
small_time_c = 0;
small_time_d = 0;
if(size_of_file <= 100000000)
{
int a;
for(a = 1;a<=10;a++)
{
start_lz4_c = clock();
dstCapacity= LZ4_compressBound(srcSize);
lz4_compressed_data_size = LZ4_compress_default(src,compressed_data,srcSize,dstCapacity);
end_lz4_c = clock();
start_lz4_d = clock();
lz4_decompressed_data_size = LZ4_decompress_safe(compressed_data,regen_buffer,lz4_compressed_data_size,srcSize);
end_lz4_d = clock();
small_time_c += ((double)(end_lz4_c-start_lz4_c))/CLOCKS_PER_SEC;
small_average_time_c = small_time_c/10;
small_time_d += ((double)(end_lz4_d-start_lz4_d))/CLOCKS_PER_SEC;
small_average_time_d = small_time_d/10;
}
printf("time %f\n",small_average_time_d);
compress_time += small_average_time_c;
decompress_time += small_average_time_d;
}
else
{
start_lz4_c = clock();
dstCapacity= LZ4_compressBound(srcSize);
LZ4_compress_default(src,compressed_data,srcSize,dstCapacity);
end_lz4_c = clock();
large_time = ((double)(end_lz4_c-start_lz4_c))/CLOCKS_PER_SEC;
//printf("time %f\n",large_time);
//compress_time += large_time;
}
//calculate time & speed
size_before_comp += srcSize;
size_after_comp += lz4_compressed_data_size;
printf("decompressed_data_size is %d\n",lz4_decompressed_data_size);
printf("decompression speed: %fMB/s\n",size_after_comp/(decompress_time *1000000));
free(src);
free(regen_buffer);
}
printf("before %d after %d\n",size_before_comp,size_after_comp);
printf("compression speed: %fMB/s\n",size_before_comp/(compress_time*1000000));
printf("compression ratio: %f\n",(float) size_before_comp/size_after_comp);
printf("time used(s): %f\n",compress_time);
printf("decompressed_data_size is %d\n",lz4_decompressed_data_size);
printf("decompression speed: %fMB/s\n",size_after_comp/(decompress_time *1000000));
return 0;
}
int splitFile(char* fileIn, size_t maxSize)
{
int result = 0;
FILE* fIn;
FILE* fOut;
char buffer[1024 * 16];
size_t size;
size_t read;
size_t written;
if ((fileIn != NULL) && (maxSize > 0))
{
fIn = fopen(fileIn, "rb");
if (fIn != NULL)
{
fOut = NULL;
result = 1; // we have at least one part
while (!feof(fIn))
{
// initialize (next) output file if no output file opened
if (fOut == NULL)
{
sprintf(buffer, "%s.%03d", fileIn, result);
fOut = fopen(buffer, "wb");
if (fOut == NULL)
{
result = -1;
break;
}
size = 0;
}
// calculate size of data to be read from input file in order to not exceed maxSize
read = sizeof(buffer);
if ((size + read) > maxSize)
{
read = maxSize - size;
}
// read data from input file
read = fread(buffer, 1, read, fIn);
if (read == 0)
{
result = -1;
break;
}
// write data to output file
written = fwrite(buffer, 1, read, fOut);
if (written != read)
{
result = -1;
break;
}
// update size counter of current output file
size += written;
if (size >= maxSize) // next split?
{
fclose(fOut);
fOut = NULL;
result++;
}
}
// clean up
if (fOut != NULL)
{
fclose(fOut);
}
fclose(fIn);
}
}
return (result);
}
Upvotes: 0
Views: 487