Reputation: 1536
C++ fast file reading line by line
I want to create a simple .OBJ parser. Code below works, but is slow. Im testing it with a 26MB file and it takes about 22 seconds to parse it. If i commented out the lines seen in the code, the time went to 17 seconds. It takes 17 seconds to iterate the file line by line and to extract data. Is there any way i can make it faster? I also tried reading the file into memory and then parsing it. It became slower.
while (std::getline(file, line)) {
if (line[0] == 'v') {
std::istringstream iss(line);
std::string type;
GLfloat x, y, z;
iss >> type >> x >> y >> z;
if (type.compare("v") == 0) {
//vertices.push_back(x);
//vertices.push_back(y);
//vertices.push_back(z);
}
else if (type.compare("vn") == 0) {
//normals.push_back(x);
//normals.push_back(y);
//normals.push_back(z);
}
else if (type.compare("vt") == 0) {
//UVs.push_back(x);
//UVs.push_back(y);
}
}
else if (line[0] == 'f') {
unsigned int size = out_vertices.size() / 3;
unsigned int vertexIndex[3], uvIndex[3], normalIndex[3];
sscanf_s(line.c_str(),
"f %d/%d/%d %d/%d/%d %d/%d/%d",
&vertexIndex[0], &uvIndex[0], &normalIndex[0],
&vertexIndex[1], &uvIndex[1], &normalIndex[1],
&vertexIndex[2], &uvIndex[2], &normalIndex[2]);
/*GLfloat* vertex = &vertices[(vertexIndex[0] - 1) * 3];
out_vertices.insert(out_vertices.end(), vertex, vertex + 3);
vertex = &vertices[(vertexIndex[1] - 1) * 3];
out_vertices.insert(out_vertices.end(), vertex, vertex + 3);
vertex = &vertices[(vertexIndex[2] - 1) * 3];
out_vertices.insert(out_vertices.end(), vertex, vertex + 3);
GLfloat* uv = &UVs[(uvIndex[0] - 1) * 2];
out_UVs.insert(out_UVs.end(), uv, uv + 2);
uv = &UVs[(uvIndex[1] - 1) * 2];
out_UVs.insert(out_UVs.end(), uv, uv + 2);
uv = &UVs[(uvIndex[2] - 1) * 2];
out_UVs.insert(out_UVs.end(), uv, uv + 2);
GLfloat* normal = &normals[(normalIndex[0] - 1) * 3];
out_normals.insert(out_normals.end(), normal, normal + 3);
normal = &normals[(normalIndex[1] - 1) * 3];
out_normals.insert(out_normals.end(), normal, normal + 3);
normal = &normals[(normalIndex[2] - 1) * 3];
out_normals.insert(out_normals.end(), normal, normal + 3);
out_indices.push_back(size);
out_indices.push_back(size + 1);
out_indices.push_back(size + 2);*/
}
}
Upvotes: 0
Views: 472
Answers (3)
Reputation: 2972
This how I did to optimize this code...
Environment : MacBook Pro, macos, Xcode for IDE. Built-in Instruments for profiling. Target File: downloaded cadnav-200615151218.rar from link you provided. Extracted APC.obj(~17M) and work on it.
Step 1 - reproduce the problem Add missing bits to the code:
typedef float GLfloat;
void reader()
{
std::ifstream file("/Users/mlifshits/Downloads/APC.obj");
Comment out unsigned int si_ze = out_vertices.size() / 3; since it is not used anywhere and the type of out_vertices unknown.
Profile:
took 537ms, time hogs: operator>>(&float), getline, scanf
Step 2 - attack first time hog which is >>(&float)
Use sscanf() in place of iss
Profile:
took 371ms (improvement!), hogs: scanf and getline
Steps 3 and 4 - attack the hogs!
looked up online about scanf and found that it's slow in reading floats. Replaced with "naive loop" from https://tinodidriksen.com/2011/05/cpp-convert-string-to-double-speed/
since getline hog is so hungry because of multiple allocations (seen in profiler), replaced it with C code using fopen() and fgets()
Profile: took 166ms (vs 546 initially!), hogs still scanf and fgets.
Optimize more? - possible very much. I stop here.
Code: #include #include #include #include #include #include
typedef float GLfloat;
//https://tinodidriksen.com/uploads/code/cpp/speed-string-to-double.cpp
float naive_atof(const char *p) {
float r = 0.0;
bool neg = false;
if (*p == '-') {
neg = true;
++p;
}
while (*p >= '0' && *p <= '9') {
r = (r*10.0) + (*p - '0');
++p;
}
if (*p == '.') {
float f = 0.0;
int n = 0;
++p;
while (*p >= '0' && *p <= '9') {
f = (f*10.0) + (*p - '0');
++p;
++n;
}
r += f / std::pow(10.0, n);
}
if (neg) {
r = -r;
}
return r;
}
void reader()
{
char line[256] = {0};
std::string type;
char xyz[3][100] = {0};
FILE*fp = fopen("/Users/limar/Downloads/APC.obj", "r");
while (fgets(line, sizeof(line), fp)) {
if (line[0] == 'v') {
GLfloat x, y, z;
sscanf(line,"%s %s %s %s", (char*)type.c_str(), xyz[0], xyz[1], xyz[2]);
x = naive_atof(xyz[0]);
y = naive_atof(xyz[1]);
z = naive_atof(xyz[2]);
if (type.compare("v") == 0) {
//vertices.push_back(x);
//vertices.push_back(y);
//vertices.push_back(z);
}
else if (type.compare("vn") == 0) {
//normals.push_back(x);
//normals.push_back(y);
//normals.push_back(z);
}
else if (type.compare("vt") == 0) {
//UVs.push_back(x);
//UVs.push_back(y);
}
}
else if (line[0] == 'f') {
//unsigned int size = out_vertices.size() / 3;
unsigned int vertexIndex[3], uvIndex[3], normalIndex[3];
sscanf(line,
"f %d/%d/%d %d/%d/%d %d/%d/%d",
&vertexIndex[0], &uvIndex[0], &normalIndex[0],
&vertexIndex[1], &uvIndex[1], &normalIndex[1],
&vertexIndex[2], &uvIndex[2], &normalIndex[2]);
/*GLfloat* vertex = &vertices[(vertexIndex[0] - 1) * 3];
out_vertices.insert(out_vertices.end(), vertex, vertex + 3);
vertex = &vertices[(vertexIndex[1] - 1) * 3];
out_vertices.insert(out_vertices.end(), vertex, vertex + 3);
vertex = &vertices[(vertexIndex[2] - 1) * 3];
out_vertices.insert(out_vertices.end(), vertex, vertex + 3);
GLfloat* uv = &UVs[(uvIndex[0] - 1) * 2];
out_UVs.insert(out_UVs.end(), uv, uv + 2);
uv = &UVs[(uvIndex[1] - 1) * 2];
out_UVs.insert(out_UVs.end(), uv, uv + 2);
uv = &UVs[(uvIndex[2] - 1) * 2];
out_UVs.insert(out_UVs.end(), uv, uv + 2);
GLfloat* normal = &normals[(normalIndex[0] - 1) * 3];
out_normals.insert(out_normals.end(), normal, normal + 3);
normal = &normals[(normalIndex[1] - 1) * 3];
out_normals.insert(out_normals.end(), normal, normal + 3);
normal = &normals[(normalIndex[2] - 1) * 3];
out_normals.insert(out_normals.end(), normal, normal + 3);
out_indices.push_back(size);
out_indices.push_back(size + 1);
out_indices.push_back(size + 2);*/
}
}
}
int main(int argc, const char * argv[]) {
reader();
return 0;
}
Upvotes: 0
Reputation: 15265
First, and most important. Rebuild your software in "Release"-mode, not in "Debug"-mode. Or, switch on optimization options on manually.
Then, after opening the file, add the following statements, after the stream is open:
constexpr size_t SizeOfIOStreamBuffer = 100'000;
static char ioBuffer[SizeOfIOStreamBuffer];
file.rdbuf()->pubsetbuf(ioBuffer, SizeOfIOStreamBuffer);
Then, for your std::vectors, please call their reserve(50000U) function.
I expect a execution time to be 1 second. (On my machine with 50MB file it was)
Please try and feed back.
Upvotes: 1
Reputation: 178
Splitting the file into smaller chunks and reading from those chunks might be able to speed up your processing.
Here is what I think might be able to help you:
Similar topic on Stackoverflow
Thanks
Upvotes: 1
Related Questions
- How to read extremely long lines from a text file fast and safe in C++?
- Is there any way to read from files faster?
- Fastest way to read in a file c++
- read huge text file line by line in C++ with buffering
- Improving C++'s reading file line by line?
- Reading text file line by line in c++
- C++ - efficiently reading a file sequentially
- C++ Best way to quickly read from larger file
- Read File line by line using C++
- very fast text file processing (C++)