Losing random characters with istringstream

Question

I am reading lines of a text file into a variable, myline and then I am trying to tokenize these lines using istringstream. However, it seems that I am losing random characters from original text file string.

cout<< myline << buff << flush; //print original text file line
istringstream iss(myline);
string sub;
while (iss >> sub) {
cout << "[" << sub << "]" << endl;
} 

If you look at my output, you can see that I have the correct string from the text file, but when I use istringstream and then print the individual tokens (seen within [] brackets), some of the tokens are prematurely truncated.

#include <iostream>
[#include]
[<iostream]
#include <sstream>
[#include]
[<sstream>]
using namespace std;
[using]
[namespace]
[st]

int main()
[int]
[main(]
{
    string str("   SOME  LONG    STRING\twith\nSPACES    ");
[string]
[str("]
[SOME]
[LONG]
[STRING\twith\nSPACES]

    istringstream iss(str);
[istringstream]
[iss(str);]

    string s;
[strin]
    while (iss >> s) {
[while]
[(iss]
[>>]
        cout << "[" << s << "]" << endl;
[cout]
[<<]
["["]
[<<]
[s]
[<<]
["]"]
[<<]
[e]
    }
    return 0;
[retur]
}

Does anybody have any ideas what I'm doing wrong? Thanks in advance!

EDIT: Here's a version of the code that will compile fully. You can run it with any text file

#include <cstring>
#include <cstdio>
#include <iostream>
#include <iomanip>
#include <string>
#include <sstream>
#include <vector>
#include <algorithm>
#include <iterator>

using namespace std;

class MyFileReader {

public:
    //constructor
    MyFileReader(const char* p);

    //destructor
    ~MyFileReader();

    //getLine()
    int getLine(char *buffer, int size);

    //getCurrentLineNumber()
    int getCurrentLineNumber();

    void tokenizeLine(vector<string>& vec);

    FILE * pFile;

};

    //constructor
    MyFileReader::MyFileReader(const char* p) {
        pFile = fopen(p, "r");
    }

    //destructor
    MyFileReader::~MyFileReader() {
        fclose(pFile);
    }

    //getLine()
    int MyFileReader::getLine(char *buffer, int size){
        char *out = fgets(buffer, size, pFile);
        if (out==NULL) {
            return -1;
        }
        char *pch = strpbrk(out,"\n");
        if (pch != NULL) {
            return 1;
        }
        else {
            return 0;
        }

    }

    int MyFileReader::getCurrentLineNumber() {
        static int mynumber=2;
        return mynumber++;
    }

    //tokenizeLine
    void MyFileReader::tokenizeLine(vector<string>& vec) {
        string myline("");
        char buff[10];
        while (1) {
            int result = getLine(buff, sizeof(buff));
            if (result == -1 ) {
                if (myline.length() > 0) 
                    cout << myline << flush;
            break;
            }
            else if (result == 0) {
                myline += buff;
            }
            else if (result == 1) {
                cout<< myline << buff << flush;
                istringstream iss(myline);
                string sub;
                while (iss >> sub) {
                    cout << "[" << sub << "]" << endl;
                } 
                myline = "";
            }
            else {
                printf("PANIC");
            }
            }
            return;
        }

    int main(int argc, char **argv) {
    vector<string> v;

    const char *filename = argv[1];
    MyFileReader f(filename);
    f.tokenizeLine(v);
    return 0;

    }

To generate the output above, I ran it on:

#include <iostream>
#include <sstream>
using namespace std;

int main()
{
    string str("   SOME  LONG    STRING\twith\nSPACES    ");

    istringstream iss(str);

    string s;
    while (iss >> s) {
        cout << "[" << s << "]" << endl;
    }
    return 0;
}

Answer 1

You line buffer is only 10 bytes long. That's not long enough to fit a full line.

void MyFileReader::tokenizeLine(vector<string>& vec) {
        string myline("");
        char buff[10];// this is too short
....

EDIT

As Dyp correctly points out your appending logic is not correct when you detect a \n in the input file.

Answer 2

The mistake is here:

else if (result == 1) {
            cout<< myline << buff << flush;
            istringstream iss(myline);
            string sub;
            while (iss >> sub) {
                cout << "[" << sub << "]" << endl;
            } 
            myline = "";
        }

If result == 1, that means that buff contains \n, it doesn't mean it only contains \n. I.e. you drop the buffer if it contains \n. If therefore the line happens to have n*10 (sizeof buffer) characters, your code works, else, the last characters of a line are not copied to myline but dropped.

The quick fix would be to:

    else if (result == 1) {
            myline += buff; // copy the rest of the line into `myline`
            cout<< myline << flush; // buff now is part of myline
            istringstream iss(myline);
            string sub;
            while (iss >> sub) {
                cout << "[" << sub << "]" << endl;
            } 
            myline = "";
        }

Though you might want to consider dropping the \n from the buffer, like:

int MyFileReader::getLine(char *buffer, int size){
    char *out = fgets(buffer, size, pFile);
    if (out==NULL) {
        return -1;
    }
    //char *pch = strpbrk(out,"\n");
    char *pch = strchr(out,'\n'); // no need to search for a string
    if (pch != NULL) {
        *pch = '\0'; // drop the '\n'
        return 1;
    }
    else {
        return 0;
    }

}

You'll have to change cout<< myline << flush; to cout<< myline << endl;, though.

---

Besides this mistake, please consider using ifstream:

#include <iostream>
#include <fstream>
#include <sstream>
using namespace std;

int main()
{
    ifstream file("test.txt");
    if(!file)
    {
        /* error */
    }else
    {
        string line;
        while(getline(file, line))
        {
            istringstream iss(line);

            string s;
            while (iss >> s) {
                cout << "[" << s << "]" << endl;
            }
        }
    }
}