Reputation: 89
When I'm running my application and load a file(25MB) through it, everything runs just fine.
But when I try loading a file(160MB) I get a System.OutOfMemoryExeption
.
Although I have been able to load the larger file at some point in time.
Is there anyway to fix this? If so, any help would be much appreciated!
My Code that loads the files:
private void openFile (string fileName)
{
List<Structs.strValidData> _header1 = new List<Structs.strValidData>();
List<Structs.strValidData> _header2 = new List<Structs.strValidData>();
List<Structs.strValidData> _header3 = new List<Structs.strValidData>();
List<Structs.strValidData> _header4 = new List<Structs.strValidData>();
var textBoxArray = new[]
{
textBoxResStart_Status1,
textBoxResStart_Status2,
textBoxResStart_Status3,
textBoxResStart_Status4,
textBoxResStart_Status5,
textBoxResStart_Status6,
textBoxResStart_Status7,
textBoxResStart_Status8,
};
var radioButtonArray = new[]
{
radioButtonResStart_SelectStr1,
radioButtonResStart_SelectStr2,
radioButtonResStart_SelectStr3,
radioButtonResStart_SelectStr4,
radioButtonResStart_SelectStr5,
radioButtonResStart_SelectStr6,
radioButtonResStart_SelectStr7,
radioButtonResStart_SelectStr8,
};
readCSV read;
read = new readCSV();
strInfo = default(Structs.strInfo);
strData = default(Structs.strData);
strSetup = default(Structs.strSetup);
strValidData = new List<Structs.strValidData>();
readID = default(Structs.ReadID);
try
{
strInfo = read.loadInfo(fileName);
strData = read.loadData(fileName);
strSetup = read.loadSetup(fileName);
readID = read.loadID(fileName);
strValidData = read.loadValidData(fileName);
var Str1 = read.loadStr1(fileName);
var Str235678 = read.loadStr235678(fileName);
var Str4 = read.loadStr4(fileName);
foreach (Structs.strValidData items in strValidData)
{
if (items.Str1_ValidData == true)
{
Str1_headers.Add(items);
}
if (items.Str2_ValidData == true ||
items.Str3_ValidData == true ||
items.Str5_ValidData == true ||
items.Str6_ValidData == true ||
items.Str7_ValidData == true ||
items.Str8_ValidData == true)
{
Str235678_headers.Add(items);
}
if (items.Str4_ValidData == true)
{
Str4_headers.Add(items);
}
}
Str1_data = combineData(Str1, Str1_headers);
Str4_data = combineData(Str4, Str4_headers);
var Str235678_CombinedData = combineData(Str235678, Str235678_headers);
foreach (Structs.strValidData items in Str235678_CombinedData)
{
if (items.Str2_ValidData == true)
{
Str2_data.Add(items);
}
if (items.Str3_ValidData == true)
{
Str3_data.Add(items);
}
if (items.Str5_ValidData == true)
{
Str5_data.Add(items);
}
if (items.Str6_ValidData == true)
{
Str6_data.Add(items);
}
if (items.Str7_ValidData == true)
{
Str7_data.Add(items);
}
if (items.Str8_ValidData == true)
{
Str8_data.Add(items);
}
}
strInfo = read.loadInfo(openDialog.FileName);
strData = read.loadData(openDialog.FileName);
strSetup = read.loadSetup(openDialog.FileName);
readID = read.loadID(openDialog.FileName);
}
catch (Exception err)
{
MessageBox.Show(err.Message);
error.logSystemError(err);
}
}
Here are the ReadCSV()
code:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using FileHelpers;
using FileHelpers.Events;
namespace Reading_Files
{
public class readCSV
{
public int strCnt = 0;
private readCSVprogressForm _waitForm;
public List<Structs.strDataImport> copyList(List<strData> copyFrom)
{
List<Structs.strDataImport> list = new List<Structs.strDataImport>();
list.AddRange(copyFrom.Select(s => copyListContents(s)));
return list;
}
public Structs.strDataImport copyListContents(strData copyFrom)
{
Structs.strDataImport data = new Structs.strDataImport();
data.sCD_TimeCP2711 = copyFrom.sCD_TimeCP2711;
data.sCD_TimeCX9020_1 = copyFrom.sCD_TimeCX9020_1;
data.sCD_TimeCX9020_2 = copyFrom.sCD_TimeCX9020_2;
data.rCD_CX9020_1_TimeDiff_DataLow = (Int32)(copyFrom.rCD_CX9020_1_TimeDiff_DataLow);
data.rCD_CX9020_2_TimeDiff_DataLow = (Int32)(copyFrom.rCD_CX9020_2_TimeDiff_DataLow);
data.iCD_NumUpper = copyFrom.iCD_NumUpper;
data.iCD_NumUpper = copyFrom.iCD_NumUpper;
data.iCD_NumLower = copyFrom.iCD_NumLower;
data.iCD_NumLower = copyFrom.iCD_NumLower;
data.bCD_1_Status = copyFrom.bCD_1_Status;
data.bCD_1_Overrange = copyFrom.bCD_1_Overrange;
data.iCD_1_Str_ID = copyFrom.iCD_1_Str_ID;
data.rCD_1_Value = copyFrom.rCD_1_Value;
data.bCD_2_Status = copyFrom.bCD_2_Status;
data.bCD_2_Overrange = copyFrom.bCD_2_Overrange;
data.iCD_2_Str_ID = copyFrom.iCD_2_Str_ID;
data.rCD_2_Value = copyFrom.rCD_2_Value;
data.bCD_3_Status = copyFrom.bCD_3_Status;
data.bCD_3_Overrange = copyFrom.bCD_3_Overrange;
data.iCD_3_Str_ID = copyFrom.iCD_3_Str_ID;
data.iCD_3_RawData = copyFrom.iCD_3_RawData;
data.rCD_3_Value = copyFrom.rCD_3_Value;
data.bCD_4_Status = copyFrom.bCD_4_Status;
data.bCD_4_Overrange = copyFrom.bCD_4_Overrange;
data.iCD_4_Str_ID = copyFrom.iCD_4_Str_ID;
data.iCD_4_RawData = copyFrom.iCD_4_RawData;
data.rCD_4_Value = copyFrom.rCD_4_Value;
data.bCD_5_Status = copyFrom.bCD_5_Status;
data.bCD_5_Overrange = copyFrom.bCD_5_Overrange;
data.iCD_5_Str_ID = copyFrom.iCD_5_Str_ID;
data.iCD_5_RawData = copyFrom.iCD_5_RawData;
data.rCD_5_Value = copyFrom.rCD_5_Value;
data.bCD_6_Status = copyFrom.bCD_6_Status;
data.bCD_6_Overrange = copyFrom.bCD_6_Overrange;
data.iCD_6_Str_ID = copyFrom.iCD_6_Str_ID;
data.iCD_6_RawData = copyFrom.iCD_6_RawData;
data.rCD_6_Value = copyFrom.rCD_6_Value;
data.bCD_7_Status = copyFrom.bCD_7_Status;
data.bCD_7_Overrange = copyFrom.bCD_7_Overrange;
data.iCD_7_Str_ID = copyFrom.iCD_7_Str_ID;
data.iCD_7_RawData = copyFrom.iCD_7_RawData;
data.rCD_7_Value = copyFrom.rCD_7_Value;
data.bCD_8_Status = copyFrom.bCD_8_Status;
data.bCD_8_Overrange = copyFrom.bCD_8_Overrange;
data.iCD_8_Str_ID = copyFrom.iCD_8_Str_ID;
data.iCD_8_RawData = copyFrom.iCD_8_RawData;
data.rCD_8_Value = copyFrom.rCD_8_Value;
data.bCD_9_Status = copyFrom.bCD_9_Status;
data.bCD_9_Overrange = copyFrom.bCD_9_Overrange;
data.iCD_9_Str_ID = copyFrom.iCD_9_Str_ID;
data.iCD_9_RawData = copyFrom.iCD_9_RawData;
data.rCD_9_Value = copyFrom.rCD_9_Value;
data.bCD_10_Status = copyFrom.bCD_10_Status;
data.bCD_10_Overrange = copyFrom.bCD_10_Overrange;
data.iCD_10_Str_ID = copyFrom.iCD_10_Str_ID;
data.iCD_10_RawData = copyFrom.iCD_10_RawData;
data.rCD_10_Value = copyFrom.rCD_10_Value;
data.bCD_11_Status = copyFrom.bCD_11_Status;
data.bCD_11_Overrange = copyFrom.bCD_11_Overrange;
data.iCD_11_Str_ID = copyFrom.iCD_11_Str_ID;
data.iCD_11_RawData = copyFrom.iCD_11_RawData;
data.rCD_11_Value = copyFrom.rCD_11_Value;
data.bCD_12_Status = copyFrom.bCD_12_Status;
data.bCD_12_Overrange = copyFrom.bCD_12_Overrange;
data.iCD_12_Str_ID = copyFrom.iCD_12_Str_ID;
data.iCD_12_RawData = copyFrom.iCD_12_RawData;
data.rCD_12_Value = copyFrom.rCD_12_Value;
data.bCD_13_Status = copyFrom.bCD_13_Status;
data.bCD_13_Overrange = copyFrom.bCD_13_Overrange;
data.iCD_13_Str_ID = copyFrom.iCD_13_Str_ID;
data.iCD_13_RawData = copyFrom.iCD_13_RawData;
data.rCD_13_Value = copyFrom.rCD_13_Value;
data.bCD_14_Status = copyFrom.bCD_14_Status;
data.bCD_14_Overrange = copyFrom.bCD_14_Overrange;
data.iCD_14_Str_ID = copyFrom.iCD_14_Str_ID;
data.iCD_14_RawData = copyFrom.iCD_14_RawData;
data.rCD_14_Value = copyFrom.rCD_14_Value;
data.bCD_15_Status = copyFrom.bCD_15_Status;
data.bCD_15_Overrange = copyFrom.bCD_15_Overrange;
data.iCD_15_Str_ID = copyFrom.iCD_15_Str_ID;
data.iCD_15_RawData = copyFrom.iCD_15_RawData;
data.rCD_15_Value = copyFrom.rCD_15_Value;
data.bCD_16_Status = copyFrom.bCD_16_Status;
data.bCD_16_Overrange = copyFrom.bCD_16_Overrange;
data.iCD_16_Str_ID = copyFrom.iCD_16_Str_ID;
data.iCD_16_RawData = copyFrom.iCD_16_RawData;
data.rCD_16_Value = copyFrom.rCD_16_Value;
data.bCD_17_Status = copyFrom.bCD_17_Status;
data.bCD_17_Overrange = copyFrom.bCD_17_Overrange;
data.iCD_17_Str_ID = copyFrom.iCD_17_Str_ID;
data.iCD_17_RawData = copyFrom.iCD_17_RawData;
data.rCD_17_Value = copyFrom.rCD_17_Value;
data.bCD_18_Status = copyFrom.bCD_18_Status;
data.bCD_18_Overrange = copyFrom.bCD_18_Overrange;
data.iCD_18_Str_ID = copyFrom.iCD_18_Str_ID;
data.iCD_18_RawData = copyFrom.iCD_18_RawData;
data.rCD_18_Value = copyFrom.rCD_18_Value;
data.bCD_19_Status = copyFrom.bCD_19_Status;
data.bCD_19_Overrange = copyFrom.bCD_19_Overrange;
data.iCD_19_Str_ID = copyFrom.iCD_19_Str_ID;
data.rCD_19_Value = copyFrom.rCD_19_Value;
data.bCD_20_Status = copyFrom.bCD_20_Status;
data.bCD_20_Overrange = copyFrom.bCD_20_Overrange;
data.iCD_20_Str_ID = copyFrom.iCD_20_Str_ID;
data.rCD_20_Value = copyFrom.rCD_20_Value;
return data;
}
public Structs.ReaStrID load_ID(string FileName)
{
var engine = new MultiRecordEngine(typeof(strInfo),
typeof(strData),
typeof(strSetup),
typeof(strID),
typeof(strData));
engine.RecordSelector = new RecordTypeSelector(strIDSelector);
var data = engine.ReadFile(FileName);
Structs.ReaStrID structure = new Structs.ReaStrID();
foreach (strID filteredData in data)
{
structure.steID[0] = filteredData._1_Ste_ID;
structure.Status[0] = filteredData._1_Status;
structure.steID[1] = filteredData._2_Ste_ID;
structure.Status[1] = filteredData._2_Status;
structure.steID[2] = filteredData._3_Ste_ID;
structure.Status[2] = filteredData._3_Status;
structure.steID[3] = filteredData._4_Ste_ID;
structure.Status[3] = filteredData._4_Status;
structure.steID[4] = filteredData._5_Ste_ID;
structure.Status[4] = filteredData._5_Status;
}
return structure;
}
public Structs.strInfo loadInfo(string FileName)
{
var engine = new MultiRecordEngine(typeof(strInfo),
typeof(strData),
typeof(strSetup),
typeof(strID),
typeof(strData));
engine.RecordSelector = new RecordTypeSelector(strInfoSelector);
var data = engine.ReadFile(FileName);
Structs.strInfo structure = new Structs.strInfo();
foreach (strInfo filteredData in data)
{
structure.Date = filteredData.Date;
structure.Description1 = filteredData.Description1;
structure.Description2 = filteredData.Description2;
structure.Description3 = filteredData.Description3;
}
return structure;
}
public Structs.strData loadData(string FileName)
{
var engine = new MultiRecordEngine(typeof(strInfo),
typeof(strData),
typeof(strSetup),
typeof(strID),
typeof(strData));
engine.RecordSelector = new RecordTypeSelector(strDataSelector);
var data = engine.ReadFile(FileName);
Structs.strData structure = new Structs.strData();
foreach (strData filteredData in data)
{
structure.iMDstr_var1_TypeID = filteredData.iMDstr_var1_TypeID;
structure.rMDstr_var1_Lenght = filteredData.rMDstr_var1_Lenght;
structure.iMDstr_var2_TypeID = filteredData.iMDstr_var2_TypeID;
structure.rMDstr_var2_Lenght = filteredData.rMDstr_var2_Lenght;
structure.iMDstr_var3_TypeID = filteredData.iMDstr_var3_TypeID;
structure.rMDstr_var3_Lenght = filteredData.rMDstr_var3_Lenght;
structure.iMDstr_var4_TypeID = filteredData.iMDstr_var4_TypeID;
structure.rMDstr_var4_Lenght = filteredData.rMDstr_var4_Lenght;
}
return structure;
}
public Structs.strSetup loadSetup(string FileName)
{
var engine = new MultiRecordEngine(typeof(strInfo),
typeof(strID),
typeof(strData),
typeof(strSetup),
typeof(strData));
engine.RecordSelector = new RecordTypeSelector(strSetupSelector);
var data = engine.ReadFile(FileName);
Structs.strSetup structure = new Structs.strSetup();
foreach (strSetup filteredData in data)
{
structure.sSSstr_Sens = filteredData.sSSstr_Sens;
structure.bSSstr_S1_A = filteredData.bSSstr_S1_A;
structure.iSSstr_S1_B = filteredData.iSSstr_S1_B;
structure.sSSstr_S1_C = filteredData.sSSstr_S1_C;
structure.rSSstr_S1_D = filteredData.rSSstr_S1_D;
structure.bSSstr_S2_A = filteredData.bSSstr_S2_A;
structure.iSSstr_S2_B = filteredData.iSSstr_S2_B;
structure.sSSstr_S2_C = filteredData.sSSstr_S2_C;
structure.rSSstr_S2_D = filteredData.rSSstr_S2_D;
structure.bSSstr_S3_A = filteredData.bSSstr_S3_A;
structure.iSSstr_S3_B = filteredData.iSSstr_S3_B;
structure.sSSstr_S3_C = filteredData.sSSstr_S3_C;
structure.iSSstr_S3_D = filteredData.iSSstr_S3_D;
}
return structure;
}
public List<Structs.str1> load1(string FileName)
{
var engine = new MultiRecordEngine(typeof(strInfo),
typeof(strData),
typeof(strSetup),
typeof(strData),
typeof(strID),
typeof(strValidData),
typeof(strStartNum),
typeof(str1),
typeof(str4));
engine.RecordSelector = new RecordTypeSelector(str1Selector);
var data = engine.ReadFile(FileName);
List<Structs.str1> list = new List<Structs.str1>();
int i = 0;
foreach (str1 data1 in data)
{
Structs.str1 structure = new Structs.str1();
structure.rGL_1_L_Positive = data1.rGL_1_L_Positive;
structure.rGL_1_L_Negative = data1.rGL_1_L_Negative;
structure.rGL_1_R_Positive = data1.rGL_1_R_Positive;
structure.rGL_1_R_Negative = data1.rGL_1_R_Negative;
list.Add(structure);
i++;
}
return list;
}
public List<Structs.str4> load4(string FileName)
{
var engine = new MultiRecordEngine(typeof(strInfo),
typeof(strData),
typeof(strSetup),
typeof(strData),
typeof(strValidData),
typeof(strStartNum),
typeof(str1),
typeof(str4));
engine.RecordSelector = new RecordTypeSelector(str4Selector);
var data = engine.ReadFile(FileName);
List<Structs.str4> list = new List<Structs.str4>();
int i = 0;
foreach (str4 data4 in data)
{
Structs.str4 structure = new Structs.str4();
structure.rGL_4_1 = data4.rGL_4_1;
structure.rGL_4_2 = data4.rGL_4_2;
structure.rGL_4_3 = data4.rGL_4_3;
structure.rGL_4_4 = data4.rGL_4_4;
structure.rGL_4_5 = data4.rGL_4_5;
structure.rGL_4_6 = data4.rGL_4_6;
structure.rGL_4_7 = data4.rGL_4_7;
structure.rGL_4_8 = data4.rGL_4_8;
list.Add(structure);
i++;
}
return list;
}
public List<Structs.strValidData> loadValidData(string FileName)
{
var engine = new MultiRecordEngine(typeof(strInfo),
typeof(strData),
typeof(strSetup),
typeof(strID),
typeof(strData),
typeof(strValidData));
engine.RecordSelector = new RecordTypeSelector(strValidDataSelector);
var data = engine.ReadFile(FileName);
List<Structs.strValidData> list = new List<Structs.strValidData>();
int i = 0;
foreach (strValidData strValidData in data)
{
Structs.strValidData structure = new Structs.strValidData();
structure._name = String.Format("strItem {0}", i + 1);
structure._index = i;
structure.str1_ValidData = strValidData.str1_ValidData;
structure.str2_ValidData = strValidData.str2_ValidData;
structure.str3_ValidData = strValidData.str3_ValidData;
structure.str4_ValidData = strValidData.str4_ValidData;
structure.str5_ValidData = strValidData.str5_ValidData;
structure.str6_ValidData = strValidData.str6_ValidData;
structure.str7_ValidData = strValidData.str7_ValidData;
structure.str8_ValidData = strValidData.str8_ValidData;
structure.str9_ValidData = strValidData.str9_ValidData;
list.Add(structure);
i++;
}
return list;
}
public List<List<Structs.strDataImport>> loadstrDataAsync(string FileName)
{
var engine_Data = new FileHelperAsyncEngine<strData>();
engine_Data.BeforeReadRecord += BeforeEventAsync;
engine_Data.AfterReadRecord += AfterEventAsync;
engine_Data.Progress += ReadProgress;
List<strData> list = new List<strData>();
List<List<Structs.strDataImport>> list2D = new List<List<Structs.strDataImport>>();
using (engine_Data.BeginReadFile(FileName))
{
var prevRowNo = 0;
var j = 0;
strCnt = 0;
foreach (strData filteredData in engine_Data)
{
if (prevRowNo > filteredData.RowNo)
{
list2D.Add(copyList(list));
list.Clear();
}
prevRowNo = filteredData.RowNo;
list.Add(filteredData);
}
list2D.Add(copyList(list));
}
return list2D;
}
private Type strIDSelector(MultiRecordEngine engine, string recordLine)
{
if (recordLine.Length == 0)
return null;
if (recordLine.Contains("** #_DATA .STATUS .STRID **"))
return typeof(strID);
else
{
return null;
}
}
private Type InfoSelector(MultiRecordEngine engine, string recordLine)
{
if (recordLine.Length == 0)
return null;
if (recordLine.Contains("** #_FILE **"))
return typeof(strInfo);
else
{
return null;
}
}
private Type strDataSelector(MultiRecordEngine engine, string recordLine)
{
if (recordLine.Length == 0)
return null;
if (recordLine.Contains("** #_DATA **"))
return typeof(strData);
else
{
return null;
}
}
private Type strSetupSelector(MultiRecordEngine engine, string recordLine)
{
if (recordLine.Length == 0)
return null;
if (recordLine.Contains("** #_SETUP **"))
return typeof(strSetup);
else
{
return null;
}
}
private Type strValidDataSelector(MultiRecordEngine engine, string recordLine)
{
if (recordLine.Length == 0)
return null;
if (recordLine.Contains("** #_VALID_DATA **"))
return typeof(strValidData);
else
{
return null;
}
}
private Type StartNumSelector(MultiRecordEngine engine, string recordLine)
{
if (recordLine.Length == 0)
return null;
if (recordLine.Contains("** #_START_NUMBER **"))
return typeof(strStartNum);
else
{
return null;
}
}
private Type str1Selector(MultiRecordEngine engine, string recordLine)
{
if (recordLine.Length == 0)
return null;
if (recordLine.Contains("** #_1 **"))
return typeof(str1);
else
{
return null;
}
}
private Type str4Selector(MultiRecordEngine engine, string recordLine)
{
if (recordLine.Length == 0)
return null;
if (recordLine.Contains("** #_4 **"))
return typeof(str4);
else
{
return null;
}
}
private void BeforeEventAsync(EngineBase engine, BeforeReadEventArgs<strData> e)
{
if (e.RecordLine != "")
{
if (Char.IsDigit(e.RecordLine, 0))
{
}
else
{
e.SkipThisRecord = true;
}
}
if (e.RecordLine.Contains("** #_VALID_DATA **;"))
{
e.SkipThisRecord = true;
}
}
private void AfterEventAsync(EngineBase engine, AfterReadEventArgs<strData> e)
{
if (e.RecordLine.Contains("** #_VALID_DATA **;"))
{
e.SkipThisRecord = true;
}
}
private void ReadProgress(object sender, ProgressEventArgs e)
{
ShowWaitForm("Opening file." + "\n" + "\n" + "Please wait...", "Open File");
_waitForm.progressBar1.Value = Convert.ToInt16(e.Percent);
}
public void ShowWaitForm(string message, string caption)
{
if (_waitForm != null && !_waitForm.IsDisposed)
{
return;
}
_waitForm = new readCSVprogressForm();
_waitForm.ShowMessage(message);
_waitForm.Text = caption;
_waitForm.TopMost = true;
_waitForm.Show();
_waitForm.Refresh();
System.Threading.Thread.Sleep(700);
Application.Idle += OnLoaded;
}
private void OnLoaded(object sender, EventArgs e)
{
Application.Idle -= OnLoaded;
_waitForm.Close();
}
}
}
Upvotes: 0
Views: 177
Reputation: 1499770
Given the comments, it sounds like the problem is really that you're using structs extensively. These should almost certainly be classes for idiomatic C#. See the design guidelines for more details of how to pick between these.
At the moment, you're loading all the values in a big List<T>
. Internally, that has an array - so it's going to be an array of your struct type. That means all the values are required to be in a single contiguous chunk of memory - and it sounds like that chunk can't be allocated.
If you change your data types to classes, then a contiguous chunk of memory will still be required - but only enough to store references to the objects you create. You'll end up using slightly more data overall (due to per-object overhead and the references to those objects) but you won't have nearly as strict a requirement on allocating a single big chunk of memory.
That's only one reason to use classes here - the reason of "this just isn't a normal use of structs" is a far bigger one, IMO.
As an aside, I'd also very strongly recommend that you start following .NET naming conventions, particularly around the use of capitalization and avoiding underscores to separate words in names. (There are other suggestions for improving the code in the question too, and I'd advise reading them all carefully.)
Upvotes: 1