MQL5 array in for loop takes too long, any better way?

Question

I'm writing a EA for strategy tester purpose only, to evaluate portfolio. So I have hundreads/thousands of trades in an array, with open price and open time, and every 1 minute bar I check in a for loop if there's any trade to open in that minute. And it's taking forever.

How can I check (in a faster way) if in an array there's a open-time that coincides with current time?

Thanks!

if(NewBar)
 {
  CopyRates(_Symbol,PERIOD_M1,0,5,candle);
  sizeAr=ArraySize(ar);
  arColumns=ArrayRange(ar,1);
  datetime candleNowTime = candle[0].time;
  datetime nextCandleTime = candle[0].time+60;      

  for(int i=0;i<sizeAr/arColumns;i++)
    {
     if(ar[i][openTime]>candleNowTime && ar[i][openTime]<nextCandleTime)
       {
        //code to open trades
       }
    }
 }

And the complete code is here:

//+------------------------------------------------------------------+
//|                                          HedgeExperienceV001.mq5 |
//|                                                      Carlos Duna |
//|                                             https://www.mql5.com |
//+------------------------------------------------------------------+
#property copyright "Carlos Duna"
#property link      "https://www.mql5.com"
#property version   "1.00"
#property tester_file "sinais.csv"
#include <Trade\Trade.mqh>
CTrade trade;
#include <Trade\SymbolInfo.mqh>
CSymbolInfo mysymbol;
#define columns 19

input double initialVolume = 0.01;  // Volume inicial
input double slFixo = 0;            // SL fixo (pips)(zero=SL do sinal)
input double tpFixo = 0;            // TP fixo (pips)(zero=TP do sinal)
input ulong  maxSlippage = 0;       // Max Deviation/Slippage(0-não usa)
input double maxPricesVariation =10;// Max % variação Bid x TP x SL x OP

MqlRates candle[];
MqlDateTime TimeStruct;
MqlDateTime dealDay;
datetime dealDayStruct;
datetime now;
int secAnterior;
int previousDay;

bool NewBar;
datetime hj;
string orderComment;
bool orderOkToOpen;

datetime inicioHedge=0;
double profitPerMagic[][2];
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
struct SLine
  {
   string            field[];
  };

SLine lines[];

string ar[][columns];
int sizeLine;
int sizeAr;
int arColumns;

double stopLossDelta;
double takeProfitDelta;

/*
lines[i].field[j]

** i= numero de cada trade, iniciando no zero (não há títulos de colunas), indo dinamicamente até a ultima linha
** j= valores dos campos

0 - tCanal
1 - str(msgId)
2 - dataHora HoraEntrada
3 - str(opType) CompraOuVenda
4 - ativo.upper()
5 - str(openPrice)
6 - str(stopLoss)
7 - str(takeProfit1)
8 - str(takeProfit2)
9 - str(takeProfit3)
10 - str(takeProfit4)
11 - str(takeProfit5)
12 - 'false' posição já foi aberta? true/false
13 - 'false' posição já foi fechada? true/false
14 - HalfCloseTime
15 - ManualCloseTime
16 - SLModify_Time
17 - SLModify_Price])
18 - magicCanal
*/

int xCanal = 0; // tCanal
int xOpCod = 1; // str(msgId)
int xDtEnt = 2; // dataHora HoraEntrada
int xBuySell=3; // str(opType) CompraOuVenda
int xAtv= 4; // ativo.upper()
int xOP = 5; // str(openPrice)
int xSL=6; // str(stopLoss)
int xTP1 = 7; // str(takeProfit1)
int xTP2 = 8; // str(takeProfit2)
int xTP3 = 9; // str(takeProfit3)
int xTP4 = 10; // str(takeProfit4)
int xTP5 = 11; // str(takeProfit5)
int xOpened = 12; // posição já foi aberta? true/false
int xClosed = 13; // posição já foi fechada? true/false
int xHalfC=14; // HalfCloseTime
int xManualC = 15; // ManualCloseTime
int xSLMTime = 16; // SLModify_Time
int xSLMPrice= 17; // SLModify_Price
int xMagic=18; // magicCanal
//+------------------------------------------------------------------+
//| Expert initialization function                                   |
//+------------------------------------------------------------------+
int OnInit()
  {
//---
   ArraySetAsSeries(candle,true);

   if(!ReadFileToArrayCSV("sinais.csv",lines))
     {
      Alert("Error, see the \"Experts\" tab for details");
      return(INIT_FAILED);
     }

   sizeLine=ArraySize(lines);

   return(INIT_SUCCEEDED);
  }
//+------------------------------------------------------------------+
//| Expert deinitialization function                                 |
//+------------------------------------------------------------------+
void OnDeinit(const int reason)
  {
//---

   ArrayFree(lines);
   ArrayFree(ar);
   double a = ProfitClosedPosition();
   Print(a);

  }
//+------------------------------------------------------------------+
//| Expert tick function                                             |
//+------------------------------------------------------------------+
void OnTick()
  {

   now=TimeCurrent();
   TimeToStruct(TimeCurrent(),TimeStruct);

   if(previousDay!=TimeStruct.day_of_year) //Runs once a day
     {
      previousDay=TimeStruct.day_of_year;

      // recria diariamente o array auxiliar ar
      ArrayResize(ar,sizeLine);
      for(int i=0;i<sizeLine;i++)
        {
         for(int j=0;j<columns;j++)
           {
            ar[i][j]=lines[i].field[j];
           }
        }

      // após array ar recriado, elimina-se o que não for ser usado no dia
      for(int i=sizeLine-1; i>=0; i--)
        {
         TimeToStruct((datetime)lines[i].field[xDtEnt],dealDay);

         // if temporario; o deal close não será por data, e sim pelo fechamento do trade, a ser sinalizado no futuro
         if(dealDay.day_of_year+2<TimeStruct.day_of_year && dealDay.year==TimeStruct.year) 
           {
            ArrayRemove(ar,i,1);
           }

         // remove entradas cujas datas de entrada sejam superiores ao dia de hoje ou que possuam flag 'closed'
         if((dealDay.day_of_year>TimeStruct.day_of_year && dealDay.year<=TimeStruct.year) || (lines[i].field[xClosed]==true))
           {
            ArrayRemove(ar,i,1);
           }
        } 

     } 

   NewBar=isNewBar();
   if(NewBar)
     {
      CopyRates(_Symbol,PERIOD_M1,0,5,candle);
      sizeAr=ArraySize(ar);
      arColumns=ArrayRange(ar,1);
      datetime candleNowTime = candle[0].time;
      datetime nextCandleTime = candle[0].time+60;      
      for(int i=0;i<sizeAr/arColumns;i++)
        {
         if(ar[i][xDtEnt]>candleNowTime && ar[i][xDtEnt]<nextCandleTime)
           {
            fAbrirOp(i);
            Print(ar[i][xCanal]," / ",ar[i][xOP]," / ",ar[i][xSL]," / ",ar[i][xTP1]);
           }

        }
     } 
  }  


//+------------------------------------------------------------------+
//| Função que transforma arquivo CSV em array                       |
//+------------------------------------------------------------------+
bool ReadFileToArrayCSV(string FileName,SLine  &Lines[])
  {
   ResetLastError();
   int h=FileOpen(FileName,FILE_READ|FILE_ANSI|FILE_CSV|FILE_COMMON,";");
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
   if(h==INVALID_HANDLE)
     {
      int ErrNum=GetLastError();
      printf("Error opening file %s # %i",FileName,ErrNum);
      return(false);
     }
   int lcnt=0; // variable for calculating lines 
   int fcnt=0; // variable for calculating line fields    
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
   while(!FileIsEnding(h))
     {
      string str=FileReadString(h);
      // new line (new element of the structure array)
      if(lcnt>=ArraySize(Lines))
        { // structure array completely filled
         ArrayResize(Lines,ArraySize(Lines)+1024); // increase the array size by 1024 elements
        }
      ArrayResize(Lines[lcnt].field,64);// change the array size in the structure
      Lines[lcnt].field[0]=str; // assign the first field value
                                // start reading other fields in the line
      fcnt=1; // till one element in the line array is occupied
      while(!FileIsLineEnding(h))
        { // read the rest of fields in the line
         str=FileReadString(h);
         if(fcnt>=ArraySize(Lines[lcnt].field))
           { // field array is completely filled
            ArrayResize(Lines[lcnt].field,ArraySize(Lines[lcnt].field)+64); // increase the array size by 64 elements
           }
         Lines[lcnt].field[fcnt]=str; // assign the value of the next field
         fcnt++; // increase the line counter
        }
      ArrayResize(Lines[lcnt].field,fcnt); // change the size of the field array according to the actual number of fields
      lcnt++; // increase the line counter
     }
   ArrayResize(Lines,lcnt); // change the array of structures (lines) according to the actual number of lines
   FileClose(h);
   return(true);
  }

//+------------------------------------------------------------------+
//| Função checar nova barra                                         |
//+------------------------------------------------------------------+
bool isNewBar(void)
  {
//--- memorize the time of opening of the last bar in the static variable
   static datetime last_time=0;
//--- current time
   datetime lastbar_time=(datetime)SeriesInfoInteger(Symbol(),PERIOD_M1,SERIES_LASTBAR_DATE);
//--- if it is the first call of the function
   if(last_time==0)
     {
      //--- set the time and exit
      last_time=lastbar_time;
      return(false);
     }
//--- if the time differs
   if(last_time!=lastbar_time)
     {
      //--- memorize the time and return true
      last_time=lastbar_time;
      return(true);
     }
//--- if we passed to this line, then the bar is not new; return false
   return(false);
  }
//+------------------------------------------------------------------+

//+------------------------------------------------------------------+
//| entrada em operação                                                                 |
//+------------------------------------------------------------------+
void fAbrirOp(int i)
  {
  if(inicioHedge==0)inicioHedge=datetime(ar[i][xDtEnt]); 
   double _TP;
   double _SL;
   orderOkToOpen=true;
   Print("iniciando ativo: ",ar[i][xAtv]);
   if(!mysymbol.Name(ar[i][xAtv]))
     {
      Print("Erro ao selecionar ativo ",ar[i][xAtv],"; operação não será aberta.");
      orderOkToOpen=false;
     }
   mysymbol.RefreshRates();
   double spread=mysymbol.Spread();
   double bid = mysymbol.Bid();
   double ask = mysymbol.Ask();
   double maxPrice = ask * (100 + maxPricesVariation)/100;
   double minPrice = bid * (100 - maxPricesVariation)/100;

//--- tuning for 3 or 5 digits
   int digits_adjust=1;
   if(mysymbol.Digits()==3 || mysymbol.Digits()==5)
      digits_adjust=10;
   stopLossDelta   = slFixo*digits_adjust;
   takeProfitDelta = tpFixo*digits_adjust;
   if(maxSlippage!=0) trade.SetDeviationInPoints(maxSlippage);

   if(double(ar[i][xOP])<minPrice || double(ar[i][xOP])>maxPrice || double(ar[i][xSL])<minPrice || double(ar[i][xSL])>maxPrice || double(ar[i][xTP1])<minPrice || double(ar[i][xTP1])>maxPrice)
     {
      Print("Erro nos níveis de preço do ativo ",ar[i][xAtv],". Bid = ",bid," / Open Price: ",ar[i][xOP]," / SL: ",ar[i][xSL]," / TP: ",ar[i][xTP1]," operação não será aberta.");
      orderOkToOpen=false;
     }

   if(orderOkToOpen)
     {
      trade.SetExpertMagicNumber(ulong(ar[i][xMagic]));
      if(ar[i][xBuySell]=="buy")
        {
         _TP = tpFixo == 0 ? double(ar[i][xTP1]) : double(ar[i][xOP]) + takeProfitDelta;
         _SL = slFixo == 0 ? double(ar[i][xSL]) : double(ar[i][xOP]) - stopLossDelta;

         if(trade.Buy(initialVolume,ar[i][xAtv],double(ar[i][xOP]),_SL,_TP,ar[i][xCanal]))
           {
            ar[i][xOpened]=true;
            Print("Ordem colocada, ticket: ",trade.ResultDeal());
            Print("preço: ",trade.ResultPrice()," , era pra ter sido por ",ar[i][xOP]);
            trade.PrintResult();
           }
         else
           {
            Print(trade.ResultRetcodeDescription());
            trade.PrintRequest();
           }
        }
      else if(ar[i][xBuySell]=="sell")
        {
         _TP = tpFixo == 0 ? double(ar[i][xTP1]) : double(ar[i][xOP]) - takeProfitDelta;
         _SL = slFixo == 0 ? double(ar[i][xSL]) : double(ar[i][xOP]) + stopLossDelta;

         if(trade.Sell(initialVolume,ar[i][xAtv],double(ar[i][xOP]),_SL,_TP,ar[i][xCanal]))
           {
            ar[i][xOpened]=true;
            Print("Ordem colocada, ticket: ",trade.ResultDeal());
            Print("preço: ",trade.ResultPrice()," , era pra ter sido por ",ar[i][xOP]);
            trade.PrintResult();
           }
         else
           {
            Print(trade.ResultRetcodeDescription());
            trade.PrintRequest();
           }
        }

     }
  } 


//+------------------------------------------------------------------+
//| Lucro acumulado do dia                                           |
//+------------------------------------------------------------------+
double ProfitClosedPosition()
  {
   int lSize;
   bool magicFound;
   double profit=0;
   int counter=1;
   HistorySelect(inicioHedge,TimeCurrent());
   int deals=HistoryDealsTotal();
   int colunas = ArrayRange(profitPerMagic,1);
//---
   for(int i=0;i<deals;i++)
     {
      ulong deal_ticket=HistoryDealGetTicket(i);
      string symbol= HistoryDealGetString(deal_ticket,DEAL_SYMBOL); 
      double _p_profit=HistoryDealGetDouble(deal_ticket,DEAL_PROFIT);
      int dealType = (int)HistoryDealGetInteger(deal_ticket,DEAL_TYPE);
      int dealMagic = (int)HistoryDealGetInteger(deal_ticket,DEAL_MAGIC);
      string   dealComment      =HistoryDealGetString(deal_ticket,DEAL_COMMENT);
      if(dealType!=0 && dealType!=1) continue;
      lSize = ArraySize(profitPerMagic);
      magicFound=false;
      if(lSize==0)ArrayResize(profitPerMagic,(lSize/colunas)+1);

      for(int j=0;j<(ArraySize(profitPerMagic)/colunas);j++)
        {
         if(profitPerMagic[j][0]==double(dealMagic))
           {
            profitPerMagic[j][1]+=_p_profit;
            magicFound=true;
            break;
           }
        }
      if(!magicFound)
        {
         ArrayResize(profitPerMagic,(lSize/colunas)+1);
         for(int j=(ArraySize(profitPerMagic)/colunas)-1;j>=0;j--)
           {
            profitPerMagic[j][0]=dealMagic;
            profitPerMagic[j][1]=profit;
            break;
           }
        }
      profit+=_p_profit;

      Print(counter+": Ativo: "+symbol+" R$ "+_p_profit+" do tipo "+dealType+" , magic: "+dealMagic," / ",dealComment);
      counter++;
     }

   for(int i=0;i<(ArraySize(profitPerMagic)/colunas);i++)
     {
      Print(profitPerMagic[i][0], " / " ,profitPerMagic[i][1]);
     }
   return(profit);
  }
//+------------------------------------------------------------------+

Answer 1

The easiest way is to use classes, each class extends CObject and you must implement Compare function. First, create a structure of entering (in=true) and exiting (in=false):

struct SEvent
{
   datetime m_time;
   double   m_price;
   bool     m_in;

   void Init(const datetime time,const double price,const bool in)
   {
    m_time=time;
    m_price=price;
    m_in=in;
   }
   int Compare(SEvent &another)const
   {
    return int(m_time-another.m_time);
   }
};

Next, create the class CDeal : public CObject

class CDeal : public CObject
{
 static int  getDir(string cmd)
 {
  if(cmd=="Sell")
     return(-1);
  if(cmd=="Buy")
     return(1);
  if(cmd=="Type" || cmd=="Balance")
     return(0);
  printf("%i %s: unknown cmd=|%s|",__LINE__,__FILE__,cmd);
  return(0);
 }
 static string getSymbol(string line)
 {
  return StringSubstr(line,0,StringLen(line)-StringLen(InpReportSuffix))+InpSymbolSuffix;
 }
public:
 intX        m_id;
 int         m_dir;
 SEvent      m_in;
 SEvent      m_out;
 double      m_lot;
 string      m_symbol;
 double      m_osl;
 double      m_otp;
 CDeal(string line)
 {
  //2019.05.13 18:27:56;Sell;0.10;EURCADm#;1.51270;;;2019.05.14 13:36:47;1.51142;;;0.10;
  string array[];
  const ushort separator=(ushort)';';
  int size=StringSplit(line,separator,array);
  if(size<11)
    {
     printf("%i %s: size=%d str=%s",__LINE__,__FILE__,size,line);
     m_dir=0;
     return;
    }
  m_dir=getDir(array[1]);
  m_lot=StringToDouble(array[2]);
  m_symbol=getSymbol(array[3]);
  m_in.Init(StringToTime(array[0]),StringToDouble(array[4]),true);
  m_osl=StringLen(array[5])>0 ? StringToDouble(array[5]) : 0;
  m_otp=StringLen(array[6])>0 ? StringToDouble(array[6]) : 0;
  m_out.Init(StringToTime(array[7]),StringToDouble(array[8]),false);
 }
~CDeal(){}
 virtual int       Compare(const CObject *node,const int mode=0) const 
 {
  CDeal *another=(CDeal*)node;
  if(mode==1)
    {
     return m_in.Compare(another.m_in);
    }
  else
    {
     return m_out.Compare(another.m_out);
    }
 }
 virtual string    toString()const
 {
  return StringFormat("%s;%d;%.2f;%s;%.5f;%.5f;%.5f;%s;%.5f;%s",TimeToString(m_in.m_time),m_dir,m_lot,m_symbol,
     m_in.m_price,m_osl,m_otp,TimeToString(m_out.m_time),m_out.m_price,m_pnl.toString());
 }
};

Then, load all CDeals from string into CArrayObj and list.Sort(1) in order to sort them by time of entry. Then loop, either from some static int cursor=0; that increments if TimeCurrent()>((CDeal*) list.At(cursor)).m_in.m_time of course keeping in mind that you may reach end of your list, or detach the element if same condition is true, and attach to another CArrayObj listOfOpenDeals. Do not forget to sort the list of existing open deals each time you add an element. Same way check whether it is time to close some deal from the list of open deals, and delete element after closing. Altogether 300 lines of code.