Multi threading not working as expected with Parallel Foreach

Question

public void GetTopRequest()
{
        DataTable dtListofTransaction = new DataTable();
        string sconnection = ConfigurationManager.AppSettings[ConfigurationManager.AppSettings["BUToProcess"].ToString()].ToString();
        string query = "select Error_Counter, MembershipEnrollmentStatus_ID_PKID, outbound_xml, type_of_transaction, ProcessSet_GUID from MES where TP = 0 and RS = 'A' and type_of_transaction = 'Initial Enrollment'";

        using (SqlConnection sqlConn = new SqlConnection(sconnection))
        {
            sqlConn.Open();

            using (SqlCommand cmd = new SqlCommand(query, sqlConn))
            {
                dtListofTransaction.Load(cmd.ExecuteReader());
                var tmpdtListofTransaction = dtListofTransaction;

                if (dtListofTransaction.Rows.Count > 0)
                {
                    var distinctListOfFamilies = (from row in tmpdtListofTransaction.AsEnumerable()
                                                  select row.Field<Guid>("ProcessSet_GUID")).Distinct().ToList();
                    int countOfFamilies = distinctListOfFamilies.Count();
                    int FamiliesToBeProcessedByThread = (countOfFamilies > 50 ? (countOfFamilies / Convert.ToInt32(ConfigurationManager.AppSettings["ThreadsToUse"])) + 1 : countOfFamilies);
                    var listOfSubscriberLists = splitList<Guid>(distinctListOfFamilies, FamiliesToBeProcessedByThread);
                    var tmplistOfSubscriberLists = listOfSubscriberLists;

                    if (countOfFamilies > 0)
                    {
                        Parallel.ForEach(listOfSubscriberLists,, new ParallelOptions { MaxDegreeOfParallelism = 4}, ac => 
                        {
                            foreach (var guid in ac)
                            {
                                Console.WriteLine(guid + " : " + DateTime.Now.ToString("MM/dd/yyyy hh:mm:ss.fff"));
                                var dbMembersToProcess = tmpdtListofTransaction.AsEnumerable().Where(p => object.Equals(p["ProcessSet_GUID"], guid) && p["outbound_xml"] != null);

                                foreach (var member in dbMembersToProcess)
                                {
                                    PushWWRequest.SendTransactions(member["outbound_xml"].ToString(), member["type_of_transaction"].ToString(), member["MembershipEnrollmentStatus_ID_PKID"].ToString(), Convert.ToInt32(member["Error_Counter"].ToString()));
                                }
                            }
                        });
                    }
                }
            }

            sqlConn.Close();
        }
    }

    public static void SendTransactions(string sRequest, string sTransactionType, string sPKID = "", int ErrorCounter = 0)
    {
        switch (sTransactionType)
        {
            case TransactionType.INITIALENROLLMENT:
                try
                {
                    CMInitialCustomerSetupTypeClient svcInitialCustomerSetupClientProxy = ClientOrmbProxy<CMInitialCustomerSetupTypeClient>();
                    CMInitialCustomerSetup initialCustomerSetupRequest = Deserialize<CMInitialCustomerSetup>(sRequest);                   
                    svcInitialCustomerSetupClientProxy.CMInitialCustomerSetup(initialCustomerSetupRequest);
                }
                catch (System.ServiceModel.CommunicationException svcExcpt)
                {
                    print svcExcpt
                }
                break;
     }
 }

I am trying to send 4 requests using 4 threads (as defined with the degree of parallelism above set to 4) at a time to process using the above code in Parallel.ForEach loop, but I am not seeing 4 requests taken simultaneously and the threads are processed in random order. I am not sure what I am doing wrong here.

Any suggestions would be helpful...Thank you.

Answer 1

Since your question is fairly trivial, I'll answer it with a bunch of other suggestions

• You shouldn't be using Parallel.ForEach with IO bound tasks, you are wasting threads and resources, the async await pattern is a better approach, it will release threads back to the operating system when waiting for an IO completion port, I'd consider an ActionBlock for the best of both worlds.

• Your code is a mess (said in the nicest possible way)

• MaxDegreeOfParallelism is only a suggestion and not a contract, TPL will decide what it needs and thinks appropriate.

• If running in parallel, there is no guarantee on what will get executed in which order, that is the idea of parallel programming, threads have a degree of freedom and order is not guaranteed

• You are creating reference copies all over the place, there is no need to do that i.e var tmpdtListofTransaction = dtListofTransaction;

• DataTable dtListofTransaction = new DataTable() should be in a using statment

• Reduce your nesting for readability, just because you can put things into a lambda statement doesn't mean you should, make it easier for yourself

• Stop using the ConfigurationManager in nested statements, it makes it impossible to read; use expression bodied properties, or store them in a variable once and be done with it

If you want to assure exactly 4 threads at once, consider making them tasks and using WaitAll, or WhenAll depending what you want, probably await Task.WhenAll(tasks);