Azure webjob, long duration

Question

I have a long duration webjob (TimerTrigger) runing.

It happens to fail from time to time.

It lasts 4 to 5 hours.

When it fails no exceptions are raised.

It may be a matter of plan wrongly dimensionned.

Is there a way to define the right plan ?

Also when restarting the webjob, previous occurence is restarted, we'd like to avoid this, and only next occurences should be restarted.

Environment : Azure WebJob, .netcore (3.1) stack, always on, 64bits, storage accounts writings, autofac dependendy injections to the webjob.

Any idea ?

Code :

        public FeedJob(IFeedConfiguration configuration, IKeyVaultService keyVaultService, IStorageTableService storageTableService, IBlobStorageService blobStorageService)
    {
        _configuration = configuration as FeedConfiguration;
        _keyVaultService = keyVaultService;
        _storageTableService = storageTableService;
        _blobStorageService = blobStorageService;
    }
    /// <summary>
    /// run periodically to request  web service
    /// </summary>
    /// <param name="myTimer"></param>
    /// <param name="log"></param>
    /// <returns></returns>
    public async Task ProcessTimerJob([TimerTrigger("%RunEvery%")] TimerInfo myTimer, ILogger log)
    {
        try
        {
            logJob = log;
            _cptTotal = _cptErrors = _cptNominals = 0;
            _dtStart = DateTime.UtcNow;
            var dtStartForReconstruction = _dtStart.AddHours(-1);

            logJob.LogInformation($"OP- =========== Job -  - =========== Started at : {_dtStart}");

            // certificate necessary to be allowed to request  (soap)
            //var Saurcertificate = X509Helper.GetCertificate(_configuration.CertificateThumbprint);
            var Saurcertificate = await _keyVaultService.GetCertificateFromKeyVaultByThumbprint(_configuration.CertificateThumbprint);

            var requests = await _blobStorageService.GetRequests(_configuration.InRequest);
            var cl = new List<DataColumn>();


            // parquet data are loaded in memory (about 17500 records)
            cl = await _blobStorageService.GetDataParquet(_dtStart);

            var borneInf = 0;
            var borneSup = cl[2].Data.Length;
            _cptTotal = 0;
            _errors = new Dictionary<string, int>();

            foreach (var req in requests)
            {
                var subEltsXml = req.Descendants().ToList();
                var endPointXml = subEltsXml.FirstOrDefault(e => e.Name == "EndPoint");
                var soapRequestXml = subEltsXml.FirstOrDefault(e => e.Name == "ReqSoap");
                _prefix = subEltsXml.FirstOrDefault(e => e.Name == "Prefix").Value;
                _title = subEltsXml.FirstOrDefault(e => e.Name == "Title").Value;

                DateTime dateFin, dateDeb;

                if (_prefix == "CCPM")
                {
                    dateDeb = _dtStart;
                    dateFin = _dtStart.AddYears(1).AddDays(-1);
                }
                else
                {
                    //CDC
                    dateFin = _dtStart;
                    dateDeb = _dtStart.AddDays(-1);
                }
                _dayFolder = $"{dateFin:yyyyMMdd}";

                logJob.LogInformation($@"OP- {_prefix} {_dtStart} =========== Job -  - =========== Extraction {_title} from  {dateDeb:yyyy-MM-dd} to {dateFin:yyyy-MM-dd}");
                logJob.LogInformation($@"OP- {_prefix} {_dtStart} =========== Job -  - =========== Extraction {_title} from pdl {borneInf} to pdl {borneSup - 1}");

                if (endPointXml != null && soapRequestXml != null)
                {
                    _cptTotalReq = _cptErrorsReq = _cptNominalsReq = 0;

                    for (int i = borneInf; i < borneSup; i++)
                    {
                        _cptTotalReq++;

                        var pdl = ((string[])cl[2].Data)[i];
                        var tarif = ((string[])cl[0].Data)[i];

                        var res = SetRequest(req, pdl, tarif, dateDeb, dateFin);

                        if (res.Item2)
                        {

                            var req2 = res.Item1;

                            var client = new RestClient(endPointXml.Value);
                            client.ClientCertificates = new X509CertificateCollection() { Saurcertificate };
                            client.Timeout = -1;

                            var request = new RestRequest(Method.POST);
                            request.AddHeader("Content-Type", "application/xml");
                            request.AddParameter("application/xml", req2.Descendants().ToList().FirstOrDefault().ToString(), ParameterType.RequestBody);

                            var response = await client.ExecuteAsync(request);

                            await WriteResultAsync(pdl, endPointXml.Value, response.Content, req2.Descendants().ToList().FirstOrDefault().ToString());
                        }

                        if (_cptTotalReq % 3000 == 0)
                        {
                            logJob.LogInformation($"OP- {_prefix} {DateTime.UtcNow} =========== Job -  - =========== {_cptTotalReq} / {borneSup - borneInf} pdls treated ");
                        }
                    }


                    _cptTotal += _cptTotalReq;
                    _cptNominals += _cptNominalsReq;
                    _cptErrors += _cptErrorsReq;
                }
            }


            await _storageTableService.WriteReport(new ReportRecord
            {
                PartitionKey = $"{_dtStart:yyyy-MM-dd-HH-mm-ss-fff}",
                RowKey = _prefix,
                DayFolder = _dayFolder,
                PdlsRetrieved = _cptTotalReq,
                PdlsTreated = _cptNominalsReq + _cptErrorsReq,
                PdlsTreatedOK = _cptNominalsReq,
                PdlsTreatedKO = _cptErrorsReq,
                PdlsNotTreated = _cptTotalReq - _cptNominalsReq - _cptErrorsReq,
                StatusesErrorXML = statusesErrorXML.ToString(),
                TariffsNotTreatedXML = tariffsNotTreatedXML.ToString()
            });

            logJob.LogInformation($@"OP- {DateTime.UtcNow}  Job -  - ================================== Total all extractions ==================================");
            logJob.LogInformation($@"OP- {DateTime.UtcNow}  Job -  - =========== Total pdls treated : {_cptTotal}");
            logJob.LogInformation($@"OP- {DateTime.UtcNow}  Job -  - =========== Total pdls OK : {_cptNominals}");
            logJob.LogInformation($@"OP- {DateTime.UtcNow}  Job -  - =========== Total pdls KO : {_cptErrors}");

            logJob.LogInformation($@"OP- {_dtStart}  Job -  - =========== Ended at : {DateTime.UtcNow}");

        }
        catch (Exception e)
        {
            logJob.LogInformation($@"OP- {_prefix} {_dtStart} =========== Job -  - =========== Exception generale at : {DateTime.UtcNow}");
            if (e.InnerException != null)
            {
                logJob.LogError($@"OP- {_prefix} {_dtStart} =========== Job -  - =========== Exception generale Message : {e.Message}");
                logJob.LogError($@"OP- {_prefix} {_dtStart} =========== Job -  - =========== Exception generale Inner Message : {e.InnerException.Message}");
            }
            else
            {
                logJob.LogError($@"OP- {_prefix} {_dtStart} =========== Job -  - =========== Exception generale Message : {e.Message}");
            }
        }
    }

Regards.