How to cancel a recurrent job when it runs concurrently twice?

Question

I have added an attribute DisableConcurrentExecution(1) on the job, but all that does is delays the execution of second instance of a job until after first one is done. I want to be able to detect when a concurrent job has been run, and then cancel it all together.

I figured, if DisableConcurrentExecution(1) will prevent two instances of same recurrent job from running at the same, it will put the second job on "retry", thus changing it's State. So I added additional custom attribute on the job, which detects failed state, like so :

public class StopConcurrentTask : JobFilterAttribute, IElectStateFilter
{
    public void OnStateElection(ElectStateContext context)
    {
        var failedState = context.CandidateState as FailedState;
        if(failedState != null && failedState.Exception != null)
        {
            if(!string.IsNullOrEmpty(failedState.Exception.Message) && failedState.Exception.Message.Contains("Timeout expired. The timeout elapsed prior to obtaining a distributed lock on"))
            {

            }
        }
    }
}

This allows me to detect whether a job failed due to being run concurrently with another instance of same job. The problem is, I can't find a way to Cancel this specific failed job and remove it from being re-run. As it is now, the job will be put on retry schedule and Hangfire will attempt to run it a number of times.

I could of course put an attribute on the Job, ensuring it does not Retry at all. However, this is not a valid solution, because I want jobs to be Retried, except if they fail due to running concurrently.

Answer 1

I actually ended up using based on Jr Tabuloc answer - it will delete a job if it has been last executed 15 seconds ago - I noticed that time between server wake up and job execution varies. Usually it is in milliseconds, but since my jobs are executed once a day, I figured 15sec won't hurt.

public class StopWakeUpExecution : JobFilterAttribute, IServerFilter
{
    public void OnPerformed(PerformedContext filterContext)
    {

    }

    public void OnPerforming(PerformingContext filterContext)
    {
        using (var connection = JobStorage.Current.GetConnection())
        {
            var recurring = connection.GetRecurringJobs().FirstOrDefault(p => p.Job.ToString() == filterContext.BackgroundJob.Job.ToString());
            TimeSpan difference = DateTime.UtcNow.Subtract(recurring.LastExecution.Value);
            if (recurring != null && difference.Seconds < 15)
            {
                // Execution was due in the past. We don't want to automaticly execute jobs after server crash though.

                var storageConnection = connection as JobStorageConnection;

                if (storageConnection == null)
                    return;

                var jobId = filterContext.BackgroundJob.Id;

                var deletedState = new DeletedState()
                {
                    Reason = "Task was due in the past. Please Execute manually if required."
                };

                using (var transaction = connection.CreateWriteTransaction())
                {
                    transaction.RemoveFromSet("retries", jobId);  // Remove from retry state
                    transaction.RemoveFromSet("schedule", jobId); // Remove from schedule state
                    transaction.SetJobState(jobId, deletedState);  // update status with failed state
                    transaction.Commit();
                }
            }
        }
    }
}

Answer 2

You can prevent retry to happen if you put validation in OnPerformed method in IServerFilter interface.

Implementation :

public class StopConcurrentTask : JobFilterAttribute, IElectStateFilter, IServerFilter
    {
        // All failed after retry will be catched here and I don't know if you still need this
        // but it is up to you
        public void OnStateElection(ElectStateContext context)
        {
            var failedState = context.CandidateState as FailedState;
            if (failedState != null && failedState.Exception != null)
            {
                if (!string.IsNullOrEmpty(failedState.Exception.Message) && failedState.Exception.Message.Contains("Timeout expired. The timeout elapsed prior to obtaining a distributed lock on"))
                {

                }
            }
        }

        public void OnPerformed(PerformedContext filterContext)
        {
            // Do your exception handling or validation here
            if (filterContext.Exception == null) return;

            using (var connection = _jobStorage.GetConnection())
            {
                var storageConnection = connection as JobStorageConnection;

                if (storageConnection == null)
                    return;

                var jobId = filterContext.BackgroundJob.Id
                // var job = storageConnection.GetJobData(jobId); -- If you want job detail

                var failedState = new FailedState(filterContext.Exception)
                {
                    Reason = "Your Exception Message or filterContext.Exception.Message"
                };

                using (var transaction = connection.GetConnection().CreateWriteTransaction())
                {
                    transaction.RemoveFromSet("retries", jobId);  // Remove from retry state
                    transaction.RemoveFromSet("schedule", jobId); // Remove from schedule state
                    transaction.SetJobState(jobId, failedState);  // update status with failed state
                    transaction.Commit();
                }
            }
        }

        public void OnPerforming(PerformingContext filterContext)
        {
           // Do nothing
        }
    }

I hope this will help you.