Swift 4: How to asynchronously use URLSessionDataTask but have the requests be in a timed queue?

Question

Basically I have some JSON data that I wish to retrieve from a bunch of URL's (all from the same host), however I can only request this data roughly every 2 seconds at minimum and only one at a time or I'll be "time banned" from the server. As you'll see below; while URLSession is very quick it also gets me time banned almost instantly when I have around 700 urls to get through.

How would I go about creating a queue in URLSession (if its functionality supports it) and while having it work asynchronously to my main thread; have it work serially on its own thread and only attempt each item in the queue after 2 seconds have past since it finished the previous request?

for url in urls {
    get(url: url)
}


func get(url: URL) {
    let session = URLSession.shared
    let task = session.dataTask(with: url, completionHandler: { (data, response, error) in

        if let error = error {
            DispatchQueue.main.async {
                print(error.localizedDescription)
            }
            return
        }
        let data = data!

        guard let response = response as? HTTPURLResponse, response.statusCode == 200 else {
            DispatchQueue.main.async {
                print("Server Error")
            }
            return
        }
        if response.mimeType == "application/json" {
            do {
                let json = try JSONSerialization.jsonObject(with: data) as! [String: Any]
                if json["success"] as! Bool == true {
                    if let count = json["total_count"] as? Int {
                        DispatchQueue.main.async {
                            self.itemsCount.append(count)
                        }
                    }
                }
            } catch {
                print(error.localizedDescription)
            }
        }
    })
    task.resume()
}

Answer 1

Recursion solves this best

import Foundation
import PlaygroundSupport

// Let asynchronous code run
PlaygroundPage.current.needsIndefiniteExecution = true

func fetch(urls: [URL]) {

    guard urls.count > 0 else {
        print("Queue finished")
        return
    }

    var pendingURLs = urls
    let currentUrl = pendingURLs.removeFirst()

    print("\(pendingURLs.count)")

    let session = URLSession.shared
    let task = session.dataTask(with: currentUrl, completionHandler: { (data, response, error) in
        print("task completed")
        if let _ = error {
            print("error received")
            DispatchQueue.main.async {
                fetch(urls: pendingURLs)
            }
            return
        }

        guard let response = response as? HTTPURLResponse, response.statusCode == 200 else {
            print("server error received")
            DispatchQueue.main.async {
                fetch(urls: pendingURLs)
            }
            return
        }
        if response.mimeType == "application/json" {
            print("json data parsed")
            DispatchQueue.main.async {
                fetch(urls: pendingURLs)
            }
        }else {
            print("unknown data")
            DispatchQueue.main.async {
                fetch(urls: pendingURLs)
            }
        }
    })

    //start execution after two seconds
    Timer.scheduledTimer(withTimeInterval: 2, repeats: false) { (timer) in
        print("resume called")
        task.resume()
    }
}

var urls = [URL]()
for _ in 0..<100 {
    if let url = URL(string: "https://google.com") {
        urls.append(url)
    }
}

fetch(urls:urls)

Answer 2

Make DispatchQueue to run your code on threads. You don't need to do this work on Main Thread. So,

// make serial queue
let queue = DispatchQueue(label: "getData")

// for delay
func wait(seconds: Double, completion: @escaping () -> Void) {
    queue.asyncAfter(deadline: .now() + seconds) { completion() }
}

// usage
for url in urls {
  wait(seconds: 2.0) {
      self.get(url: url) { (itemCount) in
         // update UI related to itemCount
      }
  }
}

By the way, Your get(url: url) function is not that great.

func get(url: URL, completionHandler: @escaping ([Int]) -> Void) {
    let session = URLSession.shared
    let task = session.dataTask(with: url, completionHandler: { (data, response, error) in

        if let error = error {
            print(error.localizedDescription)
            /* Don't need to use main thread
                DispatchQueue.main.async {
                    print(error.localizedDescription)
                }
            */
            return
        }
        let data = data!

        guard let response = response as? HTTPURLResponse, response.statusCode == 200 else {
            print("Server Error")
            /* Don't need to use main thread
                DispatchQueue.main.async {
                    print("Server Error")
                }
            */
            return
        }
        if response.mimeType == "application/json" {
            do {
                let json = try JSONSerialization.jsonObject(with: data) as! [String: Any]
                if json["success"] as! Bool == true {
                    if let count = json["total_count"] as? Int {

                        self.itemsCount.append(count)

                        // append all data that you need and pass it to completion closure
                        DispatchQueue.main.async {
                            completionHandler(self.itemsCount)
                        }
                    }
                }
            } catch {
                print(error.localizedDescription)
            }
        }
    })
    task.resume()
}

I would recommend you to learn concept of GCD(for thread) and escaping closure(for completion handler).

• GCD: https://www.raywenderlich.com/148513/grand-central-dispatch-tutorial-swift-3-part-1
• Escaping Closure: https://developer.apple.com/library/content/documentation/Swift/Conceptual/Swift_Programming_Language/Closures.html#//apple_ref/doc/uid/TP40014097-CH11-ID546

Answer 3

The easiest way is to perform recursive call:

1. Imagine you have array with your urls.

2. In place where you initially perform for loop with, replace it with single call get(url:). self.get(urls[0])

3. Then add this line at the and of response closure right after self.itemsCount.append(count):

   self.urls.removeFirst()
   Timer.scheduledTimer(withTimeInterval: 2, repeats: false) { (_) in
   self.get(url: urls[0])
   }