How to rotate between multiple forwarding proxies for outgoing requests with golang

Question

I will like to pass a list of forwarding proxy servers for POST request

Currently i am able to do it with just single forwarding proxy

serverProxy := "http://user:password@123.45.67.89:3128"

request, error := http.NewRequest("POST", httpposturl, bytes.NewBuffer(requestJSON))
request.Header.Set("Content-Type", "application/json; charset=UTF-8")

proxyURL, _ := url.Parse(serverProxy)
proxy := http.ProxyURL(proxyURL)
transport := &http.Transport{Proxy: proxy}
client := &http.Client{Transport: transport}

what i will like to do is pass a list to url.Parse and want it to use them using round robin balancing

so something like this

serverProxy := "http://user:password@123.45.67.89:3128, http://user:password@223.45.67.89:3128"

and then it will select which of the proxy servers to use and rotate them within requests

Is this possible?

UPDATE:

I want to be able to pass the rotated proxy server like this

proxyServer := roundRobin("http://round:robin@123.45.67.89:3128, http://robin:round@223.45.67.89:3128")
fmt.Println("proxy server used", proxyServer, "\n")
transport := &http.Transport{Proxy: proxyServer}
client := &http.Client{Transport: transport}

Answer 1

Here's the Montage's answer with explanation:

The requirement is to forward request through proxies in round-robin fashion

Since we are using http.Client to make request we can look at http.Client documentation to see if it provide any support for forwarding request through proxy when we look at the documentation we can see that it does support passing proxy which we can pass through http.Transport type which will be passed to http.Client. http.Transport takes proxy through Proxy field which takes in func that return *url.URL and error there are existing methods like http.ProxyURL and http.ProxyFromEnvironment provided within http package that we can use to pass proxies to http.Transport but the problem with these methods is that they only take a single proxy server which does not solve our problem at hand and hence we would require to create our own function which takes in multiple proxy servers urls and round-robin between them.

If we look at one of the existing method implemention as our base for creating our own method lets go with http.ProxyURL for our case the implementation can be found here. I have copied the implementation below

func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) {
    return func(*Request) (*url.URL, error) {
        return fixedURL, nil
    }
}

we can see that its a simple closure which takes in single url and return a closure function which then intern return the url passed in as parameter. so we can take it base and create our own round-robin clouse function

func roundRobin(proxies ...string) func(*http.Request) (*url.URL, error) {
    var urls []*url.URL
    for _, proxy := range proxies {
        u, err := url.Parse(proxy)
        if err != nil {
            log.Fatal(err)
        }
        urls = append(urls, u)
    }

    var mu sync.Mutex
    var i, lenUrls int = 0, len(urls)
    return func(r *http.Request) (*url.URL, error) {
        mu.Lock()
        i = (i + 1) % lenUrls
        u := urls[i]
        mu.Unlock()
        return u, nil
    }
}

Lets go over the roundRobin function implementation it is a variadic function which takes in proxy url(s) in string format as argument, which internally gets converted to url.URL by parsing the string using url.Parse then using the parsed url.URL to create slice of urls []*url.URL which then being used to forward request in round-robin fashion

Complete working example can be found below:

package main

import (
    "fmt"
    "log"
    "net/url"
    "net/http"
    "sync"
)

func roundRobin(proxies ...string) func(*http.Request) (*url.URL, error) {
    var urls []*url.URL
    for _, proxy := range proxies {
        u, err := url.Parse(proxy)
        if err != nil {
            log.Fatal(err)
        }
        urls = append(urls, u)
    }

    var mu sync.Mutex
    var i, lenUrls int = 0, len(urls)
    return func(r *http.Request) (*url.URL, error) {
        mu.Lock()
        i = (i + 1) % lenUrls
        u := urls[i]
        mu.Unlock()
        return u, nil
    }
}

func main() {
    proxyFn := roundRobin("http://user:password@123.45.67.89:3128", "http://user:password@223.45.67.89:3128")
    transport := &http.Transport{Proxy: proxyFn}
    client := &http.Client{Transport: transport}

    req, err := http.NewRequest("POST", "http://example.com", nil)
    req.Header.Set("Content-Type", "application/json; charset=UTF-8")

    resp, err := client.Do(req)
    if err != nil {
        fmt.Println(resp)
    }

    fmt.Println(proxyFn(nil))

    fmt.Println(proxyFn(nil))
}

Playground

Another version

package main

import (
    "fmt"
    "log"
    "net/http"
    "net/url"
    "sync"
)

func praseUrls(proxies ...string) (urls []*url.URL) {
    for _, proxy := range proxies {
        u, err := url.Parse(proxy)
        if err != nil {
            log.Fatal(err)
        }
        urls = append(urls, u)
    }
    return
}

func roundRobin(max int) func() int {
    var i int
    return func() int {
        i = (i + 1) % max
        return i
    }
}

func proxyFn(urls []*url.URL) func(*http.Request) (*url.URL, error) {
    var m sync.Mutex
    fn := roundRobin(len(urls))
    return func(*http.Request) (*url.URL, error) {
        m.Lock()
        u := urls[fn()]
        m.Unlock()
        return u, nil
    }
}

func main() {
    proxies := []string{"http://user:password@123.45.67.89:3128", "http://user:password@223.45.67.89:3128"}
    urls := praseUrls(proxies...)
    transport := &http.Transport{Proxy: proxyFn(urls)}
    client := &http.Client{Transport: transport}

    req, err := http.NewRequest("POST", "http://example.com", nil)
    req.Header.Set("Content-Type", "application/json; charset=UTF-8")

    resp, err := client.Do(req)
    if err != nil {
        fmt.Println(resp)
    }
}

Playground

Note: It would be better to pass proxy urls from env variable which would help in case any proxy server changes or new are added

Answer 2

Create a proxy function that round-robins through your proxy URLs. Use that function in your transport:

func roundRobin(urls []*url.URL) func(*http.Request) (*url.URL, error) {
    var mu sync.Mutex
    var i int
    return func(r *http.Request) (*url.URL, error) {
        mu.Lock()
        i = (i + 1) % len(urls)
        u := urls[i]
        mu.Unlock()
        return u, nil
    }
}


transport := &http.Transport{Proxy: roundRobin(yourProxyURLs)}
client := &http.Client{Transport: transport}

Answer 3

Here's the Montage's answer with code to parse a string.

func roundRobin(serverProxy string) func(*http.Request) (*url.URL, error) {
    parts := strings.Split(serverProxy, ",")
    var urls []*url.URL
    for _, part := range parts {
        u, err := url.Parse(strings.TrimSpace(part))
        if err != nil {
            log.Fatal(err)
        }
        urls = append(urls, u)
    }
    var mu sync.Mutex
    var i int
    return func(r *http.Request) (*url.URL, error) {
        mu.Lock()
        i = (i + 1) % len(urls)
        u := urls[i]
        mu.Unlock()
        return u, nil
    }
}

serverProxy := "http://user:password@123.45.67.89:3128, http://user:password@223.45.67.89:3128"
transport := &http.Transport{Proxy: roundRobin(serverProxy)}
client := &http.Client{Transport: transport}