CODEWITHSUNDEEP
pythonrandomweb-scraping
scoop realm
scoop realm

Reputation: 37

How to select a random URL form extracted URL list

I am trying to extract the proxy urls and would like to use one random url

import requests
from lxml import html
import random

def get_proxy():
    url = 'https://sslproxies.org/'
    req = requests.get(url)
    iptree = html.fromstring(req.content)
    iprange = range(1,20)
    for ips in iprange:
        https = iptree.xpath('//*[@id="proxylisttable"]/tbody/tr[%d]/td[7]//text()'%ips)
        iptd = iptree.xpath('//*[@id="proxylisttable"]/tbody/tr[%d]/td[1]//text()'%ips)
        port = iptree.xpath('//*[@id="proxylisttable"]/tbody/tr[%d]/td[2]//text()'%ips)
        for htp in https:
            if htp=="yes":
                for (ips, por) in zip(iptd, port):
                    iplist = ("https://" + ips + ":" + por)
                    print(iplist)

get_proxy()

i want to assign one random URL to a string and use it in my web scraper, but i am unable to select random one

Upvotes: 0

Views: 88

Answers (2)

lonsty
lonsty

Reputation: 195

import requests
from lxml import html
import random

def get_proxy():
    url = 'https://sslproxies.org/'
    req = requests.get(url)
    iptree = html.fromstring(req.content)
    iprange = range(1,20)
    iplist = []
    for ips in iprange:
        https = iptree.xpath('//*[@id="proxylisttable"]/tbody/tr[%d]/td[7]//text()'%ips)
        iptd = iptree.xpath('//*[@id="proxylisttable"]/tbody/tr[%d]/td[1]//text()'%ips)
        port = iptree.xpath('//*[@id="proxylisttable"]/tbody/tr[%d]/td[2]//text()'%ips)
        for htp in https:
            if htp=="yes":
                for (ips, por) in zip(iptd, port):
                    ip = ("https://" + ips + ":" + por)
                    iplist.append(ip)
    return random.choice(iplist)

get_proxy()

Upvotes: 1

ComplicatedPhenomenon
ComplicatedPhenomenon

Reputation: 4199

def get_proxy():
    url = 'https://sslproxies.org/'
    req = requests.get(url)
    iptree = html.fromstring(req.content)
    iprange = range(1,20)
    iplist =[ ]
    for ips in iprange:
        https = iptree.xpath('//*[@id="proxylisttable"]/tbody/tr[%d]/td[7]//text()'%ips)
        iptd = iptree.xpath('//*[@id="proxylisttable"]/tbody/tr[%d]/td[1]//text()'%ips)
        port = iptree.xpath('//*[@id="proxylisttable"]/tbody/tr[%d]/td[2]//text()'%ips)
        for htp in https:
            if htp=="yes":
                for (ips, por) in zip(iptd, port):
                    ip = "https://" + ips + ":" + por
                    iplist.append(ip)
     return random.choice(iplist)

Upvotes: 0

Related Questions