urllib error: Too many requests

Question

The below python program asks the user for two reddit usernames and compares their score.

import json 
from urllib import request

def obtainKarma(users_data):

    users_info = []
    for user_data in users_data:
        data = json.load(user_data)
        posts = data["data"]["children"]

        num_posts = len(posts)

        scores = []
        comments = []

        for post_id in range(num_posts):
            score = posts[post_id]["data"]["score"]
            comment = posts[post_id]["num_comments"]

            scores.append(score)
            comments.append(comment)

        users_info.append((scores,comments))


    user_id = 0
    for user_info in users_info:
        user_id+=1

        print("User"+str(user_id))
        for user_attr in user_info:
            print(user_attr)




def getUserInfo():

    count = 2

    users_data = []

    while count:
        count = count + 1

        username = input("Please enter username:\n")
        url = "https://reddit.com/user/"+username+".json"

        try:
            user_data = request.urlopen(url)
        except:
                print("No such user.\nRetry Please.\n")
                count = count + 1
                raise

        users_data.append(user_data)

    obtainKarma(users_data)

if __name__ == '__main__':
    getUserInfo()

However, when I run the program and enter a username, I get an error:

raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 429: Too Many Requests

I tried looking for similar issues but none of them satisfied to solve this specific issue. Looking at the error, it would make sense to say that the URL includes an amount of data that exceeds a specific limit? But that still sounds absurd because it is not that much of a data.

Thanks.

Answer 1

The problem seems to be resolved when you supply a User-Agent with your request.

import json
from urllib import request

def obtainKarma(users_data):

    users_info = []
    for user_data in users_data:
        data = json.loads(user_data) # I've changed 'json.load' to 'json.loads' because you want to parse a string, not a file
        posts = data["data"]["children"]

        num_posts = len(posts)

        scores = []
        comments = []

        for post_id in range(num_posts):
            score = posts[post_id]["data"]["score"]
            comment = posts[post_id]["data"]["num_comments"] # I think you forgot '["data"]' here, so I added it

            scores.append(score)
            comments.append(comment)

        users_info.append((scores,comments))


    user_id = 0
    for user_info in users_info:
        user_id+=1

        print("User"+str(user_id))
        for user_attr in user_info:
            print(user_attr)




def getUserInfo():

    count = 2

    users_data = []

    while count:
        count = count + 1

        username = input("Please enter username:\n")
        url = "https://reddit.com/user/"+username+".json"

        user_data = None

        try:
            req = request.Request(url)
            req.add_header('User-Agent', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)')
            resp = request.urlopen(req)
            user_data = resp.read().decode("utf-8")
        except Exception as e:
                print(e)
                print("No such user.\nRetry Please.\n")
                count = count + 1
                raise # why raise? --> Program will end if user is not found

        if user_data:
              print(user_data)
              users_data.append(user_data)

    obtainKarma(users_data)


if __name__ == '__main__':
    getUserInfo()

There were still other issues with your code:

You should not write json.load(user_data), because you are parsing a string. So I changed it to use json.loads(user_data).

The Python documentation for json.loads states:

Deserialize s (a str instance containing a JSON document) to a Python object using this conversion table.

And in the code comment = posts[post_id]["num_comments"], I think you forgot to index on 'data', so I changed it to comment = posts[post_id]["data"]["num_comments"]

And why are you raising the exception in the except-block? This will end the program, however it seems that you expect it not to, from looking at the following code:

print("No such user.\nRetry Please.\n")