How to scrape "Shadow Content" with BeautifulSoup

Question

I've been trying to web scrape a players Madden ratings with BeautifulSoup. I want to get the stats from this player. But, when I inspected the HTML I saw the ratings were in the "Shadow Content" part, and I can't seem to scrape that.

Answer 1

You are dealing with JavaScript website which render the inner HTML once the page fully loaded. requests will not be able to Render JS for you.

In that case you've to lookup for the API which feed the inner HTML, in your case is below:

import requests
from pprint import pp


def main(url):
    params = {
        'filter': 'iteration:* AND primaryKey:9925',
        'sort': 'iteration:Asc'
    }
    r = requests.get(url, params=params)
    pp(r.json()['docs'][0])


main('https://ratings-api.ea.com/v2/entities/m22-ratings')

Output:

{'college': 'Rutgers',
 'awareness_rating': 94,
 'throwPower_rating': 30,
 'kickReturn_rating': 74,
 'leadBlock_rating': 22,
 'strength_rating': 62,
 'bCVision_rating': 65,
 'catchInTraffic_rating': 45,
 'playAction_rating': 6,
 'pursuit_rating': 85,
 'plyrAssetname': 'McCourtyDevin_9925',
 'mediumRouteRunning_rating': 13,
 'catching_rating': 80,
 'acceleration_rating': 91,
 'spinMove_rating': 75,
 'height': 70,
 'finesseMoves_rating': 45,
 'spectacularCatch_rating': 61,
 'runBlock_rating': 35,
 'tackle_rating': 74,
 'injury_rating': 95,
 'zoneCoverage_rating': 92,
 'weight': 195,
 'plyrBirthdate': '8/13/87',
 'runningStyle_rating': 'Short Stride Loose',
 'deepRouteRunning_rating': 10,
 'firstName': 'Devin',
 'yearsPro': 11,
 'totalSalary': 10000000,
 'trucking_rating': 28,
 'throwAccuracyShort_rating': 6,
 'position': 'FS',
 'jukeMove_rating': 77,
 'playRecognition_rating': 90,
 'shortRouteRunning_rating': 15,
 'status': 'published',
 'lastName': 'McCourty',
 'jerseyNum': 32,
 'breakSack_rating': 14,
 'speed_rating': 91,
 'runBlockPower_rating': 35,
 'jumping_rating': 92,
 'toughness_rating': 77,
 'throwOnTheRun_rating': 6,
 'manCoverage_rating': 85,
 'stiffArm_rating': 25,
 'powerMoves_rating': 35,
 'iteration': 'launch-ratings',
 'release_rating': 12,
 'hitPower_rating': 67,
 'throwAccuracyMid_rating': 6,
 'kickAccuracy_rating': 24,
 'passBlockPower_rating': 35,
 'impactBlocking_rating': 67,
 'stamina_rating': 97,
 'carrying_rating': 50,
 'breakTackle_rating': 62,
 'plyrPortrait': 2043,
 'kickPower_rating': 22,
 'plyrHandedness': 'Right',
 'throwUnderPressure_rating': 23,
 'team': 'Patriots',
 'signingBonus': 6550000,
 'passBlock_rating': 35,
 'changeOfDirection_rating': 87,
 'press_rating': 86,
 'throwAccuracyDeep_rating': 6,
 'archetype': 'S_Zone',
 'blockShedding_rating': 64,
 'runBlockFinesse_rating': 35,
 'teamId': 22,
 'agility_rating': 89,
 'fullNameForSearch': 'Devin McCourty',
 'overall_rating': 92,
 'passBlockFinesse_rating': 35,
 'age': 34,
 'primaryKey': 9925}

Pickup whatever you need :).