webscraping with beautiful soup

Question

I'm trying to scrape the table from a website that contains the Current interests rates. I used python with beautiful soup but I can't locate the html parts. Please send help ! thank you.

I only need to scrape the current interest rates table, not everything else and convert it into csv file. Here is the link to my webiste: https://www.global-rates.com/en/interest-rates/libor/american-dollar/usd-libor-interest-rate-12-months.aspx here is the picture of the current interests rate table:

I tried something like this:

import bs4 
import requests
from bs4 import BeautifulSoup
import pandas as pd 

URL = 'https://www.global-rates.com/en/interest-rates/libor/american-dollar/usd-libor-interest-rate-12-months.aspx' 

response = requests.get(URL)
soup=bs4.BeautifulSoup(response.content, 'html.parser')

print(soup.title)
print(soup.title.string)
print(len(response.text))

table = soup.find('table', attrs = {'class':'tableheader'}).tbody
print(table)

columns = ['Current interest rates']
df = pd.DataFrame(columns = columns)

trs = table.find_all('tr')
for tr in trs:
    tds = tr.find_all('td')
    row = [td.text.replace('\n', '') for td in tds]
    df = df.append(pd.Series(row, index = columns), ignore_index = True)
df.to_csv('libor.csv', index = False)

but this gave me attribute errors: "None Type' object has no attribute 'tbody'

oh I also want to make automatically scraping the Mondays' interests rate if that's possible. Thank you for your help

Answer 1

Here is my attempt with just pandas

import pandas as pd

# Get all tables on page
dfs = pd.read_html('https://www.global-rates.com/en/interest-rates/libor/american-dollar/usd-libor-interest-rate-12-months.aspx')

# Find the Current interest rates table
df = [df for df in dfs if df.iloc[0][0] == 'Current interest rates'][0]

# Remove first row that contains column names
df = df.iloc[1:].copy()

# Set column names
df.columns = ['DATE','INTEREST_RATE']

# Convert date from november 02 2020 to 2020-11-02
df['DATE'] = pd.to_datetime(df['DATE'])

# Remove percentage sign from interest rate
df['INTEREST_RATE'] = df['INTEREST_RATE'].str.replace('%','').str.strip()

# Convert percentage to float type
df['INTEREST_RATE'] = df['INTEREST_RATE'].astype(float)

# Add day of the week column
df['DAY'] = df['DATE'].dt.day_name()

# Output all to CSV
df.to_csv('all_data.csv', index=False)

# Only Mondays
df_monday = df[df['DAY'] == 'Monday']

# Output only Mondays
df_monday.to_csv('monday_data.csv', index=False)

# Add day number of week (Monday = 0)
df['DAY_OF_WEEK_NUMBER'] = df['DATE'].dt.dayofweek

# Add week number of year
df['WEEK_OF_YEAR_NUMBER'] = df['DATE'].dt.weekofyear

# 1. Sort by week of year then day of week
# 2. Group by week of year
# 3. Select first record in group, which will be the earliest day available of that week
df_first_day_of_week = df.sort_values(['WEEK_OF_YEAR_NUMBER','DAY_OF_WEEK_NUMBER']).groupby('WEEK_OF_YEAR_NUMBER').first()

# # Output earliest day of the week data
df_first_day_of_week.to_csv('first_day_of_week.csv', index=False)

Answer 2

You can use this example to scrape the "Current interest rates":

import requests
import pandas as pd
from bs4 import BeautifulSoup


url = 'https://www.global-rates.com/en/interest-rates/libor/american-dollar/usd-libor-interest-rate-12-months.aspx'
soup = BeautifulSoup(requests.get(url).content, 'html.parser')

all_data = []
for row in soup.select('table:has(td:contains("Current interest rates"))[style="width:208px;border:1px solid #CCCCCC;"] tr:not(:has([colspan]))'):
    tds = [td.get_text(strip=True) for td in row.select('td')]
    all_data.append(tds)

df = pd.DataFrame(all_data, columns=['Date', 'Rate'])
print(df)
df.to_csv('data.csv', index=False)

Prints:

                Date       Rate
0   november 02 2020  0.33238 %
1    october 30 2020  0.33013 %
2    october 29 2020  0.33100 %
3    october 28 2020  0.32763 %
4    october 27 2020  0.33175 %
5    october 26 2020  0.33200 %
6    october 23 2020  0.33663 %
7    october 22 2020  0.33513 %
8    october 21 2020  0.33488 %
9    october 20 2020  0.33713 %
10   october 19 2020  0.33975 %
11   october 16 2020  0.33500 %

And saves data.csv:

---

EDIT: To get only mondays, you can do this with the dataframe:

df['Date'] = pd.to_datetime(df['Date'])
print(df[df['Date'].dt.weekday==0])

Prints:

         Date       Rate
0  2020-11-02  0.33238 %
5  2020-10-26  0.33200 %
10 2020-10-19  0.33975 %