extracting tables using BeautifulSoup

Question

I want to extract all the tables from a html file as given below using BeautifulSoup and write it to csv.

Html looks like below:

        Site Name : Aria   
            
                
                    Dn Name:
                    Aria

                        WL:
                         Meters (m)

                
                
                    River Name:
                    Ben

                        DL:
                         Meters (m)

                
                
                    Basin Name:
                    GAN

                    HFL:
                    49.4 Meters (m)

                
                
                    Div Name:

                    LGD-I

                    HFL date:
                    14-08-2017

                
            
             
            
                
                    PRESENT WL
                

                                            

                    Date: 17-07-2018 12:00
                    Value: 45.43 Meters (m)
                    Trend: Steady
                
                
                    CUMULATIVE DAILY RF
                
                

                        Date: 17-07-2018 08:30
                        Value: 0.0 Milimiters (mm)
                        

                
                                        
                                        



                            
                                
                                    NO FORECAST

I amble to extract the text from all the three table but I am not able to write it in the desired format

My code

now = datetime.datetime.now()
date = now.strftime("%d-%m-%Y")
os.chdir(r'D:\shared')


soup = BeautifulSoup(response.text,"html5lib")

tables = soup.find_all("tr")
test =[]
for table in tables:
    test.append(table.get_text())

filename = 'Water'+'-'+str(date)+'.csv'
out = open(filename, mode='ab')
writer = csv.writer(out)
writer.writerow(data)
out.close()

In output csv first table is being written in 1st column 2nd table is being written in second table and third table is written in 3rd column.

I want data in the following format:

Site Name:  Aria
Dn Name:    Aria    
WL:         Meters (m)
River Name: Ben 
DL:         Meters (m)
Basin Name: GAN
HFL:        49.4 Meters (m)
Div Name:   LGD-I)
HFL date:   14-08-2017

PRESENT WL
Date:       17-07-2018 12:00    
Value:      45.43 Meters (m)    
Trend:      Steady
CUMULATIVE 
DAILY RF
Date:       17-07-2018 08:30    
Value:      0.0 Milimiters (mm) 
NO FORECAST

Andrej Kesely · Accepted Answer

My attempt to this problem:

data = """
        Site Name : Aria
            
                
                    Dn Name:
                    Aria

                        WL:
                         Meters (m)

                
                
                    River Name:
                    Ben

                        DL:
                         Meters (m)

                
                
                    Basin Name:
                    GAN

                    HFL:
                    49.4 Meters (m)

                
                
                    Div Name:

                    LGD-I

                    HFL date:
                    14-08-2017

                
            
             
            
                
                    PRESENT WL
                

                

                    Date: 17-07-2018 12:00
                    Value: 45.43 Meters (m)
                    Trend: Steady
                
                
                    CUMULATIVE DAILY RF
                
                

                        Date: 17-07-2018 08:30
                        Value: 0.0 Milimiters (mm)
                        

                
            
                 



                            
                                
                                    NO FORECAST
                                
                            
"""

import os
import datetime
from bs4 import BeautifulSoup
from pprint import pprint
# For Python 2.7 the next line should be "from itertools import izip_longest"
from itertools import zip_longest
import csv

now = datetime.datetime.now()
date = now.strftime("%d-%m-%Y")
# os.chdir(r'D:\shared')

soup = BeautifulSoup(data, "lxml")

tables = []
for table in soup.find_all('table'):
    current_table = []
    tables.append(current_table)
    for row in table.find_all("tr"):
        for (th, td) in zip_longest(row.find_all('th'), row.find_all('td')):
            s = ("%s %s" % (th.text.strip() if th else '', td.text.strip() if td else '')).strip()
            if s:
                current_table.append(s)

tables[0].insert(0, ': '.join(w.strip() for w in soup.find('h4').text.split(':')))

for table in tables:
    for i in table:
        print(i)

filename = 'CWC-Water'+'-'+str(date)+'.csv'
out = open(filename, mode='w')
writer = csv.writer(out)
for table in zip_longest(*tables):
    writer.writerow(table)
out.close()

Prints:

Site Name: Aria
Dn Name: Aria
WL: Meters (m)
River Name: Ben
DL: Meters (m)
Basin Name: GAN
HFL: 49.4 Meters (m)
Div Name: LGD-I
HFL date: 14-08-2017
PRESENT WL
Date: 17-07-2018 12:00
Value: 45.43 Meters (m)
Trend: Steady
CUMULATIVE DAILY RF
Date: 17-07-2018 08:30
Value: 0.0 Milimiters (mm)
NO FORECAST

And outputs .csv file with following format (3-columns from tables, screenshot from LibreOffice):

Edit: - correct picture

extracting tables using BeautifulSoup

Answers (1)

Related Questions