CODEWITHSUNDEEP
pythondatabasemongodbinsert
muntazir abbas
muntazir abbas

Reputation: 57

Cannot insert Data into MongoDb: TypeError: document must be an instance of dict, bson.son.SON, bson.raw_bson.RawBSONDocument,

cannot insert data into mongodb with the help of variable but if i print the output and paste it in the db.collection.insert_many(output) then code runs properly and data is stored but i directly use db.collection.insert_many(output_final) it gives me error. please help.

import re
import json
from selenium import webdriver
from bs4 import  BeautifulSoup
import requests
from fake_useragent import  UserAgent
import pymongo
myclient = pymongo.MongoClient("mongodb://localhost:27017/")
db = myclient["db_db"]


ua          = UserAgent()
header      = {'user-agent':ua.chrome}
driver      = webdriver.Chrome('C:/Users/MUNTAZIR/Downloads/Compressed/chromedriver_win32/chromedriver.exe')
driver.get('https://www.eduvision.edu.pk/scholarships/index.php?authority=1&level=4&field=1&cat=2&type=1')
# tr = driver.find_element_by_id('ctl00_ctl42_g_7f68baae_5353_4bdd_bfe1_b88e3367234f_csr1_table')
soup = BeautifulSoup(driver.page_source, 'lxml')
scholar = soup.findAll("div", {"class": "card-content col-xs-12"})
s_output1 = ""
for s in scholar[0:1]:
        title=s.findAll("h2")[0].text
        desc = s.findAll("div", {"class": "text"})[0].text.replace("\n", "").replace('"','')
        url = "https://www.eduvision.edu.pk/scholarships/" + s.a['href']
        type= "Higher Education Commission"
        # print(type +"\n" +title +"\n" +desc +"\n" +url +"\n")
        s_output1 = ("{""\n"
             '"type"' + ":" + '" ' + type + ' ",' + "\n"
             '"title"' + ":" + '" ' + title + ' ",' + "\n"
             '"url"' + ":" + '" ' + url + ' ",' + "\n"
             '"description"' + ":" + '" ' + desc + ' "' + "\n"
             "}""\n"
             )
s_output2_d = ""
for s in scholar[1:]:
        title=s.findAll("h2")[0].text
        desc=s.findAll("div",{"class": "text"})[0].text.replace("\n", "").replace('"','')
        url = "https://www.eduvision.edu.pk/scholarships/" + s.a['href']
        type= "Higher Education Commission"
        s_output2 = (",{""\n"
             '"type"' + ":" + '" ' + type + ' ",' + "\n"
             '"title"' + ":" + '" ' + title + ' ",' + "\n"
             '"url"' + ":" + '" ' + url + ' ",' + "\n"
             '"description"' + ":" + '" ' + desc + ' "' + "\n"
             "}""\n"
             )
        s_output2_d += s_output2

output_final = ""
output_final += s_output1 + s_output2_d
print(output_final)

db.collection2.insert_many(output_final)
print("saved")
driver.close()

Upvotes: 1

Views: 761

Answers (1)

Neil
Neil

Reputation: 3291

Mongo's insert_many takes an actual list of actual dictionaries, not a string which if "evaluated" would product a list of dictionaries. So, this is valid:

db.collection.insert_many([{'a':1, 'b':2}, {'a':3, 'b':4}]) # list of dictionaries

This is not valid:

db.collection.insert_many("[{'a':1, 'b':2}, {'a':3, 'b':4}]") # string

EDIT: how to make the string into a list? Don't make a string, make a list! For example (not tested):

list_of_s_output_1s = []
for s in scholar[0:1]:
        title=s.findAll("h2")[0].text
        description = s.findAll("div", {"class": "text"})[0].text.replace("\n", "").replace('"','')
        url = "https://www.eduvision.edu.pk/scholarships/" + s.a['href']
        type= "Higher Education Commission"
        # print(type +"\n" +title +"\n" +desc +"\n" +url +"\n")
        s_output_1 = {
            "type": type, 
            "title": title, 
            "url": url, 
            "description": description
        }
        list_of_s_output_1s.append(s_output_1)

list_of_s_s_output_2s = []
for s in scholar[1:]:
        title = s.findAll("h2")[0].text
        description = s.findAll("div",{"class": "text"})[0].text.replace("\n", "").replace('"','')
        url = "https://www.eduvision.edu.pk/scholarships/" + s.a['href']
        type= "Higher Education Commission"
        s_output_2 = {
             "type": type,
             "title": title,
             "url": url,
             "description": description
            }
        list_of_s_output_2s.append(s_output_2)

output_final = list_of_s_output_1s + list_of_s_output_2s

db.collection2.insert_many(output_final)
print("saved")
driver.close()

Upvotes: 1

Related Questions