R Web scraping data from links included in subpages

Question

I am trying to create a data frame getting data from pubmed website. I have a webpage containing links to subpages from all of which I would scrape some text but my code does not work and I cannot grab the abstract text I would. I searched on other subjects here but apparently cannot solve the issue. Here's my code, yet.

              library(xml2)
              library(rvest)  
              library(tibble)
              library(dplyr)
              library(tidyverse) 
              


 link <- "https://pubmed.ncbi.nlm.nih.gov/?term=((((((%E2%80%98Food%20Supply%E2%80%99%20(MeSH))%20OR%20%E2%80%98Food%20Storage%E2%80%99%20(MeSH))%20OR%20%E2%80%98Hunger%E2%80%99(MeSH)%20OR%20food%20security%20OR%20food%20insecurity%20OR%20household%20food%20security%20OR%20global%20food%20security)%20OR%20household%20food%20insecurity)))%20AND%20((%E2%80%98Prevalence%E2%80%99%20(MeSH))%20OR%20%E2%80%98Cross-Sectional%20Studies%E2%80%99%20(MeSH)%20OR%20cross-sectional%20study%20OR%20Prevalence%20Studies%20OR%20prevalence%20study%20OR%20Cross-Sectional%20Analyses%20OR%20CrossSectional%20Analysis%20OR%20Cross%20Sectional%20Analysis%20OR%20Cross%20Sectional%20Analyses)&filter=lang.english&filter=lang.portuguese"

 # I start building variables for a data frame.

 page <- read_html(link)

 name <- page %>% 
  html_nodes(".docsum-title") %>%
  html_text()

 name_links_synopsis <- page %>%          # This takes all the links tgo the subpages
  html_nodes(".docsum-title") %>%
  html_attr("href") %>% 
  paste("https://pubmed.ncbi.nlm.nih.gov", ., sep="")

 authors <- page %>% 
  html_nodes(".full-authors") %>%
  html_text()

 PMID <- page %>% 
  html_nodes(".docsum-pmid") %>%
  html_text()

synopsis <- page %>% 
  html_nodes(".full-view-snippet") %>%
  html_text()

  pubmed <- data.frame(name, authors, name_links_synopsis, PMID, synopsis, 
 stringsAsFactors = FALSE)

  # I create a function to scrape the text of the abstract in every subpage
  
  get_pubmed = function(pubmed_link) {
     pubmed_link = "https://pubmed.ncbi.nlm.nih.gov/?term=((((((%E2%80%98Food%20Supply%E2%80%99%20(MeSH))%20OR%20%E2%80%98Food%20Storage%E2%80%99%20(MeSH))%20OR%20%E2%80%98Hunger%E2%80%99(MeSH)%20OR%20food%20security%20OR%20food%20insecurity%20OR%20household%20food%20security%20OR%20global%20food%20security)%20OR%20household%20food%20insecurity)))%20AND%20((%E2%80%98Prevalence%E2%80%99%20(MeSH))%20OR%20%E2%80%98Cross-Sectional%20Studies%E2%80%99%20(MeSH)%20OR%20cross-sectional%20study%20OR%20Prevalence%20Studies%20OR%20prevalence%20study%20OR%20Cross-Sectional%20Analyses%20OR%20CrossSectional%20Analysis%20OR%20Cross%20Sectional%20Analysis%20OR%20Cross%20Sectional%20Analyses)&filter=lang.english&filter=lang.portuguese"
     pubmed_page = read_html(pubmed_link)
     pubmed_abs = pubmed_page %>% html_nodes(".docsum-title , .docsum-title b") %>%
       html_text()
     pubmed_abs_tot =  name_links_synopsis %>% html_nodes("#eng-abstract p") %>%
      html_text()
     return(pubmed_abs_tot)
         }

R Web scraping data from links included in subpages

Answers (1)

Related Questions