CODEWITHSUNDEEP
rweb-scrapingrvest
udden2903
udden2903

Reputation: 783

Using rvest to extract both hypertext and hyperlink from a column in a table

I want to extract the hypertext and hyperlink from the column 'Name' in the following table: European Medicines Agency. My goal is to create a dataframe with one column for the name and another column for the link. Using the following code, I am able to collect the hyperlinks, but I am lost as to how I should match the links to the actual names? 

library(rvest)
library(dplyr)

page <- read_html('http://www.ema.europa.eu/ema/index.jsp?curl=pages/medicines/landing/smop_search.jsp&mid=WC0b01ac058001d127&startLetter=View%20all&applicationType=Initial%20authorisation&applicationType=Post%20authorisation&keyword=Enter%20keywords&keyword=Enter%20keywords&searchkwByEnter=false&searchType=Name&alreadyLoaded=true&status=Positive&status=Negative&jsenabled=false&orderBy=opinionDate&pageNo=1') %>%
  html_nodes('tbody a') %>% html_attr('href')

dfpage <- data.frame(page)

Upvotes: 0

Views: 420

Answers (2)

hrbrmstr
hrbrmstr

Reputation: 78792

library(rvest)
library(tidyverse)

url_template <- "http://www.ema.europa.eu/ema/index.jsp?searchType=Name&applicationType=Initial+authorisation&applicationType=Post+authorisation&searchkwByEnter=false&mid=WC0b01ac058001d127&status=Positive&status=Negative&keyword=Enter+keywords&keyword=Enter+keywords&alreadyLoaded=true&curl=pages%%2Fmedicines%%2Flanding%%2Fsmop_search.jsp&startLetter=View+all&pageNo=%s"

Get the total # of pages:

first <- sprintf(url_template, 1)

pg <- read_html(first)

html_nodes(pg, "div.pagination > ul > li:not([class])") %>%
  tail(1) %>%
  html_text(trim = TRUE) %>%
  as.numeric() -> total_pages

There are only 3 but cld be many in the future, so setup a progress bar to entertain you and scrape the table and then extract the links and add it to the table:

pb <- progress_estimated(total_pages)

sprintf(url_template, 1:total_pages) %>% 
  map_df(function(URL) {

    pb$tick()$print()

    pg <- read_html(URL)

    html_table(pg, trim = TRUE) %>%
      .[[1]] %>%
      set_names(c("name", "active_substance", "inn", "adopted", "outcome")) %>%
      as_tibble() %>%
      mutate(url = html_nodes(pg, "th[scope='row'] > a") %>% html_attr("href"))

  }) -> pending_df

glimpse(pending_df)
## Observations: 67
## Variables: 6
## $ name             <chr> "Lifmior", "Tamiflu", "Jylamvo", "Terrosa", "...
## $ active_substance <chr> "etanercept", "oseltamivir", "methotrexate", ...
## $ inn              <chr> "etanercept", "oseltamivir", "methotrexate", ...
## $ adopted          <chr> "2016-12-15", "2015-03-26", "2017-01-26", "20...
## $ outcome          <chr> "Positive", "Positive", "Positive", "Positive...
## $ url              <chr> "index.jsp?curl=pages/medicines/human/medicin...

Upvotes: 1

p0bs
p0bs

Reputation: 1034

I would use the following code:

library(rvest)
library(tidyverse)

page <- read_html('http://www.ema.europa.eu/ema/index.jsp?curl=pages/medicines/landing/smop_search.jsp&mid=WC0b01ac058001d127&startLetter=View%20all&applicationType=Initial%20authorisation&applicationType=Post%20authorisation&keyword=Enter%20keywords&keyword=Enter%20keywords&searchkwByEnter=false&searchType=Name&alreadyLoaded=true&status=Positive&status=Negative&jsenabled=false&orderBy=opinionDate&pageNo=1') %>%
  html_nodes("table") %>% 
  rvest::html_table()

data <- as_data_frame(page[[1]])

page_link <- read_html('http://www.ema.europa.eu/ema/index.jsp?curl=pages/medicines/landing/smop_search.jsp&mid=WC0b01ac058001d127&startLetter=View%20all&applicationType=Initial%20authorisation&applicationType=Post%20authorisation&keyword=Enter%20keywords&keyword=Enter%20keywords&searchkwByEnter=false&searchType=Name&alreadyLoaded=true&status=Positive&status=Negative&jsenabled=false&orderBy=opinionDate&pageNo=1') %>% 
  html_nodes(".key-detail a , .alt~ .alt th") %>% 
  html_attr('href')

link <- as_data_frame(page_link)
links <- as_data_frame(link$value[-1])

result <- cbind(data, links)

final <- result[, c("Name", "value")]

And the first row of final yields:

print(t(final[1,]))

...

      1                                                                                                                  
Name  "Natpar"                                                                                                           
value "index.jsp?curl=pages/medicines/human/medicines/003861/smops/Positive/human_smop_001096.jsp&mid=WC0b01ac058001d127"

I hope that helps. By the way, I got the correct tag by using the SelectorGadget add-in for Chrome.

Upvotes: 0

Related Questions