install.packages("rvest")

library(rvest)

library(rvest)

## 网页地址
urlb <- "http://www.sse.com.cn/market/sseindex/indexlist/s/i000001/const_list.shtml"

## 网页中数据表的xpath
xpath <- \'//*[@id="content_ab"]/div[1]/table\'

## 读入网页并提取其中的表格节点
nodes <- html_nodes(
  read_html(urlb), xpath=xpath)

## 从表格节点转换为表格列表
tables <- html_table(nodes)
restab <- tables[[1]]
head(restab)
##                                  X1                               X2
## 1 浦发银行\r\n            (600000) 白云机场\r\n            (600004)
## 2 中国国贸\r\n            (600007) 首创股份\r\n            (600008)
##                                 X3
## 1 东风汽车\r\n            (600006)
## 2 上海机场\r\n            (600009)

library(tidyverse)

pat1 <- "^(.*?)\\((.*?)\\)"
tab1 <- restab %>%
  ## 将三列合并为一列，结果为字符型向量
  reduce(c) %>% 
  ## 去掉空格和换行符，结果为字符型向量
  stringr::str_replace_all("[[:space:]]", "") %>%
  ## 提取公司简称和代码到一个矩阵行，结果为字符型矩阵
  stringr::str_match(pat1) 
tab <- tibble(
  name = tab1[,2],
  code = tab1[,3])
head(tab)
## # A tibble: 6 x 2
##   name     code  
##   <chr>    <chr> 
## 1 浦发银行 600000
## 2 中国国贸 600007
## 3 包钢股份 600010
## 4 华夏银行 600015
## 5 上港集团 600018
## 6 上海电力 600021

str(tab)
## Classes ‘tbl_df’, ‘tbl’ and \'data.frame\':    1551 obs. of  2 variables:
## $ name: chr  "浦发银行" "中国国贸" "包钢股份" "华夏银行" ...
##  $ code: chr  "600000" "600007" "600010" "600015" ...

# 加载包
library(\'rvest\')

# 指定要爬取的url
url <- \'https://www.imdb.com/search/title/?count=100&release_date=2016,2016&title_type=feature\'

# 从网页读取html代码
webpage <- read_html(url)

# 用CSS选择器获取排名部分
rank_data_html <- html_nodes(webpage,\'.text-primary\')

# 把排名转换为文本
rank_data <- html_text(rank_data_html)

# 检查一下数据
head(rank_data)

[1] "1." "2." "3." "4." "5." "6."

# 数据预处理：把排名转换为数值型
rank_data<-as.numeric(rank_data)

# 再检查一遍
head(rank_data)

[1] 1 2 3 4 5 6

# 爬取标题
title_data_html <- html_nodes(webpage,\'.lister-item-header a\')

# 转换为文本
title_data <- html_text(title_data_html)

# 检查一下
head(title_data)

[1] "Sing"          "Moana"         "Moonlight"     "Hacksaw Ridge"
[5] "Passengers"    "Trolls"

# 爬取描述
description_data_html <- html_nodes(webpage,\'.ratings-bar+ .text-muted\')

# 转为文本
description_data <- html_text(description_data_html)

# 检查一下
head(description_data)

[1] "\nIn a city of humanoid animals, a hustling theater impresario\'s attempt to save his theater with a singing competition becomes grander than he anticipates even as its finalists\' find that their lives will never be the same."

[2] "\nIn Ancient Polynesia, when a terrible curse incurred by the Demigod Maui reaches an impetuous Chieftain\'s daughter\'s island, she answers the Ocean\'s call to seek out the Demigod to set things right."

[3] "\nA chronicle of the childhood, adolescence and burgeoning adulthood of a young, African-American, gay man growing up in a rough neighborhood of Miami."

[4] "\nWWII American Army Medic Desmond T. Doss, who served during the Battle of Okinawa, refuses to kill people, and becomes the first man in American history to receive the Medal of Honor without firing a shot."

[5] "\nA spacecraft traveling to a distant colony planet and transporting thousands of people has a malfunction in its sleep chambers. As a result, two passengers are awakened 90 years early."

[6] "\nAfter the Bergens invade Troll Village, Poppy, the happiest Troll ever born, and the curmudgeonly Branch set off on a journey to rescue her friends.

# 移除 \'\n\'
description_data<-gsub("\n","",description_data)

# 再检查一下
head(description_data)

[1] "In a city of humanoid animals, a hustling theater impresario\'s attempt to save his theater with a singing competition becomes grander than he anticipates even as its finalists\' find that their lives will never be the same."

[2] "In Ancient Polynesia, when a terrible curse incurred by the Demigod Maui reaches an impetuous Chieftain\'s daughter\'s island, she answers the Ocean\'s call to seek out the Demigod to set things right."

[3] "A chronicle of the childhood, adolescence and burgeoning adulthood of a young, African-American, gay man growing up in a rough neighborhood of Miami."

[4] "WWII American Army Medic Desmond T. Doss, who served during the Battle of Okinawa, refuses to kill people, and becomes the first man in American history to receive the Medal of Honor without firing a shot."

[5] "A spacecraft traveling to a distant colony planet and transporting thousands of people has a malfunction in its sleep chambers. As a result, two passengers are awakened 90 years early."

[6] "After the Bergens invade Troll Village, Poppy, the happiest Troll ever born, and the curmudgeonly Branch set off on a journey to rescue her friends."

# 爬取runtime section
runtime_data_html <- html_nodes(webpage,\'.text-muted .runtime\')

# 转为文本
runtime_data <- html_text(runtime_data_html)

# 检查一下
head(runtime_data)

[1] "108 min" "107 min" "111 min" "139 min" "116 min" "92 min"

# 数据预处理: 去除“min”并把数字转换为数值型

runtime_data <- gsub(" min","",runtime_data)
runtime_data <- as.numeric(runtime_data)

# 再检查一下
head(rank_data)

[1] 1 2 3 4 5 6

# 爬取genre
genre_data_html <- html_nodes(webpage,\'.genre\')

# 转为文本
genre_data <- html_text(genre_data_html)

# 检查一下
head(genre_data)

[1] "\nAnimation, Comedy, Family "

[2] "\nAnimation, Adventure, Comedy "

[3] "\nDrama "

[4] "\nBiography, Drama, History "

[5] "\nAdventure, Drama, Romance "

[6] "\nAnimation, Adventure, Comedy "

# 去除“\n”
genre_data<-gsub("\n","",genre_data)

# 去除多余空格
genre_data<-gsub(" ","",genre_data)

# 每部电影只保留第一种类型
genre_data<-gsub(",.*","",genre_data)

# 转化为因子
genre_data<-as.factor(genre_data)

# 再检查一下
head(genre_data)

[1] Animation Animation Drama     Biography Adventure Animation

# 爬取IMDB rating
rating_data_html <- html_nodes(webpage,\'.ratings-imdb-rating strong\')

# 转为文本
rating_data <- html_text(rating_data_html)

# 检查一下
head(rating_data)

[1] "7.2" "7.7" "7.6" "8.2" "7.0" "6.5"

# 转为数值型
rating_data<-as.numeric(rating_data)

# 再检查一下
head(rating_data)

[1] 7.2 7.7 7.6 8.2 7.0 6.5

# 爬取votes section
votes_data_html <- html_nodes(webpage,\'.sort-num_votes-visible span:nth-child(2)\')

# 转为文本
votes_data <- html_text(votes_data_html)

# 检查一下
head(votes_data)

[1] "40,603"  "91,333"  "112,609" "177,229" "148,467" "32,497"

# 移除“，”
votes_data<-gsub(",", "", votes_data)

# 转为数值型
votes_data<-as.numeric(votes_data)

# 再检查一下
head(votes_data)

[1]  40603  91333 112609 177229 148467  32497

# 爬取directors section
directors_data_html <- html_nodes(webpage,\'.text-muted+ p a:nth-child(1)\')

# 转为文本
directors_data <- html_text(directors_data_html)

# 检查一下
head(directors_data)

[1] "Christophe Lourdelet" "Ron Clements"         "Barry Jenkins"
[4] "Mel Gibson"           "Morten Tyldum"        "Walt Dohrn"

# 转为因子
directors_data<-as.factor(directors_data)

# 爬取actors section
actors_data_html <- html_nodes(webpage,\'.lister-item-content .ghost+ a\')

# 转为文本
actors_data <- html_text(actors_data_html)

# 检查一下
head(actors_data)

[1] "Matthew McConaughey" "Auli\'i Cravalho"     "Mahershala Ali"
[4] "Andrew Garfield"     "Jennifer Lawrence"   "Anna Kendrick"

# 转为因子
actors_data<-as.factor(actors_data)

# 爬取metascore section
metascore_data_html <- html_nodes(webpage,\'.metascore\')

# 转为文本
metascore_data <- html_text(metascore_data_html)

# 检查一下
head(metascore_data)

[1] "59        " "81        " "99        " "71        " "41        "
[6] "56        "

# 去除多余空格
metascore_data<-gsub(" ","",metascore_data)

# 检查metascore data的长度
length(metascore_data)

[1] 96

library(\'ggplot2\')
qplot(data = movies_df,Runtime,fill = Genre,bins = 30)

ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))

ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))