Python爬虫实战示例-51job和豆瓣电影

2018年7月16日笔记

1.conda常用命令

1.1 列出当前环境的所有库

命令：conda list
在cmd中运行命令如下图所示：

1.2 管理环境

创建环境

命令:conda create -n {} python={}第一对大括号替换为环境的命名，第二对大括号替换为python的版本号
例如:conda create -n python27 python=2.7 这个命令就是创建一个python版本为2.7的环境，并命名为python27

列出所有环境

命令:conda info -e

进入环境

activate {}，大括号替换为虚拟环境名

环境添加库

conda install {}，大括号替换为要安装库的库名

环境删除库

conda remove {}，大括号替换为要安装库的库名

删除环境

conda remove -n {} -all，大括号替换为要删除库的库名

2. 爬虫示例

爬取豆瓣钱排名前250条信息,即下图这个网页的信息。

下面的sql语句用来创建数据库的表

drop database if exists douban; create database douban; use douban; DROP TABLE IF EXISTS `top250`; CREATE TABLE `top250` ( `director` varchar(100) DEFAULT NULL, `role` varchar(100) DEFAULT NULL, `year` varchar(100) DEFAULT NULL, `area` varchar(20) DEFAULT NULL, `genre` varchar(100) DEFAULT NULL, `title` varchar(255) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 

把豆瓣排名前250的电影信息导入mysql数据库中
下面一段代码能够成功运行的前提有两个：
1.安装库requests：pip install requests
安装库pymysql:pip install pymysql
2.修改下面代码中进入mysql数据库的用户名和密码，即修改下面这一句：
conn = pymysql.connect(host='localhost', user='root', passwd='...your password', db='douban',charset="utf8")

import requests from bs4 import BeautifulSoup as bs import pymysql if __name__ == "__main__": movieInfos = [] # 用于保存所有的电影信息 baseUrl = 'https://movie.douban.com/top250?start={}&filter=' for startIndex in range(0, 226, 25): url = baseUrl.format(startIndex) # 爬取网页 r = requests.get(url) # 获取html内容 htmlContent = r.text # 用BeautifulSoup加载html文本内容进行处理 soup = bs(htmlContent, "lxml") # 获取到页面中索引的class名为info的标签(应该有25个) movieList = soup.find_all("div", attrs={"class": "info"}) # 遍历25条电影信息 for movieItem in movieList: movieInfo = {} # 创建空字典,保存电影信息 # 获取到名为class名为hd的div标签内容 hd_div = movieItem.find("div", attrs={"class": "hd"}) # 通过bd_div获取到里面第一个span标签内容 hd_infos = hd_div.find("span").get_text().strip().split("\n") # < span class ="title" > 天堂电影院 < / span > movieInfo['title'] = hd_infos[0] # 获取到class名为bd的div标签内容 bd_div = movieItem.find("div", attrs={"class": "bd"}) # print(bd_div) # 通过bd_div获取到里面第一个p标签内容 infos = bd_div.find("p").get_text().strip().split("\n") # print(infos) #包含了两行电影信息的列表 # 获取导演和主演 infos_1 = infos[0].split("\xa0\xa0\xa0") if len(infos_1) == 2: # 获取导演,只获取排在第一位的导演名字 director = infos_1[0][4:].rstrip("...").split("/")[0] movieInfo['director'] = director # 获取主演 role = infos_1[1][4:].rstrip("...").rstrip("/").split("/")[0] movieInfo['role'] = role else: movieInfo['director'] = None movieInfo['role'] = None # 获取上映的时间/地区/电影类型 infos_2 = infos[1].lstrip().split("\xa0/\xa0") # 获取上映时间 year = infos_2[0] movieInfo['year'] = year # 获取电影地区 area = infos_2[1] movieInfo['area'] = area # 获取类型 genre = infos_2[2] movieInfo['genre'] = genre print(movieInfo) movieInfos.append(movieInfo) conn = pymysql.connect(host='localhost', user='root', passwd='...your password', db='douban',charset="utf8") # 获取游标对象 cursor = conn.cursor() # 查看结果 print('添加了{}条数据'.format(cursor.rowcount)) for movietiem in movieInfos: director = movietiem['director'] role = movietiem['role'] year = movietiem['year'] area = movietiem['area'] genre = movietiem['genre'] title = movietiem['title'] sql = 'INSERT INTO top250 values("%s","%s","%s","%s","%s","%s")' % (director, role, year, area, genre, title) # 执行sql cursor.execute(sql) # 提交 conn.commit() print('添加了{}条数据'.format(cursor.rowcount)) 

插入数据库成功截图如下：

2018年7月17日笔记

3.HTTP理解

3.1 HTTP请求格式

当浏览器向Web服务器发出请求时，它向服务器传递了一个数据块，也就是请求信息，HTTP请求信息由3部分组成：
1.请求方法URL协议/版本；2.请求头;3.请求体内容

3.2 HTTP请求方式

常见的http请求方式有get和post
Get是比较简单的http请求，直接会将发送给web服务器的数据放在请求地址的后面，即在请求地址后使用?key1=value1&ke2=value2形式传递数据，只适合数据量少，且没有安全性的请求
Post是需要发送给web服务器的数据经过编码放到请求体中，可以传递大量数据，并且有一定安全性，常用于表单提交

4.爬取51job网站信息

爬取51job网站信息并将数据持久化为excel文件

import requests from bs4 import BeautifulSoup as bs import re from urllib import parse import pandas as pd def cssFind(soup,cssSelector,nth=1): if len(soup.select(cssSelector)) >= nth: return soup.select(cssSelector)[nth-1].text else: return 0 def getSoup(url): response = requests.get(url) response.encoding = 'gbk' soup = bs(response.text,'lxml') return soup def getMaxPageNumber(url): soup = getSoup(url) maxPageNumberBefore = cssFind(soup,"span.td") pattern = "共(\d*)页" maxPageNumber = re.findall(pattern,maxPageNumberBefore)[0] return int(maxPageNumber) def getJobList(url): soup = getSoup(url) webpage_job_list = soup.select("div.dw_table div.el")[1:] job_list = [] for item in webpage_job_list: job = {} job['职位名'] = cssFind(item,"a").strip() job['公司名'] = cssFind(item,"span.t2") job['工作地点'] = cssFind(item,"span.t3") job['薪资'] = cssFind(item,"span.t4") job['发布时间'] = cssFind(item,"span.t5") job_list.append(job) return job_list def getUrl(job,page): url_before = "https://search.51job.com/list/020000,000000,0000,00,9,99,{},2," \ "{}.html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99&" \ "degreefrom=99&jobterm=99&companysize=99&lonlat=0%2C0&radius=-1&" \ "ord_field=0&confirmdate=9&dibiaoid=0&specialarea=00" url = url_before.format(parse.quote(job),page) return url if __name__ == "__main__": job = "人工智能" firstPage_url = getUrl(job,1) maxPageNumber = getMaxPageNumber(firstPage_url) job_list = [] for i in range(1,maxPageNumber+1): print("共有%d页，正在获取第%d页" %(maxPageNumber,i)) url = getUrl(job,i) job_list.extend(getJobList(url)) df = pd.DataFrame(job_list,columns=job_list[0].keys()) excel_name = "51job_{}.xlsx".format(job) df.to_excel(excel_name) print("finished!") 

5.爬取豆瓣排名前250电影信息

下面一段代码只需要修改连接mysql数据库的密码就可以运行。
sql语句写在代码中，所以代码比较长。

# coding=utf-8 from bs4 import BeautifulSoup as bs import requests import re import pymysql def cssFind(movie,cssSelector,nth=1): if len(movie.select(cssSelector)) >= nth: return movie.select(cssSelector)[nth-1].text.strip() else: return '' def reFind(pattern,sourceStr,nth=1): if len(re.findall(pattern,sourceStr)) >= nth: return re.findall(pattern,sourceStr)[nth-1] else: return '' def getConn(database ="pydb"): args = dict( host = 'localhost', user = 'root', passwd = '... your password', charset = 'utf8', db = database ) return pymysql.connect(**args) if __name__ == "__main__": #连接数据库 conn = getConn("douban") cursor = conn.cursor() #解析网页并将每条电影信息插入mysql数据库 url_before = "https://movie.douban.com/top250?start={}" flag = True for i in range(0,250,25): url = url_before.format(i) response = requests.get(url) response.encoding = 'utf-8' soup = bs(response.text, 'lxml') movie_list = soup.select("ol.grid_view li") for movie in movie_list: item = {} item['title_zh'] = cssFind(movie, "span.title") #提取标题 item['title2'] = cssFind(movie, "span.title", 2).lstrip('/').strip() #提取 item['title_other'] = cssFind(movie, "span.other").lstrip('/').strip() details = cssFind(movie, "div.bd p") pattern_director = "导演: (.*)主" item['director'] = reFind(pattern_director, details).strip('/...').strip() if item['director'] == "": item['director'] = reFind("导演: (.*)", details).strip('/...').strip() pattern_actor = "主演: (.*)" item['actor'] = reFind(pattern_actor, details).strip('/...').strip() detail2 = details.split('\n')[1] item['year'] = detail2.split('/')[0].strip() item['country'] = detail2.split('/')[1].strip() item['genre'] = detail2.split('/')[2].strip() item['rating_grade'] = cssFind(movie, "span.rating_num") item['rating_number'] = cssFind(movie, "div.star span", 4).rstrip("人评价") item['summary'] = cssFind(movie, "span.inq") if flag: drop_sql = "drop table if exists movie" cursor.execute(drop_sql) conn.commit() table_movie = ','.join(['`%s` varchar(200)'%key for key in item.keys()]) create_sql = "create table movie(`id` int primary key auto_increment,%s)" %table_movie cursor.execute(create_sql) conn.commit() flag = False table_field = ','.join(['`%s`'%key for key in item.keys()]) table_row = ','.join(['"%s"'%value for value in item.values()]) insert_sql = "insert into movie(%s) values(%s)"%(table_field, table_row) print(insert_sql) cursor.execute(insert_sql) conn.commit() conn.close()