【Python】从0开始写爬虫——豆瓣电影

from scrapy import app import re header = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36', 'Host': 'movie.douban.com', 'Accept-Language': 'zh-CN,zh;q=0.9' } movie_url = "https://movie.douban.com/subject/27132728/?tag=%E7%83%AD%E9%97%A8&from=gaia" m_id = re.search("[0-9]+", movie_url).group() # 获取soup对象 soup = app.get_soup(url=movie_url, headers=header, charset="utf-8") content = soup.find(id="content") # 抓取电影名字和上映年份 m_name = content.find("h1").find("span").string m_year = content.find(class_="year").string m_year = m_year[1:-1] # 去掉头尾的括号 # 抓取导演 info = content.find(id="info") m_directer = info.find(attrs={"rel": "v:directedBy"}).string# 上映日期 m_date = info.find(attrs={"property": "v:initialReleaseDate"}).string m_date = re.search("[-,0-9]+", m_date).group() # 类型 types = info.find_all(attrs={"property": "v:genre"}, limit=2) m_types = [] for type_ in types: m_types.append(type_.string) # 抓取主演，只取前面五个 actors = info.find(class_="actor").find_all(attrs={"rel": "v:starring"}, limit=5) m_actors = [] for actor in actors: m_actors.append(actor.string) # 片长 m_time = info.find(attrs={"property": "v:runtime"}).string m_time = re.search("[0-9]+", m_time).group() # 评分 score_info = soup.find(id="interest_sectl") m_score = score_info.find("strong").string m_stars = score_info.find_all(class_="rating_per") m_rate = [] for star in m_stars: m_rate.append(star.string[:-1]) m_votes = score_info.find(attrs={"property": "v:votes"}).string print("id", m_id, "名称", m_name, "年份 ", m_year, "导演 ", m_directer, "主演", m_actors) print("上映日期", m_date, "类型", m_types, "片长", m_time) print("评分", m_score, "星评（从5星到1星）", m_rate, "评价总数", m_votes)

id 27132728 名称 致命警告 El aviso 年份 2018 导演 丹尼尔·卡尔帕索罗 主演 ['奥拉·加里多', '劳尔·阿雷瓦洛', '贝伦·奎斯塔', 'Sergio Mur', 'Aitor Luna'] 上映日期 2018-03-23 类型 ['惊悚'] 片长 92 评分 6.4 星评（从5星到1星） ['14.2', '19.9', '41.1', '21.3', '3.5'] 评价总数 445

from scrapy import app import re def douban_movie(movie_url): header = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36', 'Host': 'movie.douban.com', 'Accept-Language': 'zh-CN,zh;q=0.9' } m_id = re.search("[0-9]+", movie_url).group() # 获取soup对象 soup = app.get_soup(url=movie_url, headers=header, charset="utf-8") content = soup.find(id="content") # 抓取电影名字和上映年份 m_name = content.find("h1").find("span").string m_year = content.find(class_="year").string m_year = m_year[1:-1] # 去掉头尾的括号 # 抓取导演 info = content.find(id="info") m_directer = info.find(attrs={"rel": "v:directedBy"}).string # 上映日期 m_date = info.find(attrs={"property": "v:initialReleaseDate"}).string m_date = re.search("[-,0-9]+", m_date).group() # 类型 types = info.find_all(attrs={"property": "v:genre"}, limit=2) m_types = [] for type_ in types: m_types.append(type_.string) # 抓取主演，只取前面五个 actors = info.find(class_="actor").find_all(attrs={"rel": "v:starring"}, limit=5) m_actors = [] for actor in actors: m_actors.append(actor.string) # 片长 m_time = info.find(attrs={"property": "v:runtime"}) m_time = re.search("[0-9]+", m_time.string).group() if m_time else 0 # 评分 score_info = soup.find(id="interest_sectl") m_score = score_info.find("strong").string m_stars = score_info.find_all(class_="rating_per") m_rate = [] for star in m_stars: m_rate.append(star.string[:-1]) m_votes = score_info.find(attrs={"property": "v:votes"}) m_votes = m_votes.string if m_votes else 0 print("id", m_id, "名称", m_name, "年份 ", m_year, "导演 ", m_directer, "主演", m_actors) print("上映日期", m_date, "类型", m_types, "片长", m_time) print("评分", m_score, "星评（从5星到1星）", m_rate, "评价总数", m_votes) douban_movie("https://movie.douban.com/subject/30236775/?from=showing") douban_movie("https://movie.douban.com/subject/26842702/?tag=%E7%83%AD%E9%97%A8&from=gaia") douban_movie("https://movie.douban.com/subject/26973784/?tag=%E6%9C%80%E6%96%B0&from=gaia") douban_movie("https://movie.douban.com/subject/30249296/?tag=%E7%83%AD%E9%97%A8&from=gaia")

id 30236775 名称 旅行吧！井底之蛙 年份 2018 导演 陈设 主演 ['王雪沁', '吴凡', '周宗禹', '强光宗', '张艺文'] 上映日期 2018-08-18 类型 ['动画', '奇幻'] 片长 78 评分 None 星评（从5星到1星） [] 评价总数 0 id 26842702 名称 燃烧 버닝 年份 2018 导演 李沧东 主演 ['刘亚仁', '史蒂文·元', '全钟瑞', '金秀京', '崔承浩'] 上映日期 2018-05-16 类型 ['剧情', '悬疑'] 片长 148 评分 7.9 星评（从5星到1星） ['25.9', '48.1', '21.4', '3.2', '1.3'] 评价总数 81519 id 26973784 名称 设局 La niebla y la doncella 年份 2017 导演 安德烈斯·M·科佩尔 主演 ['基姆·古铁雷斯', '薇洛妮卡·恩切圭', '奥拉·加里多', '罗伯托·阿拉莫'] 上映日期 2017-09-08 类型 ['惊悚'] 片长 0 评分 5.2 星评（从5星到1星） ['1.2', '9.8', '42.0', '40.0', '7.1'] 评价总数 320 id 30249296 名称 我们的侣行 第二季 年份 2018 导演 张昕宇 主演 ['张昕宇', '梁红'] 上映日期 2018-06-12 类型 ['真人秀'] 片长 0 评分 8.9 星评（从5星到1星） ['59.3', '27.8', '11.1', '1.9', '0.0'] 评价总数 342

from scrapy import app import re import datetime def douban_movie(movie_url): header = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36', 'Host': 'movie.douban.com', 'Accept-Language': 'zh-CN,zh;q=0.9' } movie = {} m_id = re.search("[0-9]+", movie_url).group() movie["id"] = m_id # 获取soup对象 soup = app.get_soup(url=movie_url, headers=header, charset="utf-8") content = soup.find(id="content") kinds = content.find(class_="episode_list") movie["type"] = 0 if kinds else 1 # 0 ：电视剧，1：电影。如果有分集数，为电视剧， 否则就是电影 # 抓取电影名字和上映年份 m_name = content.find("h1").find("span").string movie["name"] = m_name; m_year = content.find(class_="year").string m_year = m_year[1:-1] # 去掉头尾的括号 movie["year"] = m_year # 抓取导演 info = content.find(id="info") m_directer = info.find(attrs={"rel": "v:directedBy"}).string movie["directer"] = m_directer # 上映日期 m_date = info.find(attrs={"property": "v:initialReleaseDate"}).string m_date = re.search("[-,0-9]+", m_date).group() movie["date"] = datetime.datetime.strptime(m_date, '%Y-%m-%d') # 类型 categories = info.find_all(attrs={"property": "v:genre"}, limit=2) m_categories = [] for category in categories: m_categories.append(category.string) movie["categories"] = m_categories # 抓取主演，只取前面五个 actors = info.find(class_="actor").find_all(attrs={"rel": "v:starring"}, limit=5) m_actors = [] for actor in actors: m_actors.append(actor.string) movie["actors"] = m_actors # 片长 m_time = info.find(attrs={"property": "v:runtime"}) m_time = int(re.search("[0-9]+", m_time.string).group()) if m_time else 0 movie["time"] = m_time # 评分 score_info = soup.find(id="interest_sectl") m_score = score_info.find("strong").string if score_info.find("strong") else 0.0 movie["score"] = float(m_score) if m_score else 0.0 m_stars = score_info.find_all(class_="rating_per") m_rate = [] for star in m_stars: m_rate.append(float(star.string[:-1])) movie["stars"] = m_rate m_votes = score_info.find(attrs={"property": "v:votes"}) m_votes = int(m_votes.string) if m_votes else 0 movie["vote"] = m_votes return movie print(douban_movie("https://movie.douban.com/subject/30236775/?from=showing")) print(douban_movie("https://movie.douban.com/subject/26842702/?tag=%E7%83%AD%E9%97%A8&from=gaia")) print(douban_movie("https://movie.douban.com/subject/26973784/?tag=%E6%9C%80%E6%96%B0&from=gaia")) print(douban_movie("https://movie.douban.com/subject/30249296/?tag=%E7%83%AD%E9%97%A8&from=gaia"))

{'id': '30236775', 'type': 1, 'name': '旅行吧！井底之蛙', 'year': '2018', 'directer': '陈设', 'date': datetime.datetime(2018, 8, 18, 0, 0), 'categories': ['动画', '奇幻'], 'actors': ['王雪沁', '吴凡', '周宗禹', '强光宗', '张艺文'], 'time': 78, 'score': 0.0, 'stars': [], 'vote': 0} {'id': '26842702', 'type': 1, 'name': '燃烧 버닝', 'year': '2018', 'directer': '李沧东', 'date': datetime.datetime(2018, 5, 16, 0, 0), 'categories': ['剧情', '悬疑'], 'actors': ['刘亚仁', '史蒂文·元', '全钟瑞', '金秀京', '崔承浩'], 'time': 148, 'score': 7.9, 'stars': [25.9, 48.1, 21.5, 3.2, 1.3], 'vote': 83194} {'id': '26973784', 'type': 1, 'name': '设局 La niebla y la doncella', 'year': '2017', 'directer': '安德烈斯·M·科佩尔', 'date': datetime.datetime(2017, 9, 8, 0, 0), 'categories': ['惊悚'], 'actors': ['基姆·古铁雷斯', '薇洛妮卡·恩切圭', '奥拉·加里多', '罗伯托·阿拉莫'], 'time': 0, 'score': 5.2, 'stars': [1.2, 9.8, 41.8, 40.2, 7.0], 'vote': 321} {'id': '30249296', 'type': 0, 'name': '我们的侣行 第二季', 'year': '2018', 'directer': '张昕宇', 'date': datetime.datetime(2018, 6, 12, 0, 0), 'categories': ['真人秀'], 'actors': ['张昕宇', '梁红'], 'time': 0, 'score': 8.9, 'stars': [59.3, 28.8, 10.2, 1.7, 0.0], 'vote': 400}