python scrapy框架爬取haozu 数据
1.创建项目 在控制台通过scrapy startproject 创建项目 我们通过scrapy startproject haozu 创建爬虫项目 2.创建爬虫文件 在控制台 进入spiders 文件夹下 通过scrapy genspider <网站域名> scrapy genspider haozu_xzl www.haozu.com 创建爬虫文件 3.在爬虫文件中 haozu_xzl.py写代码 python version=3.6.0 -- coding: utf-8 -- import scrapyimport requestsfrom lxml import htmletree =html.etreefrom ..items import HaozuItemimport random class HaozuXzlSpider(scrapy.Spider): # scrapy crawl haozu_xzl name = 'haozu_xzl' # allowed_domains = ['www.haozu.com/sz/zuxiezilou/'] start_urls = "http://www.haozu.com/sz/zuxiezilou/" province_list = ['bj', 'sh', 'gz', 'sz', 'cd', 'cq', 'cs','dl','fz','hz','hf','nj','jian','jn','km','nb','sy', 'su','sjz','tj','wh','wx','xa','zz'] def start_requests(self): user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2' headers = {'User-Agent': user_agent} for s in self.province_list: start_url = "http://www.haozu.com/{}/zuxiezilou/".format(s) # 包含yield语句的函数是一个生成器，每次产生一个值，函数被冻结，被唤醒后再次产生一个值 yield scrapy.Request(url=start_url, headers=headers, method='GET', callback=self.parse, \ meta={"headers": headers,"city":s}) def parse(self, response): lists = response.body.decode('utf-8') selector = etree.HTML(lists) elem_list = selector.xpath('/html/body/div[2]/div[2]/div/dl[1]/dd/div[2]/div[1]/a') print(elem_list,type(elem_list)) for elem in elem_list[1:-1]: try: district = str(elem.xpath("text()"))[1:-1].replace("'",'') # district.remove(district[0]) # district.pop() print(district,type(district)) district_href =str(elem.xpath("@href"))[1:-1].replace("'",'') # district_href.remove(district_href[0]) print(district_href,type(district_href)) elem_url ="http://www.haozu.com{}".format(district_href) print(elem_url) yield scrapy.Request(url=elem_url, headers=response.meta["headers"], method='GET', callback=self.detail_url, meta={"district": district,"url":elem_url,"headers":response.meta["headers"],"city":response.meta["city"]}) except Exception as e: print(e) pass def detail_url(self, response): print("===================================================================") for i in range(1,50): # 组建url re_url = "{}o{}/".format(response.meta["url"],i) print(re_url) try: response_elem = requests.get(re_url,headers=response.meta["headers"]) seles= etree.HTML(response_elem.content) sele_list = seles.xpath("/html/body/div[3]/div[1]/ul[1]/li") for sele in sele_list: href = str(sele.xpath("./div[2]/h1/a/@href"))[1:-1].replace("'",'') print(href) href_url = "http://www.haozu.com{}".format(href) print(href_url) yield scrapy.Request(url=href_url, headers=response.meta["headers"], method='GET', callback=self.final_url, meta={"district": response.meta["district"],"city":response.meta["city"]}) except Exception as e: print(e) pass def final_url(self,response): try: body = response.body.decode('utf-8') sele_body = etree.HTML(body) #获取价格 名称 地址 item = HaozuItem() item["city"]= response.meta["city"] item['district']=response.meta["district"] item['addr'] = str(sele_body.xpath("/html/body/div[2]/div[2]/div/div/div[2]/span[1]/text()[2]"))[1:-1].replace("'",'') item['title'] = str(sele_body.xpath("/html/body/div[2]/div[2]/div/div/div[1]/h1/span/text()"))[1:-1].replace("'",'') price = str(sele_body.xpath("/html/body/div[2]/div[3]/div[2]/div[1]/span/text()"))[1:-1].replace("'",'') price_danwei=str(sele_body.xpath("/html/body/div[2]/div[3]/div[2]/div[1]/div/div/i/text()"))[1:-1].replace("'",'') print(price+price_danwei) item['price']=price+price_danwei yield item except Exception as e: print(e) pass 4.修改items.py 文件 -- coding: utf-8 -- Define here the models for your scraped items See documentation in: https://doc.scrapy.org/en/latest/topics/items.html import scrapy class HaozuItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() city = scrapy.Field() district =scrapy.Field() title = scrapy.Field() addr =scrapy.Field() price = scrapy.Field() 5修改settings.py 打开ITEM_PIPELINES = { 'haozu.pipelines.HaozuPipeline': 300,} 6 修改pipelines.py文件 这里可以自定义存储文件格式 -- coding: utf-8 -- Define your item pipelines here Don't forget to add your pipeline to the ITEM_PIPELINES setting See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html import csv class HaozuPipeline(object): def process_item(self, item, spider): f = open('./xiezilou2.csv', 'a+',encoding='utf-8',newline='') write = csv.writer(f) write.writerow((item['city'],item['district'],item['addr'],item['title'],item['price'])) print(item) return item 7.启动框架 在控制台 输入 scrapy crawl haozu_xzl 启动程序
