斗图啦

requests

BeautifulSoup4

代码

# -*- coding:utf-8 -*-
# pip install requests 框架
import requests
# pip install beautifulsoup4 框架
# pip install lxml 解析器
from bs4 import BeautifulSoup
import os

class doutuSpider(object):
    headers = {
        "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36"
    }

    def get_url(self, url):
        data = requests.get(url, headers=self.headers)
        soup = BeautifulSoup(data.content, 'lxml')
        totals = soup.findAll("a", {"class": "list-group-item"})
        for one in totals:
            sub_url = one.get('href')
            global path
            path = 'E:\\img' + '\\' + sub_url.split('/')[-1]
            os.mkdir(path)
            try:
                self.get_img_url(sub_url)
            except:
                pass
            pass
        pass

    def get_img_url(self, url):
        data = requests.get(url, headers = self.headers)
        soup = BeautifulSoup(data.content, 'lxml')
        totals = soup.findAll('div', {'class': 'artile_des'})
        for one in totals:
            img = one.find('img')
            try:
                sub_url = img.get('src')
            except Exception as e:
                raise e
            finally:
                urls = sub_url
            try:
                self.get_img(urls)
            except:
                print urls
                pass
            pass
        pass

    def get_img(self, url):
        filename = url.split('/')[-1]
        global path
        img_path = path + '\\' + filename
        img = requests.get(url, headers = self.headers)
        try:
            with open(img_path, 'wb') as f:
                f.write(img.content)
        except:
            pass
        pass

    def create(self):
        for count in range(1,10):
            url = 'https://www.doutula.com/article/list/?page={}'.format(count)
            print 'download {} page'.format(count)
            self.get_url(url)
            pass
        pass

if __name__ == '__main__':
    doutu = doutuSpider()
    doutu.create()