python之Scrapy爬虫案例：豆瓣

1. 运行命令创建项目：scrapy startproject scrapySpider
2. 进入项目目录：cd .\scrapySpider\
3. 运行命令创建爬虫：scrapy genspider douban movie.douban.com
4. 目录结构说明

   |-- scrapySpider 项目目录
   |   |-- scrapySpider 项目目录
   |   |   |-- spiders 爬虫文件目录
   |   |   |   |-- douban.py 爬虫文件
   |   |   |-- items.py 定义数据模型文件，类似于数据库表的模式或数据结构的定义
   |   |   |-- middlewares.py 定义中间件文件，用于对请求和响应进行处理和修改
   |   |   |-- pipelines.py 定义数据处理管道（Pipeline）文件，用于处理爬取到的数据的组件
   |   |   |-- settings.py 定义配置文件，用于配置 Scrapy 项目的各种设置选项和参数
   |   |-- scrapy.cfg 框架中的配置文件，用于指定项目的结构和元数据信息
   

5. 创建快代理文件scrapySpider>kuaidaili.py：https://www.kuaidaili.com/

   import requestsclass Kuaidaili():request_url = {# 获取代理ip前面'getIpSignature': 'https://auth.kdlapi.com/api/get_secret_token',# 获取代理ip'getIp': 'https://dps.kdlapi.com/api/getdps?secret_id=oy2q5xu76k4s8olx59et&num=1&signature={}'}headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36'}ip_use = '购买代理用户名'ip_password = '购买代理密码'def __init__(self):'''创建request会话对象'''self.request_session = requests.Session()self.request_session.headers.update(self.headers)# 获取代理ip签名@classmethoddef get_ip_url(cls):par = {'secret_id': 'oy2q5xu76k4s8olx59et','secret_key': '5xg6gvouc0vszfw0kxs1a8vrw1r6ity7'}response = requests.post(cls.request_url['getIpSignature'],data=par)response_data = response.json()return cls.request_url['getIp'].format(response_data['data']['secret_token'])@classmethoddef get_ip(cls):url = cls.get_ip_url()response = requests.get(url)return f'http://{cls.ip_use}:{cls.ip_password}@{response.text}/'if __name__ == '__main__':kuaidaili = Kuaidaili()print(kuaidaili.get_ip())
   

6. 爬取豆瓣案例

• douban.py

  import scrapy
  from scrapy import cmdline
  from scrapy.http import HtmlResponse,Request
  from scrapySpider.items import DoubanItemclass DoubanSpider(scrapy.Spider):name = 'douban'allowed_domains = ['movie.douban.com']start_urls = ['https://movie.douban.com/top250']def parse(self, response: HtmlResponse,**kwargs):video_list = response.xpath('//ol[@class="grid_view"]/li')for li in video_list:item = DoubanItem()item['title'] = li.xpath('.//div[@class="hd"]/a/span[1]/text()').extract_first()item['rating'] = li.xpath('.//div[@class="bd"]//span[@class="rating_num"]/text()').extract_first()item['quote'] = li.xpath('.//div[@class="bd"]//p[@class="quote"]/span/text()').extract_first()detail_url = li.xpath('.//div[@class="hd"]/a/@href').extract_first()yield Request(url=detail_url,callback=self.get_detail_info,meta={'item':item})#获取下一页数据next_page_url = response.xpath('//div[@class="paginator"]//link[@rel="next"]/@href').extract_first()if next_page_url:yield Request(url=response.urljoin(next_page_url),callback=self.parse)#重写start_requests获取多页数据# def start_requests(self):#     for i in range(0,2):#         yield Request(url=f'{self.start_urls[0]}?start={i*25}&filter=',dont_filter=True,callback=self.parse)def get_detail_info(self,response:HtmlResponse):item = response.meta['item']detail = response.xpath('//span[@class="all hidden"]/text()').extract_first()if not detail:detail = response.xpath('//div[@id="link-report-intra"]/span[1]/text()').extract_first()item['intro'] = detail.strip()request itemif __name__ == '__main__':cmdline.execute('scrapy crawl douban'.split())
  

• settings.py

  # Scrapy settings for scrapySpider project
  #
  # For simplicity, this file contains only settings considered important or
  # commonly used. You can find more settings consulting the documentation:
  #
  #     https://docs.scrapy.org/en/latest/topics/settings.html
  #     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
  #     https://docs.scrapy.org/en/latest/topics/spider-middleware.htmlBOT_NAME = "scrapySpider"SPIDER_MODULES = ["scrapySpider.spiders"]
  NEWSPIDER_MODULE = "scrapySpider.spiders"# Crawl responsibly by identifying yourself (and your website) on the user-agent
  # USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"# Obey robots.txt rules
  ROBOTSTXT_OBEY = False# Configure maximum concurrent requests performed by Scrapy (default: 16)
  #CONCURRENT_REQUESTS = 32# Configure a delay for requests for the same website (default: 0)
  # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
  # See also autothrottle settings and docs
  #DOWNLOAD_DELAY = 3
  # The download delay setting will honor only one of:
  #CONCURRENT_REQUESTS_PER_DOMAIN = 16
  #CONCURRENT_REQUESTS_PER_IP = 16# Disable cookies (enabled by default)
  #COOKIES_ENABLED = False# Disable Telnet Console (enabled by default)
  #TELNETCONSOLE_ENABLED = False# Override the default request headers:
  DEFAULT_REQUEST_HEADERS = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","Accept-Language": "en",
  }# Enable or disable spider middlewares
  # See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
  #SPIDER_MIDDLEWARES = {
  #    "scrapySpider.middlewares.ScrapyspiderSpiderMiddleware": 543,
  #}# Enable or disable downloader middlewares
  # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
  DOWNLOADER_MIDDLEWARES = {"scrapySpider.middlewares.DoubanDownloaderMiddleware": 543,
  }# Enable or disable extensions
  # See https://docs.scrapy.org/en/latest/topics/extensions.html
  # EXTENSIONS = {
  #    'scrapeops_scrapy.extension.ScrapeOpsMonitor': 500,
  # }# Configure item pipelines
  # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
  ITEM_PIPELINES = {"scrapySpider.pipelines.MysqlPipeLine": 300,"scrapySpider.pipelines.MongoPipeLine": 301,
  }# Enable and configure the AutoThrottle extension (disabled by default)
  # See https://docs.scrapy.org/en/latest/topics/autothrottle.html
  #AUTOTHROTTLE_ENABLED = True
  # The initial download delay
  #AUTOTHROTTLE_START_DELAY = 5
  # The maximum download delay to be set in case of high latencies
  #AUTOTHROTTLE_MAX_DELAY = 60
  # The average number of requests Scrapy should be sending in parallel to
  # each remote server
  #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
  # Enable showing throttling stats for every response received:
  #AUTOTHROTTLE_DEBUG = False# Enable and configure HTTP caching (disabled by default)
  # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
  #HTTPCACHE_ENABLED = True
  #HTTPCACHE_EXPIRATION_SECS = 0
  #HTTPCACHE_DIR = "httpcache"
  #HTTPCACHE_IGNORE_HTTP_CODES = []
  #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"# Set settings whose default value is deprecated to a future-proof value
  REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
  TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
  FEED_EXPORT_ENCODING = "utf-8"#日志配置
  # LOG_FILE = 'log.log'
  # LOG_FILE_APPEND = False
  # LOG_LEVEL = 'INFO'

• items.py

  # Define here the models for your scraped items
  #
  # See documentation in:
  # https://docs.scrapy.org/en/latest/topics/items.htmlimport scrapyclass DoubanItem(scrapy.Item):# define the fields for your item here like:# name = scrapy.Field()title = scrapy.Field()rating = scrapy.Field()quote = scrapy.Field()intro = scrapy.Field()
  

• pipelines.py

  # Define your item pipelines here
  #
  # Don't forget to add your pipeline to the ITEM_PIPELINES setting
  # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html# useful for handling different item types with a single interface
  from itemadapter import ItemAdapter
  import pymysql
  import pymongovideo_spider = ['douban']class DoubanPipeline:def process_item(self, item, spider):print(item)return itemclass MysqlPipeLine:def open_spider(self, spider):self.spider = spiderself.mysql = pymysql.connect(host='localhost',port=3306,user='root',password='root')self.cursor = self.mysql.cursor()# 创建video数据库和相关爬虫表if self.spider.name in video_spider:self.create_db('video')'''创建数据库'''def create_db(self,db_name):sql = f'''CREATE DATABASE IF NOT EXISTS {db_name}'''try:self.cursor.execute(sql)self.mysql.select_db(db_name)if self.spider.name == 'douban':self.create_douban_table()except Exception as e:print(f'创建{db_name}数据库失败：{e}')'''创建表douban'''def create_douban_table(self):sql = f'''CREATE TABLE IF NOT EXISTS {self.spider.name}(id INT AUTO_INCREMENT,title VARCHAR(255),rating FLOAT,quote VARCHAR(255),intro TEXT,PRIMARY KEY(id))'''try:self.cursor.execute(sql)except Exception as e:print(f'创建douban表失败：{e}')def process_item(self, item, spider):if spider.name == 'douban':sql = f'''INSERT INTO {spider.name}(title,rating,quote,intro) VALUES(%(title)s,%(rating)s,%(quote)s,%(intro)s)'''try:item['rating'] = float(item['rating'])self.cursor.execute(sql,dict(item))self.mysql.commit()except Exception as e:print(f'”{item["title"]}”插入失败：{e}')self.mysql.rollback()return itemdef close_spider(self,spider):self.mysql.close()class MongoPipeLine:def open_spider(self, spider):self.spider = spiderself.mongo = pymongo.MongoClient(host='localhost',port=27017)# 创建video数据库和相关爬虫表if self.spider.name in video_spider:self.cursor = self.mongo['video'][self.spider.name]def process_item(self, item, spider):try:self.cursor.insert_one(dict(item))except Exception as e:print(f'”{item["title"]}”插入失败：{e}')return itemdef close_spider(self, spider):self.mongo.close()
  

• middlewares.py

  # Define here the models for your spider middleware
  #
  # See documentation in:
  # https://docs.scrapy.org/en/latest/topics/spider-middleware.htmlfrom scrapy import signals
  from fake_useragent import UserAgent
  from scrapy.http import Request,HtmlResponse
  from scrapySpider.kuaidaili import Kuaidaili# useful for handling different item types with a single interface
  from itemadapter import is_item, ItemAdapterclass DoubanDownloaderMiddleware:def __init__(self):self.ua = UserAgent()self.kuaidaili = Kuaidaili()#初始化一个代理ipself.first_ip = self.kuaidaili.get_ip()@classmethoddef from_crawler(cls, crawler):s = cls()crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)return sdef process_request(self, request:Request, spider):#设置UArequest.headers['User-Agent'] = self.ua.random#设置代理request.meta['proxy'] = self.first_iprequest.meta['download_timeout'] = 5spider.logger.info(f'ip：{request.meta["proxy"]}')return Nonedef process_response(self, request, response:HtmlResponse, spider):spider.logger.info(f'ip：{request.meta["proxy"]}')if response.status == 200:return response#代理失效小重新设置代理，并返回request重新请求request.meta['proxy'] = self.kuaidaili.get_ip()request.meta['download_timeout'] = 2return requestdef spider_opened(self, spider):spider.logger.info(f'"{spider.name}"Spide')