scrapy解决管道阻塞问题采用threadpool库线程池+twisted同步语法异步编程

实现方法：process_item和download任务函数像下面编写即可，其他管道像往常一样写法

import time
import threadpool
import random
from twisted.internet import deferclass VideoPipeline:def __init__(self):self.pool = threadpool.ThreadPool(10)  # 设置线程数passdef process_item(self, item, spider):""":param item::param spider::return:"""finish_flag = random.randint(0,9)item['finish_flag'] = finish_flag# 线程池异步处理defer_item = defer.Deferred()task = threadpool.WorkRequest(self.download, args=(item,spider,finish_flag,defer_item,))self.pool.putRequest(task)return defer_itemdef download(self,item,spider,finish_flag,defer_item):"""相当于长作业"""time.sleep(random.randint(1,10))if not finish_flag:defer_item.callback(dict())defer_item.callback(item)def process_item2(self, item):""" VideoPipeline 下一个优先级管道类方法"""print(item)if __name__ == '__main__':a = VideoPipeline()for i in range(10):defer_item = a.process_item({},None)defer_item.addCallback(a.process_item2)print(11111111111)time.sleep(50)