从当当网批量获取图书信息

爬取当当网图书数据并保存到本地，使用request、lxml的etree模块、pandas保存数据为excel到本地。

爬取网页的url为：

http://search.dangdang.com/?key={}&act=input&page_index={}

其中key为搜索关键字，page_index为页码。

爬取的数据包括：爬取的数据包括：书名、作者、图书简介、出版社、出版日期、价格、评论数量。

  

代码如下：

import random
import requests
from lxml import etree
import pandas as pd
import timedata = []
data.append(['书名', '作者', '图书简介', '出版社', '出版日期', '价格', '评论数量'])
def get_book_info(url):headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'}response = requests.get(url, headers=headers)response.encoding = 'gbk'if response.status_code == 200:selector = etree.HTML(response.text)        book_list = selector.xpath('//*[@id="component_59"]/li')for book in book_list:# 书名          book_name = ''.join(book.xpath('.//p[1]/a/@title'))# 作者book_author = ''.join(book.xpath('.//p[5]/span[1]/a/@title'))# 图书简介book_intro = ''.join(book.xpath('.//p[2]/text()'))# 出版社book_publisher = ''.join(book.xpath('.//p[5]/span[3]/a/@title'))        # 出版日期    book_date = ''.join(book.xpath('.//p[5]/span[2]/text()'))# 价格book_price = ''.join(book.xpath('.//p[3]/span[1]/text()'))# 评论数量book_comments = ''.join(book.xpath('.//p[4]/a/text()'))# 随机等待时间，防止被封IPrdn = random.randint(1, 5)         print(f'等待时间：{rdn}')time.sleep(rdn)data.append([book_name, book_author, book_intro, book_publisher, book_date, book_price, book_comments])if __name__ == '__main__':    keyword = input('请输入搜索关键字：')page_index = 1while True:url = f'http://search.dangdang.com/?key={keyword}&act=input&page_index={page_index}'print(f'正在爬取第{page_index}页数据...')get_book_info(url)page_index += 1if page_index > 1:breakdf = pd.DataFrame(data[1:], columns=data[0])# 将DataFrame保存为Excel文件df.to_excel(f'{keyword}.xlsx', index=False)