爬取89ip代理、 爬取豆瓣电影

1 爬取89ip代理
2 爬取豆瓣电影

1 爬取89ip代理

import requests
from fake_useragent import UserAgent
from bs4 import BeautifulSoup
from requests.exceptions import ProxyErrorclass SpiderIP:def __init__(self):# 定义目标地址哦self.tag_url = "https://www.89ip.cn/"self.headers = {"User-Agent": UserAgent().random}def spider_index_response(self):response = requests.get(url=self.tag_url, headers=self.headers)return response.textdef create_soup(self):return BeautifulSoup(self.spider_index_response(), 'lxml')def spider_ip_port(self):soup = self.create_soup()tr_list = soup.select('div.layui-row.layui-col-space15 > div.layui-col-md8 > div > div.layui-form > table > tbody > tr')data_list = []for tr in tr_list:td_list = tr.find_all("td")ip = td_list[0].text.strip()port = td_list[1].string.strip()store = td_list[3].get_text().strip()# {"http":"http://IP:PORT"}data_list.append({"store": store, "proxies": {"http": f"http://{ip}:{port}"}})return data_listdef __spider_baidu(self, proxies):try:response = requests.get("http://httpbin.org/get", headers=self.headers, proxies=proxies, timeout=2)# 检查请求是否成功if response.status_code == 200:# 处理响应内容response.encoding = 'utf-8'  # 设置响应内容的编码格式为utf-8# 解析JSON结果data = response.text  # 获取响应信息print(data)else:print("请求失败:", response.status_code)except ProxyError:passdef test_ip(self):data_list = self.spider_ip_port()for index, data in enumerate(data_list, start=1):store = data.get("store")proxies = data.get("proxies")print(f"这是第 {index} 条数据! 运营商是 :>>> {store}")proxies = self.__spider_baidu(proxies=proxies)if proxies:print(f"当前代理可用")else:print(f"已废除")def main(self):self.test_ip()if __name__ == '__main__':s = SpiderIP()s.main()

2 爬取豆瓣电影

import requests
from fake_useragent import UserAgent
from bs4 import BeautifulSoup
from lxml import etreeclass SpiderBase:def __init__(self):self.tag_url_list = []self.headers = {"User-Agent": UserAgent().random}class SpiderTopSoup(SpiderBase):def __init__(self):super().__init__()self.tag_url_list = self.__create_tag_url_list()def __create_tag_url_list(self):tag_url_list = []for i in range(0, int(250 / 25)):if i == 0:tag_url = "https://movie.douban.com/top250"tag_url_list.append(tag_url)else:tag_url = f"https://movie.douban.com/top250?start={i * 25}"tag_url_list.append(tag_url)return tag_url_listdef __create_soup(self, page_text):return BeautifulSoup(page_text, 'lxml')def __spider_detail_data(self, soup):data_list = []div_list = soup.find_all("div", class_="item")for div in div_list:#pic_div = div.find("div", class_="pic")# 封面图链接地址img_url = pic_div.a.img.get("src")# 排名level = pic_div.em.text# 详情链接detail_url = pic_div.a.get("href")bd_a_span_list = div.find("div", class_="info").find("div", class_="hd").a.find_all("span")try:title = bd_a_span_list[0].textexcept:title = ""try:title_eg = bd_a_span_list[1].textexcept:title_eg = ""try:title_desc = bd_a_span_list[2].textexcept:title_desc = ""bd_div = div.find("div", class_="info").find("div", class_="bd")# 导演和上映日期action, publish_date = [data.replace("\xa0", "").strip() for data in bd_div.p.text.strip().split("\n")]# 评分 和 评价span_list = bd_div.find("div", class_="star").find_all("span")score = span_list[1].textcomment_num = span_list[-1].text[0:-3]# 格言try:quote = bd_div.find("p", class_="quote").span.textexcept:quote = ""data_list.append({"title": title,"title_eg": title_eg,"title_desc": title_desc,"img_url": img_url,"level": level,"detail_url": detail_url,"action": action,"publish_date": publish_date,"score": score,"comment_num": comment_num,"quote": quote,})print(data_list)return data_listdef spider_index_data(self, tag_url):response = requests.get(url=tag_url, headers=self.headers)soup = self.__create_soup(page_text=response.text)return self.__spider_detail_data(soup=soup)def main(self):data_list_all = []for tag_url in self.tag_url_list:data_list = self.spider_index_data(tag_url=tag_url)data_list_all.extend(data_list)print(len(data_list_all))if __name__ == '__main__':s = SpiderTopSoup()s.main()

版本2（建议用这个）

import requests
from fake_useragent import UserAgent
from bs4 import BeautifulSoup
from lxml import etreemovie_dict = {"title": '电影名',"title_eg": '英文名',"title_desc": '简介',"img_url": '图片链接',"level": '级别',"detail_url": '播放地址',"action": '导演和演员',"publish_date": '播放日期',"score": '评分',"comment_num": '评论数',"quote": '格言',
}class SpiderBase:def __init__(self):self.tag_url_list = []self.headers = {"User-Agent": UserAgent().random}class SpiderTopSoup(SpiderBase):def __init__(self):super().__init__()self.tag_url_list = self.__create_tag_url_list()def __create_tag_url_list(self):tag_url_list = []for i in range(0, int(250 / 25)):if i == 0:tag_url = "https://movie.douban.com/top250"tag_url_list.append(tag_url)else:tag_url = f"https://movie.douban.com/top250?start={i * 25}"tag_url_list.append(tag_url)return tag_url_listdef __create_soup(self, page_text):return BeautifulSoup(page_text, 'lxml')def __spider_detail_data(self, soup):data_list = []div_list = soup.find_all("div", class_="item")for div in div_list:#pic_div = div.find("div", class_="pic")# 封面图链接地址img_url = pic_div.a.img.get("src")# 排名level = pic_div.em.text# 详情链接detail_url = pic_div.a.get("href")bd_a_span_list = div.find("div", class_="info").find("div", class_="hd").a.find_all("span")try:title = bd_a_span_list[0].textexcept:title = ""try:title_eg = bd_a_span_list[1].textexcept:title_eg = ""try:title_desc = bd_a_span_list[2].textexcept:title_desc = ""bd_div = div.find("div", class_="info").find("div", class_="bd")# 导演和上映日期action, publish_date = [data.replace("\xa0", "").strip() for data in bd_div.p.text.strip().split("\n")]# 评分 和 评价span_list = bd_div.find("div", class_="star").find_all("span")score = span_list[1].textcomment_num = span_list[-1].text[0:-3]# 格言try:quote = bd_div.find("p", class_="quote").span.textexcept:quote = ""data_dict = {"title": title,"title_eg": title_eg,"title_desc": title_desc,"img_url": img_url,"level": level,"detail_url": detail_url,"action": action,"publish_date": publish_date,"score": score,"comment_num": comment_num,"quote": quote,}for key, value in movie_dict.items():new_dict = f"{value}: {data_dict[key]}"data_list.append(new_dict)print(data_list)return data_listdef spider_index_data(self, tag_url):response = requests.get(url=tag_url, headers=self.headers)soup = self.__create_soup(page_text=response.text)return self.__spider_detail_data(soup=soup)def main(self):data_list_all = []for tag_url in self.tag_url_list:data_list = self.spider_index_data(tag_url=tag_url)data_list_all.extend(data_list)print(len(data_list_all))if __name__ == '__main__':s = SpiderTopSoup()s.main()