python 爬取杭州小区挂牌均价

下载chrome驱动

通过chrome浏览器的 设置-帮助-关于Google Chrome 查看你所使用的Chrome版本

驱动可以从这两个地方找:

• 【推荐】https://storage.googleapis.com/chrome-for-testing-public
• http://npm.taobao.org/mirrors/chromedriver

import zipfile
import os
import requestsdef un_zip(file_name, to_dir='./'):"""unzip zip file"""zip_file = zipfile.ZipFile(file_name)if os.path.isdir(to_dir):passelse:os.mkdir(to_dir)for names in zip_file.namelist():zip_file.extract(names, to_dir)zip_file.close()def download_driver(url=None, to_dir='./', version=''):print('install chrome-driver first')if not url:url = 'http://npm.taobao.org/mirrors/chromedriver/LATEST_RELEASE'if len(version)>0:url = 'http://npm.taobao.org/mirrors/chromedriver/LATEST_RELEASE_'+versionversion = requests.get(url).content.decode('utf8')driver_file = 'http://npm.taobao.org/mirrors/chromedriver/' + version + '/chromedriver_win32.zip'else:driver_file = urlr = requests.get(driver_file)download_zip = "chromedriver_win32.zip"with open(download_zip, "wb") as code:code.write(r.content)un_zip(download_zip,  to_dir)# os.remove(download_zip)print('done')download_driver(url='https://storage.googleapis.com/chrome-for-testing-public/123.0.6312.58/win64/chromedriver-win64.zip', to_dir='./', version='')

数据库交互

把抓取的数据保存到数据库，sqlite数据库是一个db文件，可以用DBeaver工具打开，很方便

import sqlite3class DBC:def __init__(self, dbname):self.db = dbnameself.conn = Nonedef __enter__(self):self.conn = sqlite3.connect(self.db)return self.conndef __exit__(self, exc_type, exc_val, exc_tb):if exc_type is not None:self.conn.rollback()print("rollback")print(exc_type, exc_val, exc_tb)else:self.conn.commit()self.conn.close()def insert2db(data):with DBC('anjuke.db') as conn:cur = conn.cursor()cur.executemany("""insert into anjuke(community_name,year,address, tags, price, if_down, percent) values(?,?,?,?,?,?,?)""", data) with DBC('anjuke.db') as conn:cur = conn.cursor()cur.execute("""drop table if exists anjuke""")cur.execute("""create table if not exists anjuke(id INTEGER primary key  AUTOINCREMENT not null,community_name TEXT,year TEXT,address TEXT,tags TEXT,price TEXT,if_down BOOL,percent TEXT)""")

selenium 爪巴虫

import time
import traceback
from selenium import webdriverbrowser = webdriver.Chrome()
def process(url):browser.get(url)html = browser.page_sourcehtml = BeautifulSoup(html,)As = html.find_all("a", {"class": "li-row"})if len(As) == 0:raise Exception("EMPTY")data = []for A in As:price = A.find("div", {"class": "li-side"})price_value = price.find("div", {"class": "community-price"})if price_value:price_value = price_value.text.strip()minus = price.find("span")if minus:minus = 'propor-green' in minus.attrs['class']percent = price.find("span")if percent:percent = percent.text.strip()info = A.find("div", {"class": "li-info"})community_name = info.find("div", {"class": "nowrap-min li-community-title"}).textyear = info.find("span", {"class": "year"})if year:year = year.textadvantage =info.find("div", {"class":"prop-tags"})if advantage:advantage = advantage.text.strip()address = info.find("div", {"class": "props nowrap"}).find_all("span")[-1].textdata.append((community_name,year,address, advantage, price_value, minus, percent))insert2db(data)i = 0
while i < 50:  # 超过50显示重复数据i += 1url = f'https://hangzhou.anjuke.com/community/o8-p{i}'  # o2,o4,o6,o8不同排序条件print(url)try:process(url)except Exception as e:traceback.print_exc()s = input("check:")  # 抓取过程中，可能需要输入验证码i = i-1time.sleep(0.5)