python爬虫实现获取招聘信息

使用的python版本： 3.12.1

selenium版本：4.8.0

urllib版本：1.26.18

from selenium import webdriver 
from selenium.webdriver import ActionChains
import timeimport re
import xlwt
import urllib.parsedef get_html(url):chrome_driver = r"C:\chrome-win64\chromedriver.exe"options = webdriver.ChromeOptions()  # 用于配置Chrome浏览器驱动程序的行为options.add_argument("headless")  # 无界面启动options.add_experimental_option('useAutomationExtension', False)  # 禁用Chrome的自动化拓展程序options.add_experimental_option('excludeSwitches', ['enable-automation'])  # 确保浏览器不会因为启用自动化模式而出现不必要的错误或异常。options.add_argument("--disable-blink-features=AutomationControlled")  # 禁用由自动化测试或脚本控制的 Blink 功能。driver = webdriver.Chrome(chrome_options=options, executable_path=chrome_driver)# webdriver防屏蔽，不加这个就会出现滑动失败driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {"source": """Object.defineProperty(navigator, 'webdriver', {get: () => false})"""})driver.get(url)time.sleep(1)# 找到需要滑动的滑块元素slider = driver.find_element("class name","nc_bg")# 创建操作链action_chains = ActionChains(driver)# 将鼠标移动到滑块上action_chains.move_to_element(slider)# 模拟按下鼠标左键并保持不松开action_chains.click_and_hold()# 移动鼠标使滑块达到目标位置action_chains.move_by_offset(300, 0)# 松开鼠标左键action_chains.release()# 执行操作链action_chains.perform()time.sleep(10)html = driver.page_source  # 获取网页源码driver.quit()  # 清除后再退出return htmldef get_msg(excel1, sheet1):number = 0job_type = input("请输入你想要搜索的职位：")for i in range(1, 3):  # 页数自己随便改try:print("正在爬取第" + str(i) + "页数据...")result = urllib.parse.quote(job_type)  # 编码url_start = 'https://we.51job.com/api/job/search-pc?api_key=51job&keyword=' + result# 删除&timestamp参数，修改&pageSize=500url_end = '&searchType=2&function=&industry=&jobArea=090200&jobArea2=&landmark=&metro=&salary=&workYear=&degree=&companyType=&companySize=&jobType=&issueDate=&sortType=0&pageNum=' \+ str(i) + '&requestId=&pageSize=200&source=1&accountId=&pageCode=sou%7Csou%7Csoulb'url = url_start + url_endmsg = get_html(url)  # 用seleinum写的方法处理需要滑动进入，相当于这一部分可以拿到数据了msg = msg.replace('\\', '')  # 将用于转义的"\"替换为空# `(.*?)`表示任意我们想要的内容# `.*?`表示任意其他字符串reg = re.compile(r'"jobName"\s*:\s*"([^"]*)".*?'r'"cityString"\s*:\s*"([^"]*)".*?'r'"provideSalaryString"\s*:\s*"([^"]*)".*?'r'"issueDateString"\s*:\s*"([^"]*)".*?'r'"workYearString"\s*:\s*"([^"]*)".*?'r'"degreeString"\s*:\s*"([^"]*)".*?'r'"companyName"\s*:\s*"([^"]*)".*?'r'"companyTypeString"\s*:\s*"([^"]*)".*?'r'"companySizeString"\s*:\s*"([^"]*)"',re.DOTALL)  # 表示可以用.代替任意字符（包括那些换行符）items = reg.findall(msg)  # 按照正则表达式规则查找for item in items:number = number + 1print(number, item[0], item[1], item[2], item[3], item[4], item[5], item[6], item[7], item[8])sheet1.write(number, 0, number)sheet1.write(number, 1, item[0])sheet1.write(number, 2, item[6])sheet1.write(number, 3, item[1])sheet1.write(number, 4, item[7])sheet1.write(number, 5, item[2])sheet1.write(number, 6, item[5])sheet1.write(number, 7, item[4])sheet1.write(number, 8, item[8])sheet1.write(number, 9, item[3])# 表格文件保存是可以选择两种情况，# 一种在for循环里面，每写一行保存一次，这样可以放在程序中途出现异常后，文件内容啥也没有# 另一种是在for循环之外，所有内容写完再保存excel1.save("51job.xlsx")time.sleep(0.5)  # 休息间隔except Exception as e:print("except:"+str(e))passdef creat_xls(excel1):# 设置单元格格式sheet1 = excel1.add_sheet('Job', cell_overwrite_ok=True)sheet1.write(0, 0, '序号')sheet1.write(0, 1, '职位')sheet1.write(0, 2, '公司名称')sheet1.write(0, 3, '公司地点')sheet1.write(0, 4, '公司性质')sheet1.write(0, 5, '薪资')sheet1.write(0, 6, '学历要求')sheet1.write(0, 7, '工作经验')sheet1.write(0, 8, '公司规模')sheet1.write(0, 9, '发布时间')return sheet1def main():# 新建表格空间excel1 = xlwt.Workbook()  # 创建工作簿sheet1 = creat_xls(excel1)  # 创建工作表get_msg(excel1, sheet1)  # 使用函数if __name__ == '__main__':main()