爬虫下载网页文夹

爬虫下载网页pdf文件

import os
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from urllib.parse import urljoin, unquote
from tqdm import tqdm
# 设置网页的URL
base_url = "http://119/download/dzz/pdf/"# 创建保存文件的文件夹，如果文件夹不存在则创建
download_folder = "downloaded_pdfs"
os.makedirs(download_folder, exist_ok=True)def get_name(base_url):name_list=[]# 获取网页内容response = requests.get(base_url)  # 发送请求获取网页内容response.raise_for_status()  # 检查请求是否成功，若不成功则抛出异常soup = BeautifulSoup(response.text, "html.parser")  # 解析HTML内容# 查找所有文件链接for link in soup.find_all("a"):  # 遍历网页中所有的<a>标签，找到每个链接name_list.append(link.get("href"))return name_list
f1_list=get_name(base_url)
for i1 in f1_list:if i1 and i1 not in ["../"]:print(i1)f1_url = urljoin(base_url, i1)  # 将相对链接转为完整的URLf2_list=get_name(f1_url)for i2 in f2_list:   if i2 and i2 not in ["../"]:f2_url = urljoin(f1_url, i2)  # 将相对链接转为完整的URLf3_list=get_name(f2_url)for i3 in tqdm(f3_list):   if i3 and i3 not in ["../"]:f3_url = urljoin(f2_url, i3)  # 将相对链接转为完整的URLresponse = requests.get(f3_url)  # 发送请求获取网页内容response.raise_for_status()  # 检查请求是否成功，若不成功则抛出异常file_response = requests.get(f3_url)  # 发送请求下载文件内容download_folder_new = download_folder+'/'+i1+i2download_folder_new=unquote(download_folder_new)os.makedirs(download_folder_new, exist_ok=True)download_folder_path=download_folder_new+i3download_folder_path = unquote(download_folder_path)with open(download_folder_path, "wb") as f:  # 以二进制写入方式保存文件f.write(file_response.content)  # 写入下载的文件内容到本地# print(f"{download_folder_path}下载完成")  # 打印下载完成信息