【python】爬虫

下载与批量下载

import requests
#第三方库，没有下载的下载一下 pip install requests#爬虫下载图片
res=requests.get("url")
print(res.content)#二进制字节流#写文件
with open("beauty.jpg","wb")as f:f.write(res.content)#批量下载短视频
headers={...}
params={...}
res=requests.get("url/post/",params=params,headers=headers)
print(res.json())
#只拿想要的
data=res.json().get(key)
video_urls=[data.get("desc"),data.get('video').get('play_addr').get('url_list')for item in data]for title,url in video_urls:res=requests.get(url)with open(f"./document/{title}.mp4","wb")as f:f.write(res.contnt)print('ok')

获取url的小技巧：
点击下载，查看网络中的请求GET的url拿过来即可

代码转换网站 转换成爬虫代码

openpyxl库

读写操作excel文件(.xlsx格式)

pip install openpyxlimport openpyxl
#file
workbook=openpyxl.load_workbook('name.xlsx')
#sheet
sheet=workbook['Sheet']
#单元格
value=sheet['A1'].value
sheet['C2'].value='666'
#save
workbook.save('name.xlsx')#create
workbook=openpyxl.Workbook()
sheet=workbook.active#获取默认工作表
sheet.append([1,2,3,4,5,7,6,8,9])#写入一行数据for i in range(100):#多行sheet.append([i,i+1,2,3,4,5,7,6,8,9])
#save
workbook.save('new_name.xlsx')

实战：爬数据写入excel

检查->网络->创建一个请求->复制cURL

找一个网站

import requests
import openpyxl
cookies={...}
headers={...}
response=requests.get('url?r=0&count=10&categories=%70....',cookies=cookies,headers=headers,
)
#url中常有count=10,可修改成想获取的条数
print(response.text)
print(response.json())data=response.json().get("items")
for item in data:if(item.get('type')=='a'):title=item.get("title")count=item.get("rating").get("count")#save to excel
#create
workbook=openpyxl.Workbook()
sheet=workbook.active#获取默认工作表for...#多行写入data
#save
workbook.save('new_name.xlsx')