B站弹幕分析系统

视频展示，请点击。

---

尚硅谷案例

utllib的基本使用

# 使用urllib来获取百度首页的源码
import urllib.request# (1)定义一个url 就是你要访问的地址
url = 'http://www.baidu.com'# (2)模拟浏览器先服务器发送请求 response响应
response = urllib.request.urlopen(url)# (3)获取响应中的页面的源码
# read方法 返回的是字节形式的二进制数据
# 我们要将二进制转化为字符串
# 二进制--》字符串 解码 decode('编码的格式')
content = response.read().decode('utf-8')# (4)打印数据
print(content)

一个类型和六个方法

一个类型 HTTPResponse
六个方法 read readline readlines  getcode geturl getheaders

import urllib.requesturl = 'http://www.baidu.com'# 模拟浏览器先服务器发送请求 response响应
response = urllib.request.urlopen(url)# 一个类型和六个方法
# response是HTTPResponse的类型
# print(type(response))# 按照一个字节一个字节的去读
# content = response.read()
# print(content)# 返回多少个字节
# content = response.read(5)
# print(content)# 读取一行
# content = response.readline()
# print(content)# content = response.readlines()
# print(content)#返回状态码 如果是200了 那么就证明我们的逻辑没有错误
# print(response.getcode())# 返回url地址
# print(response.geturl())# 获取是一些状态信息
# print(response.getheaders())

下载

import urllib.request# 下载网页
url_page = 'http://www.baidu.com'# url代表的是下载的路径  filename文件的名字
# 在python中可以是变量的名字 也可以直接书写
urllib.request.urlretrieve(url_page, 'baidu.html')# 下载图片
url_img = 'https://tse1-mm.cn.bing.net/th/id/OIP-C.zzaLy_4i4zzfAWPn03AkdgHaFI?w=194&h=135&c=7&r=0&o=5&dpr=1.6&pid=1.7'
urllib.request.urlretrieve(url_img, 'lusi.png')
url_img = 'https://tse3-mm.cn.bing.net/th/id/OIP-C.PijFe6ZDMUUR-95IU5W_dwHaNK?w=187&h=333&c=7&r=0&o=5&dpr=1.6&pid=1.7'
urllib.request.urlretrieve(url=url_img, filename='lusi2.png')# 下载视频
url_video = 'https://vd4.bdstatic.com/mda-me989nuvejzc5iws/sc/cae_h264_nowatermark/1620626259198934606/mda-me989nuvejzc5iws.mp4?v_from_s=hkapp-haokan-hbf&auth_key=1692513321-0-0-486766e5a214bed80f7b6de930400603&bcevod_channel=searchbox_feed&cr=2&cd=0&pd=1&pt=3&logid=2121127810&vid=16710300024974486498&klogid=2121127810&abtest=111803_1-112162_1-112345_2'
urllib.request.urlretrieve(url_video, 'lunyi.mp4')

请求对象的定制(遇到了反爬)

import urllib.requesturl = 'https://www.baidu.com'# url的组成
# http/https www.baidu.com            s   wd=周杰伦       #
# 协议            主机           端口号 路径    参数        锚点headers = {'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36 Edg/115.0.1901.203'
}# 请求对象的定制《=因为urlopen方法中不能存储字典 所以headers不能传递进去
request = urllib.request.Request(url=url, headers=headers)response = urllib.request.urlopen(request)content = response.read().decode('utf-8')print(content)

编码集的演变

# https://www.baidu.com/s?wd=%E5%91%A8%E6%9D%B0%E4%BC%A6# 需求 获取 https://www.baidu.com/s?wd=周杰伦的网页源码import urllib.requesturl = 'https://www.baidu.com/s?wd=周杰伦'headers = {'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36 Edg/115.0.1901.203'
}request = urllib.request.Request(url, headers)response = urllib.request.urlopen(request)content = response.read().decode('utf-8')print(content)

UnicodeEncodeError: 'ascii' codec can't encode characters in position 11-13: ordinal not in range(128)

解决方法

# https://www.baidu.com/s?wd=%E5%91%A8%E6%9D%B0%E4%BC%A6# 需求 获取 https://www.baidu.com/s?wd=周杰伦的网页源码import urllib.requesturl = 'https://www.baidu.com/s?wd='# 将周杰伦三个字变成Unicode编码的格式
# 我们需要依赖于urllib.parse
name = urllib.parse.quote('周杰伦')url = url + nameheaders = {'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36 Edg/115.0.1901.203'
}request = urllib.request.Request(url=url, headers=headers)response = urllib.request.urlopen(request)content = response.read().decode('utf-8')print(content)

IT私塾案例

爬虫

Beautiful Soup

import re
import urllib.requestfrom bs4 import BeautifulSoupurl = "http://www.baidu.com"
response = urllib.request.urlopen(url)
html = response.read()
bs = BeautifulSoup(html, "html.parser")
# print(bs.title)
# print(bs.a)
#
# print(type(bs.head)) # <class 'bs4.element.Tag'>
# 1.Tag 标签及其内容：拿到它所找到的第一个内容
#
# print(bs.title.string)
#
# print(type(bs.title.string))
# 2.NavigableString 标签里的内容(字符串)
#
# print(bs.a.attrs)# print(type(bs))
# 3.BeautifulSoup 表示整个文档
# print(bs.name)
# print(bs)# ----------------------------------
# 文档的遍历 更多内容 搜索文档
# print(bs.head.contents[1])# 文档的搜索
# (1)find_all()
# 字符串过滤：会查找与字符串完全匹配的内容
# t_list = bs.find_all("a")
# print(t_list)# 正则表达式搜索：使用search()方法来匹配内容
# t_list = bs.find_all(re.compile("a"))# 2.kwargs 参数
# t_list = bs.find(id="head")
#
# t_list = bs.find(class_=True)
# for item in t_list:
#     print(item)# 3.text文本
# 4.选择器
# t_list = bs.select('title') # 通过标签来查找
# t_list = bs.select('.mnav') # 通过类名来查找
# t_list = bs.select('#u1') # 通过id来查找
# for item in t_list:
#     print(item)

正则表达式

豆瓣案例(爬虫)

from bs4 import BeautifulSoup  # 网页解析，获取数据
import re  # 正则表达式，进行文字匹配
import urllib.request, urllib.error  # 制定URL，获取页面数据
import xlwt  # 进行excel操作
import sqlite3  # 进行SQLite数据库操作def main():baseurl = "https://movie.douban.com/top250?start="# 1.爬取网页datalist = getData(baseurl)savepath = "豆瓣电影Top250.xls"# 3.保存数据saveData(datalist, savepath)# 影片详情链接的规则
findLink = re.compile(r'<a href="(.*?)">')  # 创建正则表达式对象，表示规则(字符串的模式)
# 影片图片
findImgsrc = re.compile(r'<img.*src="(.*?)"', re.S)  # re.S 让换行符包含在字符中
# 影片片名
findTitle = re.compile(r'<span class="title">(.*)</span>')
# 影片评分
findRating = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>')
# 找到评价人数
findJudge = re.compile(r'<span>(\d*)人评价</span>')
# 找到概况
findIng = re.compile(r'<span class="inq">(.*)</span>')
# 找到影片的相关内容
findBd = re.compile(r'<p class="">(.*?)</p>', re.S)# 爬取网页
def getData(baseurl):datalist = []for i in range(0, 10):  # 调用获取页面信息的函数10次url = baseurl + str(i * 25)html = askURL(url)  # 保存获取到的网页源码# 2.逐一解析数据soup = BeautifulSoup(html, 'html.parser')for item in soup.find_all('div', class_="item"):  # 查询符合条件的字符串，形成列表# print(item) # 测试：查看电影item全部信息data = []item = str(item)# 影片详情的链接link = re.findall(findLink, item)[0]  # re库用来通过正则表达式查找指定的字符串data.append(link)                     # 添加链接imgSrc = re.findall(findImgsrc, item)[0]data.append(imgSrc)                   # 添加图片titles = re.findall(findTitle, item)if len(titles) == 2:ctitle = titles[0]                # 添加中文名称data.append(ctitle)               # 添加外国名称otitle = titles[1].replace("/", "") # 去掉无关的符号data.append(otitle)else :data.append(titles[0])            # 添加中文名称data.append(" ")                  # 外国名称留空rating = re.findall(findRating, item)[0]data.append(rating)                   # 添加评分judgeNum = re.findall(findJudge, item)[0]data.append(judgeNum)                 # 添加评论人数ing = re.findall(findIng, item)if len(ing) != 0:ing = ing[0].replace("。", "")     # 去掉句号data.append(ing)                   # 添加概述else :data.append(" ")                   #留空bd = re.findall(findBd, item)[0]bd = re.sub('<br(\s+)?/>(\s+)?', " ", bd)bd = re.sub("/", " ", bd)data.append(bd.strip())datalist.append(data)                   # 把处理好的一部电影的信息添加进去# for item in datalist:#     print(item)return datalist# 得到指定一个URL的网页内容
def askURL(url):head = {'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36 Edg/115.0.1901.203'}request = urllib.request.Request(url, headers=head)html = ""try:response = urllib.request.urlopen(request)html = response.read().decode("utf-8")# print(html)except urllib.error.URLError as e:if hasattr(e, "code"):print(e.code)if hasattr(e, "reason"):print(e.reason)return html# 保存数据
def saveData(datalist, savepath):book = xlwt.Workbook(encoding="utf-8", style_compression=0)      # 创建workbook对象sheet = book.add_sheet('豆瓣电影Top250', cell_overwrite_ok=True)     # 创建工作表col = ('电影详情链接', '图片链接', '影片中文名', '影片外国名', '评分', '评价数', '概况', '相关信息')for i in range(0, 8):sheet.write(0, i, col[i]) # 列名for i in range(0, 250):print("第%d条" %i)data = datalist[i]for j in range(0, 8):sheet.write(i+1,j,data[j])book.save(savepath)if __name__ == "__main__":main()

数据可视化

Flask(app.py)

from flask import Flask, render_template, request
import datetimeapp = Flask(__name__)# @app.route('/')
# def hello_world():
#     return 'Hello World!'@app.route("/welcome/<name>")
def welcome(name):return "你好%s"[e-mail]("/welcome/<int:id>")
def welcome2(id):return "您好%d号会员"%id# 返回给用户渲染后的网页文件
# @app.route("/")
# def index():
#     return render_template("index.html")#  向页面传递一个变量
@app.route("/")
def index():time = datetime.date.today() # 普通变量name = ["小张","小王","小赵"]  # 列表类型task = {"任务":"打扫卫生","时间":"三小时"} # 字典类型return render_template("index.html", var = time, list = name, task = task)# 表单提交
@app.route('/test/register')
def register():return render_template("test/register.html")@app.route('/result', methods=['POST','GET'])
def result():if request.method == 'POST':result = request.formreturn render_template("test/result.html", result=result)if __name__ == '__main__':app.run()

Flask(index.html)

<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><title>Title</title>
</head>
<body>欢迎光临,今天是{{ var }}<br/>今天值班的有:<br/>{% for data in list %}      <!--用大括号和百分号括起来是控制结构，还有if--><li>{{ data }}</li>{% endfor %}任务:<br/>        <!--了解如何在页面打印表格--><table border="1">{% for key,value in task.items() %}<tr><td>{{ key }}</td><td>{{ value }}</td></tr>{% endfor %}</table>
</body>
</html>

Flask(register.html)

<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><title>Title</title>
</head>
<body>
<form action="{{ url_for('result') }}" method="post"><p>姓名：<input type="text" name="姓名"></p><p>年龄：<input type="text" name="年龄"></p><p>性别：<input type="text" name="性别"></p><p>地址：<input type="text" name="地址"></p><p><input type="submit" value="提交"></p>
</form>
</body>
</html>

Flask(result.html)

<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><title>Title</title>
</head>
<body><table border="1">{% for key,value in result.items() %}<tr><th>{{ key }}</th><td>{{ value }}</td></tr>{% endfor %}</table>
</body>
</html>