python爬虫基础实验：通过DBLP数据库获取数据挖掘顶会KDD在2023年的论文收录和相关作者信息

Task1

读取网站主页整个页面的 html 内容并解码为文本串（可使用urllib.request的相应方法），将其以UTF-8编码格式写入page.txt文件。

Code1

import urllib.requestwith urllib.request.urlopen('https://dblp.dagstuhl.de/db/conf/kdd/kdd2023.html') as response:html = response.read()html_text = html.decode()with open('page.txt','w',encoding='utf-8') as f:f.write(html_text)

Task2

打开page.txt文件，观察 Track 名称、论文标题等关键元素的组成规律。从这个文本串中提取各Track 的名称并输出（可利用字符串类型的split()和strip()方法）。

Code2

import rewith open('page.txt', 'r', encoding='utf-8') as f:content = f.read()# 使用正则表达式找到所有的 <h2 id="*"> 和 </h2> 之间的字符串
matches = re.findall(r'<h2 id=".*?">(.*?)</h2>', content)for match in matches:print(match)

Task3

可以看到， “Research Track Full Papers” 和 “Applied Data Track Full Papers” 中的论文占据了绝大多数，现欲提取这两个 Track 下的所有论文信息（包含作者列表authors、论文标题title、收录起始页startPage与终止页endPage），并按照以下格式存储到一个字典列表中，同时输出这两个 Track 各自包含的论文数量，然后把字典列表转化为 json 对象（可使用json包的相应方法），并以 2 字符缩进的方式写入kdd23.json文件中。

[{"track": "Research Track Full Papers","papers": [{"authors": ["Florian Adriaens","Honglian Wang","Aristides Gionis"],"title": "Minimizing Hitting Time between Disparate Groups with Shortcut Edges.","startPage": "1","endPage": "10"},...]}{"track": "Applied Data Track Full Papers","papers": [{"authors": ["Florian Adriaens","Honglian Wang","Aristides Gionis"],"title": "Minimizing Hitting Time between Disparate Groups with Shortcut Edges.","startPage": "1","endPage": "10"},...]}
]

Code3

import re
import jsonwith open('page.txt', 'r', encoding='utf-8') as f:content = f.read()# 定义一个列表来存储 Track 信息
tracks = []# 定义正则表达式
track_pattern = re.compile(r'<h2 id=".*?">(.*?)</h2>')
author_pattern = re.compile(r'<span itemprop="name" title=".*?">(.*?)</span>')
title_pattern = re.compile(r'<span class="title" itemprop="name">(.*?)</span>')
page_pattern = re.compile(r'<span itemprop="pagination">(.*?)-(.*?)</span>')# 找到 "Research Track Full Papers" 和 "Applied Data Science Track Full Papers" 的位置
start1 = content.find('Research Track Full Papers') - 50
start2 = content.find('Applied Data Track Full Papers') - 50
start3 = content.find('Hands On Tutorials') - 1
end = len(content)# 从整篇文本中划分出前两个Track中所有相邻"<cite"和"</cite>"之间的内容(即一篇文章的范围)
research_papers_content = re.split('<cite', content[start1:start2])[1:]
applied_papers_content = re.split('<cite', content[start2:start3])[1:]def extract_paper_info(papers_content):papers = []for paper_content in papers_content:paper_content = re.split('</cite>', paper_content)[0]papers.append(paper_content)return papersspit_research_content = extract_paper_info(research_papers_content)
spit_applied_content = extract_paper_info(applied_papers_content)# 提取每篇paper的author、title和startPage, endPage
def extract_paper_info(papers_content):papers = []for paper_content in papers_content:authors = author_pattern.findall(paper_content)titles = title_pattern.findall(paper_content)pages = page_pattern.search(paper_content)startPage, endPage = pages.groups()papers.extend([{'authors': authors, 'title': title , 'startPage': startPage , 'endPage': endPage} for title in titles])return papers# 提取 "Research Track Full Papers" 的论文信息
research_track = track_pattern.search(content[start1:start2]).group(1)
research_papers = extract_paper_info(spit_research_content)# 提取 "Applied Data Science Track Full Papers" 的论文信息
applied_track = track_pattern.search(content[start2:start3]).group(1)
#applied_papers = extract_paper_info(spit_applied_content)
applied_papers = extract_paper_info(spit_applied_content)
# 将论文信息存储到字典列表中
tracks.append({'track': research_track, 'papers': research_papers})
tracks.append({'track': applied_track, 'papers': applied_papers})# 将字典列表转换为 JSON 并写入文件
with open('kdd23.json', 'w', encoding='utf-8') as f:json.dump(tracks, f, indent=2)

Task4

基于之前爬取的页面文本，分别针对这两个 Track 前 10 篇论文的所有相关作者，爬取他们的以下信息：（1）该研究者的学术标识符orcID（有多个则全部爬取）；（2）该研究者从 2020 年至今发表的所有论文信息（包含作者authors、标题title、收录信息publishInfo和年份year）。将最终结果转化为 json 对象，并以 2 字符缩进的方式写入researchers.json文件中，相应存储格式为：

[{"researcher": "Florian Adriaens","orcID": ["0000-0001-7820-6883"],"papers": [{"authors": ["Florian Adriaens","Honglian Wang","Aristides Gionis"],"title": "Minimizing Hitting Time between Disparate Groups with Shortcut Edges.","publishInfo": "KDD 2023: 1-10","year": 2023},...]},...
]   

Code4

import re
import requests
import json
import time
import random# 打开并读取 "page.txt" 文件
with open('page.txt', 'r', encoding='utf-8') as f:content = f.read()# 定义正则表达式
author_link_pattern = re.compile(r'<span itemprop="author" itemscope itemtype="http://schema.org/Person"><a href="(.*?)" itemprop="url">')
orcID_pattern = re.compile(r'<img alt="" src="https://dblp.dagstuhl.de/img/orcid.dark.16x16.png" class="icon">(.{19})</a></li>')
researcher_pattern = re.compile(r'<head><meta charset="UTF-8"><title>dblp: (.*?)</title>')
year_pattern = re.compile(r'<span itemprop="datePublished">(.*?)</span>')# 找到 "Research Track Full Papers" 和 "Applied Data Track Full Papers" 的位置
start1 = content.find('Research Track Full Papers')
start2 = content.find('Applied Data Track Full Papers')
end = len(content)# 提取这两个部分的内容，并找到前 10 个 "persistent URL:" 之间的内容
research_papers_content = content[start1:start2].split('<cite')[1:11]
applied_papers_content = content[start2:end].split('<cite')[1:11]def extract_paper_info(papers_content):papers = []for paper_content in papers_content:paper_content = re.split('</cite>', paper_content)[0]papers.append(paper_content)return papersspit_research_content = extract_paper_info(research_papers_content)
spit_applied_content = extract_paper_info(applied_papers_content)def extract_paper_info2(paper_content):final_result = []# 使用正则表达式找到所有在 "<>" 之外的字符串outside_brackets = re.split(r'<[^>]*>', paper_content)# 遍历提取到的内容，删除含有'http'的字符串及其前面的字符串flag = -1for i in range(len(outside_brackets)):if 'http' in outside_brackets[i]:flag = ifor i in range(flag + 1 , len(outside_brackets)):if outside_brackets[i]:final_result.append(outside_brackets[i])return final_result# 定义一个列表来存储研究者信息
researchers = []# 访问每篇文章里所有作者的链接，获取作者的 orcID 和论文信息
for papers in [research_papers_content, applied_papers_content]:for paper in papers:author_links = author_link_pattern.findall(paper)for link in author_links:link_content = requests.get(link)response = link_content.text#爬虫时频繁请求服务器，可能会被网站认定为攻击行为并报错"ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接"，故采取以下两个措施#使用完后关闭响应link_content.close()  # 在各个请求之间添加随机延时等待time.sleep(random.randint(1, 3))researcher = researcher_pattern.search(response).group(1)orcID = orcID_pattern.findall(response)# 找到 "<li class="underline" title="jump to the 2020s">" 和 "<li class="underline" title="jump to the 2010s">" 之间的内容start = response.find('2020 &#8211; today')end = response.find('<header id="the2010s" class="hide-head h2">')# 提取这部分的内容，并找到所有 "</cite>" 之间的内容papers_content = response[start:end].split('</cite>')[0:-1]papers_dict = []for paper_content in papers_content:spit_content = extract_paper_info2(paper_content)year = int(year_pattern.search(paper_content).group(1))authors = []publishInfo = []for i in range(0 , len(spit_content) - 1):if spit_content[i] != ", " and (spit_content[i+1] == ", " or spit_content[i+1] == ":"):authors.append(spit_content[i])elif spit_content[i][-1] == '.':title = spit_content[i]for k in range(i+2 , len(spit_content)):publishInfo.append(spit_content[k])# 创建一个新的字典来存储每篇文章的信息paper_dict = {'authors': authors, 'title': title, 'publishInfo': ''.join(publishInfo), 'year': year}papers_dict.append(paper_dict)researchers.append({'researcher': researcher, 'orcID': orcID, 'papers': papers_dict})# 将字典列表转换为 JSON 并写入 "researchers.json" 文件
with open('researchers.json', 'w', encoding='utf-8') as f:json.dump(researchers, f, indent=2)