爬虫学习日记第八篇（爬取fofa某端口的协议排行及其机器数目，统计top200协议）

需求

找到最常用的200个协议

通过fofa搜索端口，得到协议排名前五名和对应机器的数目。
遍历端口，统计各个协议对应的机器数目（不准，但能看出个大概）

读写API

API需要会员，一天只能访问1000次。

import base64
import urllib
from time import sleep
import requests
res = {}
def onePort(j):text = 'port="' + str(j) + '"'text = base64.b64encode(text.encode("utf-8")).decode("utf-8")text = urllib.parse.quote(text)URL = f'https://fofa.info/api/v1/search/stats?fields=protocol&qbase64={text}&email=*****&key=*****'r = requests.get(URL)response_dict = r.json()print("当前端口为：",j)print(response_dict)protocols=response_dict['aggs']['protocol']for i in protocols:if i['name'] in res:res[i['name']] = res[i['name']] + i['count']else:res[i['name']] = i['count']print(res)for i in range(1,65535):onePort(i)sleep(10)

爬虫

页面动态加载，由于动态渲染的问题，有的请求返回结果为空。

单线程，未登录爬虫代码

import base64
import json
import urllib
from concurrent.futures import ThreadPoolExecutorfrom selenium import webdriver
from selenium.webdriver.chrome.service import Service
from lxml import etree
from time import sleep
#直接添加这四行代码
from selenium.webdriver.chrome.options import Options
options = Options()
options.add_argument('--headless')
# options.add_argument('--disable-gpu')failed=[]
success=[]
res = {}
def onePort(j):s = Service(r".\chromedriver.exe")driver = webdriver.Chrome(service=s,options=options)text = 'port="' + str(j) + '"'text = base64.b64encode(text.encode("utf-8")).decode("utf-8")text = urllib.parse.quote(text)print(text)driver.get("https://fofa.info/result?qbase64=" + text)sleep(7)page_text = driver.page_source# print(page_text)tree = etree.HTML(page_text)protos = tree.xpath('//div[@class="hsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel"]/div[13]//li//a/text()')nums = tree.xpath('//div[@class="hsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel"]/div[13]//li//span/text()')for i in range(len(protos)):protos[i] = protos[i].strip(' ')protos[i] = protos[i].strip('\n')protos[i] = protos[i].strip(' ')nums[i] = nums[i].strip(' ')nums[i] = nums[i].strip('\n')nums[i] = nums[i].strip(' ')nums[i] = nums[i].replace(',', '')nums[i] = int(nums[i])if protos[i] in res:res[protos[i]] = res[protos[i]] + nums[i]else:res[protos[i]] = nums[i]print(protos)print(nums)if len(protos) == 0:failed.append(j)else:success.append(j)print("当前端口号：", j)print("失败列表：", failed)print("成功列表：", success)print(res)driver.quit()for j in range(5000,10000):onePort(j)

多线程未登录代码

一定要注意多线程同时读写问题，全局变量上锁

import base64
import json
import urllib
from concurrent.futures import ThreadPoolExecutor
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from lxml import etree
from time import sleep
import threading# 直接添加这四行代码
from selenium.webdriver.chrome.options import Optionsoptions = Options()
options.add_argument('--headless')
# options.add_argument('--disable-gpu')failed = []
success = []
res = {}
lock = threading.Lock()  # 创建线程锁def onePort(j):s = Service(r".\chromedriver.exe")driver = webdriver.Chrome(service=s, options=options)text = 'port="' + str(j) + '"'text = base64.b64encode(text.encode("utf-8")).decode("utf-8")text = urllib.parse.quote(text)print(text)driver.get("https://fofa.info/result?qbase64=" + text)sleep(7)page_text = driver.page_source# print(page_text)tree = etree.HTML(page_text)protos = tree.xpath('//div[@class="hsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel"]/div[13]//li//a/text()')nums = tree.xpath('//div[@class="hsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel"]/div[13]//li//span/text()')with lock:  # 使用线程锁保护对res变量的读写操作for i in range(len(protos)):protos[i] = protos[i].strip(' ')protos[i] = protos[i].strip('\n')protos[i] = protos[i].strip(' ')nums[i] = nums[i].strip(' ')nums[i] = nums[i].strip('\n')nums[i] = nums[i].strip(' ')nums[i] = nums[i].replace(',', '')nums[i] = int(nums[i])if protos[i] in res:res[protos[i]] = res[protos[i]] + nums[i]else:res[protos[i]] = nums[i]print(protos)print(nums)if len(protos) == 0:failed.append(j)else:success.append(j)print("当前端口号：", j)print("失败列表：", failed)print("成功列表：", success)print(res)driver.quit()with ThreadPoolExecutor(30) as t:for j in range(10000,10500):# 把下载任务提交给线程池t.submit(onePort, j)

手动登录获取cookie代码

# 填写webdriver的保存目录
s = Service(r".\chromedriver.exe")
driver= webdriver.Chrome(service=s)
# 记得写完整的url 包括http和https
driver.get('https://fofa.info')
# 程序打开网页后20秒内 “手动登陆账户”
time.sleep(20)
with open('cookies.txt','w') as f:# 将cookies保存为json格式f.write(json.dumps(driver.get_cookies()))driver.close()

登录账号的单线程爬虫

from selenium import webdriver
import time
import jsonfrom selenium.webdriver.chrome.service import Serviceimport base64
import json
import urllib
from concurrent.futures import ThreadPoolExecutor
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from lxml import etree
from time import sleep
from selenium.webdriver.chrome.options import Optionsfrom selenium.webdriver.chrome.options import Options
options = Options()
# options.add_argument('--headless')
# options.add_argument('--disable-gpu')
options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 Edg/118.0.2088.46"')failed=[]
success=[]
res = {}s = Service(r".\chromedriver.exe")
driver = webdriver.Chrome(service=s, options=options)driver.get('https://fofa.info')
# 首先清除由于浏览器打开已有的cookies
driver.delete_all_cookies()with open('cookies.txt', 'r') as f:# 使用json读取cookies 注意读取的是文件 所以用load而不是loadscookies_list = json.load(f)# 将expiry类型变为intfor cookie in cookies_list:# 并不是所有cookie都含有expiry 所以要用dict的get方法来获取if isinstance(cookie.get('expiry'), float):cookie['expiry'] = int(cookie['expiry'])driver.add_cookie(cookie)# 重新发送请求(这步是非常必要的，要不然携带完cookie之后仍然在登录界面)
driver.get('https://fofa.info')
# sleep等待页面完全加载出来，这一步很关键
time.sleep(3)j=2
text = 'port="' + str(j) + '"'
text = base64.b64encode(text.encode("utf-8")).decode("utf-8")
text = urllib.parse.quote(text)
print(text)
sleep(10)
driver.get("https://fofa.info/result?qbase64=" + text)
sleep(6)
page_text = driver.page_sourceprint(page_text)tree = etree.HTML(page_text)protos = tree.xpath('//div[@class="hsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel"]/div[13]//li//a/text()')
nums = tree.xpath('//div[@class="hsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel"]/div[13]//li//span/text()')for i in range(len(protos)):protos[i] = protos[i].strip(' ')protos[i] = protos[i].strip('\n')protos[i] = protos[i].strip(' ')nums[i] = nums[i].strip(' ')nums[i] = nums[i].strip('\n')nums[i] = nums[i].strip(' ')nums[i] = nums[i].replace(',', '')nums[i] = int(nums[i])if protos[i] in res:res[protos[i]] = res[protos[i]] + nums[i]else:res[protos[i]] = nums[i]
print(protos)
print(nums)
if len(protos) == 0:failed.append(j)
else:success.append(j)
print("当前端口号：", j)
print("失败列表：", failed)
print("成功列表：", success)
print(res)driver.quit()

登录用户多线程

import threadingfrom selenium import webdriver
import time
import jsonfrom selenium.webdriver.chrome.service import Serviceimport base64
import json
import urllib
from concurrent.futures import ThreadPoolExecutor
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from lxml import etree
from time import sleep
from selenium.webdriver.chrome.options import Optionsfrom selenium.webdriver.chrome.options import Options
options = Options()
options.add_argument('--headless')
# options.add_argument('--disable-gpu')
options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 Edg/118.0.2088.46"')failed=[]
success=[]
res = {}
lock = threading.Lock()def onePort(j):s = Service(r".\chromedriver.exe")driver = webdriver.Chrome(service=s, options=options)driver.get('https://fofa.info')# 首先清除由于浏览器打开已有的cookiesdriver.delete_all_cookies()with open('cookies.txt', 'r') as f:# 使用json读取cookies 注意读取的是文件 所以用load而不是loadscookies_list = json.load(f)# 将expiry类型变为intfor cookie in cookies_list:# 并不是所有cookie都含有expiry 所以要用dict的get方法来获取if isinstance(cookie.get('expiry'), float):cookie['expiry'] = int(cookie['expiry'])driver.add_cookie(cookie)# 重新发送请求(这步是非常必要的，要不然携带完cookie之后仍然在登录界面)driver.get('https://fofa.info')# sleep等待页面完全加载出来，这一步很关键time.sleep(3)text = 'port="' + str(j) + '"'text = base64.b64encode(text.encode("utf-8")).decode("utf-8")text = urllib.parse.quote(text)print(text)driver.get("https://fofa.info/result?qbase64=" + text)sleep(6)page_text = driver.page_sourcetree = etree.HTML(page_text)protos = tree.xpath('//div[@class="hsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel"]/div[13]//li//a/text()')nums = tree.xpath('//div[@class="hsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel"]/div[13]//li//span/text()')with lock:  # 使用线程锁保护对res变量的读写操作for i in range(len(protos)):protos[i] = protos[i].strip(' ')protos[i] = protos[i].strip('\n')protos[i] = protos[i].strip(' ')nums[i] = nums[i].strip(' ')nums[i] = nums[i].strip('\n')nums[i] = nums[i].strip(' ')nums[i] = nums[i].replace(',', '')nums[i] = int(nums[i])if protos[i] in res:res[protos[i]] = res[protos[i]] + nums[i]else:res[protos[i]] = nums[i]print(protos)print(nums)if len(protos) == 0:failed.append(j)else:success.append(j)print("当前端口号：", j)print("失败列表：", failed)print("成功列表：", success)print(res)driver.quit()with ThreadPoolExecutor(1) as t:for j in range(3679,4000):# 把下载任务提交给线程池t.submit(onePort, j)