算法日志的存在核心在于搭建自检系统

        "相信每一个人执行与日志有关的任务都会遇到这样难题吧？长达几万行的日志，如果我们单纯用肉眼去一个个排查，那么恐怕所耗费的时间是以天为计量单位了。当然这是一种比较夸张的情况，根据我的项目经验，正常情况是十几个站点的人可能每天需要花费3-4个小时去排查日志或者与日志有关却能被日志替代的内容。如果我们能搭建一个智能化的系统，使得这个系统可以智能的读取日志中我们关键的信息，那么会发生什么呢？"

        有些人问，我就想用肉眼看，不行嘛？其实，"不是肉眼看不起，而是智能化日志更有性价比！"没错，如果我们搭建这样一个智能化日志自检系统，N个站n*m个团体每天都能节省n*m*k个工时去干别的事情。

NOTE：本文只是介绍一种思想，所以不会有过多的具体代码讲解，但是可以给上一个成功的案例手册，仅供参考。

———————————————————————————————————————————华丽的分割线

现场人员自检失败表计点位教程

NOTE: 如果没有“meterPoint_Self-Checking_sys.py“脚本的请联系我们进行提供！

👇

运行该脚本，参考运行命令如下（请确保此时您的工作目录处于meter/log）

#这是一条参考运行命令，请您根据您实际的情况修改-p和-t参数的具体内容
python3 meterPoint_Self-Checking_sys.py -p meterlog -t 30M00000036658634_task1703485183168_20231225141946

# @pararm:-p 是存放日志的路径，该日志包含您刚跑完测试的日志内容。
# @pararm:-t 是您任务的序号，【如下图】,Ftp图片路径下包含”task“的字符串，也就是灰色框框住的那一串正式您此次任务的序号，输入30M00000036658634_task1703485183168_20231225141946

👇

自动生成自检报表meterlog_checking.txt

里面部分关键内容如下：

👇

接下来大家请对照这张表，找到【需要现场人员自检】的【错误】进行搜索排查，有多个，可以从上往下慢慢来。

👇

以【通用类】<序号7>"该点位没有录入"作为例子，打开自检文本meterlog_checking.txt

👇

👇

如果出现无需现场人员自检的错误，需要截图一下日志中有关内容，可能后续还需提供图片我们这边进行优化。

一些使用样例图: 

可供实验的代码👑

# -*- coding: utf-8 -*-
'''
参考diamagnetic：
# 兰江
python3 meterPoint_Self-Checking_sys.py -p meterlog -t 30M00000036658634_task1703485183168_20231225141946
# 金鼎
python3 meterPoint_Self-Checking_sys.py -p meterlog -t 30M00000036658634_task1703485183168_20231225141947
'''
import re
import json
import argparse# 创建命令行参数解析器
parser = argparse.ArgumentParser()
parser.add_argument('-p', '--log_file', help='log文件路径')
parser.add_argument('-t', '--task_id', help='任务ID')
args = parser.parse_args()def extract_debug_segments(log_file):debug_segments = []with open(log_file, 'r') as file:lines = file.readlines()start_line = Noneend_line = Nonesegment = []for i, line in enumerate(lines):if 'Debug' in line or '收到请求' in line or '数据库信息' in line:if start_line is None:start_line = isegment.append(line.strip())   elif '结果放入队列待发送' in line:if start_line is not None:end_line = isegment.append(line)debug_segments.append([segment, start_line, end_line])segment = []start_line = Noneend_line = Nonereturn debug_segments
def process_request(request_str):target_index = request_str.index("{")# 按照":"分割字符串split_str = request_str[target_index:]# 获取分割后数组中最后一个索引所保存的信息json_str = split_str.strip().replace("—", "-").replace("'", "\"")objectList_request_str = json.loads(json_str)['objectList'][0]# for k in objectList_request_str:#     print(k)return objectList_request_strdef get_pointList_length(json_str):pattern = r"'Position': '(\[.*?\])'"matches = re.search(pattern, json_str)if matches is None:return 0position_list = json.loads(matches.group(1))# print("position_list:", position_list)return len(position_list)def process_sql(json_str):json_str = json_str[json_str.index("MinValue"):]json_str = "{'" + json_strjson_str = json_str.replace("'", "\"")sql_dict = json.loads(json_str)return sql_dictdef process_result(json_str):json_str = json_str[json_str.index("code"):-5]json_str = "{'" + json_str# print(json_str)json_str = json_str.replace("'", "\"")json_str = json_str.replace("None", "null")sql_dict = json.loads(json_str)return sql_dictdef contains_digit(string):pattern = r'\d'  # 正则表达式模式，匹配任意数字if re.search(pattern, string):return Trueelse:return Falsedef get_path_separator(path):if '/' in path:return '/'elif '\\' in path:return '\\'elif '\\' * 2 in path:return '\\\\'else:return Nonedef extract_work_path_tool(goal_str):split_str = get_path_separator(goal_str)pathIdx = -1splitPaths = goal_str.split(split_str)for idx, ss in enumerate(splitPaths):if ss == 'CCD':pathIdx = idxif pathIdx == -1:raise Exception("您的任务路径中没有CCD路径")work_path = splitPaths[pathIdx-1]return work_pathif __name__ == "__main__":print("---------------------------------------------------------------------")# 摄像机偏移严重+模糊Error_withoutDetctor = []# 未识别出指针Error_withoutPointer = []# 读取ftp图失败Error_loadftp = []# minIO无图Error_withoutMinioImage = []# minIO错图Error_minioErrorImage = []# 点位未录入Error_withoutId = []# 表计类型录入错误Error_clsType = []# 最大最小值设置错误Error_minMaxSet = []# 最大最小值未设置Error_withoutMinMax = []# 未打刻度点位Error_withoutPointList = []# 刻度打点错误Error_PointList = []# 未识别到任何油面表！Error_ymb = []# 画框与推理出来的油面表无匹配Error_withoutYmbMatch = []# OCR没有检测出数字Error_ocrRec = []# OCR没有检测出表盘Error_ocrDet = []# ===========================核# 获取命令行参数log_file = args.log_filework_id = args.task_iddebug_segments = extract_debug_segments(log_file)error_num = 0# not_reading_num = 0# type_num = 0ymb_num, sxb_num, bj_num = 0, 0, 0ymb_errorNum, sxb_errorNum, bj_errorNum = 0, 0, 0# 过滤一遍只剩下最新的filter_schem = {}piNums_schem = {}not_del_ids = []# 任务计算workNUms_schem = {}for idx, segment in enumerate(debug_segments):strat_line = segment[1]end_line = segment[2]for line in segment[0]:if "收到请求" in line:# print('【请求信息】: ',end='')objectList_request_str = process_request(line)# 任务IDwork_path = extract_work_path_tool(objectList_request_str['imageUrlList'][0])if not work_path in workNUms_schem:workNUms_schem[work_path] = 1else:workNUms_schem[work_path] += 1if work_path != work_id:break#点位IDextract_objectId = objectList_request_str['objectId']if not extract_objectId in filter_schem.keys():# 新增filter_schem[extract_objectId] = idxpiNums_schem[extract_objectId] = 1else:# 更新filter_schem[extract_objectId] = idxpiNums_schem[extract_objectId] += 1not_del_ids.append(idx)breakprint('|任务id                                                        |数量')print("---------------------------------------------------------------------")for wnn in workNUms_schem:if work_id == wnn:print(wnn, '     |',workNUms_schem[wnn],end='   ✔\n')else:print(wnn, '     |',workNUms_schem[wnn])print('*********************************************************************')# print(piNums_schem)# 找到第一次出现重复点位的位置print("此次任务id:{}".format(work_id))idsNums_result1 = len({key: value for key, value in piNums_schem.items() if value == 1})print("点位 [=1] 的数量：",idsNums_result1) idsNums_result2 = len({key: value for key, value in piNums_schem.items() if value > 1})print("点位 [>1] 的数量：",idsNums_result2)  print('*********************************************************************')# print(filter_schem, len(filter_schem)) # ------------------过滤结束sumWorkNum, filter_workId_num, filter_objectId_num = 0, 0, 0for idx, segment in enumerate(debug_segments):# print(segment[0],'\n',len(segment[0]))error_flag = FalseftpLoad_flag = False# print('Start Line:', segment[1])# print('End Line:', segment[2])for line in segment[0]:if "收到请求" in line:# print('【请求信息】: ',end='')objectList_request_str = process_request(line)extract_objectId = objectList_request_str['objectId']# print(extract_objectId)# print(objectList_request_str['imageUrlList'][0], work_id)# 过滤掉【不同任务】if not work_id == extract_work_path_tool(objectList_request_str['imageUrlList'][0]):filter_workId_num += 1break# 过滤掉【同任务相同点位取最新】if ( piNums_schem[extract_objectId] > 1 ) and ( idx != filter_schem[extract_objectId] ):# print(idx, filter_schem[extract_objectId])filter_objectId_num += 1break# 这里才是没被break的真正点位数量sumWorkNum += 1elif '数据库信息' in line:# print(line)if line.split("【数据库信息】")[-1] == '{}':# 数据库信息为空# print('*pointList_length:0')# print('{}')Error_withoutId.append(extract_objectId)error_num += 1breakelse:# 数据库有信息pointList_length = get_pointList_length(line)sql_schem = process_sql(line)MinValue = sql_schem['MinValue']MaxValue = sql_schem['MaxValue']meter_type = sql_schem['AlgorithmType']ImagePath = sql_schem['ImagePath']if meter_type == 'meter_v5':bj_num += 1if meter_type == 'meter_ywj':ymb_num += 1if meter_type == 'paddleocr':sxb_num += 1if meter_type == 'meter_v5':if len(MinValue)== 0 or len(MaxValue) == 0:Error_withoutMinMax.append(extract_objectId)MinValue = float(0)MaxValue = float(100)error_flag = Trueelse:MinValue = float(MinValue)MaxValue = float(MaxValue)# 表计类型录入错误(如果打点了，但表计类型不是meter_v5)if meter_type != 'meter_v5' and pointList_length != 0:Error_clsType.append(extract_objectId)error_flag = True# 未打刻度点位if meter_type == 'meter_v5' and pointList_length == 0:Error_withoutPointList.append(extract_objectId)error_flag = True# print(sql_schem, end=',')# print("*pointList_length:", pointList_length)elif '结果放入队列待发送' in line:result_schem = process_result(line)# print('【结果队列信息】：',end='')# print(result_schem)if result_schem['code'] == '2001':Error_loadftp.append(extract_objectId)ftpLoad_flag = Trueerror_flag = Truebreakif result_schem['desc'] == '未识别到任何油面表！':error_flag = TrueError_ymb.append(extract_objectId)else:splitContent = line.split("【Debug】")[-1]if "成功检测到表盘！表盘信息是" in splitContent:det_clsType = splitContent.split("：")[-1].strip().strip("").strip("[]").strip()if splitContent.split("：")[-1].strip().strip("") == "[]":Error_withoutDetctor.append(extract_objectId)error_flag = Trueif not 'sxb' in det_clsType and meter_type == 'paddleocr':Error_ocrDet.append(extract_objectId) error_flag = Trueif 'ywb' in det_clsType:ywb_minMax = [[-20, 140],[0, 160]]iter_minMax = [MinValue, MaxValue]if not iter_minMax in ywb_minMax:Error_minMaxSet.append(extract_objectId)error_flag = Trueelif 'xldlb' in det_clsType:xldlb_minMax = [[0, 3.0],[0, 10],[0, 9],[0, 1]]iter_minMax = [MinValue, MaxValue]if not iter_minMax in xldlb_minMax:Error_minMaxSet.append(extract_objectId)error_flag = True# if '动作次数' in splitContent:#     print(splitContent)# if '泄漏电流值' in splitContent:#     print(splitContent)if 'OCR没有检测出数字' in splitContent:Error_ocrRec.append(extract_objectId)error_flag = Trueif "没识别出指针" in splitContent:Error_withoutPointer.append(extract_objectId)error_flag = True# 画框与推理出来的油面表无匹配if '画框与推理出来的油面表无匹配' in splitContent:Error_withoutYmbMatch.append(extract_objectId)error_flag = Trueif len(ImagePath) == 0 or "MinIo中缺失该点位基准图" in splitContent:Error_withoutMinioImage.append(extract_objectId)error_flag = True# 用于验证if '读数结果' in splitContent and not contains_digit(splitContent):# not_reading_num +=1# 验证后 无读数个数和错误个数基本一致->代表验证成功# print(not_reading_num)continueif error_flag and not ftpLoad_flag: if meter_type == 'meter_v5':bj_errorNum += 1if meter_type == 'meter_ywj':ymb_errorNum += 1if meter_type == 'paddleocr':sxb_errorNum += 1error_num += 1elif error_flag and ftpLoad_flag:error_num += 1meter_type = ''print("错误总数比：【{}/{}】-> 即正确率：{}%".format(error_num,sumWorkNum,round((1-error_num/sumWorkNum)*100, 2)))# ===========================核# 写入# with open('meterLog_checking.txt', 'w') as output_file:with open('meterLog_checking-{}.txt'.format(work_id), 'w') as output_file:output_file.write('您这次序号为[{}]的任务:\n---------------------------------\n一共测试表计数量：[{}]个, 错误点位为：[{}]个, 未打点个数为：[{}]。\n<在此之中>\n，指针类表计成功占[{}/{}]个\n，油面表成功占[{}/{}]个\n，数显表成功占[{}/{}]个。'.format(work_id,sumWorkNum,error_num,len(Error_withoutId),bj_num - bj_errorNum, bj_num,ymb_num - ymb_errorNum, ymb_num, sxb_num - sxb_errorNum, sxb_num))# output_file.write("-> 即正确率：{}%".format(error_num,sumWorkNum,round((1-error_num/sumWorkNum)*100, 2)))output_file.write('\n')output_file.write('---------------------------------\n')output_file.write('NOTE：接下来，请您根据所需要查询的错误名称，使用<ctrl+F>的方式进行查询。\n')output_file.write('---------------------------------\n')output_file.write("【错误】可能存在摄像机偏移严重/模糊<数量：{}>：".format(str(len(set(Error_withoutDetctor)))) + "\n")output_file.write("\n".join(set(Error_withoutDetctor)))output_file.write('\n')output_file.write("【错误】未识别出指针<数量：{}>：".format(str(len(set(Error_withoutPointer)))) + "\n") output_file.write("\n".join(set(Error_withoutPointer)))output_file.write('\n')output_file.write("【错误】读取ftp图失败<数量：{}>：".format(str(len(set(Error_loadftp)))) + "\n")output_file.write("\n".join(set(Error_loadftp)))output_file.write('\n')output_file.write("【错误】minIO无图<数量：{}>：".format(str(len(set(Error_withoutMinioImage)))) + "\n")output_file.write("\n".join(set(Error_withoutMinioImage)))output_file.write('\n')output_file.write("【错误】该点位没有录入<数量：{}>：".format(str(len(set(Error_withoutId)))) + "\n")output_file.write("\n".join(set(Error_withoutId)))output_file.write('\n')output_file.write("【错误】表计类型录入错误<数量：{}>：".format(str(len(set(Error_clsType)))) + "\n")output_file.write("\n".join(set(Error_clsType)))output_file.write('\n')output_file.write("【错误】最大最小值未设置<数量：{}>：".format(str(len(set(Error_withoutMinMax)))) + "\n")output_file.write("\n".join(set(Error_withoutMinMax)))output_file.write('\n')output_file.write("【错误】未打刻度点位<数量：{}>：".format(str(len(set(Error_withoutPointList)))) + "\n")output_file.write("\n".join(set(Error_withoutPointList)))output_file.write('\n')output_file.write("【错误】最大最小值设置错误<数量：{}>：".format(str(len(set(Error_minMaxSet)))) + "\n")output_file.write("\n".join(set(Error_minMaxSet)))output_file.write('\n')output_file.write("【错误】存在刻度打点错误(暂未启用)<数量：{}>：".format(str(len(set(Error_PointList)))) + "\n")output_file.write("\n".join(set(Error_PointList)))output_file.write('\n')for ey in Error_ymb:if ey in Error_withoutYmbMatch:Error_ymb.remove(ey)output_file.write("【错误】未识别到任何油面<数量：{}>：".format(str(len(set(Error_ymb)))) + "\n")output_file.write("\n".join(set(Error_ymb)))output_file.write('\n')output_file.write("【错误】画框与推理结果无匹配<数量：{}>：".format(str(len(set(Error_withoutYmbMatch)))) + "\n")output_file.write("\n".join(set(Error_withoutYmbMatch)))output_file.write('\n')output_file.write("【错误】OCR没有检测出数字<数量：{}>：".format(str(len(set(Error_ocrRec)))) + "\n")output_file.write("\n".join(set(Error_ocrRec)))output_file.write('\n')output_file.write("【错误】OCR没有检测出表盘<数量：{}>：".format(str(len(set(Error_ocrDet)))) + "\n")output_file.write("\n".join(set(Error_ocrDet)))output_file.write('\n')print('<*总共统计数量:{}>\n<*过滤掉的非此次任务ID数量:{}>\n<*过滤掉的重复的点位ID数量:{}>'.format(len(debug_segments),filter_workId_num, filter_objectId_num))print('*********************************************************************')

可供实验的日志👑