日志分析删除

日志分析

场景

运维嫌弃生产环境打印日志过多，而且日志存储需要费用，让我们减少打印日志大小，所以需要分析日志在哪里打印的过多

解决方案

读取生产日志文件，统计分析打印日志的地方，最后删除代码中打印日志的地方

实现-LogParAnalyzer

public class LogParAnalyzer {//日志原始文件private File log;private List<Pattern> list;private ExecutorService executorService;//生成的分割文件private String subPath = "D:\\split\\";private List<File> files;public LogParAnalyzer(File log, List<String> patterns) {this.log = log;executorService = Executors.newFixedThreadPool(30);list = new ArrayList<>();try {for (String pattern : patterns) {Pattern p = Pattern.compile(pattern);list.add(p);}} catch (Exception e) {throw new RuntimeException(e);}}public void analyze() throws Exception {// 使用 try-with-resources 自动关闭 BufferedReaderint chunkSize = 100000;try (BufferedReader reader = new BufferedReader(new FileReader(log))) {File file = new File(subPath);if (!file.exists()) {file.mkdirs();}String line;List<CompletableFuture<?>> task = new ArrayList<>();int cur = 0;List<String> list = new ArrayList<>();AtomicInteger batch = new AtomicInteger(0);while ((line = reader.readLine()) != null) {//sb 会通过Arrays.copy复制字节数组，内存频繁复制list.add(line);cur++;if ((cur % chunkSize) == 0) {//深拷贝List<String> tt = list.stream().map(String::new).collect(Collectors.toList());list.clear();CompletableFuture f =CompletableFuture.runAsync(() -> processChunk(tt, batch.get()), executorService);task.add(f);batch.incrementAndGet();}}if (list.size()>0) {CompletableFuture f =CompletableFuture.runAsync(() -> processChunk(list, batch.get()), executorService);task.add(f);}//等待所有任务结束CompletableFuture.allOf(task.toArray(new CompletableFuture[0])).get();System.out.println("task execute finished");}}private void processChunk(List<String> lines, int batch) {try {System.out.println(Thread.currentThread().getName()+" execute "+ batch+".txt start");Map<String, AtomicInteger> map = new HashMap<>();try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(subPath + batch + ".txt"))) {lines.forEach(line -> {for (Pattern pattern : list) {Matcher matcher = pattern.matcher(line);if (matcher.find()) {String group = matcher.group(0);map.computeIfAbsent(group, k -> new AtomicInteger(1)).incrementAndGet();}}if (map.size() > 0) {//每个文件只保存100前100条writeBatchToFile(writer, map);}});}System.out.println(Thread.currentThread().getName()+" execute "+ batch+".txt end");} catch (Exception e) {e.printStackTrace();}}private void writeBatchToFile(BufferedWriter writer, Map<String, AtomicInteger> map) {Map<String, AtomicInteger> limit = limit(map, 100);try {for (Map.Entry<String, AtomicInteger> entry : limit.entrySet()) {writer.write(entry.getKey() + "=" + entry.getValue().get());writer.newLine();}} catch (IOException e) {e.printStackTrace();}//清除缓存map.clear();limit.clear();}public void mergeAndSort() throws Exception {files = Files.list(Paths.get(subPath)).map(Path::toFile).filter(f -> f.length() > 0).collect(Collectors.toList());// 创建 ForkJoinPoolForkJoinPool forkJoinPool = new ForkJoinPool();MergeFileTask mergeFileTask = new MergeFileTask(files.toArray(new File[0]), forkJoinPool);Path finalPath = mergeFileTask.invoke();System.out.println("final path: " + finalPath.toAbsolutePath());try (BufferedReader reader = Files.newBufferedReader(finalPath)) {String line;while ((line = reader.readLine()) != null) {System.out.println(line);}}mergeFileTask.finished();}public void finished() throws IOException {if (!CollectionUtils.isEmpty(files)){files.stream().parallel().forEach(File::delete);}Files.deleteIfExists(Paths.get(subPath));}public Map<String, AtomicInteger> limit(Map<String, AtomicInteger> map, int limit) {// 排序并过滤结果return map.entrySet().stream().sorted(Map.Entry.comparingByValue(Comparator.comparing(AtomicInteger::get).reversed())).limit(limit).collect(Collectors.toMap(Map.Entry::getKey,Map.Entry::getValue,(oldValue, newValue) -> oldValue, // 解决键冲突LinkedHashMap::new));}}

测试

@Test
public void bb() throws Exception {StopWatch stopWatch = new StopWatch();stopWatch.start();File log = new File("E:\\log_total.log\\log_total.log");//2023-09-26 11:10:00.123 INFO - none --- [main] com.example.service.UserService.create:42 - User service started successfully.//匹配出 com.example.service.UserService.create:42//c.y.c.w..*指文件名太长，缩写List<String> list = Arrays.asList("(com\\.xx\\.[\\w\\.\\*]*:\\d*)", "(c\\.y\\.c\\.[\\w\\.\\*]*:\\d*)");LogParAnalyzer logAnalyzer = new LogParAnalyzer(log, list);logAnalyzer.analyze();logAnalyzer.mergeAndSort();logAnalyzer.finished();stopWatch.stop();System.out.println(stopWatch.prettyPrint());//最后统计格式//c.y.c.s.service.impl.Service.complete:98 count: 6
}

good luck!