写的一款简易的热点词汇记录工具

项目需要对用户提交的附件、文章、搜索框内容等做热词分析。如下图：

公司有大数据团队。本着不麻烦别人就不麻烦别人的原则，写了一款简易的记录工具，原理也简单，手工在业务插入锚点，用分词器分好词，排掉字母、数字、符号、敏感词。将词汇按年度累加记录到数据库中即可。代码如下:

@Component
public class HotWordHelper {private static HotWordMapper hotWordMapper;static List<Character> FILTER_CHARS = new ArrayList<>();static {String number = "123456789abcdefghijklnmopqrstuvwxyzABCDEFGHIJKLNMOPQRSTUVWXYZ";char[] chars = number.toCharArray();for (char aChar : chars) {FILTER_CHARS.add(aChar);}}@Autowiredpublic ZYHotWordHelper(HotWordMapper hotWordMapper) {ZYHotWordHelper.hotWordMapper = hotWordMapper;}public static List<HotWord> loaderHotWordTen(String moduleCode) {LocalDate now = LocalDate.now();int year = now.getYear();return loaderHotWord(year, 10, moduleCode);}public static List<HotWord> loaderHotWord(int top, String moduleCode) {LocalDate now = LocalDate.now();int year = now.getYear();return loaderHotWord(year, top, moduleCode);}public static List<HotWord> loaderHotWord(int year, int top, String moduleCode) {LambdaQueryWrapper<HotWord> wrapper = Wrappers.lambdaQuery();wrapper.eq(HotWord::getRecordYear, year);wrapper.eq(HotWord::getModuleCode, moduleCode);return hotWordMapper.selectTop(wrapper, HotWord::getAppearTimes, top);}// 直接词汇，如字典之类的。public static void putDirectHotWord(String text, String moduleCode) {// 为不影响主业务速度，改成异常Runnable runnable = () -> putHotWord(true, text, moduleCode);AsyncExecutor.execute(runnable);}// 分析词汇public static void putAnalyzeHotWord(String text, String moduleCode) {// 为不影响主业务速度，改成异常Runnable runnable = () -> putHotWord(false, text, moduleCode);AsyncExecutor.execute(runnable);}// 附件public static void putAttachmentAsync(StringsField attachmentIds, String moduleCode) {if (ZYListUtils.isEmptyList(attachmentIds)) {return;}Runnable runnable = () -> doPutAttachmentAsync(attachmentIds, moduleCode);AsyncExecutor.execute(runnable);}// 解析附件private static void doPutAttachmentAsync(StringsField attachmentIds, String moduleCode) {FileInfoMapper fileInfoMapper = SpringUtils.getBean(FileInfoMapper.class);List<FileInfo> fileInfos = fileInfoMapper.selectBatchIds(attachmentIds);if (ZYListUtils.isEmptyList(fileInfos)) {return;}FileStoreService storeService= ZYSpringUtils.getBean(FileStoreService.class);List<FileWrapper> fileWrappers = ZYListUtils.list2list(fileInfos, FileInfo::toFileWrapper);for (FileWrapper fileWrapper : fileWrappers) {try (InputStream objectStream = storeService.getObjectStream(fileWrapper)) {String text = IOUtils.toString(objectStream, StandardCharsets.UTF_8);putAnalyzeHotWord(text, moduleCode);} catch (Exception e) {return;}}}private static void putHotWord(boolean isDirect, String text, String moduleCode) {if (ZYStrUtils.isAnyNull(text, moduleCode)) {return;}List<String> words = analyzerWords(isDirect, text);if (ZYListUtils.isEmptyList(words)) {return;}//  List<String> smallWordCompare = new ArrayList<>(words);words.removeIf(w -> {if (!matchLength(w)) {return true;}char[] chars = w.toCharArray();for (char aChar : chars) {// 不要数字字母if (FILTER_CHARS.contains(aChar)) {return true;}}// 存在误判，还是不用这段代码/*      for (String compareWord : smallWordCompare) {if (!w.equals(compareWord) && compareWord.contains(w)) {return true;}}
*/return false;});Map<String, Integer> wordCount = ZYMapUtils.countField(words, w -> w);LocalDate now = LocalDate.now();int year = now.getYear();LambdaQueryWrapper<HotWord> wrapper = Wrappers.lambdaQuery();wrapper.in(HotWord::getHotWord, words);wrapper.eq(HotWord::getRecordYear, year);wrapper.eq(HotWord::getModuleCode, moduleCode);List<HotWord> existsWords = hotWordMapper.selectList(wrapper);Map<String, HotWord> wordIdContainer = ZYListUtils.groupModel(existsWords, HotWord::getHotWord);List<HotWord> addHotWords = new ArrayList<>();List<HotWord> editHotWords = new ArrayList<>();wordCount.forEach((w, times) -> {HotWord hotWord = wordIdContainer.get(w);if (null != hotWord) {Integer appearTimes = hotWord.getAppearTimes();appearTimes += times;hotWord.setAppearTimes(appearTimes);editHotWords.add(hotWord);} else {HotWord newHotWord = new HotWord();newHotWord.setRecordYear(year);newHotWord.setAppearTimes(times);newHotWord.setHotWord(w);newHotWord.setModuleCode(moduleCode);addHotWords.add(newHotWord);}});if (ZYListUtils.isNotEmptyList(addHotWords)) {hotWordMapper.insertBatch(addHotWords);}if (ZYListUtils.isNotEmptyList(editHotWords)) {for (HotWord editHotWord : editHotWords) {hotWordMapper.updateById(editHotWord);}}}private static List<String> analyzerWords(boolean isDirect, String text) {List<String> words;if (isDirect) {words = Collections.singletonList(text);} else {words = ZYDirtyWordHelper.analyze(text);if (matchLength(text) && !words.contains(text)) {words.add(text);}}return words;}private static boolean matchLength(String text) {int length = text.length();return length > 1 && length < 6;}}

记录表中效果图,实际效果还阔以，实时性和记录速度都非常快。