Redis的BitMap实现分布式布隆过滤器

布隆过滤器（Bloom Filter）是一种高效的概率型数据结构，用于判断一个元素是否属于一个集合。它通过使用哈希函数和位数组来存储和查询数据，具有较快的插入和查询速度，并且占用空间相对较少。

引入依赖

<!--切面-->
<dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-aop</artifactId>
</dependency><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-data-redis</artifactId>
</dependency><!-- 数据库-->
<dependency><groupId>mysql</groupId><artifactId>mysql-connector-java</artifactId><version>${mysql.version}</version>
</dependency>
<dependency><groupId>com.baomidou</groupId><artifactId>mybatis-plus-boot-starter</artifactId><version>3.5.1</version>
</dependency>

properties配置

spring.datasource.url=jdbc:mysql://127.0.0.1:3306/itcast?serverTimezone=GMT%2B8&useUnicode=true&logger=Slf4JLogger&characterEncoding=utf8&autoReconnect=true&allowMultiQueries=true
spring.datasource.username=root
spring.datasource.password=root123
spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver
spring.datasource.hikari.pool-name=HikariCPDatasource
spring.datasource.hikari.minimum-idle=5
spring.datasource.hikari.idle-timeout=180000
spring.datasource.hikari.maximum-pool-size=10
spring.datasource.hikari.auto-commit=true
spring.datasource.hikari.max-lifetime=1800000
spring.datasource.hikari.connection-timeout=30000
spring.datasource.hikari.connection-test-query=SELECT 1
mybatis-plus.configuration.log-impl= org.apache.ibatis.logging.stdout.StdOutImpl
spring.redis.host=127.0.0.1
spring.redis.port=6379
spring.redis.timeout=10s
spring.redis.password=123

自定义注解

import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;/*** 此注解可作用于控制器方法，或者服务类方法** 使用示例，在目标方法上添加如下注解* <pre>*     BitMap(key = "user", id = "#id")* </pre>**/
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
@Inherited
public @interface BitMap {/*** <p>同一类集合的唯一标识符 商品表、订单表分别设置不同key</p>*/String key();/*** 支持{@code SPEL}表达式* 含义是以被调用方法参数<code>id</code>的值作为主键ID*/String id() default "#id";
}

切面

import com.example.demo.annotation.BitMap;
import com.example.demo.util.ParserUtils;
import com.example.demo.util.RedisBitMapUtils;
import com.example.demo.util.ResponseResult;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.aspectj.lang.reflect.MethodSignature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;import javax.annotation.Resource;
import java.lang.reflect.Method;
import java.util.TreeMap;/*** Redis BitMap AOP**/
@Aspect
@Component
public class BitMapAspect {private static final Logger logger = LoggerFactory.getLogger(BitMapAspect.class);@Resourceprivate RedisBitMapUtils redisBitMapUtils;@Pointcut("@annotation(com.example.demo.annotation.BitMap)")public void aspect() {}@Around("aspect()")public Object doAround(ProceedingJoinPoint point) throws Throwable {// 通过 point 对象获取方法签名信息。MethodSignature signature = (MethodSignature) point.getSignature();// 通过方法签名获取当前方法对象。Method method = signature.getMethod();// 获取当前方法上的 BitMap 注解。BitMap annotation = method.getAnnotation(BitMap.class);// 获取方法参数名和参数值的映射关系，并将结果保存到TreeMap中。TreeMap<String, Object> map = ParserUtils.createTreeMap(point, signature);// 从参数映射中获取 id 参数对应的值。String idString = ParserUtils.parse(annotation.id(), map);if (idString != null) {long id = Long.parseLong(idString);if (redisBitMapUtils.isPresent(annotation.key(), id)) {return point.proceed();} else {logger.info(String.format("当前主键ID{%d}不存在", id));return method.getReturnType().equals(ResponseResult.class) ? ResponseResult.okResult() : null;}}throw new RuntimeException("主键ID解析不正确，请按照参考格式书写");}}

RedisBitMap工具类

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.stereotype.Component;import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.io.Serializable;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;/*** {@link RedisBitMapUtils}工具类*/
@Component
public class RedisBitMapUtils {private static final Logger logger = LoggerFactory.getLogger(RedisBitMapUtils.class);@Resourceprivate   StringRedisTemplate stringRedisTemplate ;ValueOperations<String, String> opsForValue;@PostConstructpublic  void init() {opsForValue= stringRedisTemplate.opsForValue();}/*** 该方法可以方便地将一个集合中的每个元素根据给定的映射函数进行转换，* 并返回一个新的列表。如果集合为空或为null，则返回一个空列表。* @param list* @param action* @return* @param <T>* @param <R>*/public  <T, R> List<R> toList(final Collection<T> list, final Function<? super T, ? extends R> action) {Objects.requireNonNull(action);if (Objects.nonNull(list)) {return list.stream().map(action).collect(Collectors.toList());}return Collections.emptyList();}/*** <p>本方法在首次初始化以{@code key}为参数的BitMap时执行</p>* <p>首先删除Key 然后重新构建BitMap</p>** @param <T> 主键类型* @param key 每种业务分别对应不同的Key名称* @param ids 主键ID*/public  <T extends Serializable> void init(String key, Collection<T> ids) {remove(key);setBit(key, ids);}/*** <p>本方法在首次初始化以{@code key}为参数的BitMap时执行</p>* <p>首先删除Key 然后重新构建BitMap</p>** @param key    每种业务分别对应不同的Key名称* @param list   实体类对象集合* @param action 主键列（方法引用表示）* @param <T>    实体类泛型* @param <R>    主键列泛型*/public  <T, R extends Serializable> void init(String key, Collection<T> list, Function<T, R> action) {List<R> ids = toList(list, action);init(key, ids);}/*** 检查当前主键ID在Redis BitMap中是否存在 如果存在则执行函数式回调** @param key 每种业务分别对应不同的Key名称* @param id  主键ID* @return {@code R}实例*/public  <T extends Serializable, R> R ifPresent(String key, T id, Function<T, R> action) {if (getBit(key, id)) {return action.apply(id);}return null;}/*** 检查当前主键ID在Redis BitMap中是否存在 如果存在则执行函数式回调** @param key 每种业务分别对应不同的Key名称* @param id  主键ID* @return {@code R}实例*/public  <T extends Serializable, R> R ifPresent(String key, T id, Supplier<R> supplier) {if (getBit(key, id)) {return supplier.get();}return null;}/*** 检查当前主键ID在Redis BitMap中是否存在 如果存在则返回<code>true</code>** @param key 每种业务分别对应不同的Key名称* @param id  主键ID* @return 如果存在则返回<code>true</code>*/public  <T extends Serializable> boolean isPresent(String key, T id) {return getBit(key, id);}/*** 检查当前主键ID在Redis BitMap中是否存在 如果存在则返回<code>true</code>* 本方法是{@link RedisBitMapUtils#getBit(String, Serializable)}的别名方法 方便对外调用** @param key 每种业务分别对应不同的Key名称* @param id  主键ID* @return 如果存在则返回<code>true</code>*/public  <T extends Serializable> boolean checkId(String key, T id) {return getBit(key, id);}/*** 检查当前主键ID（集合）在Redis BitMap中是否存在 只返回存在的主键ID* 本方法是{@link RedisBitMapUtils#getBit(String, Serializable)}的别名方法 方便对外调用** @param key 每种业务分别对应不同的Key名称* @param ids 主键ID* @return 返回存在的主键ID*/public  <T extends Serializable> List<T> checkIds(String key, Collection<T> ids) {return ids.stream().filter(e -> checkId(key, e)).collect(Collectors.toList());}/*** 向Redis BitMap中保存主键ID** @param key 每种业务分别对应不同的Key名称* @param id  主键ID*/public  <T extends Serializable> void setBit(String key, T id) {ifOffsetValid(Objects.hash(id), e -> opsForValue.setBit(key, e, true));}/*** 向Redis BitMap中批量保存主键ID** @param <T> 主键类型* @param key 每种业务分别对应不同的Key名称* @param ids 主键ID*/public  <T extends Serializable> void setBit(String key, Collection<T> ids) {ids.forEach(id -> ifOffsetValid(Objects.hash(id), e -> opsForValue.setBit(key, e, true)));}/*** 检查当前主键ID在Redis BitMap中是否存在 如果存在则返回<code>true</code>** @param key 每种业务分别对应不同的Key名称* @param id  主键ID* @return 如果存在则返回<code>true</code>*/public  <T extends Serializable> boolean getBit(String key, T id) {return ifOffsetValid(Objects.hash(id), e -> opsForValue.getBit(key, e));}/*** 从Redis BitMap中删除当前主键ID** @param key 每种业务分别对应不同的Key名称* @param id  主键ID*/public  <T extends Serializable> void removeBit(String key, T id) {ifOffsetValid(Objects.hash(id), e -> opsForValue.setBit(key, e, false));}/*** 从Redis BitMap中批量删除主键ID** @param key 每种业务分别对应不同的Key名称* @param ids 主键ID* @param <T> 主键类型*/public  <T extends Serializable> void removeBit(String key, Collection<T> ids) {ids.forEach(id -> ifOffsetValid(Objects.hash(id), e -> opsForValue.setBit(key, e, false)));}/*** 将当前分类下的BitMap Key删除* 清空该Key下所有数据*/public  void remove(String key) {stringRedisTemplate.delete(key);}/*** <p>检查偏移量是否合法</p>* <p>Redis字符串支持字符串最大长度512M，因此支持offset的最大值为(2^32)-1</p>** @param offset 偏移量* @param action 映射规则*/private static <N extends Number> Boolean ifOffsetValid(N offset, Function<N, Boolean> action) {Objects.requireNonNull(action);//如果ID用整型表示 那么正整数范围内所有的ID均有效 最大正整数值为2147483647 约为20亿long max = (1L << 32) - 1;if (offset.intValue() >= 0 && offset.intValue() < Integer.MAX_VALUE) {return action.apply(offset);} else {// 如果偏移量类型为长整型，或者整型范围内的最大值小于0且 offset 的值小于等于 maxif (Integer.MAX_VALUE >= 0 && offset.longValue() <= max) {return action.apply(offset);} else {logger.info(String.format("偏移量{%d}越界[0,%s]，本次操作不成功！", offset.longValue(), max));return false;}}}
}

response工具类

import lombok.Data;import java.io.Serializable;
@Data
public class ResponseResult<T> implements Serializable {private Boolean success;private Integer code;private String msg;private T data;public ResponseResult() {this.success=true;this.code = HttpCodeEnum.SUCCESS.getCode();this.msg = HttpCodeEnum.SUCCESS.getMsg();}public ResponseResult(Integer code, T data) {this.code = code;this.data = data;}public ResponseResult(Integer code, String msg, T data) {this.code = code;this.msg = msg;this.data = data;}public ResponseResult(Integer code, String msg) {this.code = code;this.msg = msg;}public static ResponseResult errorResult(int code, String msg) {ResponseResult result = new ResponseResult();return result.error(code, msg);}public static ResponseResult okResult() {ResponseResult result = new ResponseResult();return result;}public static ResponseResult okResult(int code, String msg) {ResponseResult result = new ResponseResult();return result.ok(code, null, msg);}public static ResponseResult setHttpCodeEnum(HttpCodeEnum enums) {return okResult(enums.getCode(), enums.getMsg());}public static <T> ResponseResult<T> error(String message) {return new ResponseResult<T>(HttpCodeEnum.SYSTEM_ERROR.getCode(),  message);}public ResponseResult<?> error(Integer code, String msg) {this.success=false;this.code = code;this.msg = msg;return this;}public ResponseResult<?> ok(Integer code, T data) {this.success=true;this.code = code;this.data = data;return this;}public ResponseResult<?> ok(Integer code, T data, String msg) {this.success=true;this.code = code;this.data = data;this.msg = msg;return this;}public static ResponseResult ok(Object data) {ResponseResult result = new ResponseResult();result.setData(data);return result;}}

controller

import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.example.demo.annotation.BitMap;
import com.example.demo.annotation.PreventRepeatSubmit;
import com.example.demo.mapper.StuMapper;
import com.example.demo.model.ResponseResult;
import com.example.demo.model.Student;
import com.example.demo.util.RedisBitMapUtils;
import org.springframework.data.redis.core.ListOperations;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.data.redis.core.script.DefaultRedisScript;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;@RestController
@RequestMapping("/test")
@Validated
public class TestController {@Resourceprivate StuMapper stuMapper;@Resourceprivate StringRedisTemplate stringRedisTemplate;private static final String BITMAP_STU="bitmap_stu";@Resourceprivate RedisBitMapUtils redisBitMapUtils;@GetMapping("init")public com.example.demo.util.ResponseResult init(){List<Student> studentList = stuMapper.selectList(new QueryWrapper<Student>());redisBitMapUtils.init(BITMAP_STU,studentList,Student::getId);return com.example.demo.util.ResponseResult.okResult();}/*** 编程式*/@GetMapping("selectStu1/{id}")@BitMap(key = BITMAP_STU,id = "#id")public com.example.demo.util.ResponseResult selectStu1(@PathVariable Integer id){return com.example.demo.util.ResponseResult.ok(stuMapper.selectById(id));}/*** 注解式*/@GetMapping("selectStu2/{id}")public com.example.demo.util.ResponseResult selectStu2(@PathVariable Integer id){if (redisBitMapUtils.getBit(BITMAP_STU,id)){return com.example.demo.util.ResponseResult.ok(stuMapper.selectById(id));}return com.example.demo.util.ResponseResult.okResult();}}

测试

初始化biemap数据,从数据库种获取所有id并导入redis的key为bitmap_stu的数据

测试数据库存在的数据id：1，走数据库获取数据 

测试数据库的数据id：200，因为id为200不存在在数据库中，所以没有走数据库，减少了数据库压力 

 注解式也生效

达到了使用redis的bitmap实现分布式的布隆过滤器，过滤掉bitmap不存在的数据