谷粒商城----ES篇

一、product-es准备

P128

ES在内存中，所以在检索中优于mysql。ES也支持集群，数据分片存储。

需求：

• 上架的商品才可以在网站展示。
• 上架的商品需要可以被检索。

分析sku在es中如何存储

商品mapping

分析：商品上架在es中是存sku还是spu？

1）、检索的时候输入名字，是需要按照sku的title进行全文检索的
2）、检素使用商品规格，规格是spu的公共属性，每个spu是一样的
3）、按照分类id进去的都是直接列出spu的，还可以切换。
4〕、我们如果将sku的全量信息保存到es中（包括spu属性〕就太多字段了

方案1：

{skuId:1spuId:11skyTitile:华为xxprice:999saleCount:99attr:[{尺寸:5},{CPU:高通945},{分辨率:全高清}]
缺点：如果每个sku都存储规格参数(如尺寸)，会有冗余存储，因为每个sku对应的spu的规格参数都一样

方案2：

sku索引
{spuId:1skuId:11
}
attr索引
{skuId:11attr:[{尺寸:5},{CPU:高通945},{分辨率:全高清}]
}
先找到4000个符合要求的spu，再根据4000个spu查询对应的属性，封装了4000个id，long 8B*4000=32000B=32KB
1K个人检索，就是32MB结论：如果将规格参数单独建立索引，会出现检索时出现大量数据传输的问题，会引起网络网络

🚩 因此选用方案1，以空间换时间

建立product索引
最终选用的数据模型：

• { “type”: “keyword” }, # 保持数据精度问题，可以检索，但不分词
• “analyzer”: “ik_smart” # 中文分词器
• “index”: false, # 不可被检索，不生成index
• “doc_values”: false # 默认为true，不可被聚合，es就不会维护一些聚合的信息

PUT product
{"mappings":{"properties": {"skuId":{ "type": "long" },"spuId":{ "type": "keyword" },  # 不可分词"skuTitle": {"type": "text","analyzer": "ik_smart"  # 中文分词器},"skuPrice": { "type": "keyword" },  # 保证精度问题"skuImg"  : { "type": "keyword" },  # 视频中有false"saleCount":{ "type":"long" },"hasStock": { "type": "boolean" },"hotScore": { "type": "long"  },"brandId":  { "type": "long" },"catalogId": { "type": "long"  },"brandName": {"type": "keyword"}, # 视频中有false"brandImg":{"type": "keyword","index": false,  # 不可被检索，不生成index，只用做页面使用"doc_values": false # 不可被聚合，默认为true},"catalogName": {"type": "keyword" }, # 视频里有false"attrs": {"type": "nested","properties": {"attrId": {"type": "long"  },"attrName": {"type": "keyword","index": false,"doc_values": false},"attrValue": {"type": "keyword" }}}}}
}

如果检索不到商品，自己用postman测试一下，可能有的字段需要更改，你也可以把没必要的"keyword"去掉

冗余存储的字段：不用来检索，也不用来分析，节省空间

库存是bool。
检索品牌id，但是不检索品牌名字、图片
用skuTitle检索

二、nested嵌入式对象

属性是"type": “nested”,因为是内部的属性进行检索

数组类型的对象会被扁平化处理（对象的每个属性会分别存储到一起）

user.name=["aaa","bbb"]
user.addr=["ccc","ddd"]这种存储方式，可能会发生如下错误：
错误检索到{aaa,ddd}，这个组合是不存在的

数组的扁平化处理会使检索能检索到本身不存在的，为了解决这个问题，就采用了嵌入式属性，数组里是对象时用嵌入式属性（不是对象无需用嵌入式属性）

三、商品上架（ES保存）

   @Override // SpuInfoServiceImplpublic void up(Long spuId) {// 1 组装数据 查出当前spuId对应的所有sku信息List<SkuInfoEntity> skus = skuInfoService.getSkusBySpuId(spuId);// 查询这些sku是否有库存List<Long> skuids = skus.stream().map(sku -> sku.getSkuId()).collect(Collectors.toList());// 2 封装每个sku的信息// 3.查询当前sku所有可以被用来检索的规格属性List<ProductAttrValueEntity> baseAttrs = attrValueService.baseAttrListForSpu(spuId);// 得到基本属性idList<Long> attrIds = baseAttrs.stream().map(attr -> attr.getAttrId()).collect(Collectors.toList());// 过滤出可被检索的基本属性id，即search_type = 1 [数据库中目前 4、5、6、11不可检索]Set<Long> ids = new HashSet<>(attrService.selectSearchAttrIds(attrIds));// 可被检索的属性封装到SkuEsModel.Attrs中List<SkuEsModel.Attrs> attrs = baseAttrs.stream().filter(item -> ids.contains(item.getAttrId())).map(item -> {SkuEsModel.Attrs attr = new SkuEsModel.Attrs();BeanUtils.copyProperties(item, attr);return attr;}).collect(Collectors.toList());// 每件skuId是否有库存Map<Long, Boolean> stockMap = null;try {// 3.1 远程调用库存系统 查询该sku是否有库存R hasStock = wareFeignService.getSkuHasStock(skuids);// 构造器受保护 所以写成内部类对象stockMap = hasStock.getData(new TypeReference<List<SkuHasStockVo>>() {}).stream().collect(Collectors.toMap(SkuHasStockVo::getSkuId, item -> item.getHasStock()));log.warn("服务调用成功" + hasStock);} catch (Exception e) {log.error("库存服务调用失败: 原因{}", e);}Map<Long, Boolean> finalStockMap = stockMap;//防止lambda中改变// 开始封装esList<SkuEsModel> skuEsModels = skus.stream().map(sku -> {SkuEsModel esModel = new SkuEsModel();BeanUtils.copyProperties(sku, esModel);esModel.setSkuPrice(sku.getPrice());esModel.setSkuImg(sku.getSkuDefaultImg());// 4 设置库存，只查是否有库存，不查有多少if (finalStockMap == null) {esModel.setHasStock(true);} else {esModel.setHasStock(finalStockMap.get(sku.getSkuId()));}// TODO 1.热度评分  刚上架是0esModel.setHotScore(0L);// 设置品牌信息BrandEntity brandEntity = brandService.getById(esModel.getBrandId());esModel.setBrandName(brandEntity.getName());esModel.setBrandImg(brandEntity.getLogo());// 查询分类信息CategoryEntity categoryEntity = categoryService.getById(esModel.getCatalogId());esModel.setCatalogName(categoryEntity.getName());// 保存商品的属性，  查询当前sku的所有可以被用来检索的规格属性，同一spu都一样，在外面查一遍即可esModel.setAttrs(attrs);return esModel;}).collect(Collectors.toList());// 5.发给ES进行保存  gulimall-searchR r = searchFeignService.productStatusUp(skuEsModels);if (r.getCode() == 0) {// 远程调用成功baseMapper.updateSpuStatus(spuId, ProductConstant.StatusEnum.SPU_UP.getCode());} else {// 远程调用失败 TODO 接口幂等性 重试机制/*** Feign 的调用流程  Feign有自动重试机制* 1. 发送请求执行* 2.*/}}

@Slf4j
@Service
public class ProductSaveServiceImpl implements ProductSaveService {@Resourceprivate RestHighLevelClient client;/*** 将数据保存到ES* 用bulk代替index，进行批量保存* BulkRequest bulkRequest, RequestOptions options*/@Override // ProductSaveServiceImplpublic boolean productStatusUp(List<SkuEsModel> skuEsModels) throws IOException {// 1. 批量保存BulkRequest bulkRequest = new BulkRequest();// 2.构造保存请求for (SkuEsModel esModel : skuEsModels) {// 设置es索引 gulimall_productIndexRequest indexRequest = new IndexRequest(EsConstant.PRODUCT_INDEX);// 设置索引idindexRequest.id(esModel.getSkuId().toString());// json格式String jsonString = JSON.toJSONString(esModel);indexRequest.source(jsonString, XContentType.JSON);// 添加到文档bulkRequest.add(indexRequest);}// bulk批量保存BulkResponse bulk = client.bulk(bulkRequest, GuliESConfig.COMMON_OPTIONS);// TODO 是否拥有错误boolean hasFailures = bulk.hasFailures();if(hasFailures){List<String> collect = Arrays.stream(bulk.getItems()).map(item -> item.getId()).collect(Collectors.toList());log.error("商品上架错误：{}",collect);}return hasFailures;}
}

PUT product
{"mappings": {"properties": {"skuId":{"type": "long"},"spuId":{"type": "keyword"},"skuTitle":{"type": "text","analyzer": "ik_smart"},"skuPrice":{"type": "keyword"},"skuImg":{"type": "keyword","index": false,"doc_values": false},"saleCount":{"type": "long"},"hasStock":{"type": "boolean"},"hotScore":{"type": "long"},"brandId":{"type": "long"},"catalogId":{"type": "long"},"brandName":{"type":"keyword","index": false,"doc_values": false},"brandImg":{"type": "keyword","index": false,"doc_values": false},"catalogName":{"type": "keyword","index": false,"doc_values": false},"attrs":{"type": "nested","properties": {"attrId":{"type":"long"},"attrName":{"type":"keyword","index":false,"doc_values": false},"attrValue":{"type":"keyword"}}}}}
}

四、检索服务

package com.atguigu.gulimall.search.service.impl;import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.TypeReference;
import com.atguigu.common.to.es.SkuEsModel;
import com.atguigu.common.utils.R;
import com.atguigu.gulimall.search.config.GuliESConfig;
import com.atguigu.gulimall.search.constant.EsConstant;
import com.atguigu.gulimall.search.feign.ProductFeignService;
import com.atguigu.gulimall.search.service.SearchService;
import com.atguigu.gulimall.search.vo.AttrResponseVo;
import com.atguigu.gulimall.search.vo.BrandVo;
import com.atguigu.gulimall.search.vo.SearchParam;
import com.atguigu.gulimall.search.vo.SearchResult;
import lombok.extern.slf4j.Slf4j;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.NestedQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;import javax.annotation.Resource;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;/*** <p>Title: MallServiceImpl</p>* Description：* date：2020/6/12 23:06*/
@Slf4j
@Service
public class SearchServiceImpl implements SearchService {@Resourceprivate RestHighLevelClient restHighLevelClient;@Resourceprivate ProductFeignService productFeignService;@Overridepublic SearchResult search(SearchParam Param) {SearchResult result = null;// 1.准备检索请求SearchRequest searchRequest = buildSearchRequest(Param);try {// 2.执行检索请求SearchResponse response = restHighLevelClient.search(searchRequest, GuliESConfig.COMMON_OPTIONS);// 3.分析响应数据result = buildSearchResult(response, Param);} catch (IOException e) {e.printStackTrace();}return result;}/*** 准备检索请求  [构建查询语句]*/private SearchRequest buildSearchRequest(SearchParam Param) {// 帮我们构建DSL语句的SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();// 1. 模糊匹配 过滤(按照属性、分类、品牌、价格区间、库存) 先构建一个布尔Query// 1.1 mustBoolQueryBuilder boolQuery = QueryBuilders.boolQuery();if(!StringUtils.isEmpty(Param.getKeyword())){boolQuery.must(QueryBuilders.matchQuery("skuTitle",Param.getKeyword()));}// 1.2 bool - filter Catalog3Idif(StringUtils.isEmpty(Param.getCatalog3Id() != null)){boolQuery.filter(QueryBuilders.termQuery("catalogId", Param.getCatalog3Id()));}// 1.2 bool - brandId [集合]if(Param.getBrandId() != null && Param.getBrandId().size() > 0){boolQuery.filter(QueryBuilders.termsQuery("brandId", Param.getBrandId()));}// 属性查询if(Param.getAttrs() != null && Param.getAttrs().size() > 0){for (String attrStr : Param.getAttrs()) {BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();String[] s = attrStr.split("_");// 检索的id  属性检索用的值String attrId = s[0];String[] attrValue = s[1].split(":");boolQueryBuilder.must(QueryBuilders.termQuery("attrs.attrId", attrId));boolQueryBuilder.must(QueryBuilders.termsQuery("attrs.attrValue", attrValue));// 构建一个嵌入式Query 每一个必须都得生成嵌入的 nested 查询NestedQueryBuilder attrsQuery = QueryBuilders.nestedQuery("attrs", boolQueryBuilder, ScoreMode.None);boolQuery.filter(attrsQuery);}}// 1.2 bool - filter [库存]if(Param.getHasStock() != null){boolQuery.filter(QueryBuilders.termQuery("hasStock",Param.getHasStock() == 1));}// 1.2 bool - filter [价格区间]if(!StringUtils.isEmpty(Param.getSkuPrice())){RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("skuPrice");String[] s = Param.getSkuPrice().split("_");if(s.length == 2){// 有三个值 就是区间rangeQuery.gte(s[0]).lte(s[1]);}else if(s.length == 1){// 单值情况if(Param.getSkuPrice().startsWith("_")){rangeQuery.lte(s[0]);}if(Param.getSkuPrice().endsWith("_")){rangeQuery.gte(s[0]);}}boolQuery.filter(rangeQuery);}// 把以前所有条件都拿来进行封装sourceBuilder.query(boolQuery);// 1.排序if(!StringUtils.isEmpty(Param.getSort())){String sort = Param.getSort();// sort=hotScore_asc/descString[] s = sort.split("_");SortOrder order = s[1].equalsIgnoreCase("asc") ? SortOrder.ASC : SortOrder.DESC;sourceBuilder.sort(s[0], order);}// 2.分页 pageSize ： 5sourceBuilder.from((Param.getPageNum()-1) * EsConstant.PRODUCT_PASIZE);sourceBuilder.size(EsConstant.PRODUCT_PASIZE);// 3.高亮if(!StringUtils.isEmpty(Param.getKeyword())){HighlightBuilder builder = new HighlightBuilder();builder.field("skuTitle");builder.preTags("<b style='color:red'>");builder.postTags("</b>");sourceBuilder.highlighter(builder);}// 聚合分析// TODO 1.品牌聚合TermsAggregationBuilder brand_agg = AggregationBuilders.terms("brand_agg");brand_agg.field("brandId").size(50);// 品牌聚合的子聚合brand_agg.subAggregation(AggregationBuilders.terms("brand_name_agg").field("brandName").size(1));brand_agg.subAggregation(AggregationBuilders.terms("brand_img_agg").field("brandImg").size(1));// 将品牌聚合加入 sourceBuildersourceBuilder.aggregation(brand_agg);// TODO 2.分类聚合TermsAggregationBuilder catalog_agg = AggregationBuilders.terms("catalog_agg").field("catalogId").size(20);catalog_agg.subAggregation(AggregationBuilders.terms("catalog_name_agg").field("catalogName").size(1));// 将分类聚合加入 sourceBuildersourceBuilder.aggregation(catalog_agg);// TODO 3.属性聚合 attr_agg 构建嵌入式聚合NestedAggregationBuilder attr_agg = AggregationBuilders.nested("attr_agg", "attrs");// 3.1 聚合出当前所有的attrIdTermsAggregationBuilder attrIdAgg = AggregationBuilders.terms("attr_id_agg").field("attrs.attrId");// 3.1.1 聚合分析出当前attrId对应的attrNameattrIdAgg.subAggregation(AggregationBuilders.terms("attr_name_agg").field("attrs.attrName").size(1));// 3.1.2 聚合分析出当前attrId对应的所有可能的属性值attrValue	这里的属性值可能会有很多 所以写50attrIdAgg.subAggregation(AggregationBuilders.terms("attr_value_agg").field("attrs.attrValue").size(50));// 3.2 将这个子聚合加入嵌入式聚合attr_agg.subAggregation(attrIdAgg);sourceBuilder.aggregation(attr_agg);log.info("\n构建语句：->\n" + sourceBuilder.toString());SearchRequest searchRequest = new SearchRequest(new String[]{EsConstant.PRODUCT_INDEX}, sourceBuilder);return searchRequest;}/*** 构建结果数据 指定catalogId 、brandId、attrs.attrId、嵌入式查询、倒序、0-6000、每页显示两个、高亮skuTitle、聚合分析*/private SearchResult buildSearchResult(SearchResponse response, SearchParam Param) {SearchResult result = new SearchResult();// 1.返回的所有查询到的商品SearchHits hits = response.getHits();List<SkuEsModel> esModels = new ArrayList<>();if(hits.getHits() != null &&  hits.getHits().length > 0){for (SearchHit hit : hits.getHits()) {String sourceAsString = hit.getSourceAsString();// ES中检索得到的对象SkuEsModel esModel = JSON.parseObject(sourceAsString, SkuEsModel.class);if(!StringUtils.isEmpty(Param.getKeyword())){// 1.1 获取标题的高亮属性HighlightField skuTitle = hit.getHighlightFields().get("skuTitle");String highlightFields = skuTitle.getFragments()[0].string();// 1.2 设置文本高亮esModel.setSkuTitle(highlightFields);}esModels.add(esModel);}}result.setProducts(esModels);// 2.当前所有商品涉及到的所有属性信息ArrayList<SearchResult.AttrVo> attrVos = new ArrayList<>();ParsedNested attr_agg = response.getAggregations().get("attr_agg");ParsedLongTerms attr_id = attr_agg.getAggregations().get("attr_id_agg");for (Terms.Bucket bucket : attr_id.getBuckets()) {SearchResult.AttrVo attrVo = new SearchResult.AttrVo();// 2.1 得到属性的idattrVo.setAttrId(bucket.getKeyAsNumber().longValue());// 2.2 得到属性的名字String attr_name = ((ParsedStringTerms) bucket.getAggregations().get("attr_name_agg")).getBuckets().get(0).getKeyAsString();attrVo.setAttrName(attr_name);// 2.3 得到属性的所有值List<String> attr_value = ((ParsedStringTerms) bucket.getAggregations().get("attr_value_agg")).getBuckets().stream().map(item -> item.getKeyAsString()).collect(Collectors.toList());attrVo.setAttrValue(attr_value);attrVos.add(attrVo);}result.setAttrs(attrVos);// 3.当前所有商品涉及到的所有品牌信息ArrayList<SearchResult.BrandVo> brandVos = new ArrayList<>();ParsedLongTerms brand_agg = response.getAggregations().get("brand_agg");for (Terms.Bucket bucket : brand_agg.getBuckets()) {SearchResult.BrandVo brandVo = new SearchResult.BrandVo();// 3.1 得到品牌的idlong brnadId = bucket.getKeyAsNumber().longValue();brandVo.setBrandId(brnadId);// 3.2 得到品牌的名String brand_name = ((ParsedStringTerms) bucket.getAggregations().get("brand_name_agg")).getBuckets().get(0).getKeyAsString();brandVo.setBrandName(brand_name);// 3.3 得到品牌的图片String brand_img = ((ParsedStringTerms) (bucket.getAggregations().get("brand_img_agg"))).getBuckets().get(0).getKeyAsString();brandVo.setBrandImg(brand_img);brandVos.add(brandVo);}result.setBrands(brandVos);// 4.当前商品所有涉及到的分类信息ParsedLongTerms catalog_agg = response.getAggregations().get("catalog_agg");List<SearchResult.CatalogVo> catalogVos = new ArrayList<>();for (Terms.Bucket bucket : catalog_agg.getBuckets()) {// 设置分类idSearchResult.CatalogVo catalogVo = new SearchResult.CatalogVo();catalogVo.setCatalogId(Long.parseLong(bucket.getKeyAsString()));// 得到分类名ParsedStringTerms catalog_name_agg = bucket.getAggregations().get("catalog_name_agg");String catalog_name = catalog_name_agg.getBuckets().get(0).getKeyAsString();catalogVo.setCatalogName(catalog_name);catalogVos.add(catalogVo);}result.setCatalogs(catalogVos);// ================以上信息从聚合信息中获取// 5.分页信息-页码result.setPageNum(Param.getPageNum());// 总记录数long total = hits.getTotalHits().value;result.setTotal(total);// 总页码：计算得到int totalPages = (int)(total / EsConstant.PRODUCT_PASIZE + 0.999999999999);result.setTotalPages(totalPages);// 设置导航页ArrayList<Integer> pageNavs = new ArrayList<>();for (int i = 1;i <= totalPages; i++){pageNavs.add(i);}result.setPageNavs(pageNavs);// 6.构建面包屑导航功能if(Param.getAttrs() != null){List<SearchResult.NavVo> navVos = Param.getAttrs().stream().map(attr -> {SearchResult.NavVo navVo = new SearchResult.NavVo();String[] s = attr.split("_");navVo.setNavValue(s[1]);R r = productFeignService.getAttrsInfo(Long.parseLong(s[0]));// 将已选择的请求参数添加进去 前端页面进行排除result.getAttrIds().add(Long.parseLong(s[0]));if(r.getCode() == 0){AttrResponseVo data = r.getData(new TypeReference<AttrResponseVo>(){});navVo.setName(data.getAttrName());}else{// 失败了就拿id作为名字navVo.setName(s[0]);}// 拿到所有查询条件 替换查询条件String replace = replaceQueryString(Param, attr, "attrs");navVo.setLink("http://search.gulimall.com/list.html?" + replace);return navVo;}).collect(Collectors.toList());result.setNavs(navVos);}// 品牌、分类if(Param.getBrandId() != null && Param.getBrandId().size() > 0){List<SearchResult.NavVo> navs = result.getNavs();SearchResult.NavVo navVo = new SearchResult.NavVo();navVo.setName("品牌");// TODO 远程查询所有品牌R r = productFeignService.brandInfo(Param.getBrandId());if(r.getCode() == 0){List<BrandVo> brand = r.getData("data", new TypeReference<List<BrandVo>>() {});StringBuffer buffer = new StringBuffer();// 替换所有品牌IDString replace = "";for (BrandVo brandVo : brand) {buffer.append(brandVo.getBrandName() + ";");replace = replaceQueryString(Param, brandVo.getBrandId() + "", "brandId");}navVo.setNavValue(buffer.toString());navVo.setLink("http://search.gulimall.com/list.html?" + replace);}navs.add(navVo);}return result;}/*** 替换字符* key ：需要替换的key*/private String replaceQueryString(SearchParam Param, String value, String key) {String encode = null;try {encode = URLEncoder.encode(value,"UTF-8");// 浏览器对空格的编码和java的不一样encode = encode.replace("+","%20");encode = encode.replace("%28", "(").replace("%29",")");} catch (UnsupportedEncodingException e) {e.printStackTrace();}return Param.get_queryString().replace("&" + key + "=" + encode, "");}
}