6.编写正排索引切分字符串|倒排索引原理|引入jieba到项目(C++)

编写正排索引

继续编写incde.hpp

#pragma once#include <iostream>
#include <string>
#include <vector>
#include <fstream>
#include <unordered_map>
#include "util.hpp"namespace ns_index{struct DocInfo{std::string title;   //文档标题std::string content; //文档对应的去标签之后的内容std::string url;     //官网文档urluint64_t dic_id;     //文档的ID}struct InvertedElem{uint64_t doc_id;std::string word;int weight;}//倒排拉链typedef std::vector<InvertedElem> InvertedList;class Index{private://正排索引的数据结构用数组，数组的下标天然是文档的IDstd::vector<DocInfo> forward_index; //正排索引//倒排索引一定是一个关键字和一组(个)InvertedElem对应[关键字和倒排拉链的映射关系]std::unordered_map<std::string, InvertedList> inverted_index;public:Index(){}~Index(){}public://根据doc_id找到文档内容DocInfo *GetForwardIdex(uint64_t doc_id){if(doc_id >= forward_index.size()){std::cerr << "doc_id out range, error" << std::endl;return nullptr;}return &forward_index[doc_id];}//根据关键字string获得倒排拉链InvertedList *GetInvertedList(const std::string &word){auto iter = inverted_index.find(word);if(iter == inverted_index.end()){std::cerr << word << " have no InvertedList" << std::endl;return nullptr;}return &(iter->second);}//根据去标签，格式化之后的文档，构建正排和倒排索引//data/raw_html/raw.txtbool BuildIndex(const std::string &input) //parse处理完毕的数据交给我{std::ifstream in(input, std::ios::in | std::ios::binary);if(!in.is_open()){std::cerr << "sorry, " << input << " open error" << std::endl;return false;}std::string line;while(std::getline(in, line)){DocInfo * doc = BuildForwardIndex(line);if(nullptr == doc){std::cerr << "build " << line << " error" << std::endl; //for debugcontinue;}BuildInvertedIndex(*doc);}return true;}private:DocInfo *BuildForwardIndex(const std::string &line){//1.解析line，字符串切分//line -> 3 string, title, content, urlstd::vector<std::string> results;const std::string sep = '\3';  //行内分隔符ns_util::StringUtil::CutString(line, &results, sep);if(results.size() != 3){return nullptr;}//2.字符串进行填充到DocInfo中DocInfo doc;doc.title = results[0]; //titledoc.content = results[1]; //contentdoc.url = results[2]; //urldoc.doc_id = forward_index.size(); //先保存id，再插入，对应的id就是当前doc在vector中的下标//3.插入到正排索引的vector中forward_index.push_back(doc);}bool BuildInvertedIndex(const DocInfo &doc){}};}

切分字符串

打开util.hpp

#pragma once
#include <iostream>
#include <vector>
#include <string>
#include <fstream>
#include <boost/algorithm/string.hpp>namespace ns_util{class FileUtil{public:static bool ReadFile(const std::string &file_path, std::string *out){std::ifstream in(file_path, std::ios::in);if(!in.is_open()){std::cerr << "open file" << file_path << " error" << std::endl;return false;}std::string line;while(std::getline(in, line)){ //如何理解getline读取到文件结束：getline的返回值是一个&，while判断的是一个bool类型，本质是因为返
回的对象当中重载了强制类型转化*out += line;}in.close();return true;}};class StringUtil{public:static void CutString(const std::string &target, std::vector<std::string> *out, const std::string &sep){//boost splitboost::split(*out, target, boost::is_any_of(sep), boost::token_compress_on); //打开压缩分隔符}};
}

打开index.hpp

DocInfo *BuildForwardIndex(const std::string &line)
{//1.解析line，字符串切分//line -> 3 string, title, content, urlstd::vector<std::string> results;const std::string sep = '\3';  //行内分隔符ns_util::StringUtil::CutString(line, &results, sep);if(results.size() != 3){return nullptr;}//2.字符串进行填充到DocInfo中DocInfo doc;doc.title = results[0]; //titledoc.content = results[1]; //contentdoc.url = results[2]; //urldoc.doc_id = forward_index.size(); //先保存id，再插入，对应的id就是当前doc在vector中的下标//3.插入到正排索引的vector中forward_index.push_back(doc);return &forward_index.back();
}bool BuildInvertedIndex(const DocInfo &doc)
{//DocInfo{title, content, url, doc_id}//word -> 倒排拉链return true;
}

倒排索引原理

struct InvertedElem{  uint64_t doc_id;  std::string word;  int weight;  
};

倒排拉链

typedef std::vector<InvertedElem> InvertedList;  

倒排索引⼀定是⼀个关键字和⼀组(个)InvertedElem对应[关键字和倒排拉链的映射关系]

std::unordered_map<std::string, InvertedList> inverted_index;

我们拿到的⽂档内容

struct DocInfo{  std::string title;   //⽂档的标题std::string content; //⽂档对应的去标签之后的内容std::string url;     //官⽹⽂档urluint64_t doc_id;     //⽂档的ID，暂时先不做过多理解
};

⽂档：

title ： 吃葡萄  
content: 吃葡萄不吐葡萄⽪  
url: http://XXXX  
doc_id: 123

根据⽂档内容，形成⼀个或者多个InvertedElem(倒排拉链)
因为当前我们是⼀个⼀个⽂档进⾏处理的，⼀个⽂档会包含多个”词“， 都应当对应到当前的doc_id

1. 需要对 title && content都要先分词 --使⽤jieba分词

title: 吃/葡萄/吃葡萄(title_word)  
content：吃/葡萄/不吐/葡萄⽪(content_word)

词和⽂档的相关性（词频：在标题中出现的词，可以认为相关性更⾼⼀些，在内容中出现相关性低⼀些）
2. 词频统计

struct word_cnt{  title_cnt;  content_cnt;  
} 
unordered_map<std::string, word_cnt> word_cnt;  
for &word : title_word{  word_cnt[word].title_cnt++; //吃（1）/葡萄（1）/吃葡萄（1）  
} 
for &word : content_word {  word_cnt[word].content_cnt++; //吃（1）/葡萄（1）/不吐（1）/葡萄⽪（1）  
} 

知道了在⽂档中，标题和内容每个词出现的次数
3. 构建倒排拉链节点，⾃定义相关性

for &word : word_cnt{  //具体⼀个词和123⽂档的对应关系，当有多个不同的词，指向同⼀个⽂档的时候，此时该优先显⽰谁？？相关性！  struct InvertedElem elem;  elem.doc_id = 123;  0elem.word = word.first;  elem.weight = 10*word.second.title_cnt + word.second.content_cnt; inverted_index[word.first].push_back(elem);  
} 

jieba的安装和使⽤–cppjieba

克隆，复制网址链接

获取链接： git clone https://gitcode.net/mirrors/yanyiwu/cppjieba.git

如何使⽤：注意细节，我们需要⾃⼰执⾏：
cd cppjieba; cp -rf deps/limonp include/cppjieba/, 不然会编译报错

[whb@VM-0-3-centos test]$ ll  
total 372  
-rwxrwxr-x 1 whb whb 366424 Mar 28 12:11 a.out  
drwxrwxr-x 8 whb whb 4096 Mar 28 12:01 cppjieba
-rw-rw-r-- 1 whb whb  856 Mar 28 12:11 demo.cpp
lrwxrwxrwx 1 whb whb  13 Mar 28 12:05 dict -> cppjieba/dict
lrwxrwxrwx 1 whb whb  16 Mar 28 12:06 inc -> cppjieba/include
-rw-rw-r-- 1 whb whb  365 Mar 28 10:21 test.cc[whb@VM-0-3-centos test]$ cat demo.cpp  
#include "inc/cppjieba/Jieba.hpp"  
#include <iostream>  
#include <string>  
#include <vector>  
using namespace std;const char* const DICT_PATH = "./dict/jieba.dict.utf8";  
const char* const HMM_PATH = "./dict/hmm_model.utf8";  
const char* const USER_DICT_PATH = "./dict/user.dict.utf8";  
const char* const IDF_PATH = "./dict/idf.utf8";  
const char* const STOP_WORD_PATH = "./dict/stop_words.utf8";int main(int argc, char** argv) {  cppjieba::Jieba jieba(DICT_PATH,  HMM_PATH,  USER_DICT_PATH,  IDF_PATH,  STOP_WORD_PATH);  vector<string> words;  string s;  s = "⼩明硕⼠毕业于中国科学院计算所，后在⽇本京都⼤学深造";  cout << s << endl;  cout << "[demo] CutForSearch" << endl;  jieba.CutForSearch(s, words);  cout << limonp::Join(words.begin(), words.end(), "/") << endl;  return EXIT_SUCCESS;  
} 
//编写倒排索引的代码  
//注意：建⽴倒排索引的时候，要忽略⼤⼩写！！

引入jieba到项目

把jieba作为工具，写入到util.hpp中

ln -s ./test/cppjieba/include/cppjieba cppjieba

ln -s ./test/cppjieba/dict dict

如果想取消链接可以使用unlink

打开util.hpp

#pragma once
#include <iostream>
#include <vector>
#include <string>
#include <fstream>
#include <boost/algorithm/string.hpp>
#include "cppjieba/Jieba.hpp"namespace ns_util{class FileUtil{public:static bool ReadFile(const std::string &file_path, std::string *out){std::ifstream in(file_path, std::ios::in);if(!in.is_open()){std::cerr << "open file" << file_path << " error" << std::endl;return false;}std::string line;while(std::getline(in, line)){ //如何理解getline读取到文件结束：getline的返回值是一个&，while判断的是一个bool类型，本质是因为返回的对象当中重载了强制类型转化*out += line;}in.close();return true;}};class StringUtil{public:static void CutString(const std::string &target, std::vector<std::string> *out, const std::string &sep){//boost splitboost::split(*out, target, boost::is_any_of(sep), boost::token_compress_on); //打开压缩分隔符}};const char* const DICT_PATH = "./dict/jieba.dict.utf8";const char* const HMM_PATH = "./dict/hmm_model.utf8";const char* const USER_DICT_PATH = "./dict/user.dict.utf8";const char* const IDF_PATH = "./dict/idf.utf8";const char* const STOP_WORD_PATH = "./dict/stop_words.utf8";class JiebaUtil{private:static cppjieba::Jieba jieba;public:static void CutString(const std::string &src, std::vector<std::string> *out){jieba.CutForSearch(src, *out);}};cppjieba::Jieba JiebaUtil::jieba(DICT_PATH, HMM_PATH, USER_DICT_PATH, IDF_PATH, STOP_WORD_PATH);
}