来自:https://github.com/yanyiwu/cppjieba
更新子模块:
git submodule init git submodule update
引用目录:
E:\CPP\lib\04cppjieba\deps\limonp\include E:\CPP\lib\04cppjieba\include
#include <cppjieba/Jieba.hpp> using namespace std;
初始化结巴
QStringList fenci(QString title, bool isHmm = true); //分词 子酱无圣光 true= (子酱, 无, 圣光) false= (子, 酱, 无, 圣光) QStringList jjsoft::fenci(QString title, bool isHmm) { if (jieba == nullptr)//结巴 { auto dict = QApplication::applicationDirPath() + "/dict/"; const std::string DICT_PATH = QString(dict + "jieba.dict.utf8").toStdString();//最大概率法(MPSegment: Max Probability)分词所使用的词典路径 const std::string HMM_PATH = QString(dict + "hmm_model.utf8").toStdString();//隐式马尔科夫模型(HMMSegment: Hidden Markov Model)分词所使用的词典路径 const std::string USER_DICT_PATH = QString(dict + "user.dict.utf8").toStdString();//用户自定义词典路径 const std::string IDF_PATH = QString(dict + "idf.utf8").toStdString();//IDF路径 const std::string STOP_WORD_PATH = QString(dict + "stop_words.utf8").toStdString();//停用词路径 jieba = new cppjieba::Jieba(DICT_PATH, HMM_PATH, USER_DICT_PATH, IDF_PATH, STOP_WORD_PATH); //创建结巴对象 } QStringList jg; //创建结巴类 std::string s; std::vector<std::string> words;//分词结果 s = title.toStdString(); //qDebug().noquote() << "结巴分词" << title; jieba->Cut(s, words, isHmm); //标题,分隔保存位置,是否使用HMM切割 for (int i = 0; i < words.size(); i++) { jg << QString::fromStdString(words[i]); } return jg; }
分词示例:
cppjieba::Jieba jieba(DICT_PATH, HMM_PATH, USER_DICT_PATH, IDF_PATH, STOP_WORD_PATH); vector<string> words; vector<cppjieba::Word> jiebawords; string s; string result; s = "他来到了网易杭研大厦"; qDebug().noquote() << "调试s " << QString::fromStdString(s); qDebug().noquote() << "HMM切割"; jieba.Cut(s, words, true); //标题,分隔保存位置,是否使用HMM切割 for (int i = 0; i < words.size(); i++) { qDebug().noquote() << "调试" << QString::fromStdString(words[i]); }
分词方法:
//使用HMM切割 jieba.Cut(s, words, true); //无HMM切割 jieba.Cut(s, words, false); //全部剪切 jieba.CutAll(s, words); //剪切以搜索 jieba.CutForSearch(s, words); //剪切搜索带偏移量的单词 jieba.CutForSearch(s, jiebawords, true); //插入用户单词 jieba.InsertUserWord("男默女泪"); //标签 vector<pair<string, string> > tagres; jieba.Tag(s, tagres); //关键字提取 const size_t topk = 5; vector<cppjieba::KeywordExtractor::Word> keywordres; jieba.extractor.Extract(s, keywordres, topk); //单个令牌的查找标记 const int DemoTokenMaxLen = 32; char DemoTokens[][DemoTokenMaxLen] = {"拖拉机", "CEO", "123", "。"}; vector<pair<string, string> > LookupTagres(sizeof(DemoTokens) / DemoTokenMaxLen); vector<pair<string, string> >::iterator it; for (it = LookupTagres.begin(); it != LookupTagres.end(); it++) { it->first = DemoTokens[it - LookupTagres.begin()]; it->second = jieba.LookupTag(it->first); }
结果示例:
[demo] Cut With HMM 他/来到/了/网易/杭研/大厦 [demo] Cut Without HMM 他/来到/了/网易/杭/研/大厦 我来到北京清华大学 [demo] CutAll 我/来到/北京/清华/清华大学/华大/大学 小明硕士毕业于中国科学院计算所,后在日本京都大学深造 [demo] CutForSearch 小明/硕士/毕业/于/中国/科学/学院/科学院/中国科学院/计算/计算所/,/后/在/日本/京都/大学/日本京都大学/深造 [demo] Insert User Word 男默/女泪 男默女泪 [demo] CutForSearch Word With Offset [{"word": "小明", "offset": 0}, {"word": "硕士", "offset": 6}, {"word": "毕业", "offset": 12}, {"word": "于", "offset": 18}, {"word": "中国", "offset": 21}, {"word": "科学", "offset": 27}, {"word": "学院", "offset": 30}, {"word": "科学院", "offset": 27}, {"word": "中国科学院", "offset": 21}, {"word": "计算", "offset": 36}, {"word": "计算所", "offset": 36}, {"word": ",", "offset": 45}, {"word": "后", "offset": 48}, {"word": "在", "offset": 51}, {"word": "日本", "offset": 54}, {"word": "京都", "offset": 60}, {"word": "大学", "offset": 66}, {"word": "日本京都大学", "offset": 54}, {"word": "深造", "offset": 72}] [demo] Tagging 我是拖拉机学院手扶拖拉机专业的。不用多久,我就会升职加薪,当上CEO,走上人生巅峰。 [我:r, 是:v, 拖拉机:n, 学院:n, 手扶拖拉机:n, 专业:n, 的:uj, 。:x, 不用:v, 多久:m, ,:x, 我:r, 就:d, 会:v, 升职:v, 加薪:nr, ,:x, 当上:t, CEO:eng, ,:x, 走上:v, 人生:n, 巅峰:n, 。:x] [demo] Keyword Extraction 我是拖拉机学院手扶拖拉机专业的。不用多久,我就会升职加薪,当上CEO,走上人生巅峰。 [{"word": "CEO", "offset": [93], "weight": 11.7392}, {"word": "升职", "offset": [72], "weight": 10.8562}, {"word": "加薪", "offset": [78], "weight": 10.6426}, {"word": "手扶拖拉机", "offset": [21], "weight": 10.0089}, {"word": "巅峰", "offset": [111], "weight": 9.49396}]