基于发电厂知识问答库的检索式问答系统（python有代码）

╰半夏微凉° 2022-12-30 14:55 182阅读 0赞

之前写过基于倒排表的问答系统。  
[基于倒排表的电力调度知识问答系统构建][Link 1]  
问答系统所需要的数据已经提供，对于每一个问题都可以找得到相应的答案，所以可以理解为每一个样本数据是 <问题、答案>。 那系统的核心是当用户输入一个问题的时候，首先要找到跟这个问题最相近的已经存储在库里的问题，然后直接返回相应的答案即可。  
由于作者是学电气的，这里以发电厂知识文本来构建问答系统

该篇是低配版的问答系统，思路不如倒排表。

**思路**  
1，将发电厂知识问答数据集（问题.txt & 答案.txt）通过预处理，整合为格式规范的数据。  
2，基于词袋模型和TFIDF模型，采用余弦相似度作为度量标准，对测试问题语料库中的问题进行文本相似度计算，找出相似度较高的问题作为相似问题集合。  
3，将相似问题集合中的问题进行排序，同时返回其对应的答案给用户。

**文本数据集准备**  
答案.txt  
![20201226151557716.png_pic_center][]

问题.txt  
![2020122615171327.png_pic_center][]

**第一步：读取数据**

#第一步：读取数据
    
    def read_corpus(file):
        with open(file,'r',encoding='utf8',errors='ignore') as f:
            list = []
            lines = f.readlines()
            for i in lines:
                list.append(i)
        return list
    
    questions = read_corpus('./问题.txt')
    answers = read_corpus('./答案.txt')
    
    print('Example:')
    print('Question',questions[0])
    print('Answer',answers[0])

![20201226152349256.png_pic_center][]

**第二步：预处理**

#第二步：预处理
    import re
    import jieba
    
    
    def filter_out_category(input):
        new_input = re.sub('[\u4e00-\u9fa5]{2,5}\\/','',input) #过滤掉非汉字，即标点符号
        return new_input
    
    def filter_out_punctuation(input):
        new_input = re.sub('([a-zA-Z0-9])','',input)#过滤掉字母和数字
        new_input = ''.join(e for e in new_input if e.isalnum())
        return new_input
    
    def word_segmentation(input):
        new_input = ','.join(jieba.cut(input))#分词
        return new_input
    
    def preprocess_text(data):
        new_data = []
        for q in data:
            q = filter_out_category(q)#过滤掉符号
            q = filter_out_punctuation(q)#过滤掉字母和数字
            q = word_segmentation(q)#分词
            new_data.append(q)
        return new_data
    
    qlist = preprocess_text(questions)   # 更新后的
    print('questions after preprocess',qlist[0:3])

![20201226153623708.png_pic_center][]

**第三步：词袋模型和TFIDF模型**

#第三步：词袋模型和tf_idf模型
    from sklearn.feature_extraction.text import CountVectorizer
    from sklearn.feature_extraction.text import TfidfVectorizer
    
    #词袋模型
    def bow_extractor(corpus, ngram_range=(1, 1)):
        vectorizer = CountVectorizer(min_df=1, ngram_range=ngram_range)
        features = vectorizer.fit_transform(corpus)
        return vectorizer, features
    
    
    # # 词袋模型特征
    def conver2BOW(data):
        new_data = []
        for q in data:
            new_data.append(q)
        bow_vectorizer, bow_X = bow_extractor(new_data)
        return bow_vectorizer, bow_X
    bow_vectorizer, bow_X = conver2BOW(qlist)
    
    # print('BOW model')
    print('vectorizer',bow_vectorizer.get_feature_names())
    print('vector of text',bow_X[0:3].toarray())

![2020122615492199.png_pic_center][]

#tf_idf
    def tfidf_extractor(corpus, ngram_range=(1, 1)):
        vectorizer = TfidfVectorizer(min_df=1, norm='l2', smooth_idf=True, use_idf=True, ngram_range=ngram_range)
        features = vectorizer.fit_transform(corpus)
        return vectorizer, features
    
    
    # # tfidf 特征
    def conver2tfidf(data):
        new_data = []
        for q in data:
            new_data.append(q)
        tfidf_vectorizer, tfidf_X = tfidf_extractor(new_data)
        return tfidf_vectorizer, tfidf_X
    tfidf_vectorizer, tfidf_X = conver2tfidf(qlist)
    
    print('TFIDF model')
    print('vectorizer',tfidf_vectorizer.get_feature_names())
    print('vector of text',tfidf_X[0:3].toarray())

![20201226155113322.png_pic_center][]

**第四步：余弦相似度。**

#第四步：余弦相似度
    import numpy as np
    def idx_for_largest_cosine_sim(input, questions):
        list = []
        input = (input.toarray())[0]
        for question in questions:
            question = question.toarray()
            num = float(np.matmul(question, input))
            denom = np.linalg.norm(question) * np.linalg.norm(input)
    
            if denom ==0:
                cos = 0.0
            else:
                cos = num / denom
    
            list.append(cos)
    
        best_idx = list.index(max(list))
        return best_idx

**第五步：对测试问题语料库中的问题进行文本相似度计算，找出相似度较高的问题作为相似问题集合**

#第五步：问题求解
    #词袋模型求解
    def answer_bow(input):
        input = filter_out_punctuation(input)#对输入进行过滤字母和数字
        input = word_segmentation(input)#对输入进行分词
        bow = bow_vectorizer.transform([input])#对输入进行词袋模型
        best_idx = idx_for_largest_cosine_sim(bow, bow_X)#将输入和问答库的问题进行相似度计算，取出最好的哪一个
        return answers[best_idx]
    
    #tf-idf求解
    def answer_tfidf(input):
        input = filter_out_punctuation(input)#对输入进行过滤字母和数字
        input = word_segmentation(input)#对输入进行分词
        bow = tfidf_vectorizer.transform([input])#对输入进行tf-idf模型
        best_idx = idx_for_largest_cosine_sim(bow, tfidf_X)#将输入和问答库的问题进行相似度计算，取出最好的哪一个
        return answers[best_idx]

**第六步：测试**

#第六步：测试
    print('词袋 model',answer_bow("火电厂是什么"))
    print('tfidf model',answer_tfidf("火电厂"))

![2020122616035255.png_pic_center][]

**全部代码**

#!/usr/bin/env python3
    # -*- coding: utf-8 -*-
    # @Author: yudengwu(余登武）
    # @Date : 2020/12/26
    #@email:1344732766@qq.com
    #第一步：读取数据
    
    def read_corpus(file):
        with open(file,'r',encoding='utf8',errors='ignore') as f:
            list = []
            lines = f.readlines()
            for i in lines:
                list.append(i)
        return list
    
    questions = read_corpus('./问题.txt')
    answers = read_corpus('./答案.txt')
    
    #第二步：预处理
    import re
    import jieba
    
    
    def filter_out_category(input):
        new_input = re.sub('[\u4e00-\u9fa5]{2,5}\\/','',input) #过滤掉非汉字，即标点符号
        return new_input
    
    def filter_out_punctuation(input):
        new_input = re.sub('([a-zA-Z0-9])','',input)#过滤掉字母和数字
        new_input = ''.join(e for e in new_input if e.isalnum())
        return new_input
    
    def word_segmentation(input):
        new_input = ','.join(jieba.cut(input))#分词
        return new_input
    
    def preprocess_text(data):
        new_data = []
        for q in data:
            q = filter_out_category(q)#过滤掉符号
            q = filter_out_punctuation(q)#过滤掉字母和数字
            q = word_segmentation(q)#分词
            new_data.append(q)
        return new_data
    
    qlist = preprocess_text(questions)   # 更新后的问题
    
    
    #第三步：词袋模型和tf_idf模型
    from sklearn.feature_extraction.text import CountVectorizer
    from sklearn.feature_extraction.text import TfidfVectorizer
    
    
    def bow_extractor(corpus, ngram_range=(1, 1)):
        vectorizer = CountVectorizer(min_df=1, ngram_range=ngram_range)
        features = vectorizer.fit_transform(corpus)
        return vectorizer, features
    
    
    
    
    # # 词袋模型特征
    def conver2BOW(data):
        new_data = []
        for q in data:
            new_data.append(q)
        bow_vectorizer, bow_X = bow_extractor(new_data)
        return bow_vectorizer, bow_X
    bow_vectorizer, bow_X = conver2BOW(qlist)
    
    
    #tf_idf
    def tfidf_extractor(corpus, ngram_range=(1, 1)):
        vectorizer = TfidfVectorizer(min_df=1, norm='l2', smooth_idf=True, use_idf=True, ngram_range=ngram_range)
        features = vectorizer.fit_transform(corpus)
        return vectorizer, features
    
    
    # # tfidf 特征
    def conver2tfidf(data):
        new_data = []
        for q in data:
            new_data.append(q)
        tfidf_vectorizer, tfidf_X = tfidf_extractor(new_data)
        return tfidf_vectorizer, tfidf_X
    tfidf_vectorizer, tfidf_X = conver2tfidf(qlist)
    
    
    #第四步：余弦相似度
    import numpy as np
    def idx_for_largest_cosine_sim(input, questions):
        list = []
        input = (input.toarray())[0]
        for question in questions:
            question = question.toarray()
            num = float(np.matmul(question, input))
            denom = np.linalg.norm(question) * np.linalg.norm(input)
    
            if denom ==0:
                cos = 0.0
            else:
                cos = num / denom
    
            list.append(cos)
    
        best_idx = list.index(max(list))
        return best_idx
    
    #第五步：问题求解
    #词袋模型求解
    def answer_bow(input):
        input = filter_out_punctuation(input)#对输入进行过滤字母和数字
        input = word_segmentation(input)#对输入进行分词
        bow = bow_vectorizer.transform([input])#对输入进行词袋模型
        best_idx = idx_for_largest_cosine_sim(bow, bow_X)#将输入和问答库的问题进行相似度计算，取出最好的哪一个
        return answers[best_idx]
    
    #tf-idf求解
    def answer_tfidf(input):
        input = filter_out_punctuation(input)#对输入进行过滤字母和数字
        input = word_segmentation(input)#对输入进行分词
        bow = tfidf_vectorizer.transform([input])#对输入进行tf-idf模型
        best_idx = idx_for_largest_cosine_sim(bow, tfidf_X)#将输入和问答库的问题进行相似度计算，取出最好的哪一个
        return answers[best_idx]
    
    #第六步：测试
    print('词袋 model',answer_bow("火电厂是什么"))
    print('tfidf model',answer_tfidf("火电厂"))

**总结**  
该思路不如倒排表。有空参照下另一篇吧  
倒排表速度快。  
链接如下：

[基于倒排表的电力调度知识问答系统构建 ][Link 1]

![在这里插入图片描述][20201226160840454.jpg_pic_center]  
**作者：电力-余登武**  
![在这里插入图片描述][20201226160946961.gif_pic_center]

[Link 1]: https://blog.csdn.net/kobeyu652453/article/details/108901289
[20201226151557716.png_pic_center]: /images/20221120/af030cd8eea74240b7b87d7c59f5e1fb.png
[2020122615171327.png_pic_center]: /images/20221120/3af1f5971be74e1b93b2377e93f54b13.png
[20201226152349256.png_pic_center]: https://img-blog.csdnimg.cn/20201226152349256.png#pic_center
[20201226153623708.png_pic_center]: https://img-blog.csdnimg.cn/20201226153623708.png#pic_center
[2020122615492199.png_pic_center]: https://img-blog.csdnimg.cn/2020122615492199.png#pic_center
[20201226155113322.png_pic_center]: https://img-blog.csdnimg.cn/20201226155113322.png#pic_center
[2020122616035255.png_pic_center]: https://img-blog.csdnimg.cn/2020122616035255.png#pic_center
[20201226160840454.jpg_pic_center]: https://img-blog.csdnimg.cn/20201226160840454.jpg#pic_center
[20201226160946961.gif_pic_center]: https://img-blog.csdnimg.cn/20201226160946961.gif#pic_center