基于TextRank算法提取关键词——Java实现

素颜马尾好姑娘i 2022-06-06 08:17 337阅读 0赞

## 依赖 ##

<dependency>
         <groupId>com.janeluo</groupId>
         <artifactId>ikanalyzer</artifactId>
         <version>2012_u6</version>
     </dependency>

## 代码 ##

import org.apache.lucene.analysis.Analyzer;
    import org.apache.lucene.analysis.TokenStream;
    import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
    import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
    import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
    import org.wltea.analyzer.lucene.IKAnalyzer;
    
    import java.io.IOException;
    import java.io.StringReader;
    import java.util.*;
    
    /**
     * @author yuyufeng
     * @date 2017/11/3
     */
    public class Demo {
        public static void main(String[] args) {
    
            List<String> keyWords = new ArrayList<>();
            int k = 2;  //窗口大小/2
            float d = 0.85f;
            /**
             * 标点符号、常用词、以及“名词、动词、形容词、副词之外的词”
             */
            Set<String> stopWordSet = new HashSet<String>();
            stopWordSet.add("是");
            stopWordSet.add("的");
            stopWordSet.add("地");
            stopWordSet.add("从");
            stopWordSet.add("将");
            stopWordSet.add("但");
            stopWordSet.add("都");
            stopWordSet.add("和");
            stopWordSet.add("为");
            stopWordSet.add("让");
            stopWordSet.add("在");
            stopWordSet.add("由");
            stopWordSet.add("上");
            String field = "PageRank近似于一个用户，是指在Internet上随机地单击链接将会到达特定网页的可能性。通常，能够从更多地方到达的网页更为重要，因此具有更高的PageRank。每个到其他网页的链接，都增加了该网页的PageRank。具有较高PageRank的网页一般都是通过更多其他网页的链接而提高的。";
    
    
            Analyzer analyzer = new IKAnalyzer(true);
            TokenStream ts = null;
            //分词
            try {
                ts = analyzer.tokenStream("myfield", new StringReader(field));
                OffsetAttribute offset = (OffsetAttribute) ts.addAttribute(OffsetAttribute.class);
                CharTermAttribute term = (CharTermAttribute) ts.addAttribute(CharTermAttribute.class);
                TypeAttribute type = (TypeAttribute) ts.addAttribute(TypeAttribute.class);
                ts.reset();
    
                while (ts.incrementToken()) {
                    if (!stopWordSet.contains(term.toString())) {
                        keyWords.add(term.toString());
                    }
                }
                ts.end();
            } catch (IOException var14) {
                var14.printStackTrace();
            } finally {
                if (ts != null) {
                    try {
                        ts.close();
                    } catch (IOException var13) {
                        var13.printStackTrace();
                    }
                }
    
            }
    
            Map<String, Set<String>> relationWords = new HashMap<>();
    
    
            //获取每个关键词 前后k个的组合
            for (int i = 0; i < keyWords.size(); i++) {
                String keyword = keyWords.get(i);
                Set<String> keySets = relationWords.get(keyword);
                if (keySets == null) {
                    keySets = new HashSet<>();
                    relationWords.put(keyword, keySets);
                }
    
                for (int j = i - k; j <= i + k; j++) {
                    if (j < 0 || j >= keyWords.size() || j == i) {
                        continue;
                    } else {
                        keySets.add(keyWords.get(j));
                    }
                }
            }
    
           /* for (String s : relationWords.keySet()) {
                System.out.print(s+" ");
                for (String s1 : relationWords.get(s)) {
                    System.out.print(s1+" ");
                }
                System.out.println();
            }*/
    
    
            Map<String, Float> score = new HashMap<>();
            float min_diff = 0.1f; //差值最小
            int max_iter = 100;//最大迭代次数
    
            //迭代
            for (int i = 0; i < max_iter; i++) {
                Map<String, Float> m = new HashMap<>();
                float max_diff = 0;
                for (String key : relationWords.keySet()) {
                    Set<String> value = relationWords.get(key);
                    //先给每个关键词一个默认rank值
                    m.put(key, 1 - d);
                    //一个关键词的TextRank由其它成员投票出来
                    for (String other : value) {
                        int size = relationWords.get(other).size();
                        if (key.equals(other) || size == 0) {
                            continue;
                        } else {
                            m.put(key, m.get(key) + d / size * (score.get(other) == null ? 0 : score.get(other)));
                        }
                    }
    //                System.out.println("m.get(key):"+m.get(key)+" score:"+(score.get(key) == null ? 0 : score.get(key)));
                    max_diff = Math.max(max_diff, Math.abs(m.get(key) - (score.get(key) == null ? 0 : score.get(key))));
                }
    
                score = m;
                if (max_diff <= min_diff) {
                    System.out.println("迭代次数：" + i);
                    break;
                }
            }
    
            List<Score> scores = new ArrayList<>();
            for (String s : score.keySet()) {
                Score score1 = new Score();
                score1.key = s;
                score1.significance = score.get(s);
                scores.add(score1);
            }
    
            scores.sort(new Comparator<Score>() {
                @Override
                public int compare(Score o1, Score o2) {
                    if (o2.significance - o1.significance > 0) {
                        return 1;
                    } else {
                        return -1;
                    }
    
                }
            });
    
            for (Score score1 : scores) {
                System.out.println(score1);
            }
    
        }
    }
    
    class Score {
        String key;
        float significance;
    
        @Override
        public String toString() {
            return "关键词=" + key +
                    ", 重要程度=" + significance;
        }
    }

> 运行结果  
> 迭代次数：11
> 
> 关键词=网页, 重要程度=2.8311346  
> 关键词=链接, 重要程度=1.646728  
> 关键词=pagerank, 重要程度=1.6038197  
> 关键词=更多, 重要程度=1.2489531  
> 关键词=到达, 重要程度=1.1083827  
> 关键词=具有, 重要程度=0.98187566  
> 关键词=其他, 重要程度=0.9651773  
> 关键词=用户, 重要程度=0.81595975  
> 关键词=指在, 重要程度=0.8086006  
> 关键词=internet, 重要程度=0.80388165  
> 关键词=一个, 重要程度=0.787644  
> 关键词=随机, 重要程度=0.7764552  
> 关键词=单击, 重要程度=0.76052386  
> 关键词=将会, 重要程度=0.71690917  
> 关键词=能够, 重要程度=0.7066941  
> 关键词=可能性, 重要程度=0.70503104  
> 关键词=更高, 重要程度=0.7045265  
> 关键词=每个, 重要程度=0.7005399  
> 关键词=特定, 重要程度=0.6963727  
> 关键词=通过, 重要程度=0.69495517  
> 关键词=因此, 重要程度=0.69311315  
> 关键词=通常, 重要程度=0.69245243  
> 关键词=该, 重要程度=0.6918771  
> 关键词=一般, 重要程度=0.6895788  
> 关键词=都是, 重要程度=0.686642  
> 关键词=到, 重要程度=0.68152785  
> 关键词=更为重要, 重要程度=0.68064004  
> 关键词=地方, 重要程度=0.6771895  
> 关键词=近似于, 重要程度=0.6137907  
> 关键词=而, 重要程度=0.594995  
> 关键词=增加了, 重要程度=0.5508093  
> 关键词=较高, 重要程度=0.5392841  
> 关键词=提高, 重要程度=0.44995427

### 在线体验 ###

[http://www.toptool.top/index.html?tool=keywords.html][http_www.toptool.top_index.html_tool_keywords.html]

[http_www.toptool.top_index.html_tool_keywords.html]: http://www.toptool.top/index.html?tool=keywords.html