lucene 构建索引

迈不过友情╰ 2022-05-20 03:59 244阅读 0赞

`索引过程： 提取文本和创建文档 分析文档 向索引添加文档（倒排索引） 基本索引操作 向索引添加文档 addDocument(Document 使用默认分析器添加文档 addDocument(Document,Analyzer) -- 使用指定的分析器添加文档) 删除索引中的文档 ---deleteDocuments(Term) ---deleteDocuments(Term[]) ---deleteDocuments(Query) ---deleteDocuments(Query[]) maxDoc 和 numDocs()方法的区别：maxDoc（）返回索引中删除和未被删除的文档总数，后者返回索引中未被删除的文档总数。 更新索引中的文档 步骤：首先删除旧文档，再插入新文档。新文档必须包含旧文档中的所有域。 ---updateDocument(Term,Document) ---updateDocument(Term,Doducment,Analyzer) 域选项 域索引选项(Field.Index.*) Index.ANALYZED :使用分析器将域值分解成单独的词汇单元流，并使每个词汇单元能被单独搜索。适用于 普通文本区域。 Index.NOT_ANALYZED: 对域索引，但不对String 值进行分析。适用于分析哪些不能被拆分的域值，如URL,文件路径，日期，人名，使用于“精确匹配”搜索。 Index.ANALYZED_NORMS:一个扁体，不会在索引中存储 norms 信息。norm记录索引中的index-time boost信息。 Index.NOT_ANALYZED_NO_NORMS 不存储norms。 Index.NO：不被索引. 域存储选项 （Field.Store.*) Store.YES: 指定存储域值，用于搜索结果显示域值。 Store.NO：不存储域值。 CompressionTools：提供静态方法压缩和解压字节数组。 域项向量选项 Field对象其它初始化方法 Field(String name,Reader value,TermVector termVector): 使用reader来表示域值 域排序选项 对文档和域进行加权操作 文档加权： setBoost(float) 域加权： Field.setBoost(float); 索引数字：NumericField(name).set<Type>Value; 近实时搜索（near-real-time search) 索引优化： indexwriter优化方法 optimize() 将索引压缩至一个段，操作完成再返回 optimize(int maxNumSegments) ：部分优化，将所以压缩至最多 max个段。 optimize(boolean doWait)：true立即执行；false 后台线程调用合并程序 optimize(int maxNumSegments,boolean doWaint)：组合 消耗大量CPU和I/O资源。 Directory子类 SimpleFSDirectory NIOFSDirectory: java.nio.* MMapDirectory: 内存映射I/O进行文件访问。64位jre是好选择。 RAMDirectory：存入内存 FileSwitchDirectory:使用两个文件目录，根据文件扩展名在两个目录间切换使用。`

package com.bit.section2;
    
    import static org.junit.Assert.*;
    
    import java.io.IOException;
    
    import org.apache.lucene.analysis.WhitespaceAnalyzer;
    import org.apache.lucene.document.Document;
    import org.apache.lucene.document.Field;
    import org.apache.lucene.document.FieldSelectorResult;
    import org.apache.lucene.index.CorruptIndexException;
    import org.apache.lucene.index.IndexReader;
    import org.apache.lucene.index.IndexWriter;
    import org.apache.lucene.index.IndexWriterConfig;
    import org.apache.lucene.index.Term;
    import org.apache.lucene.search.IndexSearcher;
    import org.apache.lucene.search.Query;
    import org.apache.lucene.search.TermQuery;
    import org.apache.lucene.store.Directory;
    import org.apache.lucene.store.LockObtainFailedException;
    import org.apache.lucene.store.RAMDirectory;
    import org.apache.lucene.util.Version;
    import org.junit.Before;
    import org.junit.Test;
    
    import com.bit.util.TestUtil;
    
    public class IndexingTest {
    	
    	protected String[] ids = {"1","2"};
    	protected String[] unindexed = {"Netherlands","Italy"};
    	protected String[] unsorted = {"Amesterdam has lots of canals","Venice has lots of canals"};
    	protected String[] text = {"Amsterdam","Venice"};
    
    	private Directory directory;
    	
    	@Before
    	public void setUp() throws Exception {
    		directory = new RAMDirectory();
    		
    		IndexWriter writer = getWriter();
    		
    		for(int i=0;i<ids.length;i++){
    			Document doc = new Document();
    			doc.add(new Field("id", ids[i],Field.Store.YES,Field.Index.NOT_ANALYZED));
    			doc.add(new Field("country",unindexed[i],Field.Store.YES,Field.Index.NO));
    			doc.add(new Field("contents",unsorted[i],Field.Store.NO,Field.Index.ANALYZED));
    			doc.add(new Field("city",text[i],Field.Store.YES,Field.Index.ANALYZED));
    			writer.addDocument(doc);
    		}
    		writer.close();
    	}
    	
    	/**
    	 * 生成索引生成器
    	 * @return
    	 * @throws IOException 
    	 * @throws LockObtainFailedException 
    	 * @throws CorruptIndexException 
    	 */
    	private IndexWriter getWriter() throws CorruptIndexException, LockObtainFailedException, IOException {
    		return new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_36,new WhitespaceAnalyzer()));
    	}
    	@Test
    	public void test() {
    		fail("Not yet implemented");
    	}
    	
    	
    	protected int getHitCount(String fieldName,String searchString) throws IOException, Exception {
    		IndexSearcher searcher = new IndexSearcher(directory);
    		Term term = new Term(fieldName, searchString);
    		Query query = new TermQuery(term);
    		int hitCount = TestUtil.hitCount(searcher,query);
    		searcher.close();
    		return hitCount;
    	}
    
    	@Test
    	public void testIndexWriter() throws CorruptIndexException, LockObtainFailedException, IOException{
    		IndexWriter writer = getWriter();
    		assertEquals(ids.length, writer.numDocs());
    		writer.close();
    	}
    	
    	@Test
    	public void testIndexReader() throws CorruptIndexException, IOException{
    		IndexReader reader = IndexReader.open(directory);
    		assertEquals(ids.length, reader.maxDoc());
    		assertEquals(ids.length, reader.numDocs());
    		reader.close();
    	}
    	
    
    	/**
    	 * 测试删除文档
    	 */
    	@Test
    	public void testDeleteBeforeOptimize() throws IOException{
    		IndexWriter writer = getWriter();
    		assertEquals(2, writer.numDocs());
    		
    		//确认被标记删除的文档
    		writer.deleteDocuments(new Term("id", "1"));
    		writer.commit();
    		assertTrue(writer.hasDeletions());
    		
    		//确认删除一个文档并剩余一个文档
    		assertEquals(2, writer.maxDoc());
    		assertEquals(1, writer.numDocs());
    		writer.close();
    	}
    	
    	/**
    	 * 优化后的删除
    	 * @throws IOException
    	 */
    	@Test
    	public void testDeleteAfterOptimize() throws IOException{
    		IndexWriter writer = getWriter();
    		assertEquals(2, writer.numDocs());
    		
    		writer.deleteDocuments(new Term("id", "1"));
    		writer.optimize();
    		writer.commit();
    		
    		assertFalse(writer.hasDeletions());
    		assertEquals(1, writer.maxDoc());
    		assertEquals(1, writer.numDocs());
    		writer.close();
    	}
    	
    	/**
    	 * 更新文档
    	 * @throws Exception 
    	 */
    	@Test
    	public void testUpdate() throws Exception{
    		assertEquals(1, getHitCount("city", "Amsterdam"));
    		
    		IndexWriter writer = getWriter();
    		
    		Document doc = new Document();
    		doc.add(new Field("id", "1",Field.Store.YES,Field.Index.NOT_ANALYZED));
    		doc.add(new Field("country","Netherlands",Field.Store.YES,Field.Index.NO));
    		doc.add(new Field("contents","Den Haag has lots of museums",Field.Store.NO,Field.Index.ANALYZED));
    		doc.add(new Field("city","DenHaag",Field.Store.YES,Field.Index.ANALYZED));
    		
    		writer.updateDocument(new Term("id","1"),doc );
    		
    		writer.commit();
    		
    		assertEquals(0, getHitCount("city", "Amsterdam"));
    		assertEquals(1, getHitCount("city", "DenHaag"));
    	}
    	
    }