隆过滤器(BloomFilter)持久化

朴灿烈づ我的快乐病毒、 2021-06-24 16:09 333阅读 0赞

Bloomfilter运行在一台机器的内存上，不方便持久化（机器down掉就什么都没啦），也不方便分布式程序的统一去重。我们可以将数据进行持久化，这样就克服了down机的问题，常见的持久化方法包括持久化到本地磁盘或结合Redis进行持久化。本文主要介绍持久化到本地的操作。

pom.xml

<dependencies>    
            <dependency>    
                <groupId>com.google.guava</groupId>    
                <artifactId>guava</artifactId>    
                <version>22.0</version>    
            </dependency>    
    </dependencies>

1、数据持久化：

import java.io.File;
    import java.io.FileNotFoundException;
    import java.io.FileOutputStream;
    import java.io.IOException;
    import java.io.OutputStream;
    
    import com.google.common.hash.BloomFilter;
    import com.google.common.hash.Funnels;
    
    public class Demo1 {
        
        public static void main(String[] args) throws FileNotFoundException {
            
            BloomFilter<Integer> filter = BloomFilter.create(
                      Funnels.integerFunnel(),
                      500,
                      0.01);
            
            //导入数据到filter
            for(int i = 0; i < 100; i++ )
            {
                filter.put(i);
            }
            
            //数据持久化到本地
            File f= new File("d:" + File.separator + "test2"); 
            OutputStream out = null;
            out = new FileOutputStream(f);    
            
            try {
                filter.writeTo(out);
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            
            //测试验证
            for(int i = 0 ; i < 10; i++)
            {
                boolean result = filter.mightContain(i);
                
                if(result)
                {
                    System.out.println("i = " + i + " " + result);
                }
            }
        }
    }

2、读取持久化：

import java.io.File;
    import java.io.FileInputStream;
    import java.io.FileNotFoundException;
    import java.io.IOException;
    import java.io.InputStream;
    
    import com.google.common.hash.BloomFilter;
    import com.google.common.hash.Funnels;
    
    public class Demo2 {
    
        public static void main(String[] args) throws FileNotFoundException {
            
            BloomFilter<Integer> filter = BloomFilter.create(
                      Funnels.integerFunnel(),
                      500,
                      0.01);
            
            //将之前持久化的数据加载到Filter
            File f= new File("d:" + File.separator + "test2") ; 
            InputStream in = null;
            in = new FileInputStream(f); 
            try {
                filter = BloomFilter.readFrom(in,Funnels.integerFunnel());
            } catch (IOException e) {
                e.printStackTrace();
            }
            
            //测试验证
            for(int i = 0 ; i < 10; i++)
            {
                boolean result = filter.mightContain(i);
                
                if(result)
                {
                    System.out.println("i = " + i + " " + result);
                }
            }   
        }
    }