redis测试时插入千万数据教程

灰太狼 2022-09-04 11:51 1阅读 0赞

### redis测试时插入千万数据教程 ###

*  前提
 *  过程
 *   *   *  造数到文件
         *   *   *  JedisClusterPipeTest.java
         *  FileUtils
         *  每个节点pipe到redis

# 前提 #

redis集群6个节点，其中3个主节点，后面会向这个集群插入千万级别数据。

nodes.add(new HostAndPort("192.168.11.180",6379));
        nodes.add(new HostAndPort("192.168.11.180",6380));
        nodes.add(new HostAndPort("192.168.11.180",6381));
        nodes.add(new HostAndPort("192.168.11.180",6389));
        nodes.add(new HostAndPort("192.168.11.180",6390));
        nodes.add(new HostAndPort("192.168.11.180",6391));

# 过程 #

### 造数到文件 ###

##### JedisClusterPipeTest.java #####

package org.example;
    
    import org.example.util.FileUtils;
    import org.junit.Test;
    import redis.clients.jedis.*;
    import redis.clients.util.JedisClusterCRC16;
    
    import java.util.*;
    
    public class JedisClusterPipeTest {
        // 获取所有master节点的slot范围
        private TreeMap<Long, String> getAllMasterSlotRange(String host, int port) {
            TreeMap<Long, String> slotTree = new TreeMap<>();
            try{
                // redis带密码时
    //            JedisShardInfo jedisShardInfo = new JedisShardInfo(host, port);
    //            jedisShardInfo.setPassword("xxxxx");
    //            Jedis jedis = new Jedis(jedisShardInfo);
                // redis不带密码时
                Jedis jedis = new Jedis(host, port);
                // 得到所有槽位： List<List<List<Object>>> list; list.sub.sub(2). get(0):get(1) 为ip和端口
                // 具体结构信息，可以打断点自己查看
                List<Object> list = jedis.clusterSlots();
                for (Object object : list) {
                    List<Object> subList = (List<Object>) object;
                    List<Object> master = (List<Object>) subList.get(2);
                    String masterIp = new String((byte[]) master.get(0)) + ":" + master.get(1);
                    long startSlot = (long) subList.get(0);
                    long endSlot = (long) subList.get(1);
    
                    slotTree.put(startSlot, masterIp);
                    slotTree.put(endSlot, masterIp);
                }
                jedis.close();
            }catch(Exception e){
                e.printStackTrace();
            }
            return slotTree;
        }
    
        // 为每台服务器造数，key为server ip，value为造的数据集合。
        private Map<String, List<String>> getData(TreeMap<Long, String> slotHostMap, int start, int end) {
            // 为下面的key前缀分布造1000w数据
            String[] keyPres = {"key1_","key_2"};
            // key为server的ip，value为预计插入该服务器的key集合
            Map<String, List<String>> serverKeyMap = new HashMap<>();
            // key为slot id，value为该id对应的server ip
            Map<Integer, String> slotServer = new HashMap<>();
            for (String keyPre: keyPres) {
                for (int i = start; i < end; i++) {
                    String key = keyPre + i;
                    //获取槽号
                    int slot = JedisClusterCRC16.getSlot(key);
                    if (!slotServer.containsKey(slot)) {
                        slotServer.put(slot, slotHostMap.lowerEntry(Long.valueOf(slot + 1)).getValue());
                    }
                    String server = slotServer.get(slot);
                    // 把key与server ip对应起来
                    serverKeyMap.computeIfAbsent(server, serverId -> new ArrayList<>(10000)).add(key);
                }
            }
            return serverKeyMap;
        }
    
        @Test
        public void testBuildDataCmdToFile () {
            // 随意传入redis集群中一个节点ip，即可查到所有节点的slot分布
            TreeMap<Long, String> slotHostMap = getAllMasterSlotRange("192.168.11.180", 6379);
    
            System.out.println(slotHostMap.toString());
            if (slotHostMap.isEmpty()) {
                System.out.println("slot empty！");
                return;
            }
    
            // 一下子造数太多，占用太多内存，分批进行
            for (int i = 0; i < 1000_0000; i += 10_0000) {
                // 造数
                Map<String, List<String>> data = getData(slotHostMap, i, i + 10_0000);
    
                // 写入文件
                String values = " key_def_value";
                for (Map.Entry<String, List<String>> entry: data.entrySet()) {
                    // 生成命令，并拼接，待写入文件
                    StringBuilder sb = new StringBuilder();
                    for (String key: entry.getValue()) {
                        sb.append("sadd ").append(key).append(values).append("\n");
                    }
                    // entry.getKey()为server ip，比如"192.168.11.180:6379"
                    // 造的数据比较大的时候，超过idea的内存时，可以传入true，即可以追加的方式写入文件
                    FileUtils.writeText("D:\\" + entry.getKey().replace(":", "-") + "-1000w.txt", sb.toString(), true);
                }
            }
        }
    
    }

### FileUtils ###

public static void writeText(String filename, String content, boolean isAppend) {
            FileOutputStream outputStream = null;
            OutputStreamWriter outputStreamWriter = null;
            BufferedWriter writer = null;
            try {
                outputStream = new FileOutputStream(filename, isAppend);
                outputStreamWriter = new OutputStreamWriter(outputStream);
                writer = new BufferedWriter(outputStreamWriter);
                writer.write(content);
            } catch (IOException e) {
                e.printStackTrace();
            }finally {
                if(writer != null){
                    try {
                        writer.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
                if (outputStreamWriter != null){
                    try {
                        outputStreamWriter.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
                if (outputStream != null){
                    try {
                        outputStream.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }
        }

### 每个节点pipe到redis ###

上面程序执行后，可以看到D盘生成3个文件，总共数据量2000w，即set结构的key2000w个。  
![在这里插入图片描述][ff6cc861f49343a4925870e6db647dd4.png]  
上传到centos服务器/opt，执行：

cat /opt/192.168.11.180-6379-1000w.txt | redis-cli -h 192.168.11.180 -p 6379 --pipe
    cat /opt/192.168.11.180-6380-1000w.txt | redis-cli -h 192.168.11.180 -p 6380 --pipe
    cat /opt/192.168.11.180-6381-1000w.txt | redis-cli -h 192.168.11.180 -p 6381 --pipe

若服务器带密码，可以下面这样

cat /opt/192.168.11.180-6379-1000w.txt | redis-cli -h 192.168.11.180 -p 6379 -a password --pipe

–pipe无法应用于集群，但可以应用于集群的单台机器

[ff6cc861f49343a4925870e6db647dd4.png]: /images/20220829/b7aca08ed23c45f6a0a3cb2f2187f0fb.png