SpringBoot2.0+ElasticSearch网盘搜索实现

喜欢ヅ旅行 2022-01-28 13:27 303阅读 0赞

# [SpringBoot2.0+ElasticSearch网盘搜索实现][SpringBoot2.0_ElasticSearch] #

1、ES是如何实现分布式高并发全文检索  
2、简单介绍ES分片Shards分片技术  
3、为什么ES主分片对应的备分片不在同一台节点存放  
4、索引的主分片定义好后为什么不能做修改  
5、ES如何实现高可用容错方案  
6、搭建Linux上环境三台ES高可用集群环境  
7、基于ES网盘搜索引擎实现

网盘搜索引擎，写个Job 去网上爬虫 存储起来  以JSON的格式 存放在 ES服务器

网盘搜索引擎的原理：

采用ES存储抓取的数据 抓取服务器

测试数据：

##先创建索引
    
    PUT /clouddisk
     
    ##测试数据maping
    
    POST /clouddisk/_mapping/disk
    {
      "disk":{
        "properties":{
           "name":{
             "type":"text",
             "analyzer":"ik_smart",
             "search_analyzer":"ik_smart"
           },
           "source":{
             "type":"keyword"
           },
          "describe":{
             "type":"text",
             "analyzer":"ik_smart",
             "search_analyzer":"ik_smart"
           },
           "shartime":{
             "type":"date"
           },
           "browsetimes":{
             "type":"long"
           },
          "filesize":{
             "type":"float"
           },
          "sharpeople":{
             "type":"keyword"
           },
           "collectiontime":{
             "type":"date"
           },
            "baiduaddres":{
             "type":"keyword"
           }
        }
      }
      
    }
     
     
     
     
     POST /clouddisk/disk
    {
        "name": "2018史上最全美剧",
        "source": "百度云盘",
        "describe": "该课程由小明提供",
        "shartime": "2018-10-10",
        "browsetimes": 100000,
        "filesize": 4.35,
        "sharpeople": "美国东部",
        "collectiontime": "2018-11-24",
        "baiduaddres": "https://pan.baidu.com/s/1VQxFq6JnKh0KP-5aMq-WpA#list/path=%2F"
    
    }
    
    POST /clouddisk/disk
    {
        "name": "2018史上最全韩剧",
        "source": "百度云盘",
        "describe": "该课程小丽提供",
        "shartime": "2018-10-12",
        "browsetimes": 100000,
        "filesize": 6.35,
        "sharpeople": "韩国釜山",
        "collectiontime": "2018-11-24",
        "baiduaddres": "https://pan.baidu.com/s/1VQxFq6JnKh0KP-5aMq-WpA#list/path=%2F"
    
    }
    
    POST /clouddisk/disk
    {
        "name": "老友记",
        "source": "百度云盘",
        "describe": "该课程由张三提供",
        "shartime": "2018-10-10",
        "browsetimes": 100000,
        "filesize": 1.35,
        "sharpeople": "美国",
        "collectiontime": "2018-11-24",
        "baiduaddres": "https://pan.baidu.com/s/1VQxFq6JnKh0KP-5aMq-WpA#list/path=%2F"
    
    }
    
    POST /clouddisk/disk
    {
        "name": "英语雅思",
        "source": "百度云盘",
        "describe": "该课程由大海提供",
        "shartime": "2018-10-10",
        "browsetimes": 100000,
        "filesize": 1.35,
        "sharpeople": "大海",
        "collectiontime": "2018-11-24",
        "baiduaddres": "https://pan.baidu.com/s/1VQxFq6JnKh0KP-5aMq-WpA#list/path=%2F"
    
    }
    
    POST /clouddisk/disk
    {
        "name": "小猪佩奇",
        "source": "百度云盘",
        "describe": "该课程由朱佩琦出品",
        "shartime": "2018-10-10",
        "browsetimes": 100000,
        "filesize": 1.35,
        "sharpeople": "佩奇",
        "collectiontime": "2018-11-24",
        "baiduaddres": "https://pan.baidu.com/s/1VQxFq6JnKh0KP-5aMq-WpA#list/path=%2F"
    
    }

maven:

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
      <modelVersion>4.0.0</modelVersion>
      <groupId>com.toov5</groupId>
      <artifactId>springboot-esPan</artifactId>
      <version>0.0.1-SNAPSHOT</version>
      
          <parent>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-parent</artifactId>
            <version>2.0.0.RELEASE</version>
            <relativePath /> 
        </parent>
        <dependencies>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-web</artifactId>
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
            </dependency>
            <dependency>
                <groupId>org.projectlombok</groupId>
                <artifactId>lombok</artifactId>
            </dependency>
    
            <dependency>
                <groupId>com.google.collections</groupId>
                <artifactId>google-collections</artifactId>
                <version>1.0-rc2</version>
            </dependency>
            
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-freemarker</artifactId>
            </dependency>
    
        </dependencies>
      
    </project>

Dao：

package com.toov5.dao;
    
    import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
    
    import com.toov5.entity.CloudDiskEntity;
    
    public interface CloudDiskDao extends ElasticsearchRepository<CloudDiskEntity, String> {
    
    }

Entity:

package com.toov5.entity;
    
    import org.springframework.data.annotation.Id;
    import org.springframework.data.elasticsearch.annotations.Document;
    
    import lombok.Data;
    
    @Data
    @Document(indexName = "clouddisk", type = "disk")
    public class CloudDiskEntity {
    
        @Id
        private String id;
        // 名称
        private String name;
        // 来源
        private String source;
        // 描述
        private String describe;
        // 分享时间
        private String shartime;
        // 浏览次数
        private Long browsetimes;
        // 文件大小
        private Double filesize;
        // 分享人
        private String sharpeople;
        // 收录时间
        private String collectiontime;
        // 地址
        private String baiduaddres;
    
    }

Controller:

package com.toov5.controller;
    
    import java.util.List;
    import java.util.Optional;
    
    import org.apache.commons.lang.StringUtils;
    import org.elasticsearch.index.query.BoolQueryBuilder;
    import org.elasticsearch.index.query.MatchQueryBuilder;
    import org.elasticsearch.index.query.QueryBuilders;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.web.bind.annotation.PathVariable;
    import org.springframework.web.bind.annotation.RequestMapping;
    import org.springframework.web.bind.annotation.RestController;
    
    import com.google.common.collect.Lists;
    import com.toov5.dao.CloudDiskDao;
    import com.toov5.entity.CloudDiskEntity;
    
    //SpringBoot 整合 ES
    
    @RestController
    public class CloudDiskController {
        @Autowired
        private CloudDiskDao cloudDiskDao;
        
        @RequestMapping("/findById/{id}")
        public Optional<CloudDiskEntity> findById(@PathVariable String id) {
            Optional<CloudDiskEntity> findById = cloudDiskDao.findById(id);
            return  findById;
        }
        @RequestMapping("/search")
        public List<CloudDiskEntity> search(String keyWord,String describe) {
            //创建查询   查询所有的
            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();   //创建查询
            if (!StringUtils.isEmpty(keyWord)) {
                //模糊查询  一定要用ik的中文分词插件！
                MatchQueryBuilder matchQuery = QueryBuilders.matchQuery("name", keyWord);
                boolQuery.must(matchQuery);
            }
            if (!StringUtils.isEmpty(describe)) {
                //模糊查询  一定要用ik的中文分词插件！
                MatchQueryBuilder matchQuery = QueryBuilders.matchQuery("describe", describe);
                boolQuery.must(matchQuery);
            }
            
            Iterable<CloudDiskEntity> search = cloudDiskDao.search(boolQuery); //查询所有的数据
            return Lists.newArrayList(search);  //通过这个api可以进行转换  //这样查询除了所有的 然后添加match 
        }
    }

yml:

spring:
      data:
        elasticsearch:
        ####集群名称
         cluster-name: myes
        ####地址 
         cluster-nodes: 192.168.91.7:9300
      freemarker:
        # 设置模板后缀名
        suffix: .ftl
        # 设置文档类型
        content-type: text/html
        # 设置页面编码格式
        charset: UTF-8
        # 设置页面缓存
        cache: false
        # 设置ftl文件路径
        template-loader-path:
          - classpath:/templates
      # 设置静态文件路径，js,css等
      mvc:
        static-path-pattern: /static/**

启动类：

package com.toov5;
    
    import org.springframework.boot.SpringApplication;
    import org.springframework.boot.autoconfigure.SpringBootApplication;
    import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories;
    
    @SpringBootApplication
    @EnableElasticsearchRepositories(basePackages="com.toov5.dao")   //dao 代码没有@Component的情况下可以这么玩儿 而且这样比较节省代码哈哈
    public class AppEs {
       public static void main(String[] args) {
        SpringApplication.run(AppEs.class, args);
    }
    }

关键字查询时候，可以进行关键字查询，比like强悍！

![1179709-20190122010121106-1718560461.png][]

分页查询： 很简单 传入参数 Pageable 就OK了  封装好了

GET /clouddisk/disk/_search
    {
      "from": 0,
      "size":2,
       "query": {
         "match": {
           "name": "2018"
         }
       }
      
    }

page= 0 ,请求页数从0开始   2代表size 每一页代表多少条数据

前端传来参数是第几页

package com.toov5.controller;
    
    import java.util.List;
    import java.util.Optional;
    
    import org.apache.commons.lang.StringUtils;
    import org.elasticsearch.index.query.BoolQueryBuilder;
    import org.elasticsearch.index.query.MatchQueryBuilder;
    import org.elasticsearch.index.query.QueryBuilders;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.data.domain.Page;
    import org.springframework.data.domain.Pageable;
    import org.springframework.data.web.PageableDefault;
    import org.springframework.web.bind.annotation.PathVariable;
    import org.springframework.web.bind.annotation.RequestMapping;
    import org.springframework.web.bind.annotation.RestController;
    
    import com.toov5.dao.CloudDiskDao;
    import com.toov5.entity.CloudDiskEntity;
    
    //SpringBoot 整合 ES
    
    @RestController
    public class CloudDiskController {
        @Autowired
        private CloudDiskDao cloudDiskDao;
        
        @RequestMapping("/findById/{id}")
        public Optional<CloudDiskEntity> findById(@PathVariable String id) {
            Optional<CloudDiskEntity> findById = cloudDiskDao.findById(id);
            return  findById;
        }
        @RequestMapping("/search")                  //写死了 默认值
        public Page<CloudDiskEntity> search(String keyWord,@PageableDefault(page=0,value=1) Pageable pageable) {
            //创建查询   查询所有的
            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();   //创建查询
            if (!StringUtils.isEmpty(keyWord)) {
                //模糊查询  一定要用ik的中文分词插件！
                MatchQueryBuilder matchQuery = QueryBuilders.matchQuery("name", keyWord);
                boolQuery.must(matchQuery);
            }
            
            Page<CloudDiskEntity> search = cloudDiskDao.search(boolQuery,pageable);
            return search;
        }
    }

访问：

![1179709-20190122012351187-263158092.png][]

**也可以传入 页数 size数    **

![1179709-20190122012521759-116284354.png][]

整合到前端展示：

Controller：

package com.toov5.controller;
    
    import javax.servlet.http.HttpServletRequest;
    
    import org.apache.commons.lang.StringUtils;
    import org.elasticsearch.index.query.BoolQueryBuilder;
    import org.elasticsearch.index.query.MatchQueryBuilder;
    import org.elasticsearch.index.query.QueryBuilders;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.data.domain.Page;
    import org.springframework.data.domain.Pageable;
    import org.springframework.data.web.PageableDefault;
    import org.springframework.stereotype.Controller;
    import org.springframework.web.bind.annotation.RequestMapping;
    
    import com.toov5.dao.CloudDiskDao;
    import com.toov5.entity.CloudDiskEntity;
    
    @Controller
    public class PageController {
        @Autowired
        private CloudDiskDao cloudDiskDao;
        
        @RequestMapping("/search")                  //写死了 默认值
        public String search(String keyWord,@PageableDefault(page=0,value=2) Pageable pageable, HttpServletRequest req) {
            Long starTime = System.currentTimeMillis();
            //创建查询   查询所有的
            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();   //创建查询
            if (!StringUtils.isEmpty(keyWord)) {
                //模糊查询  一定要用ik的中文分词插件！
                MatchQueryBuilder matchQuery = QueryBuilders.matchQuery("name", keyWord);
                boolQuery.must(matchQuery);
            }
            
            Page<CloudDiskEntity> page = cloudDiskDao.search(boolQuery,pageable);
            req.setAttribute("page", page);
            req.setAttribute("total", page.getTotalElements());
            req.setAttribute("keyword", keyWord);
            Long endTime = System.currentTimeMillis();
            req.setAttribute("time", endTime-starTime );
            return "search";
        }
    }

页面展示：

![1179709-20190122020841076-1790117422.png][]

![1179709-20190122021040860-2105959856.png][]

页面：

<!DOCTYPE html>
    <html xmlns="http://www.w3.org/1999/xhtml" xmlns:th="http://www.thymeleaf.org">
    <head>
        <meta charset="UTF-8">
        <title>Toov5|ES网盘搜索引擎</title>
       
    
    <link href="https://cdn.bootcss.com/bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet">
    
    
    
    <script src="https://cdn.bootcss.com/bootstrap/3.3.7/css/bootstrap-theme.min.css"></script>
    
    
    
    <script src="https://cdn.bootcss.com/jquery/2.1.1/jquery.min.js"></script>
    
    
    
    <script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>
    
    </head>
    <body  style="display: block; margin: 0 auto; width: 50%; " >
          <div style="width:100%;height:60px;" align="center">
               <h2 style="color:#985f0d;">Toov5|ES网盘搜索引擎</h2>
          </div>
          <br/>
          <div align="center">
              <span style="font-size: 18px;" >检索出${total}条数据,耗时:${time}毫秒</span>
          </div>
          <br/>
          <br/>
          <div class="bs-example" data-example-id="striped-table">
          <table class="table table-bordered table-hover">
                 <thead>
                     <tr>
                         <th style="text-align:center;" scope="row">链接名称</th>
                         <th style="text-align:center;">文件大小GB</th>
                         <th style="text-align:center;">分享人</th>
                          <th style="text-align:center;">云盘地址</th>
                     </tr>
                 </thead>
                 <tbody>
                     <#list page.content as p>
                     <tr >
                         <th style="text-align: left;" >
                          <#if keyword??>
                         ${p.name?replace(keyword, '<span style="color: red">${keyword}</span>')}
                            <#else>
                           ${p.name}
                        </#if>
    
                         
                         </th>
                         <th style="text-align: center;">${p.filesize}</th>
                         <th style="text-align: center;">${p.sharpeople}</th>
                         <th style="text-align: center;"><a href="${p.baiduaddres}">云盘地址</a> </th>
                    </tr>
                   </#list>    
                 </tbody>
          </table>
      
             
          </div>
    </body>
    </html>

分页展示数据：

package com.toov5.controller;
    
    import javax.servlet.http.HttpServletRequest;
    
    import org.apache.commons.lang.StringUtils;
    import org.elasticsearch.index.query.BoolQueryBuilder;
    import org.elasticsearch.index.query.MatchQueryBuilder;
    import org.elasticsearch.index.query.QueryBuilders;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.data.domain.Page;
    import org.springframework.data.domain.Pageable;
    import org.springframework.data.web.PageableDefault;
    import org.springframework.stereotype.Controller;
    import org.springframework.web.bind.annotation.RequestMapping;
    
    import com.toov5.dao.CloudDiskDao;
    import com.toov5.entity.CloudDiskEntity;
    
    @Controller
    public class PageController {
        @Autowired
        private CloudDiskDao cloudDiskDao;
        
        @RequestMapping("/search")                  //写死了 默认值
        public String search(String keyWord,@PageableDefault(page=0,value=2) Pageable pageable, HttpServletRequest req) {
            Long starTime = System.currentTimeMillis();
            //创建查询   查询所有的
            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();   //创建查询
            if (!StringUtils.isEmpty(keyWord)) {
                //模糊查询  一定要用ik的中文分词插件！
                MatchQueryBuilder matchQuery = QueryBuilders.matchQuery("name", keyWord);
                boolQuery.must(matchQuery);
            }
                    
            Page<CloudDiskEntity> page = cloudDiskDao.search(boolQuery,pageable);
            // 计算查询总数
            long total = page.getTotalElements();
            req.setAttribute("total", page.getTotalElements());
            // 计算分页数
            int totalPage = (int) ((total - 1) / pageable.getPageSize() + 1);
            req.setAttribute("totalPage", totalPage);
            req.setAttribute("page", page);
            req.setAttribute("total", page.getTotalElements());
            req.setAttribute("keyword", keyWord);
            Long endTime = System.currentTimeMillis();
            req.setAttribute("time", endTime-starTime );
            return "search";
        }
    }

效果：

![1179709-20190122022301134-735709816.png][]

posted @ 2019-01-21 02:51 [toov5][] 阅读( ...) 评论( ...) [编辑][Link 1] 收藏

[SpringBoot2.0_ElasticSearch]: https://www.cnblogs.com/toov5/p/10296901.html
[1179709-20190122010121106-1718560461.png]: /images/20220128/93bbca151f6c48a8ba4ed8f68ed437e2.png
[1179709-20190122012351187-263158092.png]: /images/20220128/77cd7a5cd84146da9aaaf7c6e0129481.png
[1179709-20190122012521759-116284354.png]: /images/20220128/922c1576a537459eae6f901995b5a13e.png
[1179709-20190122020841076-1790117422.png]: /images/20220128/9559377c0ff84bddaf5d55ef8f762b8f.png
[1179709-20190122021040860-2105959856.png]: /images/20220128/23745fff673c4fc6b319dae03951f85e.png
[1179709-20190122022301134-735709816.png]: /images/20220128/ce2c66faf86d42fb8a31a990cd31dcab.png
[toov5]: https://www.cnblogs.com/toov5/
[Link 1]: https://i.cnblogs.com/EditPosts.aspx?postid=10296901