elasticsearch7.2深度分页实现(查询全部数据)

亦凉 2022-12-06 15:19 152阅读 0赞

用过es的朋友应该都了解es的分页模式总共有两种，一种是from,size模式，也就是常说的浅分页，这种分页模式使用起来很简单，和mysql的十分类似，但是这种方式的最大弊端是，在集群模式下，如果查询的分页数很深，很容易造成查询效率极慢甚至内存的异出的问题，解决这个问题就要提到深分页，深分页的大概思路是每一页查询会给你返回一个scrollId，类似于一个游标，记录本次查询的位置，下次使用这个有游标再去查下一页的数据，所有不会出现跨度大，查询数据多导致溢出的问题，但是它也有一个弊端就是不支持跳页。还有就是es默认的查询（也就是不加from,size）是会返回第一页10条数据，而我最近正在使用es做一个导出的功能，那么很明显默认返回一页的数据不是很友好，所有正好使用深分页可以解决。废话少说，上代码，先说下from,size的方法，这里都是使用HighLevelRestClient实现的。

// comprehensiveQueryEventVO是我封装的一个查询条件实体对象，这里是封装查询条件
    private SearchSourceBuilder getSearchSourceBuilder(ComprehensiveQueryEventVO comprehensiveQueryEventVO){
            SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
            if (comprehensiveQueryEventVO.getPageNum() != null && comprehensiveQueryEventVO.getPageSize() != null) {
                sourceBuilder.from((comprehensiveQueryEventVO.getPageNum()-1)*comprehensiveQueryEventVO.getPageSize());
                sourceBuilder.size(comprehensiveQueryEventVO.getPageSize());
            }
    SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
            if (comprehensiveQueryEventVO.getPageNum() != null && comprehensiveQueryEventVO.getPageSize() != null) {
                sourceBuilder.from((comprehensiveQueryEventVO.getPageNum()-1)*comprehensiveQueryEventVO.getPageSize());
                sourceBuilder.size(comprehensiveQueryEventVO.getPageSize());
            }
            // 拼接动态查询条件
            if (StringUtils.isNotEmpty(comprehensiveQueryEventVO.getEventCode())) {
                boolBuilder.must(QueryBuilders.termQuery("eventCode", comprehensiveQueryEventVO.getEventCode()));
            }
            return sourceBuilder;
    }
    
    
    /**
         * 查询封装，带分页
         * @param searchSourceBuilder
         * @param pageNum
         * @param pageSize
         * @param s
         * @param <T>
         * @return
         * @throws IOException
         */
        public <T> PageInfo<T> search(SearchSourceBuilder searchSourceBuilder, int pageNum, int pageSize, Class<T> s) throws Exception {
            Document declaredAnnotation = (Document )s.getDeclaredAnnotation(Document.class);
            if(declaredAnnotation == null){
                throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", s.getName()));
            }
            String indexName = declaredAnnotation.index();
            SearchRequest searchRequest = new SearchRequest(indexName);
            searchRequest.source(searchSourceBuilder);
            SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
            SearchHits hits = searchResponse.getHits();
            JSONArray jsonArray = new JSONArray();
            for (SearchHit hit : hits) {
                String sourceAsString = hit.getSourceAsString();
                JSONObject jsonObject = JSON.parseObject(sourceAsString);
                jsonArray.add(jsonObject);
            }
            log.info("返回总数为：" + hits.getTotalHits());
            int total = (int)hits.getTotalHits().value;
    
            // 封装分页
            List<T> list = jsonArray.toJavaList(s);
            PageInfo<T> page = new PageInfo<>();
            page.setList(list);
            page.setPageNum(pageNum);
            page.setPageSize(pageSize);
            page.setTotal(total);
            page.setPages(total== 0 ? 0: (total%pageSize == 0 ? total / pageSize : (total / pageSize) + 1));
            page.setHasNextPage(page.getPageNum() < page.getPages());
            return page;
        }

下面看下深分页如何实现，深分页做的导出查询，所以不需要返回分页信息，只需要返回一个集合，但是集合内部的查询是使用深分页的方式一页一页查询的，所以注意查询条件中不要封装from, to参数

/**
         * 查询封装，返回集合--不分页
         * @param searchSourceBuilder
         * @param s
         * @param <T>
         * @return
         * @throws IOException
         */
        public <T> List<T> search(SearchSourceBuilder searchSourceBuilder, Class<T> s) throws Exception {
            Document declaredAnnotation = (Document)s.getDeclaredAnnotation(Document.class);
            if(declaredAnnotation == null){
                throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", s.getName()));
            }
            String indexName = declaredAnnotation.index();
    
            final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));
            SearchRequest searchRequest = new SearchRequest(indexName);
            searchRequest.source(searchSourceBuilder);
            searchRequest.scroll(scroll);
            SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
    
            String scrollId = searchResponse.getScrollId();
            SearchHit[] searchHits = searchResponse.getHits().getHits();
            long totalHits = searchResponse.getHits().getTotalHits().value;
            log.info("es查询集合总数为{}", totalHits);
    
            JSONArray jsonArray = new JSONArray();
            while (searchHits != null && searchHits.length > 0) {
                for (SearchHit hit : searchResponse.getHits().getHits()) {
                    String sourceAsString = hit.getSourceAsString();
                    JSONObject jsonObject = JSON.parseObject(sourceAsString);
                    jsonArray.add(jsonObject);
                }
                SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
                scrollRequest.scroll(scroll);
                searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
                scrollId = searchResponse.getScrollId();
                searchHits = searchResponse.getHits().getHits();
            }
    
            // 清除滚屏
            if (scrollId != null) {
                ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
                clearScrollRequest.addScrollId(scrollId);
                ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
                boolean succeeded = clearScrollResponse.isSucceeded();
            }
    
            // 封装集合
            List<T> list = jsonArray.toJavaList(s);
            return list;
        }

好了，今天的分享就到这里，希望对你有帮助