简单使用go写入并查询 Elasticsearch

小灰灰 2022-12-22 09:35 143阅读 0赞

import

github.com/elastic/go-elasticsearch/v8

## 插入数据 ##

func create(es elasticsearch.Client, index string, count int, total int) bool {
    	var wg sync.WaitGroup
    	// Create creates a new document in the index.
    	// Returns a 409 response when a document with a same ID already exists in the index.
    	for i := count; i < total; i++ {
    		wg.Add(1)
    		k := strconv.Itoa(i)
    		var buf bytes.Buffer
    		doc := map[string]interface{}{
    			"title":   "test_title",
    			"content": "test_content",
    			"time":    time.Now().Unix(),
    			"date":    time.Now(),
    		}
    		if err := json.NewEncoder(&buf).Encode(doc); err != nil {
    			fmt.Println(err, "Error encoding doc")
    			return false
    		}
    		go func() {
    			time.Sleep(1 * time.Millisecond)
    			res, err := es.Create(index, "idx", &buf)
    			if err != nil {
    				fmt.Println(err, "Error create response")
    			}
    			wg.Done()
    			defer res.Body.Close()
    			fmt.Println(res.String())
    		}()
    	}
    	wg.Wait()
    	return true
    }

结果

![20201119140757981.png][]

## 搜索 ##

func search(client elasticsearch.Client, index string, query string) resData {
    	var buf bytes.Buffer
    	queryData := map[string]interface{}{
    		"query": map[string]interface{}{
    			"match": map[string]interface{}{
    				"name": query,
    			},
    		},
    	}
    	if err := json.NewEncoder(&buf).Encode(queryData); err != nil {
    		fmt.Println(err, "Error encoding query")
    	}
    
    	resp, err := client.Search(
    		client.Search.WithContext(context.Background()),
    		client.Search.WithIndex(index),
    		client.Search.WithBody(&buf),
    		client.Search.WithSort("time:desc"),
    		client.Search.WithFrom(10),
    		client.Search.WithSize(10),
    	)
    	if err != nil {
    		fmt.Println("err1", err)
    	}
    	//fmt.Println("resp", resp)
    	//fmt.Println("Body", resp.Body)
    	//fmt.Println("StatusCode", resp.StatusCode)
    
    	var lists lists
    	total, err := DecodeSearch(resp, &lists)
    	if err != nil {
    		fmt.Println("err2", err)
    	}
    	return resData{
    		List:  lists,
    		Total: total,
    	}
    
    }

结果

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM1MzQ5MTE0_size_16_color_FFFFFF_t_70][]

代码在 [github.com][]

在数据导入到100w时查询,出现一个错误,

"caused_by" : {
            "type" : "illegal_argument_exception",
            "reason" : "Result window is too large, from + size must be less than or equal to: [10000] but was [1000010]. See the scroll api for a more efficient way to request large data sets. This limit can be set by changing the [index.max_result_window] index level setting."
          }

经查发现是因为默认的from和size不支持这麽大,然后自己就修改下面这个参数:max\_result\_window=100000000 就行了

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM1MzQ5MTE0_size_16_color_FFFFFF_t_70 1][]

***接下来准备把数据写到10亿级别,看看查询能力........***

[20201119140757981.png]: /images/20221120/8e1646ab76644716a2ca4935e302252c.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM1MzQ5MTE0_size_16_color_FFFFFF_t_70]: /images/20221120/f5f6fb59bbdd4bc78d7dd104231f8a71.png
[github.com]: https://github.com/foxliang/Blog/tree/master/Go/my_go/elk
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM1MzQ5MTE0_size_16_color_FFFFFF_t_70 1]: /images/20221120/02bcfd8b698d46f18912331f2bc255a3.png