| commit | author | age | ||
| 56a4b8 | 1 | |
| I | 2 | # ElasticSearch 聚合(aggregations) |
| 3 | ||
| 4 | | Syntax | Description | | |
| 5 | | ----------- | ----------- | | |
| 6 | | Header | Title | | |
| 7 | | Paragraph | Text | | |
| 8 | ||
| 9 | ||
| 10 | | Syntax | Description | | |
| 11 | | ----------- | ----------- | | |
| 12 | | Header | Title | | |
| 13 | | Paragraph | Text | | |
| 14 | ||
| 15 | ### 特点 | |
| 16 | ||
| 17 | + 聚合和搜索是使用同样的数据结构，因此聚合和搜索可以是一起执行的. | |
| 18 | 这表示我们可以在一次json请求裡，同时对相同的数据进行 搜索/过滤 + 分析 | |
| 19 | + 桶 和 度量 | |
| 20 | - 桶（bucket） | |
| 21 | ||
| 22 | 1.是按照某种方式对数据进行分组，但不包括计算，因此bucket中往往会嵌套另一种聚合：metrics aggregations即度量 | |
| 23 | ||
| 24 | 2.桶可以被嵌套在其他桶里面 | |
| 25 | ||
| 26 | 3.比较常用的桶划分方式有 | |
| 27 | - Terms Aggregation：根据词条内容分组，词条内容完全匹配的为一组 | |
| 28 | - filter：一个用来过滤的桶 和用在主查询query的 "过滤filter" 的用法是一模一样的，都是过滤 | |
| 29 | - top_hits桶 : 在某个桶底下找出这个桶的前几笔hits，返回的hits格式和主查询query返回的hits格式一模一样 | |
| 30 | - Date Histogram Aggregation：根据日期阶梯分组，例如给定阶梯为周，会自动每周分为一组 | |
| 31 | - Histogram Aggregation：根据数值阶梯分组，与日期类似 | |
| 32 | - Range Aggregation：数值和日期的范围分组，指定开始和结束，然后按段分组 | |
| 33 | ||
| 34 | - 度量（metrics） | |
| 35 | ||
| 36 | 分组完成以后，我们一般会对组中的数据进行聚合运算，例如求平均值、最大、最小、求和等，这些在ES中称为度量 | |
| 37 | ||
| 38 | 常用的度量集合方式有 | |
| 39 | - Avg Aggregation：求平均值 | |
| 40 | - Max Aggregation：求最大值 | |
| 41 | - Min Aggregation：求最小值 | |
| 42 | - Percentiles Aggregation：求百分比 | |
| 43 | - Stats Aggregation：同时返回avg、max、min、sum、count等 | |
| 44 | - Sum Aggregation：求和 | |
| 45 | - Top hits Aggregation：求前几 | |
| 46 | - Value Count Aggregation：求总数 | |
| 47 | ||
| 48 | ### aggs 聚合的模板 | |
| 49 | ||
| 50 | + 当query和aggs一起存在时，会先执行query的主查询，主查询query执行完后会搜出一批结果，而这些结果才会被拿去aggs拿去做聚合 | |
| 51 | 另外要注意aggs后面会先接一层自定义的这个聚合的名字，然后才是接上要使用的聚合桶 | |
| 52 | 如果有些情况不在意查询结果是什麽，而只在意aggs的结果，可以把size设为0，如此可以让返回的hits结果集是0，加快返回的速度 | |
| 53 | ||
| 54 | + 一个aggs裡可以有很多个聚合，每个聚合彼此间都是独立的，因此可以一个聚合拿来统计数量、一个聚合拿来分析数据、一个聚合拿来计算标准差...，让一次搜索就可以把想要做的事情一次做完 | |
| 55 | ||
| 56 | + aggs可以嵌套在其他的aggs裡面，而嵌套的桶能作用的文档集范围，是外层的桶所输出的结果集 | |
| 57 | ||
| 58 | + 模板 | |
| 59 | ``` | |
| 60 | GET /test/doc/_search | |
| 61 | { | |
| 62 | "query": { ... }, | |
| 63 | "size": 0, | |
| 64 | "aggs": { | |
| 65 | "custom_name1": { //aggs后面接著的是一个自定义的name | |
| 66 | "桶": { ... } //再来才是接桶 | |
| 67 | }, | |
| 68 | "custom_name2": { //一个aggs裡可以有很多聚合 | |
| 69 | "桶": { ... } | |
| 70 | }, | |
| 71 | "custom_name3": { | |
| 72 | "桶": { | |
| 73 | ..... | |
| 74 | }, | |
| 75 | "aggs": { //aggs可以嵌套在别的aggs裡面 | |
| 76 | "in_name": { //记得使用aggs需要先自定义一个name | |
| 77 | "桶": { ... } //in_name的桶作用的文档是custom_name3的桶的结果 | |
| 78 | } | |
| 79 | } | |
| 80 | } | |
| 81 | } | |
| 82 | ``` | |
| 83 | + 结果模板 | |
| 84 | ``` | |
| 85 | { | |
| 86 | "hits": { | |
| 87 | "total": 8, | |
| 88 | "max_score": 0, | |
| 89 | "hits": [] //因为size设为0，所以没有查询结果返回 | |
| 90 | }, | |
| 91 | "aggregations": { | |
| 92 | "custom_name1": { | |
| 93 | ... | |
| 94 | }, | |
| 95 | "custom_name2": { | |
| 96 | ... | |
| 97 | }, | |
| 98 | "custom_name3": { | |
| 99 | ... , | |
| 100 | "in_name": { | |
| 101 | .... | |
| 102 | } | |
| 103 | } | |
| 104 | } | |
| 105 | } | |
| 106 | ``` | |
| 107 | ||
| 108 | ### 数据准备 | |
| 109 | ``` | |
| 110 | PUT /test | |
| 111 | { | |
| 112 | "mappings": { | |
| 113 | "dynamic": "strict", | |
| 114 | "properties": { | |
| 115 | "color": { | |
| 116 | "type": "keyword" | |
| 117 | }, | |
| 118 | "price": { | |
| 119 | "type": "long" | |
| 120 | } | |
| 121 | } | |
| 122 | } | |
| 123 | } | |
| 124 | ||
| 125 | POST /test/_doc/1 | |
| 126 | {"color":"red","price":100} | |
| 127 | ||
| 128 | POST /test/_doc/2 | |
| 129 | {"color":"green","price":500} | |
| 130 | ||
| 131 | POST /test/_doc/3 | |
| 132 | {"color":["red","blue"],"price":1000} | |
| 133 | ``` | |
| 134 | ||
| 135 | ### 示例 | |
| 136 | - trems桶 | |
| 137 | ||
| 138 | - 找出共几组颜色和组内颜色个数 | |
| 139 | ``` | |
| 140 | GET /test/_search | |
| 141 | { | |
| 142 | "size": 0, | |
| 143 | "aggs": { | |
| 144 | "my_terms": { | |
| 145 | "terms": { | |
| 146 | "field": "color" | |
| 147 | } | |
| 148 | } | |
| 149 | } | |
| 150 | } | |
| 151 | ``` | |
| 152 | 聚合结果 | |
| 153 | ``` | |
| 154 | { | |
| 155 | "aggregations" : { | |
| 156 | "my_terms" : { | |
| 157 | "doc_count_error_upper_bound" : 0, | |
| 158 | "sum_other_doc_count" : 0, | |
| 159 | "buckets" : [ | |
| 160 | { | |
| 161 | "key" : "red", | |
| 162 | "doc_count" : 2 | |
| 163 | }, | |
| 164 | { | |
| 165 | "key" : "blue", | |
| 166 | "doc_count" : 1 | |
| 167 | }, | |
| 168 | { | |
| 169 | "key" : "green", | |
| 170 | "doc_count" : 1 | |
| 171 | } | |
| 172 | ] | |
| 173 | } | |
| 174 | } | |
| 175 | } | |
| 176 | ``` | |
| 177 | - 在示例1基础上,对分组的颜色求价格平均和最小值 | |
| 178 | ``` | |
| 179 | GET /test/_search | |
| 180 | { | |
| 181 | "size": 0, | |
| 182 | "aggs": { | |
| 183 | "my_terms": { | |
| 184 | "terms": { | |
| 185 | "field": "color" | |
| 186 | }, | |
| 187 | "aggs": { | |
| 188 | "my_avg_price": { | |
| 189 | "avg": { | |
| 190 | "field": "price" | |
| 191 | } | |
| 192 | }, | |
| 193 | "my_min_price": { | |
| 194 | "min": { | |
| 195 | "field": "price" | |
| 196 | } | |
| 197 | } | |
| 198 | } | |
| 199 | } | |
| 200 | } | |
| 201 | } | |
| 202 | ``` | |
| 203 | 聚合结果 | |
| 204 | ``` | |
| 205 | { | |
| 206 | "aggregations" : { | |
| 207 | "my_terms" : { | |
| 208 | "doc_count_error_upper_bound" : 0, | |
| 209 | "sum_other_doc_count" : 0, | |
| 210 | "buckets" : [ | |
| 211 | { | |
| 212 | "key" : "red", | |
| 213 | "doc_count" : 2, | |
| 214 | "my_avg_price" : { | |
| 215 | "value" : 550.0 | |
| 216 | }, | |
| 217 | "my_min_price" : { | |
| 218 | "value" : 100.0 | |
| 219 | } | |
| 220 | }, | |
| 221 | { | |
| 222 | "key" : "blue", | |
| 223 | "doc_count" : 1, | |
| 224 | "my_avg_price" : { | |
| 225 | "value" : 1000.0 | |
| 226 | }, | |
| 227 | "my_min_price" : { | |
| 228 | "value" : 1000.0 | |
| 229 | } | |
| 230 | }, | |
| 231 | { | |
| 232 | "key" : "green", | |
| 233 | "doc_count" : 1, | |
| 234 | "my_avg_price" : { | |
| 235 | "value" : 500.0 | |
| 236 | }, | |
| 237 | "my_min_price" : { | |
| 238 | "value" : 500.0 | |
| 239 | } | |
| 240 | } | |
| 241 | ] | |
| 242 | } | |
| 243 | } | |
| 244 | ``` | |
| 245 | ||
| 246 | - filter桶 | |
| 247 | 过滤只查看红颜色的分组情况 | |
| 248 | ``` | |
| 249 | GET /test/_search | |
| 250 | { | |
| 251 | "size": 0, | |
| 252 | "aggs": { | |
| 253 | "my_fliter": { | |
| 254 | "filter": { | |
| 255 | "bool": { | |
| 256 | "must": { | |
| 257 | "terms": { | |
| 258 | "color": [ | |
| 259 | "red" | |
| 260 | ] | |
| 261 | } | |
| 262 | } | |
| 263 | } | |
| 264 | } | |
| 265 | } | |
| 266 | } | |
| 267 | } | |
| 268 | ``` | |
| 269 | 聚合结果 | |
| 270 | ``` | |
| 271 | { | |
| 272 | "aggregations" : { | |
| 273 | "my_fliter" : { | |
| 274 | "doc_count" : 2 | |
| 275 | } | |
| 276 | } | |
| 277 | } | |
| 278 | ``` | |
| 279 | ||
| 280 | filter桶和terms桶叠加嵌套使用 | |
| 281 | 过滤含有红颜色的文档,再对其中包含的颜色进行分组 | |
| 282 | ``` | |
| 283 | GET /test/_search | |
| 284 | { | |
| 285 | "size": 0, | |
| 286 | "aggs": { | |
| 287 | "my_fliter": { | |
| 288 | "filter": { | |
| 289 | "bool": { | |
| 290 | "must": { | |
| 291 | "terms": { | |
| 292 | "color": [ | |
| 293 | "red" | |
| 294 | ] | |
| 295 | } | |
| 296 | } | |
| 297 | } | |
| 298 | }, | |
| 299 | "aggs": { | |
| 300 | "my_trems": { | |
| 301 | "terms": { | |
| 302 | "field": "color" | |
| 303 | } | |
| 304 | } | |
| 305 | } | |
| 306 | } | |
| 307 | } | |
| 308 | } | |
| 309 | ``` | |
| 310 | 聚合结果 | |
| 311 | - 因为terms桶嵌套在filter桶内，所以query查询出来的文档们会先经过filter桶，如果符合filter桶，才会进入到terms桶内 | |
| 312 | - 此处通过filter桶的文档只有两笔，分别是{"color": "red"}以及{"color": ["red", "blue"]}，所以terms桶只会对这两笔文档做分组 | |
| 313 | - 这也是为什麽terms桶裡没有出现color为green的分组，因为这个文档在filter桶就被挡下来了 | |
| 314 | - 需注意的是聚合中取的是query之后文档内容,如果query中限制只查询green的文档,那么聚合将无对应内容展示 | |
| 315 | ``` | |
| 316 | { | |
| 317 | "aggregations" : { | |
| 318 | "my_fliter" : { | |
| 319 | "doc_count" : 2, | |
| 320 | "my_trems" : { | |
| 321 | "doc_count_error_upper_bound" : 0, | |
| 322 | "sum_other_doc_count" : 0, | |
| 323 | "buckets" : [ | |
| 324 | { | |
| 325 | "key" : "red", | |
| 326 | "doc_count" : 2 | |
| 327 | }, | |
| 328 | { | |
| 329 | "key" : "blue", | |
| 330 | "doc_count" : 1 | |
| 331 | } | |
| 332 | ] | |
| 333 | } | |
| 334 | } | |
| 335 | } | |
| 336 | } | |
| 337 | ``` | |
| 338 | 当然也可以先进行trems桶嵌套filter桶,意义则是分组后再进行过滤 | |
| 339 | ``` | |
| 340 | GET /test/_search | |
| 341 | { | |
| 342 | "size": 0, | |
| 343 | "aggs": { | |
| 344 | "my_trems": { | |
| 345 | "terms": { | |
| 346 | "field": "color" | |
| 347 | }, | |
| 348 | "aggs": { | |
| 349 | "my_fliter": { | |
| 350 | "filter": { | |
| 351 | "bool": { | |
| 352 | "must": { | |
| 353 | "terms": { | |
| 354 | "color": [ | |
| 355 | "red" | |
| 356 | ] | |
| 357 | } | |
| 358 | } | |
| 359 | } | |
| 360 | } | |
| 361 | } | |
| 362 | } | |
| 363 | } | |
| 364 | } | |
| 365 | } | |
| 366 | ``` | |
| 367 | 聚合结果 | |
| 368 | - 在分组中进行过滤,可以看到green中my_filter中的doc_count结果为0 | |
| 369 | - 而至于为什么bule中含有一条doc_count=1,是因为原文档是{"color":["red","blue"]} | |
| 370 | ``` | |
| 371 | { | |
| 372 | ||
| 373 | "aggregations" : { | |
| 374 | "my_trems" : { | |
| 375 | "doc_count_error_upper_bound" : 0, | |
| 376 | "sum_other_doc_count" : 0, | |
| 377 | "buckets" : [ | |
| 378 | { | |
| 379 | "key" : "red", | |
| 380 | "doc_count" : 2, | |
| 381 | "my_fliter" : { | |
| 382 | "doc_count" : 2 | |
| 383 | } | |
| 384 | }, | |
| 385 | { | |
| 386 | "key" : "blue", | |
| 387 | "doc_count" : 1, | |
| 388 | "my_fliter" : { | |
| 389 | "doc_count" : 1 | |
| 390 | } | |
| 391 | }, | |
| 392 | { | |
| 393 | "key" : "green", | |
| 394 | "doc_count" : 1, | |
| 395 | "my_fliter" : { | |
| 396 | "doc_count" : 0 | |
| 397 | } | |
| 398 | } | |
| 399 | ] | |
| 400 | } | |
| 401 | } | |
| 402 | } | |
| 403 | ``` | |
| 404 | - top_hits桶 | |
| 405 | ||
| 406 | 在某个桶底下找出这个桶的前几笔hits，返回的hits格式和主查询query返回的hits格式一模一样 | |
| 407 | ||
| 408 | 另外,该桶中不能再嵌套子聚合 | |
| 409 | Aggregator [my_top_hit] of type [top_hits] cannot accept sub-aggregations | |
| 410 | ||
| 411 | - top_hits桶支持的参数 | |
| 412 | ||
| 413 | - from、size | |
| 414 | - sort : 设置返回的hits的排序 | |
| 415 | ||
| 416 | 要注意，假设在主查询query裡已经对数据设置了排序sort，此sort并不会对aggs裡面的数据造成影响，也就是说主查询query查找出来的数据会先丢进aggs而非先经过sort，因此就算主查询设置了sort，也不会影响aggs数据裡的排序 | |
| 417 | 因此如果在top_hits桶裡的返回的hits数据想要排序，需要自己在top_hits桶裡设置sort | |
| 418 | 如果没有设置sort，默认使用主查询query所查出来的_score排序 | |
| 419 | - _source : 设置返回的字段 | |
| 420 | ||
| 421 | 按价格排序,取前两条记录 | |
| 422 | ``` | |
| 423 | GET /test/_search | |
| 424 | { | |
| 425 | "size": 0, | |
| 426 | "aggs": { | |
| 427 | "my_top_hit": { | |
| 428 | "top_hits": { | |
| 429 | "size": 2, | |
| 430 | "sort": ["price"] #默认升序asc | |
| 431 | #"sort": {"price":"desc"}这种写法也可以 | |
| 432 | } | |
| 433 | } | |
| 434 | } | |
| 435 | } | |
| 436 | ``` | |
| 437 | 聚合结果 | |
| 438 | ``` | |
| 439 | { | |
| 440 | "aggregations" : { | |
| 441 | "my_top_hit" : { | |
| 442 | "hits" : { | |
| 443 | "total" : { | |
| 444 | "value" : 3, | |
| 445 | "relation" : "eq" | |
| 446 | }, | |
| 447 | "max_score" : null, | |
| 448 | "hits" : [ | |
| 449 | { | |
| 450 | "_index" : "test", | |
| 451 | "_type" : "_doc", | |
| 452 | "_id" : "1", | |
| 453 | "_score" : null, | |
| 454 | "_source" : { | |
| 455 | "color" : "red", | |
| 456 | "price" : 100 | |
| 457 | }, | |
| 458 | "sort" : [ | |
| 459 | 100 | |
| 460 | ] | |
| 461 | }, | |
| 462 | { | |
| 463 | "_index" : "test", | |
| 464 | "_type" : "_doc", | |
| 465 | "_id" : "2", | |
| 466 | "_score" : null, | |
| 467 | "_source" : { | |
| 468 | "color" : "green", | |
| 469 | "price" : 500 | |
| 470 | }, | |
| 471 | "sort" : [ | |
| 472 | 500 | |
| 473 | ] | |
| 474 | } | |
| 475 | ] | |
| 476 | } | |
| 477 | } | |
| 478 | } | |
| 479 | } | |
| 480 | ``` | |
| 481 | ||
| 482 | ||
