文档章节

Elasticsearch mapping conflict

MrYx3en
 MrYx3en
发布于 2015/11/16 17:46
字数 997
阅读 57
收藏 0


\\

0> head -n 3 upload_ori_size.log | grep ori_size
{ "http_host": "bj.unistore.weibo.cn", "private": "public", "orilog": "lar", "mklog": "mkstr:[3|.\/images\/weibo_logo_small.png|@-他们叫我三哥哥|weibo.com\/u\/5155207263]", "ori_md5": "db302e49b603314db80e258d775e1f7f", "uid": "5155207263", "cachepath": "\/weibo_img\/filer_cache\/t_cache\/136\/098\/005CSIUfjw1ey2y4c1gjtj30dc0hsad1", "pid": "005CSIUfjw1ey2y4c1gjtj30dc0hsad1", "ori_size": "115296", "app": "mobile", "datetime": "2015-11-16 17:31:33", "@timestamp": "2015-11-16T17:31:33+0800", "message": " 2015-11-16 17:31:33#011Store: mobile 115296 005CSIUfjw1ey2y4c1gjtj30dc0hsad1 \/weibo_img\/filer_cache\/t_cache\/136\/098\/005CSIUfjw1ey2y4c1gjtj30dc0hsad1 5155207263 db302e49b603314db80e258d775e1f7f mkstr:[3|.\/images\/weibo_logo_small.png|@-他们叫我三哥哥|weibo.com\/u\/5155207263] lar public bj.unistore.weibo.cn", "host": "upl001.weibo.imgbed.bx.sinanode.com", "type": "weibo_img_upload_app_access" }

\\

0> head -n 2 service_response_time.log | grep response_time
{ "datauri": "155\/065\/9b41314fa57b5030be1a6967e67f89fa0000781f.thu?pid=66d7742cjw6dfar2jseojj", "datahost": "data11.bj.storage.t.sinaimg.cn", "response_time": "1110.89", "file_size": "5819", "action": "DOWN", "idc": "bjcnc", "datetime": "2015-11-16 17:35:06", "@timestamp": "2015-11-16T17:35:06+0800", "message": " 2015-11-16 17:35:06#011- - - - - ACCESS bjcnc DOWN 5819 1110.89 http:\/\/data11.bj.storage.t.sinaimg.cn\/155\/065\/9b41314fa57b5030be1a6967e67f89fa0000781f.thu?pid=66d7742cjw6dfar2jseojj", "host": "svc001.weibo.imgbed.bx.sinanode.com", "type": "weibo_img_service_app_access" }

\\

0> head -n 10 replicator_access.log | grep file_size
{"host":"rep001.weibo.imgbed.bx.sinanode.com", "download_speed": "612234.0", "cost_time": "0.690773", "file_size": 422915, "file_path": "\/weibo_img\/filer_cache\/t_cache\/047\/131\/b7d4e5c8gw1ey2wsz09rhj20xc18g13x.lar", "id": "7294", "level": "INFO", "fromidc": "GZ", "service": "replicate", "ms": "230", "@timestamp": "2015-11-16T16:46:03+0800", "type": "weibo_img_replicator_app_access" }

\\ service mapping

130> curl -XGET http://10.19.0.64:9200/logstash-php-2015.11.10/weibo_img_service_app/_mapping?pretty
{
  "logstash-php-2015.11.10" : {
    "mappings" : {
      "weibo_img_service_app" : {
        "dynamic_templates" : [ {
          "message_field" : {
            "mapping" : {
              "index" : "analyzed",
              "omit_norms" : true,
              "store" : false,
              "type" : "string"
            },
            "match" : "message",
            "match_mapping_type" : "string"
          }
        }, {
          "string_fields" : {
            "mapping" : {
              "ignore_above" : 256,
              "index" : "not_analyzed",
              "store" : false,
              "type" : "string",
              "doc_values" : true
            },
            "match" : "*",
            "match_mapping_type" : "string"
          }
        }, {
          "long_fields" : {
            "mapping" : {
              "type" : "long",
              "doc_values" : true
            },
            "match" : "*",
            "match_mapping_type" : "long"
          }
        }, {
          "integer_fields" : {
            "mapping" : {
              "type" : "integer",
              "doc_values" : true
            },
            "match" : "*",
            "match_mapping_type" : "integer"
          }
        }, {
          "double_fields" : {
            "mapping" : {
              "type" : "double",
              "doc_values" : true
            },
            "match" : "*",
            "match_mapping_type" : "double"
          }
        }, {
          "float_fields" : {
            "mapping" : {
              "type" : "float",
              "doc_values" : true
            },
            "match" : "*",
            "match_mapping_type" : "float"
          }
        } ],
        "_all" : {
          "enabled" : true
        },
        "properties" : {
          "@timestamp" : {
            "type" : "date",
            "doc_values" : true,
            "format" : "dateOptionalTime"
          },
          "action" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "city" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "clientip" : {
            "type" : "ip",
            "doc_values" : true
          },
          "datahost" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "datauri" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "datetime" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "details" : {
            "type" : "object",
            "enabled" : false
          },
          "end_time" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true
          },
          "error_no" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "error_type" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "file_size" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "geoip" : {
            "dynamic" : "true",
            "properties" : {
              "location" : {
                "type" : "geo_point"
              }
            }
          },
          "host" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "idc" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "jsoncontent" : {
            "dynamic" : "true",
            "properties" : {
              "content" : {
                "type" : "string",
                "index" : "not_analyzed",
                "doc_values" : true,
                "ignore_above" : 8092
              }
            }
          },
          "jsonlog" : {
            "type" : "string",
            "index" : "no"
          },
          "logsource" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "message" : {
            "type" : "string",
            "norms" : {
              "enabled" : false
            }
          },
          "originalmsg" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "pic_id" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "pic_length" : {
            "type" : "long",
            "doc_values" : true
          },
          "pic_type" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "request_header" : {
            "type" : "object",
            "enabled" : false
          },
          "response_header" : {
            "type" : "object",
            "enabled" : false
          },
          "response_time" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "rtt" : {
            "type" : "double",
            "doc_values" : true
          },
          "start_time" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true
          },
          "timestamp" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "type" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "uid" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true
          },
          "unparsed-data" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "video_buffer_duration" : {
            "type" : "long",
            "doc_values" : true
          },
          "video_duration" : {
            "type" : "double",
            "doc_values" : true
          },
          "video_request_header" : {
            "type" : "object",
            "enabled" : false
          },
          "video_response_header" : {
            "type" : "object",
            "enabled" : false
          },
          "video_time_duration" : {
            "type" : "nested",
            "properties" : {
              "duration" : {
                "type" : "string",
                "index" : "not_analyzed",
                "doc_values" : true,
                "ignore_above" : 256
              },
              "type" : {
                "type" : "string",
                "index" : "not_analyzed",
                "doc_values" : true,
                "ignore_above" : 256
              }
            }
          },
          "wm" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true
          }
        }
      }
    }
  }
}

\\ upload mapping

0> curl -XGET http://10.19.0.64:9200/logstash-php-2015.11.10/weibo_img_upload_app/_mapping?pretty
{
  "logstash-php-2015.11.10" : {
    "mappings" : {
      "weibo_img_upload_app" : {
        "dynamic_templates" : [ {
          "message_field" : {
            "mapping" : {
              "index" : "analyzed",
              "omit_norms" : true,
              "store" : false,
              "type" : "string"
            },
            "match" : "message",
            "match_mapping_type" : "string"
          }
        }, {
          "string_fields" : {
            "mapping" : {
              "ignore_above" : 256,
              "index" : "not_analyzed",
              "store" : false,
              "type" : "string",
              "doc_values" : true
            },
            "match" : "*",
            "match_mapping_type" : "string"
          }
        }, {
          "long_fields" : {
            "mapping" : {
              "type" : "long",
              "doc_values" : true
            },
            "match" : "*",
            "match_mapping_type" : "long"
          }
        }, {
          "integer_fields" : {
            "mapping" : {
              "type" : "integer",
              "doc_values" : true
            },
            "match" : "*",
            "match_mapping_type" : "integer"
          }
        }, {
          "double_fields" : {
            "mapping" : {
              "type" : "double",
              "doc_values" : true
            },
            "match" : "*",
            "match_mapping_type" : "double"
          }
        }, {
          "float_fields" : {
            "mapping" : {
              "type" : "float",
              "doc_values" : true
            },
            "match" : "*",
            "match_mapping_type" : "float"
          }
        } ],
        "_all" : {
          "enabled" : true
        },
        "properties" : {
          "@timestamp" : {
            "type" : "date",
            "doc_values" : true,
            "format" : "dateOptionalTime"
          },
          "action_type" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "app" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "cachepath" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "clientip" : {
            "type" : "ip",
            "doc_values" : true
          },
          "datetime" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "details" : {
            "type" : "object",
            "enabled" : false
          },
          "end_time" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true
          },
          "geoip" : {
            "dynamic" : "true",
            "properties" : {
              "location" : {
                "type" : "geo_point"
              }
            }
          },
          "host" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "http_host" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "jsoncontent" : {
            "dynamic" : "true",
            "properties" : {
              "content" : {
                "type" : "string",
                "index" : "not_analyzed",
                "doc_values" : true,
                "ignore_above" : 8092
              }
            }
          },
          "jsonlog" : {
            "type" : "string",
            "index" : "no"
          },
          "location" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "logsource" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "message" : {
            "type" : "string",
            "norms" : {
              "enabled" : false
            }
          },
          "mklog" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "ori_md5" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "ori_size" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "orilog" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "pic_length" : {
            "type" : "long",
            "doc_values" : true
          },
          "pid" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "private" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "request_header" : {
            "type" : "object",
            "enabled" : false
          },
          "response_header" : {
            "type" : "object",
            "enabled" : false
          },
          "rtt" : {
            "type" : "double",
            "doc_values" : true
          },
          "start_time" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true
          },
          "timestamp" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "type" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true,
            "ignore_above" : 256
          },
          "uid" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true
          },
          "video_buffer_duration" : {
            "type" : "long",
            "doc_values" : true
          },
          "video_duration" : {
            "type" : "double",
            "doc_values" : true
          },
          "video_request_header" : {
            "type" : "object",
            "enabled" : false
          },
          "video_response_header" : {
            "type" : "object",
            "enabled" : false
          },
          "video_time_duration" : {
            "type" : "nested",
            "properties" : {
              "duration" : {
                "type" : "string",
                "index" : "not_analyzed",
                "doc_values" : true,
                "ignore_above" : 256
              },
              "type" : {
                "type" : "string",
                "index" : "not_analyzed",
                "doc_values" : true,
                "ignore_above" : 256
              }
            }
          },
          "wm" : {
            "type" : "string",
            "index" : "not_analyzed",
            "doc_values" : true
          }
        }
      }
    }
  }
}


© 著作权归作者所有

共有 人打赏支持
MrYx3en
粉丝 9
博文 132
码字总数 30598
作品 0
宝鸡
系统管理员
ElasticSearch开发问题汇总(不断更新中)

1、Mapping: [译]ElasticSearch数据类型--string类型已死, 字符串数据永生 ElasticSearch动态日期映射 2、Spring Data Elasticsearch: Spring Data Elasticsearch教程...

九州暮云
07/18
0
0
elasticsearch搜索建议Completion Suggester

目的:实现淘宝、京东搜索建议功能 准备环境:安装elasticsearch, 并安装拼音插件(https://github.com/medcl/elasticsearch-analysis-pinyin)。我安装的环境为当前最新版6.2.4。 参考官网教...

杰仪
05/12
0
0
ElasticSearch排序引起的all shards failed异常原因分析

背景 注:版本为。 在我们的日志系统里需要一些系统索引,这些系统索引在应用初始化的时候就会被添加到中去,这些在中的系统索引在没有索引数据的时候,只有索引名和一些配置信息,没有信息。...

九州暮云
08/10
0
0
分布式搜索elasticsearch 中文分词集成

对于索引可能最关系的就是分词了 一般对于es 来说默认的smartcn 但效果不是很好 一个是ik的,一个是mmseg的,下面分别介绍下两者的用法,其实都差不多的,先安装插件,命令行: 安装ik插件 ...

空_明
2014/01/23
0
0
ES6-映射(mapping)

1.mapping映射概述 我们习惯上对ElasticSearch和数据库做了一些对比,索引(index)相当于数据库,类型(type)相当于数据表,映射(Mapping)相当于数据表的表结构。 ElasticSearch中的映射(M...

贾峰uk
07/02
0
0

没有更多内容

加载失败,请刷新页面

加载更多

CentOS7防火墙firewalld操作

firewalld Linux上新用的防火墙软件,跟iptables差不多的工具。 firewall-cmd 是 firewalld 的字符界面管理工具,firewalld是CentOS7的一大特性,最大的好处有两个:支持动态更新,不用重启服...

dingdayu
今天
1
0
关于组件化的最初步

一个工程可能会有多个版本,有国际版、国内版、还有针对各种不同的渠道化的打包版本、这个属于我们日常经常见到的打包差异化版本需求。 而对于工程的开发,比如以前的公司,分成了有三大块业...

DannyCoder
今天
2
0
Spring的Resttemplate发送带header的post请求

private HttpHeaders getJsonHeader() { HttpHeaders headers = new HttpHeaders(); MediaType type = MediaType.parseMediaType("application/json; charset=UTF-8"); ......

qiang123
昨天
3
0
Spring Cloud Gateway 之 Only one connection receive subscriber allowed

都说Spring Cloud Gateway好,我也来试试,可是配置了总是报下面这个错误: java.lang.IllegalStateException: Only one connection receive subscriber allowed. 困扰了我几天的问题,原来...

ThinkGem
昨天
26
0
学习设计模式——观察者模式

1. 认识观察者模式 1. 定义:定义对象之间一种一对多的依赖关系,当一个对象状态发生变化时,依赖该对象的其他对象都会得到通知并进行相应的变化。 2. 组织结构: Subject:目标对象类,会被...

江左煤郎
昨天
4
0

没有更多内容

加载失败,请刷新页面

加载更多

返回顶部
顶部