文档章节

logstash 清洗cdn日志的开发文件 配置

 大东家
发布于 2017/05/09 16:15
字数 6087
阅读 96
收藏 0
  1. 本配置主要是在针对视频点播网站做日志分析清洗的时候的开发配置,主要有三个方面可以参考配置,正则表达式以及输出时的模版
  2. 当数据量很大时,正则表达式的好坏决定了,清洗的效率也是导致效率低下最容易的节点。
  3. 发出此开发的配置,一是方便自己记录,二是方便有需求的同志们。

针对版本1.4.2

运行 logstash -f D:\logstash-1.4.2\config\shipper.config

以下是配置文件shipper.config,正则表达式,以及输出template

input {
#node part
#      file{
#	       type=>"dbagent_access"
#	       path=>"/mnt/develop/elk/logs/张小寒/access.log"
#	       start_position=>"beginning"
#      }
      file{
	       type=>"dbagent_error"
	       path=>"/mnt/develop/elk/logs/张小寒/error.log"
	       start_position=>"beginning"
       }
}

filter {
       if[type]=="cdn"
	{
               #this is a flash player error
		grok{
			break_on_match=>true
			#condition 1
		        patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
			match=>["message","%{CDN_PATTERN_HTTP_UUID}"]
			match=>["message","%{CDN_PATTERN_HTTP}"]
			match=>["message","%{CDN_PATTERN_MOVIEINFO_HTTP}"]
			match=>["message","%{CDN_PATTERN_MOVIEINFO}"]
			match=>["message","%{CDN_PATTERN_PCVCR_HTTP}"]
			match=>["message","%{CDN_PATTERN_PCVCR}"]
			match=>["message","%{CDN_PATTERN_VCR_HTTP}"]
			match=>["message","%{CDN_PATTERN_VCR}"]
			match=>["message","%{CDN_PATTERN_prvcr_HTTP}"]
			match=>["message","%{CDN_PATTERN_prvcr}"]
			add_tag=>["%{[type]}","grokked","%{[etype]}"]
		}
	     	if "_grokparsefailure" in [tags]
       		{
			drop{}
       		}


                
		if "unknown"==[chost] or ![chost]
                {#because when client ip is 'unknown', we do not need this value
                      date {
			   #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
  			    timezone=>"Etc/UTC"
			    match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		            target=>"timestamp"
                            remove_field=>["chost"]
	              }
		}else{
                   date {
			   #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
  			    timezone=>"Etc/UTC"
			    match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		            target=>"timestamp"
	              }
		}
               if [chost]
               {
		geoip {
		   source => "chost"
	           target=>"cgeo"
		   remove_field => ['[cgeo][ip]','[cgeo][postal_code]','[cgeo][dma_code]','[cgeo][area_code]','[cgeo][country_code2]','[cgeo][country_code3]','[cgeo][continent_code]','[cgeo][region_name]','[cgeo][city_name]','[cgeo][latitude]','[cgeo][longitude]','[cgeo][timezone]','[cgeo][location]']
		#database=>["path"]
                  }
                }
		#ruby{
                # in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
                #     code=>"
                #              IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
                #           "
                #}
        	mutate{
	              remove_field=>["path","@version","@timestamp","type","etype"]
           	      convert => [ "status", "integer" ]
                 }

        }else if [type]=="nginx_access"
	{
		grok{
			 break_on_match=>true
	                 #condition 1
		         match=>["message","%{NGINX_ACCESS}"]
			 patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
			 add_tag=>["%{[type]}","grokked","access"]
		}
#		if "_grokparsefailure" in [tags]
#	        {
#        	     drop{}
#        	}
                geoip {
	                  source => "chost"
	                  target=>"cgeo"
	            	  remove_field => ['[cgeo][ip]','[cgeo][postal_code]','[cgeo][dma_code]','[cgeo][area_code]','[cgeo][country_code2]','[cgeo][country_code3]','[cgeo][continent_code]','[cgeo][region_name]','[cgeo][city_name]','[cgeo][latitude]','[cgeo][longitude]','[cgeo][timezone]','[cgeo][location]']
	                  #database=>["path"]
	        }
                #ruby{
		  # in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
                 #    code=>"
		#	      IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
                 #          "
		#}
		mutate{
                   gsub => [
                        "a_month", "Jan(?:uary)?", "01",
		        "a_month", "Feb(?:ruary)?", "02",
			"a_month", "Mar(?:ch)?", "03",
			"a_month", "Apr(?:il)?", "04",
			"a_month", "May", "05",
			"a_month", "Jun(?:e)?", "06",
			"a_month", "Jul(?:y)?", "07",
			"a_month", "Aug(?:ust)?", "08",
			"a_month", "Sep(?:tember)?", "09",
			"a_month", "Oct(?:ober)?", "10",
			"a_month", "Nov(?:ember)?", "11",
			"a_month", "Dec(?:ember)?", "12"
                   ]
		   add_field =>{
			   "cdn_date" => "%{a_year}-%{a_month}-%{a_day}"
                           "timestamp" =>"%{cdn_date} %{time}"
		           }
                   remove_field=>["path","@version","@timestamp","type","time","a_day","a_month","a_year","message"]
		   convert=>["status","integer"]
		}
	     date {
			   #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
  			  timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
	        }

		

	}else if [type]=="nginx_error"
	{
		  grok{
			break_on_match=>true
			#condition 1
			patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
			match=>["message","%{NGINX_ERROR_1301}"]
                        match=>["message","%{NGINX_ERROR}"]
			add_tag=>["%{[type]}","grokked","error"]
		 }
                 if "_grokparsefailure" in [tags]
		 {
             		drop{}
                 }	
		 geoip {
			source => "chost"
			target=>"cgeo"
			remove_field => ['[cgeo][ip]','[cgeo][postal_code]','[cgeo][dma_code]','[cgeo][area_code]','[cgeo][country_code2]','[cgeo][country_code3]','[cgeo][continent_code]','[cgeo][region_name]','[cgeo][city_name]','[cgeo][latitude]','[cgeo][longitude]','[cgeo][timezone]','[cgeo][location]']
			#database=>["path"]
		 }
		ruby {
		              #note 1 ruby need to be put before mutate
			      #note 2 if not run or correctly run ruby ,please install jruby and try it again
			      #in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
			      
		              code => "
                                #IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }   
				event['interval']=event['rtime'].to_i-event['qtime'].to_i
                           "

		   }
		  mutate{
		     
                          strip => ["info"]
			  convert=>{
                                 "status"=>"integer"
				}
		 		add_field => {
				   "cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
                         	   "timestamp" =>"%{cdn_date} %{time}"
		          	 }

			 remove_field=>["path","@version","@timestamp","type","time","e_year","e_month","e_day","message","qtime","rtime"]
		  }
		date {
			  timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
	        }


	}else if [type]=="res-monitor"
	{
                 grok{
			break_on_match=>true
			#condition 1
			patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
			match=>["message","%{RESMONITOR_PATTERN_VRB}"]
                        match=>["message","%{CDNDELIVERY_PATTERN_WRN_ERRINFO}"]
			match=>["message","%{CDNDELIVERY_PATTERN_WRN_NOERR}"]
		        match=>["message","%{CDNDELIVERY_PATTERN_ERR_ERRINFO}"]
			match=>["message","%{CDNDELIVERY_PATTERN_ERR_NOERR}"]
			add_tag=>["%{[type]}","grokked","%{[etype]}"]
		 }
               if "_grokparsefailure" in [tags]
		 {
			drop{}
		}	
		ruby {
		              #note 1 ruby need to be put before mutate
			      #note 2 if not run or correctly run ruby ,please install jruby and try it again
			      #in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
			      
		              code => "
                               # IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }

				# get date from path
			      event['e_year']=event['path'][-12..-9]
			      event['e_month']=event['path'][-8..-7]
			      event['e_day']=event['path'][-6..-5]
                           "

		   }
		mutate{
			  add_field => {
				"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
   				"timestamp" =>"%{cdn_date} %{e_hour}:%{e_minute}:%{e_second}"
			  }
	         	 remove_field=>["path","@version","@timestamp","type","e_year","e_month","e_day","e_hour","e_minute","e_second"]
		  } 
	
               if [etype]=="VRB"
		{
			date {
                          timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
                          remove_field=>["message","etype"]
	        }
		}else{
			date {
                          #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
  			  timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
                          remove_field=>["etype"]
			
	        	}
		}
	}else if [type]=="cdndelivery_filecollect"
	{
               
                 grok{
			break_on_match=>true
			#condition 1
			patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
			match=>["message","%{FILECOLLECT_PATTERN_VRB}"]
                        match=>["message","%{CDNDELIVERY_PATTERN_WRN_ERRINFO}"]
			match=>["message","%{CDNDELIVERY_PATTERN_WRN_NOERR}"]
		        match=>["message","%{CDNDELIVERY_PATTERN_ERR_ERRINFO}"]
			match=>["message","%{CDNDELIVERY_PATTERN_ERR_NOERR}"]
			add_tag=>["%{[type]}","grokked","%{[etype]}"]
		 }
               if "_grokparsefailure" in [tags]
		 {
			drop{}
		}	
		ruby {
		              #note 1 ruby need to be put before mutate
			      #note 2 if not run or correctly run ruby ,please install jruby and try it again
			      #in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
			      
		              code => "
                               # IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }

				# get date from path
			      event['e_year']=event['path'][-12..-9]
			      event['e_month']=event['path'][-8..-7]
			      event['e_day']=event['path'][-6..-5]
                           "

		   }
		mutate{
			  add_field => {
				"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
   				"timestamp" =>"%{cdn_date} %{e_hour}:%{e_minute}:%{e_second}"
			  }
	         	 remove_field=>["path","@version","@timestamp","type","e_year","e_month","e_day","e_hour","e_minute","e_second"]
		  } 
	
               if[etype]=="VRB"
		{
			date {
                          timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
                         remove_field=>["message","etype"]
	        }
		}else{
			date {
                          #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
  			  timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
			 remove_field=>["etype"]
	        	}
		}
	}else if [type]=="cdndelivery_nodeschedule"
	{
                grok{
			break_on_match=>true
			#condition 1
			patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
                        #I can not find the VRB event here ,so I ignore it first
			#match=>["message","%{NODESCHEDULE_PATTERN_VRB}"]
			match=>["message","%{NODESCHEDULE_PATTERN_INF}"]
                        match=>["message","%{CDNDELIVERY_PATTERN_WRN_ERRINFO}"]
			match=>["message","%{CDNDELIVERY_PATTERN_WRN_NOERR}"]
		        match=>["message","%{CDNDELIVERY_PATTERN_ERR_ERRINFO}"]
			match=>["message","%{CDNDELIVERY_PATTERN_ERR_NOERR}"]
			add_tag=>["%{[type]}","grokked","%{[etype]}"]
		 }
               if "_grokparsefailure" in [tags]
		 {
			drop{}
		}	
		ruby {
		              #note 1 ruby need to be put before mutate
			      #note 2 if not run or correctly run ruby ,please install jruby and try it again
			      #in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
			      
		              code => "
                                #IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }

				# get date from path
			      event['e_year']=event['path'][-12..-9]
			      event['e_month']=event['path'][-8..-7]
			      event['e_day']=event['path'][-6..-5]
                           "

		   }
		mutate{
			  add_field => {
				"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
   				"timestamp" =>"%{cdn_date} %{e_hour}:%{e_minute}:%{e_second}"
			  }
	         	 remove_field=>["path","@version","@timestamp","type","e_year","e_month","e_day","e_hour","e_minute","e_second"]
		  } 
	
               if [etype]=="VRB" or [etype]=="INF"
		{
			date {
                          timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
                          remove_field=>["message","etype"]
	                }
		}else{
			date {
                          #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
  			  timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
			 remove_field=>["etype"]
	        	}
		}
	}else if [type]=="cdndelivery_storageschedule"
	{
               grok{
			break_on_match=>true
			#condition 1
			patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
                        #I can not find the VRB event here ,so I ignore it first
			#match=>["message","%{STORAGESCHEDULE_PATTERN_VRB}"]
                        match=>["message","%{CDNDELIVERY_PATTERN_WRN_ERRINFO}"]
			match=>["message","%{CDNDELIVERY_PATTERN_WRN_NOERR}"]
		        match=>["message","%{CDNDELIVERY_PATTERN_ERR_ERRINFO}"]
			match=>["message","%{CDNDELIVERY_PATTERN_ERR_NOERR}"]
			add_tag=>["%{[type]}","grokked","%{[etype]}"]
		 }
               if "_grokparsefailure" in [tags]
		 {
			drop{}
		}	
		ruby {
		              #note 1 ruby need to be put before mutate
			      #note 2 if not run or correctly run ruby ,please install jruby and try it again
			      #in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
			      
		              code => "
                               # IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }

				# get date from path
			      event['e_year']=event['path'][-12..-9]
			      event['e_month']=event['path'][-8..-7]
			      event['e_day']=event['path'][-6..-5]
                           "

		   }
		mutate{
			  add_field => {
				"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
   				"timestamp" =>"%{cdn_date} %{e_hour}:%{e_minute}:%{e_second}"
			  }
	         	 remove_field=>["path","@version","@timestamp","type","e_year","e_month","e_day","e_hour","e_minute","e_second"]
		  } 
	
               if [etype]=="VRB" 
		{
			date {
                          timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
                         remove_field=>["message","etype"]
	                 }
		}else{
			date {
                          #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
  			  timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
			 remove_field=>["etype"]
	        	}
		}
	}else if [type]=="cdndelivery_notify"
	{
		   grok{
			break_on_match=>true
			#condition 1
			patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
                        #I can not find the VRB event here ,so I ignore it first
			#match=>["message","%{NOTIFY_PATTERN_VRB}"]
                        match=>["message","%{CDNDELIVERY_PATTERN_WRN_ERRINFO}"]
			match=>["message","%{CDNDELIVERY_PATTERN_WRN_NOERR}"]
		        match=>["message","%{CDNDELIVERY_PATTERN_ERR_ERRINFO}"]
			match=>["message","%{CDNDELIVERY_PATTERN_ERR_NOERR}"]
			add_tag=>["%{[type]}","grokked","%{[etype]}"]
		 }
               if "_grokparsefailure" in [tags]
		 {
			drop{}
		}	
		ruby {
		              #note 1 ruby need to be put before mutate
			      #note 2 if not run or correctly run ruby ,please install jruby and try it again
			      #in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
			      
		              code => "
                               # IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }

				# get date from path
			      event['e_year']=event['path'][-12..-9]
			      event['e_month']=event['path'][-8..-7]
			      event['e_day']=event['path'][-6..-5]
                           "

		   }
		mutate{
			  add_field => {
				"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
   				"timestamp" =>"%{cdn_date} %{e_hour}:%{e_minute}:%{e_second}"
			  }
	         	 remove_field=>["path","@version","@timestamp","type","e_year","e_month","e_day","e_hour","e_minute","e_second"]
		  } 
	
               if[etype]=="VRB"
		{
			date {
                          timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
                         remove_field=>["message","etype"]
	        }
		}else{
			date {
                          #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
  			  timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
			 remove_field=>["etype"]
	        	}
		}
	}else if [type]=="statout"
	{
             grok{
			break_on_match=>true
			#condition 1
			patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
                        #I can not find the VRB event here ,so I ignore it first
			#match=>["message","%{NOTIFY_PATTERN_VRB}"]
                        match=>["message","%{STATOUT_PATTERN}"]
			add_tag=>["%{[type]}","grokked","statout"]
		 }
               if "_grokparsefailure" in [tags]
		 {
			drop{}
		}	
		ruby {
		              #note 1 ruby need to be put before mutate
			      #note 2 if not run or correctly run ruby ,please install jruby and try it again
			      #in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform IO.popen('hostname')
			      
		              code => "
                               # IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
				# get date from path
			      event['e_year']=event['path'][-18..-15]
			      event['e_month']=event['path'][-14..-13]
			      event['e_day']=event['path'][-12..-11]
                           "

		   }
		mutate{
			  add_field => {
				"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
   				"timestamp" =>"%{cdn_date} %{e_time}"
			  }
	         	 remove_field=>["path","@version","@timestamp","type","e_type","message","e_year","e_month","e_day","e_time"]
		  } 
	
             
			date {
                          #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
  			  timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
	        	}
	}
         else if [type]=="dbagent_access"
	{
		grok{
			 break_on_match=>true
	                 #condition 1
			 patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
		         match=>["message","%{DBAGENT_ACCESS}"]
			 add_tag=>["%{[type]}","grokked","access"]
		}
		if "_grokparsefailure" in [tags]
	        {
        	     drop{}
        	}
		if [status]=="200"
		{
                   #200 is normal http status, here we just need exception status,so drop 200
			drop{}
		}
                #ruby{
		  # in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
                 #    code=>"
		#	      IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
                 #          "
		#}
		mutate{
                   gsub => [
                        "a_month", "Jan(?:uary)?", "01",
		        "a_month", "Feb(?:ruary)?", "02",
			"a_month", "Mar(?:ch)?", "03",
			"a_month", "Apr(?:il)?", "04",
			"a_month", "May", "05",
			"a_month", "Jun(?:e)?", "06",
			"a_month", "Jul(?:y)?", "07",
			"a_month", "Aug(?:ust)?", "08",
			"a_month", "Sep(?:tember)?", "09",
			"a_month", "Oct(?:ober)?", "10",
			"a_month", "Nov(?:ember)?", "11",
			"a_month", "Dec(?:ember)?", "12"
                   ]
		   add_field =>{
			   "cdn_date" => "%{a_year}-%{a_month}-%{a_day}"
                           "timestamp" =>"%{cdn_date} %{time}"
		           }
                   remove_field=>["path","@version","@timestamp","type","time","a_day","a_month","a_year","message"]
		   convert=>["status","integer"]
		}
	     date {
			   #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
  			  timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
	        }

		

	}else if [type]=="dbagent_error"
	{
		grok{
			 break_on_match=>true
	                 #condition 1
			 patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
                         match=>["message","%{DBAGENT_ERROR_3}"]
		         match=>["message","%{DBAGENT_ERROR_2}"]
		         match=>["message","%{DBAGENT_ERROR_1}"]
			 add_tag=>["%{[type]}","grokked","error"]
		}
		if "_grokparsefailure" in [tags]
	        {
			#drop{}
        	}else
		{
		mutate{
			 add_field => {
				"cdn_date" => "%{e_year}-%{e_month}-%{e_day}"
   				"timestamp" =>"%{cdn_date} %{time}"
			  }
	         	 remove_field=>["path","@version","@timestamp","type","e_year","e_month","e_day","time"]
		  } 
	date {
		         #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the     hour value is correct, if not try remove timezone
			 timezone=>"Etc/UTC"
		         match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		         target=>"timestamp"
		}
		}
	}else if[type]=="media-info_HashValidService_info" or [type]=="media-info_HashValidService_error" or [type]=="media-info_PicKeyFramesService_info" or [type]=="media-info_PicKeyFramesService_error" or [type]=="media-info_PollCDNFileQueueService_CdnInterFaceLog_error_error" or [type]=="media-info_PollCDNFileQueueService_CdnInterFaceLog_error_info" 
or [type]=="media-info_PollCDNFileQueueService_PollCDNFileQueue_error_error" or [type]=="media-info_PollCDNFileQueueService_PollCDNFileQueue_error_info" or [type]=="media-info_CdnPost_error" or [type]=="media-info_CdnPost_info" or [type]=="media-info_PassThroughService_error" or [type]=="media-info_PassThroughService_info" or [type]=="media-info_DispatchService_DispatchService_error" or [type]=="media-info_DispatchService_DispatchService_info" or [type]=="media-info_DispatchService_MessageQueue_error" or [type]=="media-info_DispatchService_MessageQueue_info" or [type]=="media-info_AssetIntefaceWebservice_error" or [type]=="media-info_AssetIntefaceWebservice_info" or [type]=="media-info_MgBoss_Asset_error" or [type]=="media-info_MgBoss_Asset_info" or [type]=="media-info_MgBoss_CDN_error" or [type]=="media-info_MgBoss_CDN_info" or [type]=="media-info_MgBoss_DataAccess_error" or [type]=="media-info_MgBoss_DataAccess_info" or [type]=="media-info_MgBoss_EFSQL_error" or [type]=="media-info_MgBoss_EFSQL_info" or [type]=="media-info_MgBoss_Power_error" or [type]=="media-info_MgBoss_Power_info" or [type]=="media-info_MgBoss_Sys_error" or [type]=="media-info_MgBoss_Sys_info" or [type]=="media-info_MgBoss_Template_error" or [type]=="media-info_MgBoss_Template_info"
	  {
	      grok{
				 break_on_match=>true
			         #condition 1
				 patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
		                 match=>["message","%{INFOERROR_PATTERN}"]
			}
		       if "_grokparsefailure" in [tags]
			{
			     drop{}
			}
			ruby{
			  # in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
		             code=>"
		                    tmpArray=event['type'].split('_');
		                    if tmpArray.length==4
		                      event['appname']=tmpArray[1]
		                      event['category']=tmpArray[2]
		                    elsif tmpArray.length==3
		                      event['appname']=tmpArray[1]
		                    end
		                    event['index_type']=tmpArray[tmpArray.length-1]
		                   "
			}
			mutate{
				 add_field => {"timestamp" =>"%{e_date} %{e_time}"}
			 	 remove_field => ["path","@version","@timestamp","message","e_time"]
		                 remove_tag => ["multiline"]
			  } 

			date {
				 #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the     hour value is correct, if not try remove timezone
				 timezone=>"Etc/UTC"
				 match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
				 target=>"timestamp"
			}
	  }else if [type]=="media-info_LiveProgramList"
	  {
		        grok{
				 break_on_match=>true
			         #condition 1
				 patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
		                 match=>["message","%{LOGOFONTIMERUN}"]
			}
			if "_grokparsefailure" in [tags]
			{
			      drop{}
			}
			ruby{
			  # in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
		             code=>"
		                        event['appname']='LiveProgramList'
		                        event['index_type']='LiveProgramList'
		                   "
			}

			mutate{
		                add_field=>{"timestamp" =>"%{e_date} %{e_time}"}
				remove_field=>["path","@version","@timestamp","message","e_time"]
			   }
			date {
				 #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the     hour value is correct, if not try remove timezone
				 timezone=>"Etc/UTC"
				 match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
				 target=>"timestamp"
			}
	  }else if [type]=="router"
	{
		grok{
			 break_on_match=>true
	                 #condition 1
		         match=>["message","%{ROUTER_OK}"]
		         match=>["message","%{ROUTER_ERROR}"]
			 patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
			 add_tag=>["%{[type]}","grokked","%{[status]}"]
		}
		if "_grokparsefailure" in [tags]
	        {
        	     drop{}
        	}
                #geoip {
	        #          source => "chost"
	        #          target=>"cgeo"
	        #    	  remove_field => ['[cgeo][ip]','[cgeo][postal_code]','[cgeo][dma_code]','[cgeo][area_code]','[cgeo][country_code2]','[cgeo][country_code3]','[cgeo][continent_code]','[cgeo][region_name]','[cgeo][city_name]','[cgeo][latitude]','[cgeo][longitude]','[cgeo][timezone]','[cgeo][location]']
	        #          #database=>["path"]
	        #}
#                ruby{
		  # in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
 #                    code=>"
#			      IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
 #                          "
#		}
		mutate{
		   add_field =>{
			   "date"=>"%{year}-%{month}-%{day}"
                           "timestamp" =>"%{date} %{time}"
		           }
                   remove_field=>["path","@version","@timestamp","type","time","day","month","year","message","time"]
		}
	     date {
			   #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
  			  timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
	        }

	}else if [type]=="routerdll"
	{
		grok{
			 break_on_match=>true
	                 #condition 1
		         match=>["message","%{ROUTERDLL_ERR}"]
			 patterns_dir=>["/mnt/develop/elk/app/logstash-1.4.2/patterns"]
			 add_tag=>["%{[type]}","grokked","%{[etype]}"]
		}
		if "_grokparsefailure" in [tags]
	        {
        	     drop{}
        	}
                #geoip {
	        #          source => "uip"
	        #          target=>"cgeo"
	        #    	  remove_field => ['[cgeo][ip]','[cgeo][postal_code]','[cgeo][dma_code]','[cgeo][area_code]','[cgeo][country_code2]','[cgeo][country_code3]','[cgeo][continent_code]','[cgeo][region_name]','[cgeo][city_name]','[cgeo][latitude]','[cgeo][longitude]','[cgeo][timezone]','[cgeo][location]']
	        #          #database=>["path"]
	        #}
                ruby{
		  # in  linux, if hostname is no configured in hosts file, logstash get 0.0.0.0 ,so we add this code to get hostname on linux platform
                     code=>"
			     x=event['path'].index(/_/)
                             #  puts x 
                             #puts event['path']
                             #puts event['type']
			    tmpDate=event['path'][x+1..x+1+8]
			    event['date']=tmpDate[0..3]+'-'+tmpDate[4..5]+'-'+tmpDate[6..7]
#			      IO.popen('hostname') { |io| while (line = io.gets) do event['host']=line end }
                           "
		}
		mutate{
		   add_field =>{
                           "timestamp" =>"%{date} %{time}"
		           }
                   remove_field=>["path","@version","@timestamp","type","time","a_day","a_month","a_year","message","time"]
		}
	     date {
			   #use timezone is to avoid the value subtract timezone,if +8 zone,hours will be -8,in deploy situation ,please check the hour value is correct, if not try remove timezone
  			  timezone=>"Etc/UTC"
	        	  match => ["timestamp","YYYY-MM-dd HH:mm:ss"]
		          target=>"timestamp"
	        }
	}
}

output {
       #stdout { codec => rubydebug }
	if "cdn" in [tags]
	{
#this is a flash player error
#			file{
#			 path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/cdn-prvcr.log"
#			}
			elasticsearch
			{
				protocol=>"http"
				cluster=>"hunantv_cdn"
				host=>"localhost"
				bind_host=>"localhost"
				#port=>"9200"
				codec=>"json"
				template=>"/mnt/develop/elk/app/logstash-1.4.2/template/cdn.json"
				template_name=>"cdn"
				template_overwrite=>true
				index=>"cdn_%{cdn_date}"
				index_type=>"%{[tags][2]}"
			}

	}else if "nginx_access" in [tags] or  "nginx_error" in [tags]
	{              
#			file{
#			 path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/nginx.log"
#			}
			elasticsearch
			{
				protocol=>"http"
				cluster=>"hunantv_cdn"
				host=>"localhost"
				bind_host=>"localhost"
				#port=>"9200"
				codec=>"json"
				template=>"/mnt/develop/elk/app/logstash-1.4.2/template/nginx.json"
				template_name=>"nginx"
				template_overwrite=>true
				index=>"nginx_%{cdn_date}"
				index_type=>"%{[tags][2]}"
			}

	}else if "res-monitor" in [tags]
	{
	
#			file{
#				path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/res-monitor.log"
#			}
			elasticsearch
			{
				protocol=>"http"
				cluster=>"hunantv_cdn"
				host=>"localhost"
				bind_host=>"localhost"
				#port=>"9200"
				codec=>"json"
				template=>"/mnt/develop/elk/app/logstash-1.4.2/template/res-monitor.json"
				template_name=>"res-monitor"
				template_overwrite=>true
				index=>"res-monitor_%{cdn_date}"
				index_type=>"%{[tags][2]}"
			}

	}else if "cdndelivery_filecollect" in [tags]
	{
#			file{
#				path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/cdn-delivery.log"
#			}
			elasticsearch
			{
				protocol=>"http"
				cluster=>"hunantv_cdn"
				host=>"localhost"
				bind_host=>"localhost"
				#port=>"9200"
				codec=>"json"
				template=>"/mnt/develop/elk/app/logstash-1.4.2/template/cdn-delivery.json"
				template_name=>"cdn-delivery"
				template_overwrite=>true
				index=>"cdn-delivery_%{cdn_date}"
				index_type=>"%{[tags][2]}-filecollect"

			}		
	}else if "cdndelivery_nodeschedule" in [tags]
        {
#			file{
#				path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/cdn-delivery.log"
#			}
			elasticsearch
			{
				protocol=>"http"
				cluster=>"hunantv_cdn"
				host=>"localhost"
				bind_host=>"localhost"
				#port=>"9200"
				codec=>"json"
				template=>"/mnt/develop/elk/app/logstash-1.4.2/template/cdn-delivery.json"
				template_name=>"cdn-delivery"
				template_overwrite=>true
				index=>"cdn-delivery_%{cdn_date}"
				index_type=>"%{[tags][2]}-nodeschedule"
			}	
	}else if "cdndelivery_storageschedule" in [tags]
        {
#			file{
#				path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/cdn-delivery.log"
#			}
			elasticsearch
			{
				protocol=>"http"
				cluster=>"hunantv_cdn"
				host=>"localhost"
				bind_host=>"localhost"
				#port=>"9200"
				codec=>"json"
				template=>"/mnt/develop/elk/app/logstash-1.4.2/template/cdn-delivery.json"
				template_name=>"cdn-delivery"
				template_overwrite=>true
				index=>"cdn-delivery_%{cdn_date}"
				index_type=>"%{[tags][2]}-storageschedule"
			}	
	}else if "cdndelivery_notify" in [tags]
        {
#			file{
#				path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/cdn-delivery.log"
#			}
			elasticsearch
			{
				protocol=>"http"
				cluster=>"hunantv_cdn"
				host=>"localhost"
				bind_host=>"localhost"
				#port=>"9200"
				codec=>"json"
				template=>"/mnt/develop/elk/app/logstash-1.4.2/template/cdn-delivery.json"
				template_name=>"cdn-delivery"
				template_overwrite=>true
				index=>"cdn-delivery_%{cdn_date}"
				index_type=>"%{[tags][2]}-notify"
			}	
	}else if "statout" in [tags]
	{
			#file{
			#	path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/statout.log"
			#}
			elasticsearch
			{
				protocol=>"http"
				cluster=>"hunantv_cdn"
				host=>"localhost"
				bind_host=>"localhost"
				#port=>"9200"
				codec=>"json"
				template=>"/mnt/develop/elk/app/logstash-1.4.2/template/statout.json"
				template_name=>"statout"
				template_overwrite=>true
				index=>"statout_%{cdn_date}"
				index_type=>"error"
			}
	}else if "dbagent_access" in [tags]
        {
			file{
				path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/dbagent_access.log"
			}
			elasticsearch
			{
				protocol=>"http"
				cluster=>"hunantv_cdn"
				host=>"localhost"
				bind_host=>"localhost"
				#port=>"9200"
				codec=>"json"
				template=>"/mnt/develop/elk/app/logstash-1.4.2/template/dbagent.json"
				template_name=>"dbagent"
				template_overwrite=>true
				index=>"dbagent_%{cdn_date}"
				index_type=>"access"
			}	
	}else if "dbagent_error" in [tags]
        {
		file{
			path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/dbagent_error.log"
		}
			elasticsearch
			{
				protocol=>"http"
				cluster=>"hunantv_cdn"
				host=>"localhost"
				bind_host=>"localhost"
				#port=>"9200"
				codec=>"json"
				template=>"/mnt/develop/elk/app/logstash-1.4.2/template/dbagent.json"
				template_name=>"dbagent"
				template_overwrite=>true
				index=>"dbagent_%{cdn_date}"
				index_type=>"error"
			}

	}
	else if [type]=="media-info_HashValidService_info" or [type]=="media-info_HashValidService_error" or [type]=="media-info_PicKeyFramesService_info" or [type]=="media-info_PicKeyFramesService_error" or [type]=="media-info_PollCDNFileQueueService_CdnInterFaceLog_error_error" or [type]=="media-info_PollCDNFileQueueService_CdnInterFaceLog_error_info" 
or [type]=="media-info_PollCDNFileQueueService_PollCDNFileQueue_error_error" or [type]=="media-info_PollCDNFileQueueService_PollCDNFileQueue_error_info" or [type]=="media-info_CdnPost_error" or [type]=="media-info_CdnPost_info" or [type]=="media-info_PassThroughService_error" or [type]=="media-info_PassThroughService_info" or [type]=="media-info_DispatchService_DispatchService_error" or [type]=="media-info_DispatchService_DispatchService_info" or [type]=="media-info_DispatchService_MessageQueue_error" or [type]=="media-info_DispatchService_MessageQueue_info" or [type]=="media-info_AssetIntefaceWebservice_error" or [type]=="media-info_AssetIntefaceWebservice_info" or [type]=="media-info_MgBoss_Asset_error" or [type]=="media-info_MgBoss_Asset_info" or [type]=="media-info_MgBoss_CDN_error" or [type]=="media-info_MgBoss_CDN_info" or [type]=="media-info_MgBoss_DataAccess_error" or [type]=="media-info_MgBoss_DataAccess_info" or [type]=="media-info_MgBoss_EFSQL_error" or [type]=="media-info_MgBoss_EFSQL_info" or [type]=="media-info_MgBoss_Power_error" or [type]=="media-info_MgBoss_Power_info" or [type]=="media-info_MgBoss_Sys_error" or [type]=="media-info_MgBoss_Sys_info" or [type]=="media-info_MgBoss_Template_error" or [type]=="media-info_MgBoss_Template_info" or [type]=="media-info_LiveProgramList"
	  {
                #file{
                #      path=>"/mnt/develop/elk/app/logstash-1.4.2/logs/output.log"
		#}
		elasticsearch
		{
					protocol=>"http"
					cluster => "hunantv_cdn"
	    				host => "localhost"
	   				bind_host => "localhost" 
					#port=>"9200"
					codec=>"json"
					template=>"/mnt/develop/elk/app/logstash-1.4.2/template/media-info.json"
					template_name=>"media-info"
					template_overwrite=>true
					index=>"media-info_%{e_date}"
					index_type=>"%{[index_type]}"

		}
	
	  }
        else if "router" in [tags]
	{
			elasticsearch
			{
				protocol=>"http"
				cluster => "hunantv_cdn"
		    		host => "localhost"
		   		bind_host => "localhost" 
				#port=>"9200"
				codec=>"json"
				template=>"/mnt/develop/elk/app/logstash-1.4.2/template/router.json"
				template_name=>"router"
				template_overwrite=>true
				index=>"router_%{date}"
				index_type=>"%{status}"
			}

	}else if "routerdll" in [tags]
	{
			elasticsearch
			{
				protocol=>"http"
				cluster => "hunantv_cdn"
	    			host => "localhost"
	   			#bind_host => "localhost" 
				#port=>"9200"
				codec=>"json"
				template=>"/mnt/develop/elk/app/logstash-1.4.2/template/routerdll.json"
				template_name=>"routerdll"
				template_overwrite=>true
				index=>"routerdll_%{date}"

			}

	}
}

CDN 正则表达式匹配文件

# cdn pattern doc


# created by xiaohelong 2014-08-28
CDN_DATE %{YEAR}-%{MONTHNUM}-%{MONTHDAY}
CDN_TIMESTAMP %{CDN_DATE:cdn_date}[T ]%{HOUR}:?%{MINUTE}:?\d{1,2}
UUID32 [0-9a-z]{32}
HTTPSTATUS_VALUE \d+
HTTPSTATUS status=%{HTTPSTATUS_VALUE:status}
CDN_UUID32 uuid=%{UUID32:uuid}
# we only set one eventype because if we match successfully,then we will go next.
CDN_EVENTTYPE_HTTP http
CDN_EVENTTYPE_MOVIEINFO MovieInfo
CDN_EVENTTYPE_PCVCR PCVCR
CDN_EVENTTYPE_VCR VCR
CDN_EVENTTYPE_prvcr prvcr

CDN_PATTERN_HTTP_UUID %{CDN_TIMESTAMP:timestamp}.*%{CDN_EVENTTYPE_HTTP:etype}://(%{IPV4:qhost}|%{HOSTNAME:qhost}).+%{CDN_EVENTTYPE_HTTP}://(%{IPV4:rhost}|%{HOSTNAME:rhost}).+%{CDN_UUID32}.+-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_HTTP %{CDN_TIMESTAMP:timestamp}.*%{CDN_EVENTTYPE_HTTP:etype}://(%{IPV4:qhost}|%{HOSTNAME:qhost}).+%{CDN_EVENTTYPE_HTTP}://(%{IPV4:rhost}|%{HOSTNAME:rhost}).+-(%{IPV4:chost}|%{HOSTNAME:chost})

CDN_PATTERN_MOVIEINFO_HTTP %{CDN_TIMESTAMP:timestamp}.*%{CDN_EVENTTYPE_MOVIEINFO:etype}\s+.*%{HTTPSTATUS}.*-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_MOVIEINFO %{CDN_TIMESTAMP:timestamp}.*%{CDN_EVENTTYPE_MOVIEINFO:etype}--.*-(%{IPV4:chost}|%{HOSTNAME:chost})

CDN_PATTERN_PCVCR_HTTP %{CDN_TIMESTAMP:timestamp}.* %{CDN_EVENTTYPE_PCVCR:etype}\s+.*%{HTTPSTATUS}.*-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_PCVCR %{CDN_TIMESTAMP:timestamp}.* %{CDN_EVENTTYPE_PCVCR:etype}--.*-(%{IPV4:chost}|%{HOSTNAME:chost})

CDN_PATTERN_VCR_HTTP %{CDN_TIMESTAMP:timestamp}.* %{CDN_EVENTTYPE_VCR:etype}\s+.*%{HTTPSTATUS}.*-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_VCR %{CDN_TIMESTAMP:timestamp}.* %{CDN_EVENTTYPE_VCR:etype}--.*-(%{IPV4:chost}|%{HOSTNAME:chost})

CDN_PATTERN_prvcr_HTTP %{CDN_TIMESTAMP:timestamp}.* %{CDN_EVENTTYPE_prvcr:etype}\s+.*%{HTTPSTATUS}.*-(%{IPV4:chost}|%{HOSTNAME:chost})
CDN_PATTERN_prvcr %{CDN_TIMESTAMP:timestamp}.* %{CDN_EVENTTYPE_prvcr:etype}--.*-(%{IPV4:chost}|%{HOSTNAME:chost})

#above complete cdn pattern


#begin nginx access part	
NGINX_ACCESS (%{IPV4:chost}|%{HOSTNAME:chost}).+\[%{MONTHDAY:a_day}/%{MONTH:a_month}/%{YEAR:a_year}\:%{TIME:time}.*\"\s+%{NUMBER:status}.*
#(%{IPV4:chost}|%{HOSTNAME:chost}).+\[%{MONTHDAY}/%{MONTH}/%{YEAR}\:%{TIME:time}.*\"\s+%{NUMBER:status}.*
#end nginx access part

#begin nginx error part
NGINX_ERR_1301 1301
NGINX_ERROR_1301 %{YEAR:e_year}/%{MONTHNUM:e_month}/%{MONTHDAY:e_day}\s%{TIME:time}.+error.*\:\s+%{NGINX_ERR_1301:status}.*[0-9a-z]{32},.*time:%{INT:rtime}.*;t is %{INT:qtime}.*client: (%{IPV4:chost}|%{HOSTNAME:chost}).*
NGINX_ERROR %{YEAR:e_year}/%{MONTHNUM:e_month}/%{MONTHDAY:e_day}\s%{TIME:time}.+error.*\:\s+%{NUMBER:status}.*[0-9a-z]{32},.*client: (%{IPV4:chost}|%{HOSTNAME:chost}).*
#1301 %{YEAR}/%{MONTHNUM}/%{MONTHDAY}\s%{TIME}.+error.*\:\s+%{NUMBER}.*[0-9a-z]{32},%{DATA},.*time:%{INT}.*;t is %{INT}.*client: (%{IPV4:chost}|%{HOSTNAME:chost}).*
#not 1301 %{YEAR}/%{MONTHNUM}/%{MONTHDAY}\s%{TIME}.+error.*\:\s+%{NUMBER}.*[0-9a-z]{32},.*client: (%{IPV4:chost}|%{HOSTNAME:chost}).*
#end nginx error part


#begin ChenPU CDN Delivery and Resource Monitor part , all WRN,ERR use the same pattern to match
    STR_ERR ((?i)err(info|or|msg)?)
	CDNDELIVERY_EVENTTYPE_WRN WRN
	CDNDELIVERY_EVENTTYPE_ERR ERR
	#when match the WRN or ERR,please put ERRINFO pattern before no ERR
	CDNDELIVERY_PATTERN_WRN_ERRINFO %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*%{CDNDELIVERY_EVENTTYPE_WRN:etype}--%{GREEDYDATA:info}%{STR_ERR}\s*(=|=|:|:)\s*%{GREEDYDATA:errInfo}
	CDNDELIVERY_PATTERN_WRN_NOERR %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*%{CDNDELIVERY_EVENTTYPE_WRN:etype}--%{GREEDYDATA:info}

        CDNDELIVERY_PATTERN_ERR_ERRINFO %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*%{CDNDELIVERY_EVENTTYPE_ERR:etype}--%{GREEDYDATA:info}%{STR_ERR}\s*(=|=|:|:)\s*%{GREEDYDATA:errInfo}
	CDNDELIVERY_PATTERN_ERR_NOERR %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*%{CDNDELIVERY_EVENTTYPE_ERR:etype}--%{GREEDYDATA:info}

	
	#begin res_monitor part
	RESMONITOR_EVENTTYPE_VRB VRB
	RESMONITOR_PATTERN_VRB %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*\s%{RESMONITOR_EVENTTYPE_VRB:etype}.*EpochTime":%{QUOTEDSTRING:EpochTime}.*NodeId":%{NUMBER:NodeId}.*CurrentConn":%{NUMBER:CurrentConn}.*CurrentBandwidth":%{NUMBER:CurrentBandwidth}.*DiskTotalSpace":%{NUMBER:DiskTotalSpace}.*DiskFreeSpace":%{NUMBER:DiskFreeSpace}.*CpuUsage":%{NUMBER:CpuUsage}.*MemUsage":%{NUMBER:MemUsage}.*WanUsage":%{NUMBER:WanUsage}.*LanUsage":%{NUMBER:LanUsage}.*IoUsage":%{NUMBER:IoUsage}.*LanIpState":%{NUMBER:LanIpState}.*WanIpState":%{NUMBER:WanIpState}.*
	#end res_monitor part

	#begin cdn delivery part
	  	 #begin  filecollect part
	 	 FILECOLLECT_EVENTTYPE_VRB VRB
		 FILECOLLECT_PATTERN_VRB %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*\s%{FILECOLLECT_EVENTTYPE_VRB:etype}.*EpochTime":%{QUOTEDSTRING:EpochTime}.*NodeId":%{NUMBER:NodeId}.*IsInited":%{NUMBER:IsInited}.*FileName":%{QUOTEDSTRING:FileName}.*FileSize":%{INT:FileSize}.*FileHash":%{QUOTEDSTRING:FileHash}.*FileIsDel":%{INT:FileIsDel}.*
	 	  #end filecollect part
                 #begin nodeschedule part
		 NODESCHEDULE_EVENTTYPE_VRB VRB
		 NODESCHEDULE_EVENTTYPE_INF INF
                 NODESCHEDULE_PATTERN_VRB .*?
		 NODESCHEDULE_PATTERN_INF %{HOUR:e_hour}:%{MINUTE:e_minute}:%{INT:e_second}.*\s+%{NODESCHEDULE_EVENTTYPE_INF:etype}--%{DATA}\s*ContentType=%{WORD:ContentType}.*FileId=%{WORD:FileId}.*FileName=%{DATA:FileName},TorrentHash=.*%{WORD:TorrentHash}.*State=%{WORD:State}.*Progress=%{NUMBER:Progress}.*DownloadSpeed=%{NUMBER:DownloadSpeed}.*UploadSpeed=%{NUMBER:UploadSpeed}.*DownloadSpeedAvg=%{NUMBER:DownloadSpeedAvg}.*UploadSpeedAvg=%{NUMBER:UploadSpeedAvg}.*DownloadedBytes=%{NUMBER:DownloadedBytes}.*UploadedBytes=%{NUMBER:UploadedBytes}.*%{STR_ERR}=%{GREEDYDATA}.*
		 #end nodeschedule part
                 
		#begin storageschedule part
		 STORAGESCHEDULE_EVENTTYPE_VRB VRB
                 STORAGESCHEDULE_PATTERN_VRB .*?
		#end storageschedule part
                #begin notify part
                NOTIFYSCHEDULE_EVENTTYPE_VRB VRB
                NOTIFYSCHEDULE_PATTERN_VRB .*?
                #end notify part

	#end cdn delivery part
#end ChenPU CDN Delivery and Resource Monitor part
#begin shaochan statout
        STATOUT_PATTERN %{TIME:e_time}\s+.*:%{GREEDYDATA:info}
#end shaochan statout
#Begin DBAGENT access part
	DBAGENT_ACCESS (%{IPV4:chost}|%{HOSTNAME:chost}).+\[%{MONTHDAY:a_day}/%{MONTH:a_month}/%{YEAR:a_year}\:%{TIME:time}.*(POST|GET)\s*%{DATA:interface}\s+HTTP.*\s+%{INT:status}\s+%{INT}\s+.*
#End DBAGENT access part
#Begin DBAGENT error part
        DBAGENT_ERROR_3 %{YEAR:e_year}/%{MONTHNUM:e_month}/%{MONTHDAY:e_day}\s+%{TIME:time}\s+\[error\]\s+%{INT:pid}#0:\s+\*%{INT}\s+%{INT:level}\s+msg:%{DATA:msg},\s+check time:%{INT:ctime},\s+query string:\s*call\s+%{WORD:interface}.*\(%{DATA:para}\),\s+client:\s+(%{IPV4:chost}|%{HOSTNAME:chost}),.*
	DBAGENT_ERROR_2 %{YEAR:e_year}/%{MONTHNUM:e_month}/%{MONTHDAY:e_day}\s+%{TIME:time}\s+\[error\]\s+%{INT:pid}#0:\s+\*%{INT}\s+%{INT:level}\s+msg:%{DATA:msg},\s+check time:%{INT:ctime},\s+query string:\s*call\s+%{WORD:interface}.*\(%{DATA:para}\).*
	DBAGENT_ERROR_1 %{YEAR:e_year}/%{MONTHNUM:e_month}/%{MONTHDAY:e_day}\s+%{TIME:time}\s+\[error\]\s+%{INT:pid}#0:\s+%{INT:level}\s+msg:%{GREEDYDATA:msg}$.*
#End DBAGENT error part

#Begin router part
ROUTER_REQUEST_URL http://%{IP:requestServer}/.*--%{GREEDYDATA:fileLoc}
ROUTER_OK %{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day}\s*%{TIME:time},%{INT}.*-.*%{IP:chost}\s-\s%{WORD:status}\s-\s%{ROUTER_REQUEST_URL:requestURL}.*\s-\s%{NUMBER:isThirdCDN}\s-\s%{NUMBER:requestPolicyID}\s-\s%{NUMBER:rateLimited}

ROUTER_ERROR %{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day}\s%{TIME:time},.*-.*%{IP:chost}\s-\s%{WORD:status}\s-\s%{GREEDYDATA:description}.*\s-\s%{NUMBER:isThirdCDN}\s-\s%{NUMBER:requestPolicyID}\s-\s%{NUMBER:rateLimited}
#End router part

#Begin routerdll part
ROUTERDLL_ERR %{TIME:time}\.%{INT}\s%{DATA:etype}--%{GREEDYDATA:info}\s*fid=%{GREEDYDATA:fid},\s*uip=%{IP:uip},\s*pno=%{INT:pno},\s*msec=%{NUMBER:msec},\s*statusDesc=%{GREEDYDATA:statusDesc},\s*errInfo=%{GREEDYDATA:errInfo},\s*userRegionId=%{INT:userRegionId},\s*svcRegionIds=%{GREEDYDATA:svcRegionIds},\shasFileNodeIds=%{GREEDYDATA:hasFileNodeIds},\s*routeType=%{GREEDYDATA:routeType},\s*isTestSpeed=%{INT:isTestSpeed},\s*lastSvrIps=%{GREEDYDATA:lastSvrIps}
#End routerdll part

#begin media-info pattern doc
# created by xiaohelong 2014-10-14
DATE_CN (%{YEAR}-%{MONTHNUM}-%{MONTHDAY})
#begin info part 
INFOERROR_PATTERN  .*【记录时间】\s*%{DATE_CN:e_date}\s*%{TIME:e_time},%{INT}.*\s*【线程编号】\[%{INT:tid}\]\s*【信息描述】\s*%{GREEDYDATA:info}
#end info part

#begin LogOfOnTimeRun
LOGOFONTIMERUN .*Date="%{DATE_CN:e_date}\s*%{TIME:e_time}.*Type="%{DATA:status}".*Note="%{DATA:info}".*
#end LogOfOnTimeRun
#end media-info

以下均为json模版文件

cdn.json 模版文件

{
    "template": "cdn_*",
    "settings": {
        "index.number_of_shards": 5,
        "number_of_replicas": 1
    },
    "mappings": {
        "_default_": {
            "_all": {
                "enabled": "false"
            },
            "_source": {
                "enabled": true,
		"compress":true
            },
            "properties": {
                "_id": {
                    "type": "string",
                    "index": "not_analyzed"
                },
		"host":{
			"type":"string",
			"index":"not_analyzed"
		},
                "message": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "chost": {
		    "dynamic": true,
                    "type": "ip",
                    "index": "not_analyzed"
                },
                "cgeo": {
		    "dynamic": true,
                    "type": "object",
                    "properties": {
                        "country_name": {
                            "type": "string",
                            "index": "not_analyzed"
                        },
                        "real_region_name": {
                            "type": "string",
                            "index": "not_analyzed"
                        }
                    }
                }
            }
        }
    },
    "http": {
        "properties": {
            "_timestamp": {
                "enabled": true,
                "type": "date",
                "path": "timestamp",
                "format": "dateOptionalTime"
            },
            "rhost": {
                "type": "string",
                "index": "not_analyzed"
            },
            "qhost": {
                "type": "string",
                "index": "not_analyzed"
            },
            "uuid": {
                "dynamic": true,
                "type": "string",
                "index": "not_analyzed"
            }
        }
    },
    "MovieInfo": {
        "properties": {
            "_timestamp": {
                "enabled": true,
                "type": "date",
                "path": "timestamp",
                "format": "dateOptionalTime"
            },
            "status": {
                "dynamic": true,
                "type": "integer",
                "index": "not_analyzed"
            }
        }
    },
    "PCVCR": {
        "properties": {
            "_timestamp": {
                "enabled": true,
                "type": "date",
                "path": "timestamp",
                "format": "dateOptionalTime"
            },
            "status": {
                "dynamic": true,
                "type": "integer",
                "index": "not_analyzed"
            }
        }
    },
    "VCR": {
        "properties": {
            "_timestamp": {
                "enabled": true,
                "type": "date",
                "path": "timestamp",
                "format": "dateOptionalTime"
            },
            "rhost": {
                "type": "string",
                "index": "not_analyzed"
            },
            "qhost": {
                "type": "string",
                "index": "not_analyzed"
            },
            "uuid": {
                "dynamic": true,
                "type": "string",
                "index": "not_analyzed"
            }
        }
    },
    "prvcr": {
        "properties": {
            "_timestamp": {
                "enabled": true,
                "type": "date",
                "path": "timestamp",
                "format": "dateOptionalTime"
            },
            "status": {
                "dynamic": true,
                "type": "integer",
                "index": "not_analyzed"
            }
        }
    }
}
{
    "template": "cdn-delivery_*",
    "settings": {
        "index.number_of_shards": 5,
        "number_of_replicas": 1
    },
    "mappings": {
        "_default_": {
            "_all": {
                "enabled": "false"
            },
            "_source": {
                "enabled": true,
                "compress": true
            },
            "properties": {
                "_id": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "timestamp": {
                    "enabled": true,
                    "type": "date",
                    "format": "dateOptionalTime"
                },
                "host": {
                    "type": "string",
                    "index": "not_analyzed"
                }
	
            }
        },
	"VRB-filecollect":{
		"properties":{
			"EpochTime":{
				"type":"string",
				"format":"not_analyzed"
			},
			"NodeId":{
				"type":"integer"
			},
			"IsInited":{
				"type":"boolean"
			},
			"FileName":{
				"type":"string",
				"index":"not_analyzed"
			},
			"FileSize":{
				"type":"long"
			},
			"FileHash":{
				"type":"string",
				"index":"not_analyzed"
			},
			"FileIsDel":{
				"type":"boolean"
			}
		}

	},
	"ERR-filecollect":{
		"properties":{
			"message":{
				"type":"string",
				"index":"not_analyzed"
			},
			"info":{
				"type":"string",
				"index":"not_analyzed"
			},
			"errInfo":{
				"dynamic":true,
 				"type":"string",
 				"index":"not_analyzed"
			}
		}
	},
	"WRN-filecollect":{
		"properties":{
			"message":{
				"type":"string",
				"index":"not_analyzed"
			},
			"info":{
				"type":"string",
				"index":"not_analyzed"
			},
                        "errInfo":{
				"dynamic":true,
 				"type":"string",
 				"index":"not_analyzed"
			}
		}
	},
	"ERR-nodeschedule":{
		"properties":{
			"message":{
				"type":"string",
				"index":"not_analyzed"
			},
			"info":{
				"type":"string",
				"index":"not_analyzed"
			},
			"errInfo":{
				"dynamic":true,
 				"type":"string",
 				"index":"not_analyzed"
			}
		}
	},
	"WRN-nodeschedule":{
		"properties":{
			"message":{
				"type":"string",
				"index":"not_analyzed"
			},
			"info":{
				"type":"string",
				"index":"not_analyzed"
			},
                        "errInfo":{
				"dynamic":true,
 				"type":"string",
 				"index":"not_analyzed"
			}
		}
	},
        "INF-nodeschedule":{
		"properties":{
			"ContentType":{
				"type":"string",
				"index":"not_analyzed"
			},
			"FileId":{
				"type":"string",
				"index":"not_analyzed"
			},
			"FileName":{
				"type":"string",
				"index":"not_analyzed"
			},
			"TorrentHash":{
				"type":"string",
				"index":"not_analyzed"
			},
			"State":{
				"type":"string",
				"index":"not_analyzed"
			},
			"Progress":{
				"type":"float"
			},
			"DownloadSpeed":{
				"type":"float"
			},
			"UploadSpeed":{
				"type":"float"
			},
			"DownloadSpeedAvg":{
				"type":"float"
			},
			"UploadSpeedAvg":{
				"type":"float"
			},
			"DownloadedBytes":{
				"type":"float"
			},
			"UploadedBytes":{
				"type":"float"
			}
		}
	},
	"ERR-storageschedule":{
		"properties":{
			"message":{
				"type":"string",
				"index":"not_analyzed"
			},
			"info":{
				"type":"string",
				"index":"not_analyzed"
			},
			"errInfo":{
				"dynamic":true,
 				"type":"string",
 				"index":"not_analyzed"
			}
		}
	},
	"WRN-storageschedule":{
		"properties":{
			"message":{
				"type":"string",
				"index":"not_analyzed"
			},
			"info":{
				"type":"string",
				"index":"not_analyzed"
			},
                        "errInfo":{
				"dynamic":true,
 				"type":"string",
 				"index":"not_analyzed"
			}
		}
	},
	"ERR-notify":{
		"properties":{
			"message":{
				"type":"string",
				"index":"not_analyzed"
			},
			"info":{
				"type":"string",
				"index":"not_analyzed"
			},
			"errInfo":{
				"dynamic":true,
 				"type":"string",
 				"index":"not_analyzed"
			}
		}
	},
	"WRN-notify":{
		"properties":{
			"message":{
				"type":"string",
				"index":"not_analyzed"
			},
			"info":{
				"type":"string",
				"index":"not_analyzed"
			},
                        "errInfo":{
				"dynamic":true,
 				"type":"string",
 				"index":"not_analyzed"
			}
		}
	}


    }
}
{
    "template": "dbagent_*",
    "settings": {
        "index.number_of_shards": 5,
        "number_of_replicas": 0
    },
    "mappings": {
        "_default_": {
            "_all": {
                "enabled": "false"
            },
            "_source": {
                "enabled": true,
                "compress": true
            },
            "properties": {
                "_id": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "timestamp": {
                    "enabled": true,
                    "type": "date",
                    "format": "dateOptionalTime"
                },
                "host": {
                    "type": "string",
                    "index": "not_analyzed"
                }
            }
        },
	"access":{
		"properties":{
	          "status":{
			  "type":"integer"
		  },
                  "chost": {
                          "type": "ip",
                          "index": "not_analyzed"
                   },
 		  "interface":{
			   "dynamic":true,			
			   "type":"string",
			   "index":"not_analyzed"
		    }
		}
	},
        "error":{
 		"properties":{
			"pid":{
				"type":"integer"
			},
			"level":{
				"type":"integer"
			},
			"msg":{
				"type":"string",
				"index":"not_analyzed"
			},
			"ctime":{
				"dynamic":true,
				"type":"long"
			},
			"interface":{
				"dynamic":true,			
				"type":"string",
				"index":"not_analyzed"
			},
			"para":{
				"dynamic":true,				
				"type":"string",
				"index":"not_analyzed"
			},
               		"chost": {
				"dynamic":true, 	         	    	
                  	    	"type": "ip"
                        },
			"message":{
				"type":"string",
				"index":"not_analyzed"
			}
		}
	}
    }
}
{
    "template": "media-info_*",
    "settings": {
        "index.number_of_shards": 5,
        "number_of_replicas": 1
    },
    "mappings": {
        "_default_": {
            "_all": {
                "enabled": "false"
            },
            "_source": {
                "enabled": true,
                "compress": true
            },
            "properties": {
                "_id": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "timestamp": {
                    "enabled": true,
                    "type": "date",
                    "format": "dateOptionalTime"
                },
                "host": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "appname":{
                    "type":"string",
                    "index":"not_analyzed"
		},
                "category":{
                    "dynamic":true,
                    "type":"string",
                    "index":"not_analyzed"
		}
            }
        },
	"info":{
		"properties":{
			"tid":{
				"type":"integer"
			},
			"info":{
				"type":"string",
				"index":"analyzed"
			}
		}

	},
	"error":{
		"properties":{
			"tid":{
				"type":"integer"
			},
			"info":{
				"type":"string",
				"index":"analyzed"
			}
		}

	},
	"LiveProgramList":{
		"properties":{
			"status":{
				"type":"string",
				"index":"not_analyzed"
			},
                        "info":{
				"type":"string",
				"index":"analyzed"
			}
		}
	}
    }
}
{
    "template": "nginx_*",
    "settings": {
        "index.number_of_shards": 5,
        "number_of_replicas": 1
    },
    "mappings": {
        "_default_": {
            "_all": {
                "enabled": "false"
            },
            "_source": {
                "enabled": true,
                "compress": true
            },
            "properties": {
                "_id": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "timestamp": {
                    "enabled": true,
                    "type": "date",
                    "format": "dateOptionalTime"
                },
                "host": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "message": {
                    "type": "string",
                    "index": "no"
                },
                "chost": {
                    "type": "ip",
                    "index": "not_analyzed"
                },
                "cgeo": {
                    "type": "object",
                    "properties": {
                        "country_name": {
                            "type": "string",
                            "index": "not_analyzed"
                        },
                        "real_region_name": {
                            "type": "string",
                            "index": "not_analyzed"
                        }
                    }
                }
		,
		"status":{
			"type":"integer"
		}
            }
        },
	"error":{
		"properties":{
		   "interval":{
			   "dynamic":true,
			   "type":"integer"
		   }
		}

	}
    }
}
{
    "template": "nginx_access_*",
    "settings" : {
        "number_of_shards" : 5,
        "number_of_replicas" : 1,
        "index" : {
            "query" : { "default_field" : "message" }
        }
    },
    "mappings": {
        "nginx_access": { 
            "_all": { "enabled": false },
            "_source": {
                "enabled": true,
                "compress": true
            },
            "dynamic_templates": [
                {
                    "string_template" : { 
                        "match" : "*",
                        "mapping": { "type": "string", "index": "not_analyzed" },
                        "match_mapping_type" : "string"
                     } 
                 }
             ],
            "properties" : {
                "@timestamp": { "type": "date", "format": "dateOptionalTime" },
                "fid" : { "type" : "string", "index" : "not_analyzed" },
                "filename" : { "type" : "string", "index" : "analyzed" },
                "host" : { "type" : "string", "index" : "analyzed" },
                "message" : { "type" : "string", "index" : "analyzed" },
                "referer" : { "type" : "string", "index" : "analyzed" },
                "request": { "type": "string", "index" : "analyzed" },
                "sentbytes" : { "type" : "long" },
                "status" : { "type" : "short" },
                "type" : { "type" : "string", "index" : "not_analyzed" },
                "user_agent" : { "type" : "string", "index" : "analyzed" },
                "userip" : { "type" : "string", "index" : "not_analyzed" },
                "uuid" : { "type" : "string", "index" : "not_analyzed" },
                "verb" : { "type" : "string", "index" : "not_analyzed" },
                "limitrate" : { "type" : "integer" },
                "nid" : { "type" : "short" },
                "payload" : { "type" : "string", "index" : "analyzed" },
                "sign" : { "type" : "string", "index" : "not_analyzed"},
                "urgid" : { "type" : "short" },
                "srgid" : { "type" : "short" },
                "srgids" : { "type" : "string", "index" : "analyzed" },
                "t" : { "type" : "string", "index" : "not_analyzed" },
                "geoip" : {
                    "dynamic" : true,
                    "path" : "full",
                    "properties" : {
                      "location" : {
                        "type" : "geo_point"
                      }
                    },
                    "type" : "object"
                }
            }
        }
    }
}
{
    "template": "res-monitor_*",
    "settings": {
        "index.number_of_shards": 5,
        "number_of_replicas": 1
    },
    "mappings": {
        "_default_": {
            "_all": {
                "enabled": "false"
            },
            "_source": {
                "enabled": true,
                "compress": true
            },
            "properties": {
                "_id": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "timestamp": {
                    "enabled": true,
                    "type": "date",
                    "format": "dateOptionalTime"
                },
                "host": {
                    "type": "string",
                    "index": "not_analyzed"
                }
	
            }
        },
	"VRB":{
		"properties":{
			"EpochTime":{
				"type":"string",
				"format":"not_analyzed"
			},
			"NodeId":{
				"type":"integer"
			},
			"CurrentConn":{
				"type":"integer"
			},
			"CurrentBandwidth":{
				"type":"integer"
			},
			"DiskTotalSpace":{
				"type":"long"
			},
			"DiskFreeSpace":{
				"type":"long"
			},
			"CpuUsage":{
				"type":"integer"
			},
			"MemUsage":{
				"type":"integer"
			},
			"WanUsage":{
				"type":"integer"
			},
			"LanUsage":{
				"type":"integer"
			},
			"IoUsage":{
				"type":"integer"
			},
			"LanIpState":{
				"type":"integer"
			},
			"WanIpState":{
				"type":"integer"
			}
		}

	},
	"ERR":{
		"properties":{
			"message":{
				"type":"string",
				"index":"not_analyzed"
			},
			"info":{
				"type":"string",
				"index":"not_analyzed"
			},
			"errInfo":{
				"type":"string",
				"index":"not_analyzed"
			}
		}
	},
	"WRN":{
		"properties":{
			"message":{
				"type":"string",
				"index":"not_analyzed"
			},
			"info":{
				"type":"string",
				"index":"not_analyzed"
			},
                        "errInfo":{
				"dynamic":true,
 				"type":"string",
 				"index":"not_analyzed"
			}
		}
	}
    }
}
{
    "template": "router_*",
    "settings": {
        "index.number_of_shards": 5,
        "number_of_replicas": 1
    },
    "mappings": {
        "_default_": {
            "_all": {
                "enabled": "false"
            },
            "_source": {
                "enabled": true,
                "compress": true
            },
            "properties": {
                "_id": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "timestamp": {
                    "enabled": true,
                    "type": "date",
                    "format": "dateOptionalTime"
                },
                "host": {
                    "type": "string",
                    "index": "not_analyzed"
                },
		"chost":{
			"type":"ip",
			"index":"not_analyzed"
		},
		"isThirdCDN":{
			"type":"boolean"
		},
		"requestPolicyID":{
			"type":"integer"
		},
		"rateLimited":{
			"type":"long"
		}
            }
        },
	"ok":{
		"properties":{
	           "requestURL":{
			   "type":"string",
			   "index":"not_analyzed"
		   },
                   "requestServer":{
                            "type":"ip",
                            "index":"not_analyzed"
			},
                   "fileLoc":{
                             "type":"string",
                             "index":"not_analyzed"
			}
		}
	},
        "error":{
 		"properties":{
			 "description":{
				"type":"string",
				"index":"not_analyzed"
			    }

		}
	}
    }
}
{
    "template": "routerdll_*",
    "settings": {
        "index.number_of_shards": 5,
        "number_of_replicas": 1
    },
    "mappings": {
        "_default_": {
            "_all": {
                "enabled": "false"
            },
            "_source": {
                "enabled": true,
                "compress": true
            },
            "properties": {
                "_id": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "timestamp": {
                    "enabled": true,
                    "