हम लॉगस्टैश का उपयोग करके इकट्ठा, पार्स और लॉग देते हैं

 input { file { type => "some_access_log" path => [ "/var/log/vs01/*.log", "/var/log/vs02/*.log" ] exclude => [ "*.gz", "*.zip", "*.rar" ] start_position => "end" stat_interval => 1 discover_interval => 30 } } 

 type => "some_access_log" 

 path => [ "/var/log/vs01/*.log", "/var/log/vs02/*.log" ] 

 exclude => [ "*.gz", "*.zip", "*.rar" ] 

 start_position => "end" 

 stat_interval => 1 

 discover_interval => 30 

 input { tcp { type => "webserver_prod" data_timeout => 10 mode => "server" host => "192.168.3.12" port => 3337 } } 

 type => "webserver_prod" 

 data_timeout => 10 

 mode => "server" host => "192.168.3.12" port => 3337 

 input { udp { type => "webserver_prod" buffer_size => 4096 host => "192.168.3.12" port => 3337 } } 

 filter { grok { type => "some_access_log" patterns_dir => "/path/to/patterns/" pattern => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" } } 

 type => "apache_access" 

 patterns_dir => "/path/to/patterns/" 

 pattern => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" 

 NUMBER \d+ WORD \b\w+\b USERID [a-zA-Z0-9_-]+ 

 USER %{USERID} 

 CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) MAC (?:%{CISCOMAC}|%{WINDOWSMAC}) 

 filter { mutate { type => "apache_access" remove => [ "client" ] rename => [ "HOSTORIP", "client_ip" ] gsub => [ "message", "\\/", "_" ] add_field => [ "sample1", "from %{clientip}" ] } } 

 type => "apache_access" 

 remove => [ "client" ] 

 rename => [ "HOSTORIP", "client_ip" ] 

 gsub => [ "message", "\\/", "_" ] 

 add_field => [ "sample1", "from %{clientip}" ] 

 filter { date { type => "apache_access" match => [ "timestamp", "MMM dd HH:mm:ss" ] } } 

 type => "apache_access" 

 match => [ "timestamp", "MMM dd HH:mm:ss" ] 

 filter { kv { type => "custom_log" value_split => "=:" fields => ["reminder"] field_split => "\t?&" } } 

 type => "custom_log" 

 value_split => "=:" 

 fields => ["reminder"] 

 field_split => "\t?&" 

 filter { multiline { type => "java_log" pattern => "^\s" what => "previous" } } 

 type => "java_log" 

 pattern => "^\s" 

 what => "previous" 

 output { stdout { type => "custom_log" message => "IP - %{clientip}. Full message: %{@message}. End of line." } } 

 type => "custom_log" 

 message => "clIP - %{clientip}. Full message: %{@message}. End of line." 

 output { file { type => "custom_log" flush_interval => 5 gzip=> true path => "/var/log/custom/%{clientip}/%{type}" message_format => "ip: %{clientip} request:%{requri}" } } 

 type => "custom_log" 

 flush_interval => 5 

 gzip=> true 

 path => "/var/log/custom/%{clientip}/%{type}" 

 message_format => "ip: %{clientip} request:%{requri}" 

 output { elasticsearch { type => "custom_log" cluster => "es_logs" embedded => false host => "192.168.1.1" port => "19300" index => "logs-%{+YYYY.MM.dd}" } } 

 type => "custom_log" 

 cluster => "es_logs" 

 embedded => false 

 port => "19300" 

 host => "192.168.1.1" 

 index => "logs-%{+YYYY.MM.dd}" 

 output { email { type => "custom_log" from => "logstash@domain.com" to => "admin1@domain.com" cc => "admin2@domain.com" subject => "Found '%{matchName}' Alert on %{@source_host}" body => "Here is the event line %{@message}" htmlbody => "<h2>%{matchName}</h2><br/><br/><h3>Full Event</h3><br/><br/><div align='center'>%{@message}</div>" via => "sendmail" options => [ "smtpIporHost", "smtp.gmail.com", "port", "587", "domain", "yourDomain", "userName", "yourSMTPUsername", "password", "PASS", "starttls", "true", "authenticationType", "plain", "debug", "true" ] match => [ "response errors", "response,501,,or,response,301", "multiple response errors", "response,501,,and,response,301" ] } } 

 type => "custom_log" 

 from => "logstash@domain.com" to => "admin1@domain.com" cc => "admin2@domain.com" 

 subject => "Found '%{matchName}' Alert on %{@source_host}" 

 body => "Here is the event line %{@message}" htmlbody => "<h2>%{matchName}</h2><br/><br/><h3>Full Event</h3><br/><br/><div align='center'>%{@message}</div>" 

 via => "sendmail" 

 options => ... 

 match => [ "response errors", "response,501,,or,response,301", "multiple response errors", "response,501,,and,response,301" ]