1.Haproxy配置日志规则

在/etc/haproxy/haproxy.conf的frontend下增加

option httplog
option logasap
log LogServerIP local5capture request header Host len 40
capture request header X-Forwarded-For len 50
#capture request header Accept-Language len 50
capture request header Referer len 200
capture request header User-Agent len 200

2.syslog配置开启远程接收

3.Logstash配置

indexer

input {file {path => "/var/log/haproxy.log"start_position => beginningsincedb_write_interval => 0type => "HAPROXY_LOG"codec => plain {charset => "ISO-8859-1"}}
}output {#stdout { codec => rubydebug}redis {data_type => "list"key => "logstash:Haproxy_log"host => "192.168.1.2"port => 6379}
}

shipping

在logstash配置文件目录下新建patterns目录，并在这个目录下新建一个haproxy的文件

vim /usr/local/logstash2.2.2/conf/patterns/haproxy

HAPROXYTIME (?!<[0-9])%{HOUR:haproxy_hour}:%{MINUTE:haproxy_minute}(?::%{SECOND:haproxy_second})(?![0-9])
HAPROXYDATE %{MONTHDAY:haproxy_monthday}/%{MONTH:haproxy_month}/%{YEAR:haproxy_year}:%{HAPROXYTIME:haproxy_time}.%{INT:haproxy_milliseconds}

vim /usr/local/logstash2.2.2/conf/haproxy.conf

input {redis {data_type => "list"key => "logstash:Haproxy_log"host => "192.168.1.2"port => 6379threads => 5type => "HAPROXY_LOG"}
}filter {grok{patterns_dir => "./patterns"match => ["message" , "%{SYSLOGTIMESTAMP:Syslog_Timestamp} %{IPORHOST:Syslog_Server} %{SYSLOGPROG}: %{IP:Client_IP}:%{INT:Client_Port} \[%{HAPROXYDATE:Accept_Date}\] %{NOTSPACE:Frontend_Name} %{NOTSPACE:Backend_Name}/%{NOTSPACE:Server_Name} %{INT:Time_Request}/%{INT:Time_Queue}/%{INT:Time_Backend_Connect}/%{INT:Time_Backend_Response}/\+%{NOTSPACE:Time_Duration} %{INT:Http_Status_Code} \+%{NOTSPACE:Bytes_Read} %{DATA:Captured_Request_Cookie} %{DATA:Captured_Response_Cookie} %{NOTSPACE:Termination_State} %{INT:Actconn}/%{INT:Feconn}/%{INT:Beconn}/%{INT:Srvconn}/%{NOTSPACE:Retries} %{INT:Srv_Queue}/%{INT:Backend_Queue} (\{%{IPORHOST:Site}\|?((%{IP:X_Forward_IP}|)(, |)(%{IP:X_Forward_IP2}|)(, |)(%{IP:X_Forward_IP3|)(%{DATA:X_Forward_Other}|))?\|?(%{URI:Http_Referer})?\|%{GREEDYDATA:User_Agent}\})?( )( )?\"(<BADREQ>|(%{WORD:Http_Method} (%{URIPROTO:Http_Proto}://)?(?:%{USER:Http_User}(?::[^@]*)?@)?(?:%{URIHOST:Http_Host})?(?:%{URIPATHPARAM:Http_Request})?( HTTP/%{NUMBER:Http_Version})?))?\""]}useragent {source => "User_Agent"target => "ua"}if [X_Forward_IP] =~ "." {geoip {source => ["X_Forward_IP"]database => "/usr/local/logstash2.2.2/bin/GeoLiteCity.dat"}} else {geoip {source => ["Client_IP"]database => "/usr/local/logstash2.2.2/bin/GeoLiteCity.dat"}}date{match => ["Accept_Date", "dd/MMM/yyyy:HH:mm:ss.SSS" ]}mutate{remove_field => ["Syslog_Timestamp"]remove_field => ["Accept_Date"]remove_field => [ "Host" ]remove_field => [ "Syslog_Server" ]remove_field => [ "Path" ]remove_field => [ "pid" ]remove_field => [ "Client_Port" ]remove_field => [ "program" ]remove_field => [ "Haproxy_Monthday" ]remove_field => [ "Haproxy_Month" ]remove_field => [ "Haproxy_Year" ]remove_field => [ "Haproxy_Hour" ]remove_field => [ "Haproxy_Minute" ]remove_field => [ "Haproxy_Second" ]remove_field => [ "Haproxy_Milliseconds" ]remove_field => [ "Frontend_Name" ]remove_field => [ "Captured_Response_Cookie" ]remove_field => [ "Captured_Request_Cookie" ]convert => [ "Http_Status_Code","integer" ]convert => [ "Bytes_Read","integer" ]convert => [ "Time_Duration","integer" ]convert => [ "Time_Backend_Response","integer" ]convert => [ "Actconn","integer" ]convert => [ "Feconn","integer" ]convert => [ "Beconn","integer" ]convert => [ "Srvconn","integer" ]convert => [ "Retries","integer" ]convert => [ "Srv_Queue","integer" ]convert => [ "Backend_Queue","integer" ]convert => [ "Time_Request","integer" ]convert => [ "Time_Queue","integer" ]convert => [ "Time_Backend_Connect","integer" ]}
}output {#stdout { codec => rubydebug }elasticsearch {hosts => "192.168.2.240:9200"index => "logstash-haproxy-%{+YYYY.MM.dd}"}
}