Splunk

Splunk forwarder

Setup splunk-forwarder agent on Linux

cd /opt/splunkforwarder/bin/
./splunk add  forward-server splunk.acme.com:9997 --accept-license --no-prompt -auth admin:changeme
./splunk list forward-server
./splunk set  deploy-poll    splunk.acme.com:8089
./splunk show deploy-poll
./splunk enable  listen 9997
./splunk display listen
./splunk set  servername <serverName|hostname>
./splunk show servername
./splunk set  default-hostname <serverName|hostname>
./splunk show default-hostname
./splunk enable boot-start

Splunk search

Parse and visualize IIS access logs

# Example regex to process standard IIS logs would look like, saved on "Extract Fields" page
^(?:[^:\n]*:){2}\d+\s+(?P<server_ip>\d+\.\d+\.\d+\.\d+)\s+(?P<request_type>[^ ]+)\s+(?P<path>[^ ]+)(?:[^ \n]* ){2}(?P<port>\d+)\s+\-\s+(?P<source_ip>[^ ]+)(?:[^ \n]* ){3}(?P<response_code>\d+)\s+(\d+\s+)+(?P<response_time>.+)

# Show response times greater than 2 seconds duration
index="iis_access_logs" host="frontend-app-*-test"| where ResponseTime >2000 | timechart span=1m count
# | timechart function allows to visualise the results

# Count Response Codes
index="iis_access_logs" host="frontend-app-*-test"| timechart span=10m count by ResponseCode limit=10
index="iis_access_logs" host="frontend-app-*-test"| timechart span=10m count by server_ip    limit=10
# | fields ResponseCode, server_ip can be previewed on Extract Fields page

# Maximum and average response times
index="iis_access_logs" host="frontend-app-*-test"| bucket span=10s _time | streamstats time_window=60s avg(ResponseTime) | chart max(ResponseTime) avg(ResponseTime) over _time