#!/bin/bash ################################################################################################## # log_search # ################################################################################################## # This utility was made to make searching for logs much quicker on both a redbox (journal) & on # # the logserver (via messages files). # # # # # # # # # ################################################################################################## # BEGIN: Variables declare -A CONFIG CONFIG[source]="" CONFIG[mode]="" CONFIG[regex]="" CONFIG[service_search]=0 CONFIG[last_nth_days]=1 CONFIG[hostname]="N/A" CONFIG[service_filter]="N/A" declare -A SEARCH_MODES=( ['CLUSTER']='(crmd|stonith|pengine|lrmd|pacemakerd|corosync|drbd)\[' ['ISSUES']='warn|crit|fail|err' ) # END: Variables # BEGIN: Helper Functions usage () { cat <] [-m|--mode ] [--services] [-l|--last-nth-days ] [-H|--hostname ] Sources: log : When used on the log-server for a customer environment (references the hosts messages file /var/log/hosts//YYYY/MM/DD/messages*) journal : When used on a node (referenced a nodes journal for finding errors) Search Modes cluster : Searching for errors that are cluster related (matches regex: ${SEARCH_MODES[CLUSTER]}) issues : General issues that report a warning, failure, error, critical, etc (matches regex: ${SEARCH_MODES[ISSUES]}) Flags: Universal: --services : Get all the service names that return from the search --filter-regex : Will be used to filter only the services you want to Log Source: -l|--last-nth-days : (only used with log source) Search for logs in the last nth days. -H|--hostname : Hostname to search against EOF } # END: Helper Functions # BEGIN: Handle CLI Args if [[ $# -eq 0 ]]; then usage exit fi while [[ $# -ne 0 ]]; do case $1 in -s | --source ) if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(log|journal)$ ]]; then case ${2,,} in log) CONFIG[source]="log";; journal) CONFIG[source]='journal';; esac shift continue else echo "$2 is not a valid option for $1" usage exit fi ;; -m | --mode) if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(issues|cluster)$ ]]; then case ${2,,} in issues) CONFIG[mode]="issues" CONFIG[regex]=${SEARCH_MODES[ISSUES]} ;; cluster) CONFIG[mode]='cluster' CONFIG[regex]=${SEARCH_MODES[CLUSTER]} ;; esac shift continue else echo "$2 is not a valid option for $1" usage exit fi ;; --services) CONFIG[service_search]=1 ;; -H | --hostname ) if [[ "$2" != "" ]] ; then LOG_PATH=/var/log/hosts/$2 if [[ -d $LOG_PATH ]]; then CONFIG[hostname]=$2 else echo "$LOG_PATH does not exist. Either the host doesn't exist or hasn't logged to this server" usage exit fi fi ;; esac shift done # END: Handle CLI Args # BEGIN: Pre-Work Checks for KEY in ${!CONFIG[@]}; do if [[ "${CONFIG[$KEY]}" == "" ]]; then echo "$KEY was not set" exit fi done case ${CONFIG[mode]} in log) if [[ ${CONFIG[hostname]} == "N/A" ]]; then echo "A hostname was not provieded (via -H|--hostname), and it is required to do a log search" usage exit fi ;; esac # END: Pre-Work Checks # BEGIN: Work Functions gather_logs_from_journal () { local -n OUTPUT_VAR=${1:?"$FUNCNAME: Output Variable not provided"} OUTPUT_VAR=$( sudo journalctl --no-pager | grep -E ${CONFIG[regex]} ) } gather_logs_from_files () { shopt -s globstar local -a FILES=( $( ls -1r /var/log/hosts/${CONIFG[hostname]}/**/messages* ) ) local -n OUTPUT_VAR=${1:?"$FUNCNAME: Output Variable not provided"} local OUTPUT BUFFER local FILE for FILE in ${FILES[@]:0:${CONFIG[last_nth_days]}}; do BUFFER=$( zgrep -E "${CONFIG[regex]}" $FILE ) OUTPUT="$OUTPUT\n$BUFFER" done OUTPUT_VAR=$( echo -e "$OUTPUT" ) } # END: Work Functions # BEGIN: Work OUTPUT="" case ${CONFIG[source]} in log) gather_logs_from_files OUTPUT;; journal) gather_logs_from_journal OUTPUT;; esac if [[ ${CONFIG[service_search]} -eq 1 ]]; then echo "$OUTPUT" | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u elif [[ ${CONFIG[service_filter]} != "N/A" ]]; then echo "$OUTPUT" | grep -E "${CONFIG[service_filter]}" else echo "$OUTPUT" fi # END: Work