#!/bin/bash ################################################################################################## # log_search # ################################################################################################## # This utility was made to make searching for logs much quicker on both a redbox (journal) & on # # the logserver (via messages files). # # # ################################################################################################## # Author # # ########## # # Email: tristan.ancelet@acumera.com # # Work #: +1 (337) 965-1855 # # # ################################################################################################## # BEGIN: Variables declare -A CONFIG CONFIG[source]="journal" CONFIG[mode]="all" CONFIG[regex]="." CONFIG[service_search]=0 CONFIG[last_nth_days]=1 CONFIG[hostname]="N/A" CONFIG[service_filter]="N/A" CONFIG[output_file]="N/A" CONFIG[debug]=0 CONFIG[day_filter]=0 CONFIG[hour_filter]=0 CONFIG[remote_host]="N/A" declare -A SEARCH_MODES=( ['CLUSTER']='(crmd|stonith|pengine|lrmd|pacemakerd|corosync|drbd)\[' ['ISSUES']='warn|crit|fail|err' ['ALL']='.' ) # END: Variables # BEGIN: Helper Functions log () { if [[ ${CONFIG[debug]} -eq 1 ]]; then local MESSAGE=${1:?"$FUNCNAME: No message provided"} local LEVEL_NUM=${2:-0} case $LEVEL_NUM in 0) LEVEL=INFO;; 1) LEVEL=WARN;; 2) LEVEL=CRIT;; *) LEVEL=UNDEF;; esac echo "$(date) : $LEVEL : ${FUNCNAME[1]} : $MESSAGE" fi } usage () { cat <] [-m|--mode ] [--services] [-l|--last-nth-days ] [-H|--hostname ] [-d|--debug] [-o|--output-file ] Sources: log : When used on the log-server for a customer environment (references the hosts messages file /var/log/hosts//YYYY/MM/DD/messages*) journal (default) : When used on a node (referenced a nodes journal for finding errors) Search Modes cluster : Searching for errors that are cluster related (matches regex: ${SEARCH_MODES[CLUSTER]}) issues : General issues that report a warning, failure, error, critical, etc (matches regex: ${SEARCH_MODES[ISSUES]}) custom [custom-regex] : Requires you to provide your own regex as the next argument (make sure to escape your string carefully) all (default) : Just output all logs provided by your source Flags: Universal: --services : Get all the service names that return from the search -f|--filter-regex : Will be used to filter only the services you want to -d|--debug : Turns on debugging -o|--output-file : Filename to output to --remote : This will run the gathering code remotely Log Source: -l|--last-nth-days : (only used with log source) Search for logs in the last nth days. -H|--hostname : Hostname to search against Journal Source: --hours : This will narrow down to within the last X hours --days : This will narrow down to within the last X days EOF } # END: Helper Functions # BEGIN: Handle CLI Args if [[ $# -eq 0 ]]; then usage exit fi if grep -E ' (-d|--debug) ' <<< "$@" ; then CONFIG[debug]=1 log "user set debug mode" fi while [[ $# -ne 0 ]]; do log "\$1 is $1" case $1 in --hours) if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then if [[ ${CONFIG[day_filter]} -eq 0 ]]; then CONFIG[hour_filter]=$2 else echo "$1 cannot be used with --days" usage exit fi fi ;; --days) if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then if [[ ${CONFIG[hour_filter]} -eq 0 ]]; then CONFIG[day_filter]=$2 else echo "$1 cannot be used with --hours" usage exit fi fi ;; -s | --source ) if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(log|journal)$ ]]; then case ${2,,} in log) CONFIG[source]="log";; journal) CONFIG[source]='journal';; esac log "user set the source to ${CONFIG[source]}" shift continue else echo "$2 is not a valid option for $1" usage exit fi ;; -m | --mode) if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(issues|cluster|custom)$ ]]; then case ${2,,} in issues) CONFIG[mode]="issues" CONFIG[regex]=${SEARCH_MODES[ISSUES]} ;; cluster) CONFIG[mode]='cluster' CONFIG[regex]=${SEARCH_MODES[CLUSTER]} ;; custom ) CONFIG[mode]='custom' CONFIG[regex]="$3" ;; all ) CONFIG[mode]='all' CONFIG[regex]="${SEARCH_MODES[ALL]}" ;; esac log "user set search mode and regex to ${CONFIG[mode]}, ${CONFIG[regex]}" shift continue else echo "$2 is not a valid option for $1" usage exit fi ;; -o | --output-file) if [[ $2 != "" ]]; then log "user set output file to be ${CONFIG[output_file]}" CONFIG[output_file]=$2 fi ;; --services) log "user is searching for service names" CONFIG[service_search]=1 ;; -f | --filter-regex) if [[ "$2" != "" ]]; then CONFIG[service_filter]=$2 log "user provided a search term/regex ${CONFIG[service_filter]}" shift continue fi ;; -H | --hostname ) if [[ "$2" != "" ]] ; then LOG_PATH=/var/log/hosts/$2 if [[ -d $LOG_PATH ]]; then log "user set hostname to $2" CONFIG[hostname]=$2 else echo "$LOG_PATH does not exist. Either the host doesn't exist or hasn't logged to this server" usage exit fi fi ;; esac shift done # END: Handle CLI Args # BEGIN: Pre-Work Checks for KEY in ${!CONFIG[@]}; do if [[ "${CONFIG[$KEY]}" == "" ]]; then echo "$KEY was not set" exit elif [[ "${CONFIG[$KEY]}" == "N/A" ]]; then CONFIG[$KEY]="" fi done case ${CONFIG[mode]} in log) if [[ ${CONFIG[hostname]} == "" ]]; then echo "A hostname was not provieded (via -H|--hostname), and it is required to do a log search" usage exit fi ;; esac # END: Pre-Work Checks # BEGIN: Work Functions import () { declare -f gather_logs_from_journal gather_logs_from_files log CONFIG[remote_host]="" declare -p CONFIG } gather_logs_from_journal () { if [[ ${CONFIG[day_filter]} -gt 0 ]]; then sudo journalctl --since "${CONFIG[day_filter]} days ago" --no-pager | grep -E ${CONFIG[regex]} elif [[ ${CONFIG[hour_filter]} -gt 0 ]]; then sudo journalctl --since "${CONFIG[hour_filter]} hours ago" --no-pager | grep -E ${CONFIG[regex]} else sudo journalctl --no-pager | grep -E ${CONFIG[regex]} fi } gather_logs_from_files () { log "User is getting logs from the last ${CONFIG[last_nth_days]} files" local -a FILES=( $( ls -1r /var/log/hosts/${CONFIG[hostname]}/*/*/*/messages* ) ) log " Files Found: $( for FILE in ${FILES[@]}; do echo "- $FILE" done )" local OUTPUT BUFFER EXT local FILE for FILE in ${FILES[@]::${CONFIG[last_nth_days]}}; do log "$FILE being checked" EXT=$( basename $FILE | cut -d '.' -f 2 ) log "EXT was found to be $EXT" case $EXT in bz* ) GREP="bzgrep";; gz* | zip ) GREP="zgrep";; * ) GREP="grep";; esac log "GREP was set to $GREP" sudo $GREP -E "${CONFIG[regex]}" $FILE done } # END: Work Functions # BEGIN: Work case ${CONFIG[source]} in log) COMMAND=gather_logs_from_files;; journal) COMMAND=gather_logs_from_journal;; esac if [[ ${CONFIG[service_search]} -eq 1 ]]; then $COMMAND | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u | tr -d ':' elif [[ "${CONFIG[service_filter]}" ]]; then $COMMAND | grep -E "${CONFIG[service_filter]}" elif [[ "${CONFIG[output_file]}" ]]; then log "outputting all searches from user to ${CONFIG[output_file]}" $COMMAND >${CONFIG[output_file]} else $COMMAND fi # END: Work