diff --git a/log-search b/log-search index 2071b83..eb091a5 100644 --- a/log-search +++ b/log-search @@ -1,14 +1,16 @@ #!/bin/bash - ################################################################################################## # log_search # ################################################################################################## # This utility was made to make searching for logs much quicker on both a redbox (journal) & on # # the logserver (via messages files). # # # -# # -# # +################################################################################################## +# Author # # +########## # +# Email: tristan.ancelet@acumera.com # +# Work #: +1 (337) 965-1855 # # # ################################################################################################## @@ -16,17 +18,23 @@ # BEGIN: Variables declare -A CONFIG -CONFIG[source]="" -CONFIG[mode]="" -CONFIG[regex]="" +CONFIG[source]="journal" +CONFIG[mode]="all" +CONFIG[regex]="." CONFIG[service_search]=0 CONFIG[last_nth_days]=1 CONFIG[hostname]="N/A" CONFIG[service_filter]="N/A" +CONFIG[output_file]="N/A" +CONFIG[debug]=0 +CONFIG[day_filter]=0 +CONFIG[hour_filter]=0 +CONFIG[remote_host]="N/A" declare -A SEARCH_MODES=( ['CLUSTER']='(crmd|stonith|pengine|lrmd|pacemakerd|corosync|drbd)\[' ['ISSUES']='warn|crit|fail|err' + ['ALL']='.' ) # END: Variables @@ -35,25 +43,49 @@ declare -A SEARCH_MODES=( # BEGIN: Helper Functions +log () { + if [[ ${CONFIG[debug]} -eq 1 ]]; then + local MESSAGE=${1:?"$FUNCNAME: No message provided"} + local LEVEL_NUM=${2:-0} + + case $LEVEL_NUM in + 0) LEVEL=INFO;; + 1) LEVEL=WARN;; + 2) LEVEL=CRIT;; + *) LEVEL=UNDEF;; + esac + echo "$(date) : $LEVEL : ${FUNCNAME[1]} : $MESSAGE" + fi +} + + usage () { cat <] [-m|--mode ] [--services] [-l|--last-nth-days ] [-H|--hostname ] +$( basename $0 ) [-s|--source ] [-m|--mode ] [--services] [-l|--last-nth-days ] [-H|--hostname ] [-d|--debug] [-o|--output-file ] Sources: - log : When used on the log-server for a customer environment (references the hosts messages file /var/log/hosts//YYYY/MM/DD/messages*) - journal : When used on a node (referenced a nodes journal for finding errors) + log : When used on the log-server for a customer environment (references the hosts messages file /var/log/hosts//YYYY/MM/DD/messages*) + journal (default) : When used on a node (referenced a nodes journal for finding errors) Search Modes - cluster : Searching for errors that are cluster related (matches regex: ${SEARCH_MODES[CLUSTER]}) - issues : General issues that report a warning, failure, error, critical, etc (matches regex: ${SEARCH_MODES[ISSUES]}) + cluster : Searching for errors that are cluster related (matches regex: ${SEARCH_MODES[CLUSTER]}) + issues : General issues that report a warning, failure, error, critical, etc (matches regex: ${SEARCH_MODES[ISSUES]}) + custom [custom-regex] : Requires you to provide your own regex as the next argument (make sure to escape your string carefully) + all (default) : Just output all logs provided by your source Flags: Universal: - --services : Get all the service names that return from the search - --filter-regex : Will be used to filter only the services you want to + --services : Get all the service names that return from the search + -f|--filter-regex : Will be used to filter only the services you want to + -d|--debug : Turns on debugging + -o|--output-file : Filename to output to + --remote : This will run the gathering code remotely Log Source: - -l|--last-nth-days : (only used with log source) Search for logs in the last nth days. - -H|--hostname : Hostname to search against + -l|--last-nth-days : (only used with log source) Search for logs in the last nth days. + -H|--hostname : Hostname to search against + Journal Source: + --hours : This will narrow down to within the last X hours + --days : This will narrow down to within the last X days EOF } @@ -67,14 +99,43 @@ if [[ $# -eq 0 ]]; then exit fi +if grep -E ' (-d|--debug) ' <<< "$@" ; then + CONFIG[debug]=1 + log "user set debug mode" +fi + while [[ $# -ne 0 ]]; do + log "\$1 is $1" case $1 in + --hours) + if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then + if [[ ${CONFIG[day_filter]} -eq 0 ]]; then + CONFIG[hour_filter]=$2 + else + echo "$1 cannot be used with --days" + usage + exit + fi + fi + ;; + --days) + if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then + if [[ ${CONFIG[hour_filter]} -eq 0 ]]; then + CONFIG[day_filter]=$2 + else + echo "$1 cannot be used with --hours" + usage + exit + fi + fi + ;; -s | --source ) if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(log|journal)$ ]]; then case ${2,,} in log) CONFIG[source]="log";; journal) CONFIG[source]='journal';; esac + log "user set the source to ${CONFIG[source]}" shift continue else @@ -85,7 +146,7 @@ while [[ $# -ne 0 ]]; do ;; -m | --mode) - if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(issues|cluster)$ ]]; then + if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(issues|cluster|custom)$ ]]; then case ${2,,} in issues) CONFIG[mode]="issues" @@ -95,7 +156,16 @@ while [[ $# -ne 0 ]]; do CONFIG[mode]='cluster' CONFIG[regex]=${SEARCH_MODES[CLUSTER]} ;; + custom ) + CONFIG[mode]='custom' + CONFIG[regex]="$3" + ;; + all ) + CONFIG[mode]='all' + CONFIG[regex]="${SEARCH_MODES[ALL]}" + ;; esac + log "user set search mode and regex to ${CONFIG[mode]}, ${CONFIG[regex]}" shift continue else @@ -105,14 +175,31 @@ while [[ $# -ne 0 ]]; do fi ;; + -o | --output-file) + if [[ $2 != "" ]]; then + log "user set output file to be ${CONFIG[output_file]}" + CONFIG[output_file]=$2 + fi + ;; + --services) + log "user is searching for service names" CONFIG[service_search]=1 ;; + -f | --filter-regex) + if [[ "$2" != "" ]]; then + CONFIG[service_filter]=$2 + log "user provided a search term/regex ${CONFIG[service_filter]}" + shift + continue + fi + ;; -H | --hostname ) if [[ "$2" != "" ]] ; then LOG_PATH=/var/log/hosts/$2 if [[ -d $LOG_PATH ]]; then + log "user set hostname to $2" CONFIG[hostname]=$2 else echo "$LOG_PATH does not exist. Either the host doesn't exist or hasn't logged to this server" @@ -136,12 +223,14 @@ for KEY in ${!CONFIG[@]}; do if [[ "${CONFIG[$KEY]}" == "" ]]; then echo "$KEY was not set" exit + elif [[ "${CONFIG[$KEY]}" == "N/A" ]]; then + CONFIG[$KEY]="" fi done case ${CONFIG[mode]} in log) - if [[ ${CONFIG[hostname]} == "N/A" ]]; then + if [[ ${CONFIG[hostname]} == "" ]]; then echo "A hostname was not provieded (via -H|--hostname), and it is required to do a log search" usage exit @@ -156,23 +245,49 @@ esac # BEGIN: Work Functions +import () { + declare -f gather_logs_from_journal gather_logs_from_files log + CONFIG[remote_host]="" + declare -p CONFIG +} + gather_logs_from_journal () { - local -n OUTPUT_VAR=${1:?"$FUNCNAME: Output Variable not provided"} - OUTPUT_VAR=$( sudo journalctl --no-pager | grep -E ${CONFIG[regex]} ) + if [[ ${CONFIG[day_filter]} -gt 0 ]]; then + sudo journalctl --since "${CONFIG[day_filter]} days ago" --no-pager | grep -E ${CONFIG[regex]} + + elif [[ ${CONFIG[hour_filter]} -gt 0 ]]; then + sudo journalctl --since "${CONFIG[hour_filter]} hours ago" --no-pager | grep -E ${CONFIG[regex]} + + else + sudo journalctl --no-pager | grep -E ${CONFIG[regex]} + fi } gather_logs_from_files () { - shopt -s globstar - local -a FILES=( $( ls -1r /var/log/hosts/${CONIFG[hostname]}/**/messages* ) ) - local -n OUTPUT_VAR=${1:?"$FUNCNAME: Output Variable not provided"} - local OUTPUT BUFFER + log "User is getting logs from the last ${CONFIG[last_nth_days]} files" + local -a FILES=( $( ls -1r /var/log/hosts/${CONFIG[hostname]}/*/*/*/messages* ) ) + log " Files Found: +$( +for FILE in ${FILES[@]}; do + echo "- $FILE" +done +)" + + local OUTPUT BUFFER EXT local FILE - for FILE in ${FILES[@]:0:${CONFIG[last_nth_days]}}; do - BUFFER=$( zgrep -E "${CONFIG[regex]}" $FILE ) - OUTPUT="$OUTPUT\n$BUFFER" + for FILE in ${FILES[@]::${CONFIG[last_nth_days]}}; do + log "$FILE being checked" + EXT=$( basename $FILE | cut -d '.' -f 2 ) + log "EXT was found to be $EXT" + case $EXT in + bz* ) GREP="bzgrep";; + gz* | zip ) GREP="zgrep";; + * ) GREP="grep";; + esac + log "GREP was set to $GREP" + sudo $GREP -E "${CONFIG[regex]}" $FILE done - OUTPUT_VAR=$( echo -e "$OUTPUT" ) } # END: Work Functions @@ -180,19 +295,23 @@ gather_logs_from_files () { # BEGIN: Work -OUTPUT="" - case ${CONFIG[source]} in - log) gather_logs_from_files OUTPUT;; - journal) gather_logs_from_journal OUTPUT;; + log) COMMAND=gather_logs_from_files;; + journal) COMMAND=gather_logs_from_journal;; esac if [[ ${CONFIG[service_search]} -eq 1 ]]; then - echo "$OUTPUT" | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u -elif [[ ${CONFIG[service_filter]} != "N/A" ]]; then - echo "$OUTPUT" | grep -E "${CONFIG[service_filter]}" + $COMMAND | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u | tr -d ':' + +elif [[ "${CONFIG[service_filter]}" ]]; then + $COMMAND | grep -E "${CONFIG[service_filter]}" + +elif [[ "${CONFIG[output_file]}" ]]; then + log "outputting all searches from user to ${CONFIG[output_file]}" + $COMMAND >${CONFIG[output_file]} + else - echo "$OUTPUT" + $COMMAND fi # END: Work