diff --git a/log-search b/log-search index eb091a5..d7be7d1 100644 --- a/log-search +++ b/log-search @@ -30,13 +30,16 @@ CONFIG[debug]=0 CONFIG[day_filter]=0 CONFIG[hour_filter]=0 CONFIG[remote_host]="N/A" +CONFIG[local_file]="N/A" declare -A SEARCH_MODES=( - ['CLUSTER']='(crmd|stonith|pengine|lrmd|pacemakerd|corosync|drbd)\[' - ['ISSUES']='warn|crit|fail|err' + ['CLUSTER']='(crmd|stonith-ng|cib|stonith|pengine|lrmd|pacemakerd|corosync|drbd|ethmonitor)\S*\[' + ['ISSUES']='warn|crit|fail|err|' ['ALL']='.' ) +FLAG_REGEX='[\-]+\S+' + # END: Variables @@ -54,7 +57,7 @@ log () { 2) LEVEL=CRIT;; *) LEVEL=UNDEF;; esac - echo "$(date) : $LEVEL : ${FUNCNAME[1]} : $MESSAGE" + echo "$(date) : $HOSTNAME : $LEVEL : ${FUNCNAME[1]} : $MESSAGE" fi } @@ -65,7 +68,9 @@ $( basename $0 ) [-s|--source ] [-m|--mode ] [--services] [-l|--la Sources: log : When used on the log-server for a customer environment (references the hosts messages file /var/log/hosts//YYYY/MM/DD/messages*) + ilog : Log, but it will let you chose which files you will be checking against. (interactive) journal (default) : When used on a node (referenced a nodes journal for finding errors) + file : search through a local file Search Modes cluster : Searching for errors that are cluster related (matches regex: ${SEARCH_MODES[CLUSTER]}) @@ -80,12 +85,14 @@ Flags: -d|--debug : Turns on debugging -o|--output-file : Filename to output to --remote : This will run the gathering code remotely - Log Source: + File Source: + --filename : File that contains logs that you want to use for the file source + Log & iLog Source: -l|--last-nth-days : (only used with log source) Search for logs in the last nth days. -H|--hostname : Hostname to search against Journal Source: - --hours : This will narrow down to within the last X hours - --days : This will narrow down to within the last X days + --hours : This will narrow down to within the last X hours (cannot be used with --days) + --days : This will narrow down to within the last X days (cannnot be used with --hours) EOF } @@ -107,6 +114,10 @@ fi while [[ $# -ne 0 ]]; do log "\$1 is $1" case $1 in + -h | --help) + usage + exit + ;; --hours) if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then if [[ ${CONFIG[day_filter]} -eq 0 ]]; then @@ -130,10 +141,12 @@ while [[ $# -ne 0 ]]; do fi ;; -s | --source ) - if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(log|journal)$ ]]; then + if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(log|journal|ilog|file)$ ]]; then case ${2,,} in log) CONFIG[source]="log";; journal) CONFIG[source]='journal';; + ilog) CONFIG[source]="ilog";; + file) CONFIG[source]="file";; esac log "user set the source to ${CONFIG[source]}" shift @@ -150,11 +163,11 @@ while [[ $# -ne 0 ]]; do case ${2,,} in issues) CONFIG[mode]="issues" - CONFIG[regex]=${SEARCH_MODES[ISSUES]} + CONFIG[regex]=${SEARCH_MODES[${2^^}]} ;; cluster) CONFIG[mode]='cluster' - CONFIG[regex]=${SEARCH_MODES[CLUSTER]} + CONFIG[regex]=${SEARCH_MODES[${2^^}]} ;; custom ) CONFIG[mode]='custom' @@ -162,12 +175,12 @@ while [[ $# -ne 0 ]]; do ;; all ) CONFIG[mode]='all' - CONFIG[regex]="${SEARCH_MODES[ALL]}" + CONFIG[regex]=${SEARCH_MODES[${2^^}]} ;; esac log "user set search mode and regex to ${CONFIG[mode]}, ${CONFIG[regex]}" - shift - continue + shift + continue else echo "$2 is not a valid option for $1" usage @@ -179,6 +192,7 @@ while [[ $# -ne 0 ]]; do if [[ $2 != "" ]]; then log "user set output file to be ${CONFIG[output_file]}" CONFIG[output_file]=$2 + shift fi ;; @@ -186,6 +200,11 @@ while [[ $# -ne 0 ]]; do log "user is searching for service names" CONFIG[service_search]=1 ;; + --filename) + if [[ "$2" != "" ]]; then + CONFIG[local_file]=$2 + fi + ;; -f | --filter-regex) if [[ "$2" != "" ]]; then CONFIG[service_filter]=$2 @@ -195,17 +214,16 @@ while [[ $# -ne 0 ]]; do fi ;; + --remote ) + if [[ $2 != "" ]] && [[ ! $2 =~ ^$FLAG_REGEX$ ]]; then + CONFIG[remote_host]=$2 + fi + ;; + -H | --hostname ) if [[ "$2" != "" ]] ; then - LOG_PATH=/var/log/hosts/$2 - if [[ -d $LOG_PATH ]]; then - log "user set hostname to $2" - CONFIG[hostname]=$2 - else - echo "$LOG_PATH does not exist. Either the host doesn't exist or hasn't logged to this server" - usage - exit - fi + log "user set hostname to $2" + CONFIG[hostname]=$2 fi ;; @@ -229,13 +247,20 @@ for KEY in ${!CONFIG[@]}; do done case ${CONFIG[mode]} in - log) + log | ilog) if [[ ${CONFIG[hostname]} == "" ]]; then echo "A hostname was not provieded (via -H|--hostname), and it is required to do a log search" usage exit fi ;; + file) + if [[ ${CONFIG[local_file]} == "" ]]; then + echo "SOURCE(file) : File not provided" + usage + exit + fi + ;; esac # END: Pre-Work Checks @@ -246,7 +271,7 @@ esac # BEGIN: Work Functions import () { - declare -f gather_logs_from_journal gather_logs_from_files log + declare -f gather_logs_from_journal gather_logs_from_files gather_logs_from_files_interactive log gather_logs_from_local_file CONFIG[remote_host]="" declare -p CONFIG } @@ -264,8 +289,13 @@ gather_logs_from_journal () { } gather_logs_from_files () { + if [[ ! -d /var/log/hosts/${CONFIG[hostname]} ]]; then + echo "Hosts log directory for ${CONFIG[hostname]} does not exist" + exit + fi + log "User is getting logs from the last ${CONFIG[last_nth_days]} files" - local -a FILES=( $( ls -1r /var/log/hosts/${CONFIG[hostname]}/*/*/*/messages* ) ) + local -a FILES=( $( sudo ls -1r /var/log/hosts/${CONFIG[hostname]}/*/*/*/messages* ) ) log " Files Found: $( for FILE in ${FILES[@]}; do @@ -273,9 +303,7 @@ for FILE in ${FILES[@]}; do done )" - local OUTPUT BUFFER EXT - - local FILE + local FILE EXT for FILE in ${FILES[@]::${CONFIG[last_nth_days]}}; do log "$FILE being checked" EXT=$( basename $FILE | cut -d '.' -f 2 ) @@ -290,6 +318,76 @@ done done } +gather_logs_from_files_interactive () { + if [[ ! -d /var/log/hosts/${CONFIG[hostname]} ]]; then + echo "Hosts log directory for ${CONFIG[hostname]} does not exist" + exit + fi + + local -a FILE_CHOICES + local -a FILES=( $( sudo ls -1r /var/log/hosts/${CONFIG[hostname]}/*/*/*/messages* ) ) + select choice in ${FILES[@]} 'continue' quit; do + case $choice in + \?) + echo "Please try again." + ;; + quit) + echo "Exiting now" + exit + ;; + continue) + break + ;; + *) + if [[ "${FILE_CHOICES[@]}" != *$choice* ]]; then + echo "Adding ($choice) to list" + FILE_CHOICES+=( $choice ) + else + echo "That choice has already been picked. Please choose another one" + fi + ;; + + + esac + done >&2 + + + + local FILE EXT + for FILE in ${FILE_CHOICES[@]}; do + log "$FILE being checked" + EXT=$( basename $FILE | cut -d '.' -f 2 ) + log "EXT was found to be $EXT" + case $EXT in + bz* ) GREP="bzgrep";; + gz* | zip ) GREP="zgrep";; + * ) GREP="grep";; + esac + log "GREP was set to $GREP" + sudo $GREP -E "${CONFIG[regex]}" $FILE + done + +} + +gather_logs_from_local_file () { + if [[ ! -f ${CONFIG[local_file]} ]]; then + echo "$HOSTNAME : ${CONFIG[local_file]} does not exist" + exit + fi + FILE=${CONFIG[local_file]} + log "$FILE being checked" + EXT=$( basename $FILE | cut -d '.' -f 2 ) + log "EXT was found to be $EXT" + case $EXT in + bz* ) GREP="bzgrep";; + gz* | zip ) GREP="zgrep";; + * ) GREP="grep";; + esac + log "GREP was set to $GREP" + sudo $GREP -E "${CONFIG[regex]}" $FILE +} + + # END: Work Functions @@ -298,20 +396,38 @@ done case ${CONFIG[source]} in log) COMMAND=gather_logs_from_files;; journal) COMMAND=gather_logs_from_journal;; + ilog) COMMAND=gather_logs_from_files_interactive;; + file) COMMAND=gather_logs_from_local_file;; esac -if [[ ${CONFIG[service_search]} -eq 1 ]]; then - $COMMAND | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u | tr -d ':' +if [[ ${CONFIG[remote_host]} != "" ]]; then + if [[ ${CONFIG[service_search]} -eq 1 ]]; then + ssh ${CONFIG[remote_host]} "$( import ); $COMMAND" | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u | tr -d ':' -elif [[ "${CONFIG[service_filter]}" ]]; then - $COMMAND | grep -E "${CONFIG[service_filter]}" + elif [[ "${CONFIG[service_filter]}" ]]; then + ssh ${CONFIG[remote_host]} "$( import ); $COMMAND" | grep -E "${CONFIG[service_filter]}" -elif [[ "${CONFIG[output_file]}" ]]; then - log "outputting all searches from user to ${CONFIG[output_file]}" - $COMMAND >${CONFIG[output_file]} + elif [[ "${CONFIG[output_file]}" ]]; then + log "outputting all searches from user to ${CONFIG[output_file]}" + ssh ${CONFIG[remote_host]} "$( import ); $COMMAND" >${CONFIG[output_file]} + else + ssh ${CONFIG[remote_host]} "$( import ); $COMMAND" + fi else - $COMMAND + if [[ ${CONFIG[service_search]} -eq 1 ]]; then + $COMMAND | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u | tr -d ':' + + elif [[ "${CONFIG[service_filter]}" ]]; then + $COMMAND | grep -E "${CONFIG[service_filter]}" + + elif [[ "${CONFIG[output_file]}" ]]; then + log "outputting all searches from user to ${CONFIG[output_file]}" + $COMMAND >${CONFIG[output_file]} + + else + $COMMAND + fi fi # END: Work