bash-logsearch-tool/log-search

#!/bin/bash

##################################################################################################
# log_search                                                                                     #
##################################################################################################
# This utility was made to make searching for logs much quicker on both a redbox (journal) & on  #
# the logserver (via messages files).                                                            #
#                                                                                                #
##################################################################################################
# Author #                                                                                       #
##########                                                                                       #
#  Email: tristan.ancelet@acumera.com                                                            #
#  Work #: +1 (337) 965-1855                                                                     #
#                                                                                                #
##################################################################################################


# BEGIN: Variables

declare -A CONFIG
CONFIG[source]="journal"
CONFIG[mode]="all"
CONFIG[regex]="."
CONFIG[service_search]=0
CONFIG[last_nth_days]=1
CONFIG[hostname]="N/A"
CONFIG[service_filter]="N/A"
CONFIG[output_file]="N/A"
CONFIG[debug]=0
CONFIG[day_filter]=0
CONFIG[hour_filter]=0
CONFIG[remote_host]="N/A"
CONFIG[local_file]="N/A"

##########################
# An associative array containing varnames that the user want's to keep track of in log output (specific to each function log is being called from)
#
# [$FUNCNAME]="string of varnames"
#
# It will keep track of vars seperately of each function that log will be used with
##########################
declare -A LOG_WATCH_VARS


declare -A SEARCH_MODES=(
  ['CLUSTER']='(crmd|stonith-ng|cib|stonith|pengine|lrmd|pacemakerd|corosync|drbd|ethmonitor)\S*\['
  ['ISSUES']='warn|crit|fail|err|'
  ['ALL']='.'
)

FLAG_REGEX='[\-]+\S+'

# END: Variables

log_import () {
        declare -f log_watch_vars log_unwatch_vars log
        declare -p LOG_WATCH_VARS
}


# BEGIN: Helper Functions
#####################
# Function: log_watch_vars
# Usage: Provide a list of variable names to add to the watchlist for the calling function
log_watch_vars () {
	local -a REQUESTED_WATCH_VARS=( $* )
	local CALLING_FUNCTION=${FUNCNAME[1]}
	local FUNC_VARS=${LOG_WATCH_VARS[$CALLING_FUNCTION]}
	local MATCH_REGEX=${FUNC_VARS//[[:space:]]/|}

	for VARNAME in ${REQUESTED_WATCH_VARS[@]}; do
		if [[ ! "$VARNAME" =~ ^$MATCH_REGEX$ ]]; then
			FUNC_VARS+=" $VARNAME"
		else
			log "$VARNAME was provided to add to the list, but already existed there"
		fi
	done

	LOG_WATCH_VARS[$CALLING_FUNCTION]=$FUNC_VARS
}

#####################
# Function: log_unwatch_vars
# Usage: Provide a list of variable names to remove from the watchlist
log_unwatch_vars () {
	## Serialize the variable names provided by user into REGEX filter "(var1|var2|var3|var4|var_n)"
	local MATCH_REGEX="(${@// /|})"
	local CALLING_FUNCTION=${FUNCNAME[1]}
	local FUNC_VARS=${LOG_WATCH_VARS[$CALLING_FUNCTION]}
	local -a TEMP_ARRAY

	for VARNAME in $FUNC_VARS; do
		if [[ ! "$VARNAME" =~ ^$MATCH_REGEX$ ]]; then
			TEMP_ARRAY+=( $VARNAME )
		fi
	done
	LOG_WATCH_VARS[$CALLING_FUNCTION]="${TEMP_ARRAY[@]}"
}

###########
# Function: log
# Usage: Use to print out debug statements for the developer (or user) to display a log output 
#        including variable values & names
log () {
        local MESSAGE=${1:?"$FUNCNAME: No message provided"}
        local CALLING_FUNCTION=${FUNCNAME[1]}
        local FUNC_VARS=( ${LOG_WATCH_VARS[$CALLING_FUNCTION]} )
        local LEVEL=${2:-0}
        local DATE=$(date)
        local VAR_WATCH_STRING="" OUTPUT_MESSAGE=""

        if [[ ${CONFIG[debug]} -eq 1 ]]; then
          case $LEVEL in
            0) LEVEL="INFO";;
            1) LEVEL="WARN";;
            2) LEVEL="CRIT";;
            *) LEVEL="UNDEF";;
          esac

          local VARNAME VALUE
          if [[ ${#FUNC_VARS[@]} -gt 0 ]]; then
                  for VARNAME in ${FUNC_VARS[@]}; do
                          local -n VARVALUE=$VARNAME
                          if [[ $VARVALUE == "" ]]; then
                                  VALUE='N/A'
                          else
                                  VALUE=$VARVALUE
                          fi
                          if [[ $VAR_WATCH_STRING == "" ]]; then
                                  VAR_WATCH_STRING+="$VARNAME=$VALUE "
                          else
                                  VAR_WATCH_STRING+=": $VARNAME=$VALUE "
                          fi
                  done

                  OUTPUT_MESSAGE="$DATE : $HOSTNAME : $LEVEL : $VAR_WATCH_STRING : $MESSAGE"
          else
                  OUTPUT_MESSAGE="$DATE : $HOSTNAME : $LEVEL : $MESSAGE"
          fi

          echo -e "$OUTPUT_MESSAGE"
        fi
} >&2

usage () {
  cat <<EOF
$( basename $0 ) [-s|--source <source>] [-m|--mode <mode>] [--services] [-l|--last-nth-days <int>] [-H|--hostname <hostname>] [-d|--debug] [-o|--output-file <filename>]

Sources:
  Logserver:
    log                            : When used on the log-server for a customer environment (references the hosts messages file /var/log/hosts/<hostname>/YYYY/MM/DD/messages*)
    ilog                           : Log, but it will let you chose which files you will be checking against. (interactive)
  Global:
    file                           : search through a local file
    journal (default)              : When used on a node (referenced a nodes journal for finding errors)

Search Modes
  cluster                          : Searching for errors that are cluster related (matches regex: ${SEARCH_MODES[CLUSTER]})
  issues                           : General issues that report a warning, failure, error,  critical, etc (matches regex: ${SEARCH_MODES[ISSUES]})
  custom [custom-regex]            : Requires you to provide your own regex as the next argument (make sure to escape your string carefully)
  all (default)                    : Just output all logs provided by your source

Flags:
  Universal:
    --services                     : Get all the service names that return from the search
    -f|--filter-regex <regex>      : Will be used to filter only the services you want to 
    -d|--debug                     : Turns on debugging
    -o|--output-file <filename>    : Filename to output to
    --remote <hostname>            : This will run the gathering code remotely 
  File Source:
    --filename <file>              : File that contains logs that you want to use for the file source 
  Log & iLog Source:
    -l|--last-nth-days <int>       : (only used with log source) Search for logs in the last nth days.
    -H|--hostname <hostname>       : Hostname to search against
  Journal Source:
    --hours                        : This will narrow down to within the last X hours (cannot be used with --days)
    --days                         : This will narrow down to within the last X days (cannnot be used with --hours)
EOF
}

# END: Helper Functions


# BEGIN: Handle CLI Args

if [[ $# -eq 0 ]]; then
	usage
	exit
fi

if [[ "$@" =~ ^.*(-d|--debug).*$ ]]; then
	CONFIG[debug]=1
	log "user set debug mode"
fi

while [[ $# -ne 0 ]]; do
  log "\$1 is $1"
  case $1 in
	  -h | --help)
		  usage
		  exit
		  ;;
	  --hours)
		  if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then
			  if [[ ${CONFIG[day_filter]} -eq 0 ]]; then
				  CONFIG[hour_filter]=$2
				  shift 2
				  continue
			  else
				  echo "$1 cannot be used with --days"
				  usage 
				  exit
			  fi
		  fi
		  ;;

	  --days)
		  if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then
			  if [[ ${CONFIG[hour_filter]} -eq 0 ]]; then
				  CONFIG[day_filter]=$2
				  shift 2
				  continue
			  else
				  echo "$1 cannot be used with --hours"
				  usage 
				  exit
			  fi
		  fi
		  ;;

	  -s | --source )
		  if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(log|journal|ilog|file)$ ]]; then
			  CONFIG[source]=${2,,}
			  log "user set the source to ${CONFIG[source]}"
			  shift 2
			  continue
		  else
			  echo "$2 is not a valid option for $1"
			  usage
			  exit
		  fi
	  ;;

	  -m | --mode)
		  if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(issues|cluster|custom|all)$ ]]; then
		    case ${2,,} in
		      issues | cluster | all )
			      CONFIG[mode]=$2
			      CONFIG[regex]=${SEARCH_MODES[${2^^}]}
			      TOTAL=2
			      ;;
		      custom )
			      CONFIG[mode]='custom'
			      CONFIG[regex]="$3"
			      TOTAL=3
	  	    esac
		    log "user set search mode and regex to ${CONFIG[mode]}, ${CONFIG[regex]}"
		    shift $TOTAL
		    continue

		  else
			  echo "$2 is not a valid option for $1"
			  usage
			  exit
		  fi
	  ;;

	  -o | --output-file)
		  if [[ $2 != "" ]]; then
			  log "user set output file to be ${CONFIG[output_file]}"
			  CONFIG[output_file]=$2
			  shift 2
			  continue
		  fi
	  ;;

	  --services)
		  log "user is searching for service names"
		  CONFIG[service_search]=1
		  shift
		  continue
	  ;;

	  --filename)
		  if [[ "$2" != "" ]]; then
			  CONFIG[local_file]=$2
			  shift 2
			  continue
		  fi
          ;;
	  -f | --filter-regex)
		  if [[ "$2" != "" ]]; then
			  CONFIG[service_filter]=$2
			  log "user provided a search term/regex ${CONFIG[service_filter]}"
			  shift 2
			  continue
		  fi
	  ;;

	  --remote )
		  if [[ $2 != "" ]] && [[ ! $2 =~ ^$FLAG_REGEX$ ]]; then
			  CONFIG[remote_host]=$2
			  shift 2
			  continue
		  fi
	  ;;

	  -H | --hostname )
		  if [[ "$2" != "" ]] ; then
			  log "user set hostname to $2"
			  CONFIG[hostname]=$2
			  shift 2
			  continue
		  fi
	  ;;

  esac
  shift
done

# END: Handle CLI Args



# BEGIN: Pre-Work Checks

for KEY in ${!CONFIG[@]}; do
	if [[ "${CONFIG[$KEY]}" == "" ]]; then
		echo "$KEY was not set"
		exit
	elif [[ "${CONFIG[$KEY]}" == "N/A" ]]; then
		CONFIG[$KEY]=""
	fi
done

case ${CONFIG[mode]} in
	log | ilog)
		if [[ ${CONFIG[hostname]} == "" ]]; then
			echo "A hostname was not provieded (via -H|--hostname), and it is required to do a log search"
			usage
			exit
		fi
		;;
	file)
		if [[ ${CONFIG[local_file]} == "" ]]; then
			echo "SOURCE(file) : File not provided"
			usage
			exit
		fi
		;;
esac

# END: Pre-Work Checks




# BEGIN: Work Functions

import () {
	log_import
	declare -f gather_logs_from_journal gather_logs_from_files gather_logs_from_files_interactive log gather_logs_from_local_file 
	declare -p CONFIG
}

gather_logs_from_journal () {
	log_watch_vars CONFIG[day_filter] CONFIG[hour_filter] CONFIG[regex]
	log "About to load logs from journal"
	if [[ ${CONFIG[day_filter]} -gt 0 ]]; then
		sudo journalctl --since "${CONFIG[day_filter]} days ago"  --no-pager | grep -E ${CONFIG[regex]}

	elif [[ ${CONFIG[hour_filter]} -gt 0 ]]; then
		sudo journalctl --since "${CONFIG[hour_filter]} hours ago"  --no-pager | grep -E ${CONFIG[regex]}

	else
		sudo journalctl --no-pager | grep -E "${CONFIG[regex]}"
	fi
}

gather_logs_from_files () {
	log_watch_vars CONFIG[hostname] CONFIG[regex] CONFIG[last_nth_days]
	if [[ ! -d /var/log/hosts/${CONFIG[hostname]} ]]; then
		echo "Hosts log directory for ${CONFIG[hostname]} does not exist"
		exit
	i

	log "User is getting logs from the last ${CONFIG[last_nth_days]} files"
	local -a FILES=( $( sudo ls -1r /var/log/hosts/${CONFIG[hostname]}/*/*/*/messages* ) )
	log " Files Found:
$( 
for FILE in ${FILES[@]}; do
	echo "- $FILE"
done
)"

	log_watch_vars FILE EXT GREP
	local FILE EXT
	for FILE in ${FILES[@]::${CONFIG[last_nth_days]}}; do 
		log "$FILE being checked"
		EXT=$( basename $FILE | cut -d '.' -f 2 )
		log "EXT was found to be $EXT"
		case $EXT in
			bz*       ) GREP="bzgrep";;
			gz* | zip ) GREP="zgrep";;
			*         ) GREP="grep";;
		esac
		log "GREP was set to $GREP"
		sudo $GREP -E "${CONFIG[regex]}" $FILE
	done
}

gather_logs_from_files_interactive () {
	log_watch_vars CONFIG[hostname] CONFIG[regex]
	log "Preparing to obtain logs from messages files"
	if [[ ! -d /var/log/hosts/${CONFIG[hostname]} ]]; then
		echo "Hosts log directory for ${CONFIG[hostname]} does not exist"
		exit
	fi

	local -a FILE_CHOICES
	local -a FILES=( $( sudo ls -1r /var/log/hosts/${CONFIG[hostname]}/*/*/*/messages* ) )
	select choice in ${FILES[@]} 'continue'  quit; do
		case $choice in
			\?)
				echo "Please try again."
				;;
			quit)
				echo "Exiting now"
				exit
				;;
			continue)
				break
				;;
			*)
				if [[ "${FILE_CHOICES[@]}" != *$choice* ]]; then
					echo "Adding ($choice) to list"
					FILE_CHOICES+=( $choice )
				else
					echo "That choice has already been picked. Please choose another one"
				fi
				;;


		esac
	done >&2



	log_watch_vars FILE EXT GREP
	local FILE EXT
	for FILE in ${FILE_CHOICES[@]}; do 
		log "$FILE being checked"
		EXT=$( basename $FILE | cut -d '.' -f 2 )
		log "EXT was found to be $EXT"
		case $EXT in
			bz*       ) GREP="bzgrep";;
			gz* | zip ) GREP="zgrep";;
			*         ) GREP="grep";;
		esac
		log "GREP was set to $GREP"
		sudo $GREP -E "${CONFIG[regex]}" $FILE
	done

}

gather_logs_from_local_file () {
	log_watch_vars CONFIG[local_file] CONFIG[regex]
	if [[ ! -f ${CONFIG[local_file]} ]]; then
		echo "$HOSTNAME : ${CONFIG[local_file]} does not exist"
		exit
	fi
	log_watch_vars FILE EXT GREP
	FILE=${CONFIG[local_file]}
	log "$FILE being checked"
	EXT=$( basename $FILE | cut -d '.' -f 2 )
	log "EXT was found to be $EXT"
	case $EXT in
		bz*       ) GREP="bzgrep";;
		gz* | zip ) GREP="zgrep";;
		*         ) GREP="grep";;
	esac
	log "GREP was set to $GREP"
	sudo $GREP -E "${CONFIG[regex]}" $FILE
}


# END: Work Functions


# BEGIN: Work

case ${CONFIG[source]} in
	log) COMMAND=gather_logs_from_files;;
	journal) COMMAND=gather_logs_from_journal;;
	ilog) COMMAND=gather_logs_from_files_interactive;;
	file) COMMAND=gather_logs_from_local_file;;
esac

log_watch_vars COMMAND 
if [[ ${CONFIG[remote_host]} != "" ]]; then
	log "CONFIG[remote_host] was specified as ${CONFIG[remote_host]}. Going to be running search remotely"
	if [[ ${CONFIG[service_search]} -eq 1 ]]; then
		ssh ${CONFIG[remote_host]} "$( import ); $COMMAND" | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u | tr -d ':'

	elif [[ "${CONFIG[service_filter]}" ]]; then
		ssh ${CONFIG[remote_host]} "$( import ); $COMMAND" | grep -E "${CONFIG[service_filter]}"

	elif [[ "${CONFIG[output_file]}" ]]; then
		log "outputting all searches from user to ${CONFIG[output_file]}"
		ssh ${CONFIG[remote_host]} "$( import ); $COMMAND" >${CONFIG[output_file]}

	else
		ssh ${CONFIG[remote_host]} "$( import ); $COMMAND"
	fi
else
	if [[ ${CONFIG[service_search]} -eq 1 ]]; then
		$COMMAND | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u | tr -d ':'

	elif [[ "${CONFIG[service_filter]}" ]]; then
		$COMMAND | grep -E "${CONFIG[service_filter]}"

	elif [[ "${CONFIG[output_file]}" ]]; then
		log "outputting all searches from user to ${CONFIG[output_file]}"
		$COMMAND >${CONFIG[output_file]}

	else
		$COMMAND
	fi
fi | sort 

# END: Work