bash-logsearch-tool/log-search

532 lines
15 KiB
Plaintext
Raw Normal View History

2024-05-14 21:20:59 +00:00
#!/bin/bash
##################################################################################################
# log_search #
##################################################################################################
# This utility was made to make searching for logs much quicker on both a redbox (journal) & on #
# the logserver (via messages files). #
# #
2024-05-14 23:38:57 +00:00
##################################################################################################
# Author # #
########## #
# Email: tristan.ancelet@acumera.com #
# Work #: +1 (337) 965-1855 #
2024-05-14 21:20:59 +00:00
# #
##################################################################################################
# BEGIN: Variables
declare -A CONFIG
2024-05-14 23:38:57 +00:00
CONFIG[source]="journal"
CONFIG[mode]="all"
CONFIG[regex]="."
2024-05-14 21:20:59 +00:00
CONFIG[service_search]=0
CONFIG[last_nth_days]=1
CONFIG[hostname]="N/A"
CONFIG[service_filter]="N/A"
2024-05-14 23:38:57 +00:00
CONFIG[output_file]="N/A"
CONFIG[debug]=0
CONFIG[day_filter]=0
CONFIG[hour_filter]=0
CONFIG[remote_host]="N/A"
2024-05-18 18:13:42 +00:00
CONFIG[local_file]="N/A"
2024-05-14 21:20:59 +00:00
##########################
# An associative array containing varnames that the user want's to keep track of in log output (specific to each function log is being called from)
#
# [$FUNCNAME]="string of varnames"
#
# It will keep track of vars seperately of each function that log will be used with
##########################
declare -A LOG_WATCH_VARS
2024-05-14 21:20:59 +00:00
declare -A SEARCH_MODES=(
2024-05-18 18:13:42 +00:00
['CLUSTER']='(crmd|stonith-ng|cib|stonith|pengine|lrmd|pacemakerd|corosync|drbd|ethmonitor)\S*\['
['ISSUES']='warn|crit|fail|err|'
2024-05-14 23:38:57 +00:00
['ALL']='.'
2024-05-14 21:20:59 +00:00
)
2024-05-18 18:13:42 +00:00
FLAG_REGEX='[\-]+\S+'
2024-05-14 21:20:59 +00:00
# END: Variables
log_import () {
declare -f log_watch_vars log_unwatch_vars log
declare -p LOG_WATCH_VARS
}
2024-05-14 21:20:59 +00:00
# BEGIN: Helper Functions
#####################
# Function: log_watch_vars
# Usage: Provide a list of variable names to add to the watchlist for the calling function
log_watch_vars () {
local -a REQUESTED_WATCH_VARS=( $* )
local CALLING_FUNCTION=${FUNCNAME[1]}
local FUNC_VARS=${LOG_WATCH_VARS[$CALLING_FUNCTION]}
local MATCH_REGEX=${FUNC_VARS//[[:space:]]/|}
for VARNAME in ${REQUESTED_WATCH_VARS[@]}; do
if [[ ! "$VARNAME" =~ ^$MATCH_REGEX$ ]]; then
FUNC_VARS+=" $VARNAME"
else
log "$VARNAME was provided to add to the list, but already existed there"
fi
done
2024-05-14 21:20:59 +00:00
LOG_WATCH_VARS[$CALLING_FUNCTION]=$FUNC_VARS
2024-05-14 23:38:57 +00:00
}
#####################
# Function: log_unwatch_vars
# Usage: Provide a list of variable names to remove from the watchlist
log_unwatch_vars () {
## Serialize the variable names provided by user into REGEX filter "(var1|var2|var3|var4|var_n)"
local MATCH_REGEX="(${@// /|})"
local CALLING_FUNCTION=${FUNCNAME[1]}
local FUNC_VARS=${LOG_WATCH_VARS[$CALLING_FUNCTION]}
local -a TEMP_ARRAY
for VARNAME in $FUNC_VARS; do
if [[ ! "$VARNAME" =~ ^$MATCH_REGEX$ ]]; then
TEMP_ARRAY+=( $VARNAME )
fi
done
LOG_WATCH_VARS[$CALLING_FUNCTION]="${TEMP_ARRAY[@]}"
}
###########
# Function: log
# Usage: Use to print out debug statements for the developer (or user) to display a log output
# including variable values & names
log () {
local MESSAGE=${1:?"$FUNCNAME: No message provided"}
local CALLING_FUNCTION=${FUNCNAME[1]}
local FUNC_VARS=( ${LOG_WATCH_VARS[$CALLING_FUNCTION]} )
local LEVEL=${2:-0}
local DATE=$(date)
local VAR_WATCH_STRING="" OUTPUT_MESSAGE=""
if [[ ${CONFIG[debug]} -eq 1 ]]; then
case $LEVEL in
0) LEVEL="INFO";;
1) LEVEL="WARN";;
2) LEVEL="CRIT";;
*) LEVEL="UNDEF";;
esac
local VARNAME VALUE
if [[ ${#FUNC_VARS[@]} -gt 0 ]]; then
for VARNAME in ${FUNC_VARS[@]}; do
local -n VARVALUE=$VARNAME
if [[ $VARVALUE == "" ]]; then
VALUE='N/A'
else
VALUE=$VARVALUE
fi
if [[ $VAR_WATCH_STRING == "" ]]; then
VAR_WATCH_STRING+="$VARNAME=$VALUE "
else
VAR_WATCH_STRING+=": $VARNAME=$VALUE "
fi
done
OUTPUT_MESSAGE="$DATE : $HOSTNAME : $LEVEL : $VAR_WATCH_STRING : $MESSAGE"
else
OUTPUT_MESSAGE="$DATE : $HOSTNAME : $LEVEL : $MESSAGE"
fi
echo -e "$OUTPUT_MESSAGE"
fi
} >&2
2024-05-14 23:38:57 +00:00
2024-05-14 21:20:59 +00:00
usage () {
cat <<EOF
2024-05-14 23:38:57 +00:00
$( basename $0 ) [-s|--source <source>] [-m|--mode <mode>] [--services] [-l|--last-nth-days <int>] [-H|--hostname <hostname>] [-d|--debug] [-o|--output-file <filename>]
2024-05-14 21:20:59 +00:00
Sources:
Logserver:
log : When used on the log-server for a customer environment (references the hosts messages file /var/log/hosts/<hostname>/YYYY/MM/DD/messages*)
ilog : Log, but it will let you chose which files you will be checking against. (interactive)
Global:
file : search through a local file
journal (default) : When used on a node (referenced a nodes journal for finding errors)
2024-05-14 21:20:59 +00:00
Search Modes
cluster : Searching for errors that are cluster related (matches regex: ${SEARCH_MODES[CLUSTER]})
issues : General issues that report a warning, failure, error, critical, etc (matches regex: ${SEARCH_MODES[ISSUES]})
custom [custom-regex] : Requires you to provide your own regex as the next argument (make sure to escape your string carefully)
all (default) : Just output all logs provided by your source
2024-05-14 21:20:59 +00:00
Flags:
Universal:
--services : Get all the service names that return from the search
-f|--filter-regex <regex> : Will be used to filter only the services you want to
-d|--debug : Turns on debugging
-o|--output-file <filename> : Filename to output to
--remote <hostname> : This will run the gathering code remotely
2024-05-18 18:13:42 +00:00
File Source:
--filename <file> : File that contains logs that you want to use for the file source
2024-05-18 18:13:42 +00:00
Log & iLog Source:
-l|--last-nth-days <int> : (only used with log source) Search for logs in the last nth days.
-H|--hostname <hostname> : Hostname to search against
2024-05-14 23:38:57 +00:00
Journal Source:
--hours : This will narrow down to within the last X hours (cannot be used with --days)
--days : This will narrow down to within the last X days (cannnot be used with --hours)
2024-05-14 21:20:59 +00:00
EOF
}
# END: Helper Functions
# BEGIN: Handle CLI Args
if [[ $# -eq 0 ]]; then
usage
exit
fi
if [[ "$@" =~ ^.*(-d|--debug).*$ ]]; then
2024-05-14 23:38:57 +00:00
CONFIG[debug]=1
log "user set debug mode"
fi
2024-05-14 21:20:59 +00:00
while [[ $# -ne 0 ]]; do
2024-05-14 23:38:57 +00:00
log "\$1 is $1"
2024-05-14 21:20:59 +00:00
case $1 in
2024-05-18 18:13:42 +00:00
-h | --help)
usage
exit
;;
2024-05-14 23:38:57 +00:00
--hours)
if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then
if [[ ${CONFIG[day_filter]} -eq 0 ]]; then
CONFIG[hour_filter]=$2
shift 2
continue
2024-05-14 23:38:57 +00:00
else
echo "$1 cannot be used with --days"
usage
exit
fi
fi
;;
2024-05-14 23:38:57 +00:00
--days)
if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then
if [[ ${CONFIG[hour_filter]} -eq 0 ]]; then
CONFIG[day_filter]=$2
shift 2
continue
2024-05-14 23:38:57 +00:00
else
echo "$1 cannot be used with --hours"
usage
exit
fi
fi
;;
2024-05-14 21:20:59 +00:00
-s | --source )
2024-05-18 18:13:42 +00:00
if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(log|journal|ilog|file)$ ]]; then
CONFIG[source]=${2,,}
2024-05-14 23:38:57 +00:00
log "user set the source to ${CONFIG[source]}"
shift 2
2024-05-14 21:20:59 +00:00
continue
else
echo "$2 is not a valid option for $1"
usage
exit
fi
;;
-m | --mode)
if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(issues|cluster|custom|all)$ ]]; then
2024-05-14 21:20:59 +00:00
case ${2,,} in
issues | cluster | all )
CONFIG[mode]=$2
2024-05-18 18:13:42 +00:00
CONFIG[regex]=${SEARCH_MODES[${2^^}]}
TOTAL=2
2024-05-14 21:20:59 +00:00
;;
2024-05-14 23:38:57 +00:00
custom )
CONFIG[mode]='custom'
CONFIG[regex]="$3"
TOTAL=3
2024-05-14 21:20:59 +00:00
esac
2024-05-14 23:38:57 +00:00
log "user set search mode and regex to ${CONFIG[mode]}, ${CONFIG[regex]}"
shift $TOTAL
2024-05-18 18:13:42 +00:00
continue
2024-05-14 21:20:59 +00:00
else
echo "$2 is not a valid option for $1"
usage
exit
fi
;;
2024-05-14 23:38:57 +00:00
-o | --output-file)
if [[ $2 != "" ]]; then
log "user set output file to be ${CONFIG[output_file]}"
CONFIG[output_file]=$2
shift 2
continue
2024-05-14 23:38:57 +00:00
fi
;;
2024-05-14 21:20:59 +00:00
--services)
2024-05-14 23:38:57 +00:00
log "user is searching for service names"
2024-05-14 21:20:59 +00:00
CONFIG[service_search]=1
shift
continue
2024-05-14 21:20:59 +00:00
;;
2024-05-18 18:13:42 +00:00
--filename)
if [[ "$2" != "" ]]; then
CONFIG[local_file]=$2
shift 2
continue
2024-05-18 18:13:42 +00:00
fi
;;
2024-05-14 23:38:57 +00:00
-f | --filter-regex)
if [[ "$2" != "" ]]; then
CONFIG[service_filter]=$2
log "user provided a search term/regex ${CONFIG[service_filter]}"
shift 2
2024-05-14 23:38:57 +00:00
continue
fi
;;
2024-05-14 21:20:59 +00:00
2024-05-18 18:13:42 +00:00
--remote )
if [[ $2 != "" ]] && [[ ! $2 =~ ^$FLAG_REGEX$ ]]; then
CONFIG[remote_host]=$2
shift 2
continue
2024-05-18 18:13:42 +00:00
fi
;;
2024-05-14 21:20:59 +00:00
-H | --hostname )
if [[ "$2" != "" ]] ; then
2024-05-18 18:13:42 +00:00
log "user set hostname to $2"
CONFIG[hostname]=$2
shift 2
continue
2024-05-14 21:20:59 +00:00
fi
;;
esac
shift
done
# END: Handle CLI Args
# BEGIN: Pre-Work Checks
for KEY in ${!CONFIG[@]}; do
if [[ "${CONFIG[$KEY]}" == "" ]]; then
echo "$KEY was not set"
exit
2024-05-14 23:38:57 +00:00
elif [[ "${CONFIG[$KEY]}" == "N/A" ]]; then
CONFIG[$KEY]=""
2024-05-14 21:20:59 +00:00
fi
done
case ${CONFIG[mode]} in
2024-05-18 18:13:42 +00:00
log | ilog)
2024-05-14 23:38:57 +00:00
if [[ ${CONFIG[hostname]} == "" ]]; then
2024-05-14 21:20:59 +00:00
echo "A hostname was not provieded (via -H|--hostname), and it is required to do a log search"
usage
exit
fi
;;
2024-05-18 18:13:42 +00:00
file)
if [[ ${CONFIG[local_file]} == "" ]]; then
echo "SOURCE(file) : File not provided"
usage
exit
fi
;;
2024-05-14 21:20:59 +00:00
esac
# END: Pre-Work Checks
# BEGIN: Work Functions
2024-05-14 23:38:57 +00:00
import () {
log_import
declare -f gather_logs_from_journal gather_logs_from_files gather_logs_from_files_interactive log gather_logs_from_local_file
2024-05-14 23:38:57 +00:00
declare -p CONFIG
}
2024-05-14 21:20:59 +00:00
gather_logs_from_journal () {
log_watch_vars CONFIG[day_filter] CONFIG[hour_filter] CONFIG[regex]
log "About to load logs from journal"
2024-05-14 23:38:57 +00:00
if [[ ${CONFIG[day_filter]} -gt 0 ]]; then
sudo journalctl --since "${CONFIG[day_filter]} days ago" --no-pager | grep -E ${CONFIG[regex]}
elif [[ ${CONFIG[hour_filter]} -gt 0 ]]; then
sudo journalctl --since "${CONFIG[hour_filter]} hours ago" --no-pager | grep -E ${CONFIG[regex]}
else
sudo journalctl --no-pager | grep -E "${CONFIG[regex]}"
2024-05-14 23:38:57 +00:00
fi
2024-05-14 21:20:59 +00:00
}
gather_logs_from_files () {
log_watch_vars CONFIG[hostname] CONFIG[regex] CONFIG[last_nth_days]
2024-05-18 18:13:42 +00:00
if [[ ! -d /var/log/hosts/${CONFIG[hostname]} ]]; then
echo "Hosts log directory for ${CONFIG[hostname]} does not exist"
exit
i
2024-05-18 18:13:42 +00:00
2024-05-14 23:38:57 +00:00
log "User is getting logs from the last ${CONFIG[last_nth_days]} files"
2024-05-18 18:13:42 +00:00
local -a FILES=( $( sudo ls -1r /var/log/hosts/${CONFIG[hostname]}/*/*/*/messages* ) )
2024-05-14 23:38:57 +00:00
log " Files Found:
$(
for FILE in ${FILES[@]}; do
echo "- $FILE"
done
)"
log_watch_vars FILE EXT GREP
2024-05-18 18:13:42 +00:00
local FILE EXT
2024-05-14 23:38:57 +00:00
for FILE in ${FILES[@]::${CONFIG[last_nth_days]}}; do
log "$FILE being checked"
EXT=$( basename $FILE | cut -d '.' -f 2 )
log "EXT was found to be $EXT"
case $EXT in
bz* ) GREP="bzgrep";;
gz* | zip ) GREP="zgrep";;
* ) GREP="grep";;
esac
log "GREP was set to $GREP"
sudo $GREP -E "${CONFIG[regex]}" $FILE
2024-05-14 21:20:59 +00:00
done
}
2024-05-18 18:13:42 +00:00
gather_logs_from_files_interactive () {
log_watch_vars CONFIG[hostname] CONFIG[regex]
log "Preparing to obtain logs from messages files"
2024-05-18 18:13:42 +00:00
if [[ ! -d /var/log/hosts/${CONFIG[hostname]} ]]; then
echo "Hosts log directory for ${CONFIG[hostname]} does not exist"
exit
fi
local -a FILE_CHOICES
local -a FILES=( $( sudo ls -1r /var/log/hosts/${CONFIG[hostname]}/*/*/*/messages* ) )
select choice in ${FILES[@]} 'continue' quit; do
case $choice in
\?)
echo "Please try again."
;;
quit)
echo "Exiting now"
exit
;;
continue)
break
;;
*)
if [[ "${FILE_CHOICES[@]}" != *$choice* ]]; then
echo "Adding ($choice) to list"
FILE_CHOICES+=( $choice )
else
echo "That choice has already been picked. Please choose another one"
fi
;;
esac
done >&2
log_watch_vars FILE EXT GREP
2024-05-18 18:13:42 +00:00
local FILE EXT
for FILE in ${FILE_CHOICES[@]}; do
log "$FILE being checked"
EXT=$( basename $FILE | cut -d '.' -f 2 )
log "EXT was found to be $EXT"
case $EXT in
bz* ) GREP="bzgrep";;
gz* | zip ) GREP="zgrep";;
* ) GREP="grep";;
esac
log "GREP was set to $GREP"
sudo $GREP -E "${CONFIG[regex]}" $FILE
done
}
gather_logs_from_local_file () {
log_watch_vars CONFIG[local_file] CONFIG[regex]
2024-05-18 18:13:42 +00:00
if [[ ! -f ${CONFIG[local_file]} ]]; then
echo "$HOSTNAME : ${CONFIG[local_file]} does not exist"
exit
fi
log_watch_vars FILE EXT GREP
2024-05-18 18:13:42 +00:00
FILE=${CONFIG[local_file]}
log "$FILE being checked"
EXT=$( basename $FILE | cut -d '.' -f 2 )
log "EXT was found to be $EXT"
case $EXT in
bz* ) GREP="bzgrep";;
gz* | zip ) GREP="zgrep";;
* ) GREP="grep";;
esac
log "GREP was set to $GREP"
sudo $GREP -E "${CONFIG[regex]}" $FILE
}
2024-05-14 21:20:59 +00:00
# END: Work Functions
# BEGIN: Work
case ${CONFIG[source]} in
2024-05-14 23:38:57 +00:00
log) COMMAND=gather_logs_from_files;;
journal) COMMAND=gather_logs_from_journal;;
2024-05-18 18:13:42 +00:00
ilog) COMMAND=gather_logs_from_files_interactive;;
file) COMMAND=gather_logs_from_local_file;;
2024-05-14 21:20:59 +00:00
esac
log_watch_vars COMMAND
2024-05-18 18:13:42 +00:00
if [[ ${CONFIG[remote_host]} != "" ]]; then
log "CONFIG[remote_host] was specified as ${CONFIG[remote_host]}. Going to be running search remotely"
2024-05-18 18:13:42 +00:00
if [[ ${CONFIG[service_search]} -eq 1 ]]; then
ssh ${CONFIG[remote_host]} "$( import ); $COMMAND" | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u | tr -d ':'
2024-05-14 23:38:57 +00:00
2024-05-18 18:13:42 +00:00
elif [[ "${CONFIG[service_filter]}" ]]; then
ssh ${CONFIG[remote_host]} "$( import ); $COMMAND" | grep -E "${CONFIG[service_filter]}"
2024-05-14 23:38:57 +00:00
2024-05-18 18:13:42 +00:00
elif [[ "${CONFIG[output_file]}" ]]; then
log "outputting all searches from user to ${CONFIG[output_file]}"
ssh ${CONFIG[remote_host]} "$( import ); $COMMAND" >${CONFIG[output_file]}
2024-05-14 23:38:57 +00:00
2024-05-18 18:13:42 +00:00
else
ssh ${CONFIG[remote_host]} "$( import ); $COMMAND"
fi
2024-05-14 21:20:59 +00:00
else
2024-05-18 18:13:42 +00:00
if [[ ${CONFIG[service_search]} -eq 1 ]]; then
$COMMAND | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u | tr -d ':'
elif [[ "${CONFIG[service_filter]}" ]]; then
$COMMAND | grep -E "${CONFIG[service_filter]}"
elif [[ "${CONFIG[output_file]}" ]]; then
log "outputting all searches from user to ${CONFIG[output_file]}"
$COMMAND >${CONFIG[output_file]}
else
$COMMAND
fi
fi | sort
2024-05-14 21:20:59 +00:00
# END: Work