tristan/bash-logsearch-tool
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added newer script-logging features & implemented them

Browse Source
This commit is contained in:
Tristan Ancelet 2024-05-30 17:44:56 -05:00
parent 10f38a133c
commit 46d1a6a0af
1 changed files with 137 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
log-search Normal file → Executable file
View File

@ -32,6 +32,16 @@ CONFIG[hour_filter]=0
CONFIG[remote_host]="N/A" CONFIG[remote_host]="N/A"
CONFIG[local_file]="N/A" CONFIG[local_file]="N/A"
##########################
# An associative array containing varnames that the user want's to keep track of in log output (specific to each function log is being called from)
#
# [$FUNCNAME]="string of varnames"
#
# It will keep track of vars seperately of each function that log will be used with
##########################
declare -A LOG_WATCH_VARS
declare -A SEARCH_MODES=( declare -A SEARCH_MODES=(
['CLUSTER']='(crmd|stonith-ng|cib|stonith|pengine|lrmd|pacemakerd|corosync|drbd|ethmonitor)\S*\[' ['CLUSTER']='(crmd|stonith-ng|cib|stonith|pengine|lrmd|pacemakerd|corosync|drbd|ethmonitor)\S*\['
['ISSUES']='warn|crit|fail|err|' ['ISSUES']='warn|crit|fail|err|'
@ -42,25 +52,95 @@ FLAG_REGEX='[\-]+\S+'
# END: Variables # END: Variables
log_import () {
declare -f log_watch_vars log_unwatch_vars log
declare -p LOG_WATCH_VARS
}
# BEGIN: Helper Functions # BEGIN: Helper Functions
#####################
# Function: log_watch_vars
# Usage: Provide a list of variable names to add to the watchlist for the calling function
log_watch_vars () {
local -a REQUESTED_WATCH_VARS=( $* )
local CALLING_FUNCTION=${FUNCNAME[1]}
local FUNC_VARS=${LOG_WATCH_VARS[$CALLING_FUNCTION]}
local MATCH_REGEX=${FUNC_VARS//[[:space:]]/|}
log () { for VARNAME in ${REQUESTED_WATCH_VARS[@]}; do
if [[ ${CONFIG[debug]} -eq 1 ]]; then if [[ ! "$VARNAME" =~ ^$MATCH_REGEX$ ]]; then
local MESSAGE=${1:?"$FUNCNAME: No message provided"} FUNC_VARS+=" $VARNAME"
local LEVEL_NUM=${2:-0} else
log "$VARNAME was provided to add to the list, but already existed there"
fi
done
case $LEVEL_NUM in LOG_WATCH_VARS[$CALLING_FUNCTION]=$FUNC_VARS
0) LEVEL=INFO;;
1) LEVEL=WARN;;
2) LEVEL=CRIT;;
*) LEVEL=UNDEF;;
esac
echo "$(date) : $HOSTNAME : $LEVEL : ${FUNCNAME[1]} : $MESSAGE"
fi
} }
#####################
# Function: log_unwatch_vars
# Usage: Provide a list of variable names to remove from the watchlist
log_unwatch_vars () {
## Serialize the variable names provided by user into REGEX filter "(var1|var2|var3|var4|var_n)"
local MATCH_REGEX="(${@// /|})"
local CALLING_FUNCTION=${FUNCNAME[1]}
local FUNC_VARS=${LOG_WATCH_VARS[$CALLING_FUNCTION]}
local -a TEMP_ARRAY
for VARNAME in $FUNC_VARS; do
if [[ ! "$VARNAME" =~ ^$MATCH_REGEX$ ]]; then
TEMP_ARRAY+=( $VARNAME )
fi
done
LOG_WATCH_VARS[$CALLING_FUNCTION]="${TEMP_ARRAY[@]}"
}
###########
# Function: log
# Usage: Use to print out debug statements for the developer (or user) to display a log output
# including variable values & names
log () {
local MESSAGE=${1:?"$FUNCNAME: No message provided"}
local CALLING_FUNCTION=${FUNCNAME[1]}
local FUNC_VARS=( ${LOG_WATCH_VARS[$CALLING_FUNCTION]} )
local LEVEL=${2:-0}
local DATE=$(date)
local VAR_WATCH_STRING="" OUTPUT_MESSAGE=""
if [[ ${CONFIG[debug]} -eq 1 ]]; then
case $LEVEL in
0) LEVEL="INFO";;
1) LEVEL="WARN";;
2) LEVEL="CRIT";;
*) LEVEL="UNDEF";;
esac
local VARNAME VALUE
if [[ ${#FUNC_VARS[@]} -gt 0 ]]; then
for VARNAME in ${FUNC_VARS[@]}; do
local -n VARVALUE=$VARNAME
if [[ $VARVALUE == "" ]]; then
VALUE='N/A'
else
VALUE=$VARVALUE
fi
if [[ $VAR_WATCH_STRING == "" ]]; then
VAR_WATCH_STRING+="$VARNAME=$VALUE "
else
VAR_WATCH_STRING+=": $VARNAME=$VALUE "
fi
done
OUTPUT_MESSAGE="$DATE : $HOSTNAME : $LEVEL : $VAR_WATCH_STRING : $MESSAGE"
else
OUTPUT_MESSAGE="$DATE : $HOSTNAME : $LEVEL : $MESSAGE"
fi
echo -e "$OUTPUT_MESSAGE"
fi
} >&2
usage () { usage () {
cat <<EOF cat <<EOF
@ -108,7 +188,7 @@ if [[ $# -eq 0 ]]; then
exit exit
fi fi
if grep -E ' (-d|--debug) ' <<< "$@" ; then if [[ "$@" =~ ^.*(-d|--debug).*$ ]]; then
CONFIG[debug]=1 CONFIG[debug]=1
log "user set debug mode" log "user set debug mode"
fi fi
@ -124,6 +204,8 @@ while [[ $# -ne 0 ]]; do
if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then
if [[ ${CONFIG[day_filter]} -eq 0 ]]; then if [[ ${CONFIG[day_filter]} -eq 0 ]]; then
CONFIG[hour_filter]=$2 CONFIG[hour_filter]=$2
shift 2
continue
else else
echo "$1 cannot be used with --days" echo "$1 cannot be used with --days"
usage usage
@ -131,10 +213,13 @@ while [[ $# -ne 0 ]]; do
fi fi
fi fi
;; ;;
--days) --days)
if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then
if [[ ${CONFIG[hour_filter]} -eq 0 ]]; then if [[ ${CONFIG[hour_filter]} -eq 0 ]]; then
CONFIG[day_filter]=$2 CONFIG[day_filter]=$2
shift 2
continue
else else
echo "$1 cannot be used with --hours" echo "$1 cannot be used with --hours"
usage usage
@ -142,16 +227,12 @@ while [[ $# -ne 0 ]]; do
fi fi
fi fi
;; ;;
-s | --source ) -s | --source )
if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(log|journal|ilog|file)$ ]]; then if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(log|journal|ilog|file)$ ]]; then
case ${2,,} in CONFIG[source]=${2,,}
log) CONFIG[source]="log";;
journal) CONFIG[source]='journal';;
ilog) CONFIG[source]="ilog";;
file) CONFIG[source]="file";;
esac
log "user set the source to ${CONFIG[source]}" log "user set the source to ${CONFIG[source]}"
shift shift 2
continue continue
else else
echo "$2 is not a valid option for $1" echo "$2 is not a valid option for $1"
@ -161,28 +242,22 @@ while [[ $# -ne 0 ]]; do
;; ;;
-m | --mode) -m | --mode)
if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(issues|cluster|custom)$ ]]; then if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(issues|cluster|custom|all)$ ]]; then
case ${2,,} in case ${2,,} in
issues) issues | cluster | all )
CONFIG[mode]="issues" CONFIG[mode]=$2
CONFIG[regex]=${SEARCH_MODES[${2^^}]}
;;
cluster)
CONFIG[mode]='cluster'
CONFIG[regex]=${SEARCH_MODES[${2^^}]} CONFIG[regex]=${SEARCH_MODES[${2^^}]}
TOTAL=2
;; ;;
custom ) custom )
CONFIG[mode]='custom' CONFIG[mode]='custom'
CONFIG[regex]="$3" CONFIG[regex]="$3"
;; TOTAL=3
all )
CONFIG[mode]='all'
CONFIG[regex]=${SEARCH_MODES[${2^^}]}
;;
esac esac
log "user set search mode and regex to ${CONFIG[mode]}, ${CONFIG[regex]}" log "user set search mode and regex to ${CONFIG[mode]}, ${CONFIG[regex]}"
shift shift $TOTAL
continue continue
else else
echo "$2 is not a valid option for $1" echo "$2 is not a valid option for $1"
usage usage
@ -194,24 +269,30 @@ while [[ $# -ne 0 ]]; do
if [[ $2 != "" ]]; then if [[ $2 != "" ]]; then
log "user set output file to be ${CONFIG[output_file]}" log "user set output file to be ${CONFIG[output_file]}"
CONFIG[output_file]=$2 CONFIG[output_file]=$2
shift shift 2
continue
fi fi
;; ;;
--services) --services)
log "user is searching for service names" log "user is searching for service names"
CONFIG[service_search]=1 CONFIG[service_search]=1
shift
continue
;; ;;
--filename) --filename)
if [[ "$2" != "" ]]; then if [[ "$2" != "" ]]; then
CONFIG[local_file]=$2 CONFIG[local_file]=$2
shift 2
continue
fi fi
;; ;;
-f | --filter-regex) -f | --filter-regex)
if [[ "$2" != "" ]]; then if [[ "$2" != "" ]]; then
CONFIG[service_filter]=$2 CONFIG[service_filter]=$2
log "user provided a search term/regex ${CONFIG[service_filter]}" log "user provided a search term/regex ${CONFIG[service_filter]}"
shift shift 2
continue continue
fi fi
;; ;;
@ -219,6 +300,8 @@ while [[ $# -ne 0 ]]; do
--remote ) --remote )
if [[ $2 != "" ]] && [[ ! $2 =~ ^$FLAG_REGEX$ ]]; then if [[ $2 != "" ]] && [[ ! $2 =~ ^$FLAG_REGEX$ ]]; then
CONFIG[remote_host]=$2 CONFIG[remote_host]=$2
shift 2
continue
fi fi
;; ;;
@ -226,6 +309,8 @@ while [[ $# -ne 0 ]]; do
if [[ "$2" != "" ]] ; then if [[ "$2" != "" ]] ; then
log "user set hostname to $2" log "user set hostname to $2"
CONFIG[hostname]=$2 CONFIG[hostname]=$2
shift 2
continue
fi fi
;; ;;
@ -273,12 +358,14 @@ esac
# BEGIN: Work Functions # BEGIN: Work Functions
import () { import () {
declare -f gather_logs_from_journal gather_logs_from_files gather_logs_from_files_interactive log gather_logs_from_local_file log_import
CONFIG[remote_host]="" declare -f gather_logs_from_journal gather_logs_from_files gather_logs_from_files_interactive log gather_logs_from_local_file
declare -p CONFIG declare -p CONFIG
} }
gather_logs_from_journal () { gather_logs_from_journal () {
log_watch_vars CONFIG[day_filter] CONFIG[hour_filter] CONFIG[regex]
log "About to load logs from journal"
if [[ ${CONFIG[day_filter]} -gt 0 ]]; then if [[ ${CONFIG[day_filter]} -gt 0 ]]; then
sudo journalctl --since "${CONFIG[day_filter]} days ago" --no-pager | grep -E ${CONFIG[regex]} sudo journalctl --since "${CONFIG[day_filter]} days ago" --no-pager | grep -E ${CONFIG[regex]}
@ -286,15 +373,16 @@ gather_logs_from_journal () {
sudo journalctl --since "${CONFIG[hour_filter]} hours ago" --no-pager | grep -E ${CONFIG[regex]} sudo journalctl --since "${CONFIG[hour_filter]} hours ago" --no-pager | grep -E ${CONFIG[regex]}
else else
sudo journalctl --no-pager | grep -E ${CONFIG[regex]} sudo journalctl --no-pager | grep -E "${CONFIG[regex]}"
fi fi
} }
gather_logs_from_files () { gather_logs_from_files () {
log_watch_vars CONFIG[hostname] CONFIG[regex] CONFIG[last_nth_days]
if [[ ! -d /var/log/hosts/${CONFIG[hostname]} ]]; then if [[ ! -d /var/log/hosts/${CONFIG[hostname]} ]]; then
echo "Hosts log directory for ${CONFIG[hostname]} does not exist" echo "Hosts log directory for ${CONFIG[hostname]} does not exist"
exit exit
fi i
log "User is getting logs from the last ${CONFIG[last_nth_days]} files" log "User is getting logs from the last ${CONFIG[last_nth_days]} files"
local -a FILES=( $( sudo ls -1r /var/log/hosts/${CONFIG[hostname]}/*/*/*/messages* ) ) local -a FILES=( $( sudo ls -1r /var/log/hosts/${CONFIG[hostname]}/*/*/*/messages* ) )
@ -305,6 +393,7 @@ for FILE in ${FILES[@]}; do
done done
)" )"
log_watch_vars FILE EXT GREP
local FILE EXT local FILE EXT
for FILE in ${FILES[@]::${CONFIG[last_nth_days]}}; do for FILE in ${FILES[@]::${CONFIG[last_nth_days]}}; do
log "$FILE being checked" log "$FILE being checked"
@ -321,6 +410,8 @@ done
} }
gather_logs_from_files_interactive () { gather_logs_from_files_interactive () {
log_watch_vars CONFIG[hostname] CONFIG[regex]
log "Preparing to obtain logs from messages files"
if [[ ! -d /var/log/hosts/${CONFIG[hostname]} ]]; then if [[ ! -d /var/log/hosts/${CONFIG[hostname]} ]]; then
echo "Hosts log directory for ${CONFIG[hostname]} does not exist" echo "Hosts log directory for ${CONFIG[hostname]} does not exist"
exit exit
@ -355,6 +446,7 @@ gather_logs_from_files_interactive () {
log_watch_vars FILE EXT GREP
local FILE EXT local FILE EXT
for FILE in ${FILE_CHOICES[@]}; do for FILE in ${FILE_CHOICES[@]}; do
log "$FILE being checked" log "$FILE being checked"
@ -372,10 +464,12 @@ gather_logs_from_files_interactive () {
} }
gather_logs_from_local_file () { gather_logs_from_local_file () {
log_watch_vars CONFIG[local_file] CONFIG[regex]
if [[ ! -f ${CONFIG[local_file]} ]]; then if [[ ! -f ${CONFIG[local_file]} ]]; then
echo "$HOSTNAME : ${CONFIG[local_file]} does not exist" echo "$HOSTNAME : ${CONFIG[local_file]} does not exist"
exit exit
fi fi
log_watch_vars FILE EXT GREP
FILE=${CONFIG[local_file]} FILE=${CONFIG[local_file]}
log "$FILE being checked" log "$FILE being checked"
EXT=$( basename $FILE | cut -d '.' -f 2 ) EXT=$( basename $FILE | cut -d '.' -f 2 )
@ -402,7 +496,9 @@ case ${CONFIG[source]} in
file) COMMAND=gather_logs_from_local_file;; file) COMMAND=gather_logs_from_local_file;;
esac esac
log_watch_vars COMMAND
if [[ ${CONFIG[remote_host]} != "" ]]; then if [[ ${CONFIG[remote_host]} != "" ]]; then
log "CONFIG[remote_host] was specified as ${CONFIG[remote_host]}. Going to be running search remotely"
if [[ ${CONFIG[service_search]} -eq 1 ]]; then if [[ ${CONFIG[service_search]} -eq 1 ]]; then
ssh ${CONFIG[remote_host]} "$( import ); $COMMAND" | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u | tr -d ':' ssh ${CONFIG[remote_host]} "$( import ); $COMMAND" | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u | tr -d ':'
@ -430,6 +526,6 @@ else
else else
$COMMAND $COMMAND
fi fi
fi fi | sort
# END: Work # END: Work