Added newer script-logging features & implemented them
This commit is contained in:
Tristan Ancelet
parent
10f38a133c
commit
46d1a6a0af
178
log-search
Normal file → Executable file
178
log-search
Normal file → Executable file
@ -32,6 +32,16 @@ CONFIG[hour_filter]=0
|
||||
CONFIG[remote_host]="N/A"
|
||||
CONFIG[local_file]="N/A"
|
||||
|
||||
##########################
|
||||
# An associative array containing varnames that the user want's to keep track of in log output (specific to each function log is being called from)
|
||||
#
|
||||
# [$FUNCNAME]="string of varnames"
|
||||
#
|
||||
# It will keep track of vars seperately of each function that log will be used with
|
||||
##########################
|
||||
declare -A LOG_WATCH_VARS
|
||||
|
||||
|
||||
declare -A SEARCH_MODES=(
|
||||
['CLUSTER']='(crmd|stonith-ng|cib|stonith|pengine|lrmd|pacemakerd|corosync|drbd|ethmonitor)\S*\['
|
||||
['ISSUES']='warn|crit|fail|err|'
|
||||
@ -42,25 +52,95 @@ FLAG_REGEX='[\-]+\S+'
|
||||
|
||||
# END: Variables
|
||||
|
||||
log_import () {
|
||||
declare -f log_watch_vars log_unwatch_vars log
|
||||
declare -p LOG_WATCH_VARS
|
||||
}
|
||||
|
||||
|
||||
# BEGIN: Helper Functions
|
||||
#####################
|
||||
# Function: log_watch_vars
|
||||
# Usage: Provide a list of variable names to add to the watchlist for the calling function
|
||||
log_watch_vars () {
|
||||
local -a REQUESTED_WATCH_VARS=( $* )
|
||||
local CALLING_FUNCTION=${FUNCNAME[1]}
|
||||
local FUNC_VARS=${LOG_WATCH_VARS[$CALLING_FUNCTION]}
|
||||
local MATCH_REGEX=${FUNC_VARS//[[:space:]]/|}
|
||||
|
||||
log () {
|
||||
if [[ ${CONFIG[debug]} -eq 1 ]]; then
|
||||
local MESSAGE=${1:?"$FUNCNAME: No message provided"}
|
||||
local LEVEL_NUM=${2:-0}
|
||||
for VARNAME in ${REQUESTED_WATCH_VARS[@]}; do
|
||||
if [[ ! "$VARNAME" =~ ^$MATCH_REGEX$ ]]; then
|
||||
FUNC_VARS+=" $VARNAME"
|
||||
else
|
||||
log "$VARNAME was provided to add to the list, but already existed there"
|
||||
fi
|
||||
done
|
||||
|
||||
case $LEVEL_NUM in
|
||||
0) LEVEL=INFO;;
|
||||
1) LEVEL=WARN;;
|
||||
2) LEVEL=CRIT;;
|
||||
*) LEVEL=UNDEF;;
|
||||
esac
|
||||
echo "$(date) : $HOSTNAME : $LEVEL : ${FUNCNAME[1]} : $MESSAGE"
|
||||
fi
|
||||
LOG_WATCH_VARS[$CALLING_FUNCTION]=$FUNC_VARS
|
||||
}
|
||||
|
||||
#####################
|
||||
# Function: log_unwatch_vars
|
||||
# Usage: Provide a list of variable names to remove from the watchlist
|
||||
log_unwatch_vars () {
|
||||
## Serialize the variable names provided by user into REGEX filter "(var1|var2|var3|var4|var_n)"
|
||||
local MATCH_REGEX="(${@// /|})"
|
||||
local CALLING_FUNCTION=${FUNCNAME[1]}
|
||||
local FUNC_VARS=${LOG_WATCH_VARS[$CALLING_FUNCTION]}
|
||||
local -a TEMP_ARRAY
|
||||
|
||||
for VARNAME in $FUNC_VARS; do
|
||||
if [[ ! "$VARNAME" =~ ^$MATCH_REGEX$ ]]; then
|
||||
TEMP_ARRAY+=( $VARNAME )
|
||||
fi
|
||||
done
|
||||
LOG_WATCH_VARS[$CALLING_FUNCTION]="${TEMP_ARRAY[@]}"
|
||||
}
|
||||
|
||||
###########
|
||||
# Function: log
|
||||
# Usage: Use to print out debug statements for the developer (or user) to display a log output
|
||||
# including variable values & names
|
||||
log () {
|
||||
local MESSAGE=${1:?"$FUNCNAME: No message provided"}
|
||||
local CALLING_FUNCTION=${FUNCNAME[1]}
|
||||
local FUNC_VARS=( ${LOG_WATCH_VARS[$CALLING_FUNCTION]} )
|
||||
local LEVEL=${2:-0}
|
||||
local DATE=$(date)
|
||||
local VAR_WATCH_STRING="" OUTPUT_MESSAGE=""
|
||||
|
||||
if [[ ${CONFIG[debug]} -eq 1 ]]; then
|
||||
case $LEVEL in
|
||||
0) LEVEL="INFO";;
|
||||
1) LEVEL="WARN";;
|
||||
2) LEVEL="CRIT";;
|
||||
*) LEVEL="UNDEF";;
|
||||
esac
|
||||
|
||||
local VARNAME VALUE
|
||||
if [[ ${#FUNC_VARS[@]} -gt 0 ]]; then
|
||||
for VARNAME in ${FUNC_VARS[@]}; do
|
||||
local -n VARVALUE=$VARNAME
|
||||
if [[ $VARVALUE == "" ]]; then
|
||||
VALUE='N/A'
|
||||
else
|
||||
VALUE=$VARVALUE
|
||||
fi
|
||||
if [[ $VAR_WATCH_STRING == "" ]]; then
|
||||
VAR_WATCH_STRING+="$VARNAME=$VALUE "
|
||||
else
|
||||
VAR_WATCH_STRING+=": $VARNAME=$VALUE "
|
||||
fi
|
||||
done
|
||||
|
||||
OUTPUT_MESSAGE="$DATE : $HOSTNAME : $LEVEL : $VAR_WATCH_STRING : $MESSAGE"
|
||||
else
|
||||
OUTPUT_MESSAGE="$DATE : $HOSTNAME : $LEVEL : $MESSAGE"
|
||||
fi
|
||||
|
||||
echo -e "$OUTPUT_MESSAGE"
|
||||
fi
|
||||
} >&2
|
||||
|
||||
usage () {
|
||||
cat <<EOF
|
||||
@ -108,7 +188,7 @@ if [[ $# -eq 0 ]]; then
|
||||
exit
|
||||
fi
|
||||
|
||||
if grep -E ' (-d|--debug) ' <<< "$@" ; then
|
||||
if [[ "$@" =~ ^.*(-d|--debug).*$ ]]; then
|
||||
CONFIG[debug]=1
|
||||
log "user set debug mode"
|
||||
fi
|
||||
@ -124,6 +204,8 @@ while [[ $# -ne 0 ]]; do
|
||||
if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then
|
||||
if [[ ${CONFIG[day_filter]} -eq 0 ]]; then
|
||||
CONFIG[hour_filter]=$2
|
||||
shift 2
|
||||
continue
|
||||
else
|
||||
echo "$1 cannot be used with --days"
|
||||
usage
|
||||
@ -131,10 +213,13 @@ while [[ $# -ne 0 ]]; do
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
--days)
|
||||
if [[ "$2" != "" ]] & [[ $2 -gt 0 ]]; then
|
||||
if [[ ${CONFIG[hour_filter]} -eq 0 ]]; then
|
||||
CONFIG[day_filter]=$2
|
||||
shift 2
|
||||
continue
|
||||
else
|
||||
echo "$1 cannot be used with --hours"
|
||||
usage
|
||||
@ -142,16 +227,12 @@ while [[ $# -ne 0 ]]; do
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
-s | --source )
|
||||
if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(log|journal|ilog|file)$ ]]; then
|
||||
case ${2,,} in
|
||||
log) CONFIG[source]="log";;
|
||||
journal) CONFIG[source]='journal';;
|
||||
ilog) CONFIG[source]="ilog";;
|
||||
file) CONFIG[source]="file";;
|
||||
esac
|
||||
CONFIG[source]=${2,,}
|
||||
log "user set the source to ${CONFIG[source]}"
|
||||
shift
|
||||
shift 2
|
||||
continue
|
||||
else
|
||||
echo "$2 is not a valid option for $1"
|
||||
@ -161,28 +242,22 @@ while [[ $# -ne 0 ]]; do
|
||||
;;
|
||||
|
||||
-m | --mode)
|
||||
if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(issues|cluster|custom)$ ]]; then
|
||||
if [[ "$2" != "" ]] && [[ "${2,,}" =~ ^(issues|cluster|custom|all)$ ]]; then
|
||||
case ${2,,} in
|
||||
issues)
|
||||
CONFIG[mode]="issues"
|
||||
CONFIG[regex]=${SEARCH_MODES[${2^^}]}
|
||||
;;
|
||||
cluster)
|
||||
CONFIG[mode]='cluster'
|
||||
issues | cluster | all )
|
||||
CONFIG[mode]=$2
|
||||
CONFIG[regex]=${SEARCH_MODES[${2^^}]}
|
||||
TOTAL=2
|
||||
;;
|
||||
custom )
|
||||
CONFIG[mode]='custom'
|
||||
CONFIG[regex]="$3"
|
||||
;;
|
||||
all )
|
||||
CONFIG[mode]='all'
|
||||
CONFIG[regex]=${SEARCH_MODES[${2^^}]}
|
||||
;;
|
||||
TOTAL=3
|
||||
esac
|
||||
log "user set search mode and regex to ${CONFIG[mode]}, ${CONFIG[regex]}"
|
||||
shift
|
||||
shift $TOTAL
|
||||
continue
|
||||
|
||||
else
|
||||
echo "$2 is not a valid option for $1"
|
||||
usage
|
||||
@ -194,24 +269,30 @@ while [[ $# -ne 0 ]]; do
|
||||
if [[ $2 != "" ]]; then
|
||||
log "user set output file to be ${CONFIG[output_file]}"
|
||||
CONFIG[output_file]=$2
|
||||
shift
|
||||
shift 2
|
||||
continue
|
||||
fi
|
||||
;;
|
||||
|
||||
--services)
|
||||
log "user is searching for service names"
|
||||
CONFIG[service_search]=1
|
||||
shift
|
||||
continue
|
||||
;;
|
||||
|
||||
--filename)
|
||||
if [[ "$2" != "" ]]; then
|
||||
CONFIG[local_file]=$2
|
||||
CONFIG[local_file]=$2
|
||||
shift 2
|
||||
continue
|
||||
fi
|
||||
;;
|
||||
-f | --filter-regex)
|
||||
if [[ "$2" != "" ]]; then
|
||||
CONFIG[service_filter]=$2
|
||||
log "user provided a search term/regex ${CONFIG[service_filter]}"
|
||||
shift
|
||||
shift 2
|
||||
continue
|
||||
fi
|
||||
;;
|
||||
@ -219,6 +300,8 @@ while [[ $# -ne 0 ]]; do
|
||||
--remote )
|
||||
if [[ $2 != "" ]] && [[ ! $2 =~ ^$FLAG_REGEX$ ]]; then
|
||||
CONFIG[remote_host]=$2
|
||||
shift 2
|
||||
continue
|
||||
fi
|
||||
;;
|
||||
|
||||
@ -226,6 +309,8 @@ while [[ $# -ne 0 ]]; do
|
||||
if [[ "$2" != "" ]] ; then
|
||||
log "user set hostname to $2"
|
||||
CONFIG[hostname]=$2
|
||||
shift 2
|
||||
continue
|
||||
fi
|
||||
;;
|
||||
|
||||
@ -273,12 +358,14 @@ esac
|
||||
# BEGIN: Work Functions
|
||||
|
||||
import () {
|
||||
declare -f gather_logs_from_journal gather_logs_from_files gather_logs_from_files_interactive log gather_logs_from_local_file
|
||||
CONFIG[remote_host]=""
|
||||
log_import
|
||||
declare -f gather_logs_from_journal gather_logs_from_files gather_logs_from_files_interactive log gather_logs_from_local_file
|
||||
declare -p CONFIG
|
||||
}
|
||||
|
||||
gather_logs_from_journal () {
|
||||
log_watch_vars CONFIG[day_filter] CONFIG[hour_filter] CONFIG[regex]
|
||||
log "About to load logs from journal"
|
||||
if [[ ${CONFIG[day_filter]} -gt 0 ]]; then
|
||||
sudo journalctl --since "${CONFIG[day_filter]} days ago" --no-pager | grep -E ${CONFIG[regex]}
|
||||
|
||||
@ -286,15 +373,16 @@ gather_logs_from_journal () {
|
||||
sudo journalctl --since "${CONFIG[hour_filter]} hours ago" --no-pager | grep -E ${CONFIG[regex]}
|
||||
|
||||
else
|
||||
sudo journalctl --no-pager | grep -E ${CONFIG[regex]}
|
||||
sudo journalctl --no-pager | grep -E "${CONFIG[regex]}"
|
||||
fi
|
||||
}
|
||||
|
||||
gather_logs_from_files () {
|
||||
log_watch_vars CONFIG[hostname] CONFIG[regex] CONFIG[last_nth_days]
|
||||
if [[ ! -d /var/log/hosts/${CONFIG[hostname]} ]]; then
|
||||
echo "Hosts log directory for ${CONFIG[hostname]} does not exist"
|
||||
exit
|
||||
fi
|
||||
i
|
||||
|
||||
log "User is getting logs from the last ${CONFIG[last_nth_days]} files"
|
||||
local -a FILES=( $( sudo ls -1r /var/log/hosts/${CONFIG[hostname]}/*/*/*/messages* ) )
|
||||
@ -305,6 +393,7 @@ for FILE in ${FILES[@]}; do
|
||||
done
|
||||
)"
|
||||
|
||||
log_watch_vars FILE EXT GREP
|
||||
local FILE EXT
|
||||
for FILE in ${FILES[@]::${CONFIG[last_nth_days]}}; do
|
||||
log "$FILE being checked"
|
||||
@ -321,6 +410,8 @@ done
|
||||
}
|
||||
|
||||
gather_logs_from_files_interactive () {
|
||||
log_watch_vars CONFIG[hostname] CONFIG[regex]
|
||||
log "Preparing to obtain logs from messages files"
|
||||
if [[ ! -d /var/log/hosts/${CONFIG[hostname]} ]]; then
|
||||
echo "Hosts log directory for ${CONFIG[hostname]} does not exist"
|
||||
exit
|
||||
@ -355,6 +446,7 @@ gather_logs_from_files_interactive () {
|
||||
|
||||
|
||||
|
||||
log_watch_vars FILE EXT GREP
|
||||
local FILE EXT
|
||||
for FILE in ${FILE_CHOICES[@]}; do
|
||||
log "$FILE being checked"
|
||||
@ -372,10 +464,12 @@ gather_logs_from_files_interactive () {
|
||||
}
|
||||
|
||||
gather_logs_from_local_file () {
|
||||
log_watch_vars CONFIG[local_file] CONFIG[regex]
|
||||
if [[ ! -f ${CONFIG[local_file]} ]]; then
|
||||
echo "$HOSTNAME : ${CONFIG[local_file]} does not exist"
|
||||
exit
|
||||
fi
|
||||
log_watch_vars FILE EXT GREP
|
||||
FILE=${CONFIG[local_file]}
|
||||
log "$FILE being checked"
|
||||
EXT=$( basename $FILE | cut -d '.' -f 2 )
|
||||
@ -402,7 +496,9 @@ case ${CONFIG[source]} in
|
||||
file) COMMAND=gather_logs_from_local_file;;
|
||||
esac
|
||||
|
||||
log_watch_vars COMMAND
|
||||
if [[ ${CONFIG[remote_host]} != "" ]]; then
|
||||
log "CONFIG[remote_host] was specified as ${CONFIG[remote_host]}. Going to be running search remotely"
|
||||
if [[ ${CONFIG[service_search]} -eq 1 ]]; then
|
||||
ssh ${CONFIG[remote_host]} "$( import ); $COMMAND" | awk '{ print $5 }' | cut -d '[' -f 1 | sort -u | tr -d ':'
|
||||
|
||||
@ -430,6 +526,6 @@ else
|
||||
else
|
||||
$COMMAND
|
||||
fi
|
||||
fi
|
||||
fi | sort
|
||||
|
||||
# END: Work
|
||||
|
||||
Loading…
Reference in New Issue
Block a user