diff --git a/on_update_failure.sh b/on_update_failure.sh new file mode 100755 index 0000000..706ff28 --- /dev/null +++ b/on_update_failure.sh @@ -0,0 +1,184 @@ +#!/bin/bash + +# set -x + +# +############################################################################# +# Check if the ATBUpdateTool(.service) did not send U0001 (sucess), +# U0002 (activated) or U0003 (error) to ISMAS, including the case that the +# tool did not crash or was not killed under other circumstances. +# +# This script will be called in the associated atbupdatetool.service (see +# /lib/systemd/system/atbupdatetool.service). +# +############################################################################### + +readonly OUT_FILE=/tmp/out.txt +readonly SCRIPT_NAME=$(basename $0) + +if [ -z "$START_DATE" ] +then + START_DATE=$(date +"%Y-%m-%d00:00:00") +fi + +if [ -z "$STOP_DATE" ] +then + STOP_DATE=$(date +"%Y-%m-%d%H:%M:%S") +fi + +echo "EXIT_CODE=$EXIT_CODE" +echo "SERVICE_RESULT=$SERVICE_RESULT" +echo "EXIT_STATUS=$EXIT_STATUS" +echo "START_DATE=$START_DATE" +echo "STOP_DATE=$STOP_DATE" + + +last_cmd_event () { + # output json; look for start of CMD_EVENT; look for }}; extract everything between; finally re-add }}. + # could also look for ">>>", but this might change in the future. + local json=$(\ + journalctl -u atbupdatetool --since="$START_DATE" --until="$STOP_DATE" --output=json-pretty |\ + grep "#M=APISM#C=CMD_EVENT#J=" |\ + awk '{split($0, a, "#M=APISM#C=CMD_EVENT#J="); print a[2]}' |\ + awk '{split($0, a, /\}[\s]*\}/); print a[1]}' |\ + tr -d '\\' |\ + tail -1) + json+="}}" + echo "$json" +} + +last_percent_value () { + echo $(cat $OUT_FILE | sed -n "s/\(^.*\)\(\"PERCENT\"\\s*:\)\(\\s*[0-9]\+\)\(.*$\)/\3/p" | xargs | awk '{print $NF}') +} + +last_step () { + echo $(cat $OUT_FILE | sed -n "s/\(^.*\)\(\"PERCENT\"\\s*:\)\(\\s*[0-9]\+\)\(.*$\)/\1\2\3\4/p" | tail -1 | awk '{split($0, a, ","); print a[8]}' | awk '{split($0, b, " : "); print b[2]}' | tr -d '"\\') +} + +last_step_result () { + echo $(cat $OUT_FILE | sed -n "s/\(^.*\)\(\"PERCENT\"\\s*:\)\(\\s*[0-9]\+\)\(.*$\)/\1\2\3\4/p" | tail -1 | awk '{split($0, a, ","); print a[9]}' | awk '{split($0, b, " : "); print b[2]}' | tr -d '"\\') +} + +pid_atbupdatetool () { + echo $(cat $OUT_FILE | sed -n "s/\(^.*\)\(ATBUpdateTool\[\)\(\\s*[0-9]\+\)\(\].*$\)/\3/p" | uniq) +} + +last_event () { + echo $(cat $OUT_FILE | sed -n "s/\(^.*\)\(\"PERCENT\"\\s*:\)\(\\s*[0-9]\+\)\(.*$\)/\1\2\3\4/p" | tail -1 | awk '{split($0, a, ","); print a[4]}' | awk '{split($0, b, ":"); print b[2]}' | tr -d ' ' | tr -d '"\\') +} + +sendU0003 () { + LAST_CMD_EVENT=$(last_cmd_event) + echo "LAST=$LAST_CMD_EVENT" + + local json_detected=0 + local temp + local x + + if [[ ! -z "$LAST_CMD_EVENT" ]] + then + temp=$(mktemp -p /tmp) + echo "$LAST_CMD_EVENT" > "$temp" + x=$(jq .EVENT $temp) + if [ $? -eq 0 ]; then + json_detected=1 + fi + fi + + if [ $json_detected -eq 1 ] + then + # local pid=$(jq .EVENT_ID $temp) + + # if json has been detected, use "jq". + + x=$(jq .PARAMETER.PERCENT $temp | tr -d '"') + [[ -z "$x" ]] && PERCENT=0 || PERCENT=$x + + x=$(jq .PARAMETER.STEP $temp | tr -d '"') + [[ -z "$x" ]] && STEP="unknown" || STEP="$x" + + x=$(jq .PARAMETER.STEP_RESULT $temp | tr -d '"') + [[ -z "$x" ]] && STEP_RESULT="unknown" || STEP_RESULT="$x" + + x=$(pid_atbupdatetool) + [[ -z "$x" ]] && PID=99999 || PID=$x + + x=$(last_event) + [[ -z "$x" ]] && EVENT="unknown" || EVENT="$x" + + else + # cannot parse valid json. try original journal text file. + + x=$(last_percent_value) + [[ -z "$x" ]] && PERCENT=0 || PERCENT=$x + + x=$(last_step) + [[ -z "$x" ]] && STEP="unknown" || STEP="$x" + + x=$(last_step_result) + [[ -z "$x" ]] && STEP_RESULT="unknown" || STEP_RESULT="$x" + + x=$(pid_atbupdatetool) + [[ -z "$x" ]] && PID=99999 || PID=$x + + x=$(last_event) + [[ -z "$x" ]] && EVENT="unknown" || EVENT="$x" + fi + + echo "PERCENT=$PERCENT" + echo "STEP=$STEP" + echo "STEP_RESULT=$STEP_RESULT" + echo "PID=$PID" + echo "EVENT=$EVENT" + + # build json to be sent to ISMAS with "U0003" + + CURRENT_DATE=$(date +"%Y-%m-%d %H:%M:%S.000%z") + U0003Msg="{\ + \"REASON\":\"SW_UP\",\ + \"TIMESTAMP\":\"$CURRENT_DATE\",\ + \"EVENT_ID\":\"$PID\",\ + \"EVENT\":\"U0003\",\ + \"EVENTSTATE\":1,\ + \"PARAMETER\": {\ + \"PERCENT\" : $PERCENT,\ + \"RESULTCODE\" : 99,\ + \"STEP\" : \"$1 -> monitored last step: $STEP\",\ + \"STEP_RESULT\" : \"$1 -> monitored last step result: $STEP_RESULT\",\ + \"VERSION\" : \"\"\ + }\ + }" + + echo -e "U0003Msg=$U0003Msg" + + (echo "#M=APISM#C=CMD_EVENT#J=$U0003Msg"; sleep 5) | nc localhost 7777 +} + +# save journal for the last run of atbupdatetool in $OUT_FILE +echo $(journalctl -u atbupdatetool --since="$START_DATE" --until=$STOP_DATE) > $OUT_FILE + +if [[ "$EXIT_CODE" = "exited" && "$SERVICE_RESULT" = "success" && "$EXIT_STATUS" = "0" ]] +then + # almost normal run; at least there was no crash. + if cat $OUT_FILE | grep -o "U0003" | wc -l; then + echo "$SCRIPT_NAME: Failure code U0003 already sent to ISMAS" + exit 0 + fi + + if cat $OUT_FILE | grep -o "U0002" | wc -l; then + echo "$SCRIPT_NAME: Success code U0002 already sent to ISMAS" + exit 0 + else + sendU0003 "$SCRIPT_NAME: Success code U0002 not sent to ISMAS" + fi + + if cat $OUT_FILE | grep -o "U0001" | wc -l; then + echo "$SCRIPT_NAME: Success code U0001 already sent to ISMAS" + exit 0 + else + sendU0003 "$SCRIPT_NAME: Success code U0001 not sent to ISMAS" + fi +else + # something unexpected has happened. + sendU0003 "$SCRIPT_NAME: EXIT_CODE=$EXIT_CODE;SERVICE_RESULT=$SERVICE_RESULT;EXIT_STATUS=$EXIT_STATUS" +fi