ATBUpdateTool/on_update_failure.sh

#!/bin/bash

# set -x

#
#############################################################################
# Check if the ATBUpdateTool(.service) did not send U0001 (sucess),
# U0002 (activated) or U0003 (error) to ISMAS, including the case that the
# tool did not crash or was not killed under other circumstances.
#
# This script will be called in the associated atbupdatetool.service (see
# /lib/systemd/system/atbupdatetool.service).
#
###############################################################################

readonly OUT_FILE=/tmp/out.txt
readonly SCRIPT_NAME=$(basename $0)

if [ -z "$START_DATE" ]
then
    START_DATE=$(date +"%Y-%m-%d00:00:00")
fi

if [ -z "$STOP_DATE" ]
then
    STOP_DATE=$(date +"%Y-%m-%d%H:%M:%S")
fi

echo "EXIT_CODE=$EXIT_CODE"
echo "SERVICE_RESULT=$SERVICE_RESULT"
echo "EXIT_STATUS=$EXIT_STATUS"
echo "START_DATE=$START_DATE"
echo "STOP_DATE=$STOP_DATE"


last_cmd_event () {
    # output json; look for start of CMD_EVENT; look for }}; extract everything between; finally re-add }}.
    # could also look for ">>>", but this might change in the future.
    local json=$(\
	    journalctl -u atbupdatetool --since="$START_DATE" --until="$STOP_DATE" --output=json-pretty |\
	    grep "#M=APISM#C=CMD_EVENT#J="                                                              |\
	    awk '{split($0, a, "#M=APISM#C=CMD_EVENT#J="); print a[2]}'                                 |\
	    awk '{split($0, a, /\}[\s]*\}/); print a[1]}'                                               |\
	    tr -d '\\'                                                                                  |\
	    tail -1)
    json+="}}"
    echo "$json"
}

last_percent_value () {
    echo $(cat $OUT_FILE | sed -n "s/\(^.*\)\(\"PERCENT\"\\s*:\)\(\\s*[0-9]\+\)\(.*$\)/\3/p" | xargs | awk '{print $NF}')
}

last_step () {
    echo $(cat $OUT_FILE | sed -n "s/\(^.*\)\(\"PERCENT\"\\s*:\)\(\\s*[0-9]\+\)\(.*$\)/\1\2\3\4/p" | tail -1 | awk '{split($0, a, ","); print a[8]}' | awk '{split($0, b, " : "); print b[2]}' | tr -d '"\\')
}

last_step_result () {
    echo $(cat $OUT_FILE | sed -n "s/\(^.*\)\(\"PERCENT\"\\s*:\)\(\\s*[0-9]\+\)\(.*$\)/\1\2\3\4/p" | tail -1 | awk '{split($0, a, ","); print a[9]}' | awk '{split($0, b, " : "); print b[2]}' | tr -d '"\\')
}

pid_atbupdatetool () {
    echo $(cat $OUT_FILE | sed -n "s/\(^.*\)\(ATBUpdateTool\[\)\(\\s*[0-9]\+\)\(\].*$\)/\3/p" | uniq)
}

last_event () {
    echo $(cat $OUT_FILE | sed -n "s/\(^.*\)\(\"PERCENT\"\\s*:\)\(\\s*[0-9]\+\)\(.*$\)/\1\2\3\4/p" | tail -1 | awk '{split($0, a, ","); print a[4]}' | awk '{split($0, b, ":"); print b[2]}' | tr -d ' ' | tr -d '"\\') 
}

sendU0003 () {
    LAST_CMD_EVENT=$(last_cmd_event)
    echo "LAST=$LAST_CMD_EVENT"
    
    local json_detected=0
    local temp
    local x

    if [[ ! -z "$LAST_CMD_EVENT" ]]
    then
       temp=$(mktemp -p /tmp)
       echo "$LAST_CMD_EVENT" > "$temp"
       x=$(jq .EVENT $temp)
       if [ $? -eq 0 ]; then
           json_detected=1
       fi
    fi

    if [ $json_detected -eq 1 ]
    then
        # local pid=$(jq .EVENT_ID $temp)

	# if json has been detected, use "jq".

        x=$(jq .PARAMETER.PERCENT $temp | tr -d '"')
        [[ -z "$x" ]] && PERCENT=0 || PERCENT=$x

        x=$(jq .PARAMETER.STEP $temp | tr -d '"')
        [[ -z "$x" ]] && STEP="unknown" || STEP="$x"
       
        x=$(jq .PARAMETER.STEP_RESULT $temp | tr -d '"')
        [[ -z "$x" ]] && STEP_RESULT="unknown" || STEP_RESULT="$x"
    
        x=$(pid_atbupdatetool)
        [[ -z "$x" ]] && PID=99999 || PID=$x
    
        x=$(last_event)
        [[ -z "$x" ]] && EVENT="unknown" || EVENT="$x"
    
    else 
        # cannot parse valid json. try original journal text file.

        x=$(last_percent_value)
        [[ -z "$x" ]] && PERCENT=0 || PERCENT=$x

        x=$(last_step)
        [[ -z "$x" ]] && STEP="unknown" || STEP="$x"
    
        x=$(last_step_result)
        [[ -z "$x" ]] && STEP_RESULT="unknown" || STEP_RESULT="$x"
    
        x=$(pid_atbupdatetool)
        [[ -z "$x" ]] && PID=99999 || PID=$x

        x=$(last_event)
        [[ -z "$x" ]] && EVENT="unknown" || EVENT="$x"
    fi

    echo "PERCENT=$PERCENT"
    echo "STEP=$STEP"
    echo "STEP_RESULT=$STEP_RESULT"
    echo "PID=$PID"
    echo "EVENT=$EVENT"

    # build json to be sent to ISMAS with "U0003"

    CURRENT_DATE=$(date +"%Y-%m-%d %H:%M:%S.000%z")
    U0003Msg="{\
        \"REASON\":\"SW_UP\",\
        \"TIMESTAMP\":\"$CURRENT_DATE\",\
        \"EVENT_ID\":\"$PID\",\
        \"EVENT\":\"U0003\",\
        \"EVENTSTATE\":1,\
        \"PARAMETER\": {\
            \"PERCENT\" : $PERCENT,\
            \"RESULTCODE\" : 99,\
                    \"STEP\" : \"$1 -> monitored last step: $STEP\",\
                    \"STEP_RESULT\" : \"$1 -> monitored last step result: $STEP_RESULT\",\
            \"VERSION\" : \"\"\
        }\
    }"

    echo -e "U0003Msg=$U0003Msg"

    (echo "#M=APISM#C=CMD_EVENT#J=$U0003Msg"; sleep 5) | nc localhost 7777
}

# save journal for the last run of atbupdatetool in $OUT_FILE
echo $(journalctl -u atbupdatetool --since="$START_DATE" --until=$STOP_DATE) > $OUT_FILE

if [[ "$EXIT_CODE" = "exited" && "$SERVICE_RESULT" = "success" && "$EXIT_STATUS" = "0" ]]
then
    # almost normal run; at least there was no crash.	
    if cat $OUT_FILE | grep -o "U0003" | wc -l; then
        echo "$SCRIPT_NAME: Failure code U0003 already sent to ISMAS"
        exit 0
    fi

    if cat $OUT_FILE | grep -o "U0002" | wc -l; then
        echo "$SCRIPT_NAME: Success code U0002 already sent to ISMAS"
        exit 0
    else
        sendU0003 "$SCRIPT_NAME: Success code U0002 not sent to ISMAS"
    fi

    if cat $OUT_FILE | grep -o "U0001" | wc -l; then
        echo "$SCRIPT_NAME: Success code U0001 already sent to ISMAS"
        exit 0
    else
        sendU0003 "$SCRIPT_NAME: Success code U0001 not sent to ISMAS"
    fi
else
    # something unexpected has happened.
    sendU0003 "$SCRIPT_NAME: EXIT_CODE=$EXIT_CODE;SERVICE_RESULT=$SERVICE_RESULT;EXIT_STATUS=$EXIT_STATUS"
fi