#!/bin/bash

# Holler if WER worsens

set -e -u

cd "$(dirname "$0")"/..

if [[ $# -eq 1 && $1 = -v ]]; then
    verbose=true
else
    verbose=false
fi

prevmt=t/ntb/articles.nob-nno.mt-latest
gold=t/ntb/articles.gold.nno

tmp=$(mktemp -d -t wer-loss-nno-nob.XXXXXXXXXX)
trap 'rm -rf "${tmp}"' EXIT

< t/ntb/articles.orig.nob apertium -d . nob-nno_e > "${tmp}"/mt

clean () {
    sed 's/\(ORIGINAL\|KORREKTUR\):/--------------/g' "$@" \
        | tr -d '#' \
        | grep -v -e '^[*][*]*$' -e UKJENTE -e '^\[' \
        | sed 's/[.] /.\n/g' | grep .
}
clean "${prevmt}" >"${tmp}"/prev-test
clean "${gold}"   >"${tmp}"/ref
clean "${tmp}"/mt >"${tmp}"/test

prev=$(t/beam-eval-until-stable -t "${tmp}"/prev-test -r "${tmp}"/ref 2>/dev/null)
cur=$(t/beam-eval-until-stable  -t "${tmp}"/test      -r "${tmp}"/ref 2>/dev/null)

justwer () {
    sed -n '/WER/{s/[^0-9]*\([0-9][0-9]*\)[.].*/\1/;p}' \
        | awk '{print int(10*($0+0.05));exit(0)}'
    # Turn into rounded permille, only take the first one
}

werdiff () {
    echo "diff of previous ${prevmt} and current evaluation results against ${gold}:"
    echo
    diff -y <(echo "${prev}") <(echo "${cur}")
}

prevwer=$(echo "${prev}" | justwer)
curwer=$(echo "${cur}"   | justwer)
if [[ $prevwer < $curwer ]]; then
    echo
    echo "ERROR: WER increased since ${prevmt}"
    werdiff
    exit 1
elif $verbose; then
    werdiff
fi
