Spend the presses on the open question, and report what learning actually folded
Two changes to the measurement, both about not wasting a finger. The wrong-finger control is off by default now. Across every run this project has made the wrong finger has matched zero times out of forty-odd taps; the question that is still open is the false-negative rate, and each control tap spends a press that could have measured it. Pass a count to put the control back, and the summary says plainly when it was not run rather than printing a zero that reads like a result. And a trend run now reports how many frames learning folded, out of the daemon's own transcript, alongside the container size. Without that a run cannot tell "learning fired and did not help" from "learning never fired" -- which is exactly the confusion that let the first trend run measure a static template three times and look like a result. Default trial length goes to fifteen taps, since dropping the control freed the presses and ten was too few to separate three-in-ten from five-in-ten.
This commit is contained in:
parent
3e06003fd0
commit
edb0d8e530
2 changed files with 29 additions and 6 deletions
|
|
@ -40,6 +40,19 @@ UNIT=fingerprintd-test
|
||||||
BIN=/tmp/fingerprintd
|
BIN=/tmp/fingerprintd
|
||||||
COMMON="--daemon --verbose --edge-wake --sfs-root=/var/lib/fingerprintd/sfs --sfs-writable --rpmb-write"
|
COMMON="--daemon --verbose --edge-wake --sfs-root=/var/lib/fingerprintd/sfs --sfs-writable --rpmb-write"
|
||||||
|
|
||||||
|
# How many frames learning actually folded, out of the daemon's own transcript.
|
||||||
|
# Without this a trend run cannot tell "learning fired and did not help" from
|
||||||
|
# "learning never fired" -- which is exactly the confusion that made the first
|
||||||
|
# trend run measure a static template three times.
|
||||||
|
folds() {
|
||||||
|
L=$(ls -t /var/log/fingerprintd/*.log 2>/dev/null | head -1)
|
||||||
|
[ -n "$L" ] || { echo " (no transcript)"; return; }
|
||||||
|
m=$(grep -c -- "-> MATCH fid=" "$L" 2>/dev/null || echo 0)
|
||||||
|
f=$(grep -c "folded in (metric" "$L" 2>/dev/null || echo 0)
|
||||||
|
s=$(grep -c "learn: template saved" "$L" 2>/dev/null || echo 0)
|
||||||
|
echo " folds: $f frame(s) over $m matched press(es), $s save(s)"
|
||||||
|
}
|
||||||
|
|
||||||
sizes() {
|
sizes() {
|
||||||
# A template is stored twice, the container and its backup, so the sizes
|
# A template is stored twice, the container and its backup, so the sizes
|
||||||
# come in pairs. The BODY is the container minus its 4096-byte header.
|
# come in pairs. The BODY is the container minus its 4096-byte header.
|
||||||
|
|
@ -115,7 +128,7 @@ enrol)
|
||||||
echo; echo "next: fplearn.sh base" ;;
|
echo; echo "next: fplearn.sh base" ;;
|
||||||
|
|
||||||
base)
|
base)
|
||||||
N=${2:-10}; W=${3:-5}
|
N=${2:-15}; W=${3:-0}
|
||||||
echo "=== BASELINE: learning OFF ==="
|
echo "=== BASELINE: learning OFF ==="
|
||||||
restart "--learn=0" || exit 1
|
restart "--learn=0" || exit 1
|
||||||
echo "sizes before:"; sizes
|
echo "sizes before:"; sizes
|
||||||
|
|
@ -124,7 +137,9 @@ base)
|
||||||
echo; echo "next: fplearn.sh trend" ;;
|
echo; echo "next: fplearn.sh trend" ;;
|
||||||
|
|
||||||
trend)
|
trend)
|
||||||
N=${2:-10}; W=${3:-5}
|
# No wrong-finger taps by default: zero false accepts is settled over forty
|
||||||
|
# of them, and the presses are better spent on the false-negative rate.
|
||||||
|
N=${2:-15}; W=${3:-0}
|
||||||
echo "=== TREND: learning ON, three runs on one template ==="
|
echo "=== TREND: learning ON, three runs on one template ==="
|
||||||
restart "--learn=1" || exit 1
|
restart "--learn=1" || exit 1
|
||||||
echo "sizes at the start:"; sizes
|
echo "sizes at the start:"; sizes
|
||||||
|
|
@ -132,7 +147,7 @@ trend)
|
||||||
while [ $r -le 3 ]; do
|
while [ $r -le 3 ]; do
|
||||||
echo; echo "----- learning run $r of 3 -----"
|
echo; echo "----- learning run $r of 3 -----"
|
||||||
/tmp/fptrial.sh "$N" "$W"
|
/tmp/fptrial.sh "$N" "$W"
|
||||||
echo "sizes after run $r:"; sizes
|
echo "after run $r:"; sizes; folds
|
||||||
r=$((r+1))
|
r=$((r+1))
|
||||||
done
|
done
|
||||||
echo
|
echo
|
||||||
|
|
|
||||||
|
|
@ -23,12 +23,20 @@ run() { # $1 = label
|
||||||
}
|
}
|
||||||
echo "labelled trial -> $OUT"
|
echo "labelled trial -> $OUT"
|
||||||
i=1; while [ $i -le $C ]; do run "CORRECT $i/$C"; i=$((i+1)); done
|
i=1; while [ $i -le $C ]; do run "CORRECT $i/$C"; i=$((i+1)); done
|
||||||
echo; echo "===== now switch to a DIFFERENT finger ====="; sleep 3
|
# The wrong-finger half is skippable, and skipping it is the right default once
|
||||||
i=1; while [ $i -le $W ]; do run "WRONG $i/$W"; i=$((i+1)); done
|
# the question it answers is settled. Across every run this project has made the
|
||||||
|
# wrong finger has matched ZERO times out of forty-odd taps; the open question
|
||||||
|
# is the FALSE NEGATIVE rate, and each wrong-finger tap spends a press that
|
||||||
|
# could have measured that instead. Pass a count to put the control back.
|
||||||
|
if [ "$W" -gt 0 ]; then
|
||||||
|
echo; echo "===== now switch to a DIFFERENT finger ====="; sleep 3
|
||||||
|
i=1; while [ $i -le $W ]; do run "WRONG $i/$W"; i=$((i+1)); done
|
||||||
|
fi
|
||||||
echo
|
echo
|
||||||
echo "===== summary ====="
|
echo "===== summary ====="
|
||||||
awk '$1=="CORRECT"{c++; if($3=="verify-match")cm++; ct+=$4}
|
awk '$1=="CORRECT"{c++; if($3=="verify-match")cm++; ct+=$4}
|
||||||
$1=="WRONG" {w++; if($3=="verify-match")wm++; wt+=$4}
|
$1=="WRONG" {w++; if($3=="verify-match")wm++; wt+=$4}
|
||||||
END{printf " correct finger: %d/%d matched (false negatives %d) mean %d ms\n", cm,c,c-cm,(c?ct/c:0);
|
END{printf " correct finger: %d/%d matched (false negatives %d) mean %d ms\n", cm,c,c-cm,(c?ct/c:0);
|
||||||
printf " wrong finger: %d/%d matched (false ACCEPTS %d) mean %d ms\n", wm,w,wm,(w?wt/w:0)}' "$OUT"
|
if (w) printf " wrong finger: %d/%d matched (false ACCEPTS %d) mean %d ms\n", wm,w,wm,wt/w;
|
||||||
|
else printf " wrong finger: not run this trial\n"}' "$OUT"
|
||||||
echo " per-tap log: $OUT"
|
echo " per-tap log: $OUT"
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue