Make comparsion of images more reliable

Author: zzambers

tests/testlib.bash

@@ -215,7 +215,7 @@ function beforeBg() {
   if [ "x$TTL" = "x" -o "x$TTL" = "x0"  ] ; then
     echo "manual mode!"
   else
-    import  -window root "$REPORT_DIR"/diff-01.png
+    import  -window root "png24:$REPORT_DIR"/diff-01.png
   fi
 }
 
@@ -239,7 +239,7 @@ function resolveBg() {
     wait "$pid"
   else
     sleep "$TTL"
-    import  -window root "$REPORT_DIR"/$cname-02.png
+    import  -window root "png24:$REPORT_DIR"/$cname-02.png
     ps
     #pstree
     cpids="$( getDescendants "$pid" )"|| true  # gets pids of descendant processes
@@ -249,20 +249,20 @@ function resolveBg() {
       for cpid in $cpids ; do kill -9 "$cpid" ; done
     fi
     sleep 1
-    import  -window root "$REPORT_DIR"/$cname-03.png
+    import  -window root "png24:$REPORT_DIR"/$cname-03.png
     if [ "$DIFF" == "true" ] ; then
       d12=$(compareImagesSilently $cname 01 02)
       d23=$(compareImagesSilently $cname 02 03)
       d13=$(compareImagesSilently $cname 01 03 03-01)
-      if [ "$d12" -gt 15 ] ; then
-        echo "error! images 1+2 are same, should be not"
+      if [ "$d12" -lt 1 ] ; then
+        echo "error! images 1+2 are same (or too similar), should be not"
         exit 1
       fi
-      if [ "$d23" -gt 15 ] ; then
-        echo "error! images 2+3 are same, should be not"
+      if [ "$d23" -lt 1 ] ; then
+        echo "error! images 2+3 are same (or too similar), should be not"
         exit 1
       fi
-      if [ "$d13" -le 15 ] ; then
+      if [ "$d13" -gt 0 ] ; then
         echo "warning! images 1+3 are different, should be not"
         # exit 1 # The difference between first and last stage is probably not relevant, and may cause false-negatives
       fi
@@ -286,20 +286,16 @@ function compareImagesSilently() {
   if [ "x$idCompOverride" == "x" ] ; then
     idCompOverride=$id1-$id2
   fi
-  compare  -metric PSNR  "$REPORT_DIR"/"$name"-"$id1".png "$REPORT_DIR"/"$name"-"$id2".png "$REPORT_DIR"/"$name"-"$idCompOverride".png  2> "$REPORT_DIR"/res-"$idCompOverride" || r=$?
-  echo "40+ same, 10- different" 1>&2
-  # shellcheck disable=SC2155
-  # shellcheck disable=SC2002
-  local diff=$(cat "$REPORT_DIR"/res-"$idCompOverride" | sed "s;[. ].*;;")
-  # behavior of psnr has changed, zero is now reported instead of "inf":
-  # https://www.imagemagick.org/discourse-server/viewtopic.php?t=31487
-  # distinguish "inf" case using AE metric to emulate previous behavior
-  if [ "$diff" -eq 0 ] && compare -metric AE "$REPORT_DIR"/"$name"-"$id1".png "$REPORT_DIR"/"$name"-"$id2".png null: 2>&1 | grep -q '^0' ; then
-    diff="inf"
-  fi
-  if [ "$diff" == "inf" ] ; then
-    local diff=50 #same
-  fi
+  # create black and white mask of difference, which we can use later (to count different pixels)
+  compare -compose Src -highlight-color White -lowlight-color Black  "$REPORT_DIR"/"$name"-"$id1".png "$REPORT_DIR"/"$name"-"$id2".png "$REPORT_DIR"/"$name"-"$idCompOverride".png
+  local pxTotal; local pxDiff; local diff;
+  # total pixels in image
+  pxTotal="$( convert "$REPORT_DIR"/"$name"-"$idCompOverride".png -format "%[fx:w*h]" info: )"
+  # different pixels
+  # see: https://www.imagemagick.org/discourse-server/viewtopic.php?t=32078
+  pxDiff="$( convert "$REPORT_DIR"/"$name"-"$idCompOverride".png -format "%[fx:int(w*h*mean+0.5)]" info: )"
+  # percent of different pixels (rounded down)
+  diff="$( LANG=C printf '100 * %f / %f\n' "${pxDiff}" "${pxTotal}" | bc )"
   echo $diff
 }