vnc: tight: add JPEG and gradient subencoding with smooth image detection

Add gradient filter and JPEG compression with an heuristic to detect how
lossy the comppression will be. This code has been adapted from
libvncserver/tight.c.

JPEG support can be enabled/disabled at compile time with --enable-vnc-jpeg
and --disable-vnc-jpeg.

Signed-off-by: Corentin Chary <corentincj@iksaif.net>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>

configure

diff --git a/configure b/configure
index b68f01a..27791b5 100755
--- a/configure
+++ b/configure
@@ -268,6 +268,7 @@
 vde=""
 vnc_tls=""
 vnc_sasl=""
+vnc_jpeg=""
 xen=""
 linux_aio=""
 attr=""
@@ -575,6 +576,10 @@
   ;;
   --enable-vnc-sasl) vnc_sasl="yes"
   ;;
+  --disable-vnc-jpeg) vnc_jpeg="no"
+  ;;
+  --enable-vnc-jpeg) vnc_jpeg="yes"
+  ;;
   --disable-slirp) slirp="no"
   ;;
   --disable-uuid) uuid="no"
@@ -825,6 +830,8 @@
 echo "  --enable-vnc-tls         enable TLS encryption for VNC server"
 echo "  --disable-vnc-sasl       disable SASL encryption for VNC server"
 echo "  --enable-vnc-sasl        enable SASL encryption for VNC server"
+echo "  --disable-vnc-jpeg       disable JPEG lossy compression for VNC server"
+echo "  --enable-vnc-jpeg        enable JPEG lossy compression for VNC server"
 echo "  --disable-curses         disable curses output"
 echo "  --enable-curses          enable curses output"
 echo "  --disable-curl           disable curl connectivity"
@@ -1246,6 +1253,27 @@
 fi
 
 ##########################################
+# VNC JPEG detection
+if test "$vnc_jpeg" = "yes" ; then
+cat > $TMPC <<EOF
+#include <stdio.h>
+#include <jpeglib.h>
+int main(void) { struct jpeg_compress_struct s; jpeg_create_compress(&s); return 0; }
+EOF
+    vnc_jpeg_cflags=""
+    vnc_jpeg_libs="-ljpeg"
+  if compile_prog "$vnc_jpeg_cflags" "$vnc_jpeg_libs" ; then
+    vnc_jpeg=yes
+    libs_softmmu="$vnc_jpeg_libs $libs_softmmu"
+  else
+    if test "$vnc_jpeg" = "yes" ; then
+      feature_not_found "vnc-jpeg"
+    fi
+    vnc_jpeg=no
+  fi
+fi
+
+##########################################
 # fnmatch() probe, used for ACL routines
 fnmatch="no"
 cat > $TMPC << EOF
@@ -2094,6 +2122,7 @@
 echo "Mixer emulation   $mixemu"
 echo "VNC TLS support   $vnc_tls"
 echo "VNC SASL support  $vnc_sasl"
+echo "VNC JPEG support  $vnc_jpeg"
 if test -n "$sparc_cpu"; then
     echo "Target Sparc Arch $sparc_cpu"
 fi
@@ -2231,6 +2260,10 @@
   echo "CONFIG_VNC_SASL=y" >> $config_host_mak
   echo "VNC_SASL_CFLAGS=$vnc_sasl_cflags" >> $config_host_mak
 fi
+if test "$vnc_jpeg" = "yes" ; then
+  echo "CONFIG_VNC_JPEG=y" >> $config_host_mak
+  echo "VNC_JPEG_CFLAGS=$vnc_jpeg_cflags" >> $config_host_mak
+fi
 if test "$fnmatch" = "yes" ; then
   echo "CONFIG_FNMATCH=y" >> $config_host_mak
 fi