Add -o option to optimize png file size

build/sanitize-png-files.sh

Index: build/sanitize-png-files.sh
diff --git a/build/sanitize-png-files.sh b/build/sanitize-png-files.sh
index f13a191e6127f328ee1fc58092a03cc9da2e9bdf..5387a2cd8fe685e65ef89b37a27fe856b88878f9 100755
--- a/build/sanitize-png-files.sh
+++ b/build/sanitize-png-files.sh
@@ -3,7 +3,22 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-ALL_DIRS="
+# The optimization code is based on pngslim (http://goo.gl/a0XHg)
+# and executes similar pipleline to optimize the png file size.

[msw, 2013/01/16 03:43:11]

nit: "a similar"

[oshima, 2013/01/16 18:29:04]

Done.

+# The steps that requires pngoptimizercl/pngrewrite/deflopts are omitted,

[msw, 2013/01/16 03:43:11]

nits: "that require" and "deflopt"

[oshima, 2013/01/16 18:29:04]

Done.

+# but this runs allother processes, including:

[msw, 2013/01/16 03:43:11]

nit: "all other"

[oshima, 2013/01/16 18:29:04]

Done.

+# 1) various color-dependent optimization using optpng.

[msw, 2013/01/16 03:43:11]

nits: "optimizations" "optipng"

[oshima, 2013/01/16 18:29:04]

Done.

+# 2) optimize number of huffman blocks.

[msw, 2013/01/16 03:43:11]

nit: "optimize the"

[oshima, 2013/01/16 18:29:04]

Done.

+# 3) randomized huffman table.

[msw, 2013/01/16 03:43:11]

nit: "randomize the"

[oshima, 2013/01/16 18:29:04]

Done.

+# 4) Further optimize using optipng () and advdef (zlib stream).

[msw, 2013/01/16 03:43:11]

nit: remove "()"

[oshima, 2013/01/16 18:29:04]

Done.

+# Due to the step 3), each run may produce slightly different results.
+#
+# TOTO(oshima): Use different parameters for large file to reduce

[msw, 2013/01/16 03:43:11]

nits: "TODO" and "files"

[oshima, 2013/01/16 18:29:04]

Done. (moved to random_huffman_table_trial)

+# runtime.

[msw, 2013/01/16 03:43:11]

nit: this fits on the line above.

[oshima, 2013/01/16 18:29:04]

Done.

+# Note(oshima): In my experiment, advdef didn't reduce much. I'm keeping it
+# for now as it does take much time to run.

[msw, 2013/01/16 03:43:11]

nit: Did you mean "doesn't"? Otherwise, this doesn't make sense.

[oshima, 2013/01/16 18:29:04]

Done.

+
+readonly ALL_DIRS="
 ash/resources
 ui/resources
 chrome/app/theme
@@ -14,15 +29,198 @@ remoting/resources
 remoting/webapp
 "
 
-function sanitize_file {
+# Constants used for optimization
+readonly MIN_BLOCK_SIZE=128
+readonly LIMIT_BLOCKS=256
+readonly RANDOM_TRIALS=100
+
+# Global variables for stats
+TOTAL_OLD_BYTES=0
+TOTAL_NEW_BYTES=0
+TOTAL_FILE=0
+PROCESSED_FILE=0
+
+declare -a THROBBER_STR=('-' '\\' '|' '/')
+THROBBER_COUNT=0
+
+# Show throbber character at current cursor position.
+function throbber {
+  echo -ne "${THROBBER_STR[$THROBBER_COUNT]}\b"
+  let THROBBER_COUNT=($THROBBER_COUNT+1)%4
+}
+
+# Usage: process_rgb <file> <png_out_options> ...

[msw, 2013/01/16 03:43:11]

s/process_rgb/pngout_loop/

[oshima, 2013/01/16 18:29:04]

Done.

+# Optimize the png file using pngout with given option

[msw, 2013/01/16 03:43:11]

nit: "the given options"

[oshima, 2013/01/16 18:29:04]

Done.

+# using vairous block split threshold and filter types.

[msw, 2013/01/16 03:43:11]

nits: "various" and "thresholds"

[oshima, 2013/01/16 18:29:04]

Done.

+function pngout_loop {
+  local file=$1
+  shift
+  local opts=$*
+  for i in 0 128 256 512; do
+    for j in $(seq 0 5); do
+      throbber
+      pngout -q -k1 -s1 -b$i -f$j $opts $file
+    done
+  done
+}
+
+# Usage: process_grayscale <file>
+# Optimize grayscale image for all color bit depth.

[msw, 2013/01/16 03:43:11]

nit: "images" and "depths" here and lines 77 and 87.

[oshima, 2013/01/16 18:29:04]

Done.

+function process_grayscale {
+  echo -n "|gray"
+  for opt in -d1 -d2 -d4 -d8; do

[msw, 2013/01/16 03:43:11]

Can we use -d0 to minimize the bit depth without looping (here and below)?
It's okay if you just want to follow pngslim's example in this first CL, but
then maybe add a TODO to experiment with and perhaps adopt -d0 without looping.
From: http://advsys.net/ken/util/pngout.htm

[oshima, 2013/01/16 18:29:04]

I added TODO.

+    pngout_loop $file -c0 $opt

[msw, 2013/01/16 03:43:11]

According to the documentation, "If no /c# option is specified, PNGOUT will
choose the color type that can most efficiently store the image." So, can we
avoid overriding the color type or add a TODO to experiment with that?

[oshima, 2013/01/16 18:29:04]

added TODO

+  done
+}
+
+# Usage: process_grayscale_alpha <file>
+# Optimize grayscale+alpha image for all color bit depth.

[msw, 2013/01/16 03:43:11]

nit: "(with alpha)", "(w/ alpha)", or similar.

[oshima, 2013/01/16 18:29:04]

Done.

+function process_grayscale_alpha {
+  echo -n "|gray-a"
+  pngout_loop $file -c4
+  for opt in -d1 -d2 -d4 -d8; do
+    pngout_loop $file -c3 $opt
+  done
+}
+
+# Usage: process_rgb <file>
+# Optimize rgb (w w/o alpha) image for all color bit depth.

[msw, 2013/01/16 03:43:11]

nit: "(with or without alpha)", "(w/ or w/o alpha)", or similar.

[oshima, 2013/01/16 18:29:04]

Done.

+function process_rgb {
+  echo -n "|rgb"
+  for opt in -d1 -d2 -d4 -d8; do
+    pngout_loop $file -c3 $opt
+  done
+  pngout_loop $file -c2
+  pngout_loop $file -c6
+}
+
+# Usage: huffman_blocks <file>
+# Optimize huffman blocks

[msw, 2013/01/16 03:43:11]

nit: "the huffman", and add a trailing period.

[oshima, 2013/01/16 18:29:04]

Done.

+function huffman_blocks {
+  local file=$1
+  echo -n "|huffman"
+  local size=$(stat -c%s $file)
+  let MAX_BLOCKS=$size/$MIN_BLOCK_SIZE
+  if [ $MAX_BLOCKS -gt $LIMIT_BLOCKS ]; then
+    MAX_BLOCKS=$LIMIT_BLOCKS
+  fi
+  for i in $(seq 2 $MAX_BLOCKS); do
+    throbber
+    pngout -q -k1 -ks -s1 -n$i $file
+  done
+}
+
+# Usage: random_huffmantable_trial <file>

[msw, 2013/01/16 03:43:11]

nit: random_huffman_table

[oshima, 2013/01/16 18:29:04]

Done.

+# Try compressing using randomized initial huffman table.

[msw, 2013/01/16 03:43:11]

nit: "a randomized"

[oshima, 2013/01/16 18:29:04]

changed to

Try compressing by randomizing initial huffman table.

+function random_huffmantable_trial {

[msw, 2013/01/16 03:43:11]

Wow! This has got to be why this script is so slow. -s0 is the "Xtreme!
(Slowest)" strategy, but here is run 100 times, and so long as that produces a
smaller file, it's run another 100 times. I feel like we could probably devise a
more prudent algorithm/heuristic if we roughly understood the variability in the
results. Possibly add a TODO here to that effect?

[oshima, 2013/01/16 18:29:04]

what makes it really slow is that pngout takes really long to process large
files.
I guess it has O(n^2) in it. And yes, I'm going to do some experiment to reduce
the number
of loops. I moved TODO from top to this function's comment.

+  echo -n "|random"
+  local file=$1
+  local old_size=$(stat -c%s $file)
+  for i in $(seq 1 $RANDOM_TRIALS); do
+    throbber
+    pngout -q -k -ks -s0 -r $file

[msw, 2013/01/16 03:43:11]

This -k option should be -k1 if you're following pngslim; -k doesn't appear to
have any meaning without a number from the doc.

[oshima, 2013/01/16 18:29:04]

thank you for the catch! Fixed.

+  done
+  local new_size=$(stat -c%s $file)
+  if [ $new_size -lt $old_size ]; then
+    random_huffmantable_trial $file
+  fi
+}
+
+# Usage: final_comprssion <file>
+# Further compreess using optpng and advdef.

[msw, 2013/01/16 03:43:11]

nits: "compress" and "optipng"

[oshima, 2013/01/16 18:29:04]

Done.

+function final_compression {
+  echo -n "|final"
+  local file=$1
+  for i in 32k 16k 8k 4k 2k 1k 512; do

[msw, 2013/01/16 03:43:11]

Should we also try 256 here? I guess pngslim doesn't.

[oshima, 2013/01/16 18:29:04]

I'll try. I added TODO

+    throbber
+    optipng -q -nb -nc -zw$i -zc1-9 -zm1-9 -zs0-3 -f0-5 $file

[msw, 2013/01/16 03:43:11]

optipng docs claim that quiet mode is -quiet, not -q:
http://optipng.sourceforge.net/optipng-0.7.4.man.pdf
But I guess pngslim uses -q, and if it works, it's fine.

[oshima, 2013/01/16 18:29:04]

yes, it -q does work.

+  done
+  for i in $(seq 1 4); do
+    throbber
+    advdef -q -z -$i $file

[msw, 2013/01/16 03:43:11]

Strange, your code looks correct according to advdef docs:
http://advancemame.sourceforge.net/doc-advdef.html
But pngslim seems to use (incorrect?) "-z$i" instead of "-z -$i".

[oshima, 2013/01/16 18:29:04]

yes, I looked at the doc and fixed. I guess both works.

+  done
+  echo -ne "\r"
+}
+
+# Usage: optimize_size <file>
+# Performs png file optimization.
+function optimize_size {
   tput el
-  echo -ne "$1\r"
+  echo -n "$file "

[msw, 2013/01/16 03:43:11]

Does this work? (using $file before it's defined in the next line)

[oshima, 2013/01/16 18:29:04]

thank you for the catch. Fixed.

+  local file=$1
+
+  advdef -q -z -4 $file
+
+  pngout -q -s4 -c0 -force $file $file.tmp.png
+  if [ -f $file.tmp.png ]; then
+    rm $file.tmp.png
+    process_grayscale $file
+    process_grayscale_alpha $file
+  else
+    pngout -q -s4 -c4 -force $file $file.tmp.png
+    if [ -f $file.tmp.png ]; then
+      rm $file.tmp.png
+      process_grayscale_alpha $file
+    else
+      process_rgb $file
+    fi
+  fi
+
+  echo -n "|filter"
+  optipng -q -zc9 -zm8 -zs0-3 -f0-5 $file
+  pngout -q -k1 -s1 $file
+
+  huffman_blocks $file
+
+  echo -n "|strategy"
+  for i in 3 2 0; do

[msw, 2013/01/16 03:43:11]

Should we also run Strategy 1 here? I guess pngslim doesn't.

[oshima, 2013/01/16 18:29:04]

Added TODO

+    pngout -q -k1 -ks -s$i $file
+  done
+
+  random_huffmantable_trial $file
+
+  final_compression $file
+}
+
+# Usage: process_fie <file>

[msw, 2013/01/16 03:43:11]

nit: "process_file"

[oshima, 2013/01/16 18:29:04]

Done.

+function process_file {
   local file=$1
   local name=$(basename $file)
   # -rem alla removes all ancillary chunks except for tRNS
   pngcrush -d $TMP_DIR -brute -reduce -rem alla $file > /dev/null
 
-  mv "$TMP_DIR/$name" "$file"
+  if [ ! -z "$OPTIMIZE" ]; then
+    optimize_size $TMP_DIR/$name
+  fi
+}
+
+# Usage: sanitize_file <file>
+function sanitize_file {
+  local file=$1
+  local name=$(basename $file)
+  local old=$(stat -c%s $file)
+  local tmp_file=$TMP_DIR/$name
+
+  process_file $file
+
+  local new=$(stat -c%s $tmp_file)
+  let diff=$old-$new
+  let TOTAL_OLD_BYTES+=$old
+  let TOTAL_NEW_BYTES+=$new
+  let percent=($diff*100)/$old
+  let TOTAL_FILE+=1
+
+  tput el
+  if [ $new -lt $old ]; then
+    echo -ne "$file : $old => $new ($diff bytes : $percent %)\n"
+    mv "$tmp_file" "$file"
+    let PROCESSED_FILE+=1
+  else
+    if [ -z "$OPTIMIZE" ]; then
+      echo -ne "$file : skipped\r"
+    fi
+    rm $tmp_file
+  fi
 }
 
 function sanitize_dir {
@@ -32,17 +230,64 @@ function sanitize_dir {
   done
 }
 
+function install_if_not_installed {
+  local program=$1
+  dpkg -s $program > /dev/null 2>&1
+  if [ "$?" != "0" ]; then
+    read -p "Couldn't find $program. Do you want to install? (y/n)"
+    [ "$REPLY" == "y" ] && sudo apt-get install $program
+    [ "$REPLY" == "y" ] || exit
+  fi
+}
+
+function fail_if_not_installed {
+  local program=$1
+  local url=$2
+  which $program > /dev/null
+  if [ $? != 0 ]; then
+    echo "Couldn't find $program. Please download and install it from $url"
+    exit 1
+  fi
+}
+
+function show_help {
+  local program=$(basename $0)
+  echo \
+"Usage: $program [options] dir ...
+
+$program is a utility to reduce the size of png files by removing
+unnecessary chunks and compress the image.

[msw, 2013/01/16 03:43:11]

nit: "compressing"

[oshima, 2013/01/16 18:29:04]

Done.

+
+Options:
+  -o  Agressively optimize file size. Warnig: this is *VERY* slow and

[msw, 2013/01/16 03:43:11]

nits: "Aggressively" and "Warning"

[oshima, 2013/01/16 18:29:04]

Done.

+      can take hours to process all files.
+  -h  Print this help text."
+  exit 1
+}
+
 if [ ! -e ../.gclient ]; then
   echo "$0 must be run in src directory"
   exit 1
 fi
 
-# Make sure we have pngcrush installed.
-dpkg -s pngcrush > /dev/null 2>&1
-if [ "$?" != "0" ]; then
-  read -p "Couldn't fnd pngcrush. Do you want to install? (y/n)"
-  [ "$REPLY" == "y" ] && sudo apt-get install pngcrush
-  [ "$REPLY" == "y" ] || exit
+# Parse options
+while getopts oh opts
+do
+  case $opts in
+    o)
+      OPTIMIZE=true;
+      shift;;
+    [h?])
+      show_help;;
+  esac
+done
+
+# Make sure we have all necessary commands installed.
+install_if_not_installed pngcrush
+if [ ! -z "$OPTIMIZE" ]; then
+  install_if_not_installed optipng
+  fail_if_not_installed advdef "http://advancemame.sourceforge.net/comp-download.html"
+  fail_if_not_installed pngout "http://www.jonof.id.au/kenutils"
 fi
 
 # Create tmp directory for crushed png file.
@@ -51,8 +296,8 @@ TMP_DIR=$(mktemp -d)
 # Make sure we cleanup temp dir
 trap "rm -rf $TMP_DIR" EXIT
 
-# If no arguments passed, sanitize all directories.
-DIRS=$*
+# If no directories specified, sanitize all directories.

[msw, 2013/01/16 03:43:11]

nit: "directories are specified"

[oshima, 2013/01/16 18:29:04]

Done.

+DIRS=$@
 set ${DIRS:=$ALL_DIRS}
 
 for d in $DIRS; do
@@ -61,3 +306,11 @@ for d in $DIRS; do
   echo
 done
 
+# Print the results

[msw, 2013/01/16 03:43:11]

nit: Add a trailing period.

[oshima, 2013/01/16 18:29:04]

Done.

+let diff=$TOTAL_OLD_BYTES-$TOTAL_NEW_BYTES
+let percent=$diff*100/$TOTAL_OLD_BYTES
+echo "Processed $PROCESSED_FILE files (out of $TOTAL_FILE files)" \
+     "in $(date -u -d @$SECONDS +%T)s"
+echo "Result : $TOTAL_OLD_BYTES => $TOTAL_NEW_BYTES" \

[msw, 2013/01/16 03:43:11]

nit: Specify the units like "$TOTAL_OLD_BYTES bytes"

[oshima, 2013/01/16 18:29:04]

Done.

+     "($diff bytes : $percent %)"
+