GN: Build libvpx on POSIX x86

generate_gypi.sh

 #!/bin/bash -e
 #
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# This script is used to generate .gypi files and files in the config/platform
-# directories needed to build libvpx.
+# This script is used to generate .gypi, .gni files and files in the
+# config/platform directories needed to build libvpx.
 # Every time libvpx source code is updated just run this script.
 #
 # For example:
 # $ ./generate_gypi.sh
 #
-# And this will update all the .gypi and config files needed.
+# And this will update all the .gypi, .gni and config files needed.
 #
 # !!! It's highly recommended to install yasm before running this script.
 
 export LC_ALL=C
 BASE_DIR=`pwd`

[Tom Finegan, 2014/09/04 01:59:05]

Mind $()'ing this for me? It's the only backquoted command in the script.

[Alpha Left Google, 2014/09/04 23:09:48]

Done.

 LIBVPX_SRC_DIR="source/libvpx"
 LIBVPX_CONFIG_DIR="source/config"
 
-# Print gypi boilerplate header
+# Print license header.
 # $1 - Output base name
-function write_gypi_header {
-  echo "# This file is generated. Do not edit." > $1
+function write_license {
+  echo "# This file is generated. Do not edit." >> $1
   echo "# Copyright (c) 2013 The Chromium Authors. All rights reserved." >> $1

[fgalligan1, 2014/09/04 16:44:46]

2014?

[Alpha Left Google, 2014/09/04 23:09:48]

Done.

   echo "# Use of this source code is governed by a BSD-style license that can be" >> $1
   echo "# found in the LICENSE file." >> $1
   echo "" >> $1
+}
+
+# Print gypi boilerplate header.
+# $1 - Output base name
+function write_gypi_header {
   echo "{" >> $1
 }
 
-# Print gypi boilerplate footer
+# Print gypi boilerplate footer.
 # $1 - Output base name
 function write_gypi_footer {
   echo "}" >> $1
 }
 
 # Generate a gypi with a list of source files.
 # $1 - Array name for file list. This is processed with 'declare' below to
 #      regenerate the array locally.
 # $2 - Output file
-function write_file_list {
+function write_gypi {
   # Convert the first argument back in to an array.
   declare -a file_list=("${!1}")
 
+  rm -rf $2

[Tom Finegan, 2014/09/04 01:59:05]

quote

Here and throughout (where appropriate).

https://google-styleguide.googlecode.com/svn/trunk/shell.xml#Quoting

[Alpha Left Google, 2014/09/04 23:09:48]

Done.

+  write_license $2
   write_gypi_header $2
 
   echo "  'sources': [" >> $2
   for f in $file_list
   do
     echo "    '<(libvpx_source)/$f'," >> $2
   done
   echo "  ]," >> $2
 
   write_gypi_footer $2
 }
 
+# Generate a gni with a list of source files.
+# $1 - Array name for file list. This is processed with 'declare' below to
+#      regenerate the array locally.
+# $2 - GN variable name.
+# $3 - Output file.
+function write_gni {
+  # Convert the first argument back in to an array.
+  declare -a file_list=("${!1}")

[Tom Finegan, 2014/09/04 01:59:05]

I would just "local readonly" this and drop the declare -a. The () are taking
care of what declare is doing.

Also... Is this really what you want? You have an array with one element
(because of the quoting). The four loop is perfectly happy operating on the
strings in ${file_list[0]}... I don't think you even need the array.

Maybe I'm confused. Bash is hard. But:

# Quoted - 1 element
$ strings="one two three"
$ array_of_strings=("${strings}")
$ echo ${#array_of_strings[@]}
1
$ echo ${array_of_strings[@]}
one two three
$ echo ${array_of_strings[0]}
one two three

# Unquoted - N elements, 3 here.
$ strings="one two three"
$ array_of_strings=(${strings})
$ echo ${#array_of_strings[@]}
3
$ echo ${array_of_strings[@]}
one two three
$ echo ${array_of_strings[0]}
one

[Alpha Left Google, 2014/09/04 23:09:48]

Done.

+
+  echo $2 "= [" >> $3
+  for f in $file_list

[Tom Finegan, 2014/09/04 01:59:05]

If you remove the quoting above: s/$file_list/${file_list[@]}

[Alpha Left Google, 2014/09/04 23:09:48]

Done.

+  do
+    echo "  \"//third_party/libvpx/source/libvpx/$f\"," >> $3
+  done
+  echo "]" >> $3
+}
+
 # Target template function
 # $1 - Array name for file list.
 # $2 - Output file
 # $3 - Target name
 # $4 - Compiler flag
 function write_target_definition {
   declare -a sources_list=("${!1}")
 
   echo "    {" >> $2
   echo "      'target_name': '$3'," >> $2
@@ skipping 41 matching lines @@
   echo "      ]," >> $2
   fi
   echo "    }," >> $2
 }
 
 
 # Generate a gypi which applies additional compiler flags based on the file
 # name.
 # $1 - Array name for file list.
 # $2 - Output file
-function write_special_flags {
+function write_intrinsics_gypi {
   declare -a file_list=("${!1}")
 
   local mmx_sources=$(echo "$file_list" | grep '_mmx\.c$')
   local sse2_sources=$(echo "$file_list" | grep '_sse2\.c$')
   local sse3_sources=$(echo "$file_list" | grep '_sse3\.c$')
   local ssse3_sources=$(echo "$file_list" | grep '_ssse3\.c$')
   local sse4_1_sources=$(echo "$file_list" | grep '_sse4\.c$')
   local avx_sources=$(echo "$file_list" | grep '_avx\.c$')
   local avx2_sources=$(echo "$file_list" | grep '_avx2\.c$')
-
   local neon_sources=$(echo "$file_list" | grep '_neon\.c$')
 
   # Intrinsic functions and files are in flux. We can selectively generate them
   # but we can not selectively include them in libvpx.gyp. Throw some errors
   # when new targets are needed.
 
+  rm -rf $2
+  write_license $2
   write_gypi_header $2
 
   echo "  'targets': [" >> $2
 
   # x86[_64]
   if [ 0 -ne ${#mmx_sources} ]; then
     write_target_definition mmx_sources[@] $2 libvpx_intrinsics_mmx mmx
   fi
   if [ 0 -ne ${#sse2_sources} ]; then
     write_target_definition sse2_sources[@] $2 libvpx_intrinsics_sse2 sse2
@@ skipping 23 matching lines @@
   # arm neon
   if [ 0 -ne ${#neon_sources} ]; then
     write_target_definition neon_sources[@] $2 libvpx_intrinsics_neon fpu=neon
   fi
 
   echo "  ]," >> $2
 
   write_gypi_footer $2
 }
 
-# Convert a list of source files into gypi file.
+# Convert a list of source files into gypi and gni files.
 # $1 - Input file.
 # $2 - Output gypi file base. Will generate additional .gypi files when
 #      different compilation flags are required.
-function convert_srcs_to_gypi {
+function convert_srcs_to_project_files {
   # Do the following here:
   # 1. Filter .c, .h, .s, .S and .asm files.
   # 2. Move certain files to a separate include to allow applying different
   #    compiler options.
   # 3. Replace .asm.s to .asm because gyp will do the conversion.
 
   local source_list=$(grep -E '(\.c|\.h|\.S|\.s|\.asm)$' $1)
 
   # _offsets are used in pre-processing to generate files for assembly. They are
   # not part of the compiled library.
@@ skipping 15 matching lines @@
     # Select all arm neon files ending in _neon.c
     # the pattern may need to be updated if vpx_scale gets intrinics
     local intrinsic_list=$(echo "$source_list" | \
       egrep 'vp[89]/(encoder|decoder|common)/arm/neon/'  | \
       egrep '_neon.c$')
   fi
 
   # Remove these files from the main list.
   source_list=$(comm -23 <(echo "$source_list") <(echo "$intrinsic_list"))
 
-  write_file_list source_list $BASE_DIR/$2.gypi
+  local x86_list=$(echo "$source_list" | egrep '/x86/')
+
+  write_gypi source_list $BASE_DIR/$2.gypi

[Tom Finegan, 2014/09/04 01:59:05]

Looks like this ship sailed a looooong time ago, but:

Shouldn't $BASE_DIR be quoted, like, everywhere? It's just `pwd`, and there's a
chance that's going to have spaces. Is the reasoning here something along the
lines of, "This is FAR from the first thing that's going to break if someone
tries to build chromium in directory that produces a path string that contains
spaces."?

[Alpha Left Google, 2014/09/04 23:09:48]

The reasoning is: there's less than 5 people in the world who will actually roll
libvpx for libvpx and who is doing it is supposed to fix the script (if it
breaks). This is really just a helper script that makes life easier for rolling
libvpx. :)

 
   # All the files are in a single "element." Check if the first element has
   # length 0.
   if [ 0 -ne ${#intrinsic_list} ]; then
-    write_special_flags intrinsic_list[@] $BASE_DIR/$2_intrinsics.gypi
+    write_intrinsics_gypi intrinsic_list[@] $BASE_DIR/$2_intrinsics.gypi
   fi
 
+  # Write a single .gni file that includes all source files for all archs.
+  if [ 0 -ne ${#x86_list} ]; then
+    # X86 systems need to separate C and assembly files for GN.
+    local c_sources=$(echo "$source_list" | egrep '.(c|h)$')
+    local assembly_sources=$(echo "$source_list" | egrep '.asm$')
+    local mmx_sources=$(echo "$intrinsic_list" | grep '_mmx\.c$')
+    local sse2_sources=$(echo "$intrinsic_list" | grep '_sse2\.c$')
+    local sse3_sources=$(echo "$intrinsic_list" | grep '_sse3\.c$')
+    local ssse3_sources=$(echo "$intrinsic_list" | grep '_ssse3\.c$')
+    local sse4_1_sources=$(echo "$intrinsic_list" | grep '_sse4\.c$')
+    local avx_sources=$(echo "$intrinsic_list" | grep '_avx\.c$')
+    local avx2_sources=$(echo "$intrinsic_list" | grep '_avx2\.c$')
+
+    write_gni c_sources $2 $BASE_DIR/libvpx_srcs.gni
+    write_gni assembly_sources $2_assembly $BASE_DIR/libvpx_srcs.gni
+    write_gni mmx_sources $2_mmx $BASE_DIR/libvpx_srcs.gni
+    write_gni sse2_sources $2_sse2 $BASE_DIR/libvpx_srcs.gni
+    write_gni sse3_sources $2_sse3 $BASE_DIR/libvpx_srcs.gni
+    write_gni ssse3_sources $2_ssse3 $BASE_DIR/libvpx_srcs.gni
+    write_gni sse4_1_sources $2_sse4_1 $BASE_DIR/libvpx_srcs.gni
+    write_gni avx_sources $2_avx $BASE_DIR/libvpx_srcs.gni
+    write_gni avx2_sources $2_avx2 $BASE_DIR/libvpx_srcs.gni
+  else
+    local neon_sources=$(echo "$intrinsic_list" | grep '_neon\.c$')
+    write_gni source_list $2 $BASE_DIR/libvpx_srcs.gni
+    if [ 0 -ne ${#neon_sources} ]; then
+      write_gni neon_sources $2_neon $BASE_DIR/libvpx_srcs.gni
+    fi
+  fi
 }
 
 # Clean files from previous make.
 function make_clean {
   make clean > /dev/null
   rm -f libvpx_srcs.txt
 }
 
 # Lint a pair of vpx_config.h and vpx_config.asm to make sure they match.
 # $1 - Header file directory.
@@ skipping 76 matching lines @@
 # Generate Config files. "--enable-external-build" must be set to skip
 # detection of capabilities on specific targets.
 # $1 - Header file directory.
 # $2 - Config command line.
 function gen_config_files {
   ./configure $2  > /dev/null
 
   # Generate vpx_config.asm. Do not create one for mips.
   if [[ "$1" != *mipsel ]]; then
     if [[ "$1" == *x64* ]] || [[ "$1" == *ia32* ]]; then
-      egrep "#define [A-Z0-9_]+ [01]" vpx_config.h | awk '{print $2 " equ " $3}' > vpx_config.asm
+      egrep "#define [A-Z0-9_]+ [01]" vpx_config.h | awk '{print "%define " $2 " " $3}' > vpx_config.asm
     else
       egrep "#define [A-Z0-9_]+ [01]" vpx_config.h | awk '{print $2 " EQU " $3}' | perl $BASE_DIR/$LIBVPX_SRC_DIR/build/make/ads2gas.pl > vpx_config.asm
     fi
   fi
 
   cp vpx_config.* $BASE_DIR/$LIBVPX_CONFIG_DIR/$1
   make_clean
   rm -rf vpx_config.*
 }
 
 echo "Create temporary directory."
 TEMP_DIR="$LIBVPX_SRC_DIR.temp"
 rm -rf $TEMP_DIR
 cp -R $LIBVPX_SRC_DIR $TEMP_DIR
 cd $TEMP_DIR
 
-echo "Generate Config Files"
+echo "Generate config files."
 # TODO(joeyparrish) Enable AVX2 when broader VS2013 support is available
 all_platforms="--enable-external-build --enable-postproc --disable-install-srcs --enable-multi-res-encoding --enable-temporal-denoising --disable-unit-tests --disable-install-docs --disable-examples --disable-avx2"
 gen_config_files linux/ia32 "--target=x86-linux-gcc --disable-ccache --enable-pic --enable-realtime-only ${all_platforms}"
 gen_config_files linux/x64 "--target=x86_64-linux-gcc --disable-ccache --enable-pic --enable-realtime-only ${all_platforms}"
 gen_config_files linux/arm "--target=armv6-linux-gcc --enable-pic --enable-realtime-only --disable-install-bins --disable-install-libs --disable-edsp ${all_platforms}"
 gen_config_files linux/arm-neon "--target=armv7-linux-gcc --enable-pic --enable-realtime-only --disable-edsp ${all_platforms}"
 gen_config_files linux/arm-neon-cpu-detect "--target=armv7-linux-gcc --enable-pic --enable-realtime-only --enable-runtime-cpu-detect --disable-edsp ${all_platforms}"
 gen_config_files linux/arm64 "--force-target=armv8-linux-gcc --enable-pic --enable-realtime-only --disable-edsp ${all_platforms}"
 gen_config_files linux/mipsel "--target=mips32-linux-gcc --disable-fast-unaligned ${all_platforms}"
 gen_config_files linux/generic "--target=generic-gnu --enable-pic --enable-realtime-only ${all_platforms}"
@@ skipping 39 matching lines @@
 gen_rtcd_header win/ia32 x86
 gen_rtcd_header win/x64 x86_64
 gen_rtcd_header mac/ia32 x86
 gen_rtcd_header mac/x64 x86_64
 gen_rtcd_header nacl nacl
 
 echo "Prepare Makefile."
 ./configure --target=generic-gnu > /dev/null
 make_clean
 
+# Remove existing .gni file.
+rm -rf $BASE_DIR/libvpx_srcs.gni
+write_license $BASE_DIR/libvpx_srcs.gni
+
 echo "Generate X86 source list."
 config=$(print_config linux/ia32)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_x86
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_x86
 
 # Copy vpx_version.h. The file should be the same for all platforms.
 cp vpx_version.h $BASE_DIR/$LIBVPX_CONFIG_DIR
 
 echo "Generate X86_64 source list."
 config=$(print_config linux/x64)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_x86_64
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_x86_64
 
 echo "Generate ARM source list."
 config=$(print_config linux/arm)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_arm
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_arm
 
 echo "Generate ARM NEON source list."
 config=$(print_config linux/arm-neon)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_arm_neon
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_arm_neon
 
 echo "Generate ARM NEON CPU DETECT source list."
 config=$(print_config linux/arm-neon-cpu-detect)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_arm_neon_cpu_detect
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_arm_neon_cpu_detect
 
 echo "Generate ARM64 source list."
 config=$(print_config linux/arm64)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_arm64
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_arm64
 
 echo "Generate MIPS source list."
 config=$(print_config_basic linux/mipsel)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_mips
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_mips
 
 echo "Generate NaCl source list."
 config=$(print_config_basic nacl)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_nacl
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_nacl
 
 echo "Generate GENERIC source list."
 config=$(print_config_basic linux/generic)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_generic
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_generic
 
 echo "Remove temporary directory."
 cd $BASE_DIR
 rm -rf $TEMP_DIR
 
 # TODO(fgalligan): Is "--disable-fast-unaligned" needed on mipsel?
 # TODO(fgalligan): Can we turn on "--enable-realtime-only" for mipsel?