GN: Build libvpx on POSIX x86

generate_gypi.sh

Index: generate_gypi.sh
diff --git a/generate_gypi.sh b/generate_gypi.sh
index 7b44dd1cdf8a8f102c06a0e05d3394510f883026..da866631ff52e1e40d9322a16cc51767725ec5cb 100755
--- a/generate_gypi.sh
+++ b/generate_gypi.sh
@@ -4,14 +4,14 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# This script is used to generate .gypi files and files in the config/platform
-# directories needed to build libvpx.
+# This script is used to generate .gypi, .gni files and files in the
+# config/platform directories needed to build libvpx.
 # Every time libvpx source code is updated just run this script.
 #
 # For example:
 # $ ./generate_gypi.sh
 #
-# And this will update all the .gypi and config files needed.
+# And this will update all the .gypi, .gni and config files needed.
 #
 # !!! It's highly recommended to install yasm before running this script.
 
@@ -20,18 +20,23 @@ BASE_DIR=`pwd`
 LIBVPX_SRC_DIR="source/libvpx"
 LIBVPX_CONFIG_DIR="source/config"
 
-# Print gypi boilerplate header
+# Print license header.
 # $1 - Output base name
-function write_gypi_header {
-  echo "# This file is generated. Do not edit." > $1
+function write_license {
+  echo "# This file is generated. Do not edit." >> $1
   echo "# Copyright (c) 2013 The Chromium Authors. All rights reserved." >> $1

[fgalligan1, 2014/09/04 16:44:46]

2014?

[Alpha Left Google, 2014/09/04 23:09:48]

Done.

   echo "# Use of this source code is governed by a BSD-style license that can be" >> $1
   echo "# found in the LICENSE file." >> $1
   echo "" >> $1
+}
+
+# Print gypi boilerplate header.
+# $1 - Output base name
+function write_gypi_header {
   echo "{" >> $1
 }
 
-# Print gypi boilerplate footer
+# Print gypi boilerplate footer.
 # $1 - Output base name
 function write_gypi_footer {
   echo "}" >> $1
@@ -41,10 +46,12 @@ function write_gypi_footer {
 # $1 - Array name for file list. This is processed with 'declare' below to
 #      regenerate the array locally.
 # $2 - Output file
-function write_file_list {
+function write_gypi {
   # Convert the first argument back in to an array.
   declare -a file_list=("${!1}")
 
+  rm -rf $2

[Tom Finegan, 2014/09/04 01:59:05]

quote

Here and throughout (where appropriate).

https://google-styleguide.googlecode.com/svn/trunk/shell.xml#Quoting

[Alpha Left Google, 2014/09/04 23:09:48]

Done.

+  write_license $2
   write_gypi_header $2
 
   echo "  'sources': [" >> $2
@@ -57,6 +64,23 @@ function write_file_list {
   write_gypi_footer $2
 }
 
+# Generate a gni with a list of source files.
+# $1 - Array name for file list. This is processed with 'declare' below to
+#      regenerate the array locally.
+# $2 - GN variable name.
+# $3 - Output file.
+function write_gni {
+  # Convert the first argument back in to an array.
+  declare -a file_list=("${!1}")

[Tom Finegan, 2014/09/04 01:59:05]

I would just "local readonly" this and drop the declare -a. The () are taking
care of what declare is doing.

Also... Is this really what you want? You have an array with one element
(because of the quoting). The four loop is perfectly happy operating on the
strings in ${file_list[0]}... I don't think you even need the array.

Maybe I'm confused. Bash is hard. But:

# Quoted - 1 element
$ strings="one two three"
$ array_of_strings=("${strings}")
$ echo ${#array_of_strings[@]}
1
$ echo ${array_of_strings[@]}
one two three
$ echo ${array_of_strings[0]}
one two three

# Unquoted - N elements, 3 here.
$ strings="one two three"
$ array_of_strings=(${strings})
$ echo ${#array_of_strings[@]}
3
$ echo ${array_of_strings[@]}
one two three
$ echo ${array_of_strings[0]}
one

[Alpha Left Google, 2014/09/04 23:09:48]

Done.

+
+  echo $2 "= [" >> $3
+  for f in $file_list

[Tom Finegan, 2014/09/04 01:59:05]

If you remove the quoting above: s/$file_list/${file_list[@]}

[Alpha Left Google, 2014/09/04 23:09:48]

Done.

+  do
+    echo "  \"//third_party/libvpx/source/libvpx/$f\"," >> $3
+  done
+  echo "]" >> $3
+}
+
 # Target template function
 # $1 - Array name for file list.
 # $2 - Output file
@@ -118,7 +142,7 @@ function write_target_definition {
 # name.
 # $1 - Array name for file list.
 # $2 - Output file
-function write_special_flags {
+function write_intrinsics_gypi {
   declare -a file_list=("${!1}")
 
   local mmx_sources=$(echo "$file_list" | grep '_mmx\.c$')
@@ -128,13 +152,14 @@ function write_special_flags {
   local sse4_1_sources=$(echo "$file_list" | grep '_sse4\.c$')
   local avx_sources=$(echo "$file_list" | grep '_avx\.c$')
   local avx2_sources=$(echo "$file_list" | grep '_avx2\.c$')
-
   local neon_sources=$(echo "$file_list" | grep '_neon\.c$')
 
   # Intrinsic functions and files are in flux. We can selectively generate them
   # but we can not selectively include them in libvpx.gyp. Throw some errors
   # when new targets are needed.
 
+  rm -rf $2
+  write_license $2
   write_gypi_header $2
 
   echo "  'targets': [" >> $2
@@ -178,11 +203,11 @@ function write_special_flags {
   write_gypi_footer $2
 }
 
-# Convert a list of source files into gypi file.
+# Convert a list of source files into gypi and gni files.
 # $1 - Input file.
 # $2 - Output gypi file base. Will generate additional .gypi files when
 #      different compilation flags are required.
-function convert_srcs_to_gypi {
+function convert_srcs_to_project_files {
   # Do the following here:
   # 1. Filter .c, .h, .s, .S and .asm files.
   # 2. Move certain files to a separate include to allow applying different
@@ -218,14 +243,45 @@ function convert_srcs_to_gypi {
   # Remove these files from the main list.
   source_list=$(comm -23 <(echo "$source_list") <(echo "$intrinsic_list"))
 
-  write_file_list source_list $BASE_DIR/$2.gypi
+  local x86_list=$(echo "$source_list" | egrep '/x86/')
+
+  write_gypi source_list $BASE_DIR/$2.gypi

[Tom Finegan, 2014/09/04 01:59:05]

Looks like this ship sailed a looooong time ago, but:

Shouldn't $BASE_DIR be quoted, like, everywhere? It's just `pwd`, and there's a
chance that's going to have spaces. Is the reasoning here something along the
lines of, "This is FAR from the first thing that's going to break if someone
tries to build chromium in directory that produces a path string that contains
spaces."?

[Alpha Left Google, 2014/09/04 23:09:48]

The reasoning is: there's less than 5 people in the world who will actually roll
libvpx for libvpx and who is doing it is supposed to fix the script (if it
breaks). This is really just a helper script that makes life easier for rolling
libvpx. :)

 
   # All the files are in a single "element." Check if the first element has
   # length 0.
   if [ 0 -ne ${#intrinsic_list} ]; then
-    write_special_flags intrinsic_list[@] $BASE_DIR/$2_intrinsics.gypi
+    write_intrinsics_gypi intrinsic_list[@] $BASE_DIR/$2_intrinsics.gypi
   fi
 
+  # Write a single .gni file that includes all source files for all archs.
+  if [ 0 -ne ${#x86_list} ]; then
+    # X86 systems need to separate C and assembly files for GN.
+    local c_sources=$(echo "$source_list" | egrep '.(c|h)$')
+    local assembly_sources=$(echo "$source_list" | egrep '.asm$')
+    local mmx_sources=$(echo "$intrinsic_list" | grep '_mmx\.c$')
+    local sse2_sources=$(echo "$intrinsic_list" | grep '_sse2\.c$')
+    local sse3_sources=$(echo "$intrinsic_list" | grep '_sse3\.c$')
+    local ssse3_sources=$(echo "$intrinsic_list" | grep '_ssse3\.c$')
+    local sse4_1_sources=$(echo "$intrinsic_list" | grep '_sse4\.c$')
+    local avx_sources=$(echo "$intrinsic_list" | grep '_avx\.c$')
+    local avx2_sources=$(echo "$intrinsic_list" | grep '_avx2\.c$')
+
+    write_gni c_sources $2 $BASE_DIR/libvpx_srcs.gni
+    write_gni assembly_sources $2_assembly $BASE_DIR/libvpx_srcs.gni
+    write_gni mmx_sources $2_mmx $BASE_DIR/libvpx_srcs.gni
+    write_gni sse2_sources $2_sse2 $BASE_DIR/libvpx_srcs.gni
+    write_gni sse3_sources $2_sse3 $BASE_DIR/libvpx_srcs.gni
+    write_gni ssse3_sources $2_ssse3 $BASE_DIR/libvpx_srcs.gni
+    write_gni sse4_1_sources $2_sse4_1 $BASE_DIR/libvpx_srcs.gni
+    write_gni avx_sources $2_avx $BASE_DIR/libvpx_srcs.gni
+    write_gni avx2_sources $2_avx2 $BASE_DIR/libvpx_srcs.gni
+  else
+    local neon_sources=$(echo "$intrinsic_list" | grep '_neon\.c$')
+    write_gni source_list $2 $BASE_DIR/libvpx_srcs.gni
+    if [ 0 -ne ${#neon_sources} ]; then
+      write_gni neon_sources $2_neon $BASE_DIR/libvpx_srcs.gni
+    fi
+  fi
 }
 
 # Clean files from previous make.
@@ -322,7 +378,7 @@ function gen_config_files {
   # Generate vpx_config.asm. Do not create one for mips.
   if [[ "$1" != *mipsel ]]; then
     if [[ "$1" == *x64* ]] || [[ "$1" == *ia32* ]]; then
-      egrep "#define [A-Z0-9_]+ [01]" vpx_config.h | awk '{print $2 " equ " $3}' > vpx_config.asm
+      egrep "#define [A-Z0-9_]+ [01]" vpx_config.h | awk '{print "%define " $2 " " $3}' > vpx_config.asm
     else
       egrep "#define [A-Z0-9_]+ [01]" vpx_config.h | awk '{print $2 " EQU " $3}' | perl $BASE_DIR/$LIBVPX_SRC_DIR/build/make/ads2gas.pl > vpx_config.asm
     fi
@@ -339,7 +395,7 @@ rm -rf $TEMP_DIR
 cp -R $LIBVPX_SRC_DIR $TEMP_DIR
 cd $TEMP_DIR
 
-echo "Generate Config Files"
+echo "Generate config files."
 # TODO(joeyparrish) Enable AVX2 when broader VS2013 support is available
 all_platforms="--enable-external-build --enable-postproc --disable-install-srcs --enable-multi-res-encoding --enable-temporal-denoising --disable-unit-tests --disable-install-docs --disable-examples --disable-avx2"
 gen_config_files linux/ia32 "--target=x86-linux-gcc --disable-ccache --enable-pic --enable-realtime-only ${all_platforms}"
@@ -399,11 +455,15 @@ echo "Prepare Makefile."
 ./configure --target=generic-gnu > /dev/null
 make_clean
 
+# Remove existing .gni file.
+rm -rf $BASE_DIR/libvpx_srcs.gni
+write_license $BASE_DIR/libvpx_srcs.gni
+
 echo "Generate X86 source list."
 config=$(print_config linux/ia32)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_x86
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_x86
 
 # Copy vpx_version.h. The file should be the same for all platforms.
 cp vpx_version.h $BASE_DIR/$LIBVPX_CONFIG_DIR
@@ -412,49 +472,49 @@ echo "Generate X86_64 source list."
 config=$(print_config linux/x64)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_x86_64
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_x86_64
 
 echo "Generate ARM source list."
 config=$(print_config linux/arm)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_arm
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_arm
 
 echo "Generate ARM NEON source list."
 config=$(print_config linux/arm-neon)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_arm_neon
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_arm_neon
 
 echo "Generate ARM NEON CPU DETECT source list."
 config=$(print_config linux/arm-neon-cpu-detect)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_arm_neon_cpu_detect
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_arm_neon_cpu_detect
 
 echo "Generate ARM64 source list."
 config=$(print_config linux/arm64)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_arm64
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_arm64
 
 echo "Generate MIPS source list."
 config=$(print_config_basic linux/mipsel)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_mips
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_mips
 
 echo "Generate NaCl source list."
 config=$(print_config_basic nacl)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_nacl
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_nacl
 
 echo "Generate GENERIC source list."
 config=$(print_config_basic linux/generic)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_generic
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_generic
 
 echo "Remove temporary directory."
 cd $BASE_DIR