GN: Build libvpx on POSIX x86

generate_gypi.sh

 #!/bin/bash -e
 #
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# This script is used to generate .gypi files and files in the config/platform
-# directories needed to build libvpx.
+# This script is used to generate .gypi, .gni files and files in the
+# config/platform directories needed to build libvpx.
 # Every time libvpx source code is updated just run this script.
 #
 # For example:
 # $ ./generate_gypi.sh
 #
-# And this will update all the .gypi and config files needed.
+# And this will update all the .gypi, .gni and config files needed.
 #
 # !!! It's highly recommended to install yasm before running this script.
 
 export LC_ALL=C
 BASE_DIR=`pwd`
 LIBVPX_SRC_DIR="source/libvpx"
 LIBVPX_CONFIG_DIR="source/config"
 
-# Print gypi boilerplate header
+# Print license header.
 # $1 - Output base name
-function write_gypi_header {
-  echo "# This file is generated. Do not edit." > $1
+function write_license {
+  echo "# This file is generated. Do not edit." >> $1
   echo "# Copyright (c) 2013 The Chromium Authors. All rights reserved." >> $1
   echo "# Use of this source code is governed by a BSD-style license that can be" >> $1
   echo "# found in the LICENSE file." >> $1
   echo "" >> $1
+}
+
+# Print gypi boilerplate header.
+# $1 - Output base name
+function write_gypi_header {
   echo "{" >> $1
 }
 
-# Print gypi boilerplate footer
+# Print gypi boilerplate footer.
 # $1 - Output base name
 function write_gypi_footer {
   echo "}" >> $1
 }
 
 # Generate a gypi with a list of source files.
 # $1 - Array name for file list. This is processed with 'declare' below to
 #      regenerate the array locally.
 # $2 - Output file
-function write_file_list {
+function write_gypi {
   # Convert the first argument back in to an array.
   declare -a file_list=("${!1}")
 
+  rm -rf $2
+  write_license $2
   write_gypi_header $2
 
   echo "  'sources': [" >> $2
   for f in $file_list
   do
     echo "    '<(libvpx_source)/$f'," >> $2
   done
   echo "  ]," >> $2
 
   write_gypi_footer $2
 }
 
+# Generate a gni with a list of source files.
+# $1 - Array name for file list. This is processed with 'declare' below to
+#      regenerate the array locally.
+# $2 - GN variable name.
+# $3 - Output file.
+function write_gni {
+  # Convert the first argument back in to an array.
+  declare -a file_list=("${!1}")
+
+  echo $2 "= [" >> $3
+  for f in $file_list
+  do
+    echo "  \"//third_party/libvpx/source/libvpx/$f\"," >> $3
+  done
+  echo "]" >> $3
+}
+
 # Target template function
 # $1 - Array name for file list.
 # $2 - Output file
 # $3 - Target name
 # $4 - Compiler flag
 function write_target_definition {
   declare -a sources_list=("${!1}")
 
   echo "    {" >> $2
   echo "      'target_name': '$3'," >> $2
@@ skipping 41 matching lines @@
   echo "      ]," >> $2
   fi
   echo "    }," >> $2
 }
 
 
 # Generate a gypi which applies additional compiler flags based on the file
 # name.
 # $1 - Array name for file list.
 # $2 - Output file
-function write_special_flags {
+function write_intrinsics_gypi {
   declare -a file_list=("${!1}")
 
   local mmx_sources=$(echo "$file_list" | grep '_mmx\.c$')
   local sse2_sources=$(echo "$file_list" | grep '_sse2\.c$')
   local sse3_sources=$(echo "$file_list" | grep '_sse3\.c$')
   local ssse3_sources=$(echo "$file_list" | grep '_ssse3\.c$')
   local sse4_1_sources=$(echo "$file_list" | grep '_sse4\.c$')
   local avx_sources=$(echo "$file_list" | grep '_avx\.c$')
   local avx2_sources=$(echo "$file_list" | grep '_avx2\.c$')
-
   local neon_sources=$(echo "$file_list" | grep '_neon\.c$')
 
   # Intrinsic functions and files are in flux. We can selectively generate them
   # but we can not selectively include them in libvpx.gyp. Throw some errors
   # when new targets are needed.
 
+  rm -rf $2
+  write_license $2
   write_gypi_header $2
 
   echo "  'targets': [" >> $2
 
   # x86[_64]
   if [ 0 -ne ${#mmx_sources} ]; then
     write_target_definition mmx_sources[@] $2 libvpx_intrinsics_mmx mmx
   fi
   if [ 0 -ne ${#sse2_sources} ]; then
     write_target_definition sse2_sources[@] $2 libvpx_intrinsics_sse2 sse2
@@ skipping 23 matching lines @@
   # arm neon
   if [ 0 -ne ${#neon_sources} ]; then
     write_target_definition neon_sources[@] $2 libvpx_intrinsics_neon fpu=neon
   fi
 
   echo "  ]," >> $2
 
   write_gypi_footer $2
 }
 
-# Convert a list of source files into gypi file.
+# Convert a list of source files into gypi and gni files.
 # $1 - Input file.
 # $2 - Output gypi file base. Will generate additional .gypi files when
 #      different compilation flags are required.
-function convert_srcs_to_gypi {
+function convert_srcs_to_project_files {
   # Do the following here:
   # 1. Filter .c, .h, .s, .S and .asm files.
   # 2. Move certain files to a separate include to allow applying different
   #    compiler options.
   # 3. Replace .asm.s to .asm because gyp will do the conversion.
 
   local source_list=$(grep -E '(\.c|\.h|\.S|\.s|\.asm)$' $1)
 
   # _offsets are used in pre-processing to generate files for assembly. They are
   # not part of the compiled library.
@@ skipping 15 matching lines @@
     # Select all arm neon files ending in _neon.c
     # the pattern may need to be updated if vpx_scale gets intrinics
     local intrinsic_list=$(echo "$source_list" | \
       egrep 'vp[89]/(encoder|decoder|common)/arm/neon/'  | \
       egrep '_neon.c$')
   fi
 
   # Remove these files from the main list.
   source_list=$(comm -23 <(echo "$source_list") <(echo "$intrinsic_list"))
 
-  write_file_list source_list $BASE_DIR/$2.gypi
+  local x86_list=$(echo "$source_list" | egrep '/x86/')
+
+  write_gypi source_list $BASE_DIR/$2.gypi
 
   # All the files are in a single "element." Check if the first element has
   # length 0.
   if [ 0 -ne ${#intrinsic_list} ]; then
-    write_special_flags intrinsic_list[@] $BASE_DIR/$2_intrinsics.gypi
+    write_intrinsics_gypi intrinsic_list[@] $BASE_DIR/$2_intrinsics.gypi
   fi
 
+  # Write a single .gni file that includes all source files for all archs.
+  if [ 0 -ne ${#x86_list} ]; then
+    # X86 systems need to separate C and assembly files for GN.
+    local c_sources=$(echo "$source_list" | egrep '.(c|h)$')
+    local assembly_sources=$(echo "$source_list" | egrep '.asm$')
+    local mmx_sources=$(echo "$intrinsic_list" | grep '_mmx\.c$')
+    local sse2_sources=$(echo "$intrinsic_list" | grep '_sse2\.c$')
+    local sse3_sources=$(echo "$intrinsic_list" | grep '_sse3\.c$')
+    local ssse3_sources=$(echo "$intrinsic_list" | grep '_ssse3\.c$')
+    local sse4_1_sources=$(echo "$intrinsic_list" | grep '_sse4\.c$')
+    local avx_sources=$(echo "$intrinsic_list" | grep '_avx\.c$')
+    local avx2_sources=$(echo "$intrinsic_list" | grep '_avx2\.c$')
+
+    write_gni c_sources $2 $BASE_DIR/libvpx_srcs.gni
+    write_gni assembly_sources $2_assembly $BASE_DIR/libvpx_srcs.gni
+    write_gni mmx_sources $2_mmx $BASE_DIR/libvpx_srcs.gni
+    write_gni sse2_sources $2_sse2 $BASE_DIR/libvpx_srcs.gni
+    write_gni sse3_sources $2_sse3 $BASE_DIR/libvpx_srcs.gni
+    write_gni ssse3_sources $2_ssse3 $BASE_DIR/libvpx_srcs.gni
+    write_gni sse4_1_sources $2_sse4_1 $BASE_DIR/libvpx_srcs.gni
+    write_gni avx_sources $2_avx $BASE_DIR/libvpx_srcs.gni
+    write_gni avx2_sources $2_avx2 $BASE_DIR/libvpx_srcs.gni
+  else
+    local neon_sources=$(echo "$intrinsic_list" | grep '_neon\.c$')
+    write_gni source_list $2 $BASE_DIR/libvpx_srcs.gni
+    if [ 0 -ne ${#neon_sources} ]; then
+      write_gni neon_sources $2_neon $BASE_DIR/libvpx_srcs.gni
+    fi
+  fi
 }
 
 # Clean files from previous make.
 function make_clean {
   make clean > /dev/null
   rm -f libvpx_srcs.txt
 }
 
 # Lint a pair of vpx_config.h and vpx_config.asm to make sure they match.
 # $1 - Header file directory.
@@ skipping 76 matching lines @@
 # Generate Config files. "--enable-external-build" must be set to skip
 # detection of capabilities on specific targets.
 # $1 - Header file directory.
 # $2 - Config command line.
 function gen_config_files {
   ./configure $2  > /dev/null
 
   # Generate vpx_config.asm. Do not create one for mips.
   if [[ "$1" != *mipsel ]]; then
     if [[ "$1" == *x64* ]] || [[ "$1" == *ia32* ]]; then
-      egrep "#define [A-Z0-9_]+ [01]" vpx_config.h | awk '{print $2 " equ " $3}' > vpx_config.asm
+      egrep "#define [A-Z0-9_]+ [01]" vpx_config.h | awk '{print "%define " $2 " " $3}' > vpx_config.asm
     else
       egrep "#define [A-Z0-9_]+ [01]" vpx_config.h | awk '{print $2 " EQU " $3}' | perl $BASE_DIR/$LIBVPX_SRC_DIR/build/make/ads2gas.pl > vpx_config.asm
     fi
   fi
 
   cp vpx_config.* $BASE_DIR/$LIBVPX_CONFIG_DIR/$1
   make_clean
   rm -rf vpx_config.*
 }
 
 echo "Create temporary directory."
 TEMP_DIR="$LIBVPX_SRC_DIR.temp"
 rm -rf $TEMP_DIR
 cp -R $LIBVPX_SRC_DIR $TEMP_DIR
 cd $TEMP_DIR
 
-echo "Generate Config Files"
+echo "Generate config files."
 # TODO(joeyparrish) Enable AVX2 when broader VS2013 support is available
 all_platforms="--enable-external-build --enable-postproc --disable-install-srcs --enable-multi-res-encoding --enable-temporal-denoising --disable-unit-tests --disable-install-docs --disable-examples --disable-avx2"
 gen_config_files linux/ia32 "--target=x86-linux-gcc --disable-ccache --enable-pic --enable-realtime-only ${all_platforms}"
 gen_config_files linux/x64 "--target=x86_64-linux-gcc --disable-ccache --enable-pic --enable-realtime-only ${all_platforms}"
 gen_config_files linux/arm "--target=armv6-linux-gcc --enable-pic --enable-realtime-only --disable-install-bins --disable-install-libs --disable-edsp ${all_platforms}"
 gen_config_files linux/arm-neon "--target=armv7-linux-gcc --enable-pic --enable-realtime-only --disable-edsp ${all_platforms}"
 gen_config_files linux/arm-neon-cpu-detect "--target=armv7-linux-gcc --enable-pic --enable-realtime-only --enable-runtime-cpu-detect --disable-edsp ${all_platforms}"
 gen_config_files linux/arm64 "--force-target=armv8-linux-gcc --enable-pic --enable-realtime-only --disable-edsp ${all_platforms}"
 gen_config_files linux/mipsel "--target=mips32-linux-gcc --disable-fast-unaligned ${all_platforms}"
 gen_config_files linux/generic "--target=generic-gnu --enable-pic --enable-realtime-only ${all_platforms}"
@@ skipping 39 matching lines @@
 gen_rtcd_header win/ia32 x86
 gen_rtcd_header win/x64 x86_64
 gen_rtcd_header mac/ia32 x86
 gen_rtcd_header mac/x64 x86_64
 gen_rtcd_header nacl nacl
 
 echo "Prepare Makefile."
 ./configure --target=generic-gnu > /dev/null
 make_clean
 
+# Remove existing .gni file.
+rm -rf $BASE_DIR/libvpx_srcs.gni
+write_license $BASE_DIR/libvpx_srcs.gni
+
 echo "Generate X86 source list."
 config=$(print_config linux/ia32)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_x86
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_x86
 
 # Copy vpx_version.h. The file should be the same for all platforms.
 cp vpx_version.h $BASE_DIR/$LIBVPX_CONFIG_DIR
 
 echo "Generate X86_64 source list."
 config=$(print_config linux/x64)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_x86_64
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_x86_64
 
 echo "Generate ARM source list."
 config=$(print_config linux/arm)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_arm
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_arm
 
 echo "Generate ARM NEON source list."
 config=$(print_config linux/arm-neon)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_arm_neon
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_arm_neon
 
 echo "Generate ARM NEON CPU DETECT source list."
 config=$(print_config linux/arm-neon-cpu-detect)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_arm_neon_cpu_detect
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_arm_neon_cpu_detect
 
 echo "Generate ARM64 source list."
 config=$(print_config linux/arm64)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_arm64
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_arm64
 
 echo "Generate MIPS source list."
 config=$(print_config_basic linux/mipsel)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_mips
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_mips
 
 echo "Generate NaCl source list."
 config=$(print_config_basic nacl)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_nacl
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_nacl
 
 echo "Generate GENERIC source list."
 config=$(print_config_basic linux/generic)
 make_clean
 make libvpx_srcs.txt target=libs $config > /dev/null
-convert_srcs_to_gypi libvpx_srcs.txt libvpx_srcs_generic
+convert_srcs_to_project_files libvpx_srcs.txt libvpx_srcs_generic
 
 echo "Remove temporary directory."
 cd $BASE_DIR
 rm -rf $TEMP_DIR
 
 # TODO(fgalligan): Is "--disable-fast-unaligned" needed on mipsel?
 # TODO(fgalligan): Can we turn on "--enable-realtime-only" for mipsel?