Store each file as "hash/<sha1>", list of hashes as "meta/@<rev>" and serve them at "serve_file/@<r…

gce/uploader_iteration.sh

 #!/bin/bash
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # This enables bash to treat any unhandled non-zero exit codes as "exceptions"
 set -e
 source $(dirname $0)/uploader_exit_codes.sh
 
 mark_step() {
@@ skipping 44 matching lines @@
 fi
 
 mark_step 'Building DevTools frontend for revisions and uploading to Cloud Storage'
 for REVISION in $REVISION_LIST; do
     LOCAL_STORAGE_DIR=/uploader/storage
     if [ -d $LOCAL_STORAGE_DIR ]; then
         rm -rf $LOCAL_STORAGE_DIR
     fi
     ZIPS_DIR="$LOCAL_STORAGE_DIR/zips"
     REVS_DIR="$LOCAL_STORAGE_DIR/revs"
-    mkdir -p $ZIPS_DIR $REVS_DIR
+    META_DIR="$LOCAL_STORAGE_DIR/meta"
+    HASH_DIR="$LOCAL_STORAGE_DIR/hash"
+    mkdir -p $ZIPS_DIR $REVS_DIR $META_DIR $HASH_DIR
 
     if echo "$BLINK_REVISIONS_BLACKLIST" | grep -w $REVISION; then
         echo "Revision $REVISION is in the blacklist, skipping over"
         continue
     fi
 
     pushd blink
     REVISION_COMMIT=$(git log -n 1 --all --grep="git-svn-id:[^@]*@$REVISION" --pretty=oneline | awk '{ print $1; }')
     if [ -z "$REVISION_COMMIT" ]; then
         echo "@$REVISION ==> missing!"
@@ skipping 43 matching lines @@
     fi
 
     cat <<EOF > $MANIFEST_FILE_NAME
 CACHE MANIFEST
 FALLBACK:
 EOF
     for FALLBACK_ENTRY in $FALLBACK_ENTRIES; do
         echo "$FALLBACK_ENTRY $FALLBACK_ENTRY" >> $MANIFEST_FILE_NAME
     done
     echo "CACHE:" >> $MANIFEST_FILE_NAME
     find -type f -not -name $MANIFEST_FILE_NAME -print | sed -e 's#^\./##' >> $MANIFEST_FILE_NAME

[mnaganov (inactive), 2015/02/24 14:40:32]

Perhaps, we can store this list in a variable and then reuse it in the new code
to avoid duplication?

[dgozman, 2015/02/25 11:37:13]

I've added manifest file there, so it's different now.

 
+    # Copy each file to hash/<sha1>, write <sha1>:<path> to meta file.
+    META_FILE_NAME="$META_DIR/@$REVISION"

[mnaganov (inactive), 2015/02/24 14:40:32]

I suggest to move this block after creating the zip file (but before popd).
Otherwise, this creates a false impression that those files also end up in the
archive.

[dgozman, 2015/02/25 11:37:14]

Done.

+    for FILE_NAME in $(find -type f -not -name $MANIFEST_FILE_NAME); do
+        FILE_SHA1=$(sha1sum $FILE_NAME | awk '{print $1}')
+        cp $FILE_NAME "$HASH_DIR/$FILE_SHA1"
+        echo "$FILE_SHA1:${FILE_NAME##./}" >> $META_FILE_NAME
+    done;
+
     zip -9r $ZIP_FILE * || exit $EXIT_ZIP_FRONTEND
     popd
 
     ZIP_SHA1=$(sha1sum $ZIP_FILE | awk '{print $1}')
     mv $ZIP_FILE "$ZIPS_DIR/$ZIP_SHA1.zip"
     echo "$ZIP_SHA1" > "$REVS_DIR/@$REVISION"
 
     # Comment out the line below for testing.
     gsutil -m cp -R $LOCAL_STORAGE_DIR/* $GS_PATH || exit $EXIT_CS_UPLOAD
 done