Open Source Monorail

appengine/monorail/tools/backups/restore.sh

+#!/bin/bash
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is govered by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+# The existing replicas all have this prefix:
+REPLICA_PREFIX="replica-7"
+
+# The new replicas made from the restored master will have this prefix:
+NEW_REPLICA_PREFIX="replica-8"
+
+CLOUD_PROJECT="monorail-staging"
+
+DRY_RUN=true
+
+echo Restoring backups to master for $CLOUD_PROJECT. Dry run: $DRY_RUN
+echo This will delete all read replicas with the prefix "$REPLICA_PREFIX"
+echo and create a new set of replicas with the prefix "$NEW_REPLICA_PREFIX"
+echo
+echo Checking for existing read replicas to delete:
+
+EXISTING_REPLICAS=($(gcloud sql instances list --project=$CLOUD_PROJECT | grep $REPLICA_PREFIX- | awk '{print $1}'))
+
+if [ ${#EXISTING_REPLICAS[@]} -eq 0 ]; then
+  echo No replicas found with prefix $REPLICA_PREFIX
+  echo List instances to find the replica prefix by running:
+  echo gcloud sql instances list --project=$CLOUD_PROJECT
+  exit 1
+fi
+
+echo Deleting ${#EXISTING_REPLICAS[@]} existing replicas found with the prefix $REPLICA_PREFIX
+
+for r in "${EXISTING_REPLICAS[@]}"; do
+  echo Deleting $r
+  cmd="gcloud sql instances delete $r --project=$CLOUD_PROJECT"
+  echo $cmd
+  if [ $DRY_RUN == false ]; then
+    $cmd
+  fi
+done
+
+echo Checking for available backups:
+
+DUE_TIMES=($(gcloud sql backups --instance master list --project=$CLOUD_PROJECT | grep SUCCESSFUL | awk '{print $1}'))
+
+for index in ${!DUE_TIMES[*]}; do
+  echo "[$index] ${DUE_TIMES[$index]}"
+done
+
+echo "Choose one of the above due_time values."
+echo "NOTE: selecting anything besides 0 will require you to manually"
+echo "complete the rest of the restore process."
+echo "Recover from date [0: ${DUE_TIMES[0]}]:"
+read DUE_TIME_INDEX
+
+DUE_TIME=${DUE_TIMES[$DUE_TIME_INDEX]}
+
+cmd="gcloud sql instances restore-backup master --due-time=$DUE_TIME --project=$CLOUD_PROJECT --async"
+echo $cmd
+if [ $DRY_RUN == false ]; then
+  $cmd
+fi
+
+if [ "$DUE_TIME_INDEX" -ne "0" ]; then
+  echo "You've restored an older-than-latest backup. Please contact speckle-oncall@"
+  echo "to request an on-demand backup of the master before attemtping to restart replicas,"
+  echo "which this script does not do automatically in this case."
+  echo "run 'gcloud sql instances create' commands to create new replicas manually after"
+  echo "you have confirmed with speckle-oncall@ the on-demand backup is complete."
+  echo "Exiting"
+  exit 0
+fi
+
+echo "Finding restore operation ID..."
+
+RESTORE_OP_IDS=($(gcloud sql operations list --instance=master --project=$CLOUD_PROJECT | grep RESTORE_VOLUME | awk '{print $1}'))
+
+# Assume the fist RESTORE_VOLUME is the operation we want; they're listed in reverse chronological order.
+echo Waiting on restore operation ID: ${RESTORE_OP_IDS[0]}
+
+# This isn't waiting long enough. Or it says it's done before it really is.  Either way, the replica create steps fail
+# with e.g. "Failed in: CATCHING_UP Operation token: 03dd57a9-37a9-4f6f-9aa6-9c3b8ece01bd Message: Saw error in IO and/or SQL thread"
+gcloud sql operations wait ${RESTORE_OP_IDS[0]} --instance=master --project=$CLOUD_PROJECT
+
+echo Restore is finished on master. Now create the new set of read replicas with the new name prefix $NEW_REPLICA_PREFIX:
+
+for i in `seq 0 9`; do
+  cmd="gcloud sql instances create $NEW_REPLICA_PREFIX-0$i --master-instance-name master --project=$CLOUD_PROJECT --follow-gae-app=$CLOUD_PROJECT --authorized-gae-apps=$CLOUD_PROJECT --async --tier=D2"
+  echo $cmd
+  if [ $DRY_RUN == false ]; then
+    $cmd
+  fi
+done
+
+echo
+echo
+echo Backup restore is nearly complete.  Check the instances page on developer console to see when
+echo all of the replicas are "Runnable" status. Until then, you may encounter errors in issue search.
+echo In the mean time:
+echo - edit settings.py to change the db_replica_prefix variable to be "$NEW_REPLICA_PREFIX-"
+echo   Then either "make deploy_prod_backends" or "make stage_backends" for search to pick up the new prefix.
+echo   Then set the newly deploy version for besearch and besearch2 on the dev console Versons page.
+echo Follow-up:
+echo - Submit the change.
+echo - Delete old versions of besearch and besearch2 because they run up the GAE bill.