OLD | NEW |
(Empty) | |
| 1 #!/bin/bash |
| 2 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is govered by a BSD-style |
| 4 # license that can be found in the LICENSE file or at |
| 5 # https://developers.google.com/open-source/licenses/bsd |
| 6 |
| 7 # The existing replicas all have this prefix: |
| 8 REPLICA_PREFIX="replica-7" |
| 9 |
| 10 # The new replicas made from the restored master will have this prefix: |
| 11 NEW_REPLICA_PREFIX="replica-8" |
| 12 |
| 13 CLOUD_PROJECT="monorail-staging" |
| 14 |
| 15 DRY_RUN=true |
| 16 |
| 17 echo Restoring backups to master for $CLOUD_PROJECT. Dry run: $DRY_RUN |
| 18 echo This will delete all read replicas with the prefix "$REPLICA_PREFIX" |
| 19 echo and create a new set of replicas with the prefix "$NEW_REPLICA_PREFIX" |
| 20 echo |
| 21 echo Checking for existing read replicas to delete: |
| 22 |
| 23 EXISTING_REPLICAS=($(gcloud sql instances list --project=$CLOUD_PROJECT | grep $
REPLICA_PREFIX- | awk '{print $1}')) |
| 24 |
| 25 if [ ${#EXISTING_REPLICAS[@]} -eq 0 ]; then |
| 26 echo No replicas found with prefix $REPLICA_PREFIX |
| 27 echo List instances to find the replica prefix by running: |
| 28 echo gcloud sql instances list --project=$CLOUD_PROJECT |
| 29 exit 1 |
| 30 fi |
| 31 |
| 32 echo Deleting ${#EXISTING_REPLICAS[@]} existing replicas found with the prefix $
REPLICA_PREFIX |
| 33 |
| 34 for r in "${EXISTING_REPLICAS[@]}"; do |
| 35 echo Deleting $r |
| 36 cmd="gcloud sql instances delete $r --project=$CLOUD_PROJECT" |
| 37 echo $cmd |
| 38 if [ $DRY_RUN == false ]; then |
| 39 $cmd |
| 40 fi |
| 41 done |
| 42 |
| 43 echo Checking for available backups: |
| 44 |
| 45 DUE_TIMES=($(gcloud sql backups --instance master list --project=$CLOUD_PROJECT
| grep SUCCESSFUL | awk '{print $1}')) |
| 46 |
| 47 for index in ${!DUE_TIMES[*]}; do |
| 48 echo "[$index] ${DUE_TIMES[$index]}" |
| 49 done |
| 50 |
| 51 echo "Choose one of the above due_time values." |
| 52 echo "NOTE: selecting anything besides 0 will require you to manually" |
| 53 echo "complete the rest of the restore process." |
| 54 echo "Recover from date [0: ${DUE_TIMES[0]}]:" |
| 55 read DUE_TIME_INDEX |
| 56 |
| 57 DUE_TIME=${DUE_TIMES[$DUE_TIME_INDEX]} |
| 58 |
| 59 cmd="gcloud sql instances restore-backup master --due-time=$DUE_TIME --project=$
CLOUD_PROJECT --async" |
| 60 echo $cmd |
| 61 if [ $DRY_RUN == false ]; then |
| 62 $cmd |
| 63 fi |
| 64 |
| 65 if [ "$DUE_TIME_INDEX" -ne "0" ]; then |
| 66 echo "You've restored an older-than-latest backup. Please contact speckle-onca
ll@" |
| 67 echo "to request an on-demand backup of the master before attemtping to restar
t replicas," |
| 68 echo "which this script does not do automatically in this case." |
| 69 echo "run 'gcloud sql instances create' commands to create new replicas manual
ly after" |
| 70 echo "you have confirmed with speckle-oncall@ the on-demand backup is complete
." |
| 71 echo "Exiting" |
| 72 exit 0 |
| 73 fi |
| 74 |
| 75 echo "Finding restore operation ID..." |
| 76 |
| 77 RESTORE_OP_IDS=($(gcloud sql operations list --instance=master --project=$CLOUD_
PROJECT | grep RESTORE_VOLUME | awk '{print $1}')) |
| 78 |
| 79 # Assume the fist RESTORE_VOLUME is the operation we want; they're listed in rev
erse chronological order. |
| 80 echo Waiting on restore operation ID: ${RESTORE_OP_IDS[0]} |
| 81 |
| 82 # This isn't waiting long enough. Or it says it's done before it really is. Eit
her way, the replica create steps fail |
| 83 # with e.g. "Failed in: CATCHING_UP Operation token: 03dd57a9-37a9-4f6f-9aa6-9c3
b8ece01bd Message: Saw error in IO and/or SQL thread" |
| 84 gcloud sql operations wait ${RESTORE_OP_IDS[0]} --instance=master --project=$CLO
UD_PROJECT |
| 85 |
| 86 echo Restore is finished on master. Now create the new set of read replicas with
the new name prefix $NEW_REPLICA_PREFIX: |
| 87 |
| 88 for i in `seq 0 9`; do |
| 89 cmd="gcloud sql instances create $NEW_REPLICA_PREFIX-0$i --master-instance-nam
e master --project=$CLOUD_PROJECT --follow-gae-app=$CLOUD_PROJECT --authorized-g
ae-apps=$CLOUD_PROJECT --async --tier=D2" |
| 90 echo $cmd |
| 91 if [ $DRY_RUN == false ]; then |
| 92 $cmd |
| 93 fi |
| 94 done |
| 95 |
| 96 echo |
| 97 echo |
| 98 echo Backup restore is nearly complete. Check the instances page on developer c
onsole to see when |
| 99 echo all of the replicas are "Runnable" status. Until then, you may encounter er
rors in issue search. |
| 100 echo In the mean time: |
| 101 echo - edit settings.py to change the db_replica_prefix variable to be "$NEW_REP
LICA_PREFIX-" |
| 102 echo Then either "make deploy_prod_backends" or "make stage_backends" for sear
ch to pick up the new prefix. |
| 103 echo Then set the newly deploy version for besearch and besearch2 on the dev c
onsole Versons page. |
| 104 echo Follow-up: |
| 105 echo - Submit the change. |
| 106 echo - Delete old versions of besearch and besearch2 because they run up the GAE
bill. |
OLD | NEW |