Add retries to upload_dm_results

infra/bots/recipes/upload_dm_results.expected/failed_once.json

 [
   {
     "cmd": [
       "python",
       "-u",
       "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
       "[CWD]/tmp_upload",
       "511"
     ],
     "name": "makedirs tmp dir",
@@ skipping 72 matching lines @@
     ],
     "name": "find images"
   },
   {
     "cmd": [
       "gsutil",
       "cp",
       "[CWD]/dm/*",
       "gs://skia-infra-gm/dm-images-v1"
     ],
-    "name": "upload images"
+    "name": "upload images",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
   },
   {
     "cmd": [
+      "gsutil",
+      "cp",
+      "[CWD]/dm/*",
+      "gs://skia-infra-gm/dm-images-v1"
+    ],
+    "name": "upload images (attempt 2)"
+  },
+  {
+    "cmd": [
       "gsutil",
       "cp",
       "-z",
       "json,log",
       "[CWD]/tmp_upload/*",
       "gs://skia-infra-gm/dm-json-v1/2012/05/14/12/abc123/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/1337018001"
     ],
     "name": "upload JSON and logs"
   },
   {
     "name": "$result",
     "recipe_result": null,
     "status_code": 0
   }
 ]