Fix non-idempotent task that /poll reaped but returned HTTP 500.

appengine/swarming/server/task_scheduler_test.py

Index: appengine/swarming/server/task_scheduler_test.py
diff --git a/appengine/swarming/server/task_scheduler_test.py b/appengine/swarming/server/task_scheduler_test.py
index 9a97ee0999da0fc306b54e789d92636a3822f580..de3cb54d74519466a8a6ae3e182f61dd0fd3d0aa 100755
--- a/appengine/swarming/server/task_scheduler_test.py
+++ b/appengine/swarming/server/task_scheduler_test.py
@@ -1608,6 +1608,169 @@ class TaskSchedulerApiTest(test_env_handlers.AppTestBase):
     self.assertEqual(0, self.execute_tasks())
     self.assertEqual(4, len(pub_sub_calls)) # RUNNING -> COMPLETED
 
+  def test_cron_handle_bot_died_no_update_not_idempotent(self):
+    pub_sub_calls = self.mock_pub_sub()
+
+    # Test first retry, then success.
+    now = utils.utcnow()
+    request = self._gen_request(
+        properties={
+          'dimensions': {u'os': u'Windows-3.1.1', u'pool': u'default'},
+        },
+        created_ts=now,
+        expiration_ts=now+datetime.timedelta(seconds=600),
+        pubsub_topic='projects/abc/topics/def')
+    task_request.init_new_request(request, True, None)
+    _result_summary = task_scheduler.schedule_request(request, None)
+    self.assertEqual(1, self.execute_tasks())
+    self.assertEqual(0, len(pub_sub_calls))
+    bot_dimensions = {
+      u'foo': [u'bar'],
+      u'id': [u'localhost'],
+      u'os': [u'Windows', u'Windows-3.1.1'],
+      u'pool': [u'default'],
+    }
+    self._register_bot(bot_dimensions, nb_task=0)
+    request, _, run_result = task_scheduler.bot_reap_task(
+        bot_dimensions, 'abc', None)
+    self.assertEqual(
+        task_result.State.RUNNING, run_result.result_summary_key.get().state)
+    self.assertEqual(1, self.execute_tasks())
+    self.assertEqual(1, len(pub_sub_calls)) # PENDING -> RUNNING
+    self.assertEqual(1, run_result.try_number)
+    self.assertEqual(task_result.State.RUNNING, run_result.state)
+    now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
+    self.assertEqual(([], 1, 0), task_scheduler.cron_handle_bot_died('f.local'))
+    self.assertEqual(1, self.execute_tasks())
+    self.assertEqual(2, len(pub_sub_calls)) # RUNNING -> PENDING
+
+    # Refresh and compare:
+    expected = {
+      'abandoned_ts': now_1,
+      'bot_dimensions': bot_dimensions,
+      'bot_id': u'localhost',
+      'bot_version': u'abc',
+      'cipd_pins': None,
+      'children_task_ids': [],
+      'completed_ts': None,
+      'cost_usd': 0.,
+      'duration': None,
+      'exit_code': None,
+      'failure': False,
+      'id': '1d69b9f088008911',
+      'internal_failure': True,
+      'modified_ts': now_1,
+      'outputs_ref': None,
+      'server_versions': [u'v1a'],
+      'started_ts': self.now,
+      'state': task_result.State.BOT_DIED,
+      'try_number': 1,
+    }
+    self.assertEqual(expected, run_result.key.get().to_dict())
+    expected = {
+      'abandoned_ts': None,
+      'bot_dimensions': bot_dimensions,
+      'bot_id': u'localhost',
+      'bot_version': u'abc',
+      'cipd_pins': None,
+      'children_task_ids': [],
+      'completed_ts': None,
+      'costs_usd': [0.],
+      'cost_saved_usd': None,
+      'created_ts': self.now,
+      'deduped_from': None,
+      'duration': None,
+      'exit_code': None,
+      'failure': False,
+      'id': '1d69b9f088008910',
+      'internal_failure': False,
+      'modified_ts': now_1,
+      'name': u'Request name',
+      'outputs_ref': None,
+      'properties_hash': None,
+      'server_versions': [u'v1a'],
+      'started_ts': None,
+      'state': task_result.State.PENDING,
+      'tags': [
+        u'os:Windows-3.1.1',
+        u'pool:default',
+        u'priority:50',
+        u'service_account:none',
+        u'tag:1',
+        u'user:Jesus',
+      ],
+      'try_number': 1,
+      'user': u'Jesus',
+    }
+    self.assertEqual(expected, run_result.result_summary_key.get().to_dict())
+
+    # Task was retried.
+    now_2 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2)
+    bot_dimensions_second = bot_dimensions.copy()
+    bot_dimensions_second[u'id'] = [u'localhost-second']
+    self._register_bot(bot_dimensions_second, nb_task=0)
+    _request, _, run_result = task_scheduler.bot_reap_task(
+        bot_dimensions_second, 'abc', None)
+    self.assertEqual(1, self.execute_tasks())
+    self.assertEqual(3, len(pub_sub_calls)) # PENDING -> RUNNING
+    logging.info('%s', [t.to_dict() for t in task_to_run.TaskToRun.query()])
+    self.assertEqual(2, run_result.try_number)
+    self.assertEqual(
+        task_result.State.COMPLETED,
+        task_scheduler.bot_update_task(
+            run_result_key=run_result.key,
+            bot_id='localhost-second',
+            cipd_pins=None,
+            output='Foo1',
+            output_chunk_start=0,
+            exit_code=0,
+            duration=0.1,
+            hard_timeout=False,
+            io_timeout=False,
+            cost_usd=0.1,
+            outputs_ref=None,
+            performance_stats=None))
+    expected = {
+      'abandoned_ts': None,
+      'bot_dimensions': bot_dimensions_second,
+      'bot_id': u'localhost-second',
+      'bot_version': u'abc',
+      'cipd_pins': None,
+      'children_task_ids': [],
+      'completed_ts': now_2,
+      'costs_usd': [0., 0.1],
+      'cost_saved_usd': None,
+      'created_ts': self.now,
+      'deduped_from': None,
+      'duration': 0.1,
+      'exit_code': 0,
+      'failure': False,
+      'id': '1d69b9f088008910',
+      'internal_failure': False,
+      'modified_ts': now_2,
+      'name': u'Request name',
+      'outputs_ref': None,
+      'properties_hash': None,
+      'server_versions': [u'v1a'],
+      'started_ts': now_2,
+      'state': task_result.State.COMPLETED,
+      'tags': [
+        u'os:Windows-3.1.1',
+        u'pool:default',
+        u'priority:50',
+        u'service_account:none',
+        u'tag:1',
+        u'user:Jesus',
+      ],
+      'try_number': 2,
+      'user': u'Jesus',
+    }
+    self.assertEqual(expected, run_result.result_summary_key.get().to_dict())
+    self.assertEqual(0.1, run_result.key.get().cost_usd)
+
+    self.assertEqual(0, self.execute_tasks())
+    self.assertEqual(4, len(pub_sub_calls)) # RUNNING -> COMPLETED
+
   def test_cron_handle_bot_died_same_bot_denied(self):
     # Test first retry, then success.
     now = utils.utcnow()