Iteratively increase sample size for good/bad classification.

scripts/slave/recipe_modules/auto_bisect/resources/significantly_different.py

+"""Launches an anaconda environment to run some scipy hypothesis tests."""
+
+import json
+import os
+import subprocess
+import sys
+
+CONDA_SCRIPT = """

[dtu, 2016/02/05 00:18:02]

Why not put it in its own Python file?

+import json
+import sys
+
+from scipy import stats
+
+_, list_a, list_b  = sys.argv
+list_a = json.loads(list_a)
+list_b = json.loads(list_b)
+
+shapiro_p_value = stats.shapiro(list_a)[1], stats.shapiro(list_b)[1]
+mann_whitney_p_value = stats.mannwhitneyu(list_a, list_b).pvalue
+anderson_p_value = stats.anderson_ksamp([list_a, list_b]).significance_level
+welch_p_value = stats.ttest_ind(list_a, list_b, equal_var=False)[1]
+
+results = {
+    'first_sample': list_a,
+    'second_sample': list_b,
+    'shapiro_p_value': shapiro_p_value,
+    'mann_p_value': mann_whitney_p_value,
+    'anderson_p_value': mann_whitney_p_value,
+    'welch_p_value': welch_p_value,
+}
+
+print json.dumps(results)
+sys.exit(0)
+"""
+
+def main(argv, anaconda_path=None):
+  if not anaconda_path:
+    if os.name == 'nt':
+      anaconda_path = r'c:\conda-py-scientific\bin\python'
+    else:
+      anaconda_path = '/opt/conda-py-scientific/bin/python'
+
+  _, list_a, list_b, significance  = argv
+
+  p = subprocess.Popen([anaconda_path, '-', list_a, list_b],
+                       stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+  output, _ = p.communicate(input=CONDA_SCRIPT)
+  results = json.loads(output.decode())
+  if (results['shapiro_p_value'][0] < significance and

[dtu, 2016/02/05 00:18:02]

It's strange to mix logic between here and the Anaconda script. Best to put it
all in one place. You can make this script just a thin wrapper for the Anaconda
script.

+      results['shapiro_p_value'][1] < significance):
+    results['normal-y'] = True
+  else:
+    results['normal-y'] = False
+  results['significantly_different'] = bool(
+      float(results['mann_p_value']) < float(significance))
+
+  return results
+
+if __name__ == '__main__':
+  result = main(sys.argv)
+  if result:
+    print json.dumps(result, indent=4)
+    sys.exit(0)
+  sys.exit(1)