Added HTTP endpoints for Correctness counts

golden/go/analysis/analysis.go

Index: golden/go/analysis/analysis.go
diff --git a/golden/go/analysis/analysis.go b/golden/go/analysis/analysis.go
index ede75ef81bb61ee232d13f56b865c0f9c3294238..4db24694a5386370e6893d23b4a59dba6d3699e2 100644
--- a/golden/go/analysis/analysis.go
+++ b/golden/go/analysis/analysis.go
@@ -17,10 +17,10 @@ import (
 // Stores a Trace with labels and digests in memory. CommitIds, Digests and

[jcgregorio, 2014/10/18 00:09:03]

Comments for things begin with its name:

// LabeledTrace stores a Trace ...

All structs and funcs in this file.

[stephana, 2014/10/20 12:56:07]

Done.

 // Labels are of the same length, identical indices refer to the same digest.
 type LabeledTrace struct {
-	Params    map[string]string `json:"params"`
-	CommitIds []int             `json:"commitIds"`
-	Digests   []string          `json:"digests"`
-	Labels    []types.Label     `json:"labels`
+	Params    map[string]string
+	CommitIds []int
+	Digests   []string
+	Labels    []types.Label
 }
 
 func NewLabeledTrace(params map[string]string, capacity int) *LabeledTrace {
@@ -41,12 +41,14 @@ func (lt *LabeledTrace) addLabeledDigests(commitIds []int, digests []string, lab
 
 // Aggregates the Traces in tile and provides the commits that the
 // CommitIds in LabeledTrace refer to.
+// LabeledTile and LabeledTrace are used to store the cannonical information
+// extracted from the tiles. The (redundant) output data is derived from these.
 type LabeledTile struct {
-	Commits []*ptypes.Commit `json:"commits"`
+	Commits []*ptypes.Commit
 
 	// Traces are indexed by the primary key (test name). This is somewhat
 	// redundant, but this also output format.
-	Traces map[string][]*LabeledTrace `json:"traces"`
+	Traces map[string][]*LabeledTrace
 }
 
 func NewLabeledTile() *LabeledTile {
@@ -81,6 +83,43 @@ func (t *LabeledTile) getLabeledTrace(trace ptypes.Trace) (string, *LabeledTrace
 	return pKey, newLT
 }
 
+// Classification counts.
+type LabelCounts struct {
+	Unt int `json:"unt"` // Untriaged
+	Pos int `json:"pos"` // Positive
+	Neg int `json:"neg"` // Negative
+}
+
+// GUITileCounts is an output type for the aggregated label counts.
+// The tupples in Counts store: untriaged, positive, negatives
+type GUITileCounts struct {
+	Commits []*ptypes.Commit         `json:"commits"`
+	Counts  map[string][]LabelCounts `json:"counts"`
+}
+
+// GUITestCounts is an output type for a single test that contains the
+// aggregated counts over all traces and invidual trace labels.
+type GUITestCounts struct {
+	Commits    []*ptypes.Commit   `json:"commits"`
+	Aggregated []LabelCounts      `json:"aggregated"`
+	Traces     []*GUILabeledTrace `json:"traces"`
+	Labels     []IdLabel          `json:"labels"`
+}
+
+// GUILabeledTrace is an output type for the labels of a trace.
+type GUILabeledTrace struct {
+	Params map[string]string `json:"params"`
+
+	// List of commitId and Label pairs.
+	Labels []IdLabel `json:"labels"`
+}
+
+// CommitId and label pair.
+type IdLabel struct {
+	C int `json:"c"`

[jcgregorio, 2014/10/18 00:09:03]

I'm all for brevity, but this might be too much. ID and Label are pretty short.

[stephana, 2014/10/20 12:56:07]

Done.

+	L int `json:"l"`
+}
+
 // Analyzer continuously manages the tasks, like pollint for new traces
 // on disk, etc.
 type Analyzer struct {
@@ -88,9 +127,15 @@ type Analyzer struct {
 	diffStore diff.DiffStore
 	tileStore ptypes.TileStore
 
+	// Canonical data structure to hold our information about commits, digests
+	// and labels.
 	currentTile *LabeledTile
 
-	// Lock to protect the expectations and the current labeled tile.
+	// Output data structures that are derived from currentTile.
+	currentTileCounts *GUITileCounts
+	currentTestCounts map[string]*GUITestCounts
+
+	// Lock to protect the expectations and current* variables.
 	mutex sync.Mutex
 }
 
@@ -110,18 +155,18 @@ func NewAnalyzer(expStore expstorage.ExpectationsStore, tileStore ptypes.TileSto
 // Returns an entire Tile which is a collection of 'traces' over a series of
 // of commits. Each trace contains the digests and their labels based on
 // out knowledge base about digests (expectations).
-func (a *Analyzer) GetLabeledTile() *LabeledTile {
+func (a *Analyzer) GetTileCounts() (*GUITileCounts, error) {
 	a.mutex.Lock()
 	defer a.mutex.Unlock()
 
-	return a.currentTile
+	return a.currentTileCounts, nil
 }
 
-func (a *Analyzer) GetLabeledTraces(testName string) []*LabeledTrace {
+func (a *Analyzer) GetTestCounts(testName string) (*GUITestCounts, error) {
 	a.mutex.Lock()
 	defer a.mutex.Unlock()
 
-	return a.currentTile.Traces[testName]
+	return a.currentTestCounts[testName], nil
 }
 
 func (a *Analyzer) SetDigestLabels(labeledTestDigests map[string]types.TestClassification, userId string) (map[string][]*LabeledTrace, error) {
@@ -160,10 +205,15 @@ func (a *Analyzer) loop(timeBetweenPolls time.Duration) {
 			errorTileLoadingCounter.Inc(1)
 		} else {
 			newLabeledTile := a.processTile(tile)
+			newTileCounts, newTestCounts := a.getOutputCounts(newLabeledTile)
+
 			a.mutex.Lock()
 			a.currentTile = newLabeledTile
+			a.currentTileCounts = newTileCounts
+			a.currentTestCounts = newTestCounts
 			a.mutex.Unlock()
 		}
+		glog.Info("Done processing tiles.")
 		runsCounter.Inc(1)
 
 		// Sleep for a while until the next poll.
@@ -248,3 +298,53 @@ func (a *Analyzer) labelDigests(testName string, digests []string, targetLabels
 
 	return nil
 }
+
+// Derive the output counts from the given labeled tile.
+func (a *Analyzer) getOutputCounts(labeledTile *LabeledTile) (*GUITileCounts, map[string]*GUITestCounts) {
+	// Stores the aggregated counts of a tile for each test.
+	tileCountsMap := make(map[string][]LabelCounts, len(labeledTile.Traces))
+
+	// Stores the aggregated counts for each test and individual trace information.
+	testCountsMap := make(map[string]*GUITestCounts, len(labeledTile.Traces))
+
+	for testName, testTraces := range labeledTile.Traces {
+		acc := make([]LabelCounts, len(labeledTile.Commits))
+		tempTraces := make([]*GUILabeledTrace, 0, len(testTraces))
+
+		for _, oneTrace := range testTraces {
+			tempTrace := &GUILabeledTrace{
+				Params: oneTrace.Params,
+				Labels: make([]IdLabel, len(oneTrace.CommitIds)),
+			}
+
+			for i, ci := range oneTrace.CommitIds {
+				switch oneTrace.Labels[i] {
+				case types.UNTRIAGED:
+					acc[ci].Unt++
+				case types.POSITIVE:
+					acc[ci].Pos++
+				case types.NEGATIVE:
+					acc[ci].Neg++
+				}
+				tempTrace.Labels[i].C = ci
+				tempTrace.Labels[i].L = int(oneTrace.Labels[i])
+			}
+
+			tempTraces = append(tempTraces, tempTrace)
+		}
+
+		tileCountsMap[testName] = acc
+		testCountsMap[testName] = &GUITestCounts{
+			Commits:    labeledTile.Commits,
+			Aggregated: acc,
+			Traces:     tempTraces,
+		}
+	}
+
+	tileCounts := &GUITileCounts{
+		Commits: labeledTile.Commits,
+		Counts:  tileCountsMap,
+	}
+
+	return tileCounts, testCountsMap
+}