Add a command line tool for validating the tileStore.

perf/server/src/tiletool/main.go

+// tiletool is a command line application to validate a tile store.
+package main
+
+import (
+        "flag"
+        "fmt"
+        "log"
+        "time"
+)
+
+import (
+        "config"
+        "filetilestore"
+        "types"
+)
+
+// flags
+var (
+        tileDir    = flag.String("tile_dir", "/tmp/tileStore", "What directory to look for tiles in.")
+        verbose    = flag.Bool("verbose", false, "Verbose.")
+        echoHashes = flag.Bool("echo_hashes", false, "Echo Git hashes during validation.")
+)
+
+// validateTile validates a tile by confirming that all the commits are in
+// ascending order.
+//
+// Also checks that none of the commits overlap with the following tile by
+// making sure each commit appears before oldestTS.
+func validateTile(tile *types.Tile, oldestTS int64) error {
+        var lastTS int64 = 0
+        lastHash := ""
+        if *verbose {
+                fmt.Println("Number of Commits:", len(tile.Commits))
+        }
+        for i, c := range tile.Commits {
+                if *echoHashes {
+                        fmt.Println("Hash:", c.Hash, c.CommitTime)
+                }
+                if c.CommitTime > oldestTS {
+                        fmt.Printf("ERROR: Tiles out of order: Commit (%s) %d timestamp is %d, which appears after %d\n", c.Hash, i, c.CommitTime, oldestTS)

[mtklein, 2014/07/14 16:18:34]

return fmt.Errorf?

[jcgregorio, 2014/07/14 16:21:39]

Maybe eventually, but for now I only print the error while I'm debugging how
tiles got out of order.


On 2014/07/14 16:18:34, mtklein wrote:

+                }
+                if c.CommitTime < lastTS {
+                        return fmt.Errorf("Commits out of order: Commit (%s) %d timestamp is %d, which appears before (%s) %d\n", c.Hash, i, c.CommitTime, lastHash, lastTS)
+                }
+                lastTS = c.CommitTime
+                lastHash = c.Hash
+        }
+
+        if *verbose {
+                fmt.Println("Number of traces:", len(tile.Traces))
+        }
+
+        // Make sure each Trace is the right length.
+        numCommits := len(tile.Commits)
+        for _, trace := range tile.Traces {
+                if len(trace.Values) != numCommits {
+                        return fmt.Errorf("Trace lenght incorrect: Num Commits %d != Values in trace %d for Key %s", numCommits, len(trace.Values), trace.Key)
+                }
+        }
+
+        return nil
+}
+
+// validateDataset validates all the tiles stored in a TileStore.
+func validateDataset(store types.TileStore) bool {
+        index := -1
+        isValid := true
+        // If tilebuilding were instantaneous this might cause a false negative, but
+        // it's not.
+        oldestTS := time.Now().Unix()
+
+        for {
+                tile, err := store.Get(0, index)
+                if err != nil {
+                        fmt.Printf("Failed to Get(0, %d): %s\n", index, err)
+                        isValid = false
+                        break
+                }
+                if *verbose {
+                        fmt.Println("TileIndex:", tile.TileIndex)
+                }
+                // Validate the git hashes in the tile.
+                err = validateTile(tile, oldestTS)
+                oldestTS = tile.Commits[0].CommitTime
+                if err != nil {
+                        fmt.Printf("Failed to validate tile %d scale 0: %s\n", index, err)
+                        isValid = false
+                        break
+                }
+                if index > 0 && index != tile.TileIndex {
+                        fmt.Printf("Tile index inconsistent: index %d != tile.TileIndex %d\n", index, tile.TileIndex)
+                        isValid = false
+                        break
+                }
+                if tile.Scale != 0 {
+                        fmt.Printf("Tile scale isn't 0: tile.Scale %d\n", tile.Scale)
+                        isValid = false
+                        break
+                }
+                if tile.TileIndex > 0 {
+                        index = tile.TileIndex - 1
+                } else {
+                        break
+                }
+        }
+
+        return isValid
+}
+
+func main() {
+        flag.Parse()
+        valid := true
+        for _, name := range config.ALL_DATASET_NAMES {
+                fmt.Printf("Validating dataset: %s\n", string(name))
+                store := filetilestore.NewFileTileStore(*tileDir, string(name))
+                valid = valid && validateDataset(store)
+        }
+        if !valid {
+                log.Fatal("FAILED Validation.")
+        }
+}