perf: moving to tracebd

perf/go/skiaperf/main.go

 package main
 
 import (
         "encoding/json"
         "flag"
         "fmt"
         ehtml "html"
         "html/template"
         "math/rand"
         "net/http"
         "net/url"
         "path/filepath"
         "regexp"
         "runtime"
         "strconv"
         "strings"
         "time"
 
         "github.com/gorilla/mux"
         "github.com/skia-dev/glog"
         "go.skia.org/infra/go/common"
-        "go.skia.org/infra/go/filetilestore"
         "go.skia.org/infra/go/gitinfo"
         "go.skia.org/infra/go/human"
         "go.skia.org/infra/go/ingester"
         "go.skia.org/infra/go/login"
         "go.skia.org/infra/go/tiling"
+        "go.skia.org/infra/go/trace/db"
         "go.skia.org/infra/go/util"
         "go.skia.org/infra/perf/go/activitylog"
         "go.skia.org/infra/perf/go/alerting"
         "go.skia.org/infra/perf/go/annotate"
         "go.skia.org/infra/perf/go/clustering"
         "go.skia.org/infra/perf/go/config"
-        "go.skia.org/infra/perf/go/db"
+        idb "go.skia.org/infra/perf/go/db"
         "go.skia.org/infra/perf/go/parser"
         "go.skia.org/infra/perf/go/shortcut"
         "go.skia.org/infra/perf/go/stats"
         "go.skia.org/infra/perf/go/trybot"
         "go.skia.org/infra/perf/go/types"
         "go.skia.org/infra/perf/go/vec"
 )
 
 var (
         // indexTemplate is the main index.html page we serve.
@@ skipping 33 matching lines @@
         git *gitinfo.GitInfo = nil
 
         commitLinkifyRe = regexp.MustCompile("(?m)^commit (.*)$")
 )
 
 // flags
 var (
         port           = flag.String("port", ":8000", "HTTP service address (e.g., ':8000')")
         local          = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
         gitRepoDir     = flag.String("git_repo_dir", "../../../skia", "Directory location for the Skia repo.")
-        tileStoreDir   = flag.String("tile_store_dir", "/tmp/tileStore", "What directory to look for tiles in.")
         graphiteServer = flag.String("graphite_server", "skia-monitoring:2003", "Where is Graphite metrics ingestion server running.")
         apikey         = flag.String("apikey", "", "The API Key used to make issue tracker requests. Only for local testing.")
         gitRepoURL     = flag.String("git_repo_url", "https://skia.googlesource.com/skia", "The URL to pass to git clone for the source repository.")
         resourcesDir   = flag.String("resources_dir", "", "The directory to find templates, JS, and CSS files. If blank the current directory will be used.")
+        traceservice   = flag.String("trace_service", "localhost:9090", "The address of the traceservice endpoint.")
 )
 
 var (
-        nanoTileStore tiling.TileStore
+        nanoTileStore *db.Builder
 
         templates *template.Template
 )
 
 func loadTemplates() {
         templates = template.Must(template.New("").ParseFiles(
                 filepath.Join(*resourcesDir, "templates/index.html"),
                 filepath.Join(*resourcesDir, "templates/clusters.html"),
                 filepath.Join(*resourcesDir, "templates/alerting.html"),
                 filepath.Join(*resourcesDir, "templates/cl.html"),
@@ skipping 21 matching lines @@
 
 func Init() {
         rand.Seed(time.Now().UnixNano())
         if *resourcesDir == "" {
                 _, filename, _, _ := runtime.Caller(0)
                 *resourcesDir = filepath.Join(filepath.Dir(filename), "../..")
         }
 
         loadTemplates()
 
-        nanoTileStore = filetilestore.NewFileTileStore(*tileStoreDir, config.DATASET_NANO, 2*time.Minute)
-
         var err error
         git, err = gitinfo.CloneOrUpdate(*gitRepoURL, *gitRepoDir, false)
         if err != nil {
                 glog.Fatal(err)
         }
+        nanoTileStore, err = db.NewBuilder(git, *traceservice, config.INITIAL_TILE_SIZE, types.PerfTraceBuilder)
+        if err != nil {
+                glog.Fatalf("Failed to build trace/db.DB: %s", err)
+        }
 }
 
 // showcutHandler handles the POST requests of the shortcut page.
 //
 // Shortcuts are of the form:
 //
 //    {
 //       "scale": 0,
 //       "tiles": [-1],
 //       "hash": "a1092123890...",
@@ skipping 49 matching lines @@
                 glog.Errorf("Failed to write or encode output: %s", err)
         }
 }
 
 // alertingHandler returns the currently untriaged clusters.
 //
 // The return format is the same as clusteringHandler.
 func alertingHandler(w http.ResponseWriter, r *http.Request) {
         glog.Infof("Alerting Handler: %q\n", r.URL.Path)
         w.Header().Set("Content-Type", "application/json")
-        tile, err := nanoTileStore.Get(0, -1)
-        if err != nil {
-                util.ReportError(w, r, err, fmt.Sprintf("Failed to load tile."))
-                return
-        }
-
+        tile := nanoTileStore.GetTile()
         alerts, err := alerting.ListFrom(tile.Commits[0].CommitTime)
         if err != nil {
                 util.ReportError(w, r, err, "Error retrieving cluster summaries.")
                 return
         }
         enc := json.NewEncoder(w)
         if err = enc.Encode(map[string][]*types.ClusterSummary{"Clusters": alerts}); err != nil {
                 glog.Errorf("Failed to write or encode output: %s", err)
         }
 }
@@ skipping 115 matching lines @@
 //
 // Takes the following query parameters:
 //
 //   commit1 - The hash for the first commit.
 //   commit2 - The hash for the second commit.
 //   query   - A paramset in URI query format used to filter the results at each commit.
 //
 func kernelJSONHandler(w http.ResponseWriter, r *http.Request) {
         // TODO(jcgregorio) Determine the tile(s) to load based on the commit hashes,
         // possibly loading two different tiles, one for each hash.
-        tile, err := nanoTileStore.Get(0, -1)
-        if err != nil {
-                util.ReportError(w, r, err, fmt.Sprintf("Failed to load tile."))
-                return
-        }
+        tile := nanoTileStore.GetTile()
         commit1 := r.FormValue("commit1")
         commit2 := r.FormValue("commit2")
 
         // Calulate the indices where the commit falls in the tile.
         commit1Index := -1
         commit2Index := -1
 
         // Confirm that the two commits appear in the tile.
         for i, c := range tile.Commits {
                 if c.Hash == commit1 {
                         commit1Index = i
                 }
                 if c.Hash == commit2 {
                         commit2Index = i
                 }
         }
         if commit1Index == -1 || commit2Index == -1 {
                 glog.Warningf("Commits %s[%d] %s[%d]", commit1, commit1Index, commit2, commit2Index)
-                util.ReportError(w, r, err, fmt.Sprintf("Failed to find commits in tile."))
+                util.ReportError(w, r, fmt.Errorf("Failed to find commits in tile."), fmt.Sprintf("Failed to find commits in tile."))
                 return
         }
         w.Header().Set("Content-Type", "application/json")
         query := r.FormValue("query")
         q, err := url.ParseQuery(query)
         if err != nil {
                 util.ReportError(w, r, err, fmt.Sprintf("Failed to parse query parameters."))
                 return
         }
         ret := struct {
@@ skipping 54 matching lines @@
 //
 //   _k      - The K to use for k-means clustering.
 //   _stddev - The standard deviation to use when normalize traces
 //             during k-means clustering.
 //   _issue  - The Rietveld issue ID with trybot results to include.
 //
 // Additionally the rest of the query parameters as returned from
 // sk.Query.selectionsAsQuery().
 func clusteringHandler(w http.ResponseWriter, r *http.Request) {
         glog.Infof("Clustering Handler: %q\n", r.URL.Path)
-        tile, err := nanoTileStore.Get(0, -1)
-        if err != nil {
-                util.ReportError(w, r, err, fmt.Sprintf("Failed to load tile."))
-                return
-        }
+        tile := nanoTileStore.GetTile()
         w.Header().Set("Content-Type", "application/json")
         // If there are no query parameters just return with an empty set of ClusterSummaries.
         if r.FormValue("_k") == "" || r.FormValue("_stddev") == "" {
                 writeClusterSummaries(clustering.NewClusterSummaries(), w, r)
                 return
         }
 
         k, err := strconv.ParseInt(r.FormValue("_k"), 10, 32)
         if err != nil {
                 util.ReportError(w, r, err, fmt.Sprintf("_k parameter must be an integer %s.", r.FormValue("_k")))
@@ skipping 38 matching lines @@
                 }
         }
         summary, err := clustering.CalculateClusterSummaries(tile, int(k), stddev, filter)
         if err != nil {
                 util.ReportError(w, r, err, "Failed to calculate clusters.")
                 return
         }
         writeClusterSummaries(summary, w, r)
 }
 
-// getTile retrieves a tile from the disk
-func getTile(tileScale, tileNumber int) (*tiling.Tile, error) {
-        start := time.Now()
-        tile, err := nanoTileStore.Get(int(tileScale), int(tileNumber))
-        glog.Infoln("Time for tile load: ", time.Since(start).Nanoseconds())
-        if err != nil || tile == nil {
-                return nil, fmt.Errorf("Unable to get tile from tilestore: %s", err)
-        }
-        return tile, nil
-}
-
 // tileHandler accepts URIs like /tiles/0/1
 // where the URI format is /tiles/<tile-scale>/<tile-number>
 //
 // It returns JSON of the form:
 //
 //  {
 //    tiles: [20],
 //    scale: 0,
 //    paramset: {
 //      "os": ["Android", "ChromeOS", ..],
@@ skipping 31 matching lines @@
         if err != nil {
                 util.ReportError(w, r, err, "Failed parsing tile scale.")
                 return
         }
         tileNumber, err := strconv.ParseInt(match[2], 10, 0)
         if err != nil {
                 util.ReportError(w, r, err, "Failed parsing tile number.")
                 return
         }
         glog.Infof("tile: %d %d", tileScale, tileNumber)
-        tile, err := getTile(int(tileScale), int(tileNumber))
-        if err != nil {
-                util.ReportError(w, r, err, "Failed retrieving tile.")
-                return
-        }
-
+        tile := nanoTileStore.GetTile()
         guiTile := tiling.NewTileGUI(tile.Scale, tile.TileIndex)
         guiTile.Commits = tile.Commits
         guiTile.ParamSet = tile.ParamSet
         // SkpCommits goes out to the git repo, add caching if this turns out to be
         // slow.
         if skps, err := git.SkpCommits(tile); err != nil {
                 guiTile.Skps = []int{}
                 glog.Errorf("Failed to calculate skps: %s", err)
         } else {
                 guiTile.Skps = skps
@@ skipping 113 matching lines @@
         if err != nil {
                 util.ReportError(w, r, err, "Failed parsing tile scale.")
                 return
         }
         tileNumber, err := strconv.ParseInt(match[2], 10, 0)
         if err != nil {
                 util.ReportError(w, r, err, "Failed parsing tile number.")
                 return
         }
         glog.Infof("tile: %d %d", tileScale, tileNumber)
-        tile, err := getTile(int(tileScale), int(tileNumber))
-        if err != nil {
-                util.ReportError(w, r, err, "Failed retrieving tile.")
-                return
-        }
+        tile := nanoTileStore.GetTile()
         w.Header().Set("Content-Type", "application/json")
         ret := &QueryResponse{
                 Traces: []*tiling.TraceGUI{},
                 Hash:   "",
         }
         if match[3] == "" {
                 // We only want the count.
                 total := 0
                 for _, tr := range tile.Traces {
                         if tiling.Matches(tr, r.Form) {
@@ skipping 98 matching lines @@
         }
         hash := match[1]
 
         tileNum, idx, err := git.TileAddressFromHash(hash, time.Time(ingester.BEGINNING_OF_TIME))
         if err != nil {
                 glog.Infof("Did not find hash '%s', use latest: %q.\n", hash, err)
                 tileNum = -1
                 idx = -1
         }
         glog.Infof("Hash: %s tileNum: %d, idx: %d\n", hash, tileNum, idx)
-        tile, err := getTile(0, tileNum)
-        if err != nil {
-                util.ReportError(w, r, err, "Failed retrieving tile.")
-                return
-        }
+        tile := nanoTileStore.GetTile()
 
         if idx < 0 {
                 idx = len(tile.Commits) - 1 // Defaults to the last slice element.
         }
         glog.Infof("Tile: %d; Idx: %d\n", tileNum, idx)
 
         ret := SingleResponse{
                 Traces: []*SingleTrace{},
                 Hash:   tile.Commits[idx].Hash,
         }
@@ skipping 82 matching lines @@
 // calcHandler handles requests for the form:
 //
 //    /calc/?formula=filter("config=8888")
 //
 // Where the formula is any formula that parser.Eval() accepts.
 //
 // The response is the same format as queryHandler.
 func calcHandler(w http.ResponseWriter, r *http.Request) {
         glog.Infof("Calc Handler: %q\n", r.URL.Path)
         w.Header().Set("Content-Type", "application/json")
-        tile, err := nanoTileStore.Get(0, -1)
-        if err != nil {
-                util.ReportError(w, r, err, fmt.Sprintf("Failed to load tile."))
-                return
-        }
+        tile := nanoTileStore.GetTile()
         formula := r.FormValue("formula")
 
         var data interface{} = nil
         if r.FormValue("flat") == "true" {
                 resp := &FlatQueryResponse{
                         Traces: []*types.PerfTrace{},
                 }
                 if err := addFlatCalculatedTraces(resp, tile, formula); err != nil {
                         util.ReportError(w, r, err, fmt.Sprintf("Failed in /calc/ to evaluate formula."))
                         return
@@ skipping 137 matching lines @@
         fileServer := http.FileServer(http.Dir(*resourcesDir))
         return func(w http.ResponseWriter, r *http.Request) {
                 w.Header().Add("Cache-Control", string(300))
                 fileServer.ServeHTTP(w, r)
         }
 }
 
 func main() {
         defer common.LogPanic()
         // Setup DB flags.
-        dbConf := db.DBConfigFromFlags()
+        dbConf := idb.DBConfigFromFlags()
 
         common.InitWithMetrics("skiaperf", graphiteServer)
         Init()
         if !*local {
                 if err := dbConf.GetPasswordFromMetadata(); err != nil {
                         glog.Fatal(err)
                 }
         }
         if err := dbConf.InitDB(); err != nil {
                 glog.Fatal(err)
@@ skipping 39 matching lines @@
         router.HandleFunc("/help/", helpHandler)
         router.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler)
         router.HandleFunc("/logout/", login.LogoutHandler)
         router.HandleFunc("/loginstatus/", login.StatusHandler)
 
         http.Handle("/", util.LoggingGzipRequestResponse(router))
 
         glog.Infoln("Ready to serve.")
         glog.Fatal(http.ListenAndServe(*port, nil))
 }