Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(954)

Side by Side Diff: perf/go/ingest/main.go

Issue 608273002: Add versioning to perf SQL database (Closed) Base URL: https://skia.googlesource.com/buildbot@master
Patch Set: Incorporating feedback 4 Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « perf/go/db/db_test.go ('k') | perf/go/migratedb/main.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 package main 1 package main
2 2
3 // ingest is the command line tool for pulling performance data from Google 3 // ingest is the command line tool for pulling performance data from Google
4 // Storage and putting in Tiles. See the code in go/ingester for details on how 4 // Storage and putting in Tiles. See the code in go/ingester for details on how
5 // ingestion is done. 5 // ingestion is done.
6 6
7 import ( 7 import (
8 "encoding/json" 8 "encoding/json"
9 "flag" 9 "flag"
10 "net" 10 "net"
11 "net/http" 11 "net/http"
12 "os" 12 "os"
13 "strings" 13 "strings"
14 "sync" 14 "sync"
15 "time" 15 "time"
16 16
17 "github.com/golang/glog" 17 "github.com/golang/glog"
18 "github.com/rcrowley/go-metrics" 18 "github.com/rcrowley/go-metrics"
19 "skia.googlesource.com/buildbot.git/perf/go/auth" 19 "skia.googlesource.com/buildbot.git/perf/go/auth"
20 "skia.googlesource.com/buildbot.git/perf/go/config" 20 "skia.googlesource.com/buildbot.git/perf/go/config"
21 "skia.googlesource.com/buildbot.git/perf/go/db"
21 "skia.googlesource.com/buildbot.git/perf/go/flags" 22 "skia.googlesource.com/buildbot.git/perf/go/flags"
22 "skia.googlesource.com/buildbot.git/perf/go/gitinfo" 23 "skia.googlesource.com/buildbot.git/perf/go/gitinfo"
23 "skia.googlesource.com/buildbot.git/perf/go/goldingester" 24 "skia.googlesource.com/buildbot.git/perf/go/goldingester"
24 "skia.googlesource.com/buildbot.git/perf/go/ingester" 25 "skia.googlesource.com/buildbot.git/perf/go/ingester"
25 "skia.googlesource.com/buildbot.git/perf/go/trybot" 26 "skia.googlesource.com/buildbot.git/perf/go/trybot"
26 ) 27 )
27 28
28 // flags 29 // flags
29 var ( 30 var (
30 timestampFile = flag.String("timestamp_file", "/tmp/timestamp.json", "F ile where timestamp data for ingester runs will be stored.") 31 timestampFile = flag.String("timestamp_file", "/tmp/timestamp.json", "F ile where timestamp data for ingester runs will be stored.")
31 tileDir = flag.String("tile_dir", "/tmp/tileStore2/", "Path where tiles will be placed.") 32 tileDir = flag.String("tile_dir", "/tmp/tileStore2/", "Path where tiles will be placed.")
32 gitRepoDir = flag.String("git_repo_dir", "../../../skia", "Directory location for the Skia repo.") 33 gitRepoDir = flag.String("git_repo_dir", "../../../skia", "Directory location for the Skia repo.")
33 runEvery = flag.Duration("run_every", 5*time.Minute, "How often th e ingester should pull data from Google Storage.") 34 runEvery = flag.Duration("run_every", 5*time.Minute, "How often th e ingester should pull data from Google Storage.")
34 runTrybotEvery = flag.Duration("run_trybot_every", 1*time.Minute, "How o ften the ingester to pull trybot data from Google Storage.") 35 runTrybotEvery = flag.Duration("run_trybot_every", 1*time.Minute, "How o ften the ingester to pull trybot data from Google Storage.")
35 run = flag.String("run", "nano,nano-trybot,golden", "A comma separated list of ingesters to run.") 36 run = flag.String("run", "nano,nano-trybot,golden", "A comma separated list of ingesters to run.")
36 graphiteServer = flag.String("graphite_server", "skia-monitoring-b:2003" , "Where is Graphite metrics ingestion server running.") 37 graphiteServer = flag.String("graphite_server", "skia-monitoring-b:2003" , "Where is Graphite metrics ingestion server running.")
37 doOauth = flag.Bool("oauth", true, "Run through the OAuth 2.0 flo w on startup, otherwise use a GCE service account.") 38 doOauth = flag.Bool("oauth", true, "Run through the OAuth 2.0 flo w on startup, otherwise use a GCE service account.")
39 local = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
38 ) 40 )
39 41
40 func Init() { 42 func Init() {
41 metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry) 43 metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry)
42 go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, 1*time.Minute ) 44 go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, 1*time.Minute )
43 addr, _ := net.ResolveTCPAddr("tcp", *graphiteServer) 45 addr, _ := net.ResolveTCPAddr("tcp", *graphiteServer)
44 go metrics.Graphite(metrics.DefaultRegistry, 1*time.Minute, "ingest", ad dr) 46 go metrics.Graphite(metrics.DefaultRegistry, 1*time.Minute, "ingest", ad dr)
45 } 47 }
46 48
47 // Timestamps is used to read and write the timestamp file, which records the ti me 49 // Timestamps is used to read and write the timestamp file, which records the ti me
(...skipping 88 matching lines...) Expand 10 before | Expand all | Expand 10 after
136 oneStep() 138 oneStep()
137 } 139 }
138 } 140 }
139 } 141 }
140 142
141 func main() { 143 func main() {
142 flag.Parse() 144 flag.Parse()
143 flags.Log() 145 flags.Log()
144 Init() 146 Init()
145 147
148 // Initialize the database. We might not need the oauth dialog if it fai ls.
149 db.Init(db.ProdConnectionString(*local))
150
146 var client *http.Client 151 var client *http.Client
147 var err error 152 var err error
148 if *doOauth { 153 if *doOauth {
149 client, err = auth.RunFlow() 154 client, err = auth.RunFlow()
150 if err != nil { 155 if err != nil {
151 glog.Fatalf("Failed to auth: %s", err) 156 glog.Fatalf("Failed to auth: %s", err)
152 } 157 }
153 } else { 158 } else {
154 client = nil 159 client = nil
155 // Add back service account access here when it's fixed. 160 // Add back service account access here when it's fixed.
156 } 161 }
157 162
158 ingester.Init(client) 163 ingester.Init(client)
159 trybot.Init()
160 goldingester.Init() 164 goldingester.Init()
161 ts := NewTimestamps(*timestampFile) 165 ts := NewTimestamps(*timestampFile)
162 ts.Read() 166 ts.Read()
163 glog.Infof("Timestamps: %#v\n", ts.Ingester) 167 glog.Infof("Timestamps: %#v\n", ts.Ingester)
164 168
165 git, err := gitinfo.NewGitInfo(*gitRepoDir, true) 169 git, err := gitinfo.NewGitInfo(*gitRepoDir, true)
166 if err != nil { 170 if err != nil {
167 glog.Fatal("Failed loading Git info: %s\n", err) 171 glog.Fatal("Failed loading Git info: %s\n", err)
168 } 172 }
169 173
170 // ingesters is a list of all the types of ingestion we can do. 174 // ingesters is a list of all the types of ingestion we can do.
171 ingesters := map[string]Process{ 175 ingesters := map[string]Process{
172 "nano": NewIngestionProcess(ts, "ingest", git, *tileDir, config.DATASET_NANO, ingester.NanoBenchIngestion, "nano-json-v1", *runEvery), 176 "nano": NewIngestionProcess(ts, "ingest", git, *tileDir, config.DATASET_NANO, ingester.NanoBenchIngestion, "nano-json-v1", *runEvery),
173 "nano-trybot": NewIngestionProcess(ts, "trybot", git, *tileDir, config.DATASET_NANO, trybot.TrybotIngestion, "trybot/nano-json-v1", *runTrybotEv ery), 177 "nano-trybot": NewIngestionProcess(ts, "trybot", git, *tileDir, config.DATASET_NANO, trybot.TrybotIngestion, "trybot/nano-json-v1", *runTrybotEv ery),
174 "golden": NewIngestionProcess(ts, "golden", git, *tileDir, config.DATASET_GOLDEN, goldingester.GoldenIngester, "dm-json-v1", *runEvery), 178 "golden": NewIngestionProcess(ts, "golden", git, *tileDir, config.DATASET_GOLDEN, goldingester.GoldenIngester, "dm-json-v1", *runEvery),
175 } 179 }
176 180
177 for _, name := range strings.Split(*run, ",") { 181 for _, name := range strings.Split(*run, ",") {
178 glog.Infof("Process name: %s", name) 182 glog.Infof("Process name: %s", name)
179 if process, ok := ingesters[name]; ok { 183 if process, ok := ingesters[name]; ok {
180 go process() 184 go process()
181 } else { 185 } else {
182 glog.Fatalf("Not a valid ingester name: %s", name) 186 glog.Fatalf("Not a valid ingester name: %s", name)
183 } 187 }
184 } 188 }
185 189
186 select {} 190 select {}
187 } 191 }
OLDNEW
« no previous file with comments | « perf/go/db/db_test.go ('k') | perf/go/migratedb/main.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698