OLD | NEW |
1 package main | 1 package main |
2 | 2 |
3 // ingest is the command line tool for pulling performance data from Google | 3 // ingest is the command line tool for pulling performance data from Google |
4 // Storage and putting in Tiles. See the code in go/ingester for details on how | 4 // Storage and putting in Tiles. See the code in go/ingester for details on how |
5 // ingestion is done. | 5 // ingestion is done. |
6 | 6 |
7 import ( | 7 import ( |
8 "encoding/json" | 8 "encoding/json" |
9 "flag" | 9 "flag" |
10 "net" | 10 "net" |
11 "net/http" | 11 "net/http" |
12 "os" | 12 "os" |
13 "strings" | 13 "strings" |
14 "sync" | 14 "sync" |
15 "time" | 15 "time" |
16 | 16 |
17 "github.com/golang/glog" | 17 "github.com/golang/glog" |
18 "github.com/rcrowley/go-metrics" | 18 "github.com/rcrowley/go-metrics" |
19 "skia.googlesource.com/buildbot.git/perf/go/auth" | 19 "skia.googlesource.com/buildbot.git/perf/go/auth" |
20 "skia.googlesource.com/buildbot.git/perf/go/config" | 20 "skia.googlesource.com/buildbot.git/perf/go/config" |
| 21 "skia.googlesource.com/buildbot.git/perf/go/db" |
21 "skia.googlesource.com/buildbot.git/perf/go/flags" | 22 "skia.googlesource.com/buildbot.git/perf/go/flags" |
22 "skia.googlesource.com/buildbot.git/perf/go/gitinfo" | 23 "skia.googlesource.com/buildbot.git/perf/go/gitinfo" |
23 "skia.googlesource.com/buildbot.git/perf/go/goldingester" | 24 "skia.googlesource.com/buildbot.git/perf/go/goldingester" |
24 "skia.googlesource.com/buildbot.git/perf/go/ingester" | 25 "skia.googlesource.com/buildbot.git/perf/go/ingester" |
25 "skia.googlesource.com/buildbot.git/perf/go/trybot" | 26 "skia.googlesource.com/buildbot.git/perf/go/trybot" |
26 ) | 27 ) |
27 | 28 |
28 // flags | 29 // flags |
29 var ( | 30 var ( |
30 timestampFile = flag.String("timestamp_file", "/tmp/timestamp.json", "F
ile where timestamp data for ingester runs will be stored.") | 31 timestampFile = flag.String("timestamp_file", "/tmp/timestamp.json", "F
ile where timestamp data for ingester runs will be stored.") |
31 tileDir = flag.String("tile_dir", "/tmp/tileStore2/", "Path where
tiles will be placed.") | 32 tileDir = flag.String("tile_dir", "/tmp/tileStore2/", "Path where
tiles will be placed.") |
32 gitRepoDir = flag.String("git_repo_dir", "../../../skia", "Directory
location for the Skia repo.") | 33 gitRepoDir = flag.String("git_repo_dir", "../../../skia", "Directory
location for the Skia repo.") |
33 runEvery = flag.Duration("run_every", 5*time.Minute, "How often th
e ingester should pull data from Google Storage.") | 34 runEvery = flag.Duration("run_every", 5*time.Minute, "How often th
e ingester should pull data from Google Storage.") |
34 runTrybotEvery = flag.Duration("run_trybot_every", 1*time.Minute, "How o
ften the ingester to pull trybot data from Google Storage.") | 35 runTrybotEvery = flag.Duration("run_trybot_every", 1*time.Minute, "How o
ften the ingester to pull trybot data from Google Storage.") |
35 run = flag.String("run", "nano,nano-trybot,golden", "A comma
separated list of ingesters to run.") | 36 run = flag.String("run", "nano,nano-trybot,golden", "A comma
separated list of ingesters to run.") |
36 graphiteServer = flag.String("graphite_server", "skia-monitoring-b:2003"
, "Where is Graphite metrics ingestion server running.") | 37 graphiteServer = flag.String("graphite_server", "skia-monitoring-b:2003"
, "Where is Graphite metrics ingestion server running.") |
37 doOauth = flag.Bool("oauth", true, "Run through the OAuth 2.0 flo
w on startup, otherwise use a GCE service account.") | 38 doOauth = flag.Bool("oauth", true, "Run through the OAuth 2.0 flo
w on startup, otherwise use a GCE service account.") |
| 39 local = flag.Bool("local", false, "Running locally if true. As
opposed to in production.") |
38 ) | 40 ) |
39 | 41 |
40 func Init() { | 42 func Init() { |
41 metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry) | 43 metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry) |
42 go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, 1*time.Minute
) | 44 go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, 1*time.Minute
) |
43 addr, _ := net.ResolveTCPAddr("tcp", *graphiteServer) | 45 addr, _ := net.ResolveTCPAddr("tcp", *graphiteServer) |
44 go metrics.Graphite(metrics.DefaultRegistry, 1*time.Minute, "ingest", ad
dr) | 46 go metrics.Graphite(metrics.DefaultRegistry, 1*time.Minute, "ingest", ad
dr) |
45 } | 47 } |
46 | 48 |
47 // Timestamps is used to read and write the timestamp file, which records the ti
me | 49 // Timestamps is used to read and write the timestamp file, which records the ti
me |
(...skipping 88 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
136 oneStep() | 138 oneStep() |
137 } | 139 } |
138 } | 140 } |
139 } | 141 } |
140 | 142 |
141 func main() { | 143 func main() { |
142 flag.Parse() | 144 flag.Parse() |
143 flags.Log() | 145 flags.Log() |
144 Init() | 146 Init() |
145 | 147 |
| 148 // Initialize the database. We might not need the oauth dialog if it fai
ls. |
| 149 db.Init(db.ProdConnectionString(*local)) |
| 150 |
146 var client *http.Client | 151 var client *http.Client |
147 var err error | 152 var err error |
148 if *doOauth { | 153 if *doOauth { |
149 client, err = auth.RunFlow() | 154 client, err = auth.RunFlow() |
150 if err != nil { | 155 if err != nil { |
151 glog.Fatalf("Failed to auth: %s", err) | 156 glog.Fatalf("Failed to auth: %s", err) |
152 } | 157 } |
153 } else { | 158 } else { |
154 client = nil | 159 client = nil |
155 // Add back service account access here when it's fixed. | 160 // Add back service account access here when it's fixed. |
156 } | 161 } |
157 | 162 |
158 ingester.Init(client) | 163 ingester.Init(client) |
159 trybot.Init() | |
160 goldingester.Init() | 164 goldingester.Init() |
161 ts := NewTimestamps(*timestampFile) | 165 ts := NewTimestamps(*timestampFile) |
162 ts.Read() | 166 ts.Read() |
163 glog.Infof("Timestamps: %#v\n", ts.Ingester) | 167 glog.Infof("Timestamps: %#v\n", ts.Ingester) |
164 | 168 |
165 git, err := gitinfo.NewGitInfo(*gitRepoDir, true) | 169 git, err := gitinfo.NewGitInfo(*gitRepoDir, true) |
166 if err != nil { | 170 if err != nil { |
167 glog.Fatal("Failed loading Git info: %s\n", err) | 171 glog.Fatal("Failed loading Git info: %s\n", err) |
168 } | 172 } |
169 | 173 |
170 // ingesters is a list of all the types of ingestion we can do. | 174 // ingesters is a list of all the types of ingestion we can do. |
171 ingesters := map[string]Process{ | 175 ingesters := map[string]Process{ |
172 "nano": NewIngestionProcess(ts, "ingest", git, *tileDir,
config.DATASET_NANO, ingester.NanoBenchIngestion, "nano-json-v1", *runEvery), | 176 "nano": NewIngestionProcess(ts, "ingest", git, *tileDir,
config.DATASET_NANO, ingester.NanoBenchIngestion, "nano-json-v1", *runEvery), |
173 "nano-trybot": NewIngestionProcess(ts, "trybot", git, *tileDir,
config.DATASET_NANO, trybot.TrybotIngestion, "trybot/nano-json-v1", *runTrybotEv
ery), | 177 "nano-trybot": NewIngestionProcess(ts, "trybot", git, *tileDir,
config.DATASET_NANO, trybot.TrybotIngestion, "trybot/nano-json-v1", *runTrybotEv
ery), |
174 "golden": NewIngestionProcess(ts, "golden", git, *tileDir,
config.DATASET_GOLDEN, goldingester.GoldenIngester, "dm-json-v1", *runEvery), | 178 "golden": NewIngestionProcess(ts, "golden", git, *tileDir,
config.DATASET_GOLDEN, goldingester.GoldenIngester, "dm-json-v1", *runEvery), |
175 } | 179 } |
176 | 180 |
177 for _, name := range strings.Split(*run, ",") { | 181 for _, name := range strings.Split(*run, ",") { |
178 glog.Infof("Process name: %s", name) | 182 glog.Infof("Process name: %s", name) |
179 if process, ok := ingesters[name]; ok { | 183 if process, ok := ingesters[name]; ok { |
180 go process() | 184 go process() |
181 } else { | 185 } else { |
182 glog.Fatalf("Not a valid ingester name: %s", name) | 186 glog.Fatalf("Not a valid ingester name: %s", name) |
183 } | 187 } |
184 } | 188 } |
185 | 189 |
186 select {} | 190 select {} |
187 } | 191 } |
OLD | NEW |