OLD | NEW |
1 package main | 1 package main |
2 | 2 |
3 // ingest is the command line tool for pulling performance data from Google | 3 // ingest is the command line tool for pulling performance data from Google |
4 // Storage and putting in Tiles. See the code in go/ingester for details on how | 4 // Storage and putting in Tiles. See the code in go/ingester for details on how |
5 // ingestion is done. | 5 // ingestion is done. |
6 | 6 |
7 import ( | 7 import ( |
8 "encoding/json" | 8 "encoding/json" |
9 "flag" | 9 "flag" |
10 "net" | 10 "net" |
(...skipping 17 matching lines...) Expand all Loading... |
28 // flags | 28 // flags |
29 var ( | 29 var ( |
30 timestampFile = flag.String("timestamp_file", "/tmp/timestamp.json", "F
ile where timestamp data for ingester runs will be stored.") | 30 timestampFile = flag.String("timestamp_file", "/tmp/timestamp.json", "F
ile where timestamp data for ingester runs will be stored.") |
31 tileDir = flag.String("tile_dir", "/tmp/tileStore2/", "Path where
tiles will be placed.") | 31 tileDir = flag.String("tile_dir", "/tmp/tileStore2/", "Path where
tiles will be placed.") |
32 gitRepoDir = flag.String("git_repo_dir", "../../../skia", "Directory
location for the Skia repo.") | 32 gitRepoDir = flag.String("git_repo_dir", "../../../skia", "Directory
location for the Skia repo.") |
33 runEvery = flag.Duration("run_every", 5*time.Minute, "How often th
e ingester should pull data from Google Storage.") | 33 runEvery = flag.Duration("run_every", 5*time.Minute, "How often th
e ingester should pull data from Google Storage.") |
34 runTrybotEvery = flag.Duration("run_trybot_every", 1*time.Minute, "How o
ften the ingester to pull trybot data from Google Storage.") | 34 runTrybotEvery = flag.Duration("run_trybot_every", 1*time.Minute, "How o
ften the ingester to pull trybot data from Google Storage.") |
35 run = flag.String("run", "nano,nano-trybot,golden", "A comma
separated list of ingesters to run.") | 35 run = flag.String("run", "nano,nano-trybot,golden", "A comma
separated list of ingesters to run.") |
36 graphiteServer = flag.String("graphite_server", "skia-monitoring-b:2003"
, "Where is Graphite metrics ingestion server running.") | 36 graphiteServer = flag.String("graphite_server", "skia-monitoring-b:2003"
, "Where is Graphite metrics ingestion server running.") |
37 doOauth = flag.Bool("oauth", true, "Run through the OAuth 2.0 flo
w on startup, otherwise use a GCE service account.") | 37 doOauth = flag.Bool("oauth", true, "Run through the OAuth 2.0 flo
w on startup, otherwise use a GCE service account.") |
| 38 local = flag.Bool("local", false, "Running locally if true. As
opposed to in production.") |
38 ) | 39 ) |
39 | 40 |
40 func Init() { | 41 func Init() { |
41 metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry) | 42 metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry) |
42 go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, 1*time.Minute
) | 43 go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, 1*time.Minute
) |
43 addr, _ := net.ResolveTCPAddr("tcp", *graphiteServer) | 44 addr, _ := net.ResolveTCPAddr("tcp", *graphiteServer) |
44 go metrics.Graphite(metrics.DefaultRegistry, 1*time.Minute, "ingest", ad
dr) | 45 go metrics.Graphite(metrics.DefaultRegistry, 1*time.Minute, "ingest", ad
dr) |
45 } | 46 } |
46 | 47 |
47 // Timestamps is used to read and write the timestamp file, which records the ti
me | 48 // Timestamps is used to read and write the timestamp file, which records the ti
me |
(...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
149 client, err = auth.RunFlow() | 150 client, err = auth.RunFlow() |
150 if err != nil { | 151 if err != nil { |
151 glog.Fatalf("Failed to auth: %s", err) | 152 glog.Fatalf("Failed to auth: %s", err) |
152 } | 153 } |
153 } else { | 154 } else { |
154 client = nil | 155 client = nil |
155 // Add back service account access here when it's fixed. | 156 // Add back service account access here when it's fixed. |
156 } | 157 } |
157 | 158 |
158 ingester.Init(client) | 159 ingester.Init(client) |
159 » trybot.Init() | 160 » trybot.Init(*local) |
160 goldingester.Init() | 161 goldingester.Init() |
161 ts := NewTimestamps(*timestampFile) | 162 ts := NewTimestamps(*timestampFile) |
162 ts.Read() | 163 ts.Read() |
163 glog.Infof("Timestamps: %#v\n", ts.Ingester) | 164 glog.Infof("Timestamps: %#v\n", ts.Ingester) |
164 | 165 |
165 git, err := gitinfo.NewGitInfo(*gitRepoDir, true) | 166 git, err := gitinfo.NewGitInfo(*gitRepoDir, true) |
166 if err != nil { | 167 if err != nil { |
167 glog.Fatal("Failed loading Git info: %s\n", err) | 168 glog.Fatal("Failed loading Git info: %s\n", err) |
168 } | 169 } |
169 | 170 |
170 // ingesters is a list of all the types of ingestion we can do. | 171 // ingesters is a list of all the types of ingestion we can do. |
171 ingesters := map[string]Process{ | 172 ingesters := map[string]Process{ |
172 "nano": NewIngestionProcess(ts, "ingest", git, *tileDir,
config.DATASET_NANO, ingester.NanoBenchIngestion, "nano-json-v1", *runEvery), | 173 "nano": NewIngestionProcess(ts, "ingest", git, *tileDir,
config.DATASET_NANO, ingester.NanoBenchIngestion, "nano-json-v1", *runEvery), |
173 "nano-trybot": NewIngestionProcess(ts, "trybot", git, *tileDir,
config.DATASET_NANO, trybot.TrybotIngestion, "trybot/nano-json-v1", *runTrybotEv
ery), | 174 "nano-trybot": NewIngestionProcess(ts, "trybot", git, *tileDir,
config.DATASET_NANO, trybot.TrybotIngestion, "trybot/nano-json-v1", *runTrybotEv
ery), |
174 "golden": NewIngestionProcess(ts, "golden", git, *tileDir,
config.DATASET_GOLDEN, goldingester.GoldenIngester, "dm-json-v1", *runEvery), | 175 "golden": NewIngestionProcess(ts, "golden", git, *tileDir,
config.DATASET_GOLDEN, goldingester.GoldenIngester, "dm-json-v1", *runEvery), |
175 } | 176 } |
176 | 177 |
177 for _, name := range strings.Split(*run, ",") { | 178 for _, name := range strings.Split(*run, ",") { |
178 glog.Infof("Process name: %s", name) | 179 glog.Infof("Process name: %s", name) |
179 if process, ok := ingesters[name]; ok { | 180 if process, ok := ingesters[name]; ok { |
180 go process() | 181 go process() |
181 } else { | 182 } else { |
182 glog.Fatalf("Not a valid ingester name: %s", name) | 183 glog.Fatalf("Not a valid ingester name: %s", name) |
183 } | 184 } |
184 } | 185 } |
185 | 186 |
186 select {} | 187 select {} |
187 } | 188 } |
OLD | NEW |