| OLD | NEW |
| 1 package main | 1 package main |
| 2 | 2 |
| 3 // ingest is the command line tool for pulling performance data from Google | 3 // ingest is the command line tool for pulling performance data from Google |
| 4 // Storage and putting in Tiles. See the code in go/ingester for details on how | 4 // Storage and putting in Tiles. See the code in go/ingester for details on how |
| 5 // ingestion is done. | 5 // ingestion is done. |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "encoding/json" | 8 "encoding/json" |
| 9 "flag" | 9 "flag" |
| 10 "net" | 10 "net" |
| (...skipping 17 matching lines...) Expand all Loading... |
| 28 // flags | 28 // flags |
| 29 var ( | 29 var ( |
| 30 timestampFile = flag.String("timestamp_file", "/tmp/timestamp.json", "F
ile where timestamp data for ingester runs will be stored.") | 30 timestampFile = flag.String("timestamp_file", "/tmp/timestamp.json", "F
ile where timestamp data for ingester runs will be stored.") |
| 31 tileDir = flag.String("tile_dir", "/tmp/tileStore2/", "Path where
tiles will be placed.") | 31 tileDir = flag.String("tile_dir", "/tmp/tileStore2/", "Path where
tiles will be placed.") |
| 32 gitRepoDir = flag.String("git_repo_dir", "../../../skia", "Directory
location for the Skia repo.") | 32 gitRepoDir = flag.String("git_repo_dir", "../../../skia", "Directory
location for the Skia repo.") |
| 33 runEvery = flag.Duration("run_every", 5*time.Minute, "How often th
e ingester should pull data from Google Storage.") | 33 runEvery = flag.Duration("run_every", 5*time.Minute, "How often th
e ingester should pull data from Google Storage.") |
| 34 runTrybotEvery = flag.Duration("run_trybot_every", 1*time.Minute, "How o
ften the ingester to pull trybot data from Google Storage.") | 34 runTrybotEvery = flag.Duration("run_trybot_every", 1*time.Minute, "How o
ften the ingester to pull trybot data from Google Storage.") |
| 35 run = flag.String("run", "nano,nano-trybot,golden", "A comma
separated list of ingesters to run.") | 35 run = flag.String("run", "nano,nano-trybot,golden", "A comma
separated list of ingesters to run.") |
| 36 graphiteServer = flag.String("graphite_server", "skia-monitoring-b:2003"
, "Where is Graphite metrics ingestion server running.") | 36 graphiteServer = flag.String("graphite_server", "skia-monitoring-b:2003"
, "Where is Graphite metrics ingestion server running.") |
| 37 doOauth = flag.Bool("oauth", true, "Run through the OAuth 2.0 flo
w on startup, otherwise use a GCE service account.") | 37 doOauth = flag.Bool("oauth", true, "Run through the OAuth 2.0 flo
w on startup, otherwise use a GCE service account.") |
| 38 local = flag.Bool("local", false, "Running locally if true. As
opposed to in production.") |
| 38 ) | 39 ) |
| 39 | 40 |
| 40 func Init() { | 41 func Init() { |
| 41 metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry) | 42 metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry) |
| 42 go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, 1*time.Minute
) | 43 go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, 1*time.Minute
) |
| 43 addr, _ := net.ResolveTCPAddr("tcp", *graphiteServer) | 44 addr, _ := net.ResolveTCPAddr("tcp", *graphiteServer) |
| 44 go metrics.Graphite(metrics.DefaultRegistry, 1*time.Minute, "ingest", ad
dr) | 45 go metrics.Graphite(metrics.DefaultRegistry, 1*time.Minute, "ingest", ad
dr) |
| 45 } | 46 } |
| 46 | 47 |
| 47 // Timestamps is used to read and write the timestamp file, which records the ti
me | 48 // Timestamps is used to read and write the timestamp file, which records the ti
me |
| (...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 149 client, err = auth.RunFlow() | 150 client, err = auth.RunFlow() |
| 150 if err != nil { | 151 if err != nil { |
| 151 glog.Fatalf("Failed to auth: %s", err) | 152 glog.Fatalf("Failed to auth: %s", err) |
| 152 } | 153 } |
| 153 } else { | 154 } else { |
| 154 client = nil | 155 client = nil |
| 155 // Add back service account access here when it's fixed. | 156 // Add back service account access here when it's fixed. |
| 156 } | 157 } |
| 157 | 158 |
| 158 ingester.Init(client) | 159 ingester.Init(client) |
| 159 » trybot.Init() | 160 » trybot.Init(*local) |
| 160 goldingester.Init() | 161 goldingester.Init() |
| 161 ts := NewTimestamps(*timestampFile) | 162 ts := NewTimestamps(*timestampFile) |
| 162 ts.Read() | 163 ts.Read() |
| 163 glog.Infof("Timestamps: %#v\n", ts.Ingester) | 164 glog.Infof("Timestamps: %#v\n", ts.Ingester) |
| 164 | 165 |
| 165 git, err := gitinfo.NewGitInfo(*gitRepoDir, true) | 166 git, err := gitinfo.NewGitInfo(*gitRepoDir, true) |
| 166 if err != nil { | 167 if err != nil { |
| 167 glog.Fatal("Failed loading Git info: %s\n", err) | 168 glog.Fatal("Failed loading Git info: %s\n", err) |
| 168 } | 169 } |
| 169 | 170 |
| 170 // ingesters is a list of all the types of ingestion we can do. | 171 // ingesters is a list of all the types of ingestion we can do. |
| 171 ingesters := map[string]Process{ | 172 ingesters := map[string]Process{ |
| 172 "nano": NewIngestionProcess(ts, "ingest", git, *tileDir,
config.DATASET_NANO, ingester.NanoBenchIngestion, "nano-json-v1", *runEvery), | 173 "nano": NewIngestionProcess(ts, "ingest", git, *tileDir,
config.DATASET_NANO, ingester.NanoBenchIngestion, "nano-json-v1", *runEvery), |
| 173 "nano-trybot": NewIngestionProcess(ts, "trybot", git, *tileDir,
config.DATASET_NANO, trybot.TrybotIngestion, "trybot/nano-json-v1", *runTrybotEv
ery), | 174 "nano-trybot": NewIngestionProcess(ts, "trybot", git, *tileDir,
config.DATASET_NANO, trybot.TrybotIngestion, "trybot/nano-json-v1", *runTrybotEv
ery), |
| 174 "golden": NewIngestionProcess(ts, "golden", git, *tileDir,
config.DATASET_GOLDEN, goldingester.GoldenIngester, "dm-json-v1", *runEvery), | 175 "golden": NewIngestionProcess(ts, "golden", git, *tileDir,
config.DATASET_GOLDEN, goldingester.GoldenIngester, "dm-json-v1", *runEvery), |
| 175 } | 176 } |
| 176 | 177 |
| 177 for _, name := range strings.Split(*run, ",") { | 178 for _, name := range strings.Split(*run, ",") { |
| 178 glog.Infof("Process name: %s", name) | 179 glog.Infof("Process name: %s", name) |
| 179 if process, ok := ingesters[name]; ok { | 180 if process, ok := ingesters[name]; ok { |
| 180 go process() | 181 go process() |
| 181 } else { | 182 } else { |
| 182 glog.Fatalf("Not a valid ingester name: %s", name) | 183 glog.Fatalf("Not a valid ingester name: %s", name) |
| 183 } | 184 } |
| 184 } | 185 } |
| 185 | 186 |
| 186 select {} | 187 select {} |
| 187 } | 188 } |
| OLD | NEW |