| OLD | NEW |
| 1 package main | 1 package main |
| 2 | 2 |
| 3 // ingest is the command line tool for pulling performance data from Google | 3 // ingest is the command line tool for pulling performance data from Google |
| 4 // Storage and putting in Tiles. See the code in go/ingester for details on how | 4 // Storage and putting in Tiles. See the code in go/ingester for details on how |
| 5 // ingestion is done. | 5 // ingestion is done. |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "encoding/json" | 8 "encoding/json" |
| 9 "flag" | 9 "flag" |
| 10 "net" | 10 "net" |
| 11 "net/http" | 11 "net/http" |
| 12 "os" | 12 "os" |
| 13 "strings" | 13 "strings" |
| 14 "sync" | 14 "sync" |
| 15 "time" | 15 "time" |
| 16 | 16 |
| 17 "github.com/golang/glog" | 17 "github.com/golang/glog" |
| 18 "github.com/rcrowley/go-metrics" | 18 "github.com/rcrowley/go-metrics" |
| 19 "skia.googlesource.com/buildbot.git/perf/go/auth" | 19 "skia.googlesource.com/buildbot.git/perf/go/auth" |
| 20 "skia.googlesource.com/buildbot.git/perf/go/config" | 20 "skia.googlesource.com/buildbot.git/perf/go/config" |
| 21 "skia.googlesource.com/buildbot.git/perf/go/db" |
| 21 "skia.googlesource.com/buildbot.git/perf/go/flags" | 22 "skia.googlesource.com/buildbot.git/perf/go/flags" |
| 22 "skia.googlesource.com/buildbot.git/perf/go/gitinfo" | 23 "skia.googlesource.com/buildbot.git/perf/go/gitinfo" |
| 23 "skia.googlesource.com/buildbot.git/perf/go/goldingester" | 24 "skia.googlesource.com/buildbot.git/perf/go/goldingester" |
| 24 "skia.googlesource.com/buildbot.git/perf/go/ingester" | 25 "skia.googlesource.com/buildbot.git/perf/go/ingester" |
| 25 "skia.googlesource.com/buildbot.git/perf/go/trybot" | 26 "skia.googlesource.com/buildbot.git/perf/go/trybot" |
| 26 ) | 27 ) |
| 27 | 28 |
| 28 // flags | 29 // flags |
| 29 var ( | 30 var ( |
| 30 timestampFile = flag.String("timestamp_file", "/tmp/timestamp.json", "F
ile where timestamp data for ingester runs will be stored.") | 31 timestampFile = flag.String("timestamp_file", "/tmp/timestamp.json", "F
ile where timestamp data for ingester runs will be stored.") |
| 31 tileDir = flag.String("tile_dir", "/tmp/tileStore2/", "Path where
tiles will be placed.") | 32 tileDir = flag.String("tile_dir", "/tmp/tileStore2/", "Path where
tiles will be placed.") |
| 32 gitRepoDir = flag.String("git_repo_dir", "../../../skia", "Directory
location for the Skia repo.") | 33 gitRepoDir = flag.String("git_repo_dir", "../../../skia", "Directory
location for the Skia repo.") |
| 33 runEvery = flag.Duration("run_every", 5*time.Minute, "How often th
e ingester should pull data from Google Storage.") | 34 runEvery = flag.Duration("run_every", 5*time.Minute, "How often th
e ingester should pull data from Google Storage.") |
| 34 runTrybotEvery = flag.Duration("run_trybot_every", 1*time.Minute, "How o
ften the ingester to pull trybot data from Google Storage.") | 35 runTrybotEvery = flag.Duration("run_trybot_every", 1*time.Minute, "How o
ften the ingester to pull trybot data from Google Storage.") |
| 35 run = flag.String("run", "nano,nano-trybot,golden", "A comma
separated list of ingesters to run.") | 36 run = flag.String("run", "nano,nano-trybot,golden", "A comma
separated list of ingesters to run.") |
| 36 graphiteServer = flag.String("graphite_server", "skia-monitoring-b:2003"
, "Where is Graphite metrics ingestion server running.") | 37 graphiteServer = flag.String("graphite_server", "skia-monitoring-b:2003"
, "Where is Graphite metrics ingestion server running.") |
| 37 doOauth = flag.Bool("oauth", true, "Run through the OAuth 2.0 flo
w on startup, otherwise use a GCE service account.") | 38 doOauth = flag.Bool("oauth", true, "Run through the OAuth 2.0 flo
w on startup, otherwise use a GCE service account.") |
| 39 local = flag.Bool("local", false, "Running locally if true. As
opposed to in production.") |
| 38 ) | 40 ) |
| 39 | 41 |
| 40 func Init() { | 42 func Init() { |
| 41 metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry) | 43 metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry) |
| 42 go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, 1*time.Minute
) | 44 go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, 1*time.Minute
) |
| 43 addr, _ := net.ResolveTCPAddr("tcp", *graphiteServer) | 45 addr, _ := net.ResolveTCPAddr("tcp", *graphiteServer) |
| 44 go metrics.Graphite(metrics.DefaultRegistry, 1*time.Minute, "ingest", ad
dr) | 46 go metrics.Graphite(metrics.DefaultRegistry, 1*time.Minute, "ingest", ad
dr) |
| 45 } | 47 } |
| 46 | 48 |
| 47 // Timestamps is used to read and write the timestamp file, which records the ti
me | 49 // Timestamps is used to read and write the timestamp file, which records the ti
me |
| (...skipping 88 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 136 oneStep() | 138 oneStep() |
| 137 } | 139 } |
| 138 } | 140 } |
| 139 } | 141 } |
| 140 | 142 |
| 141 func main() { | 143 func main() { |
| 142 flag.Parse() | 144 flag.Parse() |
| 143 flags.Log() | 145 flags.Log() |
| 144 Init() | 146 Init() |
| 145 | 147 |
| 148 // Initialize the database. We might not need the oauth dialog if it fai
ls. |
| 149 db.Init(db.ProdConnectionString(*local)) |
| 150 |
| 146 var client *http.Client | 151 var client *http.Client |
| 147 var err error | 152 var err error |
| 148 if *doOauth { | 153 if *doOauth { |
| 149 client, err = auth.RunFlow() | 154 client, err = auth.RunFlow() |
| 150 if err != nil { | 155 if err != nil { |
| 151 glog.Fatalf("Failed to auth: %s", err) | 156 glog.Fatalf("Failed to auth: %s", err) |
| 152 } | 157 } |
| 153 } else { | 158 } else { |
| 154 client = nil | 159 client = nil |
| 155 // Add back service account access here when it's fixed. | 160 // Add back service account access here when it's fixed. |
| 156 } | 161 } |
| 157 | 162 |
| 158 ingester.Init(client) | 163 ingester.Init(client) |
| 159 trybot.Init() | |
| 160 goldingester.Init() | 164 goldingester.Init() |
| 161 ts := NewTimestamps(*timestampFile) | 165 ts := NewTimestamps(*timestampFile) |
| 162 ts.Read() | 166 ts.Read() |
| 163 glog.Infof("Timestamps: %#v\n", ts.Ingester) | 167 glog.Infof("Timestamps: %#v\n", ts.Ingester) |
| 164 | 168 |
| 165 git, err := gitinfo.NewGitInfo(*gitRepoDir, true) | 169 git, err := gitinfo.NewGitInfo(*gitRepoDir, true) |
| 166 if err != nil { | 170 if err != nil { |
| 167 glog.Fatal("Failed loading Git info: %s\n", err) | 171 glog.Fatal("Failed loading Git info: %s\n", err) |
| 168 } | 172 } |
| 169 | 173 |
| 170 // ingesters is a list of all the types of ingestion we can do. | 174 // ingesters is a list of all the types of ingestion we can do. |
| 171 ingesters := map[string]Process{ | 175 ingesters := map[string]Process{ |
| 172 "nano": NewIngestionProcess(ts, "ingest", git, *tileDir,
config.DATASET_NANO, ingester.NanoBenchIngestion, "nano-json-v1", *runEvery), | 176 "nano": NewIngestionProcess(ts, "ingest", git, *tileDir,
config.DATASET_NANO, ingester.NanoBenchIngestion, "nano-json-v1", *runEvery), |
| 173 "nano-trybot": NewIngestionProcess(ts, "trybot", git, *tileDir,
config.DATASET_NANO, trybot.TrybotIngestion, "trybot/nano-json-v1", *runTrybotEv
ery), | 177 "nano-trybot": NewIngestionProcess(ts, "trybot", git, *tileDir,
config.DATASET_NANO, trybot.TrybotIngestion, "trybot/nano-json-v1", *runTrybotEv
ery), |
| 174 "golden": NewIngestionProcess(ts, "golden", git, *tileDir,
config.DATASET_GOLDEN, goldingester.GoldenIngester, "dm-json-v1", *runEvery), | 178 "golden": NewIngestionProcess(ts, "golden", git, *tileDir,
config.DATASET_GOLDEN, goldingester.GoldenIngester, "dm-json-v1", *runEvery), |
| 175 } | 179 } |
| 176 | 180 |
| 177 for _, name := range strings.Split(*run, ",") { | 181 for _, name := range strings.Split(*run, ",") { |
| 178 glog.Infof("Process name: %s", name) | 182 glog.Infof("Process name: %s", name) |
| 179 if process, ok := ingesters[name]; ok { | 183 if process, ok := ingesters[name]; ok { |
| 180 go process() | 184 go process() |
| 181 } else { | 185 } else { |
| 182 glog.Fatalf("Not a valid ingester name: %s", name) | 186 glog.Fatalf("Not a valid ingester name: %s", name) |
| 183 } | 187 } |
| 184 } | 188 } |
| 185 | 189 |
| 186 select {} | 190 select {} |
| 187 } | 191 } |
| OLD | NEW |