Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(398)

Side by Side Diff: monitoring/go/datahopper/main.go

Issue 796883002: buildbot package: actually find commits for each build during ingestion (Closed) Base URL: https://skia.googlesource.com/buildbot@master
Patch Set: Pretty format the JSON Created 6 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « go/buildbot/testdata/venue466.json ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 /* 1 /*
2 Pulls data from multiple sources and funnels into InfluxDB. 2 Pulls data from multiple sources and funnels into InfluxDB.
3 */ 3 */
4 4
5 package main 5 package main
6 6
7 import ( 7 import (
8 "flag" 8 "flag"
9 "path"
9 "time" 10 "time"
10 11
11 "github.com/golang/glog" 12 "github.com/golang/glog"
12 influxdb "github.com/influxdb/influxdb/client" 13 influxdb "github.com/influxdb/influxdb/client"
13 "skia.googlesource.com/buildbot.git/go/buildbot" 14 "skia.googlesource.com/buildbot.git/go/buildbot"
14 "skia.googlesource.com/buildbot.git/go/common" 15 "skia.googlesource.com/buildbot.git/go/common"
16 "skia.googlesource.com/buildbot.git/go/gitinfo"
15 "skia.googlesource.com/buildbot.git/go/metadata" 17 "skia.googlesource.com/buildbot.git/go/metadata"
16 "skia.googlesource.com/buildbot.git/monitoring/go/autoroll_ingest" 18 "skia.googlesource.com/buildbot.git/monitoring/go/autoroll_ingest"
17 ) 19 )
18 20
19 const ( 21 const (
20 INFLUXDB_NAME_METADATA_KEY = "influxdb_name" 22 INFLUXDB_NAME_METADATA_KEY = "influxdb_name"
21 INFLUXDB_PASSWORD_METADATA_KEY = "influxdb_password" 23 INFLUXDB_PASSWORD_METADATA_KEY = "influxdb_password"
24 SKIA_REPO = "https://skia.googlesource.com/skia"
22 ) 25 )
23 26
24 // flags 27 // flags
25 var ( 28 var (
26 graphiteServer = flag.String("graphite_server", "localhost:2003", "Whe re is Graphite metrics ingestion server running.") 29 graphiteServer = flag.String("graphite_server", "localhost:2003", "Whe re is Graphite metrics ingestion server running.")
27 useMetadata = flag.Bool("use_metadata", true, "Load sensitive value s from metadata not from flags.") 30 useMetadata = flag.Bool("use_metadata", true, "Load sensitive value s from metadata not from flags.")
28 influxDbHost = flag.String("influxdb_host", "localhost:8086", "The I nfluxDB hostname.") 31 influxDbHost = flag.String("influxdb_host", "localhost:8086", "The I nfluxDB hostname.")
29 influxDbName = flag.String("influxdb_name", "root", "The InfluxDB us ername.") 32 influxDbName = flag.String("influxdb_name", "root", "The InfluxDB us ername.")
30 influxDbPassword = flag.String("influxdb_password", "root", "The InfluxD B password.") 33 influxDbPassword = flag.String("influxdb_password", "root", "The InfluxD B password.")
31 influxDbDatabase = flag.String("influxdb_database", "", "The InfluxDB da tabase.") 34 influxDbDatabase = flag.String("influxdb_database", "", "The InfluxDB da tabase.")
(...skipping 22 matching lines...) Expand all
54 glog.Fatalf("Failed to initialize InfluxDB client: %s", err) 57 glog.Fatalf("Failed to initialize InfluxDB client: %s", err)
55 } 58 }
56 59
57 // Data generation goroutines. 60 // Data generation goroutines.
58 go autoroll_ingest.LoadAutoRollData(dbClient, *workdir) 61 go autoroll_ingest.LoadAutoRollData(dbClient, *workdir)
59 go func() { 62 go func() {
60 // Initialize the buildbot database. 63 // Initialize the buildbot database.
61 if err := buildbot.InitDB(buildbot.ProdDatabaseConfig(*testing)) ; err != nil { 64 if err := buildbot.InitDB(buildbot.ProdDatabaseConfig(*testing)) ; err != nil {
62 glog.Fatal(err) 65 glog.Fatal(err)
63 } 66 }
64 67 » » // Create the Git repo.
68 » » skiaRepo, err := gitinfo.CloneOrUpdate(SKIA_REPO, path.Join(*wor kdir, "buildbot_git", "skia"), true)
69 » » if err != nil {
70 » » » glog.Fatal(err)
71 » » }
65 // Ingest data in a loop. 72 // Ingest data in a loop.
66 for _ = range time.Tick(30 * time.Second) { 73 for _ = range time.Tick(30 * time.Second) {
67 » » » if err := buildbot.IngestNewBuilds(); err != nil { 74 » » » skiaRepo.Update(true, true)
75 » » » if err := buildbot.IngestNewBuilds(skiaRepo); err != nil {
68 glog.Errorf("Failed to ingest new builds: %v", e rr) 76 glog.Errorf("Failed to ingest new builds: %v", e rr)
69 } 77 }
70 } 78 }
71 }() 79 }()
72 80
73 // Wait while the above goroutines generate data. 81 // Wait while the above goroutines generate data.
74 select {} 82 select {}
75 } 83 }
OLDNEW
« no previous file with comments | « go/buildbot/testdata/venue466.json ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698