Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(130)

Side by Side Diff: datahopper/go/datahopper/main.go

Issue 813443002: Overhaul database package (Closed) Base URL: https://skia.googlesource.com/buildbot@master
Patch Set: Comments/readme cleanup Created 6 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 /* 1 /*
2 Pulls data from multiple sources and funnels into InfluxDB. 2 Pulls data from multiple sources and funnels into InfluxDB.
3 */ 3 */
4 4
5 package main 5 package main
6 6
7 import ( 7 import (
8 "flag" 8 "flag"
9 "path" 9 "path"
10 "time" 10 "time"
11 11
12 "github.com/golang/glog" 12 "github.com/golang/glog"
13 influxdb "github.com/influxdb/influxdb/client" 13 influxdb "github.com/influxdb/influxdb/client"
14 "skia.googlesource.com/buildbot.git/datahopper/go/autoroll_ingest" 14 "skia.googlesource.com/buildbot.git/datahopper/go/autoroll_ingest"
15 "skia.googlesource.com/buildbot.git/go/buildbot" 15 "skia.googlesource.com/buildbot.git/go/buildbot"
16 "skia.googlesource.com/buildbot.git/go/common" 16 "skia.googlesource.com/buildbot.git/go/common"
17 "skia.googlesource.com/buildbot.git/go/database"
18 "skia.googlesource.com/buildbot.git/go/gitinfo" 17 "skia.googlesource.com/buildbot.git/go/gitinfo"
19 "skia.googlesource.com/buildbot.git/go/metadata" 18 "skia.googlesource.com/buildbot.git/go/metadata"
20 ) 19 )
21 20
22 const ( 21 const (
23 INFLUXDB_NAME_METADATA_KEY = "influxdb_name" 22 INFLUXDB_NAME_METADATA_KEY = "influxdb_name"
24 INFLUXDB_PASSWORD_METADATA_KEY = "influxdb_password" 23 INFLUXDB_PASSWORD_METADATA_KEY = "influxdb_password"
25 SKIA_REPO = "https://skia.googlesource.com/skia" 24 SKIA_REPO = "https://skia.googlesource.com/skia"
26 ) 25 )
27 26
(...skipping 27 matching lines...) Expand all
55 IsUDP: false, 54 IsUDP: false,
56 }) 55 })
57 if err != nil { 56 if err != nil {
58 glog.Fatalf("Failed to initialize InfluxDB client: %s", err) 57 glog.Fatalf("Failed to initialize InfluxDB client: %s", err)
59 } 58 }
60 59
61 // Data generation goroutines. 60 // Data generation goroutines.
62 go autoroll_ingest.LoadAutoRollData(dbClient, *workdir) 61 go autoroll_ingest.LoadAutoRollData(dbClient, *workdir)
63 go func() { 62 go func() {
64 // Initialize the buildbot database. 63 // Initialize the buildbot database.
65 » » var conf *database.DatabaseConfig 64 » » if err := buildbot.InitDB(*testing); err != nil {
66 » » if *testing {
67 » » » conf = buildbot.LocalMySQLTestDatabaseConfig("test_user" , "")
68 » » } else {
69 » » » conf = buildbot.ProdDatabaseConfig(false)
70 » » }
71 » » if err := buildbot.InitDB(conf); err != nil {
72 glog.Fatal(err) 65 glog.Fatal(err)
73 } 66 }
74 // Create the Git repo. 67 // Create the Git repo.
75 skiaRepo, err := gitinfo.CloneOrUpdate(SKIA_REPO, path.Join(*wor kdir, "buildbot_git", "skia"), true) 68 skiaRepo, err := gitinfo.CloneOrUpdate(SKIA_REPO, path.Join(*wor kdir, "buildbot_git", "skia"), true)
76 if err != nil { 69 if err != nil {
77 glog.Fatal(err) 70 glog.Fatal(err)
78 } 71 }
79 // Ingest data in a loop. 72 // Ingest data in a loop.
80 for _ = range time.Tick(30 * time.Second) { 73 for _ = range time.Tick(30 * time.Second) {
81 skiaRepo.Update(true, true) 74 skiaRepo.Update(true, true)
82 glog.Info("Ingesting builds.") 75 glog.Info("Ingesting builds.")
83 if err := buildbot.IngestNewBuilds(skiaRepo); err != nil { 76 if err := buildbot.IngestNewBuilds(skiaRepo); err != nil {
84 glog.Errorf("Failed to ingest new builds: %v", e rr) 77 glog.Errorf("Failed to ingest new builds: %v", e rr)
85 } 78 }
86 } 79 }
87 }() 80 }()
88 81
89 // Wait while the above goroutines generate data. 82 // Wait while the above goroutines generate data.
90 select {} 83 select {}
91 } 84 }
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698