Create a new pulld for systemd servers.

push/go/pulld/main.go

Index: push/go/pulld/main.go
diff --git a/push/go/pulld/main.go b/push/go/pulld/main.go
new file mode 100644
index 0000000000000000000000000000000000000000..0c241c0eec991e149f554f1f2dc0cb8732f62eba
--- /dev/null
+++ b/push/go/pulld/main.go
@@ -0,0 +1,106 @@
+// pulld is an application that monitors and pulls down new Debian packages and installs them.
+//
+// It presumes each package is monitored via systemd and not monit, so there's
+// no monitoring of monit config files is done.
+//
+// Note that it is safe for pulld to install pulld since the list of installed
+// packages is written out before 'dpkg -i' is called.
+package main
+
+import (
+	"flag"
+	"os"
+	"strings"
+	"time"
+
+	"code.google.com/p/google-api-go-client/storage/v1"
+	"github.com/skia-dev/glog"
+	"go.skia.org/infra/go/common"
+	"go.skia.org/infra/go/util"
+	"go.skia.org/infra/push/go/gsauth"
+	"go.skia.org/infra/push/go/packages"
+)
+
+var (
+	graphiteServer        = flag.String("graphite_server", "skia-monitoring:2003", "Where is Graphite metrics ingestion server running.")
+	doOauth               = flag.Bool("oauth", true, "Run through the OAuth 2.0 flow on startup, otherwise use a GCE service account.")
+	oauthCacheFile        = flag.String("oauth_cache_file", "google_storage_token.data", "Path to the file where to cache cache the oauth credentials.")
+	installedPackagesFile = flag.String("installed_packages_file", "installed_packages.json", "Path to the file where to cache the list of installed debs.")
+)
+
+// differences returns all strings that appear in server but not local.
+func differences(server, local []string) ([]string, []string) {
+	newPackages := []string{}
+	installedPackages := []string{}
+	for _, s := range server {
+		if util.In(s, local) {
+			installedPackages = append(installedPackages, s)
+		} else {
+			newPackages = append(newPackages, s)
+		}
+	}
+	return newPackages, installedPackages
+}
+
+func main() {
+	hostname, err := os.Hostname()
+	if err != nil {
+		// Never call glog before common.Init*.
+		os.Exit(1)
+	}
+	common.InitWithMetrics("pulld."+hostname, graphiteServer)
+	glog.Infof("Running with hostname: %s", hostname)
+
+	client, err := gsauth.NewClient(*doOauth, *oauthCacheFile)
+	if err != nil {
+		glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
+	}
+	glog.Info("Got authenticated client.")
+
+	store, err := storage.New(client)
+	if err != nil {
+		glog.Fatalf("Failed to create storage service client: %s", err)
+	}
+
+	for _ = range time.Tick(time.Second * 15) {
+		glog.Info("About to read package list.")
+		// Read the old and new packages from their respective storage locations.
+		serverList, err := packages.InstalledForServer(client, store, hostname)
+		if err != nil {
+			glog.Errorf("Failed to retrieve remote package list: %s", err)
+			continue
+		}
+		localList, err := packages.FromLocalFile(*installedPackagesFile)
+		if err != nil {
+			glog.Errorf("Failed to retrieve local package list: %s", err)
+			continue
+		}
+
+		// Install any new or updated packages.
+		newPackages, installed := differences(serverList.Names, localList)
+		glog.Infof("New: %v, Installed: %v", newPackages, installed)
+
+		for _, name := range newPackages {
+			// If just an appname appears w/o a packge name then that means

[rmistry, 2015/05/18 20:42:44]

"package"

[jcgregorio, 2015/05/19 19:19:04]

Done.

+			// that package hasn't been selected, so just skip it for now.
+			if len(strings.Split(name, "/")) == 1 {
+				continue
+			}
+			installed = append(installed, name)
+			if err := packages.ToLocalFile(installed, *installedPackagesFile); err != nil {
+				glog.Errorf("Failed to write local package list: %s", err)
+				continue
+			}
+			if err := packages.Install(client, store, name); err != nil {
+				glog.Errorf("Failed to install package %s: %s", name, err)
+				// Pop last name from 'installed' then rewrite the file since the
+				// install failed.
+				installed = installed[:len(installed)-1]
+				if err := packages.ToLocalFile(installed, *installedPackagesFile); err != nil {
+					glog.Errorf("Failed to rewrite local package list after install failure for %s: %s", name, err)
+				}
+				continue
+			}
+		}
+	}
+}