golden/pdfxform a pdf rasterization server

golden/go/pdfxform/main.go

Index: golden/go/pdfxform/main.go
diff --git a/golden/go/pdfxform/main.go b/golden/go/pdfxform/main.go
new file mode 100644
index 0000000000000000000000000000000000000000..25e605116a468947203606c87f586ab832f877d4
--- /dev/null
+++ b/golden/go/pdfxform/main.go
@@ -0,0 +1,506 @@
+// pdfxform is a server that rasterizes PDF documents into PNG
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/user"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/skia-dev/glog"
+	"go.skia.org/infra/go/auth"
+	"go.skia.org/infra/go/gs"
+	"go.skia.org/infra/go/pdf"
+	"go.skia.org/infra/go/util"
+	"go.skia.org/infra/perf/go/goldingester"
+	"google.golang.org/api/storage/v1"
+)
+
+////////////////////////////////////////////////////////////////////////////////
+
+const (
+	pngExt = "png"

[jcgregorio, 2015/06/26 19:21:05]

PNG_EXT

[hal.canary, 2015/06/26 22:10:52]

Done.

+	pdfExt = "pdf"

[jcgregorio, 2015/06/26 19:21:06]

PDF_EXT

[hal.canary, 2015/06/26 22:10:52]

Done.

+)
+
+////////////////////////////////////////////////////////////////////////////////
+
+// md5OfFile calculates the MD5 checksum of a file.
+func md5OfFile(path string) (string, error) {
+	md5 := md5.New()
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer util.Close(f)
+	if _, err = io.Copy(md5, f); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(md5.Sum(nil)), nil
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// removeIf is like util.Remove, but logs no error if the file does not exist.
+func removeIf(path string) {
+	if err := os.Remove(path); err != nil {
+		if !os.IsNotExist(err) {
+			glog.Errorf("Failed to Remove(%s): %v", path, err)
+		}
+	}
+}
+
+// isPDF returns true if the path appears to point to a PDF file.
+func isPDF(path string) bool {
+	f, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+	defer util.Close(f)
+	var buffer [4]byte
+	if n, err := f.Read(buffer[:]); n != 4 || err != nil {
+		return false
+	}
+	var magic = [4]byte{'%', 'P', 'D', 'F'}
+	return bytes.Equal(magic[:], buffer[:])
+}
+
+// writeTo opens a file and dumps the contents of the reader into it.
+func writeTo(path string, reader *io.ReadCloser) error {
+	defer util.Close(*reader)
+	file, err := os.Create(path)
+	if err == nil {
+		_, err = io.Copy(file, *reader)
+	}
+	return err
+}
+
+// assertNil logs the err and exits if it is not nil.
+func assertNil(err error) {
+	if err != nil {
+		errMsg := ""
+		if _, fileName, line, ok := runtime.Caller(1); ok {
+			errMsg = fmt.Sprintf("-called from: %s:%d", fileName, line)
+		}
+		glog.Fatalf("Unexpected error %s: %s", errMsg, err)
+	}
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// storageClient struct is used for uploading to cloud storage
+type storageClient struct {
+	httpClient     *http.Client
+	storageService *storage.Service
+}
+
+// getClient returns an authorized storage.Service and the
+// corresponding http.Client; if anything goes wrong, it logs a fatal
+// error.
+func getClient(cacheFilePath string) storageClient {
+	config := auth.OAuthConfig(cacheFilePath, auth.SCOPE_FULL_CONTROL)

[jcgregorio, 2015/06/26 19:21:05]

Create two command line flags:

  --local
  --oauth_cache_file

ala

   local = flag.Bool("local", false, "True if not running in prod.")
   oauthCacheFile = flag.String("oauth_cache_file", "oauth_cache.dat",
"..description..")


Then call:

	if client, err = auth.NewClient(!*local, *oauthCacheFile,
auth.SCOPE_FULL_CONTROL, auth.SCOPE_GCE); err != nil {
		glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
	}

[hal.canary, 2015/06/26 22:38:24]

Mostly done.

+	client, err := auth.RunFlow(config)
+	assertNil(err)
+	gsService, err := storage.New(client)
+	assertNil(err)
+	return storageClient{httpClient: client, storageService: gsService}
+}
+
+// gsFetch fetch the object's data from google storage
+func gsFetch(object *storage.Object, sc storageClient) (io.ReadCloser, int64, error) {
+	request, err := gs.RequestForStorageURL(object.MediaLink)
+	if err != nil {
+		return nil, -1, err
+	}
+	resp, err := sc.httpClient.Do(request)
+	if err != nil {
+		return nil, -1, err
+	}
+	if resp.StatusCode != 200 {
+		resp.Body.Close()
+		return nil, -1, fmt.Errorf("Failed to retrieve: %s %d %s", object.MediaLink, resp.StatusCode, resp.Status)
+	}
+	return resp.Body, resp.ContentLength, nil
+}
+
+// uploadFile uploads the specified file to the remote dir in Google
+// Storage. It also sets the appropriate ACLs on the uploaded file.
+// If the file already exists on the server, do nothing.
+func uploadFile(sc storageClient, input io.Reader, storageBucket, storagePath, accessControlEntity string) (bool, error) {
+	obj, _ := sc.storageService.Objects.Get(storageBucket, storagePath).Do()
+	if obj != nil {
+		return false, nil // noclobber
+	}
+	fullPath := fmt.Sprintf("gs://%s/%s", storageBucket, storagePath)
+	object := &storage.Object{Name: storagePath}
+	if _, err := sc.storageService.Objects.Insert(storageBucket, object).Media(input).Do(); err != nil {
+		return false, fmt.Errorf("Objects.Insert(%s) failed: %s", fullPath, err)
+	}
+	objectAcl := &storage.ObjectAccessControl{
+		Bucket: storageBucket, Entity: accessControlEntity, Object: storagePath, Role: "READER",
+	}
+	if _, err := sc.storageService.ObjectAccessControls.Insert(storageBucket, storagePath, objectAcl).Do(); err != nil {
+		return false, fmt.Errorf("Could not update ACL of %s: %s", fullPath, err)
+	}
+	return true, nil
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// The pdfXformer struct holds state (results, counter) and constants (bucket, directories)
+type pdfXformer struct {
+	client                 storageClient
+	storageBucket          string
+	storageJsonDirectory   string
+	storageImagesDirectory string
+	accessControlEntity    string
+	rasterizers            []pdf.Rasterizer
+	results                map[string]map[int]string
+	counter                int
+	identifier             string
+}
+
+const errorImageMd5 = "45aa8af265d16839402583df5756a7c6"
+
+// rasterizeOnce applies a single rastetizer to the given pdf file.
+// If the rasterizer fails, use the errorImage.  If everything
+// succeeds, upload the PNG.
+func (xformer *pdfXformer) rasterizeOnce(pdfPath string, rasterizerIndex int) (string, error) {
+	rasterizer := xformer.rasterizers[rasterizerIndex]
+	tempdir := filepath.Dir(pdfPath)
+	pngPath := path.Join(tempdir, fmt.Sprintf("%s.%s", rasterizer.String(), pngExt))
+	defer removeIf(pngPath)

[stephana, 2015/06/26 20:02:59]

We have a function for this already.

defer util.Remove(pngPath)

should do the trick and then you can get rid the removeIf function. 

[hal.canary, 2015/06/26 22:10:52]

I modified the name of the function to make it clear how it differs.

+	glog.Infof("> > > > rasterizing with %s", rasterizer)
+	err := rasterizer.Rasterize(pdfPath, pngPath)
+	if err != nil {
+		glog.Warningf("rasterizing %s with %s failed: %s", filepath.Base(pdfPath), rasterizer.String(), err)
+		return errorImageMd5, nil
+	}
+	md5, err := md5OfFile(pngPath)
+	if err != nil {
+		return "", err
+	}
+	f, err := os.Open(pngPath)
+	if err != nil {
+		return "", err
+	}
+	defer util.Close(f)
+	pngUploadPath := fmt.Sprintf("%s/%s.%s", xformer.storageImagesDirectory, md5, pngExt)
+	didUpload, err := uploadFile(xformer.client, f, xformer.storageBucket, pngUploadPath, xformer.accessControlEntity)
+	if err != nil {
+		return "", err
+	}
+	if didUpload {
+		glog.Infof("> > > > uploaded %s", pngUploadPath)
+	}
+	return md5, nil
+}
+
+// makeTmpDir returns a nicely-named directory for temp files in $TMPDIR
+func (xformer *pdfXformer) makeTmpDir() string {
+	if xformer.identifier == "" {
+		var host, userName string
+		if h, err := os.Hostname(); err == nil {
+			host = h
+			if i := strings.Index(host, "."); i >= 0 {
+				host = host[:i]
+			}
+		}
+		if currentUser, err := user.Current(); err == nil {
+			userName = currentUser.Username
+		}
+		userName = strings.Replace(userName, `\`, "_", -1)
+		xformer.identifier = fmt.Sprintf("%s.%s.%s.tmp.%d.", filepath.Base(os.Args[0]), host, userName, os.Getpid())

[jcgregorio, 2015/06/26 19:21:06]

When this is run in prod the exe will sit in /usr/local/bin, so you can't use
the exe directory to store data.

Add a command line flag:

   --data_dir

ala

   dataDir = flag.String("data_dir", "", "Directory to store data in.")

Use *dataDir as the place to store the data, if *dataDir is not set then you can
use ioutil.TempDir().

[hal.canary, 2015/06/26 22:10:52]

Done.

+	}
+	tempdir, err := ioutil.TempDir("", xformer.identifier)
+	assertNil(err)
+	return tempdir
+}
+
+func newResult(key map[string]string, rasterizerName, digest string) goldingester.Result {
+	keyCopy := map[string]string{}
+	for k, v := range key {
+		keyCopy[k] = v
+	}
+	keyCopy["rasterizer"] = rasterizerName
+	options := map[string]string{"ext": pngExt}
+	return goldingester.Result{Key: keyCopy, Digest: digest, Options: options}
+}
+
+// processResult rasterizes a single PDF result and returns a set of new results.
+func (xformer *pdfXformer) processResult(res goldingester.Result) []goldingester.Result {
+	rasterizedResults := []goldingester.Result{}
+	resultMap, found := xformer.results[res.Digest]
+	if found {
+		// Skip rasterizion steps: big win.
+		for index, rasterizer := range xformer.rasterizers {
+			digest, ok := resultMap[index]
+			if ok {
+				rasterizedResults = append(rasterizedResults,
+					newResult(res.Key, rasterizer.String(), digest))
+			} else {
+				glog.Errorf("missing rasterizer %s on %s", rasterizer.String(), res.Digest)
+			}
+		}
+		return rasterizedResults
+	}
+
+	tempdir := xformer.makeTmpDir()
+	defer util.RemoveAll(tempdir)
+	pdfPath := path.Join(tempdir, fmt.Sprintf("%s.pdf", res.Digest))
+	objectName := fmt.Sprintf("%s/%s.pdf", xformer.storageImagesDirectory, res.Digest)
+	storageURL := fmt.Sprintf("gs://%s/%s", xformer.storageBucket, objectName)
+	object, err := xformer.client.storageService.Objects.Get(xformer.storageBucket, objectName).Do()
+	if err != nil {
+		glog.Errorf("unable to find %s: %s", storageURL, err)
+		return []goldingester.Result{}
+	}
+	pdfData, _, err := gsFetch(object, xformer.client)
+	if err != nil {
+		glog.Errorf("unable to retrieve %s: %s", storageURL, err)
+		return []goldingester.Result{}
+	}
+	writeTo(pdfPath, &pdfData)
+	if !isPDF(pdfPath) {
+		glog.Errorf("%s is not a PDF", objectName)
+		return []goldingester.Result{}
+	}
+	resultMap = map[int]string{}
+	for index, rasterizer := range xformer.rasterizers {
+		digest, err := xformer.rasterizeOnce(pdfPath, index)
+		if err != nil {
+			glog.Errorf("rasterizer %s failed on %s.pdf: %s", rasterizer, res.Digest, err)
+			continue
+		}
+		rasterizedResults = append(rasterizedResults,
+			newResult(res.Key, rasterizer.String(), digest))
+		resultMap[index] = digest
+	}
+	xformer.results[res.Digest] = resultMap
+	return rasterizedResults
+}
+
+// processJsonFile reads a json file and produces a new json file
+// with rasterized results.
+func (xformer *pdfXformer) processJsonFile(jsonFileObject *storage.Object) {
+	jsonURL := fmt.Sprintf("gs://%s/%s", xformer.storageBucket, jsonFileObject.Name)
+	if jsonFileObject.Metadata["rasterized"] == "true" {
+		glog.Infof("> > skipping %s (already processed) {%d}", jsonURL, xformer.counter)
+		return
+	}
+	body, length, err := gsFetch(jsonFileObject, xformer.client)
+	if err != nil {
+		glog.Errorf("Failed to fetch %s", jsonURL)
+		return
+	}
+	if 0 == length {
+		util.Close(body)
+		glog.Infof("> > skipping %s (empty file) {%d}", jsonURL, xformer.counter)
+		return
+	}
+	dmstruct := goldingester.DMResults{}
+	err = json.NewDecoder(body).Decode(&dmstruct)
+	util.Close(body)
+	if err != nil {
+		glog.Errorf("Failed to parse %s", jsonURL)
+		return
+	}
+	countPdfResults := 0
+	for _, res := range dmstruct.Results {
+		if res.Options["ext"] == pdfExt {
+			countPdfResults++
+		}
+	}
+	if 0 == countPdfResults {
+		glog.Infof("> > 0 PDFs found %s {%d}", jsonURL, xformer.counter)
+		xformer.setRasterized(jsonFileObject)
+		return
+	}
+
+	glog.Infof("> > processing %d pdfs of %d results {%d}", countPdfResults, len(dmstruct.Results), xformer.counter)
+	rasterizedResults := []*goldingester.Result{}
+	i := 0
+	for _, res := range dmstruct.Results {
+		if res.Options["ext"] == pdfExt {
+			i++
+			glog.Infof("> > > processing %s.pdf [%d/%d] {%d}", res.Digest, i, countPdfResults, xformer.counter)
+			for _, rasterizedResult := range xformer.processResult(*res) {
+				rasterizedResults = append(rasterizedResults, &rasterizedResult)
+			}
+		}
+	}
+	newDMStruct := goldingester.DMResults{
+		BuildNumber: dmstruct.BuildNumber,
+		GitHash:     dmstruct.GitHash,
+		Key:         dmstruct.Key,
+		Results:     rasterizedResults,
+	}
+	newJson, err := json.Marshal(newDMStruct)
+	assertNil(err)
+
+	now := time.Now()
+	// Change the date; leave most of the rest of the path components.
+	jsonPathComponents := strings.Split(jsonFileObject.Name, "/") // []string
+	if len(jsonPathComponents) < 4 {
+		fmt.Errorf("unexpected number of path components %q", jsonPathComponents)
+		return
+	}
+	jsonPathComponents = jsonPathComponents[len(jsonPathComponents)-4:]
+	jsonPathComponents[1] += "-pdfxformer"
+	jsonUploadPath := fmt.Sprintf("%s/%d/%02d/%02d/%02d/%s",
+		xformer.storageJsonDirectory,
+		now.Year(),
+		int(now.Month()),
+		now.Day(),
+		now.Hour(),
+		strings.Join(jsonPathComponents, "/"))
+
+	_, err = uploadFile(xformer.client, bytes.NewReader(newJson), xformer.storageBucket, jsonUploadPath, xformer.accessControlEntity)
+	glog.Infof("> > wrote gs://%s/%s", xformer.storageBucket, jsonUploadPath)
+	newJsonFileObject, err := xformer.client.storageService.Objects.Get(xformer.storageBucket, jsonUploadPath).Do()
+	if err != nil {
+		glog.Errorf("Failed to find %s: %s", jsonUploadPath, err)
+	} else {
+		xformer.setRasterized(newJsonFileObject)
+	}
+	xformer.setRasterized(jsonFileObject)
+}
+
+// setRasterized sets the rasterized metadata flag of the given storage.Object
+func (xformer *pdfXformer) setRasterized(jsonFileObject *storage.Object) {
+	if nil == jsonFileObject.Metadata {
+		jsonFileObject.Metadata = map[string]string{}
+	}
+	jsonFileObject.Metadata["rasterized"] = "true"
+	_, err := xformer.client.storageService.Objects.Patch(xformer.storageBucket, jsonFileObject.Name, jsonFileObject).Do()
+	if err != nil {
+		glog.Errorf("Failed to update metadata of %s: %s", jsonFileObject.Name, err)
+	} else {
+		glog.Infof("> > Updated metadata of %s", jsonFileObject.Name)
+	}
+}
+
+// processTimeRange calls gs.GetLatestGSDirs to get a list of
+func (xformer *pdfXformer) processTimeRange(start time.Time, end time.Time) {
+	glog.Infof("Processing time range: (%s, %s)", start.Truncate(time.Second), end.Truncate(time.Second))
+	for _, dir := range gs.GetLatestGSDirs(start.Unix(), end.Unix(), xformer.storageJsonDirectory) {
+		glog.Infof("> Reading gs://%s/%s\n", xformer.storageBucket, dir)
+		requestedObjects := xformer.client.storageService.Objects.List(xformer.storageBucket).Prefix(dir).Fields(
+			"nextPageToken", "items/updated", "items/md5Hash", "items/mediaLink", "items/name", "items/metadata")
+		for requestedObjects != nil {
+			responseObjects, err := requestedObjects.Do()
+			if err != nil {
+				glog.Errorf("request %#v failed: %s", requestedObjects, err)
+				continue

[stephana, 2015/06/26 20:02:59]

wouldn't this repeat a failed request indefinitely ?

You could have a counter here and retry for a fixed number of times and then
leave the loop. 

[hal.canary, 2015/06/26 22:10:52]

Done.  Good catch.

+			}
+			for _, jsonObject := range responseObjects.Items {
+				xformer.counter++
+				glog.Infof("> > Processing object:  gs://%s/%s {%d}", xformer.storageBucket, jsonObject.Name, xformer.counter)
+				xformer.processJsonFile(jsonObject)
+			}
+			if len(responseObjects.NextPageToken) > 0 {
+				requestedObjects.PageToken(responseObjects.NextPageToken)
+			} else {
+				requestedObjects = nil
+			}
+		}
+	}
+	glog.Infof("finished time range.")
+}
+
+const errorImageData = "\x89PNG\x0D\x0A\x1A\x0A\x00\x00\x00\x0DIHDR\x00\x00" +
+	"\x00\xC1\x00\x00\x00#\x08\x06\x00\x00\x00-\xCEn\x15\x00\x00\x01\xF8IDATx" +
+	"\xDA\xED\x9C\xDB\x8E\xC3 \x0CDK\xB4\xFF\xFF\xCB\xD9\xA7V\x11\x8Ac\x1B\xCC" +
+	"-\x9Cy\xDAf)x\x0C\x03\xB6\x89\x9A\xCE\xF3<?\x00l\x8C\x03\x17\x00D\x00\x00" +
+	"\"\x00\x00\x11\x00\x80\x08\x00@\x04\x00 \x02\x00\x10\x01\x00\x88\x00\x80]" +
+	"\xF1\xB7\x8A\xA1)\xA5O\xCB\xCB\xED\x94\xD2\xEF\xEF\x1E\x97\xE8w|z\xDB\xE0" +
+	"\xF1I\xA9]w\x9CZ\xF2,\xE9\xFBXE\x00o\xC2\x0A|\xDE\xE6\xF3\xA5O\x82^\x93" +
+	"\xD1k\xE7}\xE23\xEBk\\5v\xAD\xF0j\x1A9\x01 '(\x8D\xB5\xA4\xD8\xCB\x13Gjm" +
+	"\xF3\xFF\x7F??\xC5\xD2\xD2\x98\x9A\xFDZ\xECj\xD9\xE5j\xF9h\xF1l\xFE}\xADM" +
+	"\xCE\xCB\xBA3\xB7\xF4{d\x1ER\xC2-\xEC$\x90\x16\x87\xE7\xB9\xB7\x8F([Z\x85" +
+	"XQ|\xA4>$\x1FzyFqo\xC1\xB7v\xFD\x94\x8E]\x9C\x13H\x8A\x97\x9E_\xAB!\xDA)" +
+	"\xF2m\xFBt\xE2x\xFA)\x8DU-\xBB\xB1\x87\xBB\xC6\xC7\xE3w\xEB\x84\xDF\xB5" +
+	"\xD7*mV\xBF[\xE6\xBA\xB6\xAA\xE3\x99\xCB\xAE\"\xB83&\"\x01\xEA\x95D\x95" +
+	"\x8E#M\xD4\x88\xD2\xEDu\xA1J\x0BDj\xBF\xCA<\xD5\xF8dxu\xC8cHT\xCD\xB8u" +
+	"\x15\xC9j\xE7N\xA5\xC5\xDE\xA1\xE5\xF0\xC48\xD2xI\xB1\xD21\xBB\xC2D\xB3" +
+	"\xF87\xA8\x0Ey\x9D\xA2\xC5iZ\xD5\xC4+\x84\x91\xA1\x94\x97\xFB\xDB\x04\x10" +
+	"\xCD\xB7\xF7&x\xF4Z(Z\xFBY/U\xBC\xE1\xDAn\xBF[\xF0\x06\xBEG\xCF\x1D\xE3" +
+	"\xFA\\+\xEF\xD5\x8C\x15\xD5\x7FM\xBE\xD2bW\xCC9\xCE\xB4\x10{\xCE\xA5\xC5'" +
+	"\xD3'\xC6ye\xC3\xBA\xBB\xE49C\x9E[D]\x9EX9YK\xA8Z\x9F\x1E;g\x0B\xB1\"\xC5" +
+	"^:\x97\xB56\x84\x9D\x04R\x89NR\xAD\xB5~\xAF-\x90\x9A{\x80\x11\xDCK\xEF" +
+	"\x06,c\xCC\xC87\"\xB4\xB2\x96\xE4\x8BK\xDF\xFC\xF8\x16\xD8\x1D\xBC@\x07" +
+	"\x10\x01.\x00\x88\x00\x00D\x00\x00\"\x00\x00\x11\x00\x80\x08\x00@\x04\x00" +
+	" \x02\x00\xB6\xC5?\xE4^\x82\xA6\x8A\xB0\x7F'\x00\x00\x00\x00IEND\xAEB`\x82"
+
+// uploadErrorImage should be run once to verify that the image is there

[stephana, 2015/06/26 20:03:00]

This is really not a good idea. The image should not be in here, but a command
line flag. MUCH easier to maintain down the road. And then to a test decode upon
startup. 

[jcgregorio, 2015/06/26 20:08:07]

Upon further thought I agree, should be cmd line flag.

On 2015/06/26 at 20:03:00, stephana wrote:

[hal.canary, 2015/06/26 22:10:52]

Done.

+func (xformer *pdfXformer) uploadErrorImage() {
+	// Check to see that the data is correct.
+	errorImageDataReader := bytes.NewReader([]byte(errorImageData))
+	md5sum := md5.New()
+	_, err := io.Copy(md5sum, errorImageDataReader)
+	assertNil(err)
+	if errorImageMd5 != hex.EncodeToString(md5sum.Sum(nil)) {
+		glog.Fatalf("errorImageData is corrupted")
+	}
+	_, err = errorImageDataReader.Seek(0, 0)
+	assertNil(err)
+
+	errorImagePath := fmt.Sprintf("%s/%s.png", xformer.storageImagesDirectory, errorImageMd5)
+	_, err = uploadFile(xformer.client, errorImageDataReader, xformer.storageBucket, errorImagePath, xformer.accessControlEntity)
+	assertNil(err) // If we can't upload this, we can't upload anything.
+}
+
+// Environment variables: we respect $TMPDIR
+// Arguments: glog uses -logtostderr, -log_dir
+func main() {
+	flag.Parse()

[jcgregorio, 2015/06/26 19:21:06]

Use common.InitWithMetrics.

  https://github.com/google/skia-buildbot/blob/master/go/common/common.go#L38

Also needs a command-line flag added for the graphite server.

+
+	// TODO(halcanary): where should this file exist?
+	configDir := path.Join(os.Getenv("HOME"), ".config")
+	assertNil(os.MkdirAll(configDir, 0700))

[stephana, 2015/06/26 20:03:00]

Making sure that directory exists is already implemented in fileutil:

See: 

https://github.com/google/skia-buildbot/blob/master/golden/go/filediffstore/f...

[hal.canary, 2015/06/26 22:38:24]

Done.

+
+	xformer := pdfXformer{

[stephana, 2015/06/26 20:02:59]

These should not be hard coded, but instead be flags (see skiacorrectness). The
hard coded values can be the defaults for the flag. 

There should also be a flag whether it's run local or not, in the latter case
the oauth info will come from the CGE instance's meta data. 

+		client:                 getClient(path.Join(configDir, "google_storage_token.data")),
+		storageBucket:          "chromium-skia-gm",
+		storageJsonDirectory:   "dm-json-v1",
+		storageImagesDirectory: "dm-images-v1",
+		accessControlEntity:    "domain-google.com",
+		results:                map[string]map[int]string{},
+	}
+
+	xformer.uploadErrorImage()
+
+	for _, rasterizer := range []pdf.Rasterizer{pdf.Pdfium{}, pdf.Poppler{}} {
+		if rasterizer.Enabled() {
+			xformer.rasterizers = append(xformer.rasterizers, rasterizer)
+		} else {
+			glog.Infof("rasterizer %s is disabled", rasterizer.String())
+		}
+	}
+	if len(xformer.rasterizers) == 0 {
+		glog.Fatalf("no rasterizers found")
+	}
+
+	end := time.Now()

[stephana, 2015/06/26 20:02:59]

This is not wrong, but the pattern that we use throughout is more like this: 


oneRun := func(start, end int64) {
  ... 
	xformer.processTimeRange(start, end)
  ... 
}

oneRun() 
for _ = range time.Tick(time.Minute) {
   oneRun() 
}

[hal.canary, 2015/06/26 22:10:52]

Done.

+	start := end.Add(-72 * time.Hour)
+	timeTicker := time.Tick(time.Minute)
+	for {
+		xformer.processTimeRange(start, end)
+		glog.Flush() // Flush before waiting for next tick; it may be a while.
+		_ = <-timeTicker
+		start = end
+		end = time.Now()
+	}
+}