milo/appengine/buildbot/grpc.go - Issue 2944983003: [milo] {buildbucket,buildbot,swarming,logdog} -> backends/*.

Side by Side Diff: milo/appengine/buildbot/grpc.go

Issue 2944983003: [milo] {buildbucket,buildbot,swarming,logdog} -> backends/*. (Closed)

Patch Set: Created 3 years, 6 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

« milo/appengine/backends/swarming/build.go ('K') | « milo/appengine/buildbot/expectations/win_chromium_rel_ng.246309.build.json ('k') | milo/appengine/buildbot/grpc_test.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Hide Comments ('s')

OLD	NEW
	(Empty)
1 // Copyright 2016 The LUCI Authors. All rights reserved.

2 // Use of this source code is governed under the Apache License, Version 2.0

3 // that can be found in the LICENSE file.

4

5 package buildbot

6

7 import (

8 "bytes"

9 "compress/gzip"

10 "encoding/json"

11

12 "golang.org/x/net/context"

13 "google.golang.org/grpc"

14 "google.golang.org/grpc/codes"

15

16 "github.com/golang/protobuf/ptypes/timestamp"

17 "github.com/luci/gae/service/datastore"

18 "github.com/luci/luci-go/common/iotools"

19 "github.com/luci/luci-go/common/logging"

20 milo "github.com/luci/luci-go/milo/api/proto"

21 "github.com/luci/luci-go/server/auth"

22 )

23

24 // Service is a service implementation that displays BuildBot builds.

25 type Service struct{}

26

27 var errNotFoundGRPC = grpc.Errorf(codes.NotFound, "Master Not Found")

28

29 // GetBuildbotBuildJSON implements milo.BuildbotServer.

30 func (s Service) GetBuildbotBuildJSON(c context.Context, req milo.BuildbotBuil dRequest) (

31 *milo.BuildbotBuildJSON, error) {

32

33 if req.Master == "" {

34 return nil, grpc.Errorf(codes.InvalidArgument, "No master specif ied")

35 }

36 if req.Builder == "" {

37 return nil, grpc.Errorf(codes.InvalidArgument, "No builder speci fied")

38 }

39

40 cu := auth.CurrentUser(c)

41 logging.Debugf(c, "%s is requesting %s/%s/%d",

42 cu.Identity, req.Master, req.Builder, req.BuildNum)

43

44 b, err := getBuild(c, req.Master, req.Builder, int(req.BuildNum))

45 switch {

46 case err == errBuildNotFound:

47 return nil, grpc.Errorf(codes.NotFound, "Build not found")

48 case err == errNotAuth:

49 return nil, grpc.Errorf(codes.Unauthenticated, "Unauthenticated request")

50 case err != nil:

51 return nil, err

52 }

53

54 updatePostProcessBuild(b)

55 bs, err := json.Marshal(b)

56 if err != nil {

57 return nil, err

58 }

59

60 // Marshal the build back into JSON format.

61 return &milo.BuildbotBuildJSON{Data: bs}, nil

62 }

63

64 // GetBuildbotBuildsJSON implements milo.BuildbotServer.

65 func (s Service) GetBuildbotBuildsJSON(c context.Context, req milo.BuildbotBui ldsRequest) (

66 *milo.BuildbotBuildsJSON, error) {

67

68 if req.Master == "" {

69 return nil, grpc.Errorf(codes.InvalidArgument, "No master specif ied")

70 }

71 if req.Builder == "" {

72 return nil, grpc.Errorf(codes.InvalidArgument, "No builder speci fied")

73 }

74

75 limit := req.Limit

76 if limit == 0 {

77 limit = 20

78 }

79

80 cu := auth.CurrentUser(c)

81 logging.Debugf(c, "%s is requesting %s/%s (limit %d, cursor %s)",

82 cu.Identity, req.Master, req.Builder, limit, req.Cursor)

83

84 // Perform an ACL check by fetching the master.

85 _, err := getMasterEntry(c, req.Master)

86 switch {

87 case err == errMasterNotFound:

88 return nil, grpc.Errorf(codes.NotFound, "Master not found")

89 case err == errNotAuth:

90 return nil, grpc.Errorf(codes.Unauthenticated, "Unauthenticated request")

91 case err != nil:

92 return nil, err

93 }

94

95 q := datastore.NewQuery("buildbotBuild")

96 q = q.Eq("master", req.Master).

97 Eq("builder", req.Builder).

98 Limit(limit).

99 Order("-number")

100 if req.IncludeCurrent == false {

101 q = q.Eq("finished", true)

102 }

103 // Insert the cursor or offset.

104 if req.Cursor != "" {

105 cursor, err := datastore.DecodeCursor(c, req.Cursor)

106 if err != nil {

107 return nil, grpc.Errorf(codes.InvalidArgument, "Invalid cursor: %s", err.Error())

108 }

109 q = q.Start(cursor)

110 }

111 builds, nextCursor, err := runBuildsQuery(c, q, int32(req.Limit))

112 if err != nil {

113 return nil, err

114 }

115

116 results := make([]*milo.BuildbotBuildJSON, len(builds))

117 for i, b := range builds {

118 updatePostProcessBuild(b)

119

120 // In theory we could do this in parallel, but it doesn't actual ly go faster

121 // since AppEngine is single-cored.

122 bs, err := json.Marshal(b)

123 if err != nil {

124 return nil, err

125 }

126 results[i] = &milo.BuildbotBuildJSON{Data: bs}

127 }

128 buildsJSON := &milo.BuildbotBuildsJSON{

129 Builds: results,

130 }

131 if nextCursor != nil {

132 buildsJSON.Cursor = (*nextCursor).String()

133 }

134 return buildsJSON, nil

135 }

136

137 // GetCompressedMasterJSON assembles a CompressedMasterJSON object from the

138 // provided MasterRequest.

139 func (s Service) GetCompressedMasterJSON(c context.Context, req milo.MasterReq uest) (

140 *milo.CompressedMasterJSON, error) {

141

142 if req.Name == "" {

143 return nil, grpc.Errorf(codes.InvalidArgument, "No master specif ied")

144 }

145

146 cu := auth.CurrentUser(c)

147 logging.Debugf(c, "%s is making a master request for %s", cu.Identity, r eq.Name)

148

149 entry, err := getMasterEntry(c, req.Name)

150 switch {

151 case err == errMasterNotFound:

152 return nil, errNotFoundGRPC

153 case err == errNotAuth:

154 return nil, grpc.Errorf(codes.Unauthenticated, "Unauthenticated request")

155 case err != nil:

156 return nil, err

157 }

158 // Decompress it so we can inject current build information.

159 master := &buildbotMaster{}

160 if err = decodeMasterEntry(c, entry, master); err != nil {

161 return nil, err

162 }

163 for _, slave := range master.Slaves {

164 numBuilds := 0

165 for _, builds := range slave.RunningbuildsMap {

166 numBuilds += len(builds)

167 }

168 slave.Runningbuilds = make([]*buildbotBuild, 0, numBuilds)

169 for builderName, builds := range slave.RunningbuildsMap {

170 for _, buildNum := range builds {

171 slave.Runningbuilds = append(slave.Runningbuilds , &buildbotBuild{

172 Master: req.Name,

173 Buildername: builderName,

174 Number: buildNum,

175 })

176 }

177 }

178 if err := datastore.Get(c, slave.Runningbuilds); err != nil {

179 logging.WithError(err).Errorf(c,

180 "Encountered error while trying to fetch running builds for %s: %v",

181 master.Name, slave.Runningbuilds)

182 return nil, err

183 }

184

185 for _, b := range slave.Runningbuilds {

186 updatePostProcessBuild(b)

187 }

188 }

189

190 // Also inject cached builds information.

191 for builderName, builder := range master.Builders {

192 // Get the most recent 50 buildNums on the builder to simulate w hat the

193 // cachedBuilds field looks like from the real buildbot master j son.

194 q := datastore.NewQuery("buildbotBuild").

195 Eq("finished", true).

196 Eq("master", req.Name).

197 Eq("builder", builderName).

198 Limit(50).

199 Order("-number").

200 KeysOnly(true)

201 var builds []*buildbotBuild

202 err := getBuildQueryBatcher(c).GetAll(c, q, &builds)

203 if err != nil {

204 return nil, err

205 }

206 builder.CachedBuilds = make([]int, len(builds))

207 for i, b := range builds {

208 builder.CachedBuilds[i] = b.Number

209 }

210 }

211

212 // And re-compress it.

213 gzbs := bytes.Buffer{}

214 gsw := gzip.NewWriter(&gzbs)

215 cw := iotools.CountingWriter{Writer: gsw}

216 e := json.NewEncoder(&cw)

217 if err := e.Encode(master); err != nil {

218 gsw.Close()

219 return nil, err

220 }

221 gsw.Close()

222

223 logging.Infof(c, "Returning %d bytes", cw.Count)

224

225 return &milo.CompressedMasterJSON{

226 Internal: entry.Internal,

227 Modified: &timestamp.Timestamp{

228 Seconds: entry.Modified.Unix(),

229 Nanos: int32(entry.Modified.Nanosecond()),

230 },

231 Data: gzbs.Bytes(),

232 }, nil

233 }

OLD	NEW