Merge pull request #46 from Roblox/cpu_shares

Add support for cpu shares.
This commit is contained in:
Shishir 2020-09-29 09:45:29 -07:00 committed by GitHub
commit 0b406c2320
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 77 additions and 30 deletions

View File

@ -5,6 +5,7 @@ ifndef $(GOLANG)
endif
export GO111MODULE=on
export GOOS=linux
default: build

View File

@ -29,6 +29,19 @@ import (
specs "github.com/opencontainers/runtime-spec/specs-go"
)
type ContainerConfig struct {
Image containerd.Image
ContainerName string
ContainerSnapshotName string
NetworkNamespacePath string
SecretsDir string
TaskDir string
AllocDir string
Env []string
MemoryLimit int64
CPUShares int64
}
func (d *Driver) isContainerdRunning() (bool, error) {
return d.client.IsServing(d.ctxContainerd)
}
@ -41,7 +54,7 @@ func (d *Driver) pullImage(imageName string) (containerd.Image, error) {
return d.client.Pull(d.ctxContainerd, imageName, containerd.WithPullUnpack)
}
func (d *Driver) createContainer(image containerd.Image, containerName, containerSnapshotName, containerdRuntime, netnsPath, secretsDir, taskDir, allocDir string, env []string, memoryLimit int64, config *TaskConfig) (containerd.Container, error) {
func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskConfig) (containerd.Container, error) {
if config.Command == "" && len(config.Args) > 0 {
return nil, fmt.Errorf("Command is empty. Cannot set --args without --command.")
}
@ -59,7 +72,7 @@ func (d *Driver) createContainer(image containerd.Image, containerName, containe
var opts []oci.SpecOpts
opts = append(opts, oci.WithImageConfigArgs(image, args))
opts = append(opts, oci.WithImageConfigArgs(containerConfig.Image, args))
// Enable privileged mode.
if config.Privileged {
@ -103,10 +116,13 @@ func (d *Driver) createContainer(image containerd.Image, containerName, containe
}
// Set environment variables.
opts = append(opts, oci.WithEnv(env))
opts = append(opts, oci.WithEnv(containerConfig.Env))
// Set cgroups memory limit.
opts = append(opts, oci.WithMemoryLimit(uint64(memoryLimit)))
opts = append(opts, oci.WithMemoryLimit(uint64(containerConfig.MemoryLimit)))
// Set CPU Shares.
opts = append(opts, oci.WithCPUShares(uint64(containerConfig.CPUShares)))
// Add linux devices into the container.
for _, device := range config.Devices {
@ -127,20 +143,20 @@ func (d *Driver) createContainer(image containerd.Image, containerName, containe
}
// Setup "/secrets" (NOMAD_SECRETS_DIR) in the container.
if secretsDir != "" {
secretsMount := buildMountpoint("bind", "/secrets", secretsDir, []string{"rbind", "ro"})
if containerConfig.SecretsDir != "" {
secretsMount := buildMountpoint("bind", "/secrets", containerConfig.SecretsDir, []string{"rbind", "ro"})
mounts = append(mounts, secretsMount)
}
// Setup "/local" (NOMAD_TASK_DIR) in the container.
if taskDir != "" {
taskMount := buildMountpoint("bind", "/local", taskDir, []string{"rbind", "ro"})
if containerConfig.TaskDir != "" {
taskMount := buildMountpoint("bind", "/local", containerConfig.TaskDir, []string{"rbind", "ro"})
mounts = append(mounts, taskMount)
}
// Setup "/alloc" (NOMAD_ALLOC_DIR) in the container.
if allocDir != "" {
allocMount := buildMountpoint("bind", "/alloc", allocDir, []string{"rbind", "ro"})
if containerConfig.AllocDir != "" {
allocMount := buildMountpoint("bind", "/alloc", containerConfig.AllocDir, []string{"rbind", "ro"})
mounts = append(mounts, allocMount)
}
@ -151,18 +167,18 @@ func (d *Driver) createContainer(image containerd.Image, containerName, containe
// nomad use CNI plugins e.g bridge to setup a network (and network namespace) for the container.
// CNI plugins need to be installed under /opt/cni/bin.
// network namespace is created at /var/run/netns/<id>.
// netnsPath is the path to the network namespace, which containerd joins to provide network
// for the container.
// containerConfig.NetworkNamespacePath is the path to the network namespace, which
// containerd joins to provide network for the container.
// NOTE: Only bridge networking mode is supported at this point.
if netnsPath != "" {
opts = append(opts, oci.WithLinuxNamespace(specs.LinuxNamespace{Type: specs.NetworkNamespace, Path: netnsPath}))
if containerConfig.NetworkNamespacePath != "" {
opts = append(opts, oci.WithLinuxNamespace(specs.LinuxNamespace{Type: specs.NetworkNamespace, Path: containerConfig.NetworkNamespacePath}))
}
return d.client.NewContainer(
d.ctxContainerd,
containerName,
containerd.WithRuntime(containerdRuntime, nil),
containerd.WithNewSnapshot(containerSnapshotName, image),
containerConfig.ContainerName,
containerd.WithRuntime(d.config.ContainerdRuntime, nil),
containerd.WithNewSnapshot(containerConfig.ContainerSnapshotName, containerConfig.Image),
containerd.WithNewSpec(opts...),
)
}

View File

@ -336,6 +336,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
return nil, nil, fmt.Errorf("failed to decode driver config: %v", err)
}
containerConfig := ContainerConfig{}
if driverConfig.HostNetwork && cfg.NetworkIsolation != nil {
return nil, nil, fmt.Errorf("host_network and bridge network mode are mutually exclusive, and only one of them should be set")
}
@ -347,40 +349,44 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
// Generate a random container name using docker namesgenerator package.
// https://github.com/moby/moby/blob/master/pkg/namesgenerator/names-generator.go
containerName := cfg.AllocID[:8] + "_" + namesgenerator.GetRandomName(1)
containerConfig.ContainerName = containerName
image, err := d.pullImage(driverConfig.Image)
var err error
containerConfig.Image, err = d.pullImage(driverConfig.Image)
if err != nil {
return nil, nil, fmt.Errorf("Error in pulling image: %v", err)
}
d.logger.Info(fmt.Sprintf("Successfully pulled %s image\n", image.Name()))
d.logger.Info(fmt.Sprintf("Successfully pulled %s image\n", containerConfig.Image.Name()))
// Setup environment variables.
var env []string
var secretsDir, taskDir, allocDir string
for key, val := range cfg.Env {
if skipOverride(key) {
continue
}
if key == "NOMAD_SECRETS_DIR" {
secretsDir = val
containerConfig.SecretsDir = val
}
if key == "NOMAD_TASK_DIR" {
taskDir = val
containerConfig.TaskDir = val
}
if key == "NOMAD_ALLOC_DIR" {
allocDir = val
containerConfig.AllocDir = val
}
env = append(env, fmt.Sprintf("%s=%s", key, val))
containerConfig.Env = append(containerConfig.Env, fmt.Sprintf("%s=%s", key, val))
}
containerSnapshotName := fmt.Sprintf("%s-snapshot", containerName)
var netnsPath string
containerConfig.ContainerSnapshotName = fmt.Sprintf("%s-snapshot", containerName)
if cfg.NetworkIsolation != nil && cfg.NetworkIsolation.Path != "" {
netnsPath = cfg.NetworkIsolation.Path
containerConfig.NetworkNamespacePath = cfg.NetworkIsolation.Path
}
container, err := d.createContainer(image, containerName, containerSnapshotName, d.config.ContainerdRuntime, netnsPath, secretsDir, taskDir, allocDir, env, cfg.Resources.LinuxResources.MemoryLimitBytes, &driverConfig)
// memory and cpu are coming from the resources stanza of the nomad job.
// https://www.nomadproject.io/docs/job-specification/resources
containerConfig.MemoryLimit = cfg.Resources.LinuxResources.MemoryLimitBytes
containerConfig.CPUShares = cfg.Resources.LinuxResources.CPUShares
container, err := d.createContainer(&containerConfig, &driverConfig)
if err != nil {
return nil, nil, fmt.Errorf("Error in creating container: %v", err)
}

5
go.mod
View File

@ -11,7 +11,7 @@ require (
github.com/appc/spec v0.8.11 // indirect
github.com/checkpoint-restore/go-criu v0.0.0-20191125063657-fcdcd07065c5 // indirect
github.com/containerd/cgroups v0.0.0-20200609174450-80c669f4bad0
github.com/containerd/containerd v1.3.0
github.com/containerd/containerd v1.4.1
github.com/containerd/go-cni v0.0.0-20191121212822-60d125212faf // indirect
github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd
github.com/containernetworking/plugins v0.8.3 // indirect
@ -26,6 +26,8 @@ require (
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
github.com/fsouza/go-dockerclient v1.6.0 // indirect
github.com/gogo/googleapis v1.4.0 // indirect
github.com/google/uuid v1.1.2 // indirect
github.com/hashicorp/consul v1.6.2 // indirect
github.com/hashicorp/consul-template v0.23.0
github.com/hashicorp/go-envparse v0.0.0-20190703193109-150b3a2a4611 // indirect
@ -49,6 +51,7 @@ require (
github.com/vbatts/tar-split v0.11.1 // indirect
github.com/zclconf/go-cty v1.1.1 // indirect
go4.org v0.0.0-20191010144846-132d2879e1e9 // indirect
google.golang.org/grpc v1.32.0 // indirect
)
// use lower-case sirupsen

21
go.sum
View File

@ -66,6 +66,7 @@ github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/bsm/go-vlq v0.0.0-20150828105119-ec6e8d4f5f4e/go.mod h1:N+BjUcTjSxc2mtRGSCPsat1kze3CUtvJN3/jTXlp29k=
github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/checkpoint-restore/go-criu v0.0.0-20191125063657-fcdcd07065c5 h1:950diauOSoCNaQfvLE0WaFadyI/ilKIjb6jEUQnm+hE=
github.com/checkpoint-restore/go-criu v0.0.0-20191125063657-fcdcd07065c5/go.mod h1:TrMrLQfeENAPYPRsJuq3jsqdlRh3lvi6trTZJG8+tho=
github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
@ -75,6 +76,7 @@ github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6D
github.com/circonus-labs/circonusllhist v0.1.3 h1:TJH+oke8D16535+jHExHj4nQvzlZrj7ug5D7I/orNUA=
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s=
github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
github.com/containerd/cgroups v0.0.0-20200609174450-80c669f4bad0 h1:rVearXKafAEaeQobBUHdSryM1DMDCP41vK0PtNeuWfY=
@ -84,6 +86,10 @@ github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on
github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
github.com/containerd/containerd v1.3.0 h1:xjvXQWABwS2uiv3TWgQt5Uth60Gu86LTGZXMJkjc7rY=
github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
github.com/containerd/containerd v1.4.0 h1:aWJB3lbDEaOxg1mkTBAINY2a+NsoKbAeRYefJPPRY+o=
github.com/containerd/containerd v1.4.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
github.com/containerd/containerd v1.4.1 h1:pASeJT3R3YyVn+94qEPk0SnU1OQ20Jd/T+SPKy9xehY=
github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 h1:PUD50EuOMkXVcpBIA/R95d56duJR9VxhwncsFbNnxW4=
@ -151,7 +157,10 @@ github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
github.com/elazarl/go-bindata-assetfs v0.0.0-20160803192304-e1a2a7ec64b0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
github.com/envoyproxy/go-control-plane v0.8.0/go.mod h1:GSSbY9P1neVhdY7G4wu+IK1rk/dqhiCC/4ExuWJZVuk=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.0.14/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
@ -180,6 +189,8 @@ github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6
github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/googleapis v1.1.0 h1:kFkMAZBNAn4j7K0GiZr8cRYzejq68VbheufiV3YuyFI=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/googleapis v1.4.0 h1:zgVt4UpGxcqVOw97aRGxT4svlcmdK35fynLNctY32zI=
github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@ -196,6 +207,8 @@ github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@ -206,6 +219,7 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck=
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
@ -215,6 +229,8 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
@ -484,6 +500,7 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
@ -669,6 +686,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.0.0-20180829000535-087779f1d2c9/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
@ -700,6 +718,9 @@ google.golang.org/grpc v1.22.0 h1:J0UbZOIrCAl+fpTOf8YLs4dJo8L/owV4LYVtAXQoPkw=
google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=