Skip to content

K
k3s
Commit 012ffed6 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #45255 from crassirostris/sd-logging-e2e-performance-refactoring

Automatic merge from submit-queue Implement Stackdriver Logging e2e tests using PubSub Makes tests faster and mitigates Stackdriver Logging quotas issue. Fixes https://github.com/kubernetes/kubernetes/issues/47069
parents 2d023ab0 3cada4c7
Hide whitespace changes
Inline Side-by-side
Showing with 1485 additions and 130 deletions
Godeps/Godeps.json
View file @ 012ffed6
......@@ -2821,6 +2821,10 @@
"Rev": "e3824ed33c72bf7e81da0286772c34b987520914"
},
{
"ImportPath": "google.golang.org/api/pubsub/v1",
"Rev": "e3824ed33c72bf7e81da0286772c34b987520914"
},
{
"ImportPath": "google.golang.org/grpc",
"Comment": "v1.0.4",
"Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0"
......
Godeps/LICENSES
View file @ 012ffed6
......@@ -84431,6 +84431,41 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
================================================================================
= vendor/google.golang.org/api/pubsub/v1 licensed under: =
Copyright (c) 2011 Google Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
= vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40 -
================================================================================
================================================================================
= vendor/google.golang.org/grpc licensed under: =
Copyright 2014, Google Inc.
test/e2e/cluster-logging/BUILD
View file @ 012ffed6
......@@ -27,10 +27,12 @@ go_library(
"//vendor/golang.org/x/net/context:go_default_library",
"//vendor/golang.org/x/oauth2/google:go_default_library",
"//vendor/google.golang.org/api/logging/v2beta1:go_default_library",
"//vendor/google.golang.org/api/pubsub/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/util/integer:go_default_library",
],
)
......
test/e2e/cluster-logging/es.go
View file @ 012ffed6
......@@ -39,10 +39,11 @@ var _ = framework.KubeDescribe("Cluster level logging using Elasticsearch [Featu
It("should check that logs from containers are ingested into Elasticsearch", func() {
podName := "synthlogger"
esLogsProvider, err := newEsLogsProvider(f)
framework.ExpectNoError(err, "Failed to create GCL logs provider")
framework.ExpectNoError(err, "Failed to create Elasticsearch logs provider")
err = esLogsProvider.EnsureWorking()
framework.ExpectNoError(err, "Elasticsearch is not working")
err = esLogsProvider.Init()
defer esLogsProvider.Cleanup()
framework.ExpectNoError(err, "Failed to init Elasticsearch logs provider")
err = ensureSingleFluentdOnEachNode(f, esLogsProvider.FluentdApplicationName())
framework.ExpectNoError(err, "Fluentd deployed incorrectly")
......
test/e2e/cluster-logging/es_utils.go
View file @ 012ffed6
......@@ -46,12 +46,8 @@ func newEsLogsProvider(f *framework.Framework) (*esLogsProvider, error) {
return &esLogsProvider{Framework: f}, nil
}
func (logsProvider *esLogsProvider) FluentdApplicationName() string {
return "fluentd-es"
}
// Ensures that elasticsearch is running and ready to serve requests
func (logsProvider *esLogsProvider) EnsureWorking() error {
func (logsProvider *esLogsProvider) Init() error {
f := logsProvider.Framework
// Check for the existence of the Elasticsearch service.
By("Checking the Elasticsearch service exists.")
......@@ -157,7 +153,11 @@ func (logsProvider *esLogsProvider) EnsureWorking() error {
return nil
}
func (logsProvider *esLogsProvider) ReadEntries(pod *loggingPod) []*logEntry {
func (logsProvider *esLogsProvider) Cleanup() {
// Nothing to do
}
func (logsProvider *esLogsProvider) ReadEntries(pod *loggingPod) []logEntry {
f := logsProvider.Framework
proxyRequest, errProxy := framework.GetServicesProxyRequest(f.ClientSet, f.ClientSet.Core().RESTClient().Get())
......@@ -202,7 +202,7 @@ func (logsProvider *esLogsProvider) ReadEntries(pod *loggingPod) []*logEntry {
return nil
}
entries := []*logEntry{}
entries := []logEntry{}
// Iterate over the hits and populate the observed array.
for _, e := range h {
l, ok := e.(map[string]interface{})
......@@ -223,22 +223,12 @@ func (logsProvider *esLogsProvider) ReadEntries(pod *loggingPod) []*logEntry {
continue
}
timestampString, ok := source["@timestamp"].(string)
if !ok {
framework.Logf("Timestamp not of the expected type: %T", source["@timestamp"])
continue
}
timestamp, err := time.Parse(time.RFC3339, timestampString)
if err != nil {
framework.Logf("Timestamp was not in correct format: %s", timestampString)
continue
}
entries = append(entries, &logEntry{
Payload: msg,
Timestamp: timestamp,
})
entries = append(entries, logEntry{Payload: msg})
}
return entries
}
func (logsProvider *esLogsProvider) FluentdApplicationName() string {
return "fluentd-es"
}
test/e2e/cluster-logging/sd.go
View file @ 012ffed6
......@@ -39,8 +39,9 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL", func() {
gclLogsProvider, err := newGclLogsProvider(f)
framework.ExpectNoError(err, "Failed to create GCL logs provider")
err = gclLogsProvider.EnsureWorking()
framework.ExpectNoError(err, "GCL is not working")
err = gclLogsProvider.Init()
defer gclLogsProvider.Cleanup()
framework.ExpectNoError(err, "Failed to init GCL logs provider")
err = ensureSingleFluentdOnEachNode(f, gclLogsProvider.FluentdApplicationName())
framework.ExpectNoError(err, "Fluentd deployed incorrectly")
......
test/e2e/cluster-logging/sd_load.go
View file @ 012ffed6
......@@ -38,13 +38,17 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:Stackdr
gclLogsProvider, err := newGclLogsProvider(f)
framework.ExpectNoError(err, "Failed to create GCL logs provider")
err = gclLogsProvider.Init()
defer gclLogsProvider.Cleanup()
framework.ExpectNoError(err, "Failed to init GCL logs provider")
nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
nodeCount := len(nodes)
podCount := 30 * nodeCount
loggingDuration := 10 * time.Minute
linesPerSecond := 1000 * nodeCount
linesPerPod := linesPerSecond * int(loggingDuration.Seconds()) / podCount
ingestionTimeout := 60 * time.Minute
ingestionTimeout := 20 * time.Minute
By("Running logs generator pods")
pods := []*loggingPod{}
......@@ -56,9 +60,6 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:Stackdr
defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{})
}
By("Waiting for pods to succeed")
time.Sleep(loggingDuration)
By("Waiting for all log lines to be ingested")
config := &loggingTestConfig{
LogsProvider: gclLogsProvider,
......@@ -79,12 +80,16 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:Stackdr
gclLogsProvider, err := newGclLogsProvider(f)
framework.ExpectNoError(err, "Failed to create GCL logs provider")
err = gclLogsProvider.Init()
defer gclLogsProvider.Cleanup()
framework.ExpectNoError(err, "Failed to init GCL logs provider")
nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
maxPodCount := 10
jobDuration := 1 * time.Minute
linesPerPodPerSecond := 100
testDuration := 10 * time.Minute
ingestionTimeout := 60 * time.Minute
ingestionTimeout := 20 * time.Minute
podRunDelay := time.Duration(int64(jobDuration) / int64(maxPodCount))
podRunCount := int(testDuration.Seconds())/int(podRunDelay.Seconds()) - 1
......@@ -102,9 +107,6 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:Stackdr
time.Sleep(podRunDelay)
}
By("Waiting for the last pods to finish")
time.Sleep(jobDuration)
By("Waiting for all log lines to be ingested")
config := &loggingTestConfig{
LogsProvider: gclLogsProvider,
......
test/e2e/cluster-logging/sd_utils.go
View file @ 012ffed6
......@@ -17,36 +17,46 @@ limitations under the License.
package e2e
import (
"encoding/base64"
"encoding/json"
"fmt"
"sync"
"time"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/test/e2e/framework"
"golang.org/x/net/context"
"golang.org/x/oauth2/google"
gcl "google.golang.org/api/logging/v2beta1"
pubsub "google.golang.org/api/pubsub/v1"
)
const (
// GCL doesn't support page size more than 1000
gclPageSize = 1000
// If we failed to get response from GCL, it can be a random 500 or
// quota limit exceeded. So we retry for some time in case the problem will go away.
// Quota is enforced every 100 seconds, so we have to wait for more than
// that to reliably get the next portion.
queryGclRetryDelay = 100 * time.Second
queryGclRetryTimeout = 250 * time.Second
// The amount of time to wait before considering
// Stackdriver Logging sink operational
sinkInitialDelay = 1 * time.Minute
// The limit on the number of messages to pull from PubSub
maxPullLogMessages = 100 * 1000
// The limit on the number of messages in the cache for a pod
maxCachedMessagesPerPod = 10 * 1000
// PubSub topic with log entries polling interval
gclLoggingPollInterval = 100 * time.Millisecond
)
type gclLogsProvider struct {
GclService *gcl.Service
Framework *framework.Framework
}
func (gclLogsProvider *gclLogsProvider) EnsureWorking() error {
// We assume that GCL is always working
return nil
GclService *gcl.Service
PubsubService *pubsub.Service
Framework *framework.Framework
Topic *pubsub.Topic
Subscription *pubsub.Subscription
LogSink *gcl.LogSink
LogEntryCache map[string]chan logEntry
CacheMutex *sync.Mutex
PollingStopChannel chan struct{}
}
func newGclLogsProvider(f *framework.Framework) (*gclLogsProvider, error) {
......@@ -57,75 +67,206 @@ func newGclLogsProvider(f *framework.Framework) (*gclLogsProvider, error) {
return nil, err
}
pubsubService, err := pubsub.New(hc)
if err != nil {
return nil, err
}
provider := &gclLogsProvider{
GclService: gclService,
Framework: f,
GclService: gclService,
PubsubService: pubsubService,
Framework: f,
LogEntryCache: map[string]chan logEntry{},
CacheMutex: &sync.Mutex{},
PollingStopChannel: make(chan struct{}, 1),
}
return provider, nil
}
func (logsProvider *gclLogsProvider) FluentdApplicationName() string {
return "fluentd-gcp"
func (gclLogsProvider *gclLogsProvider) Init() error {
projectId := framework.TestContext.CloudConfig.ProjectID
nsName := gclLogsProvider.Framework.Namespace.Name
topic, err := gclLogsProvider.createPubSubTopic(projectId, nsName)
if err != nil {
return fmt.Errorf("failed to create PubSub topic: %v", err)
}
gclLogsProvider.Topic = topic
subs, err := gclLogsProvider.createPubSubSubscription(projectId, nsName, topic.Name)
if err != nil {
return fmt.Errorf("failed to create PubSub subscription: %v", err)
}
gclLogsProvider.Subscription = subs
logSink, err := gclLogsProvider.createGclLogSink(projectId, nsName, nsName, topic.Name)
if err != nil {
return fmt.Errorf("failed to create Stackdriver Logging sink: %v", err)
}
gclLogsProvider.LogSink = logSink
if err = gclLogsProvider.authorizeGclLogSink(); err != nil {
return fmt.Errorf("failed to authorize log sink: %v", err)
}
framework.Logf("Waiting for log sink to become operational")
// TODO: Replace with something more intelligent
time.Sleep(sinkInitialDelay)
go gclLogsProvider.pollLogs()
return nil
}
func (gclLogsProvider *gclLogsProvider) createPubSubTopic(projectId, topicName string) (*pubsub.Topic, error) {
topicFullName := fmt.Sprintf("projects/%s/topics/%s", projectId, topicName)
topic := &pubsub.Topic{
Name: topicFullName,
}
return gclLogsProvider.PubsubService.Projects.Topics.Create(topicFullName, topic).Do()
}
func (gclLogsProvider *gclLogsProvider) createPubSubSubscription(projectId, subsName, topicName string) (*pubsub.Subscription, error) {
subsFullName := fmt.Sprintf("projects/%s/subscriptions/%s", projectId, subsName)
subs := &pubsub.Subscription{
Name: subsFullName,
Topic: topicName,
}
return gclLogsProvider.PubsubService.Projects.Subscriptions.Create(subsFullName, subs).Do()
}
// Since GCL API is not easily available from the outside of cluster
// we use gcloud command to perform search with filter
func (gclLogsProvider *gclLogsProvider) ReadEntries(pod *loggingPod) []*logEntry {
filter := fmt.Sprintf("resource.labels.pod_id=%s AND resource.labels.namespace_id=%s AND timestamp>=\"%v\"",
pod.Name, gclLogsProvider.Framework.Namespace.Name, pod.LastTimestamp.Format(time.RFC3339))
framework.Logf("Reading entries from GCL with filter '%v'", filter)
func (gclLogsProvider *gclLogsProvider) createGclLogSink(projectId, nsName, sinkName, topicName string) (*gcl.LogSink, error) {
projectDst := fmt.Sprintf("projects/%s", projectId)
filter := fmt.Sprintf("resource.labels.namespace_id=%s AND resource.labels.container_name=%s", nsName, loggingContainerName)
sink := &gcl.LogSink{
Name: sinkName,
Destination: fmt.Sprintf("pubsub.googleapis.com/%s", topicName),
Filter: filter,
}
return gclLogsProvider.GclService.Projects.Sinks.Create(projectDst, sink).Do()
}
response := getResponseSafe(gclLogsProvider.GclService, filter, "")
func (gclLogsProvider *gclLogsProvider) authorizeGclLogSink() error {
topicsService := gclLogsProvider.PubsubService.Projects.Topics
policy, err := topicsService.GetIamPolicy(gclLogsProvider.Topic.Name).Do()
if err != nil {
return err
}
var entries []*logEntry
for response != nil && len(response.Entries) > 0 {
framework.Logf("Received %d entries from GCL", len(response.Entries))
binding := &pubsub.Binding{
Role: "roles/pubsub.publisher",
Members: []string{gclLogsProvider.LogSink.WriterIdentity},
}
policy.Bindings = append(policy.Bindings, binding)
req := &pubsub.SetIamPolicyRequest{Policy: policy}
if _, err = topicsService.SetIamPolicy(gclLogsProvider.Topic.Name, req).Do(); err != nil {
return err
}
return nil
}
for _, entry := range response.Entries {
if entry.TextPayload == "" {
func (gclLogsProvider *gclLogsProvider) pollLogs() {
wait.PollUntil(gclLoggingPollInterval, func() (bool, error) {
subsName := gclLogsProvider.Subscription.Name
subsService := gclLogsProvider.PubsubService.Projects.Subscriptions
req := &pubsub.PullRequest{
ReturnImmediately: true,
MaxMessages: maxPullLogMessages,
}
resp, err := subsService.Pull(subsName, req).Do()
if err != nil {
framework.Logf("Failed to pull messaged from PubSub due to %v", err)
return false, nil
}
ids := []string{}
for _, msg := range resp.ReceivedMessages {
ids = append(ids, msg.AckId)
logEntryEncoded, err := base64.StdEncoding.DecodeString(msg.Message.Data)
if err != nil {
framework.Logf("Got a message from pubsub that is not base64-encoded: %s", msg.Message.Data)
continue
}
timestamp, parseErr := time.Parse(time.RFC3339, entry.Timestamp)
if parseErr != nil {
var gclLogEntry gcl.LogEntry
if err := json.Unmarshal(logEntryEncoded, &gclLogEntry); err != nil {
framework.Logf("Failed to decode a pubsub message '%s': %v", logEntryEncoded, err)
continue
}
entries = append(entries, &logEntry{
Timestamp: timestamp,
Payload: entry.TextPayload,
})
podName := gclLogEntry.Resource.Labels["pod_id"]
ch := gclLogsProvider.getCacheChannel(podName)
ch <- logEntry{Payload: gclLogEntry.TextPayload}
}
nextToken := response.NextPageToken
if nextToken == "" {
break
if len(ids) > 0 {
ackReq := &pubsub.AcknowledgeRequest{AckIds: ids}
if _, err = subsService.Acknowledge(subsName, ackReq).Do(); err != nil {
framework.Logf("Failed to ack: %v", err)
}
}
response = getResponseSafe(gclLogsProvider.GclService, filter, response.NextPageToken)
return false, nil
}, gclLogsProvider.PollingStopChannel)
}
func (gclLogsProvider *gclLogsProvider) Cleanup() {
gclLogsProvider.PollingStopChannel <- struct{}{}
if gclLogsProvider.LogSink != nil {
projectId := framework.TestContext.CloudConfig.ProjectID
sinkNameId := fmt.Sprintf("projects/%s/sinks/%s", projectId, gclLogsProvider.LogSink.Name)
sinksService := gclLogsProvider.GclService.Projects.Sinks
if _, err := sinksService.Delete(sinkNameId).Do(); err != nil {
framework.Logf("Failed to delete LogSink: %v", err)
}
}
return entries
if gclLogsProvider.Subscription != nil {
subsService := gclLogsProvider.PubsubService.Projects.Subscriptions
if _, err := subsService.Delete(gclLogsProvider.Subscription.Name).Do(); err != nil {
framework.Logf("Failed to delete PubSub subscription: %v", err)
}
}
if gclLogsProvider.Topic != nil {
topicsService := gclLogsProvider.PubsubService.Projects.Topics
if _, err := topicsService.Delete(gclLogsProvider.Topic.Name).Do(); err != nil {
framework.Logf("Failed to delete PubSub topic: %v", err)
}
}
}
func getResponseSafe(gclService *gcl.Service, filter string, pageToken string) *gcl.ListLogEntriesResponse {
for start := time.Now(); time.Since(start) < queryGclRetryTimeout; time.Sleep(queryGclRetryDelay) {
response, err := gclService.Entries.List(&gcl.ListLogEntriesRequest{
ProjectIds: []string{
framework.TestContext.CloudConfig.ProjectID,
},
OrderBy: "timestamp desc",
Filter: filter,
PageSize: int64(gclPageSize),
PageToken: pageToken,
}).Do()
if err == nil {
return response
func (gclLogsProvider *gclLogsProvider) ReadEntries(pod *loggingPod) []logEntry {
var entries []logEntry
ch := gclLogsProvider.getCacheChannel(pod.Name)
polling_loop:
for {
select {
case entry := <-ch:
entries = append(entries, entry)
default:
break polling_loop
}
}
return entries
}
func (logsProvider *gclLogsProvider) FluentdApplicationName() string {
return "fluentd-gcp"
}
framework.Logf("Failed to get response from GCL due to %v, retrying", err)
func (gclLogsProvider *gclLogsProvider) getCacheChannel(podName string) chan logEntry {
gclLogsProvider.CacheMutex.Lock()
defer gclLogsProvider.CacheMutex.Unlock()
if ch, ok := gclLogsProvider.LogEntryCache[podName]; ok {
return ch
}
return nil
newCh := make(chan logEntry, maxCachedMessagesPerPod)
gclLogsProvider.LogEntryCache[podName] = newCh
return newCh
}
test/e2e/cluster-logging/utils.go
View file @ 012ffed6
......@@ -33,13 +33,16 @@ import (
const (
// Duration of delay between any two attempts to check if all logs are ingested
ingestionRetryDelay = 100 * time.Second
ingestionRetryDelay = 30 * time.Second
// Amount of requested cores for logging container in millicores
loggingContainerCpuRequest = 10
// Amount of requested memory for logging container in bytes
loggingContainerMemoryRequest = 10 * 1024 * 1024
// Name of the container used for logging tests
loggingContainerName = "logging-container"
)
var (
......@@ -51,26 +54,21 @@ var (
type loggingPod struct {
// Name of the pod
Name string
// If we didn't read some log entries, their
// timestamps should be no less than this timestamp.
// Effectively, timestamp of the last ingested entry
// for which there's no missing entry before it
LastTimestamp time.Time
// Cache of ingested and read entries
Occurrences map[int]*logEntry
Occurrences map[int]logEntry
// Number of lines expected to be ingested from this pod
ExpectedLinesNumber int
}
type logEntry struct {
Payload string
Timestamp time.Time
Payload string
}
type logsProvider interface {
Init() error
Cleanup()
ReadEntries(*loggingPod) []logEntry
FluentdApplicationName() string
EnsureWorking() error
ReadEntries(*loggingPod) []*logEntry
}
type loggingTestConfig struct {
......@@ -81,7 +79,7 @@ type loggingTestConfig struct {
MaxAllowedFluentdRestarts int
}
func (entry *logEntry) getLogEntryNumber() (int, bool) {
func (entry logEntry) getLogEntryNumber() (int, bool) {
submatch := logEntryMessageRegex.FindStringSubmatch(entry.Payload)
if submatch == nil || len(submatch) < 2 {
return 0, false
......@@ -96,10 +94,8 @@ func createLoggingPod(f *framework.Framework, podName string, nodeName string, t
createLogsGeneratorPod(f, podName, nodeName, totalLines, loggingDuration)
return &loggingPod{
Name: podName,
// It's used to avoid querying logs from before the pod was started
LastTimestamp: time.Now(),
Occurrences: make(map[int]*logEntry),
Name: podName,
Occurrences: make(map[int]logEntry),
ExpectedLinesNumber: totalLines,
}
}
......@@ -113,7 +109,7 @@ func createLogsGeneratorPod(f *framework.Framework, podName string, nodeName str
RestartPolicy: api_v1.RestartPolicyNever,
Containers: []api_v1.Container{
{
Name: podName,
Name: loggingContainerName,
Image: "gcr.io/google_containers/logs-generator:v0.1.0",
Env: []api_v1.EnvVar{
{
......@@ -146,7 +142,7 @@ func waitForSomeLogs(f *framework.Framework, config *loggingTestConfig) error {
podHasIngestedLogs := make([]bool, len(config.Pods))
podWithIngestedLogsCount := 0
for start := time.Now(); podWithIngestedLogsCount < len(config.Pods) && time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
for start := time.Now(); time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
for podIdx, pod := range config.Pods {
if podHasIngestedLogs[podIdx] {
continue
......@@ -167,6 +163,10 @@ func waitForSomeLogs(f *framework.Framework, config *loggingTestConfig) error {
}
}
}
if podWithIngestedLogsCount == len(config.Pods) {
break
}
}
if podWithIngestedLogsCount < len(config.Pods) {
......@@ -189,7 +189,7 @@ func waitForFullLogsIngestion(f *framework.Framework, config *loggingTestConfig)
missingByPod[podIdx] = pod.ExpectedLinesNumber
}
for start := time.Now(); totalMissing > 0 && time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
for start := time.Now(); time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
missing := 0
for podIdx, pod := range config.Pods {
if missingByPod[podIdx] == 0 {
......@@ -203,6 +203,8 @@ func waitForFullLogsIngestion(f *framework.Framework, config *loggingTestConfig)
totalMissing = missing
if totalMissing > 0 {
framework.Logf("Still missing %d lines in total", totalMissing)
} else {
break
}
}
......@@ -245,19 +247,14 @@ func pullMissingLogsCount(logsProvider logsProvider, pod *loggingPod) int {
if err != nil {
framework.Logf("Failed to get missing lines count from pod %s due to %v", pod.Name, err)
return pod.ExpectedLinesNumber
} else if missingOnPod > 0 {
framework.Logf("Pod %s is missing %d lines", pod.Name, missingOnPod)
} else {
framework.Logf("All logs from pod %s are ingested", pod.Name)
}
return missingOnPod
}
func getMissingLinesCount(logsProvider logsProvider, pod *loggingPod) (int, error) {
entries := logsProvider.ReadEntries(pod)
framework.Logf("Got %d entries from provider", len(entries))
for _, entry := range entries {
lineNumber, ok := entry.getLogEntryNumber()
if !ok {
......@@ -271,17 +268,6 @@ func getMissingLinesCount(logsProvider logsProvider, pod *loggingPod) (int, erro
}
}
for i := 0; i < pod.ExpectedLinesNumber; i++ {
entry, ok := pod.Occurrences[i]
if !ok {
break
}
if entry.Timestamp.After(pod.LastTimestamp) {
pod.LastTimestamp = entry.Timestamp
}
}
return pod.ExpectedLinesNumber - len(pod.Occurrences), nil
}
......
vendor/BUILD
View file @ 012ffed6
......@@ -362,6 +362,7 @@ filegroup(
"//vendor/google.golang.org/api/googleapi:all-srcs",
"//vendor/google.golang.org/api/logging/v2beta1:all-srcs",
"//vendor/google.golang.org/api/monitoring/v3:all-srcs",
"//vendor/google.golang.org/api/pubsub/v1:all-srcs",
"//vendor/google.golang.org/grpc:all-srcs",
"//vendor/gopkg.in/gcfg.v1:all-srcs",
"//vendor/gopkg.in/inf.v0:all-srcs",
......
vendor/google.golang.org/api/pubsub/v1/BUILD 0 → 100644
View file @ 012ffed6
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["pubsub-gen.go"],
tags = ["automanaged"],
deps = [
"//vendor/golang.org/x/net/context:go_default_library",
"//vendor/golang.org/x/net/context/ctxhttp:go_default_library",
"//vendor/google.golang.org/api/gensupport:go_default_library",
"//vendor/google.golang.org/api/googleapi:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)
vendor/google.golang.org/api/pubsub/v1/pubsub-api.json 0 → 100644
View file @ 012ffed6
{
"kind": "discovery#restDescription",
"description": "Provides reliable, many-to-many, asynchronous messaging between applications.\n",
"servicePath": "",
"rootUrl": "https://pubsub.googleapis.com/",
"basePath": "",
"ownerDomain": "google.com",
"name": "pubsub",
"batchPath": "batch",
"revision": "20170329",
"documentationLink": "https://cloud.google.com/pubsub/docs",
"id": "pubsub:v1",
"title": "Google Cloud Pub/Sub API",
"ownerName": "Google",
"discoveryVersion": "v1",
"resources": {
"projects": {
"resources": {
"topics": {
"methods": {
"get": {
"description": "Gets the configuration of a topic.",
"response": {
"$ref": "Topic"
},
"parameterOrder": [
"topic"
],
"httpMethod": "GET",
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"parameters": {
"topic": {
"location": "path",
"description": "The name of the topic to get.\nFormat is `projects/{project}/topics/{topic}`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/topics/[^/]+$"
}
},
"flatPath": "v1/projects/{projectsId}/topics/{topicsId}",
"path": "v1/{+topic}",
"id": "pubsub.projects.topics.get"
},
"publish": {
"flatPath": "v1/projects/{projectsId}/topics/{topicsId}:publish",
"path": "v1/{+topic}:publish",
"id": "pubsub.projects.topics.publish",
"description": "Adds one or more messages to the topic. Returns `NOT_FOUND` if the topic\ndoes not exist. The message payload must not be empty; it must contain\n either a non-empty data field, or at least one attribute.",
"request": {
"$ref": "PublishRequest"
},
"response": {
"$ref": "PublishResponse"
},
"parameterOrder": [
"topic"
],
"httpMethod": "POST",
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"parameters": {
"topic": {
"description": "The messages in the request will be published on this topic.\nFormat is `projects/{project}/topics/{topic}`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/topics/[^/]+$",
"location": "path"
}
}
},
"testIamPermissions": {
"response": {
"$ref": "TestIamPermissionsResponse"
},
"parameterOrder": [
"resource"
],
"httpMethod": "POST",
"parameters": {
"resource": {
"location": "path",
"description": "REQUIRED: The resource for which the policy detail is being requested.\nSee the operation documentation for the appropriate value for this field.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/topics/[^/]+$"
}
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"flatPath": "v1/projects/{projectsId}/topics/{topicsId}:testIamPermissions",
"path": "v1/{+resource}:testIamPermissions",
"id": "pubsub.projects.topics.testIamPermissions",
"request": {
"$ref": "TestIamPermissionsRequest"
},
"description": "Returns permissions that a caller has on the specified resource.\nIf the resource does not exist, this will return an empty set of\npermissions, not a NOT_FOUND error.\n\nNote: This operation is designed to be used for building permission-aware\nUIs and command-line tools, not for authorization checking. This operation\nmay \"fail open\" without warning."
},
"delete": {
"flatPath": "v1/projects/{projectsId}/topics/{topicsId}",
"id": "pubsub.projects.topics.delete",
"path": "v1/{+topic}",
"description": "Deletes the topic with the given name. Returns `NOT_FOUND` if the topic\ndoes not exist. After a topic is deleted, a new topic may be created with\nthe same name; this is an entirely new topic with none of the old\nconfiguration or subscriptions. Existing subscriptions to this topic are\nnot deleted, but their `topic` field is set to `_deleted-topic_`.",
"httpMethod": "DELETE",
"parameterOrder": [
"topic"
],
"response": {
"$ref": "Empty"
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"parameters": {
"topic": {
"location": "path",
"description": "Name of the topic to delete.\nFormat is `projects/{project}/topics/{topic}`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/topics/[^/]+$"
}
}
},
"list": {
"response": {
"$ref": "ListTopicsResponse"
},
"parameterOrder": [
"project"
],
"httpMethod": "GET",
"parameters": {
"pageToken": {
"description": "The value returned by the last `ListTopicsResponse`; indicates that this is\na continuation of a prior `ListTopics` call, and that the system should\nreturn the next page of data.",
"type": "string",
"location": "query"
},
"pageSize": {
"description": "Maximum number of topics to return.",
"format": "int32",
"type": "integer",
"location": "query"
},
"project": {
"location": "path",
"description": "The name of the cloud project that topics belong to.\nFormat is `projects/{project}`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+$"
}
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"flatPath": "v1/projects/{projectsId}/topics",
"path": "v1/{+project}/topics",
"id": "pubsub.projects.topics.list",
"description": "Lists matching topics."
},
"setIamPolicy": {
"response": {
"$ref": "Policy"
},
"parameterOrder": [
"resource"
],
"httpMethod": "POST",
"parameters": {
"resource": {
"location": "path",
"description": "REQUIRED: The resource for which the policy is being specified.\nSee the operation documentation for the appropriate value for this field.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/topics/[^/]+$"
}
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"flatPath": "v1/projects/{projectsId}/topics/{topicsId}:setIamPolicy",
"path": "v1/{+resource}:setIamPolicy",
"id": "pubsub.projects.topics.setIamPolicy",
"request": {
"$ref": "SetIamPolicyRequest"
},
"description": "Sets the access control policy on the specified resource. Replaces any\nexisting policy."
},
"create": {
"flatPath": "v1/projects/{projectsId}/topics/{topicsId}",
"id": "pubsub.projects.topics.create",
"path": "v1/{+name}",
"request": {
"$ref": "Topic"
},
"description": "Creates the given topic with the given name.",
"httpMethod": "PUT",
"parameterOrder": [
"name"
],
"response": {
"$ref": "Topic"
},
"parameters": {
"name": {
"location": "path",
"description": "The name of the topic. It must have the format\n`\"projects/{project}/topics/{topic}\"`. `{topic}` must start with a letter,\nand contain only letters (`[A-Za-z]`), numbers (`[0-9]`), dashes (`-`),\nunderscores (`_`), periods (`.`), tildes (`~`), plus (`+`) or percent\nsigns (`%`). It must be between 3 and 255 characters in length, and it\nmust not start with `\"goog\"`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/topics/[^/]+$"
}
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
]
},
"getIamPolicy": {
"flatPath": "v1/projects/{projectsId}/topics/{topicsId}:getIamPolicy",
"id": "pubsub.projects.topics.getIamPolicy",
"path": "v1/{+resource}:getIamPolicy",
"description": "Gets the access control policy for a resource.\nReturns an empty policy if the resource exists and does not have a policy\nset.",
"httpMethod": "GET",
"parameterOrder": [
"resource"
],
"response": {
"$ref": "Policy"
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"parameters": {
"resource": {
"description": "REQUIRED: The resource for which the policy is being requested.\nSee the operation documentation for the appropriate value for this field.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/topics/[^/]+$",
"location": "path"
}
}
}
},
"resources": {
"subscriptions": {
"methods": {
"list": {
"flatPath": "v1/projects/{projectsId}/topics/{topicsId}/subscriptions",
"id": "pubsub.projects.topics.subscriptions.list",
"path": "v1/{+topic}/subscriptions",
"description": "Lists the name of the subscriptions for this topic.",
"httpMethod": "GET",
"parameterOrder": [
"topic"
],
"response": {
"$ref": "ListTopicSubscriptionsResponse"
},
"parameters": {
"pageSize": {
"location": "query",
"description": "Maximum number of subscription names to return.",
"format": "int32",
"type": "integer"
},
"topic": {
"location": "path",
"description": "The name of the topic that subscriptions are attached to.\nFormat is `projects/{project}/topics/{topic}`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/topics/[^/]+$"
},
"pageToken": {
"location": "query",
"description": "The value returned by the last `ListTopicSubscriptionsResponse`; indicates\nthat this is a continuation of a prior `ListTopicSubscriptions` call, and\nthat the system should return the next page of data.",
"type": "string"
}
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
]
}
}
}
}
},
"subscriptions": {
"methods": {
"list": {
"httpMethod": "GET",
"response": {
"$ref": "ListSubscriptionsResponse"
},
"parameterOrder": [
"project"
],
"parameters": {
"pageToken": {
"location": "query",
"description": "The value returned by the last `ListSubscriptionsResponse`; indicates that\nthis is a continuation of a prior `ListSubscriptions` call, and that the\nsystem should return the next page of data.",
"type": "string"
},
"pageSize": {
"location": "query",
"description": "Maximum number of subscriptions to return.",
"format": "int32",
"type": "integer"
},
"project": {
"description": "The name of the cloud project that subscriptions belong to.\nFormat is `projects/{project}`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+$",
"location": "path"
}
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"flatPath": "v1/projects/{projectsId}/subscriptions",
"id": "pubsub.projects.subscriptions.list",
"path": "v1/{+project}/subscriptions",
"description": "Lists matching subscriptions."
},
"setIamPolicy": {
"request": {
"$ref": "SetIamPolicyRequest"
},
"description": "Sets the access control policy on the specified resource. Replaces any\nexisting policy.",
"response": {
"$ref": "Policy"
},
"parameterOrder": [
"resource"
],
"httpMethod": "POST",
"parameters": {
"resource": {
"description": "REQUIRED: The resource for which the policy is being specified.\nSee the operation documentation for the appropriate value for this field.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/subscriptions/[^/]+$",
"location": "path"
}
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"flatPath": "v1/projects/{projectsId}/subscriptions/{subscriptionsId}:setIamPolicy",
"path": "v1/{+resource}:setIamPolicy",
"id": "pubsub.projects.subscriptions.setIamPolicy"
},
"create": {
"response": {
"$ref": "Subscription"
},
"parameterOrder": [
"name"
],
"httpMethod": "PUT",
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"parameters": {
"name": {
"location": "path",
"description": "The name of the subscription. It must have the format\n`\"projects/{project}/subscriptions/{subscription}\"`. `{subscription}` must\nstart with a letter, and contain only letters (`[A-Za-z]`), numbers\n(`[0-9]`), dashes (`-`), underscores (`_`), periods (`.`), tildes (`~`),\nplus (`+`) or percent signs (`%`). It must be between 3 and 255 characters\nin length, and it must not start with `\"goog\"`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/subscriptions/[^/]+$"
}
},
"flatPath": "v1/projects/{projectsId}/subscriptions/{subscriptionsId}",
"path": "v1/{+name}",
"id": "pubsub.projects.subscriptions.create",
"description": "Creates a subscription to a given topic.\nIf the subscription already exists, returns `ALREADY_EXISTS`.\nIf the corresponding topic doesn't exist, returns `NOT_FOUND`.\n\nIf the name is not provided in the request, the server will assign a random\nname for this subscription on the same project as the topic, conforming\nto the\n[resource name format](https://cloud.google.com/pubsub/docs/overview#names).\nThe generated name is populated in the returned Subscription object.\nNote that for REST API requests, you must specify a name in the request.",
"request": {
"$ref": "Subscription"
}
},
"acknowledge": {
"flatPath": "v1/projects/{projectsId}/subscriptions/{subscriptionsId}:acknowledge",
"path": "v1/{+subscription}:acknowledge",
"id": "pubsub.projects.subscriptions.acknowledge",
"description": "Acknowledges the messages associated with the `ack_ids` in the\n`AcknowledgeRequest`. The Pub/Sub system can remove the relevant messages\nfrom the subscription.\n\nAcknowledging a message whose ack deadline has expired may succeed,\nbut such a message may be redelivered later. Acknowledging a message more\nthan once will not result in an error.",
"request": {
"$ref": "AcknowledgeRequest"
},
"response": {
"$ref": "Empty"
},
"parameterOrder": [
"subscription"
],
"httpMethod": "POST",
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"parameters": {
"subscription": {
"description": "The subscription whose message is being acknowledged.\nFormat is `projects/{project}/subscriptions/{sub}`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/subscriptions/[^/]+$",
"location": "path"
}
}
},
"getIamPolicy": {
"httpMethod": "GET",
"parameterOrder": [
"resource"
],
"response": {
"$ref": "Policy"
},
"parameters": {
"resource": {
"location": "path",
"description": "REQUIRED: The resource for which the policy is being requested.\nSee the operation documentation for the appropriate value for this field.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/subscriptions/[^/]+$"
}
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"flatPath": "v1/projects/{projectsId}/subscriptions/{subscriptionsId}:getIamPolicy",
"id": "pubsub.projects.subscriptions.getIamPolicy",
"path": "v1/{+resource}:getIamPolicy",
"description": "Gets the access control policy for a resource.\nReturns an empty policy if the resource exists and does not have a policy\nset."
},
"modifyAckDeadline": {
"response": {
"$ref": "Empty"
},
"parameterOrder": [
"subscription"
],
"httpMethod": "POST",
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"parameters": {
"subscription": {
"location": "path",
"description": "The name of the subscription.\nFormat is `projects/{project}/subscriptions/{sub}`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/subscriptions/[^/]+$"
}
},
"flatPath": "v1/projects/{projectsId}/subscriptions/{subscriptionsId}:modifyAckDeadline",
"path": "v1/{+subscription}:modifyAckDeadline",
"id": "pubsub.projects.subscriptions.modifyAckDeadline",
"description": "Modifies the ack deadline for a specific message. This method is useful\nto indicate that more time is needed to process a message by the\nsubscriber, or to make the message available for redelivery if the\nprocessing was interrupted. Note that this does not modify the\nsubscription-level `ackDeadlineSeconds` used for subsequent messages.",
"request": {
"$ref": "ModifyAckDeadlineRequest"
}
},
"get": {
"description": "Gets the configuration details of a subscription.",
"response": {
"$ref": "Subscription"
},
"parameterOrder": [
"subscription"
],
"httpMethod": "GET",
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"parameters": {
"subscription": {
"location": "path",
"description": "The name of the subscription to get.\nFormat is `projects/{project}/subscriptions/{sub}`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/subscriptions/[^/]+$"
}
},
"flatPath": "v1/projects/{projectsId}/subscriptions/{subscriptionsId}",
"path": "v1/{+subscription}",
"id": "pubsub.projects.subscriptions.get"
},
"testIamPermissions": {
"description": "Returns permissions that a caller has on the specified resource.\nIf the resource does not exist, this will return an empty set of\npermissions, not a NOT_FOUND error.\n\nNote: This operation is designed to be used for building permission-aware\nUIs and command-line tools, not for authorization checking. This operation\nmay \"fail open\" without warning.",
"request": {
"$ref": "TestIamPermissionsRequest"
},
"response": {
"$ref": "TestIamPermissionsResponse"
},
"parameterOrder": [
"resource"
],
"httpMethod": "POST",
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"parameters": {
"resource": {
"location": "path",
"description": "REQUIRED: The resource for which the policy detail is being requested.\nSee the operation documentation for the appropriate value for this field.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/subscriptions/[^/]+$"
}
},
"flatPath": "v1/projects/{projectsId}/subscriptions/{subscriptionsId}:testIamPermissions",
"path": "v1/{+resource}:testIamPermissions",
"id": "pubsub.projects.subscriptions.testIamPermissions"
},
"modifyPushConfig": {
"response": {
"$ref": "Empty"
},
"parameterOrder": [
"subscription"
],
"httpMethod": "POST",
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"parameters": {
"subscription": {
"location": "path",
"description": "The name of the subscription.\nFormat is `projects/{project}/subscriptions/{sub}`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/subscriptions/[^/]+$"
}
},
"flatPath": "v1/projects/{projectsId}/subscriptions/{subscriptionsId}:modifyPushConfig",
"path": "v1/{+subscription}:modifyPushConfig",
"id": "pubsub.projects.subscriptions.modifyPushConfig",
"description": "Modifies the `PushConfig` for a specified subscription.\n\nThis may be used to change a push subscription to a pull one (signified by\nan empty `PushConfig`) or vice versa, or change the endpoint URL and other\nattributes of a push subscription. Messages will accumulate for delivery\ncontinuously through the call regardless of changes to the `PushConfig`.",
"request": {
"$ref": "ModifyPushConfigRequest"
}
},
"pull": {
"request": {
"$ref": "PullRequest"
},
"description": "Pulls messages from the server. Returns an empty list if there are no\nmessages available in the backlog. The server may return `UNAVAILABLE` if\nthere are too many concurrent pull requests pending for the given\nsubscription.",
"response": {
"$ref": "PullResponse"
},
"parameterOrder": [
"subscription"
],
"httpMethod": "POST",
"parameters": {
"subscription": {
"description": "The subscription from which messages should be pulled.\nFormat is `projects/{project}/subscriptions/{sub}`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/subscriptions/[^/]+$",
"location": "path"
}
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"flatPath": "v1/projects/{projectsId}/subscriptions/{subscriptionsId}:pull",
"path": "v1/{+subscription}:pull",
"id": "pubsub.projects.subscriptions.pull"
},
"delete": {
"flatPath": "v1/projects/{projectsId}/subscriptions/{subscriptionsId}",
"path": "v1/{+subscription}",
"id": "pubsub.projects.subscriptions.delete",
"description": "Deletes an existing subscription. All messages retained in the subscription\nare immediately dropped. Calls to `Pull` after deletion will return\n`NOT_FOUND`. After a subscription is deleted, a new one may be created with\nthe same name, but the new one has no association with the old\nsubscription or its topic unless the same topic is specified.",
"response": {
"$ref": "Empty"
},
"parameterOrder": [
"subscription"
],
"httpMethod": "DELETE",
"parameters": {
"subscription": {
"description": "The subscription to delete.\nFormat is `projects/{project}/subscriptions/{sub}`.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/subscriptions/[^/]+$",
"location": "path"
}
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
]
}
}
},
"snapshots": {
"methods": {
"setIamPolicy": {
"flatPath": "v1/projects/{projectsId}/snapshots/{snapshotsId}:setIamPolicy",
"path": "v1/{+resource}:setIamPolicy",
"id": "pubsub.projects.snapshots.setIamPolicy",
"request": {
"$ref": "SetIamPolicyRequest"
},
"description": "Sets the access control policy on the specified resource. Replaces any\nexisting policy.",
"response": {
"$ref": "Policy"
},
"parameterOrder": [
"resource"
],
"httpMethod": "POST",
"parameters": {
"resource": {
"description": "REQUIRED: The resource for which the policy is being specified.\nSee the operation documentation for the appropriate value for this field.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/snapshots/[^/]+$",
"location": "path"
}
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
]
},
"testIamPermissions": {
"request": {
"$ref": "TestIamPermissionsRequest"
},
"description": "Returns permissions that a caller has on the specified resource.\nIf the resource does not exist, this will return an empty set of\npermissions, not a NOT_FOUND error.\n\nNote: This operation is designed to be used for building permission-aware\nUIs and command-line tools, not for authorization checking. This operation\nmay \"fail open\" without warning.",
"response": {
"$ref": "TestIamPermissionsResponse"
},
"parameterOrder": [
"resource"
],
"httpMethod": "POST",
"parameters": {
"resource": {
"description": "REQUIRED: The resource for which the policy detail is being requested.\nSee the operation documentation for the appropriate value for this field.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/snapshots/[^/]+$",
"location": "path"
}
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"flatPath": "v1/projects/{projectsId}/snapshots/{snapshotsId}:testIamPermissions",
"path": "v1/{+resource}:testIamPermissions",
"id": "pubsub.projects.snapshots.testIamPermissions"
},
"getIamPolicy": {
"description": "Gets the access control policy for a resource.\nReturns an empty policy if the resource exists and does not have a policy\nset.",
"response": {
"$ref": "Policy"
},
"parameterOrder": [
"resource"
],
"httpMethod": "GET",
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/pubsub"
],
"parameters": {
"resource": {
"location": "path",
"description": "REQUIRED: The resource for which the policy is being requested.\nSee the operation documentation for the appropriate value for this field.",
"required": true,
"type": "string",
"pattern": "^projects/[^/]+/snapshots/[^/]+$"
}
},
"flatPath": "v1/projects/{projectsId}/snapshots/{snapshotsId}:getIamPolicy",
"path": "v1/{+resource}:getIamPolicy",
"id": "pubsub.projects.snapshots.getIamPolicy"
}
}
}
}
}
},
"parameters": {
"pp": {
"location": "query",
"description": "Pretty-print response.",
"type": "boolean",
"default": "true"
},
"bearer_token": {
"description": "OAuth bearer token.",
"type": "string",
"location": "query"
},
"oauth_token": {
"location": "query",
"description": "OAuth 2.0 token for the current user.",
"type": "string"
},
"upload_protocol": {
"description": "Upload protocol for media (e.g. \"raw\", \"multipart\").",
"type": "string",
"location": "query"
},
"prettyPrint": {
"description": "Returns response with indentations and line breaks.",
"type": "boolean",
"default": "true",
"location": "query"
},
"fields": {
"description": "Selector specifying which fields to include in a partial response.",
"type": "string",
"location": "query"
},
"uploadType": {
"description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\").",
"type": "string",
"location": "query"
},
"callback": {
"location": "query",
"description": "JSONP",
"type": "string"
},
"$.xgafv": {
"description": "V1 error format.",
"type": "string",
"enumDescriptions": [
"v1 error format",
"v2 error format"
],
"location": "query",
"enum": [
"1",
"2"
]
},
"alt": {
"type": "string",
"enumDescriptions": [
"Responses with Content-Type of application/json",
"Media download with context-dependent Content-Type",
"Responses with Content-Type of application/x-protobuf"
],
"location": "query",
"description": "Data format for response.",
"default": "json",
"enum": [
"json",
"media",
"proto"
]
},
"key": {
"description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.",
"type": "string",
"location": "query"
},
"access_token": {
"description": "OAuth access token.",
"type": "string",
"location": "query"
},
"quotaUser": {
"description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters.",
"type": "string",
"location": "query"
}
},
"schemas": {
"Policy": {
"description": "Defines an Identity and Access Management (IAM) policy. It is used to\nspecify access control policies for Cloud Platform resources.\n\n\nA `Policy` consists of a list of `bindings`. A `Binding` binds a list of\n`members` to a `role`, where the members can be user accounts, Google groups,\nGoogle domains, and service accounts. A `role` is a named list of permissions\ndefined by IAM.\n\n**Example**\n\n {\n \"bindings\": [\n {\n \"role\": \"roles/owner\",\n \"members\": [\n \"user:mike@example.com\",\n \"group:admins@example.com\",\n \"domain:google.com\",\n \"serviceAccount:my-other-app@appspot.gserviceaccount.com\",\n ]\n },\n {\n \"role\": \"roles/viewer\",\n \"members\": [\"user:sean@example.com\"]\n }\n ]\n }\n\nFor a description of IAM and its features, see the\n[IAM developer's guide](https://cloud.google.com/iam).",
"type": "object",
"properties": {
"etag": {
"description": "`etag` is used for optimistic concurrency control as a way to help\nprevent simultaneous updates of a policy from overwriting each other.\nIt is strongly suggested that systems make use of the `etag` in the\nread-modify-write cycle to perform policy updates in order to avoid race\nconditions: An `etag` is returned in the response to `getIamPolicy`, and\nsystems are expected to put that etag in the request to `setIamPolicy` to\nensure that their change will be applied to the same version of the policy.\n\nIf no `etag` is provided in the call to `setIamPolicy`, then the existing\npolicy is overwritten blindly.",
"format": "byte",
"type": "string"
},
"version": {
"description": "Version of the `Policy`. The default version is 0.",
"format": "int32",
"type": "integer"
},
"bindings": {
"description": "Associates a list of `members` to a `role`.\nMultiple `bindings` must not be specified for the same `role`.\n`bindings` with no members will result in an error.",
"type": "array",
"items": {
"$ref": "Binding"
}
}
},
"id": "Policy"
},
"Topic": {
"description": "A topic resource.",
"type": "object",
"properties": {
"name": {
"description": "The name of the topic. It must have the format\n`\"projects/{project}/topics/{topic}\"`. `{topic}` must start with a letter,\nand contain only letters (`[A-Za-z]`), numbers (`[0-9]`), dashes (`-`),\nunderscores (`_`), periods (`.`), tildes (`~`), plus (`+`) or percent\nsigns (`%`). It must be between 3 and 255 characters in length, and it\nmust not start with `\"goog\"`.",
"type": "string"
}
},
"id": "Topic"
},
"ModifyAckDeadlineRequest": {
"description": "Request for the ModifyAckDeadline method.",
"type": "object",
"properties": {
"ackDeadlineSeconds": {
"description": "The new ack deadline with respect to the time this request was sent to\nthe Pub/Sub system. For example, if the value is 10, the new\nack deadline will expire 10 seconds after the `ModifyAckDeadline` call\nwas made. Specifying zero may immediately make the message available for\nanother pull request.\nThe minimum deadline you can specify is 0 seconds.\nThe maximum deadline you can specify is 600 seconds (10 minutes).",
"format": "int32",
"type": "integer"
},
"ackIds": {
"description": "List of acknowledgment IDs.",
"type": "array",
"items": {
"type": "string"
}
}
},
"id": "ModifyAckDeadlineRequest"
},
"SetIamPolicyRequest": {
"description": "Request message for `SetIamPolicy` method.",
"type": "object",
"properties": {
"policy": {
"description": "REQUIRED: The complete policy to be applied to the `resource`. The size of\nthe policy is limited to a few 10s of KB. An empty policy is a\nvalid policy but certain Cloud Platform services (such as Projects)\nmight reject them.",
"$ref": "Policy"
}
},
"id": "SetIamPolicyRequest"
},
"ModifyPushConfigRequest": {
"description": "Request for the ModifyPushConfig method.",
"type": "object",
"properties": {
"pushConfig": {
"$ref": "PushConfig",
"description": "The push configuration for future deliveries.\n\nAn empty `pushConfig` indicates that the Pub/Sub system should\nstop pushing messages from the given subscription and allow\nmessages to be pulled and acknowledged - effectively pausing\nthe subscription if `Pull` is not called."
}
},
"id": "ModifyPushConfigRequest"
},
"PubsubMessage": {
"description": "A message data and its attributes. The message payload must not be empty;\nit must contain either a non-empty data field, or at least one attribute.",
"type": "object",
"properties": {
"data": {
"description": "The message payload.",
"format": "byte",
"type": "string"
},
"attributes": {
"description": "Optional attributes for this message.",
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"messageId": {
"description": "ID of this message, assigned by the server when the message is published.\nGuaranteed to be unique within the topic. This value may be read by a\nsubscriber that receives a `PubsubMessage` via a `Pull` call or a push\ndelivery. It must not be populated by the publisher in a `Publish` call.",
"type": "string"
},
"publishTime": {
"description": "The time at which the message was published, populated by the server when\nit receives the `Publish` call. It must not be populated by the\npublisher in a `Publish` call.",
"format": "google-datetime",
"type": "string"
}
},
"id": "PubsubMessage"
},
"Binding": {
"description": "Associates `members` with a `role`.",
"type": "object",
"properties": {
"members": {
"description": "Specifies the identities requesting access for a Cloud Platform resource.\n`members` can have the following values:\n\n* `allUsers`: A special identifier that represents anyone who is\n on the internet; with or without a Google account.\n\n* `allAuthenticatedUsers`: A special identifier that represents anyone\n who is authenticated with a Google account or a service account.\n\n* `user:{emailid}`: An email address that represents a specific Google\n account. For example, `alice@gmail.com` or `joe@example.com`.\n\n\n* `serviceAccount:{emailid}`: An email address that represents a service\n account. For example, `my-other-app@appspot.gserviceaccount.com`.\n\n* `group:{emailid}`: An email address that represents a Google group.\n For example, `admins@example.com`.\n\n* `domain:{domain}`: A Google Apps domain name that represents all the\n users of that domain. For example, `google.com` or `example.com`.\n\n",
"type": "array",
"items": {
"type": "string"
}
},
"role": {
"description": "Role that is assigned to `members`.\nFor example, `roles/viewer`, `roles/editor`, or `roles/owner`.\nRequired",
"type": "string"
}
},
"id": "Binding"
},
"AcknowledgeRequest": {
"description": "Request for the Acknowledge method.",
"type": "object",
"properties": {
"ackIds": {
"description": "The acknowledgment ID for the messages being acknowledged that was returned\nby the Pub/Sub system in the `Pull` response. Must not be empty.",
"type": "array",
"items": {
"type": "string"
}
}
},
"id": "AcknowledgeRequest"
},
"Empty": {
"description": "A generic empty message that you can re-use to avoid defining duplicated\nempty messages in your APIs. A typical example is to use it as the request\nor the response type of an API method. For instance:\n\n service Foo {\n rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty);\n }\n\nThe JSON representation for `Empty` is empty JSON object `{}`.",
"type": "object",
"properties": {},
"id": "Empty"
},
"ListTopicsResponse": {
"description": "Response for the `ListTopics` method.",
"type": "object",
"properties": {
"topics": {
"description": "The resulting topics.",
"type": "array",
"items": {
"$ref": "Topic"
}
},
"nextPageToken": {
"description": "If not empty, indicates that there may be more topics that match the\nrequest; this value should be passed in a new `ListTopicsRequest`.",
"type": "string"
}
},
"id": "ListTopicsResponse"
},
"ListTopicSubscriptionsResponse": {
"description": "Response for the `ListTopicSubscriptions` method.",
"type": "object",
"properties": {
"nextPageToken": {
"description": "If not empty, indicates that there may be more subscriptions that match\nthe request; this value should be passed in a new\n`ListTopicSubscriptionsRequest` to get more subscriptions.",
"type": "string"
},
"subscriptions": {
"description": "The names of the subscriptions that match the request.",
"type": "array",
"items": {
"type": "string"
}
}
},
"id": "ListTopicSubscriptionsResponse"
},
"PullResponse": {
"description": "Response for the `Pull` method.",
"type": "object",
"properties": {
"receivedMessages": {
"description": "Received Pub/Sub messages. The Pub/Sub system will return zero messages if\nthere are no more available in the backlog. The Pub/Sub system may return\nfewer than the `maxMessages` requested even if there are more messages\navailable in the backlog.",
"type": "array",
"items": {
"$ref": "ReceivedMessage"
}
}
},
"id": "PullResponse"
},
"ReceivedMessage": {
"description": "A message and its corresponding acknowledgment ID.",
"type": "object",
"properties": {
"ackId": {
"description": "This ID can be used to acknowledge the received message.",
"type": "string"
},
"message": {
"$ref": "PubsubMessage",
"description": "The message."
}
},
"id": "ReceivedMessage"
},
"PushConfig": {
"description": "Configuration for a push delivery endpoint.",
"type": "object",
"properties": {
"pushEndpoint": {
"description": "A URL locating the endpoint to which messages should be pushed.\nFor example, a Webhook endpoint might use \"https://example.com/push\".",
"type": "string"
},
"attributes": {
"additionalProperties": {
"type": "string"
},
"description": "Endpoint configuration attributes.\n\nEvery endpoint has a set of API supported attributes that can be used to\ncontrol different aspects of the message delivery.\n\nThe currently supported attribute is `x-goog-version`, which you can\nuse to change the format of the pushed message. This attribute\nindicates the version of the data expected by the endpoint. This\ncontrols the shape of the pushed message (i.e., its fields and metadata).\nThe endpoint version is based on the version of the Pub/Sub API.\n\nIf not present during the `CreateSubscription` call, it will default to\nthe version of the API used to make such call. If not present during a\n`ModifyPushConfig` call, its value will not be changed. `GetSubscription`\ncalls will always return a valid version, even if the subscription was\ncreated without this attribute.\n\nThe possible values for this attribute are:\n\n* `v1beta1`: uses the push format defined in the v1beta1 Pub/Sub API.\n* `v1` or `v1beta2`: uses the push format defined in the v1 Pub/Sub API.",
"type": "object"
}
},
"id": "PushConfig"
},
"TestIamPermissionsResponse": {
"description": "Response message for `TestIamPermissions` method.",
"type": "object",
"properties": {
"permissions": {
"description": "A subset of `TestPermissionsRequest.permissions` that the caller is\nallowed.",
"type": "array",
"items": {
"type": "string"
}
}
},
"id": "TestIamPermissionsResponse"
},
"PullRequest": {
"description": "Request for the `Pull` method.",
"type": "object",
"properties": {
"returnImmediately": {
"description": "If this field set to true, the system will respond immediately even if\nit there are no messages available to return in the `Pull` response.\nOtherwise, the system may wait (for a bounded amount of time) until at\nleast one message is available, rather than returning no messages. The\nclient may cancel the request if it does not wish to wait any longer for\nthe response.",
"type": "boolean"
},
"maxMessages": {
"description": "The maximum number of messages returned for this request. The Pub/Sub\nsystem may return fewer than the number specified.",
"format": "int32",
"type": "integer"
}
},
"id": "PullRequest"
},
"ListSubscriptionsResponse": {
"description": "Response for the `ListSubscriptions` method.",
"type": "object",
"properties": {
"nextPageToken": {
"description": "If not empty, indicates that there may be more subscriptions that match\nthe request; this value should be passed in a new\n`ListSubscriptionsRequest` to get more subscriptions.",
"type": "string"
},
"subscriptions": {
"description": "The subscriptions that match the request.",
"type": "array",
"items": {
"$ref": "Subscription"
}
}
},
"id": "ListSubscriptionsResponse"
},
"PublishRequest": {
"description": "Request for the Publish method.",
"type": "object",
"properties": {
"messages": {
"description": "The messages to publish.",
"type": "array",
"items": {
"$ref": "PubsubMessage"
}
}
},
"id": "PublishRequest"
},
"PublishResponse": {
"description": "Response for the `Publish` method.",
"type": "object",
"properties": {
"messageIds": {
"description": "The server-assigned ID of each published message, in the same order as\nthe messages in the request. IDs are guaranteed to be unique within\nthe topic.",
"type": "array",
"items": {
"type": "string"
}
}
},
"id": "PublishResponse"
},
"Subscription": {
"description": "A subscription resource.",
"type": "object",
"properties": {
"ackDeadlineSeconds": {
"description": "This value is the maximum time after a subscriber receives a message\nbefore the subscriber should acknowledge the message. After message\ndelivery but before the ack deadline expires and before the message is\nacknowledged, it is an outstanding message and will not be delivered\nagain during that time (on a best-effort basis).\n\nFor pull subscriptions, this value is used as the initial value for the ack\ndeadline. To override this value for a given message, call\n`ModifyAckDeadline` with the corresponding `ack_id` if using\npull.\nThe minimum custom deadline you can specify is 10 seconds.\nThe maximum custom deadline you can specify is 600 seconds (10 minutes).\nIf this parameter is 0, a default value of 10 seconds is used.\n\nFor push delivery, this value is also used to set the request timeout for\nthe call to the push endpoint.\n\nIf the subscriber never acknowledges the message, the Pub/Sub\nsystem will eventually redeliver the message.",
"format": "int32",
"type": "integer"
},
"name": {
"description": "The name of the subscription. It must have the format\n`\"projects/{project}/subscriptions/{subscription}\"`. `{subscription}` must\nstart with a letter, and contain only letters (`[A-Za-z]`), numbers\n(`[0-9]`), dashes (`-`), underscores (`_`), periods (`.`), tildes (`~`),\nplus (`+`) or percent signs (`%`). It must be between 3 and 255 characters\nin length, and it must not start with `\"goog\"`.",
"type": "string"
},
"topic": {
"description": "The name of the topic from which this subscription is receiving messages.\nFormat is `projects/{project}/topics/{topic}`.\nThe value of this field will be `_deleted-topic_` if the topic has been\ndeleted.",
"type": "string"
},
"pushConfig": {
"$ref": "PushConfig",
"description": "If push delivery is used with this subscription, this field is\nused to configure it. An empty `pushConfig` signifies that the subscriber\nwill pull and ack messages using API methods."
}
},
"id": "Subscription"
},
"TestIamPermissionsRequest": {
"description": "Request message for `TestIamPermissions` method.",
"type": "object",
"properties": {
"permissions": {
"description": "The set of permissions to check for the `resource`. Permissions with\nwildcards (such as '*' or 'storage.*') are not allowed. For more\ninformation see\n[IAM Overview](https://cloud.google.com/iam/docs/overview#permissions).",
"type": "array",
"items": {
"type": "string"
}
}
},
"id": "TestIamPermissionsRequest"
}
},
"protocol": "rest",
"icons": {
"x32": "http://www.google.com/images/icons/product/search-32.gif",
"x16": "http://www.google.com/images/icons/product/search-16.gif"
},
"version": "v1",
"baseUrl": "https://pubsub.googleapis.com/",
"canonicalName": "Pubsub",
"auth": {
"oauth2": {
"scopes": {
"https://www.googleapis.com/auth/cloud-platform": {
"description": "View and manage your data across Google Cloud Platform services"
},
"https://www.googleapis.com/auth/pubsub": {
"description": "View and manage Pub/Sub topics and subscriptions"
}
}
}
}
}
vendor/google.golang.org/api/pubsub/v1/pubsub-gen.go 0 → 100644
View file @ 012ffed6
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment