From b5770f1c1b5d43ceeec514a3aa078612fd4ef14e Mon Sep 17 00:00:00 2001 From: Evan Damon Date: Mon, 5 Nov 2018 13:22:12 -0800 Subject: [PATCH 1/2] [feat] Add support for time ranges longer than one hour --- cmd/logshare-cli/main.go | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/cmd/logshare-cli/main.go b/cmd/logshare-cli/main.go index 575f434..9b1ed3a 100644 --- a/cmd/logshare-cli/main.go +++ b/cmd/logshare-cli/main.go @@ -113,18 +113,40 @@ func run(conf *config) func(c *cli.Context) error { if err != nil { return errors.Wrap(err, "failed to fetch field names") } + } else if conf.iterate { + // Iterate over the time range in 1h (3600s) blocks, making sure to + // stay within the request rate limit (1 request/5 seconds) + hourStart := conf.startTime + hourEnd := hourStart + 3600 + var previousRequest time.Time + for hourStart < conf.endTime { + if hourEnd > conf.endTime { + hourEnd = conf.endTime + } + time.Sleep(5*time.Second - time.Since(previousRequest)) + previousRequest = time.Now() + meta, err = client.GetFromTimestamp( + conf.zoneID, hourStart, hourEnd, conf.count) + if err != nil { + return errors.Wrap(err, "failed to fetch via timestamp") + } + log.Printf("HTTP status %d | %dms | %s", + meta.StatusCode, meta.Duration, meta.URL) + log.Printf("Retrieved %d logs", meta.Count) + hourStart = hourEnd + hourEnd += 3600 + } } else { meta, err = client.GetFromTimestamp( conf.zoneID, conf.startTime, conf.endTime, conf.count) if err != nil { return errors.Wrap(err, "failed to fetch via timestamp") } + log.Printf("HTTP status %d | %dms | %s", + meta.StatusCode, meta.Duration, meta.URL) + log.Printf("Retrieved %d logs", meta.Count) } - log.Printf("HTTP status %d | %dms | %s", - meta.StatusCode, meta.Duration, meta.URL) - log.Printf("Retrieved %d logs", meta.Count) - return nil } } @@ -143,6 +165,7 @@ func parseFlags(conf *config, c *cli.Context) error { conf.listFields = c.Bool("list-fields") conf.googleStorageBucket = c.String("google-storage-bucket") conf.googleProjectID = c.String("google-project-id") + conf.iterate = c.Bool("iterate") return conf.Validate() } @@ -161,6 +184,7 @@ type config struct { listFields bool googleStorageBucket string googleProjectID string + iterate bool } func (conf *config) Validate() error { @@ -181,6 +205,10 @@ func (conf *config) Validate() error { return errors.New("Both google-storage-bucket and google-project-id must be provided to upload to Google Storage") } + if conf.endTime > (conf.startTime+3600) && conf.iterate == false { + return errors.New("Time range too long; ranges longer than 1h0m0s require the --iterate option") + } + return nil } @@ -246,4 +274,8 @@ var flags = []cli.Flag{ Name: "google-project-id", Usage: "Project ID of the Google Cloud Storage Bucket to upload logs to", }, + cli.BoolFlag{ + Name: "iterate", + Usage: "Iterate over time ranges longer than normally allowed by the Logpull Rest API", + }, } From f3a1424a29f80d0e6b1e84344c7a75b7d7c1836f Mon Sep 17 00:00:00 2001 From: Evan Damon Date: Mon, 19 Nov 2018 16:49:48 -0800 Subject: [PATCH 2/2] Merge changes from upstream and update README with new flag --- README.md | 2 ++ cmd/logshare-cli/main.go | 22 +++++++++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 2d0e48e..5ee5c2f 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,8 @@ GLOBAL OPTIONS: --list-fields List the available log fields for use with the --fields flag --google-storage-bucket value Full URI to a Google Cloud Storage Bucket to upload logs to --google-project-id value Project ID of the Google Cloud Storage Bucket to upload logs to + --skip-create-bucket Do not attempt to create the bucket specified by --google-storage-bucket + --iterate Iterate over time ranges longer than normally allowed by the Logpull Rest API --help, -h show help --version, -v print the version ``` diff --git a/cmd/logshare-cli/main.go b/cmd/logshare-cli/main.go index 9b1ed3a..751f717 100644 --- a/cmd/logshare-cli/main.go +++ b/cmd/logshare-cli/main.go @@ -38,7 +38,7 @@ func main() { } } -func setupGoogleStr(projectID string, bucketName string, filename string) (*gcs.Writer, error) { +func setupGoogleStr(projectID string, bucketName string, filename string, skipCreateBucket bool) (*gcs.Writer, error) { gCtx := context.Background() gClient, error := gcs.NewClient(gCtx) @@ -48,11 +48,13 @@ func setupGoogleStr(projectID string, bucketName string, filename string) (*gcs. gBucket := gClient.Bucket(bucketName) - if error = gBucket.Create(gCtx, projectID, nil); strings.Contains(error.Error(), "409") { - log.Printf("Bucket %v already exists.\n", bucketName) - error = nil - } else if error != nil { - return nil, error + if !skipCreateBucket { + if error = gBucket.Create(gCtx, projectID, nil); strings.Contains(error.Error(), "409") { + log.Printf("Bucket %v already exists.\n", bucketName) + error = nil + } else if error != nil { + return nil, error + } } obj := gBucket.Object(filename) @@ -82,7 +84,7 @@ func run(conf *config) func(c *cli.Context) error { if conf.googleStorageBucket != "" { fileName := "cloudflare_els_" + conf.zoneID + "_" + strconv.Itoa(int(time.Now().Unix())) + ".json" - gcsWriter, err := setupGoogleStr(conf.googleProjectID, conf.googleStorageBucket, fileName) + gcsWriter, err := setupGoogleStr(conf.googleProjectID, conf.googleStorageBucket, fileName, conf.skipCreateBucket) if err != nil { return err } @@ -165,6 +167,7 @@ func parseFlags(conf *config, c *cli.Context) error { conf.listFields = c.Bool("list-fields") conf.googleStorageBucket = c.String("google-storage-bucket") conf.googleProjectID = c.String("google-project-id") + conf.skipCreateBucket = c.Bool("skip-create-bucket") conf.iterate = c.Bool("iterate") return conf.Validate() @@ -184,6 +187,7 @@ type config struct { listFields bool googleStorageBucket string googleProjectID string + skipCreateBucket bool iterate bool } @@ -274,6 +278,10 @@ var flags = []cli.Flag{ Name: "google-project-id", Usage: "Project ID of the Google Cloud Storage Bucket to upload logs to", }, + cli.BoolFlag{ + Name: "skip-create-bucket", + Usage: "Do not attempt to create the bucket specified by --google-storage-bucket", + }, cli.BoolFlag{ Name: "iterate", Usage: "Iterate over time ranges longer than normally allowed by the Logpull Rest API",