[feat] Add support for time ranges longer than one hour

README.md

@@ -76,6 +76,7 @@ GLOBAL OPTIONS:
    --google-storage-bucket value  Full URI to a Google Cloud Storage Bucket to upload logs to
    --google-project-id value      Project ID of the Google Cloud Storage Bucket to upload logs to
    --skip-create-bucket           Do not attempt to create the bucket specified by --google-storage-bucket
+   --iterate                      Iterate over time ranges longer than normally allowed by the Logpull Rest API
    --help, -h                     show help
    --version, -v                  print the version
 ```

cmd/logshare-cli/main.go

@@ -115,18 +115,40 @@ func run(conf *config) func(c *cli.Context) error {
 			if err != nil {
 				return errors.Wrap(err, "failed to fetch field names")
 			}
+		} else if conf.iterate {
+			// Iterate over the time range in 1h (3600s) blocks, making sure to
+			// stay within the request rate limit (1 request/5 seconds)
+			hourStart := conf.startTime
+			hourEnd := hourStart + 3600
+			var previousRequest time.Time
+			for hourStart < conf.endTime {
+				if hourEnd > conf.endTime {
+					hourEnd = conf.endTime
+				}
+				time.Sleep(5*time.Second - time.Since(previousRequest))
+				previousRequest = time.Now()
+				meta, err = client.GetFromTimestamp(
+					conf.zoneID, hourStart, hourEnd, conf.count)
+				if err != nil {
+					return errors.Wrap(err, "failed to fetch via timestamp")
+				}
+				log.Printf("HTTP status %d | %dms | %s",
+					meta.StatusCode, meta.Duration, meta.URL)
+				log.Printf("Retrieved %d logs", meta.Count)
+				hourStart = hourEnd
+				hourEnd += 3600
+			}
 		} else {
 			meta, err = client.GetFromTimestamp(
 				conf.zoneID, conf.startTime, conf.endTime, conf.count)
 			if err != nil {
 				return errors.Wrap(err, "failed to fetch via timestamp")
 			}
+			log.Printf("HTTP status %d | %dms | %s",
+				meta.StatusCode, meta.Duration, meta.URL)
+			log.Printf("Retrieved %d logs", meta.Count)
 		}
 
-		log.Printf("HTTP status %d | %dms | %s",
-			meta.StatusCode, meta.Duration, meta.URL)
-		log.Printf("Retrieved %d logs", meta.Count)
-
 		return nil
 	}
 }
@@ -146,6 +168,7 @@ func parseFlags(conf *config, c *cli.Context) error {
 	conf.googleStorageBucket = c.String("google-storage-bucket")
 	conf.googleProjectID = c.String("google-project-id")
 	conf.skipCreateBucket = c.Bool("skip-create-bucket")
+	conf.iterate = c.Bool("iterate")
 
 	return conf.Validate()
 }
@@ -165,6 +188,7 @@ type config struct {
 	googleStorageBucket string
 	googleProjectID     string
 	skipCreateBucket    bool
+	iterate             bool
 }
 
 func (conf *config) Validate() error {
@@ -185,6 +209,10 @@ func (conf *config) Validate() error {
 		return errors.New("Both google-storage-bucket and google-project-id must be provided to upload to Google Storage")
 	}
 
+	if conf.endTime > (conf.startTime+3600) && conf.iterate == false {
+		return errors.New("Time range too long; ranges longer than 1h0m0s require the --iterate option")
+	}
+
 	return nil
 }
 
@@ -254,4 +282,8 @@ var flags = []cli.Flag{
 		Name:  "skip-create-bucket",
 		Usage: "Do not attempt to create the bucket specified by --google-storage-bucket",
 	},
+	cli.BoolFlag{
+		Name:  "iterate",
+		Usage: "Iterate over time ranges longer than normally allowed by the Logpull Rest API",
+	},
 }