Enhance GCP plugin with Cloud Logging API source and new request log table

Makefile

@@ -6,7 +6,9 @@ PLUGIN_BINARY = $(PLUGIN_DIR)/tailpipe-plugin-gcp.plugin
 VERSION_JSON = $(PLUGIN_DIR)/version.json
 VERSIONS_JSON = $(TAILPIPE_INSTALL_DIR)/plugins/versions.json
 
+.PHONY: install
+
 install:
 	go build -o $(PLUGIN_BINARY) -tags "${BUILD_TAGS}" *.go
 	$(PLUGIN_BINARY) metadata > $(VERSION_JSON)
-	rm -f $(VERSIONS_JSON)
+	rm -f $(VERSIONS_JSON)

docs/sources/gcp_cloud_logging_api.md

@@ -0,0 +1,40 @@
+---
+title: "Source: gcp_cloud_logging_api - Collect logs from GCP Cloud Logging API"
+description: "Allows users to collect logs from Google Cloud Platform (GCP) Cloud Logging API."
+---
+
+# Source: gcp_cloud_logging_api - Obtain logs from GCP Cloud Logging API
+
+The Google Cloud Platform (GCP) Cloud Logging API provides access to all logs for all GCP services. It allows you to view and manage logs for your GCP projects, services, and applications.
+
+This source is currently configured only for request logs from Google Load Balancer and Cloud Armor logs, from the source log name `projects/project-id/logs/requests`
+
+Using this source, currently you can collect, filter, and analyze request logs that have been enriched with Cloud Armor rule findings, in order to collect metrics on blocking or analyze to eliminate false positive findings that would block wanted application requests.
+
+Any other log type except for audit logs are of the `logEntry` type, and this source can potentially collect them with minor changes to the source code.  (Tables must still be created for each type of log)
+
+## Example Configurations
+
+### Collect request logs
+
+Collect all of the request logs for a project.
+
+```hcl
+connection "gcp" "my_project" {
+  project = "my-gcp-project"
+}
+
+partition "gcp_requests_log" "my_logs" {
+  source "gcp_cloud_logging_api" {
+    connection = connection.gcp.my_project
+  }
+}
+```
+
+
+## Arguments
+
+| Argument   | Type             | Required | Default                  | Description                                                                                                                   |
+|------------|------------------|----------|--------------------------|-------------------------------------------------------------------------------------------------------------------------------|
+| connection | `connection.gcp` | No       | `connection.gcp.default` | The [GCP connection](https://hub.tailpipe.io/plugins/turbot/gcp#connection-credentials) to use to connect to the GCP account. |
+| log_types  | List(String)     | No       | []                       | This could any type of non-audit log that confirms to the [logEntry data model](https://cloud.google.com/logging/docs/log-entry-data-model) and is stored in the `_Default` logging bucket in a GCP project.  The only restriction is what tables are supported in Tailpipe.  Currently the only supported type is `requests`

docs/tables/gcp_requests_log/index.md

@@ -0,0 +1,186 @@
+---
+title: "Tailpipe Table: gcp_requests_log - Query GCP request logs"
+description: "GCP request logs capture network requests from GCP Load Balancers containing fields for the results of Cloud Armor analysis."
+---
+
+# Table: gcp_requests_log - Query GCP request logs
+
+The `gcp_requests_log` table allows you to query data from GCP audit logs. This table provides detailed information about API calls made within your Google Cloud environment, including the event name, resource affected, user identity, and more.
+
+## Configure
+
+Create a [partition](https://tailpipe.io/docs/manage/partition) for `gcp_requests_log`:
+
+```sh
+vi ~/.tailpipe/config/gcp.tpc
+```
+
+```hcl
+connection "gcp" "logging_account" {
+  project = "my-gcp-project"
+}
+
+partition "gcp_requests_log" "my_logs" {
+  source "gcp_storage_bucket" {
+    connection = connection.gcp.logging_account
+  }
+}
+```
+OR
+
+```hcl
+connection "gcp" "logging_account" {
+  project = "my-gcp-project"
+}
+
+partition "gcp_requests_log" "my_logs" {
+  source "gcp_cloud_logging_api" {
+    connection = connection.gcp.logging_account
+  }
+}
+```
+
+## Collect
+
+[Collect](https://tailpipe.io/docs/manage/collection) logs for all `gcp_requests_log` partitions:
+
+```sh
+tailpipe collect gcp_requests_log
+```
+
+Or for a single partition:
+
+```sh
+tailpipe collect gcp_requests_log.my_logs
+```
+
+## Query
+
+
+### Blocked Requests
+
+Count how many requests were blocked by Cloud Armor across all policies
+
+```sql
+SELECT
+  enforced_security_policy.name AS policy_name,
+  count(*) AS total_blocked_requests
+FROM
+  gcp_requests_log
+WHERE
+  enforced_security_policy ->> 'outcome' = 'DENY'
+GROUP BY
+  policy_name
+ORDER BY
+  total_blocked_requests DESC
+```
+
+### Top 10 events
+
+List the 10 most blocked OWASP Core Rule Set rules
+
+```sql
+SELECT
+  enforced_security_policy ->> 'preconfigured_expr_id' AS rule_id,
+  COUNT(*) AS total_occurrences
+FROM
+  gcp_requests_log
+WHERE
+  enforced_security_policy ->> 'outcome' = 'DENY'
+  AND LENGTH(enforced_security_policy ->> 'preconfigured_expr_id') > 0
+GROUP BY
+  rule_id
+ORDER BY
+  total_occurrences DESC
+LIMIT 10
+```
+
+
+## Example Configurations
+
+### Collect logs from a Storage bucket
+
+Collect request logs stored in a Storage bucket that use the [default log file name format](https://hub.tailpipe.io/plugins/turbot/gcp/tables/gcp_audit_log#gcp_storage_bucket).
+
+```hcl
+connection "gcp" "logging_account" {
+  project = "my-gcp-project"
+}
+
+partition "gcp_requests_log" "my_logs" {
+  source "gcp_storage_bucket" {
+    connection = connection.gcp.logging_account
+    bucket     = "gcp-cloudarmor-logs-bucket"
+  }
+}
+```
+
+### Collect logs from a Storage bucket with a prefix
+
+Collect audit logs stored with a GCS key prefix.
+
+```hcl
+partition "gcp_requests_log" "my_logs_prefix" {
+  source "gcp_storage_bucket" {
+    connection = connection.gcp.logging_account
+    bucket     = "gcp-cloudarmor-logs-bucket"
+    prefix     = "my/prefix/"
+  }
+}
+```
+
+### Collect logs from a Storage Bucket for a single project
+
+Collect audit logs for a specific project.
+
+```hcl
+partition "gcp_requests_log" "my_logs_prefix" {
+  filter = "log_name like 'projects/my-project-name/logs/requests/%'"
+
+  source "gcp_storage_bucket" {
+    connection = connection.gcp.logging_account
+    bucket     = "gcp-cloudarmor-logs-bucket"
+  }
+}
+```
+
+### Collect logs from Cloud Logging API
+
+Collect request logs directly via the Cloud Logging API.  *Note that rate limiting is currently not implemented and this could impact ability to collect a large number of logs*
+
+```hcl
+connection "gcp" "my_project" {
+  project = "my-gcp-project"
+}
+
+partition "gcp_requests_log" "my_logs" {
+  source "gcp_cloud_logging_api" {
+    connection = connection.gcp.my_project
+  }
+}
+```
+
+### Collect other types of logs from Cloud Logging API
+
+The Cloud Logging API source can be expanded upon to retrieve logs other than requests, if the appropriate table is created to enrich and save them.  The log name attribute is the filter used for this, and it assumes that a table / partition have been made that match the data type.
+
+Example: Collecting GCP Dataflow logs
+
+```hcl
+partition "gcp_dataflow_log" "my_logs_prefix" {
+  filter = "log_name like 'projects/my-project-name/logs/dataflow.googleapis.com%'"
+  source "gcp_cloud_logging_api" {
+    connection = connection.gcp.my_project
+  }
+}
+```
+
+## Source Defaults
+
+### gcp_storage_bucket
+
+This table sets the following defaults for the [gcp_storage_bucket](https://hub.tailpipe.io/plugins/turbot/gcp/sources/gcp_storage_bucket#arguments):
+
+| Argument      | Default |
+|--------------|---------|
+| file_layout   | `requests/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}_%{DATA:end_time}_%{DATA:suffix}.json` |

gcp/plugin.go

@@ -4,8 +4,10 @@ import (
 	"github.com/turbot/go-kit/helpers"
 	"github.com/turbot/tailpipe-plugin-gcp/config"
 	"github.com/turbot/tailpipe-plugin-gcp/sources/audit_log_api"
+	"github.com/turbot/tailpipe-plugin-gcp/sources/cloud_logging_api"
 	"github.com/turbot/tailpipe-plugin-gcp/sources/storage_bucket"
 	"github.com/turbot/tailpipe-plugin-gcp/tables/audit_log"
+	"github.com/turbot/tailpipe-plugin-gcp/tables/requests_log"
 	"github.com/turbot/tailpipe-plugin-sdk/plugin"
 	"github.com/turbot/tailpipe-plugin-sdk/row_source"
 	"github.com/turbot/tailpipe-plugin-sdk/table"
@@ -20,9 +22,11 @@ func init() {
 	// 1. row struct
 	// 2. table implementation
 	table.RegisterTable[*audit_log.AuditLog, *audit_log.AuditLogTable]()
+	table.RegisterTable[*requests_log.RequestsLog, *requests_log.RequestsLogTable]()
 
 	// register sources
 	row_source.RegisterRowSource[*audit_log_api.AuditLogAPISource]()
+	row_source.RegisterRowSource[*cloud_logging_api.CloudLoggingAPISource]()
 	row_source.RegisterRowSource[*storage_bucket.GcpStorageBucketSource]()
 }
 

go.mod

@@ -82,7 +82,7 @@ require (
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/gertd/go-pluralize v0.2.1 // indirect
 	github.com/go-git/go-git/v5 v5.13.0 // indirect
-	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.4 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect

go.sum

@@ -372,8 +372,8 @@ github.com/go-git/go-git/v5 v5.13.0/go.mod h1:Wjo7/JyVKtQgUNdXYXIepzWfJQkUEIGvkv
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE=
-github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA=
+github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E=
+github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=