Add support for time partitioning and expiry

.editorconfig

@@ -0,0 +1,8 @@
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2

.gcloudignore

@@ -0,0 +1,17 @@
+# This file specifies files that are *not* uploaded to Google Cloud Platform
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+node_modules
+#!include:.gitignore

.gitignore

@@ -0,0 +1,52 @@
+node_modules/**
+
+# Serverless
+.serverless
+.env*
+tmp
+.coveralls.yml
+logs.js
+# Google
+keyfile.json
+
+# Logs
+*.log
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
+node_modules
+
+# IDE
+**/.idea
+
+# OS
+.vscode
+cloudflare-gcp.code-workspace
+
+security-events/.env.yml
+security-events/node_modules
+security-events/scc_key.json
+security-events/.DS_Store
+security-events/local.js
+
+cli/node_modules
+
+logpush-to-bigquery/.DS_Store
+logpush-to-bigquery/node_modules
+cli/dist
+cli/confs

README.md

@@ -1,27 +1,6 @@
-# GCS To Big Query Cloud Function
-Google Cloud Function Code to push json files from GCS to Big Query
+# Cloudflare + Google Cloud | Integrations
+Integrate Cloudflare Enterprise Log Push with BigQuery and Security Command Center on Google Cloud.
 
-### Requirements
-If using the Google Cloud UI to upload then there are no local requirements.
+* [Cloudflare Log Push to BigQuery](https://github.com/cloudflare/cloudflare-gcp/tree/master/logpush-to-bigquery)
 
-[gcloud SDK](https://cloud.google.com/sdk/downloads) need to be setup to run from command line.
-
-### Setup
-Config Variables:
-   "DATASET" - BigQuery Dataset to write to. Will be created if necessaray.
-   "TABLE" - BigQuery Table to write to. Will be created if necessaray.
-
-When deploying through gcloud or in UI, the following need to be specified:
-   * name: any name can be used, but if entry-point is not specified then the function name will be used.
-   * trigger-bucket: The Google Storage Bucket that will trigger the Cloud Function on file create. Must already exist.
-   * stage-bucket: Where the Cloud Function code should be kept. Should not be the same as trigger bucket in case you update the Cloud Function.
-   * entry-point: Should always be "jsonLoad" unless the Cloud Function name is also "jsonLoad"
-
-
-### Deploying from gcloud cli
-~~~ 
-gcloud beta functions deploy <name of the cloud function> 
---trigger-resource=<trigger-bucket-name> --trigger-event google.storage.object.finalize 
---source=<path to gcsToBigQuery repository on your workstation> --stage-bucket=<gs://gcs-bucket> 
---entry-point=jsonLoad
-~~~ 
+----

cli/.gcloudignore

@@ -0,0 +1,16 @@
+# This file specifies files that are *not* uploaded to Google Cloud Platform
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+node_modules

cli/cli.js

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+exports = require('yargs')
+  .commandDir('cmds')
+  .demandCommand()
+  .help()
+  .argv

cli/cloudshell-security-events.md

@@ -0,0 +1,71 @@
+# Cloudflare Security Events
+
+## Enter the Cloud Shell subdirectory for the project you want to use for the Cloudflare SCC integration
+<walkthrough-project-setup></walkthrough-project-setup>
+
+## Set Cloudshell to the project where you'd like to deploy the integration
+```sh
+gcloud config set project {{project-id}}
+```
+
+## Install dependencies and CLI:
+Run:
+```bash
+sh install.sh
+```
+
+## Set Environment Variables and write deployment files
+Run:
+```bash
+./setup setEnv
+```
+
+## Build configuration files:
+Run:
+```bash
+./setup buildConf
+```
+**Tip:** If you need to edit `security-events/.env.yml` in the future, use this command to update the configuration files before redeploying.
+
+## Enable the necessary Cloud APIs to run the Cloudflare integration
+```bash
+./setup enableAPIs
+```
+
+## Create Cloud Scheduler event (deployed via Pub/Sub)
+```bash
+./setup setSchedule
+```
+
+## Create a service account key for SCC
+Run:
+```bash
+./setup getServiceAcctKey
+```
+
+**Tip:** This may throw some errors but if the final message succeeds, you're probably ok. Having issues? Make sure you're using the correct account:
+```bash
+gcloud config set account MY_GCP_ACCT_EMAIL
+```
+
+## Deploy integration
+```bash
+./setup deploy
+```
+
+
+## Done!
+The configuration file, `.env.yml` can be modified here:
+```sh
+cd cloudflare-gcp/security-events
+nano .env.yml
+```
+
+Then, to rebuild the necessary configuration files:
+```sh
+cd cloudflare-gcp/cli
+./setup buildConf
+```
+
+<walkthrough-directive-name param-name="conclusion-trophy">
+</walkthrough-directive-name>

cli/cmds/account.js

@@ -0,0 +1,16 @@
+const child = require('child_process')
+
+exports.command = 'account'
+
+exports.describe = 'Get or set current Google account configuration'
+
+exports.builder = {
+  dir: {
+    default: '.'
+  }
+}
+
+exports.handler = (argv) => {
+  if (argv.set) return child.execSync(`gcloud config set account ${argv.set}`).toString()
+  return child.execSync(`gcloud config get-value account --format=object`).toString()
+}

cli/cmds/buildConf.js

@@ -0,0 +1,72 @@
+const { info, success, err } = require('../utils/logger')
+const { baseDir, envDir, deploymentDir } = require('../utils/paths')
+const vinyl = require('vinyl-fs')
+const replace = require('gulp-frep')
+const fs = require('fs-extra')
+const del = require('del')
+
+require('env-yaml').config({ path: envDir })
+
+exports.command = 'buildConf'
+
+exports.describe = 'Build or rebuild configuration files after changing .env.yml'
+
+exports.builder = {
+  dir: {
+    default: '.'
+  }
+}
+
+exports.handler = (argv) => {
+  const patterns = [
+    {
+      pattern: /SERVICE_ACCOUNT/g,
+      replacement: `${process.env.SERVICE_ACCOUNT}`
+    },
+    {
+      pattern: /PROJECT_ID/g,
+      replacement: `${process.env.PROJECT_ID}`
+    },
+    {
+      pattern: /SOURCE_ID/g,
+      replacement: `${process.env.SOURCE_ID}`
+    },
+    {
+      pattern: /BUCKET_NAME/g,
+      replacement: `${process.env.BUCKET_NAME}`
+    },
+    {
+      pattern: /REGION/g,
+      replacement: `${process.env.REGION}`
+    },
+    {
+      pattern: /DEPLOYMENT_DIR/g,
+      replacement: `${deploymentDir}`
+    },
+    {
+      pattern: /BASE_DIR/g,
+      replacement: `${baseDir}`
+    },
+    {
+      pattern: /GCLOUD_ORG/g,
+      replacement: `${process.env.SOURCE_ID}`.split('/')[1]
+    },
+    {
+      pattern: /\*.*\s\*\s\*\s\*\s*\s\*/gi,
+      replacement: `*/${process.env.INTERVAL.replace('m', '')} * * * *`
+    }
+  ]
+  setTimeout(function () {
+    try {
+      vinyl.src([`templates/*.yml`, `!templates/setEnvPrompts.yml`])
+        .pipe(replace(patterns))
+        .pipe(vinyl.dest(`confs`))
+    } catch (e) {
+      throw err(e)
+    } finally {
+      success(`\n\nService Account Key created and environment variables set. To modify this file, use \n\n$ nano ${envDir}\n\n${fs.readFileSync(envDir)}`)
+      info(`Project: ${process.env.PROJECT_ID}, Org: ${process.env.GCLOUD_ORG}`)
+      success(`Click next -->`)
+    }
+  }, 1500)
+}

cli/cmds/deploy.js

@@ -0,0 +1,37 @@
+const yaml = {
+  read: require('read-yaml')
+}
+const child = require('child_process')
+const { info, success, err } = require('../utils/logger')
+const { baseDir, deploymentDir, envDir } = require('../utils/paths')
+require('env-yaml').config({ path: envDir })
+
+exports.command = 'deploy'
+
+exports.describe = 'Deploy Cloud Functions'
+
+exports.builder = {
+  dir: {
+    default: '.'
+  }
+}
+
+exports.handler = async function deploy (argv) {
+  const sh = cmd => child.execSync(cmd).toString()
+  let file = yaml.read.sync(`${baseDir}/confs/deploy.yml`)
+  console.log(file.create_bucket)
+  try {
+    sh(file.create_bucket.main)
+    success(`Bucket created. Starting function deployment`)
+  } catch (e) {
+    info(`Bucket already created. Starting function deployment`)
+  } finally {
+    let cmdString = ''
+    for (let [k, v] of Object.entries(file.deploy_function)) {
+      cmdString += ` --${k}=${v}`
+    }
+    cmdString = 'gcloud beta functions deploy FirewallEventsToSecurityCenter' + cmdString
+    cmdString = cmdString.trimRight().trimLeft()
+    sh(cmdString)
+  }
+}

cli/cmds/enableAPIs.js

@@ -0,0 +1,39 @@
+const shP = require('exec-sh').promise
+const yaml = {
+  read: require('read-yaml')
+}
+
+exports.command = 'enableAPIs'
+
+exports.describe = 'Enable APIs'
+
+exports.builder = {
+  dir: {
+    default: '.'
+  }
+}
+
+exports.handler = async function enableAPIs () {
+  const { info, success, err } = require('../utils/logger')
+  const { baseDir } = require('../utils/paths')
+  let cmds = yaml.read.sync(`${baseDir}/confs/enableAPIs.yml`)
+  cmds = Object.values(cmds.apis)
+  let i = 1
+  const runCmds = cmds.map(async cmd => {
+    let out
+    try {
+      out = await shP(`${cmd}`, false)
+    } catch (e) {
+      if (e instanceof TypeError) err(e)
+    }
+    return out
+  })
+
+  for (const cmd of runCmds) {
+    await cmd
+    success(`${i++}/${cmds.length} .. ${cmds[i - 2]} succeeded`)
+    if (i === runCmds.length) {
+      setTimeout(() => { info('\nClick Next -->') }, 2000)
+    }
+  }
+}