feat(CG-1339): add aws ebs snapshot

Author: James Zhou

src/enums/resources.ts

@@ -23,6 +23,7 @@ export default {
   sqsQueue: 'aws_sqs_queue',
   iamGroup: 'aws_iam_group',
   snsTopic: 'aws_sns_topic',
+  ebsSnapshot: 'aws_ebs_snapshot',
   ebsVolume: 'aws_ebs_volume',
   iamPolicy: 'aws_iam_policy',
   vpnGateway: 'aws_vpn_gateway',

src/enums/schemasMap.ts

@@ -31,6 +31,7 @@ export default {
   [services.dmsReplicationInstance]: 'awsDmsReplicationInstance',
   [services.dynamodb]: 'awsDynamoDbTable',
   [services.ebs]: 'awsEbs',
+  [services.ebsSnapshot]: 'awsEbsSnapshot',
   [services.ec2Instance]: 'awsEc2',
   [services.ecr]: 'awsEcr',
   [services.ecsCluster]: 'awsEcsCluster',

src/enums/serviceAliases.ts

@@ -17,6 +17,7 @@ export default {
   [services.codebuild]: 'codebuilds',
   [services.configurationRecorder]: 'configurationRecorders',
   [services.dmsReplicationInstance]: 'dmsReplicationInstances',
+  [services.ebsSnapshot]: 'ebsSnapshots',
   [services.ec2Instance]: 'ec2Instances',
   [services.ecsCluster]: 'ecsClusters',
   [services.ecsContainer]: 'ecsContainers',

src/enums/serviceMap.ts

@@ -23,6 +23,7 @@ import CognitoIdentityPool from '../services/cognitoIdentityPool'
 import CognitoUserPool from '../services/cognitoUserPool'
 import DynamoDB from '../services/dynamodb'
 import EBS from '../services/ebs'
+import EBSSnapshot from '../services/ebsSnapshot'
 import EC2 from '../services/ec2'
 import EcsCluster from '../services/ecsCluster'
 import EcsContainer from '../services/ecsContainer'
@@ -133,6 +134,7 @@ export default {
   [services.cognitoUserPool]: CognitoUserPool,
   [services.configurationRecorder]: ConfigurationRecorder,
   [services.ebs]: EBS,
+  [services.ebsSnapshot]: EBSSnapshot,
   [services.ec2Instance]: EC2,
   [services.ecr]: ECR,
   [services.efs]: EFS,

src/enums/services.ts

@@ -25,6 +25,7 @@ export default {
   dmsReplicationInstance: 'dmsReplicationInstance',
   dynamodb: 'dynamodb',
   ebs: 'ebs',
+  ebsSnapshot: 'ebsSnapshot',
   ec2Instance: 'ec2Instance',
   ecr: 'ecr',
   ecsCluster: 'ecsCluster',

src/properties/logger.ts

@@ -249,6 +249,13 @@ export default {
   doneFetchingEbsData: '✅ Done fetching EBS Data ✅',
   fetchedEbsVolumes: (num: number): string => `Fetched ${num} EBS Volumes`,
   lookingForEbs: 'Looking for EBS volumes for EC2 instances...',
+  /**
+   * EBS Snapshot
+   */
+    fetchingEbsSnapshotData: 'Fetching EBS Snapshot data for this AWS account via the AWS SDK...',
+    doneFetchingEbsSnapshotData: '✅ Done fetching EBS Snapshot Data ✅',
+    fetchedEbsSnapshots: (num: number): string => `Fetched ${num} EBS Snapshots`,
+    lookingForEbsSnapshot: 'Looking for EBS Snapshots...',
   /**
    * EC2
    */

src/services/ebs/connections.ts

@@ -0,0 +1,70 @@
+import isEmpty from 'lodash/isEmpty'
+
+import EC2, {
+  Volume,
+  Snapshot,
+  TagList,
+} from 'aws-sdk/clients/ec2'
+
+import { ServiceConnection } from '@cloudgraph/sdk'
+
+import services from '../../enums/services'
+
+
+/**
+ * EBS
+ */
+
+export default ({
+    service: volume,
+    data,
+    region,
+    account,
+  }: {
+    account: string
+    data: { name: string; data: { [property: string]: any[] } }[]
+    service: Volume & {
+      region: string
+      Tags?: TagList
+    }
+    region: string
+}): { [key: string]: ServiceConnection[] } => {
+    const connections: ServiceConnection[] = []
+
+    const {
+      VolumeId: id,
+      SnapshotId: snapshotId,
+      Tags: tags,
+    } = volume
+
+    /**
+     * Find EBS Snapshot
+     * related to this EBS Volume
+     */
+    const ebsSnapshots: {
+      name: string
+      data: { [property: string]: Snapshot[] }
+    } = data.find(({ name }) => name === services.ebsSnapshot)
+
+    if (ebsSnapshots?.data?.[region]) {
+      const snapshotInRegion: Snapshot[] = ebsSnapshots.data[region].filter(
+        ({ SnapshotId }: Snapshot) => SnapshotId === snapshotId
+      )
+  
+      if (!isEmpty(snapshotInRegion)) {
+        for (const sh of snapshotInRegion) {
+          connections.push({
+            id: sh.SnapshotId,
+            resourceType: services.ebsSnapshot,
+            relation: 'child',
+            field: 'ebsSnapshots',
+          })
+        }
+      }
+    }
+
+    const ebsResult = {
+      [id]: connections,
+    }
+    return ebsResult
+}

src/services/ebs/index.ts

@@ -2,12 +2,15 @@ import { Service } from '@cloudgraph/sdk'
 import BaseService from '../base'
 import format from './format'
 import getData from './data'
+import getConnections from './connections'
 import mutation from './mutation'
 
 export default class EBS extends BaseService implements Service {
   format = format.bind(this)
 
   getData = getData.bind(this)
 
+  getConnections = getConnections.bind(this)
+
   mutation = mutation
 }

src/services/ebs/schema.graphql

@@ -15,6 +15,7 @@ type awsEbs implements awsBaseService @key(fields: "arn") {
   ec2Instance: [awsEc2] @hasInverse(field: ebs)
   asg: [awsAsg] @hasInverse(field: ebs)
   emrInstance: [awsEmrInstance] @hasInverse(field: ebs)
+  ebsSnapshots: [awsEbsSnapshot] @hasInverse(field: ebs)
 }
 
 type awsEbsAttachment

src/services/ebsSnapshot/data.ts

@@ -0,0 +1,189 @@
+import EC2, {
+  DescribeSnapshotsResult,
+  DescribeSnapshotsRequest,
+  Snapshot,
+  CreateVolumePermission,
+  DescribeSnapshotAttributeResult,
+} from 'aws-sdk/clients/ec2'
+import { Config } from 'aws-sdk/lib/config'
+import { AWSError } from 'aws-sdk/lib/error'
+
+import groupBy from 'lodash/groupBy'
+import isEmpty from 'lodash/isEmpty'
+
+import CloudGraph from '@cloudgraph/sdk'
+
+import { AwsTag, TagMap } from '../../types'
+
+import { initTestEndpoint } from '../../utils'
+import AwsErrorLog from '../../utils/errorLog'
+import { convertAwsTagsToTagMap } from '../../utils/format'
+import awsLoggerText from '../../properties/logger'
+
+/**
+ * EBS Snapshot
+ */
+
+const lt = { ...awsLoggerText }
+const { logger } = CloudGraph
+const serviceName = 'EBS'
+const errorLog = new AwsErrorLog(serviceName)
+const endpoint = initTestEndpoint(serviceName)
+
+export interface RawAwsEBSSnapshot extends Omit<Snapshot, 'Tags'> {
+  region: string
+  Permissions?: CreateVolumePermission[]
+  Tags?: TagMap
+}
+
+const listEbsSnapshotAttribute = async ({
+  ec2,
+  snapshotId,
+}: {
+  ec2: EC2
+  snapshotId: string
+}): Promise<CreateVolumePermission[]> =>
+  new Promise(resolve =>
+    ec2.describeSnapshotAttribute(
+      {
+        Attribute: 'createVolumePermission',
+        SnapshotId: snapshotId,
+      },
+      (err: AWSError, data: DescribeSnapshotAttributeResult) => {
+        if (err) {
+          errorLog.generateAwsErrorLog({
+            functionName: 'ec2:describeSnapshotAttribute',
+            err,
+          })
+        }
+
+        if (isEmpty(data)) {
+          return resolve([])
+        }
+
+        return resolve(data?.CreateVolumePermissions)
+      }
+    )
+  )
+
+const listEbsSnapshots = async ({
+  ec2,
+  region,
+  nextToken: NextToken = '',
+  ebsSnapshotData,
+  resolveRegion,
+}: {
+  ec2: EC2
+  region: string
+  nextToken?: string
+  ebsSnapshotData: RawAwsEBSSnapshot[]
+  resolveRegion: () => void
+}): Promise<void> => {
+  let args: DescribeSnapshotsRequest = {
+    OwnerIds: ['self']
+  }
+
+  if (NextToken) {
+    args = { ...args, NextToken }
+  }
+
+  ec2.describeSnapshots(
+    args,
+    async (err: AWSError, data: DescribeSnapshotsResult) => {
+      if (err) {
+        errorLog.generateAwsErrorLog({
+          functionName: 'ec2:describeSnapshots',
+          err,
+        })
+      }
+
+      /**
+       * No EBS snapshot data for this region
+       */
+      if (isEmpty(data)) {
+        return resolveRegion()
+      }
+
+      const { NextToken: nextToken, Snapshots: snapshots } = data || {}
+      logger.debug(lt.fetchedEbsSnapshots(snapshots.length))
+
+      /**
+       * No EBS Snapshot Found
+       */
+
+      if (isEmpty(snapshots)) {
+        return resolveRegion()
+      }
+
+      /**
+       * Check to see if there are more
+       */
+
+      if (nextToken) {
+        listEbsSnapshots({ region, nextToken, ec2, ebsSnapshotData, resolveRegion })
+      }
+
+      const ebsVolumes = []
+
+      for (const { Tags, SnapshotId, ...snapshot } of snapshots) {
+        let snapshotAttributes: CreateVolumePermission[] = []
+        if (SnapshotId) {
+          snapshotAttributes = await listEbsSnapshotAttribute({
+            ec2,
+            snapshotId: SnapshotId,
+          })
+        }
+        ebsVolumes.push({
+          ...snapshot,
+          region,
+          SnapshotId,
+          Permissions: snapshotAttributes,
+          Tags: convertAwsTagsToTagMap(Tags as AwsTag[]),
+        })
+      }
+
+      ebsSnapshotData.push(...ebsVolumes)
+
+      /**
+       * If this is the last page of data then return the instances
+       */
+
+      if (!nextToken) {
+        resolveRegion()
+      }
+    }
+  )
+}
+
+export default async ({
+  regions,
+  config,
+}: {
+  regions: string
+  config: Config
+}): Promise<{
+  [region: string]: Snapshot & { region: string }[]
+}> =>
+  new Promise(async resolve => {
+    const ebsSnapshotData: RawAwsEBSSnapshot[] = []
+    const volumePromises: Promise<void>[] = []
+
+    // Get all the EBS data for each region with its snapshots
+    for (const region of regions.split(',')) {
+      const ec2 = new EC2({ ...config, region, endpoint })
+
+      volumePromises.push(
+        new Promise<void>(resolveRegion =>
+          listEbsSnapshots({ ec2, region, ebsSnapshotData, resolveRegion })
+        )
+      )
+    }
+
+    logger.debug(lt.fetchingEbsSnapshotData)
+    // Fetch EBS volumes
+    await Promise.all(volumePromises)
+
+    errorLog.reset()
+
+    resolve(groupBy(ebsSnapshotData, 'region'))
+  })