Feature/lambdafication revised

.gitignore

@@ -7,6 +7,9 @@ output.json
 vscode/*
 .vscode/
 .idea
+.local
+template.packaged.yaml
+config.*
 coverage/
 .nyc_output
-config.js
+config.js

LambdaInstall.md

@@ -0,0 +1,56 @@
+# CloudSploit Lambda Deployment Guide
+
+The `Makefile` has all of the commands required to build, package, and deploy CloudSploit to AWS Lambda. It makes use of environment variables for configuration.
+
+## Environment Variable COnfiguration
+
+|   Environment Variable    | Description
+|---------------------------|-------------
+| `ARTIFACT_BUCKET`         | The name of the bucket to upload the zipped lambda code to
+| `STACK_NAME`              | The name to give the cloudformation stack
+| `DEFAULT_ROLE_NAME`       | The default to use for the name of the role for CloudSploit to assume
+| `SECRETS_MANAGER_PREFIX`  | The prefix to be used for secrets manager secrets
+| `BUCKET_NAME`             | The name of the bucket to write output to
+| `BUCKET_PREFIX`           | The prefix to write within the bucket
+| `CREATE_BUCKET`           | Whether or not to create the bucket, using the given bucket name ("no" or "yes")
+| `SNS_TOPIC`               | The topic to subscribe the CloudSploit lambda to
+| `SCHEDULE`                | A schedule expression to run the lambda on
+| `SCHEDULED_ACCOUNT_ID`    | If schedule provided, the account ID to scan
+| `SCHEDULED_ROLE_NAME`     | If schedule provided, the role name to assume
+| `SCHEDULED_EXTERNAL_ID`   | If schedule provided, the external ID to use when assuming the role
+
+## Sample Config File
+
+```bash
+export ARTIFACT_BUCKET=mybucket
+export STACK_NAME=cloudsploit-test
+export DEFAULT_ROLE_NAME=cloudsploit-role
+export SECRETS_MANAGER_PREFIX="/cloudsploit/secrets/"
+export BUCKET_NAME=cloudsploit-output-bucket
+export BUCKET_PREFIX="/"
+export CREATE_BUCKET=yes
+export SNS_TOPIC=""
+export SCHEDULE=""
+export SCHEDULED_ACCOUNT_ID=""
+export SCHEDULED_ROLE_NAME=""
+export SCHEDULED_EXTERNAL_ID=""
+```
+
+## Deploy Process
+1. Create a file with the above variables called `config.ENVIRONMENT-DESIGNATOR`, where ENVIRONMENT-DESIGNATOR can be your environment (dev, stage, prod)
+2. run `make deploy env=ENVIRONMENT-DESIGNATOR`
+    * Make deploy will install the npm modules, build the lambda SAM package, and deploy the lambda as a Cloudformation Template.
+
+## Lambda Trigger
+The Lambda can be triggered by a StepFunction or via the AWS CLI using the following event structure:
+```json
+{
+    "aws": {
+        "roleArn": "arn:aws:iam::ACCOUNTID:role/ROLENAME"
+    },
+    "s3Prefix": "PREFIX_FOR_FINDINGS_FROM_THIS_SPECIFIC_INVOCATION"
+}
+```
+Substitute the appropriate values for ACCOUNTID, ROLENAME and PREFIX_FOR_FINDINGS_FROM_THIS_SPECIFIC_INVOCATION
+
+More details on Lambda Invocation options are in the lambda_event_details.md file

Makefile

@@ -0,0 +1,60 @@
+
+ifndef env
+# $(error env is not set)
+	env ?= dev
+endif
+
+ifdef CONFIG
+	include $(CONFIG)
+	export
+else
+	include config.$(env)
+	export
+endif
+
+ifndef STACK_NAME
+$(error STACK_NAME is not set)
+endif
+
+.PHONY: $(FUNCTIONS)
+
+# # Run all tests
+# test: cfn-validate
+# 	cd lambda && $(MAKE) test
+
+# # Do everything
+# clean:
+# 	cd lambda && $(MAKE) clean
+
+build:
+	npm install
+	# because for some reason, the node modules were dated to 1985
+	find ./node_modules/* -mtime +10950 -exec touch {} \;
+
+package: build
+	aws cloudformation package \
+	  --s3-bucket ${ARTIFACT_BUCKET} \
+	  --template-file ./cloudformation/template.yaml \
+	  --output-template-file ./template.packaged.yaml
+
+deploy: package
+	aws cloudformation deploy \
+	  --template-file ./template.packaged.yaml \
+	  --capabilities CAPABILITY_IAM \
+	  --no-fail-on-empty-changeset \
+	  --stack-name ${STACK_NAME} \
+	  --parameter-overrides \
+	    DefaultRoleName=${DEFAULT_ROLE_NAME} \
+	    SecretsManagerPrefix=${SECRETS_MANAGER_PREFIX} \
+	    BucketName=${BUCKET_NAME} \
+	    BucketPrefix=${BUCKET_PREFIX} \
+	    CreateBucket=${CREATE_BUCKET} \
+	    SNSTopic=${SNS_TOPIC} \
+	    Schedule=${SCHEDULE} \
+	    ScheduledAccountId=${SCHEDULED_ACCOUNT_ID} \
+	    ScheduledRoleName=${SCHEDULED_ROLE_NAME} \
+	    ScheduledExternalId=${SCHEDULED_EXTERNAL_ID}
+
+sync-scorecards:
+	aws s3 sync s3://$(BUCKET_NAME)/$(BUCKET_PREFIX) Results/$(STACK_NAME)
+	open Results/$(STACK_NAME)

cloudformation/deploy.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -euxo pipefail
+
+npm install
+
+aws cloudformation package \
+  --s3-bucket ${ARTIFACT_BUCKET} \
+  --template-file ./cloudformation/template.yaml \
+  --output-template-file ./template.packaged.yaml
+
+aws cloudformation deploy \
+  --template-file ./template.packaged.yaml \
+  --capabilities CAPABILITY_IAM \
+  --no-fail-on-empty-changeset \
+  --stack-name ${STACK_NAME} \
+  --parameter-overrides \
+    DefaultRoleName=${DEFAULT_ROLE_NAME} \
+    SecretsManagerPrefix=${SECRETS_MANAGER_PREFIX} \
+    BucketName=${BUCKET_NAME} \
+    BucketPrefix=${BUCKET_PREFIX} \
+    CreateBucket=${CREATE_BUCKET} \
+    SNSTopic=${SNS_TOPIC} \
+    Schedule=${SCHEDULE} \
+    ScheduledAccountId=${SCHEDULED_ACCOUNT_ID} \
+    ScheduledRoleName=${SCHEDULED_ROLE_NAME} \
+    ScheduledExternalId=${SCHEDULED_EXTERNAL_ID}

cloudformation/template.yaml

@@ -0,0 +1,140 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Description: Deploy a lambda version of CloudSploit Scanning Engine, triggered by SNS Message
+
+
+Parameters:
+  DefaultRoleName:
+    Description: The default role name to assume when one is not provided
+    Type: String
+
+  SecretsManagerPrefix:
+    Description: Prefix of secrets manager secrets for access to provider credentials
+    Type: String
+    Default: ''
+
+  BucketName:
+    Description: Name of the bucket to hold scan reports
+    Type: String
+  BucketPrefix:
+    Description: The prefix to upload the scan reports under
+    Type: String
+    Default: ''
+  CreateBucket:
+    Description: Create the bucket or use an existing bucket
+    Type: String
+    Default: 'no'
+    AllowedValues:
+      - 'yes'
+      - 'no'
+
+  SNSTopic:
+    Description: ARN from the inventory stack that triggers parallel actions across all accounts
+    Type: String
+    Default: ''
+
+  Schedule:
+    Description: A schedule to be applied to the lambda
+    Type: String
+    Default: ''
+  ScheduledAccountId:
+    Description: Only used if schedule is specified, scan this account ID when invoked
+    Type: String
+    Default: ''
+  ScheduledRoleName:
+    Description: Only used if schedule is specified, assume this role name to scan
+    Type: String
+    Default: ''
+  ScheduledExternalId:
+    Description: Only used if schedule is specified, use this external id to assume the role
+    Type: String
+    NoEcho: True
+    Default: ''
+
+
+Conditions:
+  SubscribeLambda: !Not [ !Equals [ !Ref SNSTopic, '' ] ]
+  CreateSchedule: !Not [ !Equals [ !Ref Schedule, ''] ]
+  ShouldCreateBucket: !Equals [ !Ref CreateBucket, 'yes' ]
+
+
+Resources:
+  CloudsploitScanner:
+    Type: AWS::Serverless::Function
+    Properties:
+      FunctionName: !Sub ${AWS::StackName}-scanner
+      Description: Execute a CloudSploit Scan on the specific AWS account
+      Timeout: 900
+      MemorySize: 3008
+      Handler: lambda_index.handler
+      Runtime: nodejs10.x
+      CodeUri: ..
+      Environment:
+        Variables:
+          DEFAULT_ROLE_NAME: !Ref DefaultRoleName
+          RESULT_BUCKET: !Ref BucketName
+          RESULT_PREFIX: !Ref BucketPrefix
+          SECRET_PREFIX: !Ref SecretsManagerPrefix
+      Policies:
+        - Version: '2012-10-17'
+          Statement:
+            - Action: s3:PutObject
+              Effect: Allow
+              Resource:
+                - !Sub arn:aws:s3:::${BucketName}/*
+                - !Sub arn:aws:s3:::${BucketName}
+            - Effect: Allow
+              Action: sts:AssumeRole
+              Resource: '*'
+            - Effect: Allow
+              Action: secretsmanager:GetSecretValue
+              Resource: !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretsManagerPrefix}/*
+
+  CloudsploitScannerSchedule:
+    Condition: CreateSchedule
+    Type: AWS::Events::Rule
+    Properties:
+      ScheduleExpression: !Ref Schedule
+      State: ENABLED
+      Targets:
+        - Arn: !GetAtt CloudsploitScanner.Arn
+          Id: CloudsploitScannerScheduleLambdaTarget
+          Input: !Sub |
+            {
+              "aws": {
+                "accountID": "${ScheduledAccountId}",
+                "roleArn": "${ScheduledRoleName}",
+                "externalId": "${ScheduledExternalId}"
+              }
+            }
+
+  CloudsploitScannerSNSPermission:
+    Condition: SubscribeLambda
+    Type: AWS::Lambda::Permission
+    Properties:
+      FunctionName: !GetAtt CloudsploitScanner.Arn
+      Principal: sns.amazonaws.com
+      SourceArn: !Ref SNSTopic
+      Action: lambda:invokeFunction
+
+  CloudsploitScannerSubscription:
+    Condition: SubscribeLambda
+    Type: AWS::SNS::Subscription
+    Properties:
+      Endpoint: !GetAtt CloudsploitScanner.Arn
+      Protocol: lambda
+      TopicArn: !Ref SNSTopic
+
+  CloudsploitResultsBucket:
+    Condition: ShouldCreateBucket
+    Type: AWS::S3::Bucket
+    Properties:
+      BucketName: !Ref BucketName
+
+
+Outputs:
+  CloudsploutLambdaArn:
+    Value: !GetAtt CloudsploitScanner.Arn
+
+  CloudsploutLambdaName:
+    Value: !Ref CloudsploitScanner

collectors/aws/collector.js

@@ -954,7 +954,7 @@ var collect = function(AWSConfig, settings, callback) {
                     globalServices.indexOf(service) === -1) return regionCb();
                 if (!collection[serviceLower][callKey][region]) collection[serviceLower][callKey][region] = {};
 
-                var LocalAWSConfig = JSON.parse(JSON.stringify(AWSConfig));
+                var LocalAWSConfig = Object.assign({}, AWSConfig);
                 LocalAWSConfig.region = region;
 
                 if (callObj.override) {
@@ -1058,7 +1058,7 @@ var collect = function(AWSConfig, settings, callback) {
                             !collection[callObj.reliesOnService][callObj.reliesOnCall][region].data ||
                             !collection[callObj.reliesOnService][callObj.reliesOnCall][region].data.length)) return regionCb();
 
-                        var LocalAWSConfig = JSON.parse(JSON.stringify(AWSConfig));
+                        var LocalAWSConfig = Object.assign({}, AWSConfig);
                         if (callObj.deleteRegion) {
                             //delete LocalAWSConfig.region;
                             LocalAWSConfig.region = settings.govcloud ? 'us-gov-west-1' : settings.china ? 'cn-north-1' : 'us-east-1';