From e7123c1095f3af040a3136c1e3e1b42942c44284 Mon Sep 17 00:00:00 2001 From: Himanshu Garg Date: Thu, 19 May 2022 13:01:57 -0700 Subject: [PATCH 1/5] add linter shell script --- linter.sh | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100755 linter.sh diff --git a/linter.sh b/linter.sh new file mode 100755 index 0000000..80548a1 --- /dev/null +++ b/linter.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# +# Copyright (c) 2022 Circle Internet Financial Trading Company Limited. +# All rights reserved. +# +# Circle Internet Financial Trading Company Limited CONFIDENTIAL +# This file includes unpublished proprietary source code of Circle Internet +# Financial Trading Company Limited, Inc. The copyright notice above does not +# evidence any actual or intended publication of such source code. Disclosure +# of this source code or any related proprietary information is strictly +# prohibited without the express written permission of Circle Internet Financial +# Trading Company Limited. +# + +## linter.sh - Helper script to lint yml files +## Requires the yamllint package. can be installed apt-get -y install yamllint +## Usage : ./linter.sh "FILE TO CHECK" +if [ $# -ne 1 ] +then + echo "Usage: $0 YAML file" >&2 + exit 1 +fi +FILE="$1" +# ensure yamllint package is installed +yamllint -v &> /dev/null +if [ $? -ne 0 ] +then + echo "Package yamllint does not exist. Please install yamllint" >&2 + exit 1 +fi + + +### Steps +if [ $FILE == *.yml ] || [ $FILE == *.yaml ] ; then + yamllint $FILE +else + echo "File $FILE does not exist or it isnt a yml or yaml file. Please check your file" + exit 1 +fi + + From 8756d518c7b516670efe93600ae3f832bfcf8bf6 Mon Sep 17 00:00:00 2001 From: Himanshu Garg Date: Tue, 31 May 2022 11:05:27 -0700 Subject: [PATCH 2/5] check access --- s3_push.sh | 50 +++++++++++++++++++++++++++++++++++++++++ scanner.sh | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100755 s3_push.sh create mode 100755 scanner.sh diff --git a/s3_push.sh b/s3_push.sh new file mode 100755 index 0000000..552ae8d --- /dev/null +++ b/s3_push.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# +# Copyright (c) 2022 Circle Internet Financial Trading Company Limited. +# All rights reserved. +# +# Circle Internet Financial Trading Company Limited CONFIDENTIAL +# This file includes unpublished proprietary source code of Circle Internet +# Financial Trading Company Limited, Inc. The copyright notice above does not +# evidence any actual or intended publication of such source code. Disclosure +# of this source code or any related proprietary information is strictly +# prohibited without the express written permission of Circle Internet Financial +# Trading Company Limited. +# + +## scanner.sh - Helper script to scan images using VirusTotalAPI +## Requires VIRUS_TOTAL_API_KEY env var + +if [ $# -ne 1 ] +then + echo "Usage: $0 Dir to be synced" >&2 + exit 1 +fi + +sync_to_s3 () { + echo "Uploading file to Virus total" + request=$(curl -sSL --request POST \ + --url "https://www.virustotal.com/vtapi/v2/file/scan" \ + --form "apikey=${VIRUS_TOTAL_API_KEY}" \ + --form "file=${FILE}") + resource=$(echo $request | jq .resource | tr -d '"') + + echo "Checking status of uploaded file" + response=$(curl -sSL --request GET \ + --url "https://www.virustotal.com/vtapi/v2/file/report?apikey=${VIRUS_TOTAL_API_KEY}&resource=${resource}") + + malware_found=$(echo $response | jq -r .positives | tr -d '"') + if [[ $malware_found -ne 0 ]] + then + echo "Malware Found" + exit 1 + fi + +} + +if [ $FILE == *.jpg ] || [ $FILE == *.png ] || [ $FILE == *.svg ] ; then + check_file_for_malware +else + echo "File $FILE does not exist or it isn't a .jpg, .svg or .png file. Please check your file" + exit 1 +fi diff --git a/scanner.sh b/scanner.sh new file mode 100755 index 0000000..818f0b2 --- /dev/null +++ b/scanner.sh @@ -0,0 +1,66 @@ +#!/bin/sh +# +# Copyright (c) 2022 Circle Internet Financial Trading Company Limited. +# All rights reserved. +# +# Circle Internet Financial Trading Company Limited CONFIDENTIAL +# This file includes unpublished proprietary source code of Circle Internet +# Financial Trading Company Limited, Inc. The copyright notice above does not +# evidence any actual or intended publication of such source code. Disclosure +# of this source code or any related proprietary information is strictly +# prohibited without the express written permission of Circle Internet Financial +# Trading Company Limited. +# + +## scanner.sh - Helper script to scan images using VirusTotalAPI +## Requires VIRUS_TOTAL_API_KEY env var + +if [ $# -ne 1 ] +then + echo "Usage: $0 Path to image to be scanned" >&2 + exit 1 +fi + +FILE="$1" + +if [ -f $FILE ] +then + echo "$FILE exists." +else + echo "$FILE does not exist or isn't a file" + exit 1 +fi + +if [[ -z "${VIRUS_TOTAL_API_KEY}" ]] +then + echo "VIRUS_TOTAL_API_KEY env var not defined" + exit 1 +fi + +check_file_for_malware () { + echo "Uploading file to Virus total" + request=$(curl -sSL --request POST \ + --url "https://www.virustotal.com/vtapi/v2/file/scan" \ + --form "apikey=${VIRUS_TOTAL_API_KEY}" \ + --form "file=${FILE}") + resource=$(echo $request | jq .resource | tr -d '"') + + echo "Checking status of uploaded file" + response=$(curl -sSL --request GET \ + --url "https://www.virustotal.com/vtapi/v2/file/report?apikey=${VIRUS_TOTAL_API_KEY}&resource=${resource}") + + malware_found=$(echo $response | jq -r .positives | tr -d '"') + if [[ $malware_found -ne 0 ]] + then + echo "Malware Found" + exit 1 + fi + +} + +if [ $FILE == *.jpg ] || [ $FILE == *.png ] || [ $FILE == *.svg ] ; then + check_file_for_malware +else + echo "File $FILE does not exist or it isn't a .jpg, .svg or .png file. Please check your file" + exit 1 +fi From 67bb9e6dae8b989bfb75171bd44679ce72ed41bb Mon Sep 17 00:00:00 2001 From: Himanshu Garg Date: Tue, 31 May 2022 12:09:36 -0700 Subject: [PATCH 3/5] change to use linter.py --- .github/workflows/lintscan.yml | 12 +++++----- .github/workflows/s3.yml | 30 ------------------------- linter.sh | 41 ---------------------------------- s3_push.sh | 2 +- 4 files changed, 8 insertions(+), 77 deletions(-) delete mode 100644 .github/workflows/s3.yml delete mode 100755 linter.sh diff --git a/.github/workflows/lintscan.yml b/.github/workflows/lintscan.yml index 8cff786..bde15f2 100644 --- a/.github/workflows/lintscan.yml +++ b/.github/workflows/lintscan.yml @@ -23,11 +23,13 @@ jobs: - uses: actions/checkout@v3 # Runs a single command using the runners shell - - name: Install yamllint package - run: sudo apt-get install -y yamllint + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt - name: Run Linter - run: ./linter.sh ecosystem_schema.yml - + run: ./linter.py + virus_scanner: # The type of runner that the job will run on runs-on: ubuntu-latest @@ -36,7 +38,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - + - name: Run Virus Scanner env: # Or as an environment variable VIRUS_TOTAL_API_KEY: ${{ secrets.VIRUS_TOTAL_API_KEY }} diff --git a/.github/workflows/s3.yml b/.github/workflows/s3.yml deleted file mode 100644 index acc7d80..0000000 --- a/.github/workflows/s3.yml +++ /dev/null @@ -1,30 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: DeployS3 - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the master branch - push: - branches: [ master ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 - - # Runs a single command using the runners shell - - name: Upload file to S3 - env: # Or as an environment variable - BUCKET: ${{ secrets.S3BUCKET }} - run: ./s3_push.sh $BUCKET ecosystem_schema.yml diff --git a/linter.sh b/linter.sh deleted file mode 100755 index ffc30c4..0000000 --- a/linter.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2022 Circle Internet Financial Trading Company Limited. -# All rights reserved. -# -# Circle Internet Financial Trading Company Limited CONFIDENTIAL -# This file includes unpublished proprietary source code of Circle Internet -# Financial Trading Company Limited, Inc. The copyright notice above does not -# evidence any actual or intended publication of such source code. Disclosure -# of this source code or any related proprietary information is strictly -# prohibited without the express written permission of Circle Internet Financial -# Trading Company Limited. -# - -## linter.sh - Helper script to lint yml files -## Requires the yamllint package. can be installed apt-get -y install yamllint -## Usage : ./linter.sh "FILE TO CHECK" -if [ $# -ne 1 ] -then - echo "Usage: $0 YAML file" >&2 - exit 1 -fi -FILE="$1" -# ensure yamllint package is installed -yamllint -v &> /dev/null -if [ $? -ne 0 ] -then - echo "Package yamllint does not exist. Please install yamllint" >&2 - exit 1 -fi - - -### Steps -if [[ $FILE == *.yml ]] || [[ $FILE == *.yaml ]] ; then - yamllint $FILE -else - echo "File $FILE does not exist or it isnt a yml or yaml file. Please check your file" - exit 1 -fi - - diff --git a/s3_push.sh b/s3_push.sh index 691f486..d00e060 100755 --- a/s3_push.sh +++ b/s3_push.sh @@ -28,7 +28,7 @@ else exit 1 fi -aws s3 cp $FILE s3://$BUCKET/$FILE +aws s3 cp $FILE s3://${BUCKET}/${FILE} #aws s3 sync catalog s3://$BUCKET/catalog --acl public-read --exclude "*" --include "*.yml" --include "*.yaml" From 2c3698e49628f38aba4a0b11881b9670a9ddf0ba Mon Sep 17 00:00:00 2001 From: Himanshu Garg Date: Tue, 31 May 2022 12:33:44 -0700 Subject: [PATCH 4/5] sign commit From 4a90cd899b1ee9ede2247362278320fb386500ae Mon Sep 17 00:00:00 2001 From: Himanshu Garg Date: Tue, 31 May 2022 12:46:39 -0700 Subject: [PATCH 5/5] fix error --- .github/workflows/lintscan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lintscan.yml b/.github/workflows/lintscan.yml index 3d6762b..bc8e500 100644 --- a/.github/workflows/lintscan.yml +++ b/.github/workflows/lintscan.yml @@ -27,7 +27,7 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt - name: Run Linter - run: ./linter.py + run: python linter.py virus_scanner: # The type of runner that the job will run on