Skip to content

Commit e79ce76

Browse files
jacobmsftjacobmsft
committed
Initial checkin/move to the codeql-container registry
1 parent 8cca5f7 commit e79ce76

File tree

10 files changed

+293
-0
lines changed

10 files changed

+293
-0
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ __pycache__/
33
*.py[cod]
44
*$py.class
55

6+
# VS
7+
/.vs/**
8+
69
# C extensions
710
*.so
811

‎Dockerfile‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
FROM ubuntu:20.04 AS codeql_base
2+
LABEL maintainer="Github codeql team"
3+
4+
# tzdata install needs to be non-interactive
5+
ENV DEBIAN_FRONTEND=noninteractive
6+
7+
# install/update basics and python
8+
RUN apt-get update && \
9+
apt-get upgrade -y && \
10+
apt-get install -y --no-install-recommends \
11+
software-properties-common \
12+
vim \
13+
curl \
14+
wget \
15+
git \
16+
build-essential \
17+
unzip \
18+
apt-transport-https \
19+
python3.8 \
20+
python3-venv \
21+
python3-pip \
22+
python3-setuptools \
23+
python3-dev \
24+
gnupg \
25+
g++ \
26+
make \
27+
gcc \
28+
apt-utils \
29+
rsync \
30+
file \
31+
gettext && \
32+
apt-get clean
33+
34+
# Clone our setup and run scripts
35+
#RUN git clone https://github.com/microsoft/codeql-container /usr/local/startup_scripts
36+
RUN mkdir -p /usr/local/startup_scripts
37+
RUN ls -al /usr/local/startup_scripts
38+
COPY container /usr/local/startup_scripts/
39+
RUN pip3 install --upgrade pip \
40+
&& pip3 install -r /usr/local/startup_scripts/requirements.txt
41+
42+
# Install latest codeQL
43+
ENV CODEQL_HOME /usr/local/codeql-home
44+
# record the latest version of the codeql-cli
45+
RUN python3 /usr/local/startup_scripts/get-latest-codeql-version.py > /tmp/codeql_version
46+
RUN mkdir -p ${CODEQL_HOME} \
47+
${CODEQL_HOME}/codeql-cli \
48+
${CODEQL_HOME}/codeql-repo \
49+
${CODEQL_HOME}/codeql-go-repo \
50+
/opt/codeql
51+
52+
RUN CODEQL_VERSION=$(cat /tmp/codeql_version) && \
53+
wget -q https://github.com/github/codeql-cli-binaries/releases/download/${CODEQL_VERSION}/codeql-linux64.zip -O /tmp/codeql_linux.zip && \
54+
unzip /tmp/codeql_linux.zip -d ${CODEQL_HOME}/codeql-cli && \
55+
rm /tmp/codeql_linux.zip
56+
57+
# get the latest codeql queries and record the HEAD
58+
RUN git clone https://github.com/github/codeql ${CODEQL_HOME}/codeql-repo && \
59+
git --git-dir ${CODEQL_HOME}/codeql-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-repo-last-commit
60+
RUN git clone https://github.com/github/codeql-go ${CODEQL_HOME}/codeql-go-repo && \
61+
git --git-dir ${CODEQL_HOME}/codeql-go-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-go-repo-last-commit
62+
63+
ENV PATH="${CODEQL_HOME}/codeql-cli/codeql:${PATH}"
64+
65+
# Pre-compile our queries to save time later
66+
#RUN codeql query compile --threads=0 ${CODEQL_HOME}/codelq-repo/*/ql/src/codeql-suites/*-.qls
67+
#RUN codeql query compile --threads=0 ${CODEQL_HOME}/codelq-go-repo/ql/src/codeql-suites/*-.qls
68+
#ENTRYPOINT ["python3", "/usr/local/startup_scripts/setup.py"]

‎container/get-latest-codeql-version.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/python3
2+
# get the parent directory of the script, to link libs
3+
4+
import os
5+
import sys
6+
sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
7+
8+
from libs.github import get_latest_github_repo_version
9+
10+
def main():
11+
latest_release = get_latest_github_repo_version("github/codeql-cli-binaries")
12+
print(latest_release.title)
13+
14+
main()

‎container/libs/__init__.py‎

Whitespace-only changes.

‎container/libs/codeql.py‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
from sys import exit
2+
from re import search
3+
from zipfile import ZipFile
4+
from libs.utils import *
5+
from logging import getLogger
6+
from libs.github import get_latest_github_repo_version
7+
8+
logger = getLogger('codeql-container')
9+
10+
class CodeQL:
11+
"""
12+
Codeql related logic, downloading, installing etc
13+
"""
14+
CODEQL_HOME = None
15+
16+
CODEQL_GITHUB_URL = 'https://github.com/github/codeql-cli-binaries'
17+
CODEQL_QUERIES_URL = 'https://github.com/github/codeql'
18+
CODEQL_GO_QUERIES_URL = 'https://github.com/github/codeql-go'
19+
20+
TEMP_DIR ='/tmp'
21+
22+
# Error codes
23+
ERROR_EXECUTING_COMMAND = 1
24+
ERROR_EXECUTING_CODEQL = 2
25+
ERROR_UNKNOWN_OS = 3
26+
ERROR_GIT_COMMAND = 4
27+
28+
def __init__(self, codeql_base_dir):
29+
self.CODEQL_HOME = codeql_base_dir
30+
31+
def download_and_install_latest_codeql(self, github_version):
32+
"""
33+
Download and install the latest codeql-cli from the github repo releases
34+
"""
35+
download_url = None
36+
download_path = None
37+
if os_name == 'posix':
38+
download_url = f'https://github.com/github/codeql-cli-binaries/releases/download/{github_version.title}/codeql-linux64.zip'
39+
download_path = f'{self.TEMP_DIR}/codeql_linux.zip'
40+
elif os_name == 'nt':
41+
download_url = f'https://github.com/github/codeql-cli-binaries/releases/download/{github_version.title}/codeql-win64.zip'
42+
download_path = f'{self.TEMP_DIR}/codeql_windows.zip'
43+
else:
44+
exit(ERROR_UNKNOWN_OS)
45+
46+
logger.info(f'Downloading codeql-cli version {github_version.title}...')
47+
check_output_wrapper(f"wget -q {download_url} -O {download_path}", shell=True).decode("utf-8")
48+
self.install_codeql_cli(download_path)
49+
#rm /tmp/codeql_linux.zip
50+
51+
def download_and_install_latest_codeql_queries(self):
52+
"""
53+
Download and install the latest codeql queries from the github repo
54+
"""
55+
logger.info("Downloading codeql queries...")
56+
codeql_repo_dir = f'{self.CODEQL_HOME}/codeql-repo'
57+
wipe_and_create_dir(codeql_repo_dir)
58+
ret1 = check_output_wrapper(f'git clone {self.CODEQL_QUERIES_URL} {codeql_repo_dir}', shell=True)
59+
60+
codeql_go_repo_dir = f'{self.CODEQL_HOME}/codeql-go-repo'
61+
wipe_and_create_dir(codeql_go_repo_dir)
62+
ret2 = check_output_wrapper(f'git clone {self.CODEQL_GO_QUERIES_URL} {codeql_go_repo_dir}', shell=True)
63+
if ret1 is CalledProcessError or ret2 is CalledProcessError:
64+
logger.error("Could not run git command")
65+
exit(ERROR_GIT_COMMAND)
66+
67+
def get_current_local_version(self):
68+
ret_string = check_output_wrapper(f'{self.CODEQL_HOME}/codeql-cli/codeql/codeql version', shell=True).decode("utf-8")
69+
if ret_string is CalledProcessError:
70+
logger.error("Could not run codeql command")
71+
exit(ERROR_EXECUTING_CODEQL)
72+
version_match = search("Version: ([0-9.]+)\.", ret_string)
73+
if not version_match:
74+
logger.error("Could not determine existing codeql version")
75+
exit(ERROR_EXECUTING_CODEQL)
76+
version = f'v{version_match.group(1)}'
77+
return version
78+
79+
def get_latest_codeql_github_version(self):
80+
return get_latest_github_repo_version("github/codeql-cli-binaries")
81+
82+
def install_codeql_cli(self, download_path):
83+
logger.info("Installing codeql-cli...")
84+
codeql_dir = f'{self.CODEQL_HOME}/codeql-cli'
85+
wipe_and_create_dir(codeql_dir)
86+
ret1 = check_output_wrapper(f'unzip {download_path} -d {codeql_dir}', shell=True)

‎container/libs/github.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
from datetime import datetime, MINYEAR
2+
from github import Github, GitRelease, Repository, GithubException
3+
4+
def get_latest_github_repo_version(repo):
5+
client = Github()
6+
repo = client.get_repo(repo)
7+
releases = repo.get_releases()
8+
latest_release = get_latest_github_release(releases)
9+
return latest_release
10+
11+
def get_latest_github_release(releases):
12+
latest_release = None
13+
latest_date = datetime(MINYEAR, 1, 1)
14+
for release in releases:
15+
if release.created_at > latest_date:
16+
latest_date = release.created_at
17+
latest_release = release
18+
return latest_release

‎container/libs/utils.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
from sys import exit
2+
from shutil import rmtree
3+
from os import environ, mkdir, name as os_name
4+
from subprocess import check_output, CalledProcessError
5+
from logging import getLogger
6+
7+
logger = getLogger('codeql-container')
8+
9+
# get secrets from the env
10+
def get_env_variable(self, var_name, optional=False):
11+
"""
12+
Retrieve an environment variable. Any failures will cause an exception
13+
to be thrown.
14+
"""
15+
try:
16+
return environ[var_name]
17+
except KeyError:
18+
if optional:
19+
return False
20+
else:
21+
error_msg = f'Error: You must set the {var_name} environment variable.'
22+
raise Exception(error_msg)
23+
24+
def check_output_wrapper(*args, **kwargs):
25+
"""
26+
Thin wrapper around subprocess
27+
"""
28+
29+
logger.debug('Executing %s, %s', args, kwargs)
30+
try:
31+
return check_output(*args, **kwargs)
32+
except CalledProcessError as msg:
33+
logger.warning('Error %s,%s,%s from command.', msg.returncode, msg.output, msg.stderr)
34+
logger.debug('Output: %s', msg.output)
35+
sys.exit(ERROR_EXECUTING_COMMAND);
36+
37+
def wipe_and_create_dir(dirname):
38+
rmtree(dirname)
39+
mkdir(dirname)

‎container/requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
PyGithub==1.43.7

‎container/setup.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/usr/bin/python3
2+
3+
import os
4+
import sys
5+
from logging import Logger, getLogger, INFO
6+
from sys import path as syspath
7+
from libs.utils import *
8+
from libs.github import *
9+
from libs.codeql import *
10+
11+
# get the parent directory of the script, to link libs
12+
sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
13+
14+
CODEQL_HOME = environ['CODEQL_HOME']
15+
16+
logger = getLogger('codeql-container')
17+
logger.setLevel(INFO)
18+
19+
def setup():
20+
"""
21+
Download and install the latest codeql cli
22+
Download and install the latest codeql queries
23+
"""
24+
25+
# check version and download the latest version
26+
get_latest_codeql()
27+
# install vscode?
28+
# clone codeql libs
29+
# setup vscode + codeql
30+
# wait for user
31+
32+
33+
def get_latest_codeql():
34+
# what version do we have?
35+
codeql = CodeQL(CODEQL_HOME)
36+
current_installed_version = codeql.get_current_local_version()
37+
logger.info(f'Current codeql version: {current_installed_version}')
38+
latest_online_version = codeql.get_latest_codeql_github_version()
39+
if current_installed_version != latest_online_version.title:
40+
# we got a newer version online, download and install it
41+
codeql.download_and_install_latest_codeql(latest_online_version)
42+
# get the latest queries regardless (TODO: Optimize by storing and checking the last commit hash?)
43+
codeql.download_and_install_latest_codeql_queries()
44+
45+
setup()
46+

‎container/startup.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import os
2+
3+
# should we update the local copy of codeql-cli if a new version is available?
4+
CHECK_LATEST_CODEQL_CLI = False
5+
6+
# should we update the local copy of codeql queries if a new version is available?
7+
CHECK_LATEST_QUERIES = False
8+
9+
10+
# if we are downloading new queries, should we precompile them
11+
#(makes query execution faster, but building the container build slower).
12+
PRECOMPILE_QUERIES = False
13+
14+
def main():
15+
# get all the command-line args/envs required
16+
# check if the latest codeql cli need to be downloaded
17+
# check if the latest codeql queries need to be downloaded
18+
# check if we need to precompile the new queries

0 commit comments

Comments
 (0)