diff --git a/.vscode/settings.json b/.vscode/settings.json index 495cf39..dc02ca1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -36,7 +36,7 @@ "editor.formatOnSave": true, "[python]": { "editor.codeActionsOnSave": { - "source.organizeImports": true + "source.organizeImports": "explicit" }, "editor.defaultFormatter": "ms-python.python" }, diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ecd0a15 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,94 @@ +# Base image +FROM pytorch/pytorch:1.9.0-cuda10.2-cudnn7-runtime + +# Ensure noninteractive apt installs and use bash for RUN so conda works later +ARG DEBIAN_FRONTEND=noninteractive +SHELL ["/bin/bash", "-lc"] + +# Copy files (Singularity %files) +COPY cli.sh /cli.sh + +COPY old_requirements.txt /requirements.txt + +# Environment (Singularity %environment) +ENV SINGULARITY=true \ +PATH="$PATH:/GloVe/build" + +# Make CLI executable (part of %post) +RUN chmod u+x /cli.sh + +# Update & base packages +RUN apt-get update && \ +apt-get install -y --no-install-recommends \ +wget curl git build-essential cmake \ +graphviz zip unzip vim libexpat1-dev \ +gnupg bash sudo && \ +rm -rf /var/lib/apt/lists/* + +RUN pip install --no-cache-dir dgl-cu102 -f https://data.dgl.ai/wheels/repo.html +RUN pip install --no-cache-dir -r /requirements.txt + + +# Install GloVe (from source) +RUN cd / && \ + git clone https://github.com/stanfordnlp/GloVe.git && \ + cd GloVe && make + +# Build & install cppcheck 2.5 from source +RUN cd / && \ + curl -L https://github.com/danmar/cppcheck/archive/refs/tags/2.5.tar.gz -o cppcheck2.5.tar.gz && \ + mkdir -p /cppcheck && mv cppcheck2.5.tar.gz /cppcheck && \ + cd /cppcheck && tar -xzf cppcheck2.5.tar.gz && \ + cd cppcheck-2.5 && mkdir build && cd build && \ + cmake .. && cmake --build . && make install && \ + rm -rf /cppcheck + +# Install Joern (non-interactive script drive similar to %post) +# Note: Running as root in Docker, so no sudo needed. +RUN apt-get update && apt-get install -y --no-install-recommends openjdk-8-jdk && \ + rm -rf /var/lib/apt/lists/* && \ + cd / && \ + wget https://github.com/ShiftLeftSecurity/joern/releases/latest/download/joern-install.sh && \ + chmod +x ./joern-install.sh && \ + printf 'Y\n/bin/joern\ny\n/usr/local/bin\n\n' | ./joern-install.sh --interactive && \ + rm -f /joern-install.sh + +# Install Miniconda (silent) and put it on PATH +ENV CONDA_DIR=/root/miniconda3 +ENV PATH=$CONDA_DIR/bin:$PATH +RUN cd / && \ + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + bash Miniconda3-latest-Linux-x86_64.sh -b -p "$CONDA_DIR" && \ + rm -f Miniconda3-latest-Linux-x86_64.sh && \ + conda clean -y --all + +# Install RATS (from archived tarball) +RUN cd / && \ + curl -L https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/rough-auditing-tool-for-security/rats-2.4.tgz -o rats-2.4.tgz && \ + tar -xzf rats-2.4.tgz && \ + cd rats-2.4 && ./configure && make && make install && \ + cd / && rm -rf rats-2.4 rats-2.4.tgz + +# Python tools & dependencies +# - flawfinder via pip +# - requirements (kept same name mapping as Singularity) +# - DGL CUDA 10.2 wheel +# - pygraphviz via conda (conda-forge for reliability) +# - NLTK + punkt +RUN pip install --no-cache-dir flawfinder +# RUN conda install -y -c conda-forge pygraphviz +RUN pip install --no-cache-dir nltk + +RUN python -c 'import nltk; nltk.download("punkt")' && \ + conda clean -y --all && \ + rm -rf /root/.cache/pip + +# Default working directory +RUN mkdir -p linevd +WORKDIR /linevd + +# Run script (Singularity %runscript) +# ENTRYPOINT ["/bin/bash", "/cli.sh"] +# ENTRYPOINT ["bash"] +# If you prefer to allow overriding while still defaulting, you could use: +# CMD [] diff --git a/Singularity b/Singularity index a5d8401..21065a5 100644 --- a/Singularity +++ b/Singularity @@ -3,7 +3,7 @@ From:pytorch/pytorch:1.9.0-cuda10.2-cudnn7-runtime %files cli.sh /cli.sh - requirements.txt /requirements.txt + old_requirements.txt /requirements.txt %environment export SINGULARITY=true diff --git a/get_old_reqirements.py b/get_old_reqirements.py new file mode 100644 index 0000000..401a5f2 --- /dev/null +++ b/get_old_reqirements.py @@ -0,0 +1,52 @@ +import requests +from datetime import datetime, timezone + +def get_version_at_time(package_name, target_date): + """ + Get the latest version of a package available on PyPI at a specific UTC date. + :param package_name: str, name of the package + :param target_date: datetime, UTC datetime to check + :return: str, version or None if not found + """ + url = f"https://pypi.org/pypi/{package_name}/json" + resp = requests.get(url) + if resp.status_code != 200: + print(f"Package {package_name} not found on PyPI.") + return None + + data = resp.json() + releases = data.get("releases", {}) + latest_version = None + latest_time = None + + for version, files in releases.items(): + for file in files: + upload_time = file.get("upload_time_iso_8601") + if upload_time: + upload_dt = datetime.fromisoformat(upload_time.replace("Z", "+00:00")) + if upload_dt <= target_date: + if (latest_time is None) or (upload_dt > latest_time): + latest_time = upload_dt + latest_version = version + + return latest_version + +if __name__ == "__main__": + infile = "requirements.txt" + new_requirements = [] + date_str = "2022-03-22" + target_dt = datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc) + + with open(infile, "r") as f: + packages = [line.strip() for line in f if line.strip() and not line.startswith("#")] + for package in packages: + if ">=" in package or "==" in package: + new_requirements.append(package) + continue + version = get_version_at_time(package, target_dt) + new_requirements.append(f"{package}=={version}" if version else package) + + with open("old_requirements.txt", "w") as f: + for req in new_requirements: + f.write(req + "\n") + print("Old requirements saved to old_requirements.txt") \ No newline at end of file diff --git a/old_requirements.txt b/old_requirements.txt new file mode 100644 index 0000000..47a6851 --- /dev/null +++ b/old_requirements.txt @@ -0,0 +1,41 @@ +tqdm>=4.61.2 +numpy>=1.20.1 +jupyterlab==4.0.0a22 +gdown==4.4.0 +matplotlib==3.5.1 +gensim==4.1.2 +joblib==1.1.0 +lightgbm==3.3.2 +xgboost==1.5.2 +nltk==3.7 +seaborn==0.11.2 +pytest==7.1.1 +scikit-learn==1.0.2 +scipy==1.7.3 +pandas==1.3.5 +fastparquet==0.8.0 +spacy==3.2.3 +imbalanced-learn==0.9.0 +dgl==0.9.0 +networkx==2.6.3 +pydot==1.4.2 +graphviz==0.19.1 +python-Levenshtein==0.12.2 +tensorboard==2.8.0 +python-igraph==0.9.9 +unidiff==0.7.3 +fuzzywuzzy==0.18.0 +python-Levenshtein==0.12.2 +libclang==13.0.0 +pandarallel==1.6.1 +ipywidgets==7.7.0 +transformers==4.17.0 +torchtext==0.12.0 +torchsummary==1.5.1 +torchinfo==1.6.3 +tsne_torch==1.0.1 +pytorch-lightning==1.5.10 +torch_scatter==2.0.9 +ujson==5.1.0 +unidecode==1.3.4 +ray[tune]==2.0.0 diff --git a/setup_env.sh b/setup_env.sh new file mode 100755 index 0000000..95d3c6e --- /dev/null +++ b/setup_env.sh @@ -0,0 +1,3 @@ +source /opt/conda/bin/activate +pip uninstall -y dgl +pip install dgl==0.9.0 diff --git a/start_container.sh b/start_container.sh new file mode 100755 index 0000000..b8b4f4d --- /dev/null +++ b/start_container.sh @@ -0,0 +1 @@ +docker run --shm-size=4gb --gpus=all --mount type=bind,src=.,dst=/linevd -it --entrypoint bash linevd:latest