From cb0625ef3226088f127743d6bc52cf4ffba327e5 Mon Sep 17 00:00:00 2001 From: Artem Goldenberg Date: Mon, 25 Dec 2023 05:23:07 +0300 Subject: [PATCH 1/3] Dockerfiles for python and c++ --- .github/workflows/CI.yml | 73 ++++++++++++++++++++++------------------ Makefile | 23 ++++++++++++- config.json | 2 +- py_server/.dockerignore | 34 +++++++++++++++++++ py_server/Dockerfile | 18 ++++++++++ py_server/Makefile | 12 +++++-- server/.dockerignore | 6 ++++ server/Dockerfile | 18 ++++++++++ server/Makefile | 12 ++++++- 9 files changed, 160 insertions(+), 38 deletions(-) create mode 100644 py_server/.dockerignore create mode 100644 py_server/Dockerfile create mode 100644 server/.dockerignore create mode 100644 server/Dockerfile diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9554a42c..5cf31263 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -20,55 +20,64 @@ jobs: - uses: actions/checkout@v3 - name: Format run: make test.client + + # Docker: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + + # - name: Docker Test + # run: make test.docker V=2 Python: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: install-python - uses: actions/setup-python@v4 - id: python - with: - python-version: "3.10" + # - name: install-python + # uses: actions/setup-python@v4 + # id: python + # with: + # python-version: "3.10" - - name: Build - run: make build.py_server - env: - python: ${{steps.python.outputs.python-path}} + # - name: Build + # run: make build.py_server + # env: + # python: ${{steps.python.outputs.python-path}} - name: Test - run: make test.py_server + run: make test.docker.py_server CPP: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 - - name: compiler-update - run: | - sudo apt update - sudo apt install -y g++-11 - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 90 - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 90 - sudo apt install libzip-dev + # - name: compiler-update + # run: | + # sudo apt update + # sudo apt install -y g++-11 + # sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 90 + # sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 90 + # sudo apt install libzip-dev - - name: install-cpprest - run: sudo apt install libcpprest-dev + # - name: install-cpprest + # run: sudo apt install libcpprest-dev - - name: install-boost - uses: MarkusJx/install-boost@v2.4.4 - id: install-boost - with: - boost_version: 1.81.0 - platform_version: 20.04 + # - name: install-boost + # uses: MarkusJx/install-boost@v2.4.4 + # id: install-boost + # with: + # boost_version: 1.81.0 + # platform_version: 20.04 - - name: Build - run: make build.server - env: - boost: ${{steps.install-boost.outputs.BOOST_ROOT}} + # - name: Build + # run: make build.server + # env: + # boost: ${{steps.install-boost.outputs.BOOST_ROOT}} - name: Test - run: make test.server - env: - boost: ${{steps.install-boost.outputs.BOOST_ROOT}} + run: make test.docker.server + # run: make test.server + # env: + # boost: ${{steps.install-boost.outputs.BOOST_ROOT}} diff --git a/Makefile b/Makefile index 7553d7cf..46c9381b 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,18 @@ -.PHONY: help build.% test.% clean +.PHONY: help docker build.% test.% clean .DEFAULT_GOAL = help targets = py_server server export CONFIG_PATH = $(abspath config.json) +ifeq ($(shell which node),) +$(error 'node' not found, see README about node) +endif + +clientPort = $(shell node -p "require('$(CONFIG_PATH)').client.PORT") +pythonPort = $(shell node -p "require('$(CONFIG_PATH)').py_server.PORT") +# pythonPort = $(call getField,PORT) + MAKEFLAGS += -k ifneq ($V,2) # comment next line and Makefile will print what it's doing @@ -20,6 +28,8 @@ Example (runs tests in py_server): make test.py_server V=2 Targets: -------- +docker: launches everything in 3 docker containers +test.docker: start docker containers, run tests in them and exit run.client: starts the client run.py_server: starts the py_server run.server: starts the c++ server @@ -46,6 +56,17 @@ help: export HELP := $(HELP) help: @echo "$$HELP" +docker: export WEBPORT=$(clientPort) +docker: export PYPORT=$(pythonPort) +docker: compose.yml + docker compose up + +test.docker.py_server: + $(MAKE) docker -C py_server -e + +test.docker.server: + $(MAKE) docker -C server -e + $(addprefix run.,py_server client): run.%: $(MAKE) -C $* -e diff --git a/config.json b/config.json index e7e25041..53d0aff7 100644 --- a/config.json +++ b/config.json @@ -5,7 +5,7 @@ }, "py_server": { - "HOST": "localhost", + "HOST": "0.0.0.0", "PORT": 3000, "SECRET_KEY": "minecraft", "SQLALCHEMY_DATABASE_URI": "sqlite:///database.db", diff --git a/py_server/.dockerignore b/py_server/.dockerignore new file mode 100644 index 00000000..5ae4464d --- /dev/null +++ b/py_server/.dockerignore @@ -0,0 +1,34 @@ +# Include any files or directories that you don't want to be copied to your +# container here (e.g., local build artifacts, temporary files, etc.). + +**/.DS_Store +**/__pycache__ +**/.mypy_cache +**/.pytest_cache +**/.venv +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/bin +**/charts +**/docker-compose* +**/compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +**/install +LICENSE +README.md diff --git a/py_server/Dockerfile b/py_server/Dockerfile new file mode 100644 index 00000000..529e7796 --- /dev/null +++ b/py_server/Dockerfile @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1 + +FROM node:21-alpine3.18 + +RUN apk add --no-cache make +RUN apk add --update --no-cache python3 + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +EXPOSE ${PYPORT} + +WORKDIR /app +COPY . . + +ENV CONFIG_PATH=/app/config.json + +CMD make build && make test diff --git a/py_server/Makefile b/py_server/Makefile index c30780e5..71d741b2 100644 --- a/py_server/Makefile +++ b/py_server/Makefile @@ -2,7 +2,7 @@ app = mlcraft dirEnv = .venv dirBin = $(dirEnv)/bin -.PHONY: help default build format test pytest typecheck formatcheck clean +.PHONY: help default docker build format test pytest typecheck formatcheck clean .DEFAULT_GOAL = default ifneq ($V,2) @@ -22,7 +22,9 @@ $(error 'node' not found, see README about node) endif getField = $(shell node -p "require('$(CONFIG_PATH)').py_server.$1") -flags = -h $(call getField,HOST) -p $(call getField,PORT) +port = $(call getField,PORT) +host = $(call getField,HOST) +flags = -h $(host) -p $(port) python = $(shell which 'python3.10' || which 'python3' || which 'python') ifeq ("$(python)","") @@ -57,6 +59,11 @@ default: install @echo "---------------------------------------------------------------------------" $(dirBin)/flask --app $(app) run --debug $(flags) +docker: export PYPORT := $(port) +docker: Dockerfile + docker build -t $(app)/python:test . + docker run -v $(CONFIG_PATH):/app/config.json $(app)/python:test + build: install format: install @@ -83,4 +90,3 @@ $(dirEnv): clean: $(RM) install - $(RM) -r instance \ No newline at end of file diff --git a/server/.dockerignore b/server/.dockerignore new file mode 100644 index 00000000..828d801e --- /dev/null +++ b/server/.dockerignore @@ -0,0 +1,6 @@ +bin +Build +model_data +.cache +*.xcodeproj +*.entitlements diff --git a/server/Dockerfile b/server/Dockerfile new file mode 100644 index 00000000..8d413c9b --- /dev/null +++ b/server/Dockerfile @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1 + +FROM node:21-slim + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y build-essential +RUN apt install -y libboost-dev +RUN apt-get install -y libcpprest-dev +RUN apt-get install -y libx11-dev libpng-dev libzip-dev + +EXPOSE ${CPORT} + +WORKDIR /app +COPY . . + +ENV CONFIG_PATH=/app/config.json + +CMD make build && make test diff --git a/server/Makefile b/server/Makefile index b3f865fa..1e11f2ec 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,3 +1,4 @@ +app = mlcraft dirApi = api dirCore = core dirTests = tests @@ -39,7 +40,7 @@ ifneq ($V,2) .SILENT: endif .DEFAULT_GOAL = default -.PHONY: default serve test test.% clean checkboost +.PHONY: default docker serve test test.% clean checkboost include $(wildcard $(dirBuild)/*.d) @@ -60,6 +61,7 @@ help: @echo @echo "Available targets:" @echo "default: builds and runs core/main.cpp file" + @echo "docker: tests application inside the docker container" @echo "serve: builds and runs api/server.cpp file" @echo "test: builds and runs all tests in the tests directory" @echo "test.: runs specific test, for example 'make test.Blob'" @@ -80,6 +82,14 @@ help: build: $(core) $(server) $(testTargets) +### DOCKER ### + +docker: export CPORT := $(port) +docker: Dockerfile + docker build -t $(app)/cpp:test . + docker run -v $(CONFIG_PATH):/app/config.json $(app)/cpp:test + + ### CORE ### coreFiles = $(wildcard $(dirCore)/*.cpp) From 75cb666e0e7b4ff3ee662326a2fc3d5b0610032e Mon Sep 17 00:00:00 2001 From: Artem Goldenberg Date: Mon, 25 Dec 2023 17:29:01 +0300 Subject: [PATCH 2/3] make docker --- Makefile | 5 +++-- client/.dockerignore | 3 +++ compose.yml | 52 +++++++++++++++++++++++++++++++++++++++++++ config.json | 4 ++-- py_server/Dockerfile | 4 ++-- server/.dockerignore | 1 + server/core/Shape.cpp | 1 - 7 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 client/.dockerignore create mode 100644 compose.yml diff --git a/Makefile b/Makefile index 46c9381b..4854fdc3 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ endif clientPort = $(shell node -p "require('$(CONFIG_PATH)').client.PORT") pythonPort = $(shell node -p "require('$(CONFIG_PATH)').py_server.PORT") -# pythonPort = $(call getField,PORT) +cppPort = $(shell node -p "require('$(CONFIG_PATH)').cpp_server.PORT") MAKEFLAGS += -k ifneq ($V,2) @@ -58,8 +58,9 @@ help: docker: export WEBPORT=$(clientPort) docker: export PYPORT=$(pythonPort) +docker: export CPORT=$(cppPort) docker: compose.yml - docker compose up + docker compose up --build --remove-orphans test.docker.py_server: $(MAKE) docker -C py_server -e diff --git a/client/.dockerignore b/client/.dockerignore new file mode 100644 index 00000000..f5d5f9a9 --- /dev/null +++ b/client/.dockerignore @@ -0,0 +1,3 @@ +node_modules +package-lock.json +**/.DS_Store diff --git a/compose.yml b/compose.yml new file mode 100644 index 00000000..f671b6d5 --- /dev/null +++ b/compose.yml @@ -0,0 +1,52 @@ +name: mlcraft + +services: + web: + build: + context: client + dockerfile_inline: | + FROM node:21-alpine3.18 + RUN apk add --update --no-cache make + WORKDIR /app + COPY . . + ports: + - ${WEBPORT}:${WEBPORT} + command: make + volumes: + - ${CONFIG_PATH}:/app/config.json + environment: + - CONFIG_PATH=/app/config.json + depends_on: + - pyServer + # FROM python:${PYTHON_VERSION}-slim + + pyServer: + image: mlcraft/python + build: py_server + ports: + - ${PYPORT}:${PYPORT} + command: make + volumes: + - ${CONFIG_PATH}:/app/config.json + - saveme:/app/.venv + - ./py_server/instance/database.db:/app/instance/database.db + environment: + - CONFIG_PATH=/app/config.json + depends_on: + - cppServer + + cppServer: + image: mlcraft/cpp + build: server + ports: + - ${CPORT}:${CPORT} + command: make serve V=2 + volumes: + - ${CONFIG_PATH}:/app/config.json + environment: + - CONFIG_PATH=/app/config.json + + +volumes: + database: + saveme: \ No newline at end of file diff --git a/config.json b/config.json index 53d0aff7..169e17d3 100644 --- a/config.json +++ b/config.json @@ -1,6 +1,6 @@ { "client": { - "HOST": "localhost", + "HOST": "0.0.0.0", "PORT": 2001 }, @@ -13,7 +13,7 @@ }, "cpp_server": { - "HOST": "localhost", + "HOST": "0.0.0.0", "PORT": 2000, "BOOST_ROOT": "" } diff --git a/py_server/Dockerfile b/py_server/Dockerfile index 529e7796..44ef16e1 100644 --- a/py_server/Dockerfile +++ b/py_server/Dockerfile @@ -2,8 +2,8 @@ FROM node:21-alpine3.18 -RUN apk add --no-cache make -RUN apk add --update --no-cache python3 +RUN apk add --update --no-cache make +RUN apk add --no-cache python3 ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 diff --git a/server/.dockerignore b/server/.dockerignore index 828d801e..a60df7c3 100644 --- a/server/.dockerignore +++ b/server/.dockerignore @@ -1,3 +1,4 @@ +**/.DS_Store bin Build model_data diff --git a/server/core/Shape.cpp b/server/core/Shape.cpp index 7f4cc167..6d4c6ab5 100644 --- a/server/core/Shape.cpp +++ b/server/core/Shape.cpp @@ -64,7 +64,6 @@ size_t Shape::operator [] (int i) const { size_t Shape::stride(int i) const { assert(i >= 0 && i < 4); - size_t result = 1; switch (i) { case 0: return dims[1] * dims[2] * dims[3]; case 1: return dims[2] * dims[3]; From 3a337ce30c20866647f96ca972368118546238fa Mon Sep 17 00:00:00 2001 From: Artem Goldenberg Date: Mon, 25 Dec 2023 20:00:26 +0300 Subject: [PATCH 3/3] some dockery docker --- .github/workflows/CI.yml | 45 ---------------------------------------- client/index.html | 2 +- compose.yml | 1 + py_server/Dockerfile | 2 +- py_server/pyproject.toml | 1 + 5 files changed, 4 insertions(+), 47 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5cf31263..63db673c 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -20,31 +20,12 @@ jobs: - uses: actions/checkout@v3 - name: Format run: make test.client - - # Docker: - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - - # - name: Docker Test - # run: make test.docker V=2 Python: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - # - name: install-python - # uses: actions/setup-python@v4 - # id: python - # with: - # python-version: "3.10" - - # - name: Build - # run: make build.py_server - # env: - # python: ${{steps.python.outputs.python-path}} - - name: Test run: make test.docker.py_server @@ -53,31 +34,5 @@ jobs: steps: - uses: actions/checkout@v3 - # - name: compiler-update - # run: | - # sudo apt update - # sudo apt install -y g++-11 - # sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 90 - # sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 90 - # sudo apt install libzip-dev - - # - name: install-cpprest - # run: sudo apt install libcpprest-dev - - # - name: install-boost - # uses: MarkusJx/install-boost@v2.4.4 - # id: install-boost - # with: - # boost_version: 1.81.0 - # platform_version: 20.04 - - # - name: Build - # run: make build.server - # env: - # boost: ${{steps.install-boost.outputs.BOOST_ROOT}} - - name: Test run: make test.docker.server - # run: make test.server - # env: - # boost: ${{steps.install-boost.outputs.BOOST_ROOT}} diff --git a/client/index.html b/client/index.html index a4624df7..f486ec95 100644 --- a/client/index.html +++ b/client/index.html @@ -2,6 +2,6 @@ diff --git a/compose.yml b/compose.yml index 84dbf293..89871e65 100644 --- a/compose.yml +++ b/compose.yml @@ -14,6 +14,7 @@ services: command: make volumes: - ${CONFIG_PATH}:/app/config.json + - ./client:/app environment: - CONFIG_PATH=/app/config.json depends_on: diff --git a/py_server/Dockerfile b/py_server/Dockerfile index 44ef16e1..13750ae0 100644 --- a/py_server/Dockerfile +++ b/py_server/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -FROM node:21-alpine3.18 +FROM --platform=linux/amd64 node:21-alpine3.18 RUN apk add --update --no-cache make RUN apk add --no-cache python3 diff --git a/py_server/pyproject.toml b/py_server/pyproject.toml index efbc3eb8..ceadc84d 100644 --- a/py_server/pyproject.toml +++ b/py_server/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "black == 23.10.1", "pytest == 7.4.2", "mypy == 1.6.1", + "cython == 3.0.7", "matplotlib == 3.8.2", ]