From e93bd7b55092f00b2510695115b674f757c28ba8 Mon Sep 17 00:00:00 2001 From: Eduard Poliakov Date: Mon, 3 Jun 2024 15:02:54 +0300 Subject: [PATCH 1/6] Add two PostgreSQL, MinioS3 and backup scripts. --- backup/backup_and_upload.log | 172 +++++++++++++++++++++++++++++++++++ backup/backup_and_upload.sh | 83 +++++++++++++++++ docker-compose.yaml | 46 ++++++++++ 3 files changed, 301 insertions(+) create mode 100644 backup/backup_and_upload.log create mode 100644 backup/backup_and_upload.sh create mode 100644 docker-compose.yaml diff --git a/backup/backup_and_upload.log b/backup/backup_and_upload.log new file mode 100644 index 0000000..6f103be --- /dev/null +++ b/backup/backup_and_upload.log @@ -0,0 +1,172 @@ + +[2024-05-24 15:32:22] Starting backup process. +[2024-05-24 15:32:22] Creating backup of Similar_Design_Finder_Dev database. +[2024-05-24 15:32:22] Successfully created backup of Similar_Design_Finder_Dev database. +[2024-05-24 15:32:22] Creating backup of Similar_Design_Finder_Prod database. +[2024-05-24 15:32:23] Successfully created backup of Similar_Design_Finder_Prod database. +[2024-05-24 15:32:23] Uploading Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-24 15:32:26] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-24 15:32:26] Uploading Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-24 15:32:29] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-24 15:32:29] Deleting local backups. +[2024-05-24 15:32:29] Backup process completed. +[2024-05-24 15:32:40] Starting backup process. +[2024-05-24 15:32:40] Creating backup of Similar_Design_Finder_Dev database. +[2024-05-24 15:32:40] Successfully created backup of Similar_Design_Finder_Dev database. +[2024-05-24 15:32:40] Creating backup of Similar_Design_Finder_Prod database. +[2024-05-24 15:32:40] Successfully created backup of Similar_Design_Finder_Prod database. +[2024-05-24 15:32:40] Uploading Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-24 15:32:43] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-24 15:32:43] Uploading Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-24 15:32:45] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-24 15:32:45] Deleting local backups. +[2024-05-24 15:32:45] Backup process completed. +[2024-05-24 15:35:01] Starting backup process. +[2024-05-24 15:35:01] Creating backup of Similar_Design_Finder_Dev database. +[2024-05-24 15:35:02] Successfully created backup of Similar_Design_Finder_Dev database. +[2024-05-24 15:35:02] Creating backup of Similar_Design_Finder_Prod database. +[2024-05-24 15:35:02] Successfully created backup of Similar_Design_Finder_Prod database. +[2024-05-24 15:35:02] Uploading Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-24 15:35:04] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-24 15:35:04] Uploading Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-24 15:35:06] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-24 15:35:06] Deleting local backups. +[2024-05-24 15:35:06] Backup process completed. +[2024-05-25 03:00:01] Starting backup process. +[2024-05-25 03:00:01] Creating backup of Similar_Design_Finder_Dev database. +[2024-05-25 03:00:01] Successfully created backup of Similar_Design_Finder_Dev database. +[2024-05-25 03:00:01] Creating backup of Similar_Design_Finder_Prod database. +[2024-05-25 03:00:02] Successfully created backup of Similar_Design_Finder_Prod database. +[2024-05-25 03:00:02] Uploading Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-25 03:00:05] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-25 03:00:05] Uploading Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-25 03:00:07] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-25 03:00:07] Deleting local backups. +[2024-05-25 03:00:07] Backup process completed. +[2024-05-26 03:03:05] Starting backup process. +[2024-05-26 03:03:05] Creating backup of Similar_Design_Finder_Dev database. +[2024-05-26 03:04:01] Successfully created backup of Similar_Design_Finder_Dev database. +[2024-05-26 03:04:01] Creating backup of Similar_Design_Finder_Prod database. +[2024-05-26 03:04:02] Successfully created backup of Similar_Design_Finder_Prod database. +[2024-05-26 03:04:02] Uploading Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-26 03:04:04] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-26 03:04:04] Uploading Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-26 03:04:07] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-26 03:04:07] Deleting local backups. +[2024-05-26 03:04:07] Backup process completed. +[2024-05-27 03:00:01] Starting backup process. +[2024-05-27 03:00:01] Creating backup of Similar_Design_Finder_Dev database. +[2024-05-27 03:00:01] Successfully created backup of Similar_Design_Finder_Dev database. +[2024-05-27 03:00:01] Creating backup of Similar_Design_Finder_Prod database. +[2024-05-27 03:00:02] Successfully created backup of Similar_Design_Finder_Prod database. +[2024-05-27 03:00:02] Uploading Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-27 03:00:04] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-27 03:00:04] Uploading Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-27 03:00:07] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-27 03:00:07] Deleting local backups. +[2024-05-27 03:00:07] Backup process completed. +[2024-05-27 20:08:47] Starting backup process. +[2024-05-27 20:08:47] Creating backup of Similar_Design_Finder_Dev database. +[2024-05-27 20:08:47] Successfully created backup of Similar_Design_Finder_Dev database. +[2024-05-27 20:08:47] Creating backup of Similar_Design_Finder_Prod database. +[2024-05-27 20:08:48] Successfully created backup of Similar_Design_Finder_Prod database. +[2024-05-27 20:08:48] Creating backup of MinIO data. +[2024-05-27 20:08:51] Successfully created backup of MinIO data. +[2024-05-27 20:08:51] Uploading Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-27 20:08:54] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-27 20:08:54] Uploading Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-27 20:08:56] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-27 20:08:56] Uploading MinIO data backup to Google Drive. +[2024-05-27 20:08:59] Successfully uploaded MinIO data backup to Google Drive. +[2024-05-27 20:08:59] Deleting local backups. +[2024-05-27 20:08:59] Backup process completed. +[2024-05-27 20:12:40] Starting backup process. +[2024-05-27 20:12:40] Creating backup of Similar_Design_Finder_Dev database. +[2024-05-27 20:12:41] Successfully created backup of Similar_Design_Finder_Dev database. +[2024-05-27 20:12:41] Creating backup of Similar_Design_Finder_Prod database. +[2024-05-27 20:12:42] Successfully created backup of Similar_Design_Finder_Prod database. +[2024-05-27 20:12:42] Creating backup of MinIO data. +[2024-05-27 20:12:43] Successfully created backup of MinIO data. +[2024-05-27 20:12:43] Uploading Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-27 20:12:46] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-27 20:12:46] Uploading Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-27 20:12:48] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-27 20:12:48] Uploading MinIO data backup to Google Drive. +[2024-05-27 20:12:51] Successfully uploaded MinIO data backup to Google Drive. +[2024-05-27 20:12:51] Deleting local backups. +[2024-05-27 20:12:51] Backup process completed. +[2024-05-28 03:00:01] Starting backup process. +[2024-05-28 03:00:01] Creating backup of Similar_Design_Finder_Dev database. +[2024-05-28 03:00:02] Successfully created backup of Similar_Design_Finder_Dev database. +[2024-05-28 03:00:02] Creating backup of Similar_Design_Finder_Prod database. +[2024-05-28 03:00:02] Successfully created backup of Similar_Design_Finder_Prod database. +[2024-05-28 03:00:02] Creating backup of MinIO data. +[2024-05-28 03:00:03] Successfully created backup of MinIO data. +[2024-05-28 03:00:03] Uploading Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-28 03:00:06] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-28 03:00:06] Uploading Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-28 03:00:09] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-28 03:00:09] Uploading MinIO data backup to Google Drive. +[2024-05-28 03:00:11] Successfully uploaded MinIO data backup to Google Drive. +[2024-05-28 03:00:11] Deleting local backups. +[2024-05-28 03:00:11] Backup process completed. +[2024-05-29 03:00:01] Starting backup process. +[2024-05-29 03:00:01] Creating backup of Similar_Design_Finder_Dev database. +[2024-05-29 03:00:02] Successfully created backup of Similar_Design_Finder_Dev database. +[2024-05-29 03:00:02] Creating backup of Similar_Design_Finder_Prod database. +[2024-05-29 03:00:02] Successfully created backup of Similar_Design_Finder_Prod database. +[2024-05-29 03:00:02] Creating backup of MinIO data. +[2024-05-29 03:00:03] Successfully created backup of MinIO data. +[2024-05-29 03:00:03] Uploading Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-29 03:00:06] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-29 03:00:06] Uploading Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-29 03:00:09] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-29 03:00:09] Uploading MinIO data backup to Google Drive. +[2024-05-29 03:00:11] Successfully uploaded MinIO data backup to Google Drive. +[2024-05-29 03:00:11] Deleting local backups. +[2024-05-29 03:00:11] Backup process completed. +[2024-05-30 03:00:01] Starting backup process. +[2024-05-30 03:00:01] Creating backup of Similar_Design_Finder_Dev database. +[2024-05-30 03:00:02] Successfully created backup of Similar_Design_Finder_Dev database. +[2024-05-30 03:00:02] Creating backup of Similar_Design_Finder_Prod database. +[2024-05-30 03:00:03] Successfully created backup of Similar_Design_Finder_Prod database. +[2024-05-30 03:00:03] Creating backup of MinIO data. +[2024-05-30 03:00:04] Successfully created backup of MinIO data. +[2024-05-30 03:00:04] Uploading Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-30 03:00:07] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. +[2024-05-30 03:00:07] Uploading Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-30 03:00:09] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. +[2024-05-30 03:00:09] Uploading MinIO data backup to Google Drive. +[2024-05-30 03:00:11] Successfully uploaded MinIO data backup to Google Drive. +[2024-05-30 03:00:11] Deleting local backups. +[2024-05-30 03:00:11] Backup process completed. +[2024-06-02 03:00:01] Starting backup process. +[2024-06-02 03:00:01] Creating backup of Similar_Design_Finder_Dev database. +[2024-06-02 03:00:03] Successfully created backup of Similar_Design_Finder_Dev database. +[2024-06-02 03:00:03] Creating backup of Similar_Design_Finder_Prod database. +[2024-06-02 03:00:04] Successfully created backup of Similar_Design_Finder_Prod database. +[2024-06-02 03:00:04] Creating backup of MinIO data. +[2024-06-02 03:00:05] Successfully created backup of MinIO data. +[2024-06-02 03:00:05] Uploading Similar_Design_Finder_Dev database backup to Google Drive. +[2024-06-02 03:00:08] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. +[2024-06-02 03:00:08] Uploading Similar_Design_Finder_Prod database backup to Google Drive. +[2024-06-02 03:00:10] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. +[2024-06-02 03:00:10] Uploading MinIO data backup to Google Drive. +[2024-06-02 03:00:12] Successfully uploaded MinIO data backup to Google Drive. +[2024-06-02 03:00:12] Deleting local backups. +[2024-06-02 03:00:12] Backup process completed. +[2024-06-03 03:00:01] Starting backup process. +[2024-06-03 03:00:01] Creating backup of Similar_Design_Finder_Dev database. +[2024-06-03 03:00:02] Successfully created backup of Similar_Design_Finder_Dev database. +[2024-06-03 03:00:02] Creating backup of Similar_Design_Finder_Prod database. +[2024-06-03 03:00:03] Successfully created backup of Similar_Design_Finder_Prod database. +[2024-06-03 03:00:03] Creating backup of MinIO data. +[2024-06-03 03:00:04] Successfully created backup of MinIO data. +[2024-06-03 03:00:04] Uploading Similar_Design_Finder_Dev database backup to Google Drive. +[2024-06-03 03:00:07] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. +[2024-06-03 03:00:07] Uploading Similar_Design_Finder_Prod database backup to Google Drive. +[2024-06-03 03:00:09] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. +[2024-06-03 03:00:09] Uploading MinIO data backup to Google Drive. +[2024-06-03 03:00:11] Successfully uploaded MinIO data backup to Google Drive. +[2024-06-03 03:00:11] Deleting local backups. +[2024-06-03 03:00:11] Backup process completed. diff --git a/backup/backup_and_upload.sh b/backup/backup_and_upload.sh new file mode 100644 index 0000000..4086e95 --- /dev/null +++ b/backup/backup_and_upload.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Date and time for the filename +timestamp=$(date +'%Y%m%d_%H%M%S') + +# Path to the directory where the script is located +backup_dir="$(dirname "$0")" + +# Path to the log file +log_file="$backup_dir/backup_and_upload.log" + +# Function to write messages to the log file +log_message() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" >> "$log_file" +} + +# Create the log file if it doesn't exist +touch "$log_file" + +# Log the start of the backup process +log_message "Starting backup process." + +# Create a backup of the Similar_Design_Finder_Dev database +log_message "Creating backup of Similar_Design_Finder_Dev database." +docker exec -t $(docker-compose -f "$backup_dir/../docker-compose.yaml" ps -q postgres_dev) pg_dumpall -U admin > "$backup_dir/pg_cluster_backup_dev_$timestamp.sql" +if [ $? -eq 0 ]; then + log_message "Successfully created backup of Similar_Design_Finder_Dev database." +else + log_message "Error creating backup of Similar_Design_Finder_Dev database." +fi + +# Create a backup of the Similar_Design_Finder_Prod database +log_message "Creating backup of Similar_Design_Finder_Prod database." +docker exec -t $(docker-compose -f "$backup_dir/../docker-compose.yaml" ps -q postgres_prod) pg_dumpall -U admin > "$backup_dir/pg_cluster_backup_prod_$timestamp.sql" +if [ $? -eq 0 ]; then + log_message "Successfully created backup of Similar_Design_Finder_Prod database." +else + log_message "Error creating backup of Similar_Design_Finder_Prod database." +fi + +# Create a backup of the MinIO data +log_message "Creating backup of MinIO data." +docker run --rm --volumes-from $(docker-compose -f "$backup_dir/../docker-compose.yaml" ps -q minio) -v "$backup_dir":/backup busybox tar czvf /backup/minio_backup_$timestamp.tar.gz /data +if [ $? -eq 0 ]; then + log_message "Successfully created backup of MinIO data." +else + log_message "Error creating backup of MinIO data." +fi + +# Upload backups to Google Drive +log_message "Uploading Similar_Design_Finder_Dev database backup to Google Drive." +rclone copy "$backup_dir/pg_cluster_backup_dev_$timestamp.sql" gdrive: +if [ $? -eq 0 ]; then + log_message "Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive." +else + log_message "Error uploading Similar_Design_Finder_Dev database backup to Google Drive." +fi + +log_message "Uploading Similar_Design_Finder_Prod database backup to Google Drive." +rclone copy "$backup_dir/pg_cluster_backup_prod_$timestamp.sql" gdrive: +if [ $? -eq 0 ]; then + log_message "Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive." +else + log_message "Error uploading Similar_Design_Finder_Prod database backup to Google Drive." +fi + +log_message "Uploading MinIO data backup to Google Drive." +rclone copy "$backup_dir/minio_backup_$timestamp.tar.gz" gdrive: +if [ $? -eq 0 ]; then + log_message "Successfully uploaded MinIO data backup to Google Drive." +else + log_message "Error uploading MinIO data backup to Google Drive." +fi + +# Delete local backups after uploading +log_message "Deleting local backups." +rm "$backup_dir/pg_cluster_backup_dev_$timestamp.sql" +rm "$backup_dir/pg_cluster_backup_prod_$timestamp.sql" +rm "$backup_dir/minio_backup_$timestamp.tar.gz" + +# Log the completion of the backup process +log_message "Backup process completed." + diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..f6c6151 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,46 @@ +version: '3.8' +services: + postgres_dev: + image: pgvector/pgvector:pg16 + environment: + POSTGRES_USER: admin + POSTGRES_PASSWORD: admin + POSTGRES_DB: Similar_Design_Finder_Dev + volumes: + - pgdata_dev:/var/lib/postgresql/data + ports: + - "5433:5432" + restart: always + + postgres_prod: + image: pgvector/pgvector:pg16 + environment: + POSTGRES_USER: admin + POSTGRES_PASSWORD: admin + POSTGRES_DB: Similar_Design_Finder_Prod + volumes: + - pgdata_prod:/var/lib/postgresql/data + ports: + - "5432:5432" + restart: always + + minio: + image: docker.io/bitnami/minio:2022 + ports: + - '9000:9000' + - '9001:9001' + volumes: + - minio_data:/data + environment: + - MINIO_ROOT_USER=admin + - MINIO_ROOT_PASSWORD=admin2024 + - MINIO_DEFAULT_BUCKETS=default + restart: always + +volumes: + pgdata_dev: + external: true + pgdata_prod: + external: true + minio_data: + external: true From 13ecde13e744f4ba7df4e0ac99a0ef6e9b87a4d3 Mon Sep 17 00:00:00 2001 From: Eduard Poliakov Date: Mon, 3 Jun 2024 21:48:47 +0300 Subject: [PATCH 2/6] Deletion backup_and_upload.log, add .gitignore --- .gitignore | 116 +++++++++++++++++++++++ backup/backup_and_upload.log | 172 ----------------------------------- 2 files changed, 116 insertions(+), 172 deletions(-) create mode 100644 .gitignore delete mode 100644 backup/backup_and_upload.log diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2934357 --- /dev/null +++ b/.gitignore @@ -0,0 +1,116 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# DotEnv configuration +.env + +# Database +*.db +*.rdb + +# Pycharm +.idea + +# VS Code +.vscode/ + +# Spyder +.spyproject/ + +# Jupyter NB Checkpoints +.ipynb_checkpoints/ + +# exclude data from source control by default +# /data/ + +# Mac OS-specific storage files +.DS_Store + +# vim +*.swp +*.swo + +# Mypy cache +.mypy_cache/ + +# Pyre cache +.python-version + +# Venv environment +/venv/ + +# Model folder +/models/ +/data/ +# data/* +# !data/processed/ +# data/processed/* +# !data/processed/predictions.csv + + + + +# MLFlow +/mlruns/ + + + + +# MLFlow +/mlruns/ +/data diff --git a/backup/backup_and_upload.log b/backup/backup_and_upload.log deleted file mode 100644 index 6f103be..0000000 --- a/backup/backup_and_upload.log +++ /dev/null @@ -1,172 +0,0 @@ - -[2024-05-24 15:32:22] Starting backup process. -[2024-05-24 15:32:22] Creating backup of Similar_Design_Finder_Dev database. -[2024-05-24 15:32:22] Successfully created backup of Similar_Design_Finder_Dev database. -[2024-05-24 15:32:22] Creating backup of Similar_Design_Finder_Prod database. -[2024-05-24 15:32:23] Successfully created backup of Similar_Design_Finder_Prod database. -[2024-05-24 15:32:23] Uploading Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-24 15:32:26] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-24 15:32:26] Uploading Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-24 15:32:29] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-24 15:32:29] Deleting local backups. -[2024-05-24 15:32:29] Backup process completed. -[2024-05-24 15:32:40] Starting backup process. -[2024-05-24 15:32:40] Creating backup of Similar_Design_Finder_Dev database. -[2024-05-24 15:32:40] Successfully created backup of Similar_Design_Finder_Dev database. -[2024-05-24 15:32:40] Creating backup of Similar_Design_Finder_Prod database. -[2024-05-24 15:32:40] Successfully created backup of Similar_Design_Finder_Prod database. -[2024-05-24 15:32:40] Uploading Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-24 15:32:43] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-24 15:32:43] Uploading Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-24 15:32:45] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-24 15:32:45] Deleting local backups. -[2024-05-24 15:32:45] Backup process completed. -[2024-05-24 15:35:01] Starting backup process. -[2024-05-24 15:35:01] Creating backup of Similar_Design_Finder_Dev database. -[2024-05-24 15:35:02] Successfully created backup of Similar_Design_Finder_Dev database. -[2024-05-24 15:35:02] Creating backup of Similar_Design_Finder_Prod database. -[2024-05-24 15:35:02] Successfully created backup of Similar_Design_Finder_Prod database. -[2024-05-24 15:35:02] Uploading Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-24 15:35:04] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-24 15:35:04] Uploading Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-24 15:35:06] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-24 15:35:06] Deleting local backups. -[2024-05-24 15:35:06] Backup process completed. -[2024-05-25 03:00:01] Starting backup process. -[2024-05-25 03:00:01] Creating backup of Similar_Design_Finder_Dev database. -[2024-05-25 03:00:01] Successfully created backup of Similar_Design_Finder_Dev database. -[2024-05-25 03:00:01] Creating backup of Similar_Design_Finder_Prod database. -[2024-05-25 03:00:02] Successfully created backup of Similar_Design_Finder_Prod database. -[2024-05-25 03:00:02] Uploading Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-25 03:00:05] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-25 03:00:05] Uploading Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-25 03:00:07] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-25 03:00:07] Deleting local backups. -[2024-05-25 03:00:07] Backup process completed. -[2024-05-26 03:03:05] Starting backup process. -[2024-05-26 03:03:05] Creating backup of Similar_Design_Finder_Dev database. -[2024-05-26 03:04:01] Successfully created backup of Similar_Design_Finder_Dev database. -[2024-05-26 03:04:01] Creating backup of Similar_Design_Finder_Prod database. -[2024-05-26 03:04:02] Successfully created backup of Similar_Design_Finder_Prod database. -[2024-05-26 03:04:02] Uploading Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-26 03:04:04] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-26 03:04:04] Uploading Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-26 03:04:07] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-26 03:04:07] Deleting local backups. -[2024-05-26 03:04:07] Backup process completed. -[2024-05-27 03:00:01] Starting backup process. -[2024-05-27 03:00:01] Creating backup of Similar_Design_Finder_Dev database. -[2024-05-27 03:00:01] Successfully created backup of Similar_Design_Finder_Dev database. -[2024-05-27 03:00:01] Creating backup of Similar_Design_Finder_Prod database. -[2024-05-27 03:00:02] Successfully created backup of Similar_Design_Finder_Prod database. -[2024-05-27 03:00:02] Uploading Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-27 03:00:04] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-27 03:00:04] Uploading Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-27 03:00:07] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-27 03:00:07] Deleting local backups. -[2024-05-27 03:00:07] Backup process completed. -[2024-05-27 20:08:47] Starting backup process. -[2024-05-27 20:08:47] Creating backup of Similar_Design_Finder_Dev database. -[2024-05-27 20:08:47] Successfully created backup of Similar_Design_Finder_Dev database. -[2024-05-27 20:08:47] Creating backup of Similar_Design_Finder_Prod database. -[2024-05-27 20:08:48] Successfully created backup of Similar_Design_Finder_Prod database. -[2024-05-27 20:08:48] Creating backup of MinIO data. -[2024-05-27 20:08:51] Successfully created backup of MinIO data. -[2024-05-27 20:08:51] Uploading Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-27 20:08:54] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-27 20:08:54] Uploading Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-27 20:08:56] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-27 20:08:56] Uploading MinIO data backup to Google Drive. -[2024-05-27 20:08:59] Successfully uploaded MinIO data backup to Google Drive. -[2024-05-27 20:08:59] Deleting local backups. -[2024-05-27 20:08:59] Backup process completed. -[2024-05-27 20:12:40] Starting backup process. -[2024-05-27 20:12:40] Creating backup of Similar_Design_Finder_Dev database. -[2024-05-27 20:12:41] Successfully created backup of Similar_Design_Finder_Dev database. -[2024-05-27 20:12:41] Creating backup of Similar_Design_Finder_Prod database. -[2024-05-27 20:12:42] Successfully created backup of Similar_Design_Finder_Prod database. -[2024-05-27 20:12:42] Creating backup of MinIO data. -[2024-05-27 20:12:43] Successfully created backup of MinIO data. -[2024-05-27 20:12:43] Uploading Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-27 20:12:46] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-27 20:12:46] Uploading Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-27 20:12:48] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-27 20:12:48] Uploading MinIO data backup to Google Drive. -[2024-05-27 20:12:51] Successfully uploaded MinIO data backup to Google Drive. -[2024-05-27 20:12:51] Deleting local backups. -[2024-05-27 20:12:51] Backup process completed. -[2024-05-28 03:00:01] Starting backup process. -[2024-05-28 03:00:01] Creating backup of Similar_Design_Finder_Dev database. -[2024-05-28 03:00:02] Successfully created backup of Similar_Design_Finder_Dev database. -[2024-05-28 03:00:02] Creating backup of Similar_Design_Finder_Prod database. -[2024-05-28 03:00:02] Successfully created backup of Similar_Design_Finder_Prod database. -[2024-05-28 03:00:02] Creating backup of MinIO data. -[2024-05-28 03:00:03] Successfully created backup of MinIO data. -[2024-05-28 03:00:03] Uploading Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-28 03:00:06] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-28 03:00:06] Uploading Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-28 03:00:09] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-28 03:00:09] Uploading MinIO data backup to Google Drive. -[2024-05-28 03:00:11] Successfully uploaded MinIO data backup to Google Drive. -[2024-05-28 03:00:11] Deleting local backups. -[2024-05-28 03:00:11] Backup process completed. -[2024-05-29 03:00:01] Starting backup process. -[2024-05-29 03:00:01] Creating backup of Similar_Design_Finder_Dev database. -[2024-05-29 03:00:02] Successfully created backup of Similar_Design_Finder_Dev database. -[2024-05-29 03:00:02] Creating backup of Similar_Design_Finder_Prod database. -[2024-05-29 03:00:02] Successfully created backup of Similar_Design_Finder_Prod database. -[2024-05-29 03:00:02] Creating backup of MinIO data. -[2024-05-29 03:00:03] Successfully created backup of MinIO data. -[2024-05-29 03:00:03] Uploading Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-29 03:00:06] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-29 03:00:06] Uploading Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-29 03:00:09] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-29 03:00:09] Uploading MinIO data backup to Google Drive. -[2024-05-29 03:00:11] Successfully uploaded MinIO data backup to Google Drive. -[2024-05-29 03:00:11] Deleting local backups. -[2024-05-29 03:00:11] Backup process completed. -[2024-05-30 03:00:01] Starting backup process. -[2024-05-30 03:00:01] Creating backup of Similar_Design_Finder_Dev database. -[2024-05-30 03:00:02] Successfully created backup of Similar_Design_Finder_Dev database. -[2024-05-30 03:00:02] Creating backup of Similar_Design_Finder_Prod database. -[2024-05-30 03:00:03] Successfully created backup of Similar_Design_Finder_Prod database. -[2024-05-30 03:00:03] Creating backup of MinIO data. -[2024-05-30 03:00:04] Successfully created backup of MinIO data. -[2024-05-30 03:00:04] Uploading Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-30 03:00:07] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. -[2024-05-30 03:00:07] Uploading Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-30 03:00:09] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. -[2024-05-30 03:00:09] Uploading MinIO data backup to Google Drive. -[2024-05-30 03:00:11] Successfully uploaded MinIO data backup to Google Drive. -[2024-05-30 03:00:11] Deleting local backups. -[2024-05-30 03:00:11] Backup process completed. -[2024-06-02 03:00:01] Starting backup process. -[2024-06-02 03:00:01] Creating backup of Similar_Design_Finder_Dev database. -[2024-06-02 03:00:03] Successfully created backup of Similar_Design_Finder_Dev database. -[2024-06-02 03:00:03] Creating backup of Similar_Design_Finder_Prod database. -[2024-06-02 03:00:04] Successfully created backup of Similar_Design_Finder_Prod database. -[2024-06-02 03:00:04] Creating backup of MinIO data. -[2024-06-02 03:00:05] Successfully created backup of MinIO data. -[2024-06-02 03:00:05] Uploading Similar_Design_Finder_Dev database backup to Google Drive. -[2024-06-02 03:00:08] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. -[2024-06-02 03:00:08] Uploading Similar_Design_Finder_Prod database backup to Google Drive. -[2024-06-02 03:00:10] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. -[2024-06-02 03:00:10] Uploading MinIO data backup to Google Drive. -[2024-06-02 03:00:12] Successfully uploaded MinIO data backup to Google Drive. -[2024-06-02 03:00:12] Deleting local backups. -[2024-06-02 03:00:12] Backup process completed. -[2024-06-03 03:00:01] Starting backup process. -[2024-06-03 03:00:01] Creating backup of Similar_Design_Finder_Dev database. -[2024-06-03 03:00:02] Successfully created backup of Similar_Design_Finder_Dev database. -[2024-06-03 03:00:02] Creating backup of Similar_Design_Finder_Prod database. -[2024-06-03 03:00:03] Successfully created backup of Similar_Design_Finder_Prod database. -[2024-06-03 03:00:03] Creating backup of MinIO data. -[2024-06-03 03:00:04] Successfully created backup of MinIO data. -[2024-06-03 03:00:04] Uploading Similar_Design_Finder_Dev database backup to Google Drive. -[2024-06-03 03:00:07] Successfully uploaded Similar_Design_Finder_Dev database backup to Google Drive. -[2024-06-03 03:00:07] Uploading Similar_Design_Finder_Prod database backup to Google Drive. -[2024-06-03 03:00:09] Successfully uploaded Similar_Design_Finder_Prod database backup to Google Drive. -[2024-06-03 03:00:09] Uploading MinIO data backup to Google Drive. -[2024-06-03 03:00:11] Successfully uploaded MinIO data backup to Google Drive. -[2024-06-03 03:00:11] Deleting local backups. -[2024-06-03 03:00:11] Backup process completed. From 3fd35b49a7ca22f5823ac6e7ef14466aff1bb863 Mon Sep 17 00:00:00 2001 From: Eduard Poliakov Date: Wed, 5 Jun 2024 08:55:22 +0300 Subject: [PATCH 3/6] Add information to the README.md file --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eafe9f5..191762d 100644 --- a/README.md +++ b/README.md @@ -1 +1,9 @@ -# mlsd-infra \ No newline at end of file +# MLSD-Infra +This repository contains the infrastructure code for the MLSD project. + +## Requirements +- Two data bases: one for development and one for production +- One S3 data base in Minio + + +We also have backups that are made once a day in the 3AM using cronjobs and rclone sending the backups to Google Drive. From a409bfd6ef79177c564f9c734a7da54f13bf841a Mon Sep 17 00:00:00 2001 From: Eduard Poliakov Date: Mon, 17 Jun 2024 18:02:23 +0300 Subject: [PATCH 4/6] Add --- docker-compose.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index f6c6151..36ed1c9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,9 +3,9 @@ services: postgres_dev: image: pgvector/pgvector:pg16 environment: - POSTGRES_USER: admin - POSTGRES_PASSWORD: admin - POSTGRES_DB: Similar_Design_Finder_Dev + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB_DEV} volumes: - pgdata_dev:/var/lib/postgresql/data ports: @@ -15,9 +15,9 @@ services: postgres_prod: image: pgvector/pgvector:pg16 environment: - POSTGRES_USER: admin - POSTGRES_PASSWORD: admin - POSTGRES_DB: Similar_Design_Finder_Prod + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB_PROD} volumes: - pgdata_prod:/var/lib/postgresql/data ports: @@ -32,9 +32,9 @@ services: volumes: - minio_data:/data environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=admin2024 - - MINIO_DEFAULT_BUCKETS=default + - MINIO_ROOT_USER=${MINIO_ROOT_USER} + - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} + - MINIO_DEFAULT_BUCKETS=${MINIO_DEFAULT_BUCKETS} restart: always volumes: From 01b72629943da4942c41aed1746735607d2262de Mon Sep 17 00:00:00 2001 From: Eduard Poliakov Date: Mon, 17 Jun 2024 18:02:43 +0300 Subject: [PATCH 5/6] Add CI (template) and hide variables in .env --- .github/workflows/ci.yml | 64 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..51e98b1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,64 @@ +name: CI for + +on: + workflow_dispatch: + push: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + python-version: ["3.11.2"] + + steps: + - name: Checkout to mlsd-infra + uses: actions/checkout@v2 + + - name: Checkout to Similar-Design-Finder + uses: actions/checkout@v2 + with: + repository: https://github.com/SimulatorML/Similar-Design-Finder + path: repoB + token: ${{ secrets.REPO_B_ACCESS_TOKEN }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python3 - + echo "export PATH=$HOME/.local/bin:\$PATH" >> $GITHUB_ENV + + - name: Install dependencies + working-directory: repoB + run: | + poetry install + + - name: Format with ruff (black replacement) + working-directory: repoB + run: | + poetry run ruff check . --fix + + - name: Sort and clean imports with isort + working-directory: repoB + run: | + poetry run isort . + + - name: Set PYTHONPATH + run: echo "PYTHONPATH=$GITHUB_WORKSPACE/repoB" >> $GITHUB_ENV + + - name: Test with pytest + working-directory: repoB + run: | + poetry run pytest + + - name: Run pre-commit checks + working-directory: repoB + run: | + poetry run pre-commit run --all-files From 163766f254f695f3def65965835e85844de33031 Mon Sep 17 00:00:00 2001 From: Eduard Poliakov Date: Fri, 28 Jun 2024 16:39:58 +0300 Subject: [PATCH 6/6] Add different passwords to databases --- docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 36ed1c9..2a400ae 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,7 +4,7 @@ services: image: pgvector/pgvector:pg16 environment: POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PASSWORD: ${POSTGRES_DB_DEV_PASSWORD} POSTGRES_DB: ${POSTGRES_DB_DEV} volumes: - pgdata_dev:/var/lib/postgresql/data @@ -16,7 +16,7 @@ services: image: pgvector/pgvector:pg16 environment: POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PASSWORD: ${POSTGRES_DB_PROD_PASSWORD} POSTGRES_DB: ${POSTGRES_DB_PROD} volumes: - pgdata_prod:/var/lib/postgresql/data