From 2ccdd4eb32284c742350d100e53210aac412ab18 Mon Sep 17 00:00:00 2001 From: Ivareh Date: Mon, 12 Aug 2024 15:38:38 +0200 Subject: [PATCH 1/4] 379 Updated prestart and docker with improved restart script for backend_data_retrieval --- .../data_retrieval/prestart.sh | 12 ++++++++---- src/docker-compose.yml | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/backend_data_retrieval/data_retrieval/prestart.sh b/src/backend_data_retrieval/data_retrieval/prestart.sh index e3e299e1b..9ebd22335 100755 --- a/src/backend_data_retrieval/data_retrieval/prestart.sh +++ b/src/backend_data_retrieval/data_retrieval/prestart.sh @@ -1,10 +1,14 @@ -#! /usr/bin/env bash +#!/bin/bash python dataretrieval_pre_start.py # Create initial data in DB python modifier_data_deposit/deposit_modifier_data.py -while true; do - python external_data_retrieval/main.py -done \ No newline at end of file + + +while true +do + python external_data_retrieval/main.py || echo "App crashed... restarting..." >&2 + echo "Press Ctrl-C to quit." && sleep 1 +done diff --git a/src/docker-compose.yml b/src/docker-compose.yml index bb298ed3a..56ec63532 100644 --- a/src/docker-compose.yml +++ b/src/docker-compose.yml @@ -82,6 +82,7 @@ services: backend-data-retrieval: image: backend_data_retrieval:1.0.0 + restart: always depends_on: db: condition: service_healthy From bd7757a6b121a41c5d80675f9f35fa735dd24f01 Mon Sep 17 00:00:00 2001 From: Ivareh Date: Tue, 13 Aug 2024 07:26:48 +0200 Subject: [PATCH 2/4] 379 Update parameters and added sys._exit() to poe_api retrieval --- src/.env | 4 ++-- .../data_retrieval/poe_api_retrieval/poe_api.py | 6 ++++-- src/docker-compose.override.yml | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/.env b/src/.env index 322724525..d047e94ad 100644 --- a/src/.env +++ b/src/.env @@ -16,7 +16,7 @@ DOCKER_IMAGE_BACKEND=pom_backend DOCKER_IMAGE_FRONTEND=pom_frontend # Backend -PRIVATIZE_API=False +PRIVATIZE_API= TESTING=True BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173,https://localhost,https://localhost:5173" PROJECT_NAME="pathofmodifiers" @@ -31,7 +31,7 @@ SMTP_USER=pomodifiers@outlook.com SMTP_PASSWORD=changethis # Data retrieval -MANUAL_NEXT_CHANGE_ID=False +MANUAL_NEXT_CHANGE_ID= NEXT_CHANGE_ID=2464293076-2443587902-2368433646-2624156651-2552811860 POE_PUBLIC_STASHES_AUTH_TOKEN=changethis CURRENT_SOFTCORE_LEAGUE=Necropolis diff --git a/src/backend_data_retrieval/data_retrieval/external_data_retrieval/data_retrieval/poe_api_retrieval/poe_api.py b/src/backend_data_retrieval/data_retrieval/external_data_retrieval/data_retrieval/poe_api_retrieval/poe_api.py index 12cd1d295..2734b5449 100644 --- a/src/backend_data_retrieval/data_retrieval/external_data_retrieval/data_retrieval/poe_api_retrieval/poe_api.py +++ b/src/backend_data_retrieval/data_retrieval/external_data_retrieval/data_retrieval/poe_api_retrieval/poe_api.py @@ -1,3 +1,4 @@ +import os import requests import time import logging @@ -459,5 +460,6 @@ def dump_stream(self, track_progress: bool = True) -> Iterator[pd.DataFrame]: print("Finished transformation phase.") current_time = time.perf_counter() time_since_launch = current_time - self.time_of_launch - if time_since_launch > 3600: - raise ProgramRunTooLongException + if time_since_launch > 1: + print("Program has run for more than an hour, shutting down.") + os._exit(1) diff --git a/src/docker-compose.override.yml b/src/docker-compose.override.yml index 5bbd9a537..13cd77a9b 100644 --- a/src/docker-compose.override.yml +++ b/src/docker-compose.override.yml @@ -55,7 +55,7 @@ services: command: /start-reload.sh backend-data-retrieval: - restart: "no" + restart: always ports: - "8889:8889" volumes: From 2faa9b4831832f5e3dbdbc3b1d53381e0849fa05 Mon Sep 17 00:00:00 2001 From: Ivareh Date: Tue, 13 Aug 2024 07:28:19 +0200 Subject: [PATCH 3/4] 379 Changed sys exit code from 1 to 0 --- .../data_retrieval/poe_api_retrieval/poe_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend_data_retrieval/data_retrieval/external_data_retrieval/data_retrieval/poe_api_retrieval/poe_api.py b/src/backend_data_retrieval/data_retrieval/external_data_retrieval/data_retrieval/poe_api_retrieval/poe_api.py index 2734b5449..6ddf08273 100644 --- a/src/backend_data_retrieval/data_retrieval/external_data_retrieval/data_retrieval/poe_api_retrieval/poe_api.py +++ b/src/backend_data_retrieval/data_retrieval/external_data_retrieval/data_retrieval/poe_api_retrieval/poe_api.py @@ -462,4 +462,4 @@ def dump_stream(self, track_progress: bool = True) -> Iterator[pd.DataFrame]: time_since_launch = current_time - self.time_of_launch if time_since_launch > 1: print("Program has run for more than an hour, shutting down.") - os._exit(1) + os._exit(0) From d0e08420b98de8e499022218689abcb98782f359 Mon Sep 17 00:00:00 2001 From: Ivareh Date: Tue, 13 Aug 2024 07:29:04 +0200 Subject: [PATCH 4/4] 379 Changed time_since_launch from 1 to 3600 in poe_api --- .../data_retrieval/poe_api_retrieval/poe_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend_data_retrieval/data_retrieval/external_data_retrieval/data_retrieval/poe_api_retrieval/poe_api.py b/src/backend_data_retrieval/data_retrieval/external_data_retrieval/data_retrieval/poe_api_retrieval/poe_api.py index 6ddf08273..8e72a3c4a 100644 --- a/src/backend_data_retrieval/data_retrieval/external_data_retrieval/data_retrieval/poe_api_retrieval/poe_api.py +++ b/src/backend_data_retrieval/data_retrieval/external_data_retrieval/data_retrieval/poe_api_retrieval/poe_api.py @@ -460,6 +460,6 @@ def dump_stream(self, track_progress: bool = True) -> Iterator[pd.DataFrame]: print("Finished transformation phase.") current_time = time.perf_counter() time_since_launch = current_time - self.time_of_launch - if time_since_launch > 1: + if time_since_launch > 3600: print("Program has run for more than an hour, shutting down.") os._exit(0)