diff --git a/.gitignore b/.gitignore index ff4d5d0a..01d6def6 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__ *.svg *.swo *.swp +*.tar.gz **/build diff --git a/README.md b/README.md index ef22abfd..f0ce16a1 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,18 @@ Quick overview of what you will be able to see and manage through this project # Install +## System requirements when using macOS +```bash +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +brew install python@3.9 +brew install virtualenv +brew install graphviz +export GRAPHVIZ_DIR="$(brew --prefix graphviz)" +export C_INCLUDE_PATH=/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/Headers +export CPLUS_INCLUDE_PATH=/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/Headers +pip install pygraphviz --global-option=build_ext --global-option="-I$GRAPHVIZ_DIR/include" --global-option="-L$GRAPHVIZ_DIR/lib" +``` + ## System requirements when using ubuntu ```bash sudo apt-add-repository ppa:mutlaqja/ppa @@ -100,6 +112,13 @@ sudo apt-get install \ pip install -r requirements.txt ``` +## Build and use docker for development (for macos dev for instance) + +```console + cd ./docker + ./build_images.sh latest linux/amd64 +``` + ## Building the nice reporting / latex reports ```bash sudo apt-get update diff --git a/ScopeSimulator/data/bsc5.dat.gz b/ScopeSimulator/data/bsc5.dat.gz old mode 100755 new mode 100644 diff --git a/apps/Arduino/configure_tty.sh b/apps/Arduino/configure_tty.sh old mode 100755 new mode 100644 diff --git a/apps/Arduino/upload.sh b/apps/Arduino/upload.sh old mode 100755 new mode 100644 diff --git a/apps/indi_lemarchand_box.sh b/apps/indi_lemarchand_box.sh old mode 100755 new mode 100644 diff --git a/apps/launch_PAWS.sh b/apps/launch_PAWS.sh old mode 100755 new mode 100644 diff --git a/apps/launch_arduino_capture.py b/apps/launch_arduino_capture.py old mode 100755 new mode 100644 diff --git a/apps/launch_backyard_conf.sh b/apps/launch_backyard_conf.sh old mode 100755 new mode 100644 diff --git a/apps/launch_indi_ASI120MC_alone.sh b/apps/launch_indi_ASI120MC_alone.sh old mode 100755 new mode 100644 diff --git a/apps/launch_indi_aag.sh b/apps/launch_indi_aag.sh old mode 100755 new mode 100644 diff --git a/apps/launch_indi_eqmod_simu.sh b/apps/launch_indi_eqmod_simu.sh old mode 100755 new mode 100644 diff --git a/apps/launch_indi_gemini2_alone.sh b/apps/launch_indi_gemini2_alone.sh old mode 100755 new mode 100644 diff --git a/apps/launch_indi_gemini_asi.sh b/apps/launch_indi_gemini_asi.sh old mode 100755 new mode 100644 diff --git a/apps/launch_indi_gemini_canon.sh b/apps/launch_indi_gemini_canon.sh old mode 100755 new mode 100644 diff --git a/apps/launch_indi_scope_controller_debug.sh b/apps/launch_indi_scope_controller_debug.sh old mode 100755 new mode 100644 diff --git a/apps/launch_indi_simu.sh b/apps/launch_indi_simu.sh old mode 100755 new mode 100644 diff --git a/apps/launch_indi_ticfocus.sh b/apps/launch_indi_ticfocus.sh old mode 100755 new mode 100644 diff --git a/apps/launch_weather_capture.py b/apps/launch_weather_capture.py old mode 100755 new mode 100644 diff --git a/apps/plot_weather.py b/apps/plot_weather.py old mode 100755 new mode 100644 diff --git a/apps/prototyping/Solvercpp/build.sh b/apps/prototyping/Solvercpp/build.sh old mode 100755 new mode 100644 diff --git a/apps/prototyping/messaging/listen_mqtt.sh b/apps/prototyping/messaging/listen_mqtt.sh old mode 100755 new mode 100644 diff --git a/apps/prototyping/messaging/send_mqtt.sh b/apps/prototyping/messaging/send_mqtt.sh old mode 100755 new mode 100644 diff --git a/conf_files/dockers/influxdb-remoteobservatory/config/config.yml b/conf_files/dockers/influxdb-remoteobservatory/config/config.yml old mode 100755 new mode 100644 diff --git a/conf_files/dockers/mosquitto-remoteobservatory/mosquitto.conf b/conf_files/dockers/mosquitto-remoteobservatory/mosquitto.conf old mode 100755 new mode 100644 diff --git a/conf_files/dockers/telegraf-remoteobservatory/telegraf.conf b/conf_files/dockers/telegraf-remoteobservatory/telegraf.conf old mode 100755 new mode 100644 diff --git a/conf_files/indi_driver_conf/Shelyak Spox_config.xml b/conf_files/indi_driver_conf/Shelyak Spox_config.xml index efdb410d..460a324d 100644 --- a/conf_files/indi_driver_conf/Shelyak Spox_config.xml +++ b/conf_files/indi_driver_conf/Shelyak Spox_config.xml @@ -1,5 +1,5 @@ - + Off @@ -7,12 +7,12 @@ On - + 1000 - + On @@ -29,7 +29,7 @@ Off - + On @@ -46,7 +46,7 @@ Off - + On diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..308bed16 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,266 @@ +ARG BASE_IMAGE=ubuntu:24.04 +FROM ${BASE_IMAGE} + +# prevent keyboard related user input to be asked for +ENV DEBIAN_FRONTEND noninteractive + +# Useless docker cache for pip +ENV PIP_NO_CACHE_DIR 1 + +# Versioning +ENV INDI_VERSION v2.0.8 +ENV PYINDI_VERSION v1.9.1 +ENV INDI_3RD_PARTY_VERSION v2.0.8 +ENV PYTHON_VERSION 3.12.3 + +# Generic install / utilities / dev +RUN apt-get update && apt-get --assume-yes --quiet install --no-install-recommends \ + build-essential \ + cmake \ + gettext \ + git \ + libgraphviz-dev \ + libz3-dev \ + python3 \ + python3-dev \ + python3-numpy-dev \ + python3-pip \ + python3-setuptools \ + python3-six \ + software-properties-common \ + vim && \ + apt-get clean + +# pyenv dependencies +RUN apt-get --assume-yes --quiet install --no-install-recommends \ + build-essential \ + curl \ + libbz2-dev \ + libffi-dev \ + liblzma-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + llvm \ + make \ + python3-openssl \ + tk-dev \ + wget \ + xz-utils \ + zlib1g-dev + +## Install astrometry.net packages +# Check cat /usr/local/astrometry/etc/astrometry.cfg to make sure destination path is correct +# 4100-series built from the Tycho-2 catalog; scales 7-19 available, good for images wider than 1 degree. Recommended. +# Mean satellite observation epoch ~J1991.5 +# Epoch of the Tycho-2 catalog J2000.0 +# Reference system ICRS +# see https://heasarc.gsfc.nasa.gov/W3Browse/all/tycho2.html + +# 5200-series, LIGHT version built from Tycho-2 + Gaia-DR2; scales 0-6 available, good for images narrower than 1 degree. Combine with 4100-series for broader scale coverage. +# The LIGHT version contains smaller files with no additional Gaia-DR2 information tagged along. Recommended. +# Gaia DR2 astrometry consistently uses the ICRS reference system and provides stellar coordinates valid for epoch J2015.5 +# see https://www.cosmos.esa.int/web/cheops-guest-observers-programme/coordinates +RUN apt-get --assume-yes --quiet install --no-install-recommends libcairo2-dev libnetpbm11-dev \ + && mkdir -p $HOME/projects/astrometry.net \ + && cd $HOME/projects/astrometry.net \ + && wget https://astrometry.net/downloads/astrometry.net-0.95.tar.gz \ + && tar -xvf ./*.tar.gz \ + && cd astrometry.net-0.95 \ + && export NETPBM_LIB="-L/usr/lib -lnetpbm" \ + && export NETPBM_INC="-I/usr/include" \ + && ./configure --prefix=/usr \ + && make reconfig \ + && make all -j8 \ + && make install \ + && ln -s /usr/local/astrometry/bin/an-fitstopnm /usr/local/bin/an-fitstopnm +# && wget --recursive --no-parent --no-host-directories --cut-dirs=6 --accept "*.fits" --continue --directory-prefix=/usr/local/astrometry/data/ https://portal.nersc.gov/project/cosmo/temp/dstn/index-5200/LITE/ + +# Downloading gcloud package +RUN curl -sSL https://sdk.cloud.google.com > /tmp/gcl && bash /tmp/gcl --install-dir=/opt/gcloud --disable-prompts +ENV PATH $PATH:/opt/gcloud/google-cloud-sdk/bin + +# Now Download astrometry.net index files +RUN gsutil -m cp gs://astrometry_data/* /usr/local/astrometry/data/ + +## Indi dependencies for pre-packages binaries +#RUN apt-add-repository ppa:mutlaqja/ppa && apt-get --assume-yes --quiet install --no-install-recommends \ +# gsc \ +# libcfitsio-dev \ +# libnova-dev \ +# libindi1 \ +# indi-bin \ +# kstars-bleeding \ +# swig + +# Dependencies to build indi from sources + pyindi-client that is linked with binaries through swig +# astrometry.net was built from source +RUN apt-get --assume-yes --quiet install --no-install-recommends \ + cdbs \ + dkms \ + fxload \ + libboost-regex-dev \ + libcfitsio-dev \ + libcurl4-gnutls-dev \ + libdc1394-dev \ + libev-dev \ + libfftw3-dev \ + libftdi-dev \ + libftdi1-dev \ + libgps-dev \ + libgsl0-dev \ + libjpeg-dev \ + libkrb5-dev \ + libnova-dev \ + libraw-dev \ + libtiff5-dev \ + libusb-1.0-0-dev \ + libusb-dev \ + swig + +# Build indi from sources +RUN mkdir -p $HOME/projects \ + && cd $HOME/projects \ + && git clone https://github.com/indilib/indi.git \ + && cd $HOME/projects/indi \ + && git checkout $INDI_VERSION \ + && cd $HOME/projects \ + && mkdir -p build/indi \ + && cd build/indi \ + && cmake -DCMAKE_INSTALL_PREFIX=/usr $HOME/projects/indi \ + && make -j8 \ + && make install + +# Dependencies to build indi-3rd party from sources +RUN apt-get --assume-yes --quiet install --no-install-recommends \ + libnova-dev \ + libcfitsio-dev \ + libusb-1.0-0-dev \ + zlib1g-dev \ + libgsl-dev \ + libjpeg-dev \ + libcurl4-gnutls-dev \ + libtiff-dev \ + libftdi-dev \ + libraw-dev \ + libdc1394-dev \ + libgphoto2-dev \ + libboost-dev \ + libboost-regex-dev \ + librtlsdr-dev \ + liblimesuite-dev \ + libftdi1-dev \ + libgps-dev \ + libavcodec-dev \ + libavdevice-dev \ + libzmq3-dev + +# Additional dependencies with indi-3rd party - clone the whole thing +RUN mkdir -p $HOME/projects \ + && cd $HOME/projects \ + && git clone https://github.com/indilib/indi-3rdparty.git \ + && cd $HOME/projects/indi-3rdparty \ + && git checkout $INDI_3RD_PARTY_VERSION + +RUN for i in indi-duino libasi indi-asi libplayerone indi-playerone indi-shelyak libaltaircam; \ + do cd $HOME/projects \ + && mkdir -p $HOME/projects/build/$i \ + && cd $HOME/projects/build/$i \ + && cmake -DCMAKE_INSTALL_PREFIX=/usr $HOME/projects/indi-3rdparty/$i \ + && make -j8 \ + && make install; \ + done + +## OpenPhd2 might later be replaced by the original source ? +#RUN add-apt-repository ppa:pch/phd2 && apt-get --assume-yes --quiet install --no-install-recommends \ +# phd2 +# Dependencies to build phd2 from sources +RUN apt-get --assume-yes --quiet install --no-install-recommends \ + libwxgtk3.2-dev + +RUN mkdir -p $HOME/projects \ + && cd $HOME/projects \ + && git clone https://github.com/gnthibault/phd2.git \ + && mkdir -p build/phd2 \ + && cd build/phd2 \ + && cmake -DCMAKE_INSTALL_PREFIX=/usr $HOME/projects/phd2 \ + && make -j8 \ + && make install + +RUN cp /opt/remote_observatory/infrastructure/phd2.service /etc/systemd/system/ +RUN chmod 644 /etc/systemd/system/phd2.service + +## Dependencies for nice reporting / latex reports +RUN apt-get --assume-yes --quiet install --no-install-recommends \ + texlive-latex-recommended \ + texlive-publishers \ + texlive-bibtex-extra \ + texlive-science + +# Using bash for lower level scripting from now-on +SHELL ["/bin/bash", "-l", "-c"] +RUN echo 'export PS1="\u@\h \w> "' | cat - /root/.profile > temp && mv temp /root/.profile + +# Actual application code and configs +RUN mkdir -p /opt/remote_observatory +ADD code.tar.gz /opt/remote_observatory + +# Python environment +RUN curl https://pyenv.run | bash +RUN echo 'export PYENV_ROOT=/root/.pyenv' >> /root/.bashrc +RUN echo '[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"' >> /root/.bashrc +RUN echo 'eval "$(pyenv init -)"' >> /root/.bashrc +RUN pyenv install -v $PYTHON_VERSION +RUN pyenv global $PYTHON_VERSION + +# Python virtual environment +ENV VIRTUAL_ENV=/opt/remote_observatory_venv +RUN python -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN echo 'source /opt/remote_observatory_venv/bin/activate' >> /root/.bashrc + +# Python packages +#COPY requirements.txt . +RUN pip install -r /opt/remote_observatory/requirements.txt + +# Build and install pyindi-client +RUN mkdir -p $HOME/projects \ + && cd $HOME/projects \ + && git clone https://github.com/indilib/pyindi-client.git \ + && cd $HOME/projects/pyindi-client \ + && git checkout $PYINDI_VERSION \ + && pip install -e . + + +# Indi webmanager for dev +RUN pip install indiweb==0.1.8 +RUN cp /opt/remote_observatory/infrastructure/indiwebmanager*.service /etc/systemd/system/ +RUN chmod 644 /etc/systemd/system/indiwebmanager.service +RUN chmod 644 /etc/systemd/system/indiwebmanager_guiding_camera.service +RUN chmod 644 /etc/systemd/system/indiwebmanager_pointing_camera.service +RUN chmod 644 /etc/systemd/system/indiwebmanager_science_camera.service + +#RUN systemctl daemon-reload +#RUN systemctl enable indiwebmanager.service + + +# # Add Docker's official GPG key: +# sudo apt-get update +# sudo apt-get install ca-certificates curl +# sudo install -m 0755 -d /etc/apt/keyrings +# sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +# sudo chmod a+r /etc/apt/keyrings/docker.asc + +# # Add the repository to Apt sources: +# echo \ +# "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ +# $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ +# sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +# sudo apt-get update && sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +# Build with +# ./build_images.sh latest linux/amd64 +# launch with +# docker run --user $(id -u):$(id -g) -it --rm -v /main/machine/volume:/docker/mount/point --net host gnthibault/remote_observatory:latest diff --git a/docker/build_images.sh b/docker/build_images.sh new file mode 100755 index 00000000..311810fa --- /dev/null +++ b/docker/build_images.sh @@ -0,0 +1,137 @@ +#!/bin/bash + +DOCKER_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +MAIN_REPO_DIR="$DOCKER_DIR/.." +DOCKER_URL_REPO="gnthibault" +DOCKER_BASE_IMAGE="ubuntu:24.04" +DOCKER_ARM64_BASE_IMAGE="arm64v8/ubuntu:24.04" + + +docker_clean() { + echo "-- DOCKER BUILD -- Now performs cleaning" + + # Now clean all exited containers + echo "-- (Cleaning exited containers)" + docker container prune --filter "until=24h" -f + #for docker_hash in $(docker ps --quiet --filter=status=exited); + #do + # docker rm -f $docker_hash + #done + + # Now clean all dangling images + echo "-- (Cleaning dangling image)" + docker image prune -a --filter "until=4h" -f + #for docker_hash in $(docker images --quiet --filter=unused=true); + #do + # docker rmi -f $docker_hash + #done + + # Now clean all dangling volume + echo "-- (Cleaning associated ressources (volumes, network)" + docker volume prune -f + docker network prune --filter "until=24h" -f + #for volume in $(docker volume ls -qf 'dangling=true'); + #do + # docker volume rm $volume + #done +} + +docker_tag() { + #$1 is the image name + #$2 is the actual tag + echo "--docker tag $1:$2 $DOCKER_URL_REPO/$1:$2" + docker tag $1:$2 $DOCKER_URL_REPO/$1:$2 + # For local kaniko please check https://github.com/GoogleContainerTools/kaniko?tab=readme-ov-file#running-kaniko-in-docker + # kaniko --dockerfile=Dockerfile --context=/path/to/build/context --destination=my-registry.com/my-image:latest +} + +docker_push() { + #$1 is the image name + #$2 is the actual tag + echo "--docker push $DOCKER_URL_REPO/$1:$2" + docker push $DOCKER_URL_REPO/$1:$2 + # For local kaniko please check https://github.com/GoogleContainerTools/kaniko?tab=readme-ov-file#running-kaniko-in-docker + # kaniko --dockerfile=Dockerfile --context=/path/to/build/context --destination=my-registry.com/my-image:latest --push +} + +docker_build () { + #$1 the image name + #$2 the directory where Dockerfile is located + #$3 the tag + #"$4": the docker build options, note the "" are importants + IMAGE_NAME=$1:$3 + #remove leading and trailing double quote + PARAMETERS=$(sed -e 's/^"//' -e 's/"$//' <<<"$4") + PARAMETERS="$PARAMETERS --no-cache=true" + echo "--docker build $PARAMETERS -t $IMAGE_NAME $2" + # Use --progress=plain for debugging purpose + docker build --progress=plain $PARAMETERS -t $IMAGE_NAME $2 # 2>&1 >/dev/null + # For local kaniko please check https://github.com/GoogleContainerTools/kaniko?tab=readme-ov-file#running-kaniko-in-docker + # kaniko --dockerfile=Dockerfile --context=/path/to/build/context --destination=my-image:latest --build-arg KEY=VALUE + # you might want to use "--tar-path $APP_NAME.tar --no-push" and then use crane to push to remote +} + +process_build() { + #$1 the image name + #$2 is the target ARCH parameter + #$3 the (git) tag to be checked out and embedded in the image + + # define path and image name + NAME=$1 + TAG=$2 + TARGET_ARCH=$3 + + # Specify CPU/GPU related build parameters + if [ TARGET_ARCH = "linux/arm64" ] + then + DOCKER_BASE_IMAGE=$DOCKER_ARM64_BASE_IMAGE + else + DOCKER_BASE_IMAGE=$DOCKER_BASE_IMAGE + fi + + #Build a tar out of the code repo + tar --directory $MAIN_REPO_DIR \ + -cvzf $DOCKER_DIR/code.tar.gz \ + --exclude "*.log"\ + --exclude "*.tar.gz" \ + --exclude "*.zip" \ + --exclude "*.pyc" . + + # Now build the actual production image + echo "-- DOCKER BUILD -- Now building image" + # Takes at input, the tag ($1) and directory ($2) whether it should embed + # a copy of the codebase $3 + docker_build $NAME . $TAG \ + "\"--build-arg BASE_IMAGE=$DOCKER_BASE_IMAGE\ + --build-arg PYTHON_REQ=PYTHON_REQ_FILE\ + --platform $TARGET_ARCH\"" + + # tag the freshly generated image + # docker_tag $NAME $TAG + # push the tagged image to the repo + # docker_push $NAME $TAG + + echo "-- DOCKER BUILD -- Finished building image" +} + +# Main script +if [[ -n "$1" ]]; then + img_name="remote_observatory" + image_tag=$1 +else + echo "Missing argument 1, please specify either a tag or latest" + exit +fi + +if [[ -n "$2" ]]; then + target_arch=$2 +else + echo "Missing argument 2, please specify either linux/amd64 or linux/arm64" + exit +fi + +#Make sure the script fails if one non zero error code is returned +set -e + +# Perform actual work +process_build $img_name $image_tag $target_arch \ No newline at end of file diff --git a/documentation/old_dataset_exploration/script.sh b/documentation/old_dataset_exploration/script.sh old mode 100755 new mode 100644 diff --git a/infrastructure/indiwebmanager.service b/infrastructure/indiwebmanager.service index 14ba64ac..63f0706b 100644 --- a/infrastructure/indiwebmanager.service +++ b/infrastructure/indiwebmanager.service @@ -5,9 +5,9 @@ After=multi-user.target [Service] Type=idle # MUST SET YOUR USERNAME HERE. -User=gnthibault +User=root Environment=LD_LIBRARY_PATH=/usr/local/lib:/usr/lib -ExecStart=/home/gnthibault/.local/bin/indi-web -v --xmldir /home/gnthibault/projects/RemoteObservatory/conf_files/indi_driver_xml --conf /home/gnthibault/projects/RemoteObservatory/conf_files/indi_driver_conf +ExecStart=/bin/indi-web -v --xmldir /opt/remote_observatory/infrastructure/conf_files/indi_driver_xml --conf /opt/remote_observatory/conf_files/indi_driver_conf Restart=always RestartSec=5 diff --git a/infrastructure/onstep_info.txt b/infrastructure/onstep_info.txt new file mode 100644 index 00000000..df2cb9e2 --- /dev/null +++ b/infrastructure/onstep_info.txt @@ -0,0 +1,25 @@ +https://onstep.groups.io/g/main/wiki/26586#AXIS1 --> AXIS1_LIMIT_MIN et MAX et pareil pour AXIS2 + +AXIS1_LIMIT_MIN + + Default Value: -180 + Other Values: -90 to -270 or -360 (degrees) + Notes: Minimum Hour Angle for Equatorial Mount modes (to -270 degrees) or minimum Azimuth for AltAzm mode (to -360 degrees.) + Reminder: + To disable limits in Azimuth (unlimited range) set to -360 and also set AXIS1_LIMIT_MAX to 360. + During tracking and gotos if a limit is broken motion is stopped by canceling any goto and turning tracking off. Keep in mind that up to SLEW_RAPID_STOP_DIST (Config.h) distance should be allowed for to stop the motors/mount once the limit is detected. + +LIMIT_SENSE + + Default Value: OFF + Other Values: OFF, ON, ON_PULLUP, ON_PULLDOWN + Notes: Limit sense switches can be wired so several can be used where any switch change causes OnStep to stop tracking and slews (allowing for SLEW_RAPID_STOP_DIST.) This is most often implemented as a panic switch the user can press and/or lever switches on the mount axes. + Associated Options: + LIMIT_SENSE_STATE + Default Value: LOW + Other Values: LOW, HIGH + Notes: + Use LOW for NO (normally open) switches. If multiple switches of this type are used wired them in parallel. + Use HIGH for NC (normally closed) switches. If multiple switches of this type are used wired them in series. + Reminder: For my recent PCB designs this input DOES have a pullup resistor along with a bypass capacitor to provide basic ESD protection. + diff --git a/infrastructure/phd2.service b/infrastructure/phd2.service index d7018394..9ae9eae8 100644 --- a/infrastructure/phd2.service +++ b/infrastructure/phd2.service @@ -14,11 +14,11 @@ After=multi-user.target [Service] Type=idle -User=gnthibault +User=root Environment=LD_LIBRARY_PATH=/usr/local/lib:/usr/lib Environment="DISPLAY=:0" -# "XAUTHORITY=/home/gnthibault/.Xauthority" -ExecStartPre=rm -f /home/gnthibault/phd2.1 +# "XAUTHORITY=/root/.Xauthority" +ExecStartPre=rm -f /root/phd2.1 ExecStart=/usr/bin/phd2 Restart=always RestartSec=3 diff --git a/requirements.txt b/requirements.txt index 92913cb0..05a73328 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,33 +1,34 @@ -astroalign==1.0.4 -astroplan==0.8 -astropy==4.1 -astroquery==0.4.1 +astroalign==2.5.1 +astroplan==0.10 +astropy==6.1.2 +astroquery==0.4.7 erdantic==0.6.0 -influxdb-client==1.35.0 +influxdb-client==1.44.0 gcn-kafka==0.3.0 -lxml==4.9.1 +graphviz==0.20.1 +lxml==5.2.2 ntplib==0.3.4 -matplotlib==3.3.4 +matplotlib==3.7.2 meshcat==0.3.2 -numpy==1.22.0 -opencv-python==4.3.0.36 -photutils==1.0.2 +numpy==1.26.4 +opencv-python==4.8.0.76 +photutils==1.9.0 pyaml==20.4.0 pydantic==2.2.1 pydantic-core==2.6.1 -pygraphviz==1.6 -PyLaTeX==1.3.2 +#pygraphviz==1.6 +PyLaTeX==1.4.2 pymongo==3.11.3 paho-mqtt==1.6.1 -pyserial==3.4 +#pyserial==3.4 pytz==2021.3 pyzmq==25.1.0 -rawpy==0.16.0 requests==2.31.0 rinohtype==0.3.1 -sep==1.1.1 -serial==0.0.97 +sep==1.2.1 +#serial==0.0.97 +setuptools==70.0.0 Sphinx==1.8.1 transitions==0.7.1 tzwhere==3.0.3 -watchdog==0.8.3 +watchdog==4.0.1 \ No newline at end of file diff --git a/scripts/Aladin.jar b/scripts/Aladin.jar old mode 100755 new mode 100644 diff --git a/scripts/solve_field.sh b/scripts/solve_field.sh old mode 100755 new mode 100644 diff --git a/utils/database.py b/utils/database.py index 75f2a627..16d603b5 100644 --- a/utils/database.py +++ b/utils/database.py @@ -9,6 +9,9 @@ from warnings import warn import weakref +# DB +from sqlmodel import Field, Session, SQLModel, create_engine, select + # Local from Service.HostTimeService import HostTimeService from utils import serializers as json_util @@ -122,6 +125,127 @@ def clear_current(self, type): """ raise NotImplementedError +############################################################################### +# SQLite implementation +############################################################################### + +class TransactionalFileDB(metaclass=abc.ABCMeta): + def __init__(self, db_name=None, collection_names=list(), logger=None, + serv_time=HostTimeService(), **kwargs): + """ + Init base class for db instances. + + Args: + db_name: Name of the database, typically panoptes or + panoptes_testing. + collection_names (list of str): Names of the valid collections. + logger: (Optional) logger to use for warnings. + """ + super().__init__(db_name=db_name, **kwargs) + self.db_folder = db_name + self.sqlite_file_path = os.path.join(self.db_folder, "database.db") + # Set up storage directory. + self._storage_dir = self.db_folder + os.makedirs(self._storage_dir, exist_ok=True) + self.engine = None + self.create_db_and_tables() + + def create_db_and_tables(self): + sqlite_url = f"sqlite:///{self.sqlite_file_path}" + connect_args = {"check_same_thread": False} + self.engine = create_engine(sqlite_url, echo=True, connect_args=connect_args) + SQLModel.metadata.create_all(self.engine) + + def _warn(self, *args, **kwargs): + if self.logger: + self.logger.warning(*args, **kwargs) + else: + warn(*args) + + def validate_collection(self, collection): + if collection not in self.collection_names: + msg = f"Collection type {collection} not available" + self._warn(msg) + # Can't import utils.error earlier + from utils.error import InvalidCollection + raise InvalidCollection(msg) + + @abc.abstractclassmethod + def insert_current(self, collection, obj, store_permanently=True): + """Insert an object into both the `current` collection and the + collection provided. + + Args: + collection (str): Name of valid collection within the db. + obj (dict or str): Object to be inserted. + store_permanently (bool): Whether to also update the collection, + defaults to True. + + Returns: + str: identifier of inserted record. If `store_permanently` is True, + will be the identifier of the object in the `collection`, + otherwise will be the identifier of object in the `current` + collection. These may or may not be the same. + Returns None if unable to insert into the collection. + """ + raise NotImplementedError + + @abc.abstractclassmethod + def insert(self, collection, obj): + """Insert an object into the collection provided. + + The `obj` to be stored in a collection should include the `type` + and `date` metadata as well as a `data` key that contains the actual + object data. If these keys are not provided then `obj` will be wrapped + in a corresponding object that does contain the metadata. + + Args: + collection (str): Name of valid collection within the db. + obj (dict or str): Object to be inserted. + + Returns: + str: identifier of inserted record in `collection`. + Returns None if unable to insert into the collection. Can be + considered as an object_id + """ + raise NotImplementedError + + @abc.abstractclassmethod + def get_current(self, collection): + """Returns the most current record for the given collection + + Args: + collection (str): Name of the collection to get most current from + + Returns: + dict|None: Current object of the collection or None. + """ + raise NotImplementedError + + @abc.abstractclassmethod + def find(self, collection, obj_id): + """Find an object by it's identifier. + + Args: + collection (str): Collection to search for object. + obj_id (ObjectID|str): Record identifier returned earlier by insert + or insert_current. + + Returns: + dict|None: Object matching identifier or None. + """ + raise NotImplementedError + + @abc.abstractclassmethod + def clear_current(self, type): + """Clear the current record of a certain type + + Args: + type (str): The type of entry in the current collection that + should be cleared. + """ + raise NotImplementedError + ############################################################################### # MongoDB implementation diff --git a/utils/datamodel.py b/utils/datamodel.py index 847ea639..7c917be4 100644 --- a/utils/datamodel.py +++ b/utils/datamodel.py @@ -1,7 +1,10 @@ from datetime import datetime from enum import Enum -from typing import List, Optional from pydantic import BaseModel +from sqlmodel import Field, Session, SQLModel, create_engine, select +from typing import List, Optional + +# TODO TN: https://sqlmodel.tiangolo.com/tutorial/fastapi/simple-hero-api/ class Sequence(BaseModel): id: str