From 46a152f4a14d35ab4988c884cb6f8a9f3e8a0d0c Mon Sep 17 00:00:00 2001 From: Kyle Ferriter Date: Sat, 29 Mar 2025 21:38:07 -0400 Subject: [PATCH 1/4] Update compose port bindings to localhost only --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 43d4348e..a8bf3672 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: command: seqrepo-rest-service -w /usr/local/share/seqrepo/2024-12-20 network_mode: bridge ports: - - 5000:5000 + - localhost:5000:5000 uta: # Test: @@ -26,7 +26,7 @@ services: volumes: - uta_vol:/var/lib/postgresql/data ports: - - 5432:5432 + - localhost:5432:5432 volumes: seqrepo_vol: From 449b57be6459076567bd581a54463d0fe7776337 Mon Sep 17 00:00:00 2001 From: Kyle Ferriter Date: Sat, 29 Mar 2025 22:29:30 -0400 Subject: [PATCH 2/4] Add examples using UTA download cache and local seqrepo volume mount --- .../seqrepo-copy-local.compose.yaml | 53 +++++++++++++++++++ .../seqrepo-mount-local.compose.yaml | 46 ++++++++++++++++ docs/setup_help/docker-compose.md | 24 +++++++++ 3 files changed, 123 insertions(+) create mode 100644 docs/setup_help/docker-compose-examples/seqrepo-copy-local.compose.yaml create mode 100644 docs/setup_help/docker-compose-examples/seqrepo-mount-local.compose.yaml create mode 100644 docs/setup_help/docker-compose.md diff --git a/docs/setup_help/docker-compose-examples/seqrepo-copy-local.compose.yaml b/docs/setup_help/docker-compose-examples/seqrepo-copy-local.compose.yaml new file mode 100644 index 00000000..86432e0e --- /dev/null +++ b/docs/setup_help/docker-compose-examples/seqrepo-copy-local.compose.yaml @@ -0,0 +1,53 @@ +#### +# This is a docker-compose example to run seqrepo-rest-service by first populating the seqrepo_vol +# with a local copy of the seqrepo data instead of re-downloading it. +# Assumes SEQREPO_ROOT_DIR is set to the directory where your seqrepo data is stored. +# e.g. if you ahve a seqrepo db at ~/.local/share/seqrepo/2024-12-20, SEQREPO_ROOT_DIR +# should be set to ~/.local/share/seqrepo. +# +# It also includes a volume uta_dl_cache which enables the UTA database to be reconstructed +# from scratch using postgres restore without re-downloading the archive file. +#### + +services: + seqrepo_local_populator: + # image: alpine + image: eeacms/rsync + volumes: + - seqrepo_vol:/usr/local/share/seqrepo + - $SEQREPO_ROOT_DIR:/seqrepo + command: > + /bin/sh -c "rsync -a --delete /seqrepo/2024-12-20/ /usr/local/share/seqrepo/2024-12-20/" + + seqrepo-rest-service: + # Test: curl http://localhost:5000/seqrepo/1/sequence/refseq:NM_000551.3 + image: biocommons/seqrepo-rest-service:0.2.2 + volumes: + - seqrepo_vol:/usr/local/share/seqrepo + depends_on: + seqrepo_local_populator: + condition: service_completed_successfully + command: seqrepo-rest-service -w /usr/local/share/seqrepo/2024-12-20 + ports: + - 127.0.0.1:5001:5000 + + uta: + # Test: + # psql -XAt postgres://anonymous@localhost/uta -c 'select count(*) from uta_20241220.transcript' + # 314227 + image: biocommons/uta:uta_20241220 + environment: + - POSTGRES_PASSWORD=some-password-that-you-make-up + volumes: + - uta_dl_cache:/tmp + - uta_vol:/var/lib/postgresql/data + ports: + - 127.0.0.1:5433:5432 + +volumes: + seqrepo_vol: + external: true + uta_vol: + external: true + uta_dl_cache: + external: true diff --git a/docs/setup_help/docker-compose-examples/seqrepo-mount-local.compose.yaml b/docs/setup_help/docker-compose-examples/seqrepo-mount-local.compose.yaml new file mode 100644 index 00000000..c01ccc15 --- /dev/null +++ b/docs/setup_help/docker-compose-examples/seqrepo-mount-local.compose.yaml @@ -0,0 +1,46 @@ +#### +# This is a docker-compose example to run seqrepo-rest-service by mounting +# a local directory of seqrepo instead of re-downloading it or storing it in a volume. +# Assumes SEQREPO_ROOT_DIR is set to the directory where your seqrepo data is stored. +# e.g. if you have a seqrepo db at ~/.local/share/seqrepo/2024-12-20, SEQREPO_ROOT_DIR +# should be set to ~/.local/share/seqrepo. +# +# It also includes a volume uta_dl_cache which enables the UTA database to be reconstructed +# from scratch using postgres restore without re-downloading the archive file. +#### + +services: + + seqrepo-rest-service: + # Test: curl http://localhost:5000/seqrepo/1/sequence/refseq:NM_000551.3 + image: biocommons/seqrepo-rest-service:0.2.2 + volumes: + # Recommended to mount read-only to avoid accidentally modififying host files + - $SEQREPO_ROOT_DIR:/usr/local/share/seqrepo:ro + depends_on: + seqrepo_local_populator: + condition: service_completed_successfully + command: seqrepo-rest-service -w /usr/local/share/seqrepo/2024-12-20 + ports: + - 127.0.0.1:5001:5000 + + uta: + # Test: + # psql -XAt postgres://anonymous@localhost/uta -c 'select count(*) from uta_20241220.transcript' + # 314227 + image: biocommons/uta:uta_20241220 + environment: + - POSTGRES_PASSWORD=some-password-that-you-make-up + volumes: + - uta_dl_cache:/tmp + - uta_vol:/var/lib/postgresql/data + ports: + - 127.0.0.1:5433:5432 + +volumes: + seqrepo_vol: + external: true + uta_vol: + external: true + uta_dl_cache: + external: true diff --git a/docs/setup_help/docker-compose.md b/docs/setup_help/docker-compose.md new file mode 100644 index 00000000..cba3b8b9 --- /dev/null +++ b/docs/setup_help/docker-compose.md @@ -0,0 +1,24 @@ +The compose.yaml file in this repo can be used by Docker Compose. It has also been tested to work with Podman Compose. + +See full specification: +https://github.com/compose-spec/compose-spec/blob/main/00-overview.md + + +The default docker compose in this repo assumes you have not downloaded the UTA or SeqRepo databases out of band, but either ran the compose from scratch which runs the `biocommons/seqrepo` or `biocommons/uta` containers in a way that populates their databases if they don't exist, or you ran those containers on your own previously, such that the volumes `seqrepo_vol` and `uta_vol` are already populated. + +If you already have a SeqRepo database directory on your local filesystem, you can mount it directly as a volume, or can copy it into a docker volume (recommended if your local disk space is not a concern). + + +For an example of mounting a local seqrepo dir directly to the seqrepo-rest-service container see [seqrepo-mount-local.compose.yaml](./docker-compose-examples/seqrepo-mount-local.compose.yaml) + + +For an example of populating a docker volume with a local seqrepo dir see [seqrepo-copy-local.compose.yaml](./docker-compose-examples/seqrepo-copy-local.compose.yaml) + + +In both of the above examples, a volume called `uta_dl_cache` is used. UTA downloads a postgres dump archive if its database is not populated yet, and stores the archive in `/tmp`. If we make `/tmp` a persistent volume, we can destroy the UTA container and the `uta_vol` and recreate it from scratch using the compressed archive in `uta_dl_cache`. The latest UTA compressed postgres dump for `biocommons/uta:uta_20241220` is 344MB, while the uncompressed postgres database created from it is 6GB. + +To run one of the compose files that uses a local seqrepo, with a seqrepo root dir in `~/.local/share/seqrepo/`, run + +``` +$ SEQREPO_ROOT_DIR=$HOME/.local/share/seqrepo/ docker-compose -f $(pwd)/docs/setup_help/docker-compose-examples/seqrepo-copy-local.compose.yaml up +``` From 85935045544a153946822393bc5782b0f5dff779 Mon Sep 17 00:00:00 2001 From: Kyle Ferriter Date: Sat, 29 Mar 2025 22:30:02 -0400 Subject: [PATCH 3/4] Rename docker-compose.yml to compose.yaml --- docker-compose.yml => compose.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker-compose.yml => compose.yaml (100%) diff --git a/docker-compose.yml b/compose.yaml similarity index 100% rename from docker-compose.yml rename to compose.yaml From 0ef819bba3196757d5032b086c1bede46164d70a Mon Sep 17 00:00:00 2001 From: Kyle Ferriter Date: Sat, 29 Mar 2025 22:36:06 -0400 Subject: [PATCH 4/4] Mount seqrepo copy source as read-only. Remove unused depends_on from seqrepo-mount-local.compose.yaml --- .../docker-compose-examples/seqrepo-copy-local.compose.yaml | 2 +- .../docker-compose-examples/seqrepo-mount-local.compose.yaml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/setup_help/docker-compose-examples/seqrepo-copy-local.compose.yaml b/docs/setup_help/docker-compose-examples/seqrepo-copy-local.compose.yaml index 86432e0e..8624951e 100644 --- a/docs/setup_help/docker-compose-examples/seqrepo-copy-local.compose.yaml +++ b/docs/setup_help/docker-compose-examples/seqrepo-copy-local.compose.yaml @@ -15,7 +15,7 @@ services: image: eeacms/rsync volumes: - seqrepo_vol:/usr/local/share/seqrepo - - $SEQREPO_ROOT_DIR:/seqrepo + - $SEQREPO_ROOT_DIR:/seqrepo:ro command: > /bin/sh -c "rsync -a --delete /seqrepo/2024-12-20/ /usr/local/share/seqrepo/2024-12-20/" diff --git a/docs/setup_help/docker-compose-examples/seqrepo-mount-local.compose.yaml b/docs/setup_help/docker-compose-examples/seqrepo-mount-local.compose.yaml index c01ccc15..2a6dd2ff 100644 --- a/docs/setup_help/docker-compose-examples/seqrepo-mount-local.compose.yaml +++ b/docs/setup_help/docker-compose-examples/seqrepo-mount-local.compose.yaml @@ -17,9 +17,6 @@ services: volumes: # Recommended to mount read-only to avoid accidentally modififying host files - $SEQREPO_ROOT_DIR:/usr/local/share/seqrepo:ro - depends_on: - seqrepo_local_populator: - condition: service_completed_successfully command: seqrepo-rest-service -w /usr/local/share/seqrepo/2024-12-20 ports: - 127.0.0.1:5001:5000