diff --git a/.env b/.env new file mode 100644 index 0000000000..6b414b86bb --- /dev/null +++ b/.env @@ -0,0 +1,9 @@ +ELASTIC_PASSWORD=elastic +KIBANA_PASSWORD=kibana +STACK_VERSION=8.17.2 +CLUSTER_NAME=docker-cluster +LICENSE=basic +ES_PORT=9200 +KIBANA_PORT=5601 +# Increase or decrease based on the available host memory (in bytes) +MEM_LIMIT=1073741824 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3b88843b46..827c3e57e2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ docker-compose.debug.override.yml bin/ target/ export/ +temp/ +set_java_17.sh # Scripts docker-env.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0fe0e3a87e..106ef1954f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,6 +10,7 @@ variables: DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE:$DOCKER_TAG DOCKER_PULL_SECRET: docker-registry-secret APP_NAME: $CI_PROJECT_NAME + K8S_NAMESPACE_PREFIX: biosamples before_script: - echo $CI_BUILD_REF @@ -17,177 +18,236 @@ before_script: - apk update && apk add git stages: -# - build + - build - package - config - deploy -maven-package: +maven-package-all-apps: image: ${CI_REGISTRY_IMAGE}/eclipse-temurin:17-jdk - stage: package + stage: build script: - - './mvnw -q deploy -P embl-ebi -s ci_settings.xml -DskipTests -Dmaven.source.skip=true' - - mkdir deployment - - cp webapps/core/target/webapps-core-*.war deployment/webapps-core.war - - cp webapps/core-v2/target/webapps-core-v2*.jar deployment/webapps-core-v2.jar - - cp agents/solr/target/agents-solr-*.jar deployment/agents-solr.jar - - cp agents/uploadworkers/target/agents-uploadworkers-*.jar deployment/agents-uploadworkers.jar - # - cp pipelines/curation/target/pipelines-curation-*.jar deployment/pipelines-curation.jar - # - cp pipelines/ena/target/pipelines-ena-*.jar deployment/pipelines-ena.jar - # - cp pipelines/ncbi-ena-link/target/pipelines-ncbi-ena-link-*.jar deployment/pipelines-ncbi-ena-link.jar - - cp pipelines/sample-release/target/pipelines-sample-release-*.jar deployment/pipelines-sample-release.jar - # - cp pipelines/sample-post-release-action/target/pipelines-sample-post-release-action*.jar deployment/pipelines-sample-post-release-action.jar - - cp pipelines/ncbi/target/pipelines-ncbi-*.jar deployment/pipelines-ncbi.jar - - cp pipelines/reindex/target/pipelines-reindex-*.jar deployment/pipelines-reindex.jar - - cp pipelines/sample-transformation-dtol/target/pipelines-sample-transformation-dtol-*.jar deployment/pipelines-sample-transformation-dtol.jar + - './mvnw -q package -P embl-ebi -s ci_settings.xml -DskipTests -Dmaven.source.skip=true' artifacts: paths: - - deployment - -#maven-package-webapps-core: -# image: ${CI_REGISTRY_IMAGE}/eclipse-temurin:17-jdk -# stage: build -# script: -# - './mvnw -q deploy -pl webapps/core -am -P embl-ebi -s ci_settings.xml -DskipTests -Dmaven.source.skip=true' -# artifacts: -# paths: -# - webapps/core/target/webapps-core-*.war + - webapps/core/target/webapps-core-*.war + - webapps/core-v2/target/webapps-core-v2-*.jar + - agents/uploadworkers/target/agents-uploadworkers-*.jar + - pipelines/reindex/target/pipelines-reindex-*.jar + - pipelines/curation/target/pipelines-curation-*.jar + - pipelines/curami/target/pipelines-curami-*.jar + - pipelines/copydown/target/pipelines-copydown-*.jar -#build_docker_image: -# stage: package -# image: docker:stable -# services: -# - docker:stable-dind -# before_script: -# - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY" -# script: -# - docker build --build-arg DOCKER_REGISTRY=${CI_REGISTRY_IMAGE} -t $DOCKER_IMAGE_NAME -f webapps/core/Dockerfile . -# - docker push $DOCKER_IMAGE_NAME -# after_script: -# - docker logout ${CI_REGISTRY_IMAGE} - -#deploy-dev-bsd-v1: -# image: dtzar/helm-kubectl:3.11.0 -# stage: deploy -# script: -# - cd webapps/core -# - sed -i "s|%DOCKER_IMAGE%|$DOCKER_IMAGE_NAME|g" core-deployment.yaml -# - kubectl config set-cluster bsd-cluster --server="${K8_HL_SERVER}" -# - kubectl config set clusters.bsd-cluster.certificate-authority-data ${K8_HL_CERTIFICATE_AUTHORITY_DATA} -# - kubectl config set-credentials bsd-user --token="${K8_HL_CREDENTIALS}" -# - kubectl config set-context bsd-context --cluster=bsd-cluster --user=bsd-user -# - kubectl config use-context bsd-context -# - kubectl apply -f core-deployment.yaml --namespace=biosamples-dev -# - kubectl apply -f core-service.yaml --namespace=biosamples-dev -# when: manual +build_and_push_docker_images: + stage: package + image: docker:stable + services: + - docker:stable-dind + before_script: + - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY" + parallel: + matrix: + - APP_NAME: "webapp-core" + DOCKERFILE_TARGET: "webapp-core" + - APP_NAME: "webapp-core-v2" + DOCKERFILE_TARGET: "webapp-core-v2" + - APP_NAME: "agents-uploadworkers" + DOCKERFILE_TARGET: "agents-uploadworkers" + - APP_NAME: "pipelines-reindex" + DOCKERFILE_TARGET: "pipelines-reindex" + - APP_NAME: "pipelines-curation" + DOCKERFILE_TARGET: "pipelines-curation" + - APP_NAME: "pipelines-curami" + DOCKERFILE_TARGET: "pipelines-curami" + - APP_NAME: "pipelines-copydown" + DOCKERFILE_TARGET: "pipelines-copydown" + script: + - | + DOCKER_IMAGE_NAME="$CI_REGISTRY_IMAGE/$APP_NAME:$DOCKER_TAG" + echo "Building and pushing $DOCKER_IMAGE_NAME with target $DOCKERFILE_TARGET" + docker build --target $DOCKERFILE_TARGET -t $DOCKER_IMAGE_NAME -f k8s/Dockerfile . + docker push $DOCKER_IMAGE_NAME + + # Also tag as latest for the current branch + DOCKER_LATEST_NAME="$CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_REF_SLUG-latest" + docker tag $DOCKER_IMAGE_NAME $DOCKER_LATEST_NAME + docker push $DOCKER_LATEST_NAME + echo $DOCKER_LATEST_NAME + after_script: + - docker logout $CI_REGISTRY -clone-config-dev: +clone-config: stage: config script: - git clone https://$BSD_INTERNAL_USER:$BSD_INTERNAL_PASS@gitlab.ebi.ac.uk/biosamples/biosamples-internal.git - mkdir config - - cp -r biosamples-internal/script/* deployment/ - - cp -r biosamples-internal/dev/* config/ - - cp biosamples-internal/deployment/deploy.sh ./ + - cp -r biosamples-internal/k8s/* config/ artifacts: paths: - - deployment - config - - deploy.sh only: - dev + - main + - biosamples-search -clone-config-preproduction: - stage: config - script: - - git clone https://$BSD_INTERNAL_USER:$BSD_INTERNAL_PASS@gitlab.ebi.ac.uk/biosamples/biosamples-internal.git - - mkdir config - - cp -r biosamples-internal/script/* deployment/ - - cp -r biosamples-internal/wwwdev/* config/ - - cp biosamples-internal/deployment/deploy.sh ./ - artifacts: - paths: - - deployment - - config - - deploy.sh +deploy_k8s_primary_dev: + variables: + ENVIRONMENT_NAME: primary_dev + K8S_NAMESPACE: biosamples-dev + environment: + name: primary_dev + url: https://wwwdev.ebi.ac.uk/biosamples + only: + - dev + - main + - biosamples-search when: manual + extends: .kube_deploy_script -clone-config-production: - stage: config - script: - - git clone https://$BSD_INTERNAL_USER:$BSD_INTERNAL_PASS@gitlab.ebi.ac.uk/biosamples/biosamples-internal.git - - mkdir config - - cp -r biosamples-internal/script/* deployment/ - - cp -r biosamples-internal/www/* config/ - - cp biosamples-internal/deployment/deploy.sh ./ - - cp biosamples-internal/deployment/deploy_agents_solr.sh ./ - artifacts: - paths: - - deployment - - config - - deploy.sh - - deploy_agents_solr.sh +deploy_k8s_primary_prod: + variables: + ENVIRONMENT_NAME: primary_prod + K8S_NAMESPACE: biosamples-prod + environment: + name: primary_prod + url: https://www.ebi.ac.uk/biosamples + only: + - dev + - main + - biosamples-search when: manual - rules: - - if: $CI_COMMIT_REF_NAME == "master" + extends: .kube_deploy_script -.deploy-template: &deploy-template - image: ${CI_REGISTRY_IMAGE}/ubuntu:latest - stage: deploy - script: - - chmod +x deploy.sh - - ./deploy.sh $BSD_NODE_NAME $BSD_HOST_NAME +deploy_k8s_fallback_prod: + variables: + ENVIRONMENT_NAME: fallback_prod + K8S_NAMESPACE: biosamples-prod + environment: + name: fallback_prod + url: https://www.ebi.ac.uk/biosamples + only: + - dev + - main + - biosamples-search when: manual - rules: - - if: $CI_COMMIT_REF_NAME == "master" + extends: .kube_deploy_script -deploy-dev: - <<: *deploy-template - rules: - - if: $CI_COMMIT_REF_NAME == "dev" +deploy_pipeline_k8s_primary_prod: variables: - BSD_NODE_NAME: bsd_dev - BSD_HOST_NAME: wp-np2-44 + ENVIRONMENT_NAME: primary_prod + K8S_NAMESPACE: biosamples-prod environment: - name: dev-${BSD_HOST_NAME} - url: http://${BSD_HOST_NAME}:8081/biosamples/ + name: primary_prod + url: https://wwwdev.ebi.ac.uk/biosamples + only: + - dev + - main + - biosamples-search + when: manual + extends: .kube_deploy_jobs_script -deploy-preproduction: - <<: *deploy-template - rules: - - if: $CI_COMMIT_REF_NAME == "refactoring_1_2024" - - if: $CI_COMMIT_REF_NAME == "dev" - - if: $CI_COMMIT_REF_NAME == "master" +deploy_pipeline_k8s_primary_dev: variables: - BSD_NODE_NAME: bsd_dev - BSD_HOST_NAME: wp-np2-40 + ENVIRONMENT_NAME: primary_dev + K8S_NAMESPACE: biosamples-dev environment: - name: dev-${BSD_HOST_NAME} - url: http://${BSD_HOST_NAME}:8081/biosamples/ + name: primary_dev + url: https://wwwdev.ebi.ac.uk/biosamples + only: + - dev + - main + - biosamples-search + when: manual + extends: .kube_deploy_jobs_script -deploy-production: - <<: *deploy-template - parallel: - matrix: - - BSD_NODE_NAME: bsd_prod - BSD_HOST_NAME: [ wp-p2m-40, wp-p2m-41, wp-p1m-40, wp-p1m-41 ] +deploy_cronjobs_k8s_primary_dev: + variables: + ENVIRONMENT_NAME: primary_dev + K8S_NAMESPACE: biosamples-dev environment: - name: prod-${BSD_HOST_NAME} - url: http://${BSD_HOST_NAME}:8081/biosamples/ + name: primary_dev + url: https://wwwdev.ebi.ac.uk/biosamples + only: + - dev + - main + - biosamples-search + when: manual + extends: .kube_deploy_cronjobs_script -deploy-production-solr: - image: ${CI_REGISTRY_IMAGE}/ubuntu:latest +.kube_deploy_script: stage: deploy + image: dtzar/helm-kubectl:3.16 + tags: ["dind"] + services: + - docker:27-dind script: - - chmod +x deploy.sh - - ./deploy_agents_solr.sh ${BSD_NODE_NAME} ${BSD_SOLR_HOST_NAME} - when: manual - rules: - - if: '$BSD_NODE_NAME == "bsd_prod" && $CI_COMMIT_REF_NAME == "master"' - parallel: - matrix: - - BSD_NODE_NAME: bsd_prod - BSD_SOLR_HOST_NAME: [ wp-p1m-42, wp-p2m-42 ] - environment: - name: prod-solr-${BSD_HOST_NAME} - url: http://${BSD_SOLR_HOST_NAME}.ebi.ac.uk:8983/solr + - echo $K8S_NAMESPACE + - kubectl config set-context --current --namespace=${K8S_NAMESPACE} + - | + kubectl create configmap application-properties \ + --from-file=./config/${ENVIRONMENT_NAME}/application.properties \ + --dry-run=client -o yaml | kubectl apply -f - + - kubectl delete secret $DOCKER_PULL_SECRET || true + - | + kubectl create secret docker-registry $DOCKER_PULL_SECRET \ + --docker-server=$CI_REGISTRY \ + --docker-username=$CI_REGISTRY_USER \ + --docker-password=$CI_REGISTRY_PASSWORD + - | + helm upgrade --install $APP_NAME ./k8s/helm \ + --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ + --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ + --set image.repository=$CI_REGISTRY_IMAGE/webapp-core \ + --set image.tag=$DOCKER_TAG \ + --set biosamples.context.path=/biosamples + - | + helm upgrade --install ${APP_NAME}-v2 ./k8s/helm \ + --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ + --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ + --set image.repository=$CI_REGISTRY_IMAGE/webapp-core-v2 \ + --set image.tag=$DOCKER_TAG \ + --set biosamples.context.path=/biosamples/v2 + - | + helm upgrade --install ${APP_NAME}-uploadworkers ./k8s/helm \ + --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ + --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ + --set image.repository=$CI_REGISTRY_IMAGE/agents-uploadworkers \ + --set image.tag=$DOCKER_TAG + +.kube_deploy_jobs_script: + stage: deploy + image: dtzar/helm-kubectl:3.16 + tags: [ "dind" ] + services: + - docker:27-dind + script: + - echo $K8S_NAMESPACE + - kubectl config set-context --current --namespace=${K8S_NAMESPACE} + - kubectl delete secret $DOCKER_PULL_SECRET || true + - | + kubectl create secret docker-registry $DOCKER_PULL_SECRET \ + --docker-server=$CI_REGISTRY \ + --docker-username=$CI_REGISTRY_USER \ + --docker-password=$CI_REGISTRY_PASSWORD + - | + helm upgrade --install biosamples-reindex ./k8s/jobs \ + --values ./k8s/jobs/values-${ENVIRONMENT_NAME}.yaml \ + --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ + --set image.repository=$CI_REGISTRY_IMAGE/pipelines-reindex \ + --set image.tag=$DOCKER_TAG + +.kube_deploy_cronjobs_script: + stage: deploy + image: dtzar/helm-kubectl:3.16 + tags: [ "dind" ] + services: + - docker:27-dind + script: + - echo $K8S_NAMESPACE + - kubectl config set-context --current --namespace=${K8S_NAMESPACE} + - | + helm upgrade --install biosamples-pipeline ./k8s/cronjobs \ + --values ./k8s/cronjobs/values-${ENVIRONMENT_NAME}.yaml \ + --set "imagePullSecrets[0].name=gitlab-argo-secret" \ + --set image.repository=$CI_REGISTRY_IMAGE \ + --set image.tag=$DOCKER_TAG diff --git a/README.adoc b/README.adoc index 3340abe54e..9d8152747f 100644 --- a/README.adoc +++ b/README.adoc @@ -63,6 +63,12 @@ git clone https://github.com/EBIBioSamples/biosamples-v4.git cd biosamples-v4 ./mvnw -T 2C package ---- +. Clone and build `biosamples-search` image ++ +[source,sh] +---- +./build-biosamples-search-image.sh +---- . Start BioSamples on your machine + [source,sh] diff --git a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/Application.java b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/Application.java index 52acfb017b..d2c20d4b0b 100644 --- a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -12,11 +12,9 @@ import org.apache.http.HeaderElement; import org.apache.http.HeaderElementIterator; -import org.apache.http.HttpHost; import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; import org.apache.http.conn.ConnectionKeepAliveStrategy; -import org.apache.http.conn.routing.HttpRoute; import org.apache.http.impl.client.cache.CacheConfig; import org.apache.http.impl.client.cache.CachingHttpClientBuilder; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; @@ -68,8 +66,8 @@ public MongoAccessionService mongoSampleAccessionService( mongoOperations); } - - // todo I Had to add restTemplate bean as a temporary workaround as there seems to be a problem with dependencies after refactor. + // todo I Had to add restTemplate bean as a temporary workaround as there seems to be a problem + // with dependencies after refactor. // We need to sort out dependency problem and remove this unused dependency. @Bean @@ -80,12 +78,11 @@ public RestTemplate restTemplate(final RestTemplateCustomizer restTemplateCustom } @Bean - public RestTemplateCustomizer restTemplateCustomizer(final BioSamplesProperties bioSamplesProperties) { + public RestTemplateCustomizer restTemplateCustomizer( + final BioSamplesProperties bioSamplesProperties) { return restTemplate -> { - final ConnectionKeepAliveStrategy keepAliveStrategy = (response, context) -> { - final HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); while (it.hasNext()) { @@ -115,8 +112,7 @@ public RestTemplateCustomizer restTemplateCustomizer(final BioSamplesProperties final RequestConfig config = RequestConfig.custom() .setConnectTimeout(timeout * 1000) - .setConnectionRequestTimeout( - timeout * 1000) + .setConnectionRequestTimeout(timeout * 1000) .setSocketTimeout(timeout * 1000) .build(); final HttpClient httpClient = diff --git a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java index aa59fb0797..8c41cfe6ab 100644 --- a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java +++ b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java @@ -62,14 +62,18 @@ public void receiveMessageFromBioSamplesFileUploaderQueue(final String mongoFile handleMessage(mongoFileId); } - private void handleMessage(final String submissionId) { + private void handleMessage(String submissionId) { + submissionId = submissionId.replace("\"", ""); final Optional fileUploadOptional = mongoFileUploadRepository.findById(submissionId); - final MongoFileUpload mongoFileUpload = - fileUploadOptional.orElseThrow( - () -> - new GlobalExceptions.UploadInvalidException( - "Could not find file upload record for submissionId: " + submissionId)); + if (fileUploadOptional.isEmpty()) { + log.error("Could not find file upload record for submissionId: {}", submissionId); + // todo here exception means there is no progress from the queue reading loop. + // We can send this to dead letter or something for monitoring. + return; + } + + final MongoFileUpload mongoFileUpload = fileUploadOptional.get(); try { validationResult = new ValidationResult(); @@ -311,6 +315,7 @@ private Sample buildAndPersistSample( } catch (final Exception e) { persisted = false; handleUnauthorizedWhilePersistence(sampleName, accession, sampleWithAccession, e); + throw new GlobalExceptions.SampleValidationException(e.getMessage()); } if (sampleWithAccession && persisted) { diff --git a/build-biosamples-search-image.sh b/build-biosamples-search-image.sh new file mode 100755 index 0000000000..6389ed7497 --- /dev/null +++ b/build-biosamples-search-image.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Clones EBIBioSamples/biosamples-search, builds with Gradle (Java 24), and builds a Docker image. + +set -euo pipefail + +# Configuration (override via environment variables if needed) +REPO_URL="${REPO_URL:-https://github.com/EBIBioSamples/biosamples-search}" +CLONE_DIR="${CLONE_DIR:-./temp/biosamples-search}" +IMAGE_TAG="${IMAGE_TAG:-biosamples-search:latest}" +DOCKER_CONTEXT="${DOCKER_CONTEXT:-}" # If empty, auto-detect a Dockerfile +GRADLE_TASKS="${GRADLE_TASKS:-build}" # e.g., assemble, build +GRADLE_ARGS="${GRADLE_ARGS:---no-daemon -x test}" # add/remove -x test as needed +USE_DOCKER_GRADLE="${USE_DOCKER_GRADLE:-1}" # 1 = use Dockerized Gradle (JDK 24), 0 = use local Gradle +GRADLE_DOCKER_IMAGE="${GRADLE_DOCKER_IMAGE:-gradle:8.14-jdk24}" # Gradle with JDK 24 + +# Checks +command -v git >/dev/null 2>&1 || { echo "git is required"; exit 1; } +command -v docker >/dev/null 2>&1 || { echo "docker is required"; exit 1; } + +echo "[1/3] Cloning repository: $REPO_URL" +if [ -d "$CLONE_DIR" ]; then + echo "Directory '$CLONE_DIR' already exists. Using it." +else + git clone --depth 1 "$REPO_URL" "$CLONE_DIR" +fi + +cd "$CLONE_DIR" + + +echo "[2/3] Detecting Dockerfile" +if [ -n "${DOCKER_CONTEXT}" ] && [ -f "${DOCKER_CONTEXT}/Dockerfile" ]; then + CONTEXT_DIR="${DOCKER_CONTEXT}" +else + DETECTED=$(find . -maxdepth 3 -type f -name Dockerfile | head -n 1 || true) + if [ -n "$DETECTED" ]; then + CONTEXT_DIR="$(dirname "$DETECTED")" + else + echo "No Dockerfile found. Set DOCKER_CONTEXT to the directory that contains a Dockerfile." + exit 1 + fi +fi + +echo "[3/3] Building Docker image: ${IMAGE_TAG}" +# Check if buildx is available, otherwise use regular docker build +if docker buildx version >/dev/null 2>&1; then + echo "Using BuildKit (buildx available)" + DOCKER_BUILDKIT=1 docker build -t "${IMAGE_TAG}" "${CONTEXT_DIR}" +else + echo "BuildKit not available, using standard docker build" + docker build -t "${IMAGE_TAG}" "${CONTEXT_DIR}" +fi + +echo "${IMAGE_TAG} built successfully" + +docker compose up -d elastic + +cd - +rm -rf ./temp/biosamples-search \ No newline at end of file diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/ClientProperties.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/ClientProperties.java index 840e904c11..ce00f52024 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/ClientProperties.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/ClientProperties.java @@ -100,13 +100,10 @@ public class ClientProperties { @Value("${biosamples.webapp.core.facet.cache.maxage:86400}") private int webappCoreFacetCacheMaxAge; - @Value("${biosamples.schema.validator.uri:http://localhost:8085/validate}") - private URI biosamplesSchemaValidatorServiceUri; - - @Value("${biosamples.schemaValidator:http://localhost:3020/validate}") + @Value("${biosamples.schema.validator.url:http://localhost:3020/validate}") private String schemaValidator; - @Value("${biosamples.schemaStore:http://localhost:8085}") + @Value("${biosamples.schema.store.url:http://localhost:8085}") private String schemaStore; @Value("${biosamples.schema.default:BSDC00001}") diff --git a/core/pom.xml b/core/pom.xml index ad2e8b2111..9411337254 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -53,7 +53,6 @@ org.springframework.boot spring-boot-starter-test - 1.5.15.RELEASE test diff --git a/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java b/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java index 315c3e48ea..abae07ab0a 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java @@ -11,10 +11,10 @@ package uk.ac.ebi.biosamples.messaging; public class MessagingConstants { - public static final String INDEXING_EXCHANGE = "biosamples.forindexing.solr"; - public static final String INDEXING_QUEUE = "biosamples.tobeindexed.solr"; - public static final String REINDEXING_EXCHANGE = "biosamples.reindex.solr"; - public static final String REINDEXING_QUEUE = "biosamples.reindex.solr"; - public static final String UPLOAD_QUEUE = "biosamples.uploaded.files"; + public static final String INDEXING_EXCHANGE = "biosamples.indexing"; + public static final String INDEXING_QUEUE = "biosamples.indexing.es"; + public static final String REINDEXING_QUEUE = "biosamples.reindexing.es"; + public static final String UPLOAD_EXCHANGE = "biosamples.uploaded.files.exchange"; + public static final String UPLOAD_QUEUE = "biosamples.uploaded.files"; } diff --git a/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java b/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java index e9d592c412..b65b8e3f3c 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java @@ -12,8 +12,6 @@ import org.springframework.amqp.core.*; import org.springframework.amqp.rabbit.annotation.EnableRabbit; -import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter; -import org.springframework.amqp.support.converter.MessageConverter; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import uk.ac.ebi.biosamples.messaging.MessagingConstants; @@ -45,13 +43,6 @@ public Exchange indexingExchange() { .build(); } - @Bean(name = "reindexingExchange") - public Exchange reindexingExchange() { - return ExchangeBuilder.directExchange(MessagingConstants.REINDEXING_EXCHANGE) - .durable(true) - .build(); - } - @Bean(name = "uploadExchange") public Exchange uploadExchange() { return ExchangeBuilder.fanoutExchange(MessagingConstants.UPLOAD_EXCHANGE).durable(true).build(); @@ -69,7 +60,7 @@ public Binding indexBinding() { @Bean(name = "reindexingBinding") public Binding reindexBinding() { return BindingBuilder.bind(reindexingQueue()) - .to(reindexingExchange()) + .to(indexingExchange()) .with(MessagingConstants.REINDEXING_QUEUE) .noargs(); } @@ -84,8 +75,8 @@ public Binding uploadBinding() { // enable messaging in json // note that this class is not the same as the http MessageConverter class - @Bean - public MessageConverter getJackson2MessageConverter() { - return new Jackson2JsonMessageConverter(); - } + // @Bean + // public MessageConverter getJackson2MessageConverter() { + // return new Jackson2JsonMessageConverter(); + // } } diff --git a/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java b/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java index 958a4c6edc..b92ae0bf67 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java @@ -10,6 +10,8 @@ */ package uk.ac.ebi.biosamples.service; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -24,7 +26,6 @@ import uk.ac.ebi.biosamples.core.model.Relationship; import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.messaging.MessagingConstants; -import uk.ac.ebi.biosamples.messaging.model.MessageContent; import uk.ac.ebi.biosamples.mongo.service.SampleReadService; @Service @@ -32,11 +33,13 @@ public class MessagingService { private final Logger log = LoggerFactory.getLogger(getClass()); private final SampleReadService sampleReadService; private final AmqpTemplate amqpTemplate; + private final ObjectMapper objectMapper; public MessagingService( - final SampleReadService sampleReadService, final AmqpTemplate amqpTemplate) { + SampleReadService sampleReadService, AmqpTemplate amqpTemplate, ObjectMapper objectMapper) { this.sampleReadService = sampleReadService; this.amqpTemplate = amqpTemplate; + this.objectMapper = objectMapper; } public void sendFileUploadedMessage(final String fileId) { @@ -70,10 +73,33 @@ void fetchThenSendMessage( updateInverseRelationships(sample.get(), existingRelationshipTargets); // send the original sample with the extras as related samples - amqpTemplate.convertAndSend( - MessagingConstants.INDEXING_EXCHANGE, - MessagingConstants.INDEXING_QUEUE, - MessageContent.build(sample.get(), null, related, false)); + // amqpTemplate.convertAndSend( + // MessagingConstants.INDEXING_EXCHANGE, + // MessagingConstants.INDEXING_QUEUE, + // MessageContent.build(sample.get(), null, related, false)); + + try { + String json = objectMapper.writeValueAsString(sample.get()); + log.info("Sending message for indexing: {}", sample.get().getAccession()); + // amqpTemplate.send(MessagingConstants.INDEXING_EXCHANGE, + // MessagingConstants.INDEXING_QUEUE, new Message(json.getBytes(StandardCharsets.UTF_8))); + amqpTemplate.convertAndSend( + MessagingConstants.INDEXING_EXCHANGE, MessagingConstants.INDEXING_QUEUE, json); + related.forEach( + s -> { + try { + amqpTemplate.convertAndSend( + MessagingConstants.INDEXING_EXCHANGE, + MessagingConstants.INDEXING_QUEUE, + objectMapper.writeValueAsString(s)); + } catch (JsonProcessingException e) { + log.error("Failed to convert sample to JSON: {}", s.getAccession(), e); + // throw new RuntimeException(e); + } + }); + } catch (Exception e) { + log.error("Failed to convert sample to JSON: {}", sample.get().getAccession(), e); + } } } diff --git a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java index 7ceecb839a..d3828ed4f6 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java @@ -75,7 +75,7 @@ public Optional> getCompatibleFilters( * @param filters a collection of filters * @return the corresponding filter query */ - List getFilterQuery(final Collection filters) { + public List getFilterQuery(final Collection filters) { if (filters == null || filters.size() == 0) { return Collections.emptyList(); } @@ -126,7 +126,7 @@ List getFilterQuery(final Collection filters) { * * @return a filter query for public and domain relevant samples */ - Optional getPublicFilterQuery(final String webinSubmissionAccountId) { + public Optional getPublicFilterQuery(final String webinSubmissionAccountId) { // check if this is a read superuser if (webinSubmissionAccountId != null && webinSubmissionAccountId.equalsIgnoreCase( diff --git a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java index 38780c569f..f3e3abc6dc 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java @@ -60,7 +60,10 @@ public Page fetchSolrSampleByText( // solr degrades with high page and size params, use cursor instead if (pageable.getPageNumber() > 500 || pageable.getPageSize() > 200) { - log.warn("Max page number/size exceeded, returning error to the user. Number: {}, size: {}", pageable.getPageNumber(), pageable.getPageSize()); + log.warn( + "Max page number/size exceeded, returning error to the user. Number: {}, size: {}", + pageable.getPageNumber(), + pageable.getPageSize()); throw new GlobalExceptions.PaginationException(); } diff --git a/docker-compose.yml b/docker-compose.yml index ca2986349a..b4a20ccca8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2' services: biosamples-webapps-core: @@ -29,8 +28,9 @@ services: - spring.data.mongodb.auto-index-creation=true - biosamples.mongo.sample.writeConcern=1 - BIOSAMPLES_NEO_URL=bolt://neo4j:7687 - - biosamples.schemaValidator=http://json-schema-validator:3020/validate - - biosamples.schemaStore=https://wwwdev.ebi.ac.uk/biosamples/schema-store + - biosamples.schema.validator.url=http://json-schema-validator:3020/validate + - biosamples.schema.store.url=https://wwwdev.ebi.ac.uk/biosamples/schema-store + - biosamples.search.host=biosamples-search - SPRING_RABBITMQ_HOST=rabbitmq - SPRING_RABBITMQ_PUBLISHER-CONFIRMS=true - SPRING_RABBITMQ_PUBLISHER-RETURNS=true @@ -88,7 +88,6 @@ services: - GOOGLE_APPLICATION_CREDENTIALS=/secrets/prj-int-dev-omics-apps-mon-1fba094e79b2.json ports: - 8081:8080 - - 9090:9090 - 8000:8000 biosamples-webapps-core-v2: @@ -120,8 +119,8 @@ services: - SPRING_RABBITMQ_LISTENER_SIMPLE_PREFETCH=100 - SPRING_RABBITMQ_LISTENER_SIMPLE_TRANSACTION-SIZE=25 - BIOSAMPLES_NEO_URL=bolt://neo4j:7687 - - biosamples.schemaValidator=http://json-schema-validator:3020/validate - - biosamples.schemaStore=https://wwwdev.ebi.ac.uk/biosamples/schema-store + - biosamples.schema.validator.url=http://json-schema-validator:3020/validate + - biosamples.schema.store.url=https://wwwdev.ebi.ac.uk/biosamples/schema-store - spring.jackson.serialization-inclusion=non_null - spring.jackson.serialization.WRITE_NULL_MAP_VALUES=false - spring.jackson.serialization.indent_output=true @@ -359,6 +358,7 @@ services: links: - biosamples-webapps-core - neo4j + - mongo command: - java - -jar @@ -369,6 +369,7 @@ services: - LOGGING_FILE=/logs/integration.log - BIOSAMPLES_LEGACYAPIKEY=fooqwerty - BIOSAMPLES_NEO_URL=bolt://neo4j:7687 + - spring.data.mongodb.uri=mongodb://mongo:27017/biosamples json-schema-validator: image: quay.io/ebi-ait/biovalidator:2.0.1 @@ -406,7 +407,7 @@ services: - rabbitmq_data:/var/lib/rabbitmq/mnesia solr: - image: solr:8.11.2-slim + image: solr:8.11.2 mem_limit: 1g ports: - 8983:8983 @@ -434,6 +435,70 @@ services: - neo_import:/import - logs:/logs + + biosamples-search: + image: biosamples-search:latest + ports: + - "8083:8080" + - "9090:9090" + environment: + - spring.elasticsearch.username=elastic + - spring.elasticsearch.password=elastic + - spring.elasticsearch.uris=http://elastic:9200 + - spring.rabbitmq.host=rabbitmq + links: + - elastic + elastic-setup: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + user: "0" + command: > + bash -c ' + if [ x${ELASTIC_PASSWORD} == x ]; then + echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; + exit 1; + elif [ x${KIBANA_PASSWORD} == x ]; then + echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + exit 1; + fi; + echo "Waiting for Elasticsearch availability"; + until curl -s http://elastic:9200 | grep -q "missing authentication credentials"; do sleep 30; done; + echo "Setting kibana_system password"; + until curl -s -X POST -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" http://elastic:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "All done!"; + ' + healthcheck: + test: [ "CMD-SHELL", "[ -f config/certs/es01/es01.crt ]" ] + interval: 1s + timeout: 5s + retries: 120 + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - es_data1:/usr/share/elasticsearch/data + ports: + - ${ES_PORT}:9200 + environment: + - node.name=elastic + - discovery.type=single-node + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - xpack.security.http.ssl.enabled=false + mem_limit: 1g + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s http://localhost:9200 | grep -q 'missing authentication credentials'", + ] + interval: 10s + timeout: 10s + retries: 120 + + volumes: solr_data: null mongo_data: null @@ -442,3 +507,7 @@ volumes: neo_plugins: null neo_data: null neo_import: null + es_data1: + driver: local + kibana_data: + driver: local diff --git a/docker-debug.sh b/docker-debug.sh index 6f1229a1df..79759ad1fd 100755 --- a/docker-debug.sh +++ b/docker-debug.sh @@ -41,7 +41,8 @@ echo "checking mongo is up" curl http://localhost:8983/solr/samples/config -H 'Content-type:application/json' -d'{"set-property" : {"updateHandler.autoCommit.maxTime":5000, "updateHandler.autoCommit.openSearcher":"true", "query.documentCache.size":1024, "query.filterCache.size":1024, "query.filterCache.autowarmCount":128, "query.queryResultCache.size":1024, "query.queryResultCache.autowarmCount":128}}' #profile any queries that take longer than 100 ms -docker-compose run --rm mongo mongo --eval 'db.setProfilingLevel(1)' mongo:27017/biosamples +#don't use run, spins up a new container, use eval to use existing container +docker-compose exec mongo mongo biosamples --eval 'db.setProfilingLevel(1)' docker-compose -f docker-compose.yml -f docker-compose.debug.override.yml -f docker-compose.override.yml up -d biosamples-webapps-core diff --git a/docker-integration.sh b/docker-integration.sh index 5445dfea10..1a67f2b9c2 100755 --- a/docker-integration.sh +++ b/docker-integration.sh @@ -1,25 +1,25 @@ -#!/bin/bash -set -e - -./docker-webapp.sh --clean - - -docker-compose up -d biosamples-agents-solr -docker-compose up -d biosamples-agents-upload-workers - -#ARGS=--spring.profiles.active=big -for X in 1 2 3 4 5 6 -do - echo "============================================================================================================" - echo "=================================== STARTING INTEGRATION TESTS PHASE-"$X "=====================================" - echo "============================================================================================================" - #java -jar integration/target/integration-4.0.0-SNAPSHOT.jar --phase=$X $ARGS $@ - docker-compose run --rm --service-ports biosamples-integration java -jar integration-5.3.15-SNAPSHOT.jar --phase=$X $ARGS $@ - sleep 10 #solr is configured to commit every 5 seconds - -done - -#leave the agent up at the end -docker-compose up -d biosamples-agents-solr - -echo "Successfully completed" +#!/bin/bash +set -e + +./docker-webapp.sh --clean + + +#docker-compose up -d biosamples-agents-solr +docker-compose up -d biosamples-agents-upload-workers + +#ARGS=--spring.profiles.active=big +for X in 1 2 3 4 5 6 +do + echo "============================================================================================================" + echo "=================================== STARTING INTEGRATION TESTS PHASE-"$X "=====================================" + echo "============================================================================================================" + #java -jar integration/target/integration-4.0.0-SNAPSHOT.jar --phase=$X $ARGS $@ + docker-compose run --rm --service-ports biosamples-integration java -jar integration-5.3.15-SNAPSHOT.jar --phase=$X $ARGS $@ + sleep 10 #solr is configured to commit every 5 seconds + +done + +#leave the agent up at the end +#docker-compose up -d biosamples-agents-solr + +echo "Successfully completed" diff --git a/docker-webapp.sh b/docker-webapp.sh index c8b6990f8f..45220866ed 100755 --- a/docker-webapp.sh +++ b/docker-webapp.sh @@ -29,9 +29,9 @@ docker-compose build #start up the webapps (and dependencies) #docker-compose up -d --remove-orphans solr rabbitmq mongo neo4j json-schema-validator schema-store -docker-compose up -d --remove-orphans solr rabbitmq mongo neo4j json-schema-validator -echo "checking solr is up" -./http-status-check -u http://localhost:8983 -t 30 +docker-compose up -d --remove-orphans elastic rabbitmq mongo neo4j json-schema-validator +#echo "checking solr is up" +#./http-status-check -u http://localhost:9200 -t 30 echo "checking rabbitmq is up" ./http-status-check -u http://localhost:15672 -t 30 echo "checking mongo is up" @@ -45,14 +45,29 @@ echo "checking neo4j is up" #configure solr -curl http://localhost:8983/solr/samples/config -H 'Content-type:application/json' -d'{"set-property" : {"updateHandler.autoCommit.maxTime":1000, "updateHandler.autoCommit.openSearcher":"true", "updateHandler.autoSoftCommit.maxDocs":1, "query.documentCache.size":1024, "query.filterCache.size":1024, "query.filterCache.autowarmCount":128, "query.queryResultCache.size":1024, "query.queryResultCache.autowarmCount":128}}' +#curl http://localhost:8983/solr/samples/config -H 'Content-type:application/json' -d'{"set-property" : {"updateHandler.autoCommit.maxTime":1000, "updateHandler.autoCommit.openSearcher":"true", "updateHandler.autoSoftCommit.maxDocs":1, "query.documentCache.size":1024, "query.filterCache.size":1024, "query.filterCache.autowarmCount":128, "query.queryResultCache.size":1024, "query.queryResultCache.autowarmCount":128}}' #configure schema-store #profile any queries that take longer than 100 ms -docker-compose run --rm mongo mongo --eval 'db.setProfilingLevel(1)' mongo:27017/biosamples +#don't use run, spins up a new container, use eval to use existing container +docker-compose exec mongo mongo biosamples --eval 'db.setProfilingLevel(1)' +until curl -s http://localhost:9200 | grep -q "missing authentication credentials"; do sleep 30; done; +# create ES index +curl -X DELETE "http://localhost:9200/samples" -u "elastic:elastic" + +curl -X PUT "http://localhost:9200/samples" \ + -H "Content-Type: application/json" \ + -u "elastic:elastic" \ + --data-binary @es_index.json + + +docker-compose up -d biosamples-search +sleep 20 +echo "checking biosamples-search is up" +./http-status-check -u http://localhost:8083/actuator/health -t 600 docker-compose up -d biosamples-webapps-core sleep 40 diff --git a/es_index.json b/es_index.json new file mode 100644 index 0000000000..0bb7591cad --- /dev/null +++ b/es_index.json @@ -0,0 +1,194 @@ +{ +"settings": { + "number_of_shards": 1, + "number_of_replicas": 0 +}, +"mappings": { + "properties": { + "sample_full_text": { + "type": "text" + }, + "_class": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "accession": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "characteristics": { + "type": "nested", + "properties": { + "key": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "value": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "create": { + "type": "date" + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "externalReferences": { + "type": "nested", + "properties": { + "url": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "name": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "publications": { + "type": "nested", + "properties": { + "pubmed_id": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "relationships": { + "type": "nested", + "properties": { + "source": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "target": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "type": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "release": { + "type": "date" + }, + "sraAccession": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "status": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "submitted": { + "type": "date" + }, + "submittedVia": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "taxId": { + "type": "long", + "copy_to": "sample_full_text" + }, + "update": { + "type": "date" + }, + "webinSubmissionAccountId": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } +} +} diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/FileDownloadIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/FileDownloadIntegration.java index d5b9e81a40..9b0945f946 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/FileDownloadIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/FileDownloadIntegration.java @@ -65,8 +65,7 @@ protected void phaseThree() { throw new IntegrationTestFailException("Invalid format in samples.json", Phase.THREE); } } catch (final IOException e) { - // TODO: @Isuru to check please! - /*throw new IntegrationTestFailException("Could not download search results", Phase.THREE);*/ + throw new IntegrationTestFailException("Could not download search results", Phase.THREE); } } diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java index 3e0a265118..832eca5a5f 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java @@ -276,13 +276,37 @@ private Sample getArrayExpressSampleTest() { private void facetEndpointShouldReturnAllFacetValuesWhenFacetFilterIsAvailable() { String url = clientProperties.getBiosamplesClientUri() + "/samples/facets?facet=SRA accession"; + log.info("Calling facet endpoint URL: {}", url); try { ResponseEntity response = restTemplate.getForEntity(url, String.class); + log.info("Response status: {}", response.getStatusCode()); + log.info("Response body: {}", response.getBody()); + JsonNode node = objectMapper.readTree(response.getBody()); - JsonNode facets = node.get("_embedded").get("facets"); + JsonNode embedded = node.get("_embedded"); + if (embedded == null) { + log.error( + "Response does not contain '_embedded' field. Full response: {}", response.getBody()); + throw new IntegrationTestFailException( + "Facet endpoint response does not contain '_embedded' field. Response: " + + response.getBody(), + Phase.SIX); + } + + JsonNode facets = embedded.get("facets"); + if (facets == null) { + log.error( + "Response does not contain 'facets' field. '_embedded' content: {}", + embedded.toString()); + throw new IntegrationTestFailException( + "Facet endpoint response does not contain 'facets' field. Response: " + + response.getBody(), + Phase.SIX); + } + for (JsonNode facet : facets) { if ("SRA accession".equals(facet.get("label").asText())) { - if (facet.get("content").size() <= 10) { + if (facet.get("content").size() < 10) { throw new IntegrationTestFailException( "Facet count should be larger than 10 when facet filters are being used", Phase.SIX); diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java index fff3907823..81ac2d654f 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java @@ -14,6 +14,7 @@ import java.util.Optional; import java.util.SortedSet; import java.util.TreeSet; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; @@ -21,6 +22,7 @@ import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component +@Slf4j public class RestPrivateSampleIntegration extends AbstractIntegration { private final ClientProperties clientProperties; @@ -95,7 +97,7 @@ protected void phaseSix() {} private Sample getSampleWithReleaseDateToday() { final String name = "RestPrivateSampleIntegration_sample_1"; - final Instant release = Instant.now(); + final Instant release = Instant.now().minusSeconds(3600); final SortedSet attributes = new TreeSet<>(); attributes.add(Attribute.build("description", "Fake sample with today(now) release date")); diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/SitemapIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/SitemapIntegration.java index 7e9d8db3c4..8178cd32cd 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/SitemapIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/SitemapIntegration.java @@ -63,7 +63,7 @@ protected void phaseTwo() { lookupTable.put(Objects.requireNonNull(sample.getContent()).getAccession(), Boolean.FALSE); } - if (samples.size() <= 0) { + if (samples.isEmpty()) { throw new RuntimeException("No search results found!"); } diff --git a/k8s/Dockerfile b/k8s/Dockerfile new file mode 100644 index 0000000000..cc21ef98de --- /dev/null +++ b/k8s/Dockerfile @@ -0,0 +1,33 @@ +FROM eclipse-temurin:17-jdk-alpine as base +WORKDIR /app + +FROM base as webapp-core +COPY webapps/core/target/webapps-core-*.war app.war +EXPOSE 8080 +CMD ["java", "-jar", "app.war"] + +FROM base as webapp-core-v2 +COPY webapps/core-v2/target/webapps-core-*.jar app.jar +EXPOSE 8080 +CMD ["java", "-jar", "app.jar"] + +FROM base as agents-uploadworkers +COPY agents/uploadworkers/target/agents-uploadworkers-*.jar app.jar +EXPOSE 8080 +CMD ["java", "-jar", "app.jar"] + +FROM base as pipelines-reindex +COPY pipelines/reindex/target/pipelines-reindex-*.jar app.jar +CMD ["java", "-jar", "app.jar"] + +FROM base as pipelines-curation +COPY pipelines/curation/target/pipelines-curation-*.jar app.jar +CMD ["java", "-jar", "app.jar"] + +FROM base as pipelines-curami +COPY pipelines/curami/target/pipelines-curami-*.jar app.jar +CMD ["java", "-jar", "app.jar"] + +FROM base as pipelines-copydown +COPY pipelines/copydown/target/pipelines-copydown-*.jar app.jar +CMD ["java", "-jar", "app.jar"] diff --git a/k8s/cronjobs/Chart.yaml b/k8s/cronjobs/Chart.yaml new file mode 100644 index 0000000000..1322d750ef --- /dev/null +++ b/k8s/cronjobs/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: cronjobs +description: Argo workflow dag for pipelines +type: application +version: 0.1.0 +appVersion: "1.0" diff --git a/k8s/cronjobs/templates/_helpers.tpl b/k8s/cronjobs/templates/_helpers.tpl new file mode 100644 index 0000000000..57677a38f7 --- /dev/null +++ b/k8s/cronjobs/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cronjobs.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cronjobs.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cronjobs.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cronjobs.labels" -}} +helm.sh/chart: {{ include "helm.chart" . }} +{{ include "cronjobs.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cronjobs.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cronjobs.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "cronjobs.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "cronjobs.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml new file mode 100644 index 0000000000..d74627d06b --- /dev/null +++ b/k8s/cronjobs/templates/workflow.yaml @@ -0,0 +1,92 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + name: {{ include "cronjobs.fullname" . }} + namespace: {{ .Values.workflow.namespace }} + labels: + app.kubernetes.io/name: {{ include "cronjobs.name" . }} +spec: + serviceAccountName: argo-workflow-sa + entrypoint: {{ .Values.job.name }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + parallelism: 2 + ttlStrategy: + secondsAfterSuccess: {{ .Values.workflow.ttlStrategy.secondsAfterSuccess }} + secondsAfterFailure: {{ .Values.workflow.ttlStrategy.secondsAfterFailure }} + + templates: + - name: {{ .Values.job.name }} + dag: + tasks: + - name: curation + template: curation + - name: curami + template: curami + dependencies: + - curation + - name: copydown + template: copydown + + - name: curation + retryStrategy: + limit: {{ .Values.workflow.retryStrategy.limit }} + container: + image: "{{ .Values.image.repository }}/pipelines-curation:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SPRING_CONFIG_LOCATION + value: /config/application.properties + command: + {{- toYaml .Values.job.command | nindent 10 }} + args: + {{- toYaml .Values.job.args | nindent 10 }} + resources: + {{- toYaml .Values.job.resources | nindent 10 }} + volumeMounts: + {{- toYaml .Values.volumeMounts | nindent 10 }} + volumes: + {{- toYaml .Values.volumes | nindent 8 }} + + - name: curami + retryStrategy: + limit: {{ .Values.workflow.retryStrategy.limit }} + container: + image: "{{ .Values.image.repository }}/pipelines-curami:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SPRING_CONFIG_LOCATION + value: /config/application.properties + command: + {{- toYaml .Values.job.command | nindent 10 }} + args: + {{- toYaml .Values.job.args | nindent 10 }} + resources: + {{- toYaml .Values.job.resources | nindent 10 }} + volumeMounts: + {{- toYaml .Values.volumeMounts | nindent 10 }} + volumes: + {{- toYaml .Values.volumes | nindent 8 }} + + - name: copydown + retryStrategy: + limit: {{ .Values.workflow.retryStrategy.limit }} + container: + image: "{{ .Values.image.repository }}/pipelines-copydown:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SPRING_CONFIG_LOCATION + value: /config/application.properties + command: + {{- toYaml .Values.job.command | nindent 10 }} + args: + {{- toYaml .Values.job.args | nindent 10 }} + resources: + {{- toYaml .Values.job.resources | nindent 10 }} + volumeMounts: + {{- toYaml .Values.volumeMounts | nindent 10 }} + volumes: + {{- toYaml .Values.volumes | nindent 8 }} + diff --git a/k8s/cronjobs/values-primary_dev.yaml b/k8s/cronjobs/values-primary_dev.yaml new file mode 100644 index 0000000000..156124fc83 --- /dev/null +++ b/k8s/cronjobs/values-primary_dev.yaml @@ -0,0 +1,2 @@ +rabbitmq: + host: wp-np2-40.ebi.ac.uk diff --git a/k8s/cronjobs/values.yaml b/k8s/cronjobs/values.yaml new file mode 100644 index 0000000000..02b61e3bc8 --- /dev/null +++ b/k8s/cronjobs/values.yaml @@ -0,0 +1,40 @@ +image: + repositoryBase: __set + repository: __set-in-commandline__ + pullPolicy: IfNotPresent + tag: latest +imagePullSecrets: + - name: __set-in-commandline__ + +job: + name: pipelines + command: ["java", "-jar", "/app/app.jar"] + args: [] + resources: + limits: + cpu: "500m" + memory: "1Gi" + requests: + cpu: "200m" + memory: "512Mi" + +workflow: + ttlStrategy: + secondsAfterSuccess: 3600 + secondsAfterFailure: 7200 + retryStrategy: + limit: 1 + backoff: + duration: "10s" + factor: 1 + maxDuration: "1m" + +volumes: + - name: config-volume + configMap: + name: application-properties + +volumeMounts: + - name: config-volume + mountPath: /config/application.properties + subPath: application.properties diff --git a/k8s/helm/.helmignore b/k8s/helm/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/k8s/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8s/helm/Chart.yaml b/k8s/helm/Chart.yaml new file mode 100644 index 0000000000..ec79288853 --- /dev/null +++ b/k8s/helm/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: helm +description: A Helm chart for Kubernetes +type: application +version: 0.1.0 +appVersion: "1.16.0" diff --git a/k8s/helm/templates/NOTES.txt b/k8s/helm/templates/NOTES.txt new file mode 100644 index 0000000000..62f1b422e2 --- /dev/null +++ b/k8s/helm/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "helm.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch its status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "helm.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "helm.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "helm.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/k8s/helm/templates/_helpers.tpl b/k8s/helm/templates/_helpers.tpl new file mode 100644 index 0000000000..ba04c300df --- /dev/null +++ b/k8s/helm/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "helm.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "helm.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "helm.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "helm.labels" -}} +helm.sh/chart: {{ include "helm.chart" . }} +{{ include "helm.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "helm.selectorLabels" -}} +app.kubernetes.io/name: {{ include "helm.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "helm.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "helm.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/k8s/helm/templates/deployment.yaml b/k8s/helm/templates/deployment.yaml new file mode 100644 index 0000000000..16711cb69a --- /dev/null +++ b/k8s/helm/templates/deployment.yaml @@ -0,0 +1,106 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "helm.fullname" . }} + labels: + {{- include "helm.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "helm.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "helm.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "helm.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SERVER_SERVLET_CONTEXT-PATH + value: {{ .Values.biosamples.context.path }} + - name: SERVER_CONTEXT-PATH + value: {{ .Values.biosamples.context.path }} + - name: BIOSAMPLES.SCHEMA.VALIDATOR.URL + value: {{ .Values.biosamples.schema.validator.url }} + - name: BIOSAMPLES.SCHEMA.STORE.URL + value: {{ .Values.biosamples.schema.store.url }} + - name: SPRING_CONFIG_LOCATION + value: /config/application.properties + - name: SPRING_DATA_MONGODB_URI + valueFrom: + secretKeyRef: + name: biosamples-mongodb + key: connection-string + - name: SPRING_RABBITMQ_HOST + value: {{ .Values.rabbitmq.host }} + - name: SPRING_RABBITMQ_USERNAME + valueFrom: + secretKeyRef: + name: rabbitmq-user + key: username + - name: SPRING_RABBITMQ_PASSWORD + valueFrom: + secretKeyRef: + name: rabbitmq-user + key: password + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP +{{/* {{- with .Values.livenessProbe }}*/}} +{{/* livenessProbe:*/}} +{{/* {{- toYaml . | nindent 12 }}*/}} +{{/* {{- end }}*/}} +{{/* {{- with .Values.readinessProbe }}*/}} +{{/* readinessProbe:*/}} +{{/* {{- toYaml . | nindent 12 }}*/}} +{{/* {{- end }}*/}} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/k8s/helm/templates/hpa.yaml b/k8s/helm/templates/hpa.yaml new file mode 100644 index 0000000000..28c087ea4f --- /dev/null +++ b/k8s/helm/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "helm.fullname" . }} + labels: + {{- include "helm.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "helm.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/k8s/helm/templates/ingress.yaml b/k8s/helm/templates/ingress.yaml new file mode 100644 index 0000000000..5bdb7911f7 --- /dev/null +++ b/k8s/helm/templates/ingress.yaml @@ -0,0 +1,43 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "helm.fullname" . }} + labels: + {{- include "helm.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- with .pathType }} + pathType: {{ . }} + {{- end }} + backend: + service: + name: {{ include "helm.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/k8s/helm/templates/service.yaml b/k8s/helm/templates/service.yaml new file mode 100644 index 0000000000..de450fc579 --- /dev/null +++ b/k8s/helm/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "helm.fullname" . }} + labels: + {{- include "helm.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "helm.selectorLabels" . | nindent 4 }} diff --git a/k8s/helm/templates/serviceaccount.yaml b/k8s/helm/templates/serviceaccount.yaml new file mode 100644 index 0000000000..d47046559d --- /dev/null +++ b/k8s/helm/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "helm.serviceAccountName" . }} + labels: + {{- include "helm.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/k8s/helm/templates/tests/test-connection.yaml b/k8s/helm/templates/tests/test-connection.yaml new file mode 100644 index 0000000000..bf1c65f248 --- /dev/null +++ b/k8s/helm/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "helm.fullname" . }}-test-connection" + labels: + {{- include "helm.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "helm.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/k8s/helm/values-fallback_prod.yaml b/k8s/helm/values-fallback_prod.yaml new file mode 100644 index 0000000000..34a46d3c46 --- /dev/null +++ b/k8s/helm/values-fallback_prod.yaml @@ -0,0 +1,6 @@ +rabbitmq: + host: wp-p2m-42.ebi.ac.uk +biosamples: + search: + host: biosamples-search-helm + port: 9090 diff --git a/k8s/helm/values-primary_dev.yaml b/k8s/helm/values-primary_dev.yaml new file mode 100644 index 0000000000..fa65361164 --- /dev/null +++ b/k8s/helm/values-primary_dev.yaml @@ -0,0 +1,6 @@ +rabbitmq: + host: wp-np2-40.ebi.ac.uk +biosamples: + search: + host: biosamples-search-helm + port: 9090 \ No newline at end of file diff --git a/k8s/helm/values-primary_prod.yaml b/k8s/helm/values-primary_prod.yaml new file mode 100644 index 0000000000..2a8d877e09 --- /dev/null +++ b/k8s/helm/values-primary_prod.yaml @@ -0,0 +1,6 @@ +rabbitmq: + host: wp-p1m-42.ebi.ac.uk +biosamples: + search: + host: biosamples-search-helm + port: 9090 diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml new file mode 100644 index 0000000000..cabbe5c045 --- /dev/null +++ b/k8s/helm/values.yaml @@ -0,0 +1,94 @@ +biosamples: + context: + path: / + schema: + validator: + url: http://biovalidator-service:3020/biosamples/biovalidator/validate + store: + url: http://json-schema-store:8080/biosamples/schema-store +service: + type: NodePort + port: 8080 +livenessProbe: + httpGet: + path: /actuator/health/liveness + port: 8080 +readinessProbe: + httpGet: + path: /actuator/health/readiness + port: 8080 + +replicaCount: 1 +image: + repository: __set-in-commandline__ + pullPolicy: IfNotPresent + tag: latest + +imagePullSecrets: + - name: __set-in-commandline__ +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + automount: true + annotations: {} + name: "" +podAnnotations: {} +podLabels: {} +podSecurityContext: {} + # fsGroup: 2000 +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +volumes: + - name: config-volume + configMap: + name: application-properties + +volumeMounts: + - name: config-volume + mountPath: /config/application.properties + subPath: application.properties + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/k8s/jobs/Chart.yaml b/k8s/jobs/Chart.yaml new file mode 100644 index 0000000000..1d20fc91bc --- /dev/null +++ b/k8s/jobs/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: pipelines +description: A Helm chart for Kubernetes +type: application +version: 0.1.0 +appVersion: "1.16.0" diff --git a/k8s/jobs/templates/_helpers.tpl b/k8s/jobs/templates/_helpers.tpl new file mode 100644 index 0000000000..ba04c300df --- /dev/null +++ b/k8s/jobs/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "helm.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "helm.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "helm.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "helm.labels" -}} +helm.sh/chart: {{ include "helm.chart" . }} +{{ include "helm.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "helm.selectorLabels" -}} +app.kubernetes.io/name: {{ include "helm.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "helm.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "helm.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/k8s/jobs/templates/job.yaml b/k8s/jobs/templates/job.yaml new file mode 100644 index 0000000000..e4bd82e65c --- /dev/null +++ b/k8s/jobs/templates/job.yaml @@ -0,0 +1,50 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "helm.fullname" . }}-job + labels: + app: {{ include "helm.name" . }} + chart: {{ include "helm.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + backoffLimit: {{ .Values.backoffLimit }} + template: + metadata: + labels: + app: {{ include "helm.name" . }} + release: {{ .Release.Name }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + restartPolicy: {{ .Values.restartPolicy }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SPRING_DATA_MONGODB_URI + valueFrom: + secretKeyRef: + name: biosamples-mongodb + key: connection-string + - name: SPRING_RABBITMQ_HOST + value: {{ .Values.rabbitmq.host }} + - name: SPRING_RABBITMQ_USERNAME + valueFrom: + secretKeyRef: + name: rabbitmq-user + key: username + - name: SPRING_RABBITMQ_PASSWORD + valueFrom: + secretKeyRef: + name: rabbitmq-user + key: password + {{- range .Values.env }} + - name: {{ .name }} + value: "{{ .value }}" + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} diff --git a/k8s/jobs/values-fallback_prod.yaml b/k8s/jobs/values-fallback_prod.yaml new file mode 100644 index 0000000000..97b5d91bdc --- /dev/null +++ b/k8s/jobs/values-fallback_prod.yaml @@ -0,0 +1,2 @@ +rabbitmq: + host: wp-p2m-42.ebi.ac.uk diff --git a/k8s/jobs/values-primary_dev.yaml b/k8s/jobs/values-primary_dev.yaml new file mode 100644 index 0000000000..156124fc83 --- /dev/null +++ b/k8s/jobs/values-primary_dev.yaml @@ -0,0 +1,2 @@ +rabbitmq: + host: wp-np2-40.ebi.ac.uk diff --git a/k8s/jobs/values-primary_prod.yaml b/k8s/jobs/values-primary_prod.yaml new file mode 100644 index 0000000000..ecfbe90c39 --- /dev/null +++ b/k8s/jobs/values-primary_prod.yaml @@ -0,0 +1,2 @@ +rabbitmq: + host: wp-p1m-42.ebi.ac.uk diff --git a/k8s/jobs/values.yaml b/k8s/jobs/values.yaml new file mode 100644 index 0000000000..e02efa6fdc --- /dev/null +++ b/k8s/jobs/values.yaml @@ -0,0 +1,22 @@ +image: + repository: biosamples-v4/pipelines-reindex + tag: "1.0.0" + pullPolicy: IfNotPresent + +resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "1" + memory: "2Gi" + +env: + - name: SPRING_PROFILES_ACTIVE + value: "batch" + - name: JAVA_OPTS + value: "-Xms512m -Xmx1g" + +restartPolicy: Never + +backoffLimit: 3 \ No newline at end of file diff --git a/logback/pom.xml b/logback/pom.xml new file mode 100644 index 0000000000..9de50b0079 --- /dev/null +++ b/logback/pom.xml @@ -0,0 +1,24 @@ + + 4.0.0 + logback + jar + + + uk.ac.ebi.biosamples + biosamples + 5.3.15-SNAPSHOT + ../ + + + + + org.springframework.boot + spring-boot-starter + + + ch.qos.logback + logback-classic + + + diff --git a/logback/src/main/resources/logback-shared.xml b/logback/src/main/resources/logback-shared.xml new file mode 100644 index 0000000000..db439d0c25 --- /dev/null +++ b/logback/src/main/resources/logback-shared.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + ${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) + %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} + %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}} + + + + + + + + + + + + + + + + + + diff --git a/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java b/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java index 888bc3fe5e..c65b287944 100644 --- a/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java +++ b/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java @@ -26,20 +26,20 @@ import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountEntry; import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; -import uk.ac.ebi.biosamples.service.FacetService; import uk.ac.ebi.biosamples.service.SamplePageService; +import uk.ac.ebi.biosamples.service.facet.FacetingService; @Component public class AnalyticsApplicationRunner implements ApplicationRunner { private static final Logger LOG = LoggerFactory.getLogger(AnalyticsApplicationRunner.class); private final AnalyticsService analyticsService; private final PipelineFutureCallback pipelineFutureCallback; - private final FacetService facetService; + private final FacetingService facetService; private final SamplePageService samplePageService; public AnalyticsApplicationRunner( final AnalyticsService analyticsService, - final FacetService facetService, + final FacetingService facetService, final SamplePageService samplePageService) { this.analyticsService = analyticsService; this.facetService = facetService; diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/HelpdeskActionApplicationRunner.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/HelpdeskActionApplicationRunner.java index 18ad765d9b..fe43f92c05 100644 --- a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/HelpdeskActionApplicationRunner.java +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/HelpdeskActionApplicationRunner.java @@ -76,11 +76,17 @@ public void run(ApplicationArguments args) { } case "changeStatusOfSamplesFromFile" -> { - final List accessions = - sampleStatusUpdater.parseFileAndGetSampleAccessionList( - "C:\\Users\\dgupta\\AtlantECO-samples-to-suppress.txt"); + final String file = + args.containsOption("file") ? args.getOptionValues("file").get(0) : null; + + if (file != null) { + final List accessions = + sampleStatusUpdater.parseFileAndGetSampleAccessionList(file); - sampleStatusUpdater.processSamples(accessions, null); + sampleStatusUpdater.processSamples(accessions, SampleStatus.PUBLIC); + } else { + throw new RuntimeException("File is not provided"); + } } case "updateSampleRelationships" -> { diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java index c059fabe09..8b0082ce29 100644 --- a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java @@ -28,7 +28,6 @@ @Component public class SampleChecklistComplianceHandlerEVA { - private static final Logger log = LoggerFactory.getLogger(SampleChecklistComplianceHandlerEVA.class); @@ -38,7 +37,6 @@ public class SampleChecklistComplianceHandlerEVA { "geographic location (region and locality)"; private static final String COLLECTION_DATE = "collection_date"; private static final String COLLECTION_DATE_WITHOUT_UNDERSCORE = "collection date"; - private static final String NCBI_MIRRORING_WEBIN_ID = "Webin-842"; private final BioSamplesClient bioSamplesWebinClient; private final PipelinesProperties pipelinesProperties; diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleStatusUpdater.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleStatusUpdater.java index 05d493438c..b284a32bc8 100644 --- a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleStatusUpdater.java +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleStatusUpdater.java @@ -137,13 +137,18 @@ private void handleSample(final Sample sample, final SampleStatus toMakeStatus) LocalDateTime.now(ZoneOffset.UTC) .plusYears(100) .toEpochSecond(ZoneOffset.UTC))) + .withStatus(SampleStatus.PRIVATE) .build(); } else { log.info("{} is already private", accession); } } else if (toMakeStatus == SampleStatus.PUBLIC) { if (sample.getRelease().isAfter(Instant.now())) { - updatedSample = Sample.Builder.fromSample(sample).withRelease(Instant.now()).build(); + updatedSample = + Sample.Builder.fromSample(sample) + .withRelease(Instant.now()) + .withStatus(SampleStatus.PUBLIC) + .build(); } else { log.info("{} is already public", accession); } diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java index baf6127629..109ac0013f 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java @@ -66,10 +66,10 @@ public class PipelinesProperties { @Value("${biosamples.pipelines.copydown.domain:self.BiosampleCopydown}") private String copydownDomain; - @Value("${biosamples.schemaValidator:http://localhost:3020/validate}") + @Value("${biosamples.schema.validator.url:http://localhost:3020/validate}") private String schemaValidator; - @Value("${biosamples.schemaStore:http://localhost:8085/api/v2/schemas}") + @Value("${biosamples.schema.store.url:http://localhost:8085/api/v2/schemas}") private String schemaStore; @Value( diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineLastRun.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineLastRun.java new file mode 100644 index 0000000000..d1362319b6 --- /dev/null +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineLastRun.java @@ -0,0 +1,28 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.model; + +import java.time.LocalDate; +import lombok.Builder; +import lombok.Getter; +import lombok.extern.jackson.Jacksonized; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +@Document +@Jacksonized +@Builder +@Getter +public class PipelineLastRun { + @Id private String id; + private PipelineName pipelineName; + private LocalDate lastRunDate; +} diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java new file mode 100644 index 0000000000..348121d002 --- /dev/null +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java @@ -0,0 +1,22 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.repository; + +import java.util.Optional; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; +import uk.ac.ebi.biosamples.model.PipelineLastRun; +import uk.ac.ebi.biosamples.model.PipelineName; + +@Repository +public interface PipelineLastRunRepository extends MongoRepository { + Optional findFirstByPipelineName(PipelineName pipelineName); +} diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java new file mode 100644 index 0000000000..6c3fbdb1d9 --- /dev/null +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java @@ -0,0 +1,46 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import java.time.LocalDate; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.model.PipelineLastRun; +import uk.ac.ebi.biosamples.model.PipelineName; +import uk.ac.ebi.biosamples.repository.PipelineLastRunRepository; + +@Service +@Slf4j +@RequiredArgsConstructor +public class PipelineHelperService { + private final PipelineLastRunRepository pipelineLastRunRepository; + + public PipelineLastRun getLastRunDate(PipelineName pipelineName) { + return pipelineLastRunRepository + .findFirstByPipelineName(pipelineName) + .orElse( + PipelineLastRun.builder() + .pipelineName(pipelineName) + .lastRunDate(LocalDate.EPOCH) + .build()); + } + + public void updateLastRunDate(PipelineLastRun pipelineLastRun, LocalDate lastRunDate) { + PipelineLastRun updated = + PipelineLastRun.builder() + .id(pipelineLastRun.getId()) + .pipelineName(pipelineLastRun.getPipelineName()) + .lastRunDate(lastRunDate) + .build(); + pipelineLastRunRepository.save(updated); + } +} diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java index 534ac9f137..a61b27dba2 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java @@ -17,10 +17,7 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.ApplicationArguments; @@ -34,6 +31,15 @@ public class PipelineUtils { private static final Logger log = LoggerFactory.getLogger(PipelineUtils.class); + public static Collection getLastRunFilters(LocalDate lastRunDate, LocalDate startDate) { + Filter fromDateFilter = + new DateRangeFilter.DateRangeFilterBuilder("update") + .from(lastRunDate.atStartOfDay().toInstant(ZoneOffset.UTC)) + .until(startDate.atStartOfDay().toInstant(ZoneOffset.UTC)) + .build(); + return List.of(fromDateFilter); + } + public static Collection getDateFilters( final ApplicationArguments args, final String dateType) { final Collection filters = new ArrayList<>(); diff --git a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java index 39841e7dfa..86d5948409 100644 --- a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -10,21 +10,8 @@ */ package uk.ac.ebi.biosamples; -import org.apache.http.HeaderElement; -import org.apache.http.HeaderElementIterator; -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.conn.ConnectionKeepAliveStrategy; -import org.apache.http.conn.routing.HttpRoute; -import org.apache.http.impl.client.cache.CacheConfig; -import org.apache.http.impl.client.cache.CachingHttpClientBuilder; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.http.message.BasicHeaderElementIterator; -import org.apache.http.protocol.HTTP; -import org.apache.http.protocol.HttpContext; import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.boot.web.client.RestTemplateCustomizer; @@ -34,7 +21,8 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.configuration.ExclusionConfiguration; import uk.ac.ebi.biosamples.service.EnaConfig; @@ -51,94 +39,30 @@ }) @Import(ExclusionConfiguration.class) @EnableCaching +@EnableWebSecurity +@EnableMongoRepositories(basePackages = "uk.ac.ebi.biosamples.repository") public class Application { public static void main(final String[] args) { - final ConfigurableApplicationContext ctx = SpringApplication.run(Application.class, args); + SpringApplication app = new SpringApplication(Application.class); + app.setWebApplicationType(WebApplicationType.NONE); + + final ConfigurableApplicationContext ctx = app.run(args); PipelineUtils.exitPipeline(ctx); } + @Bean + public RestTemplate restTemplate(final RestTemplateCustomizer restTemplateCustomizer) { + final RestTemplate restTemplate = new RestTemplate(); + restTemplateCustomizer.customize(restTemplate); + return restTemplate; + } + @Bean public RestTemplateCustomizer restTemplateCustomizer( final BioSamplesProperties bioSamplesProperties, - final PipelinesProperties piplinesProperties) { - return new RestTemplateCustomizer() { - @Override - public void customize(final RestTemplate restTemplate) { - - // use a keep alive strategy to try to make it easier to maintain connections for - // reuse - final ConnectionKeepAliveStrategy keepAliveStrategy = - new ConnectionKeepAliveStrategy() { - @Override - public long getKeepAliveDuration( - final HttpResponse response, final HttpContext context) { - - // check if there is a non-standard keep alive header present - final HeaderElementIterator it = - new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); - while (it.hasNext()) { - final HeaderElement he = it.nextElement(); - final String param = he.getName(); - final String value = he.getValue(); - if (value != null && param.equalsIgnoreCase("timeout")) { - return Long.parseLong(value) * 1000; - } - } - // default to 60s if no header - return 60 * 1000; - } - }; - - // set a number of connections to use at once for multiple threads - final PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = - new PoolingHttpClientConnectionManager(); - poolingHttpClientConnectionManager.setMaxTotal(piplinesProperties.getConnectionCountMax()); - poolingHttpClientConnectionManager.setDefaultMaxPerRoute( - piplinesProperties.getConnectionCountDefault()); - poolingHttpClientConnectionManager.setMaxPerRoute( - new HttpRoute(HttpHost.create(piplinesProperties.getZooma())), - piplinesProperties.getConnectionCountZooma()); - poolingHttpClientConnectionManager.setMaxPerRoute( - new HttpRoute(HttpHost.create(bioSamplesProperties.getOls())), - piplinesProperties.getConnectionCountOls()); - - // set a local cache for cacheable responses - final CacheConfig cacheConfig = - CacheConfig.custom() - .setMaxCacheEntries(1024) - .setMaxObjectSize(1024 * 1024) // max size of 1Mb - // number of entries x size of entries = 1Gb total cache size - .setSharedCache(false) // act like a browser cache not a middle-hop cache - .build(); - - // set a timeout limit - // TODO put this in application.properties - final int timeout = 60; // in seconds - final RequestConfig config = - RequestConfig.custom() - .setConnectTimeout(timeout * 1000) // time to establish the connection with the - // remote host - .setConnectionRequestTimeout( - timeout * 1000) // maximum time of inactivity between two - // data packets - .setSocketTimeout(timeout * 1000) - .build(); // time to wait for a connection from the connection - // manager/pool - - // make the actual client - final HttpClient httpClient = - CachingHttpClientBuilder.create() - .setCacheConfig(cacheConfig) - .useSystemProperties() - .setConnectionManager(poolingHttpClientConnectionManager) - .setKeepAliveStrategy(keepAliveStrategy) - .setDefaultRequestConfig(config) - .build(); - - // and wire it into the resttemplate - restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory(httpClient)); - } - }; + final PipelinesProperties pipelinesProperties) { + return new PipelinesHelper() + .getRestTemplateCustomizer(bioSamplesProperties, pipelinesProperties); } } diff --git a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java index 3280aa5355..f1bfb3fed7 100644 --- a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java +++ b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java @@ -12,6 +12,7 @@ import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentLinkedQueue; @@ -29,7 +30,10 @@ import uk.ac.ebi.biosamples.core.model.PipelineAnalytics; import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.model.PipelineLastRun; +import uk.ac.ebi.biosamples.model.PipelineName; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; +import uk.ac.ebi.biosamples.service.PipelineHelperService; import uk.ac.ebi.biosamples.utils.PipelineUtils; import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @@ -37,27 +41,35 @@ @Component public class CopydownApplicationRunner implements ApplicationRunner { private static final Logger LOG = LoggerFactory.getLogger(CopydownApplicationRunner.class); + private static final PipelineName PIPELINE_NAME = PipelineName.COPYDOWN; private final BioSamplesClient bioSamplesClient; private final PipelinesProperties pipelinesProperties; private final AnalyticsService analyticsService; private final PipelineFutureCallback pipelineFutureCallback; + private final PipelineHelperService pipelineHelperService; public CopydownApplicationRunner( final BioSamplesClient bioSamplesClient, final PipelinesProperties pipelinesProperties, - final AnalyticsService analyticsService) { + final AnalyticsService analyticsService, + PipelineHelperService pipelineHelperService) { this.bioSamplesClient = bioSamplesClient; this.pipelinesProperties = pipelinesProperties; this.analyticsService = analyticsService; + this.pipelineHelperService = pipelineHelperService; pipelineFutureCallback = new PipelineFutureCallback(); } @Override public void run(final ApplicationArguments args) throws Exception { - final Collection filters = PipelineUtils.getDateFilters(args, "update"); + PipelineLastRun pipelineLastRun = pipelineHelperService.getLastRunDate(PIPELINE_NAME); + LocalDate lastRunDate = pipelineLastRun.getLastRunDate(); + LocalDate startDate = LocalDate.now(); + final Collection filters = PipelineUtils.getLastRunFilters(lastRunDate, startDate); final Instant startTime = Instant.now(); LOG.info("Pipeline started at {}", startTime); + LOG.info("Processing samples from {}", lastRunDate); long sampleCount = 0; try (final AdaptiveThreadPoolExecutor executorService = @@ -89,6 +101,7 @@ public void run(final ApplicationArguments args) throws Exception { LOG.info("waiting for futures"); // wait for anything to finish ThreadUtils.checkAndCallbackFutures(futures, 0, pipelineFutureCallback); + pipelineHelperService.updateLastRunDate(pipelineLastRun, startDate); } catch (final Exception e) { LOG.error("Pipeline failed to finish successfully", e); throw e; diff --git a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/Application.java index c3551f905c..de7a03296b 100644 --- a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -34,7 +34,9 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.configuration.ExclusionConfiguration; import uk.ac.ebi.biosamples.service.EnaConfig; @@ -51,6 +53,8 @@ }) @Import(ExclusionConfiguration.class) @EnableCaching +@EnableWebSecurity +@EnableMongoRepositories(basePackages = "uk.ac.ebi.biosamples.repository") public class Application { public static void main(final String[] args) { diff --git a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java index 1885cdb1a2..c0a5a25f80 100644 --- a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java +++ b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java @@ -13,6 +13,7 @@ import java.io.*; import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentLinkedQueue; @@ -33,9 +34,12 @@ import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.core.model.SampleAnalytics; import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.model.PipelineLastRun; +import uk.ac.ebi.biosamples.model.PipelineName; import uk.ac.ebi.biosamples.mongo.model.MongoCurationRule; import uk.ac.ebi.biosamples.mongo.repository.MongoCurationRuleRepository; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; +import uk.ac.ebi.biosamples.service.PipelineHelperService; import uk.ac.ebi.biosamples.utils.PipelineUtils; import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @@ -43,6 +47,7 @@ @Component public class CuramiApplicationRunner implements ApplicationRunner { private static final Logger LOG = LoggerFactory.getLogger(CuramiApplicationRunner.class); + private static final PipelineName PIPELINE_NAME = PipelineName.CURAMI; private final BioSamplesClient bioSamplesClient; private final PipelinesProperties pipelinesProperties; @@ -50,25 +55,32 @@ public class CuramiApplicationRunner implements ApplicationRunner { private final MongoCurationRuleRepository repository; private final AnalyticsService analyticsService; private final PipelineFutureCallback pipelineFutureCallback; + private final PipelineHelperService pipelineHelperService; public CuramiApplicationRunner( final BioSamplesClient bioSamplesClient, final PipelinesProperties pipelinesProperties, final MongoCurationRuleRepository repository, - final AnalyticsService analyticsService) { + final AnalyticsService analyticsService, + PipelineHelperService pipelineHelperService) { this.bioSamplesClient = bioSamplesClient; this.pipelinesProperties = pipelinesProperties; this.repository = repository; this.analyticsService = analyticsService; + this.pipelineHelperService = pipelineHelperService; curationRules = new HashMap<>(); pipelineFutureCallback = new PipelineFutureCallback(); } @Override public void run(final ApplicationArguments args) throws Exception { - final Collection filters = PipelineUtils.getDateFilters(args, "update"); + PipelineLastRun pipelineLastRun = pipelineHelperService.getLastRunDate(PIPELINE_NAME); + LocalDate lastRunDate = pipelineLastRun.getLastRunDate(); + LocalDate startDate = LocalDate.now(); + final Collection filters = PipelineUtils.getLastRunFilters(lastRunDate, startDate); final Instant startTime = Instant.now(); LOG.info("Pipeline started at {}", startTime); + LOG.info("Processing samples from {}", lastRunDate); long sampleCount = 0; final SampleAnalytics sampleAnalytics = new SampleAnalytics(); @@ -104,6 +116,7 @@ public void run(final ApplicationArguments args) throws Exception { LOG.info("Waiting for all scheduled tasks to finish"); ThreadUtils.checkAndCallbackFutures(futures, 0, pipelineFutureCallback); + pipelineHelperService.updateLastRunDate(pipelineLastRun, startDate); } catch (final Exception e) { LOG.error("Pipeline failed to finish successfully", e); throw e; diff --git a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/Application.java index 83dd6af292..98cee4c9b1 100644 --- a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -23,6 +23,7 @@ import org.apache.http.message.BasicHeaderElementIterator; import org.apache.http.protocol.HTTP; import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.boot.web.client.RestTemplateCustomizer; @@ -32,7 +33,9 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.configuration.ExclusionConfiguration; import uk.ac.ebi.biosamples.service.EnaConfig; @@ -49,9 +52,14 @@ }) @Import(ExclusionConfiguration.class) @EnableCaching +@EnableWebSecurity +@EnableMongoRepositories(basePackages = "uk.ac.ebi.biosamples.repository") public class Application { public static void main(final String[] args) { - final ConfigurableApplicationContext ctx = SpringApplication.run(Application.class, args); + SpringApplication app = new SpringApplication(Application.class); + app.setWebApplicationType(WebApplicationType.NONE); + + final ConfigurableApplicationContext ctx = app.run(args); PipelineUtils.exitPipeline(ctx); } diff --git a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java index 484cb62cd4..216a69a8c9 100644 --- a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java +++ b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java @@ -12,6 +12,7 @@ import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentLinkedQueue; @@ -31,7 +32,10 @@ import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.core.service.CurationApplicationService; import uk.ac.ebi.biosamples.curation.service.IriUrlValidatorService; +import uk.ac.ebi.biosamples.model.PipelineLastRun; +import uk.ac.ebi.biosamples.model.PipelineName; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; +import uk.ac.ebi.biosamples.service.PipelineHelperService; import uk.ac.ebi.biosamples.utils.PipelineUtils; import uk.ac.ebi.biosamples.utils.ols.OlsProcessor; import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; @@ -40,6 +44,8 @@ @Component public class CurationApplicationRunner implements ApplicationRunner { private static final Logger LOG = LoggerFactory.getLogger(CurationApplicationRunner.class); + private static final PipelineName PIPELINE_NAME = PipelineName.CURATION; + private final BioSamplesClient bioSamplesClient; private final PipelinesProperties pipelinesProperties; private final OlsProcessor olsProcessor; @@ -47,6 +53,7 @@ public class CurationApplicationRunner implements ApplicationRunner { private final AnalyticsService analyticsService; private final PipelineFutureCallback pipelineFutureCallback; private final IriUrlValidatorService iriUrlValidatorService; + private final PipelineHelperService pipelineHelperService; public CurationApplicationRunner( final BioSamplesClient bioSamplesClient, @@ -54,20 +61,27 @@ public CurationApplicationRunner( final OlsProcessor olsProcessor, final CurationApplicationService curationApplicationService, final AnalyticsService analyticsService, - final IriUrlValidatorService iriUrlValidatorService) { + final IriUrlValidatorService iriUrlValidatorService, + PipelineHelperService pipelineHelperService) { this.bioSamplesClient = bioSamplesClient; this.pipelinesProperties = pipelinesProperties; this.olsProcessor = olsProcessor; this.curationApplicationService = curationApplicationService; this.analyticsService = analyticsService; this.iriUrlValidatorService = iriUrlValidatorService; + this.pipelineHelperService = pipelineHelperService; pipelineFutureCallback = new PipelineFutureCallback(); } @Override public void run(final ApplicationArguments args) throws Exception { + PipelineLastRun pipelineLastRun = pipelineHelperService.getLastRunDate(PIPELINE_NAME); + LocalDate lastRunDate = pipelineLastRun.getLastRunDate(); + LocalDate startDate = LocalDate.now(); final Instant startTime = Instant.now(); - final Collection filters = PipelineUtils.getDateFilters(args, "update"); + final Collection filters = PipelineUtils.getLastRunFilters(lastRunDate, startDate); + LOG.info("Pipeline started at {}", startTime); + LOG.info("Processing samples from {}", lastRunDate); long sampleCount = 0; try (final AdaptiveThreadPoolExecutor executorService = @@ -107,6 +121,7 @@ public void run(final ApplicationArguments args) throws Exception { LOG.info("waiting for futures"); // wait for anything to finish ThreadUtils.checkAndCallbackFutures(futures, 0, pipelineFutureCallback); + pipelineHelperService.updateLastRunDate(pipelineLastRun, startDate); } catch (final Exception e) { LOG.error("Pipeline failed to finish successfully", e); diff --git a/pipelines/ncbi-ena-link/src/main/java/uk/ac/ebi/biosamples/ena/NcbiEnaLinkRunner.java b/pipelines/ncbi-ena-link/src/main/java/uk/ac/ebi/biosamples/ena/NcbiEnaLinkRunner.java index 3e04258c66..2522e7555f 100644 --- a/pipelines/ncbi-ena-link/src/main/java/uk/ac/ebi/biosamples/ena/NcbiEnaLinkRunner.java +++ b/pipelines/ncbi-ena-link/src/main/java/uk/ac/ebi/biosamples/ena/NcbiEnaLinkRunner.java @@ -30,6 +30,7 @@ import uk.ac.ebi.biosamples.mongo.util.PipelineCompletionStatus; import uk.ac.ebi.biosamples.service.EraProDao; import uk.ac.ebi.biosamples.service.SampleRetrievalResult; +import uk.ac.ebi.biosamples.service.SampleCallbackResult; import uk.ac.ebi.biosamples.utils.PipelineUniqueIdentifierGenerator; import uk.ac.ebi.biosamples.utils.PipelineUtils; import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; diff --git a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/Application.java index 1ffca15837..d4bac8652f 100644 --- a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -11,8 +11,10 @@ package uk.ac.ebi.biosamples; import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.web.client.RestTemplateCustomizer; import org.springframework.cache.annotation.EnableCaching; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; @@ -22,6 +24,8 @@ import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.configuration.ExclusionConfiguration; import uk.ac.ebi.biosamples.service.EnaConfig; import uk.ac.ebi.biosamples.service.EnaSampleToBioSampleConversionService; @@ -39,6 +43,7 @@ @EnableCaching(proxyTargetClass = true) @EnableAsync @EnableScheduling +@EnableWebSecurity public class Application { // this is needed to read nonstrings from properties files @@ -49,7 +54,25 @@ public static PropertySourcesPlaceholderConfigurer getPropertySourcesPlaceholder } public static void main(final String[] args) { - final ConfigurableApplicationContext ctx = SpringApplication.run(Application.class, args); + SpringApplication app = new SpringApplication(Application.class); + app.setWebApplicationType(WebApplicationType.NONE); + + final ConfigurableApplicationContext ctx = app.run(args); PipelineUtils.exitPipeline(ctx); } + + @Bean + public RestTemplate restTemplate(final RestTemplateCustomizer restTemplateCustomizer) { + final RestTemplate restTemplate = new RestTemplate(); + restTemplateCustomizer.customize(restTemplate); + return restTemplate; + } + + @Bean + public RestTemplateCustomizer restTemplateCustomizer( + final BioSamplesProperties bioSamplesProperties, + final PipelinesProperties pipelinesProperties) { + return new PipelinesHelper() + .getRestTemplateCustomizer(bioSamplesProperties, pipelinesProperties); + } } diff --git a/pipelines/pom.xml b/pipelines/pom.xml index 3f454b9158..82a5d05f59 100644 --- a/pipelines/pom.xml +++ b/pipelines/pom.xml @@ -18,7 +18,7 @@ embl-ebi - + @@ -37,7 +37,7 @@ taxonimport reindex chain - + diff --git a/pipelines/reindex/pom.xml b/pipelines/reindex/pom.xml index ba406741e8..99d57cb27d 100644 --- a/pipelines/reindex/pom.xml +++ b/pipelines/reindex/pom.xml @@ -13,6 +13,21 @@ + + + org.springframework.hateoas + spring-hateoas + 1.3.4 + + + org.springframework + spring-webmvc + + + org.springframework.boot + spring-boot-starter-tomcat + + uk.ac.ebi.biosamples core diff --git a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java index 9a62af2f13..07afc3fd87 100644 --- a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -10,17 +10,35 @@ */ package uk.ac.ebi.biosamples; +import org.apache.http.HeaderElement; +import org.apache.http.HeaderElementIterator; +import org.apache.http.HttpHost; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.conn.ConnectionKeepAliveStrategy; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.impl.client.cache.CacheConfig; +import org.apache.http.impl.client.cache.CachingHttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicHeaderElementIterator; +import org.apache.http.protocol.HTTP; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.web.client.RestTemplateCustomizer; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.configuration.ExclusionConfiguration; +import uk.ac.ebi.biosamples.security.service.BioSamplesWebSecurityConfig; import uk.ac.ebi.biosamples.service.EnaConfig; import uk.ac.ebi.biosamples.service.EnaSampleToBioSampleConversionService; import uk.ac.ebi.biosamples.service.EraProDao; +import uk.ac.ebi.biosamples.service.PipelineHelperService; import uk.ac.ebi.biosamples.utils.PipelineUtils; @SpringBootApplication(exclude = DataSourceAutoConfiguration.class) @@ -28,7 +46,16 @@ excludeFilters = { @ComponentScan.Filter( type = FilterType.ASSIGNABLE_TYPE, - value = {EnaConfig.class, EraProDao.class, EnaSampleToBioSampleConversionService.class}) + value = { + EnaConfig.class, + EraProDao.class, + EnaSampleToBioSampleConversionService.class, + BioSamplesWebSecurityConfig.class, + PipelineHelperService.class + }), + @ComponentScan.Filter( + type = FilterType.REGEX, + pattern = "uk\\.ac\\.ebi\\.biosamples\\.service\\.validation\\..*") }) @Import(ExclusionConfiguration.class) public class Application { @@ -37,4 +64,73 @@ public static void main(final String[] args) { final ConfigurableApplicationContext ctx = SpringApplication.run(Application.class, args); PipelineUtils.exitPipeline(ctx); } + + // todo I Had to add restTemplate bean as a temporary workaround as there seems to be a problem + // with dependencies after refactor. + // We need to sort out dependency problem and remove this unused dependency. + + @Bean + public RestTemplate restTemplate(final RestTemplateCustomizer restTemplateCustomizer) { + final RestTemplate restTemplate = new RestTemplate(); + restTemplateCustomizer.customize(restTemplate); + return restTemplate; + } + + @Bean + public RestTemplateCustomizer restTemplateCustomizer( + final BioSamplesProperties bioSamplesProperties, + final PipelinesProperties pipelinesProperties) { + return restTemplate -> { + final ConnectionKeepAliveStrategy keepAliveStrategy = + (response, context) -> { + final HeaderElementIterator it = + new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); + while (it.hasNext()) { + final HeaderElement he = it.nextElement(); + final String param = he.getName(); + final String value = he.getValue(); + if (value != null && param.equalsIgnoreCase("timeout")) { + return Long.parseLong(value) * 1000; + } + } + + return 60 * 1000; + }; + + final PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = + new PoolingHttpClientConnectionManager(); + poolingHttpClientConnectionManager.setMaxTotal(pipelinesProperties.getConnectionCountMax()); + poolingHttpClientConnectionManager.setDefaultMaxPerRoute( + pipelinesProperties.getConnectionCountDefault()); + poolingHttpClientConnectionManager.setMaxPerRoute( + new HttpRoute(HttpHost.create(pipelinesProperties.getZooma())), + pipelinesProperties.getConnectionCountZooma()); + poolingHttpClientConnectionManager.setMaxPerRoute( + new HttpRoute(HttpHost.create(bioSamplesProperties.getOls())), + pipelinesProperties.getConnectionCountOls()); + + final CacheConfig cacheConfig = + CacheConfig.custom() + .setMaxCacheEntries(1024) + .setMaxObjectSize(1024 * 1024) + .setSharedCache(false) + .build(); + final int timeout = 60; + final RequestConfig config = + RequestConfig.custom() + .setConnectTimeout(timeout * 1000) + .setConnectionRequestTimeout(timeout * 1000) + .setSocketTimeout(timeout * 1000) + .build(); + final HttpClient httpClient = + CachingHttpClientBuilder.create() + .setCacheConfig(cacheConfig) + .useSystemProperties() + .setConnectionManager(poolingHttpClientConnectionManager) + .setKeepAliveStrategy(keepAliveStrategy) + .setDefaultRequestConfig(config) + .build(); + restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory(httpClient)); + }; + } } diff --git a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java index 9865e3ee55..ebefaba09d 100644 --- a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java +++ b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java @@ -10,6 +10,7 @@ */ package uk.ac.ebi.biosamples; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.*; import java.util.concurrent.*; import org.apache.commons.lang.IncompleteArgumentException; @@ -28,7 +29,6 @@ import uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter; import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.messaging.MessagingConstants; -import uk.ac.ebi.biosamples.messaging.model.MessageContent; import uk.ac.ebi.biosamples.mongo.model.MongoSample; import uk.ac.ebi.biosamples.mongo.service.SampleReadService; import uk.ac.ebi.biosamples.utils.PipelineUtils; @@ -46,19 +46,22 @@ */ @Component public class ReindexRunner implements ApplicationRunner { - private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationRunner.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ReindexRunner.class); private final AmqpTemplate amqpTemplate; private final SampleReadService sampleReadService; private final MongoOperations mongoOperations; + private final ObjectMapper objectMapper; @Autowired public ReindexRunner( - final AmqpTemplate amqpTemplate, - final SampleReadService sampleReadService, - final MongoOperations mongoOperations) { + AmqpTemplate amqpTemplate, + SampleReadService sampleReadService, + MongoOperations mongoOperations, + ObjectMapper objectMapper) { this.amqpTemplate = amqpTemplate; this.sampleReadService = sampleReadService; this.mongoOperations = mongoOperations; + this.objectMapper = objectMapper; } @Override @@ -103,7 +106,8 @@ public void run(final ApplicationArguments args) throws Exception { futures.put( accession, executor.submit( - new SampleIndexingCallable(accession, sampleReadService, amqpTemplate))); + new SampleIndexingCallable( + accession, sampleReadService, amqpTemplate, objectMapper))); ThreadUtils.checkFutures(futures, 1000); } @@ -121,14 +125,17 @@ private static class SampleIndexingCallable implements Callable { private final String accession; private final SampleReadService sampleReadService; private final AmqpTemplate amqpTemplate; + private final ObjectMapper objectMapper; public SampleIndexingCallable( - final String accession, - final SampleReadService sampleReadService, - final AmqpTemplate amqpTemplate) { + String accession, + SampleReadService sampleReadService, + AmqpTemplate amqpTemplate, + ObjectMapper objectMapper) { this.accession = accession; this.sampleReadService = sampleReadService; this.amqpTemplate = amqpTemplate; + this.objectMapper = objectMapper; } @Override @@ -153,19 +160,21 @@ private boolean fetchSampleAndSendMessage(final boolean isRetry) { if (sampleOptional.isPresent()) { try { + String json = objectMapper.writeValueAsString(sampleOptional.get()); amqpTemplate.convertAndSend( - MessagingConstants.REINDEXING_EXCHANGE, - MessagingConstants.REINDEXING_QUEUE, - MessageContent.build(sampleOptional.get(), null, related, false)); + MessagingConstants.INDEXING_EXCHANGE, MessagingConstants.INDEXING_QUEUE, json); + // amqpTemplate.convertAndSend( + // MessagingConstants.REINDEXING_EXCHANGE, + // MessagingConstants.REINDEXING_QUEUE, + // MessageContent.build(sampleOptional.get(), null, related, false)); return true; } catch (final Exception e) { LOGGER.error( - String.format( - "Failed to convert sample to message and send to queue for %s", accession), - e); + "Failed to convert sample to message and send to queue for {}", accession, e); } } else { + LOGGER.warn("Failed to fetch sample for {}", accession); final String errorMessage = isRetry ? String.format("Failed to fetch sample after retrying for %s", accession) diff --git a/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java b/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java index 8e1c1a5938..76209e82f4 100644 --- a/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java +++ b/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java @@ -15,6 +15,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.*; import org.junit.Test; import org.junit.runner.RunWith; @@ -36,6 +37,7 @@ public class ReindexRunnerTest { @Mock private AmqpTemplate amqpTemplate; @Mock private MongoOperations mongoOperations; @Mock private SampleReadService sampleReadService; + @Mock private ObjectMapper objectMapper; private final List accessions = Arrays.asList("ACCESSION1", "ACCESSION2", "ACCESSION3"); @@ -106,7 +108,7 @@ public MongoSample next() { .thenReturn(Optional.empty()) .thenReturn(Optional.of(sample3)); final ReindexRunner reindexRunner = - new ReindexRunner(amqpTemplate, sampleReadService, mongoOperations); + new ReindexRunner(amqpTemplate, sampleReadService, mongoOperations, objectMapper); reindexRunner.run(applicationArguments); } } diff --git a/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/Application.java index 4432161faa..003bdc0b0e 100644 --- a/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -30,11 +30,12 @@ import uk.ac.ebi.biosamples.service.EraProDao; import uk.ac.ebi.biosamples.utils.PipelineUtils; -@SpringBootApplication(exclude = { - DataSourceAutoConfiguration.class, - SecurityAutoConfiguration.class, - UserDetailsServiceAutoConfiguration.class -}) +@SpringBootApplication( + exclude = { + DataSourceAutoConfiguration.class, + SecurityAutoConfiguration.class, + UserDetailsServiceAutoConfiguration.class + }) @ComponentScan( excludeFilters = { @ComponentScan.Filter( diff --git a/pom.xml b/pom.xml index 6515a8f9bb..e1550eccf6 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,7 @@ integration client properties + logback core @@ -108,6 +109,11 @@ gitlab-maven https://gitlab.ebi.ac.uk/api/v4/projects/1143/packages/maven + + + gitlab-biosamples-search + https://gitlab.ebi.ac.uk/api/v4/projects/5516/packages/maven + @@ -146,6 +152,15 @@ -parameters + + org.apache.maven.plugins + maven-clean-plugin + 3.2.0 + + false + false + + org.apache.maven.plugins diff --git a/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java b/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java index 577493a9c9..08230d2003 100644 --- a/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java +++ b/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java @@ -97,13 +97,10 @@ public class BioSamplesProperties { @Value("${biosamples.webapp.core.facet.cache.maxage:86400}") private int webappCoreFacetCacheMaxAge; - @Value("${biosamples.schema.validator.uri:http://localhost:8085/validate}") - private URI biosamplesSchemaValidatorServiceUri; - - @Value("${biosamples.schemaValidator:http://localhost:3020/validate}") + @Value("${biosamples.schema.validator.url:http://localhost:3020/validate}") private String schemaValidator; - @Value("${biosamples.schemaStore:http://localhost:8085}") + @Value("${biosamples.schema.store.url:http://localhost:8085}") private String schemaStore; @Value("${biosamples.schema.default:BSDC00001}") @@ -115,6 +112,12 @@ public class BioSamplesProperties { @Value("${biosamples.bulksubmisison.webin.superuser.validation:false}") private boolean enableBulkSubmissionWebinSuperUserValidation; + @Value("${biosamples.search.host:biosamples-search-helm}") + private String biosamplesSearchHost; + + @Value("${biosamples.search.port:9090}") + private int biosamplesSearchPort; + public int getBiosamplesClientConnectionCountMax() { return connectionCountMax; } diff --git a/update-versions.sh b/update-versions.sh index 9e5cc477aa..b387715aa1 100755 --- a/update-versions.sh +++ b/update-versions.sh @@ -64,4 +64,10 @@ echo "Updating docker-compose and shell files to the new version" find . -name "docker-*.yml" -or -name "docker-*.sh" | xargs sed -i.versionsBackup "s/$LAST_VERSION/$NEW_VERSION/g" || exit 1 -echo "Version update complete!" + +echo "Clearing up resource..." +./mvnw versions:commit +find . -name "*.versionsBackup" -delete + +echo "Version update complete: $LAST_VERSION -> $NEW_VERSION" + diff --git a/webapps/core-v2/pom.xml b/webapps/core-v2/pom.xml index 0aaed09930..46ff856969 100644 --- a/webapps/core-v2/pom.xml +++ b/webapps/core-v2/pom.xml @@ -24,6 +24,11 @@ properties 5.3.15-SNAPSHOT + + uk.ac.ebi.biosamples + logback + 5.3.15-SNAPSHOT + org.springframework.boot diff --git a/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/Application.java b/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/Application.java index ddcbe92533..5014006026 100644 --- a/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -101,7 +101,7 @@ public RestTemplate restTemplate() { return new RestTemplate(); } - @Value("${spring.cloud.gcp.project-id}") + @Value("${spring.cloud.gcp.project-id:no_project}") private String enaGcpProject; @Autowired private Environment environment; diff --git a/webapps/core-v2/src/main/resources/logback-spring.xml b/webapps/core-v2/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000..e87793051d --- /dev/null +++ b/webapps/core-v2/src/main/resources/logback-spring.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/webapps/core/core-deployment.yaml b/webapps/core/core-deployment.yaml index 045616baaa..813543a466 100644 --- a/webapps/core/core-deployment.yaml +++ b/webapps/core/core-deployment.yaml @@ -27,9 +27,9 @@ spec: value: "wp-np2-40" - name: SPRING_RABBITMQ_PORT value: "5672" - - name: biosamples.schemaValidator + - name: biosamples.schema.validator.url value: "https://wwwdev.ebi.ac.uk/biosamples/biovalidator/validate" - - name: biosamples.schemaStore + - name: biosamples.schema.store.url value: "https://wwwdev.ebi.ac.uk/biosamples/schema-store" - name: SERVER_SERVLET_CONTEXT_PATH value: /biosamples diff --git a/webapps/core/pom.xml b/webapps/core/pom.xml index 0cd1f73f19..a967e0b1ec 100644 --- a/webapps/core/pom.xml +++ b/webapps/core/pom.xml @@ -26,6 +26,21 @@ properties 5.3.15-SNAPSHOT + + uk.ac.ebi.biosamples + logback + 5.3.15-SNAPSHOT + + + uk.ac.ebi.biosamples.search + proto + 0.0.1-SNAPSHOT + + + com.google.protobuf + protobuf-java + 3.25.5 + org.springframework.boot diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java index 62cf3d3f73..a49a35df0e 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java @@ -51,10 +51,7 @@ public ResponseEntity download( @RequestParam(name = "text", required = false) final String text, @RequestParam(name = "filter", required = false) final String[] filter, @RequestParam(name = "zip", required = false, defaultValue = "true") final boolean zip, - @RequestParam(name = "format", required = false) - final String - format, // there is no easy way to set accept header in html for downloading large - // files + @RequestParam(name = "format", required = false) final String format, @RequestParam(name = "count", required = false, defaultValue = "100000") final int count, final HttpServletResponse response, final HttpServletRequest request) { diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java index 3265b7e08b..c58a470911 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java @@ -22,18 +22,19 @@ import org.springframework.web.bind.annotation.*; import uk.ac.ebi.biosamples.core.model.facet.Facet; import uk.ac.ebi.biosamples.core.model.filter.Filter; -import uk.ac.ebi.biosamples.service.FacetService; import uk.ac.ebi.biosamples.service.FilterService; +import uk.ac.ebi.biosamples.service.facet.FacetingService; import uk.ac.ebi.biosamples.utils.LinkUtils; @RestController @ExposesResourceFor(Facet.class) @RequestMapping("/samples/facets") public class SampleFacetController { - private final FacetService facetService; + private final FacetingService facetService; private final FilterService filterService; - public SampleFacetController(final FacetService facetService, final FilterService filterService) { + public SampleFacetController( + final FacetingService facetService, final FilterService filterService) { this.facetService = facetService; this.filterService = filterService; } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java index 105d7b2dba..0b14efd410 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java @@ -42,6 +42,7 @@ import uk.ac.ebi.biosamples.security.model.AuthorizationProvider; import uk.ac.ebi.biosamples.service.*; import uk.ac.ebi.biosamples.service.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.facet.FacetingService; /** * Primary controller for HTML operations. @@ -57,7 +58,7 @@ public class SampleHtmlController { private final SampleService sampleService; private final SamplePageService samplePageService; private final JsonLDService jsonLDService; - private final FacetService facetService; + private final FacetingService facetService; private final FilterService filterService; private final BioSamplesProperties bioSamplesProperties; private final WebinAuthenticationService webinAuthenticationService; @@ -66,7 +67,7 @@ public SampleHtmlController( final SampleService sampleService, final SamplePageService samplePageService, final JsonLDService jsonLDService, - final FacetService facetService, + final FacetingService facetService, final FilterService filterService, final BioSamplesProperties bioSamplesProperties, final WebinAuthenticationService webinAuthenticationService) { diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java index 422ef7d184..daca984ae5 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java @@ -11,14 +11,14 @@ package uk.ac.ebi.biosamples.service; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; @@ -30,9 +30,8 @@ import uk.ac.ebi.biosamples.mongo.repository.MongoSampleRepository; import uk.ac.ebi.biosamples.mongo.service.MongoSampleToSampleConverter; import uk.ac.ebi.biosamples.mongo.service.SampleReadService; -import uk.ac.ebi.biosamples.solr.model.SolrSample; +import uk.ac.ebi.biosamples.service.search.SearchService; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; -import uk.ac.ebi.biosamples.solr.service.SolrSampleService; /** * Service layer business logic for centralising repository access and conversions between different @@ -41,19 +40,26 @@ * @author faulcon */ @Service +@Slf4j public class SamplePageService { - - private final Logger log = LoggerFactory.getLogger(getClass()); - - @Autowired private MongoSampleRepository mongoSampleRepository; - @Autowired private MongoCurationLinkRepository mongoCurationLinkRepository; - - // TODO use a ConversionService to manage all these - @Autowired private MongoSampleToSampleConverter mongoSampleToSampleConverter; - - @Autowired private SampleReadService sampleService; - - @Autowired private SolrSampleService solrSampleService; + private final MongoSampleRepository mongoSampleRepository; + private final MongoCurationLinkRepository mongoCurationLinkRepository; + private final MongoSampleToSampleConverter mongoSampleToSampleConverter; + private final SampleReadService sampleService; + private final SearchService searchService; + + public SamplePageService( + MongoSampleRepository mongoSampleRepository, + MongoCurationLinkRepository mongoCurationLinkRepository, + MongoSampleToSampleConverter mongoSampleToSampleConverter, + SampleReadService sampleService, + @Qualifier("elasticSearchService") SearchService searchService) { + this.mongoSampleRepository = mongoSampleRepository; + this.mongoCurationLinkRepository = mongoCurationLinkRepository; + this.mongoSampleToSampleConverter = mongoSampleToSampleConverter; + this.sampleService = sampleService; + this.searchService = searchService; + } public Page getSamplesOfExternalReference(final String urlHash, final Pageable pageable) { final Page pageMongoSample = @@ -79,15 +85,15 @@ public Page getSamplesByText( final Pageable pageable, final boolean applyCurations) { long startTime = System.nanoTime(); - final Page pageSolrSample = - solrSampleService.fetchSolrSampleByText(text, filters, webinSubmissionAccountId, pageable); + final Page accessionPage = + searchService.searchForAccessions( + text, new HashSet<>(filters), webinSubmissionAccountId, pageable); long endTime = System.nanoTime(); - log.trace("Got solr page in " + ((endTime - startTime) / 1000000) + "ms"); + log.trace("Got search page in {}ms", (endTime - startTime) / 1000000); startTime = System.nanoTime(); final Page>> pageFutureSample; - pageFutureSample = - pageSolrSample.map(ss -> sampleService.fetchAsync(ss.getAccession(), applyCurations)); + pageFutureSample = accessionPage.map(a -> sampleService.fetchAsync(a, applyCurations)); final Page pageSample = pageFutureSample.map( @@ -103,7 +109,7 @@ public Page getSamplesByText( } }); endTime = System.nanoTime(); - log.trace("Got mongo page content in " + ((endTime - startTime) / 1000000) + "ms"); + log.trace("Got mongo page content in {}ms", (endTime - startTime) / 1000000); return pageSample; } @@ -117,19 +123,19 @@ public CursorArrayList getSamplesByText( cursorMark = validateCursor(cursorMark); size = validatePageSize(size); - final CursorArrayList cursorSolrSample = - solrSampleService.fetchSolrSampleByText( - text, filters, webinSubmissionAccountId, cursorMark, size); + final CursorArrayList cursorAccessionList = + searchService.searchForAccessions( + text, new HashSet<>(filters), webinSubmissionAccountId, cursorMark, size); final List>> listFutureSample; listFutureSample = - cursorSolrSample.stream() - .map(s -> sampleService.fetchAsync(s.getAccession(), applyCurations)) + cursorAccessionList.stream() + .map(a -> sampleService.fetchAsync(a, applyCurations)) .collect(Collectors.toList()); final List listSample = collectSampleFutures(listFutureSample); - return new CursorArrayList<>(listSample, cursorSolrSample.getNextCursorMark()); + return new CursorArrayList<>(listSample, cursorAccessionList.getNextCursorMark()); } private List collectSampleFutures(final List>> listFutureSample) { diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java index 5e2781d543..ff4ef8fc3d 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java @@ -17,20 +17,21 @@ import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.mongo.model.MongoAnalytics; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; +import uk.ac.ebi.biosamples.service.facet.FacetingService; import uk.ac.ebi.biosamples.solr.service.SolrFacetService; import uk.ac.ebi.biosamples.solr.service.SolrFieldService; @Service public class StatService { - private final FacetService facetService; + private final FacetingService facetService; private final FilterService filterService; private final AnalyticsService analyticsService; private final SolrFacetService solrFacetService; private final SolrFieldService solrFieldService; public StatService( - final FacetService facetService, + final FacetingService facetService, final FilterService filterService, final AnalyticsService analyticsService, final SolrFacetService solrFacetService, diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java new file mode 100644 index 0000000000..aca16a0668 --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java @@ -0,0 +1,116 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.facet; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.StatusRuntimeException; +import io.micrometer.core.annotation.Timed; +import java.util.*; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import uk.ac.ebi.biosamples.BioSamplesProperties; +import uk.ac.ebi.biosamples.core.model.facet.*; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountEntry; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.search.grpc.*; +import uk.ac.ebi.biosamples.service.search.SearchFilterMapper; + +@Service("elasticFacetService") +@RequiredArgsConstructor +@Slf4j +public class ElasticFacetService implements FacetService { + private final BioSamplesProperties bioSamplesProperties; + + @Override + @Timed("biosamples.facet.page.elastic") + public List getFacets( + String searchTerm, + Set filters, + String webinId, + Pageable facetFieldPageInfo, + Pageable facetValuesPageInfo, + String facetField, + List facetFields) { + + ManagedChannel channel = + ManagedChannelBuilder.forAddress( + bioSamplesProperties.getBiosamplesSearchHost(), + bioSamplesProperties.getBiosamplesSearchPort()) + .usePlaintext() + .build(); + SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); + FacetResponse response; + try { + FacetRequest.Builder builder = FacetRequest.newBuilder(); + if (StringUtils.hasText(searchTerm)) { + builder.setText(searchTerm); + } + builder.addAllFilters(SearchFilterMapper.getSearchFilters(filters, webinId)); + if (facetFields != null) { + builder.addAllFacets(facetFields); + } + builder.setSize(facetFieldPageInfo.getPageSize()); + + response = stub.getFacets(builder.build()); + } catch (StatusRuntimeException e) { + log.error("Failed to fetch samples from remote server", e); + throw new RuntimeException("Failed to fetch samples from remote server", e); + } finally { + channel.shutdown(); + } + + List facets = response.getFacetsList(); + return convertToFacets(facets); + } + + public static List convertToFacets( + List grpcFacets) { + return grpcFacets.stream().map(ElasticFacetService::convertFacet).toList(); + } + + static Facet convertFacet(uk.ac.ebi.biosamples.search.grpc.Facet grpcFacet) { + Facet.Builder facetBuilder = + switch (grpcFacet.getType()) { + case "attr" -> + new AttributeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + case "dt" -> + new DateRangeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + case "rel" -> + new RelationFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + case "extd" -> + new ExternalReferenceDataFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + // case "sdata" -> new DateRangeFacet.Builder(grpcFacet.getField(), + // grpcFacet.getCount()); + default -> + new AttributeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + }; + return facetBuilder.build(); + } + + static LabelCountListContent convertToLabelCounts(Map labelCounts) { + List labelCountEntries = + labelCounts.entrySet().stream() + .map(e -> LabelCountEntry.build(e.getKey(), e.getValue())) + .toList(); + return new LabelCountListContent(labelCountEntries); + } +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetService.java new file mode 100644 index 0000000000..a6d05f11a2 --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetService.java @@ -0,0 +1,28 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.facet; + +import java.util.List; +import java.util.Set; +import org.springframework.data.domain.Pageable; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.filter.Filter; + +public interface FacetService { + List getFacets( + String searchTerm, + Set filters, + String webinId, + Pageable facetFieldPageInfo, + Pageable facetValuesPageInfo, + String facetField, + List facetFields); +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetingService.java similarity index 78% rename from webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetService.java rename to webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetingService.java index 96d2de2e22..0875e1222f 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetingService.java @@ -8,28 +8,29 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.service; +package uk.ac.ebi.biosamples.service.facet; import java.util.Collection; +import java.util.HashSet; import java.util.List; import org.apache.solr.client.solrj.util.ClientUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; import uk.ac.ebi.biosamples.core.model.facet.Facet; import uk.ac.ebi.biosamples.core.model.filter.Filter; -import uk.ac.ebi.biosamples.solr.service.SolrFacetService; @Service -public class FacetService { +public class FacetingService { private final Logger log = LoggerFactory.getLogger(getClass()); - private final SolrFacetService solrFacetService; + private final FacetService facetService; - public FacetService(final SolrFacetService solrFacetService) { - this.solrFacetService = solrFacetService; + public FacetingService(@Qualifier("elasticFacetService") FacetService facetService) { + this.facetService = facetService; } public List getFacets( @@ -57,8 +58,14 @@ public List getFacets( final long startTime = System.nanoTime(); final String escapedText = text == null ? null : ClientUtils.escapeQueryChars(text); final List facets = - solrFacetService.getFacets( - escapedText, filters, facetPageable, facetValuePageable, facetField, facetFields); + facetService.getFacets( + escapedText, + new HashSet<>(filters), + null, + facetPageable, + facetValuePageable, + facetField, + facetFields); final long endTime = System.nanoTime(); log.trace("Got solr facets in " + ((endTime - startTime) / 1000000) + "ms"); diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SearchFacetMapper.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SearchFacetMapper.java new file mode 100644 index 0000000000..6ae53efaf0 --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SearchFacetMapper.java @@ -0,0 +1,114 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.facet; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; +import uk.ac.ebi.biosamples.search.grpc.*; + +public class SearchFacetMapper { + + public static List getSearchFilters( + Set filters, String webinId) { + List grpcFilters = new ArrayList<>(); + grpcFilters.add(getPrivateSearchFilter(webinId)); + if (!CollectionUtils.isEmpty(filters)) { + getSearchFilters(filters, grpcFilters); + } + return grpcFilters; + } + + private static void getSearchFilters( + Set filters, List grpcFilters) { + for (uk.ac.ebi.biosamples.core.model.filter.Filter filter : filters) { + Filter.Builder filterBuilder = Filter.newBuilder(); + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AccessionFilter f) { + f.getContent() + .ifPresent( + accession -> + filterBuilder.setAccession( + AccessionFilter.newBuilder().setAccession(accession))); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.NameFilter f) { + f.getContent() + .ifPresent(name -> filterBuilder.setName(NameFilter.newBuilder().setName(name))); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AuthenticationFilter f) { + f.getContent() + .ifPresent( + auth -> { // todo domain + filterBuilder.setWebin(WebinIdFilter.newBuilder().setWebinId(auth)); + }); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter f) { + DateRangeFilter.DateField dateField = + switch (f.getLabel()) { + case "update" -> DateRangeFilter.DateField.UPDATE; + case "create" -> DateRangeFilter.DateField.CREATE; + case "release" -> DateRangeFilter.DateField.RELEASE; + case "submitted" -> DateRangeFilter.DateField.SUBMITTED; + default -> throw new IllegalArgumentException("Unknown date field " + f.getLabel()); + }; + + f.getContent() + .ifPresent( + dateRange -> + filterBuilder.setDateRange( + DateRangeFilter.newBuilder() + .setField(dateField) + .setFrom(dateRange.getFrom().toString()) + .setTo(dateRange.getUntil().toString()))); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AttributeFilter f) { + AttributeFilter.Builder attributeFilterBuilder = AttributeFilter.newBuilder(); + attributeFilterBuilder.setField(f.getLabel()); + f.getContent() + .ifPresent( + attribute -> + attributeFilterBuilder.addAllValues(List.of(attribute))); // todo aggregation + filterBuilder.setAttribute(attributeFilterBuilder); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.RelationFilter f) { + RelationshipFilter.Builder relationshipFilterBuilder = RelationshipFilter.newBuilder(); + relationshipFilterBuilder.setType(f.getLabel()); + f.getContent().ifPresent(relationshipFilterBuilder::setTarget); + filterBuilder.setRelationship(relationshipFilterBuilder); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.InverseRelationFilter f) { + RelationshipFilter.Builder relationshipFilterBuilder = RelationshipFilter.newBuilder(); + relationshipFilterBuilder.setType(f.getLabel()); + f.getContent().ifPresent(relationshipFilterBuilder::setSource); + filterBuilder.setRelationship(relationshipFilterBuilder); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.ExternalReferenceDataFilter f) { + ExternalRefFilter.Builder externalRefFilterBuilder = ExternalRefFilter.newBuilder(); + externalRefFilterBuilder.setArchive(f.getLabel()); + f.getContent().ifPresent(externalRefFilterBuilder::setAccession); + filterBuilder.setExternal(externalRefFilterBuilder); + } + + // todo SraAccessionFilter, Structured data filter + + grpcFilters.add(filterBuilder.build()); + } + } + + private static Filter getPrivateSearchFilter(String webinId) { + PublicFilter.Builder publicFilterBuilder = PublicFilter.newBuilder(); + if (StringUtils.hasText(webinId)) { + publicFilterBuilder.setWebinId(webinId); + } + return Filter.newBuilder().setPublic(publicFilterBuilder.build()).build(); + } +} diff --git a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFacetService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SolrFacetService.java similarity index 95% rename from core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFacetService.java rename to webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SolrFacetService.java index 7a5a53b4dd..4746634b52 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFacetService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SolrFacetService.java @@ -26,11 +26,12 @@ import uk.ac.ebi.biosamples.core.model.facet.Facet; import uk.ac.ebi.biosamples.core.model.facet.FacetHelper; import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.service.facet.FacetService; import uk.ac.ebi.biosamples.solr.model.field.SolrSampleField; import uk.ac.ebi.biosamples.solr.repo.SolrSampleRepository; -@Service -public class SolrFacetService { +@Service("solrFacetService") +public class SolrFacetService implements FacetService { private static final int TIME_ALLOWED = 55; private final SolrSampleRepository solrSampleRepository; private final SolrFieldService solrFieldService; @@ -45,9 +46,11 @@ public SolrFacetService( this.solrFilterService = solrFilterService; } + @Override public List getFacets( final String searchTerm, - final Collection filters, + final Set filters, + final String webinId, final Pageable facetFieldPageInfo, final Pageable facetValuesPageInfo, final String facetField, @@ -125,7 +128,14 @@ public List getFacets( final Pageable facetFieldPageInfo, final Pageable facetValuesPageInfo) { - return getFacets(searchTerm, filters, facetFieldPageInfo, facetValuesPageInfo, null, null); + return getFacets( + searchTerm, + new HashSet<>(filters), + null, + facetFieldPageInfo, + facetValuesPageInfo, + null, + null); } private List> getFacetFields( diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java new file mode 100644 index 0000000000..c489f07c6c --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java @@ -0,0 +1,182 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.search; + +import com.google.protobuf.Timestamp; +import com.google.protobuf.util.Timestamps; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.StatusRuntimeException; +import io.micrometer.core.annotation.Timed; +import java.time.Instant; +import java.util.List; +import java.util.Set; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.*; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import uk.ac.ebi.biosamples.BioSamplesProperties; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.search.grpc.*; +import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; + +@Service("elasticSearchService") +@RequiredArgsConstructor +@Slf4j +public class ElasticSearchService implements SearchService { + private final BioSamplesProperties bioSamplesProperties; + + @Override + @Timed("biosamples.search.page.elastic") + public Page searchForAccessions( + String searchTerm, Set filters, String webinId, Pageable pageable) { + ManagedChannel channel = + ManagedChannelBuilder.forAddress( + bioSamplesProperties.getBiosamplesSearchHost(), + bioSamplesProperties.getBiosamplesSearchPort()) + .usePlaintext() + .build(); + SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); + SearchResponse response; + try { + SearchRequest.Builder builder = SearchRequest.newBuilder(); + if (StringUtils.hasText(searchTerm)) { + builder.setText(searchTerm); + } + builder.addAllFilters(SearchFilterMapper.getSearchFilters(filters, webinId)); + builder.setSize(pageable.getPageSize()); + builder.setNumber(pageable.getPageNumber()); + builder.addAllSort(pageable.getSort().stream().map(Sort.Order::toString).toList()); + + response = stub.searchSamples(builder.build()); + } catch (StatusRuntimeException e) { + log.error("Failed to fetch samples from remote server", e); + throw new RuntimeException("Failed to fetch samples from remote server", e); + } finally { + channel.shutdown(); + } + + List accessions = response.getAccessionsList(); + Sort sort = + Sort.by( + response.getSortList().stream() + .map(s -> new Sort.Order(Sort.Direction.ASC, s)) + .toList()); + PageRequest page = PageRequest.of(response.getNumber(), response.getSize(), sort); + long totalElements = response.getTotalElements(); + SearchAfter searchAfter = response.getSearchAfter(); + + return new SearchAfterPage<>( + accessions, page, totalElements, searchAfter.getUpdate(), searchAfter.getAccession()); + } + + @Override + @Timed("biosamples.search.cursor.elastic") + public CursorArrayList searchForAccessions( + String searchTerm, Set filters, String webinId, String cursor, int size) { + SearchAfter searchAfter = null; + String[] cursorParts = cursor.split(","); + if (cursorParts.length == 2) { + Instant update = Instant.parse(cursorParts[0].trim()); + String accession = cursorParts[1].trim(); + searchAfter = + SearchAfter.newBuilder() + .setUpdate( + Timestamp.newBuilder() + .setSeconds(update.getEpochSecond()) + .setNanos(update.getNano()) + .build()) + .setAccession(accession) + .build(); + } + + ManagedChannel channel = + ManagedChannelBuilder.forAddress( + bioSamplesProperties.getBiosamplesSearchHost(), + bioSamplesProperties.getBiosamplesSearchPort()) + .usePlaintext() + .build(); + SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); + SearchResponse response; + try { + SearchRequest.Builder builder = SearchRequest.newBuilder(); + if (StringUtils.hasText(searchTerm)) { + builder.setText(searchTerm); + } + builder.addAllFilters(SearchFilterMapper.getSearchFilters(filters, webinId)); + builder.setSize(size); + if (searchAfter != null) { + builder.setSearchAfter(searchAfter); + } + + response = stub.searchSamples(builder.build()); + } catch (StatusRuntimeException e) { + log.error("Failed to fetch samples from remote server", e); + throw new RuntimeException("Failed to fetch samples from remote server", e); + } finally { + channel.shutdown(); + } + + List accessions = response.getAccessionsList(); + SearchAfter newSearchAfter = response.getSearchAfter(); + + if (StringUtils.hasText(newSearchAfter.getAccession())) { + cursor = + Timestamps.toString(newSearchAfter.getUpdate()) + "," + newSearchAfter.getAccession(); + } + + return new CursorArrayList<>(accessions, cursor); + } + + /*public OutputStream searchForAccessionsStream(String searchTerm, Set filters, String webinId, String cursor, int size) { + SearchAfter searchAfter = null; + String[] cursorParts = cursor.split(","); + if (cursorParts.length == 2) { + Instant update = Instant.parse(cursorParts[0].trim()); + String accession = cursorParts[1].trim(); + searchAfter = SearchAfter.newBuilder() + .setUpdate(Timestamp.newBuilder().setSeconds(update.getEpochSecond()).setNanos(update.getNano()).build()) + .setAccession(accession).build(); + } + + ManagedChannel channel = ManagedChannelBuilder.forAddress(bioSamplesProperties.getBiosamplesSearchHost(), 9090).usePlaintext().build(); + SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); + Iterator response; + try { + StreamRequest.Builder builder = StreamRequest.newBuilder(); + if (StringUtils.hasText(searchTerm)) { + builder.setText(searchTerm); + } + builder.addAllFilters(SearchFilterMapper.getSearchFilters(filters, webinId)); + if (searchAfter != null) { + builder.setSearchAfter(searchAfter); + } + + response = stub.streamSamples(builder.build()); + } catch (StatusRuntimeException e) { + log.warn("Failed to fetch samples from remote server", e); + throw new RuntimeException("Failed to fetch samples from remote server", e); + } finally { + channel.shutdown(); + } + + List accessionList = new ArrayList<>(); + while (response.hasNext()) { + StreamResponse streamResponse = response.next(); + searchAfter = streamResponse.getSearchAfter(); + accessionList.add(streamResponse.getAccession()); + cursor = Timestamps.toString(searchAfter.getUpdate()) + "," + searchAfter.getAccession(); + } + + return new CursorArrayList<>(accessionList, cursor); + }*/ +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java new file mode 100644 index 0000000000..2efb33d0b2 --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java @@ -0,0 +1,32 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.search; + +import com.google.protobuf.Timestamp; +import java.time.Instant; +import java.util.List; +import lombok.Getter; +import lombok.NonNull; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; + +@Getter +public class SearchAfterPage extends PageImpl { + private final Instant update; + private final String accession; + + public SearchAfterPage( + List content, Pageable pageable, long total, @NonNull Timestamp update, String accession) { + super(content, pageable, total); + this.update = Instant.ofEpochSecond(update.getSeconds(), update.getNanos()); + this.accession = accession; + } +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchFilterMapper.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchFilterMapper.java new file mode 100644 index 0000000000..83f3f4508c --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchFilterMapper.java @@ -0,0 +1,166 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.search; + +import java.util.*; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; +import uk.ac.ebi.biosamples.core.model.filter.AuthenticationFilter; +import uk.ac.ebi.biosamples.core.model.filter.ExternalReferenceDataFilter; +import uk.ac.ebi.biosamples.core.model.filter.InverseRelationFilter; +import uk.ac.ebi.biosamples.core.model.filter.RelationFilter; +import uk.ac.ebi.biosamples.search.grpc.*; + +public class SearchFilterMapper { + + public static List getSearchFilters( + Set filters, String webinId) { + List grpcFilters = new ArrayList<>(); + grpcFilters.add(getPrivateSearchFilter(webinId)); + if (!CollectionUtils.isEmpty(filters)) { + getSearchFilters(filters, grpcFilters); + } + return grpcFilters; + } + + private static void getSearchFilters( + Set filters, List grpcFilters) { + Map filterMap = new HashMap<>(); + for (uk.ac.ebi.biosamples.core.model.filter.Filter filter : filters) { + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AccessionFilter f) { + getAccessionSearchFilter(f, grpcFilters); + } else if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.NameFilter f) { + getNameSearchFilter(f, grpcFilters); + } else if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AuthenticationFilter f) { + getAuthSearchFilter(f, grpcFilters); + } else if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter f) { + getDateRangeSearchFilter(f, grpcFilters); + } else if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AttributeFilter f) { + getAttributeSearchFilter(f, filterMap); + } else if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.RelationFilter f) { + getRelationshipSearchFilter(f, grpcFilters); + } else if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.InverseRelationFilter f) { + getInverseRelationshipSearchFilter(f, grpcFilters); + } else if (filter + instanceof uk.ac.ebi.biosamples.core.model.filter.ExternalReferenceDataFilter f) { + getExternalReferenceSearchFilter(f, grpcFilters); + } else { + // todo SraAccessionFilter, Structured data filter + throw new RuntimeException("Unsupported filter type " + filter.getClass().getName()); + } + } + filterMap.forEach((k, v) -> grpcFilters.add(v.build())); // allows OR filter for attributes + } + + private static void getAuthSearchFilter(AuthenticationFilter f, List grpcFilters) { + f.getContent() + .ifPresent( + auth -> { // todo domain + grpcFilters.add( + Filter.newBuilder() + .setWebin(WebinIdFilter.newBuilder().setWebinId(auth)) + .build()); + }); + } + + private static void getNameSearchFilter( + uk.ac.ebi.biosamples.core.model.filter.NameFilter f, List grpcFilters) { + f.getContent() + .ifPresent( + name -> + grpcFilters.add( + Filter.newBuilder().setName(NameFilter.newBuilder().setName(name)).build())); + } + + private static void getAccessionSearchFilter( + uk.ac.ebi.biosamples.core.model.filter.AccessionFilter f, List grpcFilters) { + f.getContent() + .ifPresent( + accession -> + grpcFilters.add( + Filter.newBuilder() + .setAccession(AccessionFilter.newBuilder().setAccession(accession)) + .build())); + } + + private static void getExternalReferenceSearchFilter( + ExternalReferenceDataFilter f, List grpcFilters) { + ExternalRefFilter.Builder externalRefFilterBuilder = ExternalRefFilter.newBuilder(); + f.getContent().ifPresent(externalRefFilterBuilder::setArchive); + grpcFilters.add(Filter.newBuilder().setExternal(externalRefFilterBuilder).build()); + } + + private static void getInverseRelationshipSearchFilter( + InverseRelationFilter f, List grpcFilters) { + RelationshipFilter.Builder relationshipFilterBuilder = RelationshipFilter.newBuilder(); + relationshipFilterBuilder.setType(f.getLabel()); + f.getContent().ifPresent(relationshipFilterBuilder::setSource); + grpcFilters.add(Filter.newBuilder().setRelationship(relationshipFilterBuilder).build()); + } + + private static void getRelationshipSearchFilter(RelationFilter f, List grpcFilters) { + RelationshipFilter.Builder relationshipFilterBuilder = RelationshipFilter.newBuilder(); + if (f.getLabel().equalsIgnoreCase("relationship type")) { + f.getContent().ifPresent(relationshipFilterBuilder::setType); + } else if (f.getLabel().equalsIgnoreCase("relationship source")) { + f.getContent().ifPresent(relationshipFilterBuilder::setSource); + } else if (f.getLabel().equalsIgnoreCase("relationship target")) { + f.getContent().ifPresent(relationshipFilterBuilder::setTarget); + } + + grpcFilters.add(Filter.newBuilder().setRelationship(relationshipFilterBuilder).build()); + } + + private static void getDateRangeSearchFilter( + uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter f, List grpcFilters) { + f.getContent() + .ifPresent( + dateRange -> { + DateRangeFilter.DateField dateField = + switch (f.getLabel()) { + case "update" -> DateRangeFilter.DateField.UPDATE; + case "create" -> DateRangeFilter.DateField.CREATE; + case "release" -> DateRangeFilter.DateField.RELEASE; + case "submitted" -> DateRangeFilter.DateField.SUBMITTED; + default -> + throw new IllegalArgumentException("Unknown date field " + f.getLabel()); + }; + grpcFilters.add( + Filter.newBuilder() + .setDateRange( + DateRangeFilter.newBuilder() + .setField(dateField) + .setFrom(dateRange.getFrom().toString()) + .setTo(dateRange.getUntil().toString())) + .build()); + }); + } + + private static void getAttributeSearchFilter( + uk.ac.ebi.biosamples.core.model.filter.AttributeFilter filter, + Map filterMap) { + Filter.Builder fb = + filterMap.getOrDefault( + filter.getLabel(), + Filter.newBuilder() + .setAttribute(AttributeFilter.newBuilder().setField(filter.getLabel()))); + filter.getContent().ifPresent(attribute -> fb.getAttributeBuilder().addValues(attribute)); + filterMap.putIfAbsent(filter.getLabel(), fb); + } + + private static Filter getPrivateSearchFilter(String webinId) { + PublicFilter.Builder publicFilterBuilder = PublicFilter.newBuilder(); + if (StringUtils.hasText(webinId)) { + publicFilterBuilder.setWebinId(webinId); + } + return Filter.newBuilder().setPublic(publicFilterBuilder.build()).build(); + } +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java new file mode 100644 index 0000000000..79584a65e7 --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java @@ -0,0 +1,25 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.search; + +import java.util.Set; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; + +public interface SearchService { + Page searchForAccessions( + String searchTerm, Set filters, String webinId, Pageable pageable); + + CursorArrayList searchForAccessions( + String searchTerm, Set filters, String webinId, String cursor, int size); +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java new file mode 100644 index 0000000000..db63203ab8 --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java @@ -0,0 +1,50 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.search; + +import io.micrometer.core.annotation.Timed; +import java.util.List; +import java.util.Set; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.model.SolrSample; +import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; +import uk.ac.ebi.biosamples.solr.service.SolrSampleService; + +@Service("solrSearchService") +@RequiredArgsConstructor +@Slf4j +public class SolrSearchService implements SearchService { + private final SolrSampleService solrSampleService; + + @Override + @Timed("biosamples.search.page.solr") + public Page searchForAccessions( + String searchTerm, Set filters, String webinId, Pageable pageable) { + return solrSampleService + .fetchSolrSampleByText(searchTerm, filters, webinId, pageable) + .map(SolrSample::getAccession); + } + + @Override + @Timed("biosamples.search.cursor.solr") + public CursorArrayList searchForAccessions( + String searchTerm, Set filters, String webinId, String cursor, int size) { + CursorArrayList samples = + solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, cursor, size); + List accessions = samples.stream().map(SolrSample::getAccession).toList(); + return new CursorArrayList<>(accessions, samples.getNextCursorMark()); + } +} diff --git a/webapps/core/src/main/resources/logback-spring.xml b/webapps/core/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000..e87793051d --- /dev/null +++ b/webapps/core/src/main/resources/logback-spring.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/webapps/core/src/main/resources/templates/fragments/facets.html b/webapps/core/src/main/resources/templates/fragments/facets.html index 8204be2c46..b13c21ee8b 100644 --- a/webapps/core/src/main/resources/templates/fragments/facets.html +++ b/webapps/core/src/main/resources/templates/fragments/facets.html @@ -24,6 +24,7 @@ +
Showing approximate facet counts for faster response