From f5d83744f4ade5610054ea6193a5cf19cf94c9cb Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Wed, 25 Jun 2025 10:19:03 +0100 Subject: [PATCH 01/86] search library integration --- pom.xml | 5 ++ webapps/core/pom.xml | 5 ++ .../service/search/ESSearchService.java | 47 +++++++++++++++++++ .../service/search/SearchService.java | 13 +++++ .../service/search/SolrSearchService.java | 35 ++++++++++++++ 5 files changed, 105 insertions(+) create mode 100644 webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ESSearchService.java create mode 100644 webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java create mode 100644 webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java diff --git a/pom.xml b/pom.xml index 049e1341d..3c9afcbdf 100644 --- a/pom.xml +++ b/pom.xml @@ -107,6 +107,11 @@ biosamples-v4 https://gitlab.ebi.ac.uk/api/v4/projects/2669/packages/maven + + + gitlab-maven + https://gitlab.ebi.ac.uk/api/v4/projects/5516/packages/maven + diff --git a/webapps/core/pom.xml b/webapps/core/pom.xml index 59278683b..c7b892f9c 100644 --- a/webapps/core/pom.xml +++ b/webapps/core/pom.xml @@ -66,6 +66,11 @@ models-curami 5.3.12-SNAPSHOT + + uk.ac.ebi.biosamples.search + proto + 0.0.1-SNAPSHOT + org.springframework.boot diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ESSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ESSearchService.java new file mode 100644 index 000000000..5feebc41d --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ESSearchService.java @@ -0,0 +1,47 @@ +package uk.ac.ebi.biosamples.service.search; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.StatusRuntimeException; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.search.grpc.*; +import uk.ac.ebi.biosamples.solr.model.SolrSample; +import uk.ac.ebi.biosamples.solr.service.SolrSampleService; + +import java.util.List; +import java.util.Set; + +@Service +@RequiredArgsConstructor +@Slf4j +public class ESSearchService implements SearchService { + private final SolrSampleService solrSampleService; + + @Override + public List searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable) { + + ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 9090).usePlaintext().build(); + SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); + SearchResponse response = null; + try { + response = stub.searchSamples(SearchRequest.newBuilder().setText(searchTerm).build()); + } catch (StatusRuntimeException e) { + log.warn("Failed to fetch samples from remote server", e); + } + + return response.getAccessionsList(); + } + + @Override + public List searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size) { + return solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, cursor, size) + .stream() + .map(SolrSample::getAccession) + .toList(); + } +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java new file mode 100644 index 000000000..bb2f19d15 --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java @@ -0,0 +1,13 @@ +package uk.ac.ebi.biosamples.service.search; + +import org.springframework.data.domain.Pageable; +import uk.ac.ebi.biosamples.model.filter.Filter; + +import java.util.List; +import java.util.Set; + +public interface SearchService { + List searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable); + + List searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size); +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java new file mode 100644 index 000000000..a5cfd0664 --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java @@ -0,0 +1,35 @@ +package uk.ac.ebi.biosamples.service.search; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.model.SolrSample; +import uk.ac.ebi.biosamples.solr.service.SolrSampleService; + +import java.util.List; +import java.util.Set; + +@Service +@RequiredArgsConstructor +@Slf4j +public class SolrSearchService implements SearchService { + private final SolrSampleService solrSampleService; + + @Override + public List searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable) { + return solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, pageable).getContent() + .stream() + .map(SolrSample::getAccession) + .toList(); + } + + @Override + public List searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size) { + return solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, cursor, size) + .stream() + .map(SolrSample::getAccession) + .toList(); + } +} From 022894fef453d8707c728ba1a9eab5b76915b590 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Thu, 26 Jun 2025 10:10:58 +0100 Subject: [PATCH 02/86] extract solr search service --- .../service/search/ESSearchService.java | 47 ---------------- .../service/search/ElasticSearchService.java | 56 +++++++++++++++++++ .../service/search/SearchService.java | 7 ++- .../service/search/SolrSearchService.java | 18 +++--- 4 files changed, 70 insertions(+), 58 deletions(-) delete mode 100644 webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ESSearchService.java create mode 100644 webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ESSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ESSearchService.java deleted file mode 100644 index 5feebc41d..000000000 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ESSearchService.java +++ /dev/null @@ -1,47 +0,0 @@ -package uk.ac.ebi.biosamples.service.search; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.data.domain.Pageable; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestTemplate; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.search.grpc.*; -import uk.ac.ebi.biosamples.solr.model.SolrSample; -import uk.ac.ebi.biosamples.solr.service.SolrSampleService; - -import java.util.List; -import java.util.Set; - -@Service -@RequiredArgsConstructor -@Slf4j -public class ESSearchService implements SearchService { - private final SolrSampleService solrSampleService; - - @Override - public List searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable) { - - ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 9090).usePlaintext().build(); - SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); - SearchResponse response = null; - try { - response = stub.searchSamples(SearchRequest.newBuilder().setText(searchTerm).build()); - } catch (StatusRuntimeException e) { - log.warn("Failed to fetch samples from remote server", e); - } - - return response.getAccessionsList(); - } - - @Override - public List searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size) { - return solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, cursor, size) - .stream() - .map(SolrSample::getAccession) - .toList(); - } -} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java new file mode 100644 index 000000000..4590090a2 --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java @@ -0,0 +1,56 @@ +package uk.ac.ebi.biosamples.service.search; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.StatusRuntimeException; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.search.grpc.SearchGrpc; +import uk.ac.ebi.biosamples.search.grpc.SearchRequest; +import uk.ac.ebi.biosamples.search.grpc.SearchResponse; +import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; + +import java.util.Iterator; +import java.util.Set; + +@Service("elasticSearchService") +@RequiredArgsConstructor +@Slf4j +public class ElasticSearchService implements SearchService { + + @Override + public Page searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable) { + + ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 9090).usePlaintext().build(); + SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); + SearchResponse response; + try { + response = stub.searchSamples(SearchRequest.newBuilder().setText(searchTerm).build()); + } catch (StatusRuntimeException e) { + log.error("Failed to fetch samples from remote server", e); + throw new RuntimeException("Failed to fetch samples from remote server", e); + } + + response.getAccessionsList(); + return Page.empty(); + } + + @Override + public CursorArrayList searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size) { + ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 9090).usePlaintext().build(); + SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); + Iterator response; + try { + response = stub.streamSamples(SearchRequest.newBuilder().setText(searchTerm).build()); + } catch (StatusRuntimeException e) { + log.warn("Failed to fetch samples from remote server", e); + throw new RuntimeException("Failed to fetch samples from remote server", e); + } + + return new CursorArrayList<>(response.next().getAccessionsList(), cursor); + } +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java index bb2f19d15..f41b55344 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java @@ -1,13 +1,14 @@ package uk.ac.ebi.biosamples.service.search; +import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; -import java.util.List; import java.util.Set; public interface SearchService { - List searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable); + Page searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable); - List searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size); + CursorArrayList searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size); } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java index a5cfd0664..04a451b2d 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java @@ -2,34 +2,36 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; import uk.ac.ebi.biosamples.model.filter.Filter; import uk.ac.ebi.biosamples.solr.model.SolrSample; +import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; import uk.ac.ebi.biosamples.solr.service.SolrSampleService; import java.util.List; import java.util.Set; -@Service +@Service("solrSearchService") @RequiredArgsConstructor @Slf4j public class SolrSearchService implements SearchService { private final SolrSampleService solrSampleService; @Override - public List searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable) { - return solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, pageable).getContent() - .stream() - .map(SolrSample::getAccession) - .toList(); + public Page searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable) { + return solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, pageable) + .map(SolrSample::getAccession); } @Override - public List searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size) { - return solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, cursor, size) + public CursorArrayList searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size) { + CursorArrayList samples = solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, cursor, size); + List accessions = samples .stream() .map(SolrSample::getAccession) .toList(); + return new CursorArrayList<>(accessions, samples.getNextCursorMark()); } } From 37bde6bf816a0d760b5e8f8bb6b162ed60013706 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 30 Jun 2025 12:19:01 +0100 Subject: [PATCH 03/86] feat: es integration --- .../biosamples/service/AccessionsService.java | 57 -------------- .../biosamples/service/AccessionsService.java | 55 ++++++++++++++ .../biosamples/service/SamplePageService.java | 74 ++++++++++--------- .../service/search/ElasticSearchService.java | 6 +- .../service/search/SolrSearchService.java | 3 + 5 files changed, 102 insertions(+), 93 deletions(-) delete mode 100644 utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java create mode 100644 webapps/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java b/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java deleted file mode 100644 index cda397ea1..000000000 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java +++ /dev/null @@ -1,57 +0,0 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.util.Collection; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageRequest; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.solr.model.SolrSample; -import uk.ac.ebi.biosamples.solr.service.SolrSampleService; -import uk.ac.ebi.biosamples.utils.LinkUtils; - -@Service -public class AccessionsService { - private final SolrSampleService solrSampleService; - private final FilterService filterService; - - public AccessionsService( - final SolrSampleService solrSampleService, final FilterService filterService) { - this.solrSampleService = solrSampleService; - this.filterService = filterService; - } - - public Page getAccessions( - final String text, - final String[] requestfilters, - final String webinSubmissionAccountId, - final Integer page, - final Integer size) { - final PageRequest pageable = PageRequest.of(page, size); - final String decodedText = LinkUtils.decodeText(text); - final String[] decodedFilter = LinkUtils.decodeTexts(requestfilters); - final Collection filtersAfterDecode = filterService.getFiltersCollection(decodedFilter); - - return fetchAccessions(pageable, decodedText, filtersAfterDecode, webinSubmissionAccountId); - } - - private Page fetchAccessions( - final PageRequest pageable, - final String decodedText, - final Collection filtersAfterDecode, - final String webinSubmissionAccountId) { - final Page pageSolrSample = - solrSampleService.fetchSolrSampleByText( - decodedText, filtersAfterDecode, webinSubmissionAccountId, pageable); - return pageSolrSample.map(SolrSample::getAccession); - } -} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java new file mode 100644 index 000000000..ec13a0ffa --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java @@ -0,0 +1,55 @@ +/* + * Copyright 2021 EMBL - European Bioinformatics Institute + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package uk.ac.ebi.biosamples.service; + +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.service.search.SearchService; +import uk.ac.ebi.biosamples.utils.LinkUtils; + +import java.util.Collection; +import java.util.HashSet; + +@Service +public class AccessionsService { + private final FilterService filterService; + private final SearchService searchService; + + public AccessionsService(FilterService filterService, + @Qualifier("solrSearchService") SearchService searchService) { + this.filterService = filterService; + this.searchService = searchService; + } + + public Page getAccessions(String text, + String[] requestFilters, + String webinSubmissionAccountId, + Integer page, + Integer size) { + PageRequest pageable = PageRequest.of(page, size); + String decodedText = LinkUtils.decodeText(text); + String[] decodedFilter = LinkUtils.decodeTexts(requestFilters); + Collection filtersAfterDecode = filterService.getFiltersCollection(decodedFilter); + + return fetchAccessions(pageable, decodedText, filtersAfterDecode, webinSubmissionAccountId); + } + + private Page fetchAccessions(PageRequest pageable, + String decodedText, + Collection filtersAfterDecode, + String webinSubmissionAccountId) { + return searchService.searchForAccessions( + decodedText, new HashSet<>(filtersAfterDecode), webinSubmissionAccountId, pageable); + } +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java index 18f320c67..55daaa083 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java @@ -10,15 +10,8 @@ */ package uk.ac.ebi.biosamples.service; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; @@ -30,9 +23,16 @@ import uk.ac.ebi.biosamples.mongo.repository.MongoSampleRepository; import uk.ac.ebi.biosamples.mongo.service.MongoSampleToSampleConverter; import uk.ac.ebi.biosamples.mongo.service.SampleReadService; -import uk.ac.ebi.biosamples.solr.model.SolrSample; +import uk.ac.ebi.biosamples.service.search.SearchService; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; -import uk.ac.ebi.biosamples.solr.service.SolrSampleService; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.stream.Collectors; /** * Service layer business logic for centralising repository access and conversions between different @@ -41,19 +41,25 @@ * @author faulcon */ @Service +@Slf4j public class SamplePageService { - - private final Logger log = LoggerFactory.getLogger(getClass()); - - @Autowired private MongoSampleRepository mongoSampleRepository; - @Autowired private MongoCurationLinkRepository mongoCurationLinkRepository; - - // TODO use a ConversionService to manage all these - @Autowired private MongoSampleToSampleConverter mongoSampleToSampleConverter; - - @Autowired private SampleReadService sampleService; - - @Autowired private SolrSampleService solrSampleService; + private final MongoSampleRepository mongoSampleRepository; + private final MongoCurationLinkRepository mongoCurationLinkRepository; + private final MongoSampleToSampleConverter mongoSampleToSampleConverter; + private final SampleReadService sampleService; + private final SearchService searchService; + + public SamplePageService(MongoSampleRepository mongoSampleRepository, + MongoCurationLinkRepository mongoCurationLinkRepository, + MongoSampleToSampleConverter mongoSampleToSampleConverter, + SampleReadService sampleService, + @Qualifier("solrSearchService") SearchService searchService) { + this.mongoSampleRepository = mongoSampleRepository; + this.mongoCurationLinkRepository = mongoCurationLinkRepository; + this.mongoSampleToSampleConverter = mongoSampleToSampleConverter; + this.sampleService = sampleService; + this.searchService = searchService; + } public Page getSamplesOfExternalReference(final String urlHash, final Pageable pageable) { final Page pageMongoSample = @@ -79,15 +85,14 @@ public Page getSamplesByText( final Pageable pageable, final boolean applyCurations) { long startTime = System.nanoTime(); - final Page pageSolrSample = - solrSampleService.fetchSolrSampleByText(text, filters, webinSubmissionAccountId, pageable); + final Page accessionPage = + searchService.searchForAccessions(text, new HashSet<>(filters), webinSubmissionAccountId, pageable); long endTime = System.nanoTime(); - log.trace("Got solr page in " + ((endTime - startTime) / 1000000) + "ms"); + log.trace("Got search page in {}ms", (endTime - startTime) / 1000000); startTime = System.nanoTime(); final Page>> pageFutureSample; - pageFutureSample = - pageSolrSample.map(ss -> sampleService.fetchAsync(ss.getAccession(), applyCurations)); + pageFutureSample = accessionPage.map(a -> sampleService.fetchAsync(a, applyCurations)); final Page pageSample = pageFutureSample.map( @@ -103,7 +108,7 @@ public Page getSamplesByText( } }); endTime = System.nanoTime(); - log.trace("Got mongo page content in " + ((endTime - startTime) / 1000000) + "ms"); + log.trace("Got mongo page content in {}ms", (endTime - startTime) / 1000000); return pageSample; } @@ -117,19 +122,18 @@ public CursorArrayList getSamplesByText( cursorMark = validateCursor(cursorMark); size = validatePageSize(size); - final CursorArrayList cursorSolrSample = - solrSampleService.fetchSolrSampleByText( - text, filters, webinSubmissionAccountId, cursorMark, size); + final CursorArrayList cursorAccessionList = + searchService.searchForAccessions(text, new HashSet<>(filters), webinSubmissionAccountId, cursorMark, size); final List>> listFutureSample; listFutureSample = - cursorSolrSample.stream() - .map(s -> sampleService.fetchAsync(s.getAccession(), applyCurations)) + cursorAccessionList.stream() + .map(a -> sampleService.fetchAsync(a, applyCurations)) .collect(Collectors.toList()); final List listSample = collectSampleFutures(listFutureSample); - return new CursorArrayList<>(listSample, cursorSolrSample.getNextCursorMark()); + return new CursorArrayList<>(listSample, cursorAccessionList.getNextCursorMark()); } private List collectSampleFutures(final List>> listFutureSample) { diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java index 4590090a2..8753398d5 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java @@ -3,6 +3,7 @@ import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.StatusRuntimeException; +import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Page; @@ -23,6 +24,7 @@ public class ElasticSearchService implements SearchService { @Override + @Timed("biosamples.search.page.elastic") public Page searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable) { ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 9090).usePlaintext().build(); @@ -40,6 +42,7 @@ public Page searchForAccessions(String searchTerm, Set filters, } @Override + @Timed("biosamples.search.cursor.elastic") public CursorArrayList searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size) { ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 9090).usePlaintext().build(); SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); @@ -51,6 +54,7 @@ public CursorArrayList searchForAccessions(String searchTerm, Set(response.next().getAccessionsList(), cursor); + SearchResponse searchResponse = response.next(); + return new CursorArrayList<>(searchResponse.getAccessionsList(), searchResponse.getPage()..getSearchAfter()); } } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java index 04a451b2d..c1ecceed2 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java @@ -1,5 +1,6 @@ package uk.ac.ebi.biosamples.service.search; +import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Page; @@ -20,12 +21,14 @@ public class SolrSearchService implements SearchService { private final SolrSampleService solrSampleService; @Override + @Timed("biosamples.search.page.solr") public Page searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable) { return solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, pageable) .map(SolrSample::getAccession); } @Override + @Timed("biosamples.search.cursor.solr") public CursorArrayList searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size) { CursorArrayList samples = solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, cursor, size); List accessions = samples From 7f60e8c09e435fe56113169e0da92f4c998cff90 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 7 Jul 2025 10:50:21 +0100 Subject: [PATCH 04/86] reindex runnder, and MessageConfig changes before moving to ES --- .../java/uk/ac/ebi/biosamples/ReindexRunner.java | 15 +++++++++++---- .../uk/ac/ebi/biosamples/ReindexRunnerTest.java | 4 +++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java index 441ab0f1b..8451d4a09 100644 --- a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java +++ b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java @@ -12,6 +12,8 @@ import java.util.*; import java.util.concurrent.*; + +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang.IncompleteArgumentException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,15 +50,17 @@ public class ReindexRunner implements ApplicationRunner { private final AmqpTemplate amqpTemplate; private final SampleReadService sampleReadService; private final MongoOperations mongoOperations; + private final ObjectMapper objectMapper; @Autowired public ReindexRunner( final AmqpTemplate amqpTemplate, final SampleReadService sampleReadService, - final MongoOperations mongoOperations) { + final MongoOperations mongoOperations, ObjectMapper objectMapper) { this.amqpTemplate = amqpTemplate; this.sampleReadService = sampleReadService; this.mongoOperations = mongoOperations; + this.objectMapper = objectMapper; } @Override @@ -101,7 +105,7 @@ public void run(final ApplicationArguments args) throws Exception { futures.put( accession, executor.submit( - new SampleIndexingCallable(accession, sampleReadService, amqpTemplate))); + new SampleIndexingCallable(accession, sampleReadService, amqpTemplate, objectMapper))); ThreadUtils.checkFutures(futures, 1000); } @@ -119,14 +123,16 @@ private static class SampleIndexingCallable implements Callable { private final String accession; private final SampleReadService sampleReadService; private final AmqpTemplate amqpTemplate; + private final ObjectMapper objectMapper; public SampleIndexingCallable( final String accession, final SampleReadService sampleReadService, - final AmqpTemplate amqpTemplate) { + final AmqpTemplate amqpTemplate, ObjectMapper objectMapper) { this.accession = accession; this.sampleReadService = sampleReadService; this.amqpTemplate = amqpTemplate; + this.objectMapper = objectMapper; } @Override @@ -154,7 +160,8 @@ private boolean fetchSampleAndSendMessage(final boolean isRetry) { amqpTemplate.convertAndSend( Messaging.REINDEXING_EXCHANGE, Messaging.REINDEXING_QUEUE, - MessageContent.build(sampleOptional.get(), null, related, false)); + objectMapper.writeValueAsString(sampleOptional.get())); +// MessageContent.build(sampleOptional.get(), null, related, false)); return true; } catch (final Exception e) { diff --git a/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java b/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java index 6183228ac..3b9192b24 100644 --- a/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java +++ b/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java @@ -16,6 +16,8 @@ import static org.mockito.Mockito.when; import java.util.*; + +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; @@ -106,7 +108,7 @@ public MongoSample next() { .thenReturn(Optional.empty()) .thenReturn(Optional.of(sample3)); final ReindexRunner reindexRunner = - new ReindexRunner(amqpTemplate, sampleReadService, mongoOperations); + new ReindexRunner(amqpTemplate, sampleReadService, mongoOperations, new ObjectMapper()); reindexRunner.run(applicationArguments); } } From 0dd52ed1f82dc2906fa24e90d6e22a726a3e1d18 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 29 Jul 2025 15:35:39 +0100 Subject: [PATCH 05/86] merge with dev --- .gitlab-ci.yml | 83 +- RELEASE.adoc | 6 + agents/pom.xml | 2 +- agents/solr/pom.xml | 22 +- .../biosamples/solr/MessageHandlerSolr.java | 12 +- .../uk/ac/ebi/biosamples/solr/SolrRunner.java | 8 +- .../solr/MessageHandlerSolrTest.java | 6 +- agents/uploadworkers/pom.xml | 48 +- .../runner/FileUploadMessageQueueRunner.java | 8 +- .../FileUploadSubmissionService.java | 10 +- client/client/pom.xml | 6 +- .../biosamples/client/BioSamplesClient.java | 17 +- .../client/BioSamplesWebinGetClient.java | 4 +- .../service/CurationRetrievalService.java | 10 +- .../service/CurationSubmissionService.java | 2 +- .../service/SampleCursorRetrievalService.java | 15 +- .../service/SamplePageRetrievalService.java | 4 +- .../service/SampleRetrievalService.java | 2 +- .../service/SampleRetrievalServiceV2.java | 2 +- .../service/SampleSubmissionService.java | 2 +- .../service/SampleSubmissionServiceV2.java | 4 +- .../StructuredDataSubmissionService.java | 2 +- .../client/utils/ClientProperties.java | 2 +- .../utils/IterableResourceFetchAll.java | 73 +- .../model/ResourceSerializationTest.java | 1 + client/pom.xml | 2 +- client/starter/pom.xml | 10 +- .../client/BioSamplesAutoConfiguration.java | 4 +- commons/pom.xml | 55 - core/pom.xml | 183 ++ .../advice/AccessControlControllerAdvice.java | 58 +- .../ModelAttributeControllerAdvice.java | 60 +- .../SampleConversionControllerAdvice.java | 58 +- .../SampleNotAccessibleControllerAdvice.java | 56 +- .../SampleNotFoundControllerAdvice.java | 56 +- .../SchemaValidationControllerAdvice.java | 58 +- .../ac/ebi/biosamples/core}/ModelConfig.java | 68 +- .../ebi/biosamples/core}/model/Accession.java | 96 +- .../biosamples/core}/model/AccessionType.java | 66 +- .../ebi/biosamples/core}/model/Attribute.java | 448 +-- .../biosamples/core}/model/Certificate.java | 198 +- .../ebi/biosamples/core}/model/Contact.java | 590 ++-- .../ebi/biosamples/core}/model/Curation.java | 664 ++-- .../biosamples/core}/model/CurationLink.java | 186 +- .../biosamples/core}/model/CurationRule.java | 148 +- .../core}/model/ExternalReference.java | 232 +- .../biosamples/core}/model/Organization.java | 422 +-- .../core}/model/PipelineAnalytics.java | 166 +- .../biosamples/core}/model/Publication.java | 238 +- .../biosamples/core}/model/Relationship.java | 258 +- .../core}/model/RelationshipType.java | 64 +- .../ac/ebi/biosamples/core}/model/Sample.java | 2148 ++++++------- .../core}/model/SampleAnalytics.java | 166 +- .../core}/model/SampleCompareResult.java | 58 +- .../core}/model/SampleFacetValue.java | 76 +- .../biosamples/core}/model/SampleStatus.java | 54 +- .../core}/model/SubmissionReceipt.java | 2 +- .../core}/model/SubmittedViaType.java | 42 +- .../core}/model/facet/AttributeFacet.java | 218 +- .../core}/model/facet/DateRangeFacet.java | 266 +- .../facet/ExternalReferenceDataFacet.java | 214 +- .../biosamples/core}/model/facet/Facet.java | 132 +- .../core}/model/facet/FacetHelper.java | 234 +- .../core}/model/facet/FacetType.java | 100 +- .../model/facet/InverseRelationFacet.java | 216 +- .../core}/model/facet/RelationFacet.java | 216 +- .../model/facet/content/FacetContent.java | 33 +- .../model/facet/content/LabelCountEntry.java | 148 +- .../facet/content/LabelCountListContent.java | 90 +- .../model/facet/content/RangeCountEntry.java | 186 +- .../facet/content/RangeCountListContent.java | 90 +- .../core}/model/filter/AccessionFilter.java | 170 +- .../core}/model/filter/AttributeFilter.java | 190 +- .../model/filter/AuthenticationFilter.java | 4 +- .../core}/model/filter/DateRangeFilter.java | 542 ++-- .../filter/ExternalReferenceDataFilter.java | 190 +- .../biosamples/core}/model/filter/Filter.java | 118 +- .../core}/model/filter/FilterType.java | 146 +- .../model/filter/InverseRelationFilter.java | 192 +- .../core}/model/filter/NameFilter.java | 170 +- .../core}/model/filter/RelationFilter.java | 192 +- .../core}/model/structured/AbstractData.java | 4 +- .../model/structured/StructuredCell.java | 2 +- .../model/structured/StructuredData.java | 6 +- .../model/structured/StructuredDataEntry.java | 2 +- .../model/structured/StructuredDataTable.java | 2 +- .../model/structured/StructuredDataType.java | 2 +- .../model/structured/StructuredEntry.java | 2 +- .../model/structured/StructuredTable.java | 2 +- .../core}/service/AccessionSerializer.java | 60 +- .../core}/service/AttributeValidator.java | 72 +- .../service/CharacteristicDeserializer.java | 166 +- .../service/CharacteristicSerializer.java | 230 +- .../service/CurationApplicationService.java | 236 +- .../service/CustomInstantDeserializer.java | 90 +- .../service/CustomInstantSerializer.java | 70 +- .../service/ExternalReferenceService.java | 208 +- .../core}/service/ExternalReferenceUtils.java | 178 +- .../core}/service/FileDownloadSerializer.java | 216 +- .../core}/service/FilterBuilder.java | 172 +- .../service/HttpOlsUrlResolutionService.java | 188 +- .../core}/service/RelationshipValidator.java | 128 +- .../service/SampleRelationshipUtils.java | 96 +- .../biosamples/core}/service/SampleUtils.java | 240 +- .../core}/service/SampleValidator.java | 164 +- .../content/FacetContentDeserializer.java | 107 +- .../structured/AbstractDataDeserializer.java | 4 +- .../structured/AbstractDataSerializer.java | 4 +- .../model/AttributeRecommendation.java | 152 +- .../curami}/model/CuramiRecommendation.java | 186 +- .../curami}/model/SampleRecommendation.java | 60 +- .../curami}/service/CuramiUtils.java | 112 +- .../curami}/service/DataLoader.java | 192 +- .../exception}/GlobalExceptions.java | 336 +- .../jsonld}/model/BioSchemasContext.java | 92 +- .../jsonld}/model/BioschemasObject.java | 26 +- .../jsonld}/model/JsonLDDataCatalog.java | 290 +- .../jsonld}/model/JsonLDDataRecord.java | 352 +-- .../jsonld}/model/JsonLDDataset.java | 138 +- .../jsonld}/model/JsonLDDefinedTerm.java | 124 +- .../jsonld}/model/JsonLDPropertyValue.java | 170 +- .../jsonld}/model/JsonLDSample.java | 286 +- .../jsonld}/model/JsonLDStructuredValue.java | 94 +- .../jsonld}/service/ContextDeserializer.java | 120 +- .../jsonld}/service/ContextSerializer.java | 98 +- .../SampleToJsonLDSampleRecordConverter.java | 242 +- .../messaging/MessagingConstants.java | 40 +- .../messaging/config}/MessageConfig.java | 177 +- .../messaging/model}/MessageContent.java | 122 +- .../messaging/service}/MessageUtils.java | 62 +- .../ebi/biosamples/mongo/MongoProperties.java | 44 +- .../biosamples/mongo/config}/MongoConfig.java | 88 +- .../mongo/model/MongoAnalytics.java | 162 +- .../mongo/model/MongoAuthChangeRecord.java | 0 .../mongo/model/MongoCertificate.java | 290 +- .../biosamples/mongo/model/MongoCuration.java | 696 ++--- .../mongo/model/MongoCurationLink.java | 288 +- .../mongo/model/MongoCurationRule.java | 214 +- .../mongo/model/MongoExternalReference.java | 300 +- .../mongo/model/MongoFileUpload.java | 180 +- .../biosamples/mongo/model/MongoPipeline.java | 160 +- .../mongo/model/MongoRelationship.java | 260 +- .../biosamples/mongo/model/MongoSample.java | 450 +-- .../mongo/model/MongoSampleMessage.java | 150 +- .../biosamples/mongo/model/MongoSequence.java | 72 +- .../mongo/model/MongoStructuredData.java | 162 +- .../repository/MongoAnalyticsRepository.java | 0 .../repository/MongoAuthChangeRepository.java | 0 .../MongoCurationLinkRepository.java | 0 .../repository/MongoCurationRepository.java | 0 .../MongoCurationRuleRepository.java | 0 .../repository/MongoFileUploadRepository.java | 0 .../repository/MongoPipelineRepository.java | 0 .../MongoSampleMessageRepository.java | 0 .../repository/MongoSampleRepository.java | 0 .../MongoSampleRepositoryCustom.java | 0 .../repository/MongoSampleRepositoryImpl.java | 0 .../MongoStructuredDataRepository.java | 0 .../mongo/service/AnalyticsService.java | 260 +- ...ertificateToMongoCertificateConverter.java | 50 +- ...ationLinkToMongoCurationLinkConverter.java | 62 +- .../mongo/service/CurationReadService.java | 382 +-- .../CurationToMongoCurationConverter.java | 66 +- .../service/CustomWriteConcernResolver.java | 94 +- ...enceToMongoExternalReferenceConverter.java | 56 +- .../mongo/service/MongoAccessionService.java | 414 +-- ...ongoCertificateToCertificateConverter.java | 56 +- ...goCurationLinkToCurationLinkConverter.java | 62 +- .../MongoCurationToCurationConverter.java | 66 +- ...ReferenceToExternalReferenceConverter.java | 54 +- .../MongoInverseRelationshipService.java | 128 +- ...goRelationshipToRelationshipConverter.java | 52 +- .../service/MongoSampleToSampleConverter.java | 294 +- ...ructuredDataToStructuredDataConverter.java | 58 +- ...ationshipToMongoRelationshipConverter.java | 52 +- .../mongo/service/SampleReadService.java | 318 +- .../service/SampleToMongoSampleConverter.java | 170 +- ...redDataToMongoStructuredDataConverter.java | 52 +- .../BioSamplesFileUploadSubmissionStatus.java | 38 +- .../mongo/util/PipelineCompletionStatus.java | 32 +- .../mongo/util/SampleNameAccessionPair.java | 74 +- .../ebi/biosamples/neo4j/NeoProperties.java | 56 +- .../biosamples/neo4j/model/CypherQuery.java | 70 +- .../ebi/biosamples/neo4j/model/GraphLink.java | 156 +- .../ebi/biosamples/neo4j/model/GraphNode.java | 192 +- .../neo4j/model/GraphRelationship.java | 106 +- .../neo4j/model/GraphSearchQuery.java | 122 +- .../neo4j/model/NeoExternalEntity.java | 94 +- .../neo4j/model/NeoRelationship.java | 90 +- .../ebi/biosamples/neo4j/model/NeoSample.java | 253 +- .../neo4j/repo/NeoSampleRepository.java | 524 ++-- .../neo4j/service/NeoRelationshipService.java | 26 +- .../neo4j/service/NeoSampleConverter.java | 42 +- .../biosamples/security/model}/AuthRealm.java | 32 +- .../security/model}/AuthRequestWebin.java | 62 +- .../biosamples/security}/model/AuthToken.java | 95 +- .../model}/AuthorizationProvider.java | 30 +- .../model}/FileUploaderAuthRequest.java | 2 +- .../security/model}/UserAuthentication.java | 4 +- .../service}/AccessControlService.java | 188 +- ...CrossSourceIngestAccessControlService.java | 168 +- .../BioSamplesTokenAuthenticationFilter.java | 2 +- .../BioSamplesTokenAuthenticationService.java | 3 +- .../service}/BioSamplesTokenHandler.java | 5 +- .../service}/BioSamplesWebSecurityConfig.java | 2 +- .../biosamples/service/AccessionsService.java | 57 + .../ExceptionAsTextHttpMessageConverter.java | 114 +- .../ebi/biosamples/service/FilterService.java | 87 +- .../biosamples/service/MessagingService.java | 272 +- .../service/StructuredDataService.java | 244 +- .../taxonomy/TaxonomyClientService.java | 120 +- .../service/taxonomy/ZoomaService.java | 200 +- .../validation/ElixirSchemaValidator.java | 280 +- .../service}/validation/JsonSchema.java | 176 +- .../validation/SchemaValidationService.java | 166 +- .../validation/ValidationRequest.java | 78 +- .../service}/validation/ValidatorI.java | 42 +- .../biosamples/sitemap}/model/XmlSitemap.java | 89 +- .../sitemap}/model/XmlSitemapIndex.java | 66 +- .../sitemap}/model/XmlSitemapSet.java | 64 +- .../ebi/biosamples/sitemap}/model/XmlUrl.java | 288 +- .../biosamples/sitemap}/model/XmlUrlSet.java | 94 +- .../sitemap}/model/package-info.java | 14 +- .../solr/CustomInstantSolrDeserializer.java | 70 +- .../solr/CustomInstantSolrSerializer.java | 72 +- .../uk/ac/ebi/biosamples/solr/SolrConfig.java | 40 +- .../ebi/biosamples/solr/model/SolrSample.java | 600 ++-- .../model/field/FilterCriteriaBuilder.java | 38 +- .../model/field/SolrSampleAccessionField.java | 172 +- .../field/SolrSampleAttributeValueField.java | 192 +- .../field/SolrSampleAuthenticationField.java | 4 +- .../solr/model/field/SolrSampleDateField.java | 240 +- .../SolrSampleExternalReferenceDataField.java | 204 +- .../solr/model/field/SolrSampleField.java | 284 +- .../field/SolrSampleInverseRelationField.java | 206 +- .../solr/model/field/SolrSampleNameField.java | 174 +- .../model/field/SolrSampleRelationField.java | 202 +- .../model/strategy/FacetFetchStrategy.java | 90 +- .../strategy/RegularFacetFetchStrategy.java | 354 +-- .../biosamples/solr/repo/CursorArrayList.java | 76 +- .../solr/repo/SolrSampleRepository.java | 34 +- .../solr/repo/SolrSampleRepositoryCustom.java | 184 +- .../solr/repo/SolrSampleRepositoryImpl.java | 382 +-- .../service/SampleToSolrSampleConverter.java | 527 ++-- .../solr/service/SolrFacetService.java | 420 +-- .../solr/service/SolrFieldService.java | 290 +- .../solr/service/SolrFilterService.java | 323 +- .../solr/service/SolrSampleService.java | 264 +- .../ac/ebi/biosamples/utils/ClientUtils.java | 170 +- .../uk/ac/ebi/biosamples/utils/LinkUtils.java | 128 +- .../biosamples/utils/json/JsonFragmenter.java | 110 +- .../biosamples/utils}/ols/OlsProcessor.java | 480 +-- .../ebi/biosamples/utils}/ols/OlsResult.java | 58 +- .../utils/phenopacket/OLSDataRetriever.java | 270 +- .../phenopacket/PhenopacketAttribute.java | 128 +- .../PhenopacketConversionHelper.java | 198 +- .../phenopacket/PhenopacketConverter.java | 1156 +++---- .../thread}/AdaptiveThreadPoolExecutor.java | 504 +-- .../biosamples/utils/thread}/ThreadUtils.java | 168 +- .../utils/upload/Characteristics.java | 56 +- .../utils/upload/FileUploadUtils.java | 1414 ++++----- .../utils/upload/ValidationResult.java | 112 +- .../src/main/resources/abbreviations.csv | 544 ++-- .../src/main/resources/attributes.csv | 2036 ++++++------ .../src/main/resources/curations.csv | 6 +- .../src/main/resources/public.explore.der | Bin .../src/main/resources/public.old.explore.der | Bin .../src/main/resources/public.prod.der | Bin .../biosamples/core}/model/AttributeTest.java | 124 +- .../core}/model/ContactComparisonTest.java | 312 +- .../biosamples/core}/model/CurationTest.java | 102 +- .../core}/model/FilterSerializationTest.java | 324 +- .../model/RelationshipValidationTests.java | 76 +- .../core}/model/SampleBuilderTest.java | 50 +- .../core}/model/SampleTaxIdTest.java | 212 +- .../core}/model/SerializationTest.java | 396 +-- .../HttpOlsUrlResolutionServiceTest.java | 9 +- .../jsonld}/model/BioschemasContextTest.java | 160 +- .../service/ContextDeserializerTest.java | 124 +- .../service/ContextSerializerTest.java | 80 +- ...mpleToJsonLDSampleRecordConverterTest.java | 140 +- .../mongo}/model/MongoSerializationTest.java | 13 +- .../service/CurationReadServiceTest.java | 8 +- .../solr/SolrSampleTestConfiguration.java | 42 +- .../ebi/biosamples/solr/SolrServiceTests.java | 85 + .../ebi/biosamples/utils/LinkUtilsTest.java | 94 +- .../utils}/ols/OlsProcessorTest.java | 256 +- .../utils}/ols/TestApplication.java | 64 +- .../phenopacket/OLSDataRetrieverTest.java | 74 +- .../phenopacket/PhenopacketConverterTest.java | 252 +- .../validation/TestApplication.java | 64 +- .../src/test}/logback-test.xml | 22 +- .../src/test/resources/TEST1.json | 138 +- .../src/test/resources/TEST2.json | 28 +- .../src/test/resources/core}/TEST1.json | 154 +- .../src/test/resources/core}/TEST1.xml | 116 +- .../src/test/resources/core}/TEST2.json | 36 +- .../src/test/resources/core}/curation.json | 36 +- .../ols-responses/FBcv_0003016.json | 186 +- .../ols-responses/NCBITaxon_3702.json | 1216 ++++---- .../resources}/ols-responses/NCIT_C14207.json | 270 +- .../ols-responses/PATO_0000384.json | 2736 ++++++++--------- .../ols-responses/invalid-term.json | 24 +- .../phenopacket/basicPhenopacketJson.json | 64 +- .../resources/phenopacket/biosample1.json | 148 +- .../resources/phenopacket/biosample3.json | 172 +- .../resources/phenopacket/biosamples2.json | 168 +- .../phenopacket/diseaseTestPhenopacket.json | 70 +- .../individualMappingTestPhenopacket.json | 74 +- .../resources/phenopacket/phenopacket1.json | 100 +- .../resources/phenopacket/phenopacket2.json | 114 +- .../resources/phenopacket/phenopacket3.json | 140 +- .../resources/phenopacket/phenopacket_1.json | 136 +- .../phenopacket/phenotypeTestPhenopacket.json | 84 +- docker-agents.sh | 10 +- docker-compose.yml | 26 +- docker-integration.big.sh | 46 +- docker-integration.sh | 2 +- docker-test.sh | 32 +- integration/pom.xml | 81 +- .../ebi/biosamples/AbstractIntegration.java | 8 +- .../ac/ebi/biosamples/AmrDataIntegration.java | 10 +- .../uk/ac/ebi/biosamples/Application.java | 6 + .../uk/ac/ebi/biosamples/BigIntegration.java | 6 +- .../uk/ac/ebi/biosamples/ETagIntegration.java | 6 +- .../ac/ebi/biosamples/JsonLdIntegration.java | 6 +- .../JsonSchemaValidationIntegration.java | 3 +- .../biosamples/PhenopacketIntegration.java | 4 +- .../biosamples/RestCurationIntegration.java | 10 +- .../RestExternalReferenceIntegration.java | 2 +- .../ebi/biosamples/RestFacetIntegration.java | 4 +- .../ebi/biosamples/RestFilterIntegration.java | 8 +- .../uk/ac/ebi/biosamples/RestIntegration.java | 2 +- .../RestIntegrationSRAAccessioningV2.java | 2 +- .../ac/ebi/biosamples/RestIntegrationV2.java | 2 +- .../RestPrivateSampleIntegration.java | 3 +- .../RestSampleStatusIntegration.java | 4 +- .../ebi/biosamples/RestSearchIntegration.java | 4 +- .../biosamples/SamplesGraphIntegration.java | 2 +- .../ac/ebi/biosamples/SitemapIntegration.java | 41 +- .../StructuredDataGenericIntegration.java | 4 +- .../ebi/biosamples/XmlSearchIntegration.java | 2 +- messaging/pom.xml | 24 - models/README.md | 13 - models/core/pom.xml | 68 - .../core/src/main/xsd/BioSD/BioSDSchema.xsd | 184 -- .../ResultQuerySampleSchema.xsd | 48 - .../ResultQuerySampleGroupSchema.xsd | 47 - models/curami/pom.xml | 32 - models/jsonld/pom.xml | 38 - models/mongo/pom.xml | 41 - models/neo4j/pom.xml | 27 - models/pom.xml | 32 - models/security/pom.xml | 22 - .../src/main/resources/abbreviations.csv | 273 -- .../src/main/resources/attributes.csv | 1019 ------ .../security/src/main/resources/curations.csv | 4 - models/sitemap/pom.xml | 61 - models/solr/pom.xml | 49 - .../solr/SolrSampleTestConfiguration.java | 11 - .../ebi/biosamples/solr/SolrServiceTests.java | 94 - pipelines/analytics/pom.xml | 28 +- .../AnalyticsApplicationRunner.java | 10 +- .../uk/ac/ebi/biosamples/Application.java | 18 - pipelines/chain/pom.xml | 21 +- .../auth/services/AuthChangeHandler.java | 4 +- .../SamplesCrawlerAuthChangeHandler.java | 92 + .../SampleChecklistComplianceHandlerEVA.java | 4 +- .../SampleExternalReferenceHandler.java | 6 +- .../services/SampleRelationshipHandler.java | 4 +- .../helpdesk/services/SampleRestoreIPK.java | 4 +- .../services/SampleStatusUpdater.java | 8 +- {utils/pipeline => pipelines/common}/pom.xml | 159 +- .../biosamples/PipelineApplicationRunner.java | 228 +- .../biosamples/PipelineFutureCallback.java | 74 +- .../uk/ac/ebi/biosamples/PipelineResult.java | 52 +- .../biosamples/PipelineSampleCallable.java | 96 +- .../uk/ac/ebi/biosamples/PipelinesHelper.java | 218 +- .../ebi/biosamples/PipelinesProperties.java | 332 +- .../configuration/ExclusionConfiguration.java | 42 +- .../biosamples/model/AccessionFtpUrlPair.java | 78 +- .../ac/ebi/biosamples/model/PipelineName.java | 52 +- .../service/AmrDataLoaderService.java | 466 +-- .../service/BioSampleConverter.java | 744 ++--- .../ac/ebi/biosamples/service/EnaConfig.java | 74 +- ...aSampleToBioSampleConversionConstants.java | 102 +- ...EnaSampleToBioSampleConversionService.java | 385 +-- .../service/EnaSampleXmlEnhancer.java | 754 ++--- .../ac/ebi/biosamples/service/EnaXmlUtil.java | 102 +- .../ac/ebi/biosamples/service/EraProDao.java | 542 ++-- .../ebi/biosamples/service/EraproSample.java | 300 +- .../service/SampleCallbackResult.java | 104 +- .../biosamples/service/TaxonomyService.java | 42 +- .../ac/ebi/biosamples/service/XmlUtils.java | 312 +- .../PipelineUniqueIdentifierGenerator.java | 56 +- .../ebi/biosamples/utils/PipelineUtils.java | 338 +- .../ebi/biosamples/utils/XmlFragmenter.java | 0 .../ebi/biosamples/utils/XmlPathBuilder.java | 0 .../biosamples/utils/XmlPathBuilderTest.java | 11 + pipelines/copydown/pom.xml | 16 +- .../copydown/CopydownApplicationRunner.java | 10 +- .../copydown/SampleCopydownCallable.java | 2 +- pipelines/curami/pom.xml | 25 +- .../curation/CuramiApplicationRunner.java | 12 +- .../curation/SampleCuramiCallable.java | 6 +- pipelines/curation/pom.xml | 21 +- .../curation/CurationApplicationRunner.java | 14 +- .../curation/SampleCurationCallable.java | 10 +- .../curation/MockBioSamplesClient.java | 6 +- .../curation/NonSavingApplication.java | 2 +- .../curation/SampleCurationCallableTest.java | 8 +- .../biosamples/curation/TestApplication.java | 6 +- pipelines/ena/pom.xml | 20 +- .../ebi/biosamples/ena/EnaImportRunner.java | 4 +- pipelines/export/pom.xml | 15 +- .../ebi/biosamples/export/ExportRunner.java | 2 +- pipelines/ncbi-ena-link/pom.xml | 20 +- .../biosamples/ena/NcbiEnaLinkCallable.java | 12 +- .../ebi/biosamples/ena/NcbiEnaLinkRunner.java | 4 +- pipelines/ncbi/pom.xml | 15 +- .../java/uk/ac/ebi/biosamples/ncbi/Ncbi.java | 10 +- .../biosamples/ncbi/NcbiElementCallable.java | 12 +- .../ncbi/NcbiElementCallableFactory.java | 2 +- .../biosamples/ncbi/NcbiFragmentCallback.java | 4 +- .../service/NcbiAmrConversionService.java | 2 +- .../service/NcbiSampleConversionService.java | 6 +- .../biosamples/NcbiAmrConvertionTests.java | 6 +- .../biosamples/NcbiBaseConverterTests.java | 6 +- .../NcbiNotSuppressedBaseConverterTests.java | 4 +- .../uk/ac/ebi/biosamples/TestApplication.java | 2 +- .../biosamples/ncbi/MockBioSamplesClient.java | 4 +- .../ebi/biosamples/ncbi/NcbiDatesTests.java | 4 +- .../ncbi/NcbiElementCallableTest.java | 4 +- pipelines/neoexport/pom.xml | 11 +- .../biosamples/neoexport/NeoCsvExporter.java | 2 +- .../neoexport/NeoExportCallable.java | 2 +- .../biosamples/neoexport/NeoExportRunner.java | 12 +- pipelines/pom.xml | 14 +- pipelines/reindex/pom.xml | 15 +- .../uk/ac/ebi/biosamples/ReindexRunner.java | 29 +- .../ac/ebi/biosamples/ReindexRunnerTest.java | 6 +- pipelines/sample-post-release-action/pom.xml | 21 +- ...plePostReleaseActionApplicationRunner.java | 8 +- .../SamplePostReleaseActionCallable.java | 4 +- pipelines/sample-release/pom.xml | 6 +- .../samplerelease/SampleReleaseCallable.java | 8 +- .../samplerelease/SampleReleaseRunner.java | 2 +- pipelines/sample-transformation-dtol/pom.xml | 26 +- .../TransformationApplicationRunner.java | 10 +- .../curation/TransformationCallable.java | 6 +- pipelines/taxonimport/pom.xml | 25 +- .../TaxonImportApplicationRunner.java | 8 +- pipelines/zooma/pom.xml | 21 +- .../biosamples/zooma/SampleZoomaCallable.java | 8 +- .../zooma/ZoomaApplicationRunner.java | 12 +- pom.xml | 23 +- properties/pom.xml | 2 +- scripts/data/samples/sample_ERC000022.json | 134 - scripts/data/samples/sample_ERC000023.json | 107 - scripts/load_test_bulk_sample_submission.py | 190 -- scripts/requirements.txt | 14 - utils/client/pom.xml | 21 - utils/ols/pom.xml | 36 - .../examples/ols-responses/FBcv_0003016.json | 94 - utils/pom.xml | 23 - utils/thread/pom.xml | 14 - utils/validation/pom.xml | 45 - .../ols-responses/NCBITaxon_3702.json | 609 ---- .../examples/ols-responses/NCIT_C14207.json | 136 - .../examples/ols-responses/PATO_0000384.json | 1369 --------- .../examples/ols-responses/invalid-term.json | 13 - .../src/test/resources/logback-test.xml | 12 - utils/webapp/pom.xml | 100 - webapps/core-v2/pom.xml | 47 +- .../controller/BulkActionControllerV2.java | 15 +- .../controller/SampleGetControllerV2.java | 6 +- .../ebi/biosamples/service/SampleService.java | 770 +++++ .../service}/WebinAuthenticationService.java | 21 +- .../BulkActionControllerV2Test.java | 4 +- webapps/core/Dockerfile | 5 + .../core/core-dependencies-local-only.yaml | 107 + webapps/core/core-deployment.yaml | 37 + webapps/core/core-service.yaml | 12 + webapps/core/pom.xml | 72 +- .../controller/AccessionsGetController.java | 2 +- .../controller/BioschemasGetController.java | 12 +- .../controller/CurationController.java | 4 +- .../controller/FileDownloadController.java | 4 +- .../controller/FileUploadController.java | 6 +- .../controller/FileUploadLoginController.java | 10 +- .../controller/RecommendationController.java | 6 +- .../SampleCurationLinksController.java | 6 +- .../controller/SampleFacetController.java | 4 +- .../controller/SampleHtmlController.java | 14 +- .../controller/SampleRestController.java | 12 +- .../controller/SamplesRestController.java | 26 +- .../SchemaValidationController.java | 8 +- .../controller/SitemapController.java | 23 +- .../biosamples/controller/StatController.java | 2 +- .../controller/StructuredDataController.java | 6 +- .../biosamples/service/AccessionsService.java | 55 - .../CurationLinkResourceAssembler.java | 4 +- .../service/CurationPersistService.java | 6 +- .../service/CurationResourceAssembler.java | 2 +- .../ebi/biosamples/service/FacetService.java | 4 +- .../service/FileDownloadInputStream.java | 5 +- .../service/FileDownloadService.java | 3 +- .../ebi/biosamples/service/JsonLDService.java | 4 +- .../service/RecommendationService.java | 10 +- .../service/SampleManipulationService.java | 4 +- .../biosamples/service/SamplePageService.java | 4 +- .../service/SampleResourceAssembler.java | 2 +- .../service/SampleRestResponseBodyAdvice.java | 2 +- .../ebi/biosamples/service/SampleService.java | 1537 +++++---- .../SamplesRestResponseBodyAdvice.java | 2 +- .../ebi/biosamples/service/StatService.java | 4 +- .../service/WebinAuthenticationService.java | 314 ++ .../service/certification/Certifier.java | 2 +- .../service/certification/CertifyService.java | 2 +- .../service/certification/Interrogator.java | 2 +- .../service/certification/Validator.java | 4 +- .../service/search/ElasticSearchService.java | 14 +- .../service/search/SearchService.java | 2 +- .../service/search/SolrSearchService.java | 2 +- .../service/upload/FileUploadService.java | 31 +- .../resources/templates/fragments/footer.html | 2 +- .../java/uk/ac/ebi/biosamples/EtagTests.java | 12 +- .../ac/ebi/biosamples/StructuredDataTest.java | 4 +- .../certification/service/CertifierTest.java | 4 +- .../service/CertifyServiceTest.java | 4 +- .../certification/service/CuratorTest.java | 4 +- .../service/FileRecorderTest.java | 4 +- .../certification/service/IdentifierTest.java | 4 +- .../service/InterrogatorTest.java | 4 +- .../certification/service/ValidatorTest.java | 4 +- .../controller/SampleRestControllerTest.java | 2 +- .../controller/SamplesRestControllerTest.java | 6 +- .../biosamples/docs/ApiDocumentationTest.java | 19 +- .../biosamples/docs/DocumentationHelper.java | 8 +- .../service/FileDownloadInputStreamTest.java | 7 +- .../service/RecommendationServiceTest.java | 8 +- .../security/AccessControlServiceTest.java | 1 + .../taxonomy/TaxonomyClientServiceTest.java | 2 +- webapps/pom.xml | 2 +- 544 files changed, 31277 insertions(+), 34839 deletions(-) delete mode 100644 commons/pom.xml create mode 100644 core/pom.xml rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/advice/AccessControlControllerAdvice.java (94%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/advice/ModelAttributeControllerAdvice.java (97%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/advice/SampleConversionControllerAdvice.java (94%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotAccessibleControllerAdvice.java (94%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotFoundControllerAdvice.java (94%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/advice/SchemaValidationControllerAdvice.java (94%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/ModelConfig.java (96%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/Accession.java (91%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/AccessionType.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/Attribute.java (95%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/Certificate.java (94%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/Contact.java (95%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/Curation.java (97%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/CurationLink.java (92%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/CurationRule.java (95%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/ExternalReference.java (96%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/Organization.java (95%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/PipelineAnalytics.java (92%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/Publication.java (95%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/Relationship.java (95%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/RelationshipType.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/Sample.java (95%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/SampleAnalytics.java (92%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/SampleCompareResult.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/SampleFacetValue.java (94%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/SampleStatus.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/SubmissionReceipt.java (96%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/SubmittedViaType.java (92%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/facet/AttributeFacet.java (89%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/facet/DateRangeFacet.java (90%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/facet/ExternalReferenceDataFacet.java (89%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/facet/Facet.java (90%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/facet/FacetHelper.java (96%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/facet/FacetType.java (95%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/facet/InverseRelationFacet.java (89%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/facet/RelationFacet.java (89%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/facet/content/FacetContent.java (84%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/facet/content/LabelCountEntry.java (94%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/facet/content/LabelCountListContent.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/facet/content/RangeCountEntry.java (95%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/facet/content/RangeCountListContent.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/filter/AccessionFilter.java (92%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/filter/AttributeFilter.java (92%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/filter/AuthenticationFilter.java (95%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/filter/DateRangeFilter.java (95%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/filter/ExternalReferenceDataFilter.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/filter/Filter.java (91%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/filter/FilterType.java (96%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/filter/InverseRelationFilter.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/filter/NameFilter.java (91%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/filter/RelationFilter.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/structured/AbstractData.java (92%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/structured/StructuredCell.java (96%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/structured/StructuredData.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/structured/StructuredDataEntry.java (94%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/structured/StructuredDataTable.java (98%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/structured/StructuredDataType.java (96%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/structured/StructuredEntry.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/model/structured/StructuredTable.java (98%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/AccessionSerializer.java (90%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/AttributeValidator.java (91%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/CharacteristicDeserializer.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/CharacteristicSerializer.java (94%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/CurationApplicationService.java (92%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/CustomInstantDeserializer.java (94%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/CustomInstantSerializer.java (94%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/ExternalReferenceService.java (96%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/ExternalReferenceUtils.java (95%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/FileDownloadSerializer.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/FilterBuilder.java (94%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/HttpOlsUrlResolutionService.java (95%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/RelationshipValidator.java (92%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/SampleRelationshipUtils.java (91%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/SampleUtils.java (94%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/SampleValidator.java (90%) rename {models/core/src/main/java/uk/ac/ebi/biosamples/model => core/src/main/java/uk/ac/ebi/biosamples/core/service}/facet/content/FacetContentDeserializer.java (88%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/structured/AbstractDataDeserializer.java (93%) rename {models/core/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/core}/service/structured/AbstractDataSerializer.java (93%) rename {models/curami/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/curami}/model/AttributeRecommendation.java (95%) rename {models/curami/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/curami}/model/CuramiRecommendation.java (95%) rename {models/curami/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/curami}/model/SampleRecommendation.java (91%) rename {models/curami/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/curami}/service/CuramiUtils.java (89%) rename {models/curami/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/curami}/service/DataLoader.java (92%) rename {commons/src/main/java/uk/ac/ebi/biosamples/exceptions => core/src/main/java/uk/ac/ebi/biosamples/exception}/GlobalExceptions.java (97%) rename {models/jsonld/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/jsonld}/model/BioSchemasContext.java (88%) rename {models/jsonld/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/jsonld}/model/BioschemasObject.java (92%) rename {models/jsonld/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/jsonld}/model/JsonLDDataCatalog.java (96%) rename {models/jsonld/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/jsonld}/model/JsonLDDataRecord.java (95%) rename {models/jsonld/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/jsonld}/model/JsonLDDataset.java (96%) rename {models/jsonld/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/jsonld}/model/JsonLDDefinedTerm.java (94%) rename {models/jsonld/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/jsonld}/model/JsonLDPropertyValue.java (95%) rename {models/jsonld/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/jsonld}/model/JsonLDSample.java (95%) rename {models/jsonld/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/jsonld}/model/JsonLDStructuredValue.java (94%) rename {models/jsonld/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/jsonld}/service/ContextDeserializer.java (94%) rename {models/jsonld/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/jsonld}/service/ContextSerializer.java (92%) rename {models/jsonld/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/jsonld}/service/SampleToJsonLDSampleRecordConverter.java (91%) rename messaging/src/main/java/uk/ac/ebi/biosamples/Messaging.java => core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java (86%) rename {messaging/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/messaging/config}/MessageConfig.java (73%) rename {messaging/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/messaging/model}/MessageContent.java (91%) rename {messaging/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/messaging/service}/MessageUtils.java (95%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/MongoProperties.java (97%) rename {models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo => core/src/main/java/uk/ac/ebi/biosamples/mongo/config}/MongoConfig.java (92%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoAnalytics.java (92%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoAuthChangeRecord.java (100%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCertificate.java (94%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCuration.java (96%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationLink.java (91%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationRule.java (93%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoExternalReference.java (96%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoFileUpload.java (97%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoPipeline.java (97%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoRelationship.java (96%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSample.java (94%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSampleMessage.java (96%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSequence.java (96%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoStructuredData.java (89%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoAnalyticsRepository.java (100%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoAuthChangeRepository.java (100%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationLinkRepository.java (100%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationRepository.java (100%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationRuleRepository.java (100%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoFileUploadRepository.java (100%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoPipelineRepository.java (100%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleMessageRepository.java (100%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepository.java (100%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepositoryCustom.java (100%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepositoryImpl.java (100%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoStructuredDataRepository.java (100%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/AnalyticsService.java (95%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/CertificateToMongoCertificateConverter.java (93%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationLinkToMongoCurationLinkConverter.java (94%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationReadService.java (96%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationToMongoCurationConverter.java (94%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/CustomWriteConcernResolver.java (97%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/ExternalReferenceToMongoExternalReferenceConverter.java (93%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoAccessionService.java (96%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCertificateToCertificateConverter.java (94%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationLinkToCurationLinkConverter.java (94%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationToCurationConverter.java (94%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoExternalReferenceToExternalReferenceConverter.java (94%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoInverseRelationshipService.java (97%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoRelationshipToRelationshipConverter.java (93%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoSampleToSampleConverter.java (94%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoStructuredDataToStructuredDataConverter.java (93%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/RelationshipToMongoRelationshipConverter.java (93%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleReadService.java (95%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleToMongoSampleConverter.java (92%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/service/StructuredDataToMongoStructuredDataConverter.java (92%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/util/BioSamplesFileUploadSubmissionStatus.java (97%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/util/PipelineCompletionStatus.java (97%) rename {models/mongo => core}/src/main/java/uk/ac/ebi/biosamples/mongo/util/SampleNameAccessionPair.java (97%) rename {models/neo4j => core}/src/main/java/uk/ac/ebi/biosamples/neo4j/NeoProperties.java (97%) rename {models/neo4j => core}/src/main/java/uk/ac/ebi/biosamples/neo4j/model/CypherQuery.java (96%) rename {models/neo4j => core}/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphLink.java (94%) rename {models/neo4j => core}/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphNode.java (96%) rename {models/neo4j => core}/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphRelationship.java (92%) rename {models/neo4j => core}/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphSearchQuery.java (96%) rename {models/neo4j => core}/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoExternalEntity.java (90%) rename {models/neo4j => core}/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoRelationship.java (90%) rename {models/neo4j => core}/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoSample.java (92%) rename {models/neo4j => core}/src/main/java/uk/ac/ebi/biosamples/neo4j/repo/NeoSampleRepository.java (96%) rename {models/neo4j => core}/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoRelationshipService.java (98%) rename {models/neo4j => core}/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoSampleConverter.java (92%) rename {models/security/src/main/java/uk/ac/ebi/biosamples/model/auth => core/src/main/java/uk/ac/ebi/biosamples/security/model}/AuthRealm.java (91%) rename {models/security/src/main/java/uk/ac/ebi/biosamples/model/auth => core/src/main/java/uk/ac/ebi/biosamples/security/model}/AuthRequestWebin.java (93%) rename {models/security/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/security}/model/AuthToken.java (89%) rename {models/security/src/main/java/uk/ac/ebi/biosamples/model/auth => core/src/main/java/uk/ac/ebi/biosamples/security/model}/AuthorizationProvider.java (92%) rename {models/security/src/main/java/uk/ac/ebi/biosamples/model/auth => core/src/main/java/uk/ac/ebi/biosamples/security/model}/FileUploaderAuthRequest.java (94%) rename {utils/webapp/src/main/java/uk/ac/ebi/biosamples/security => core/src/main/java/uk/ac/ebi/biosamples/security/model}/UserAuthentication.java (94%) rename {utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/security => core/src/main/java/uk/ac/ebi/biosamples/security/service}/AccessControlService.java (92%) rename {utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/security => core/src/main/java/uk/ac/ebi/biosamples/security/service}/BioSamplesCrossSourceIngestAccessControlService.java (91%) rename {utils/webapp/src/main/java/uk/ac/ebi/biosamples/security => core/src/main/java/uk/ac/ebi/biosamples/security/service}/BioSamplesTokenAuthenticationFilter.java (98%) rename {utils/webapp/src/main/java/uk/ac/ebi/biosamples/security => core/src/main/java/uk/ac/ebi/biosamples/security/service}/BioSamplesTokenAuthenticationService.java (93%) rename {utils/webapp/src/main/java/uk/ac/ebi/biosamples/security => core/src/main/java/uk/ac/ebi/biosamples/security/service}/BioSamplesTokenHandler.java (90%) rename {utils/webapp/src/main/java/uk/ac/ebi/biosamples/security => core/src/main/java/uk/ac/ebi/biosamples/security/service}/BioSamplesWebSecurityConfig.java (98%) create mode 100644 core/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/service/ExceptionAsTextHttpMessageConverter.java (95%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/service/FilterService.java (93%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java (90%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/service/StructuredDataService.java (95%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/TaxonomyClientService.java (93%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/ZoomaService.java (97%) rename {utils/validation/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/service}/validation/ElixirSchemaValidator.java (96%) rename {utils/validation/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/service}/validation/JsonSchema.java (93%) rename {utils/validation/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/service}/validation/SchemaValidationService.java (92%) rename {utils/validation/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/service}/validation/ValidationRequest.java (93%) rename {utils/validation/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/service}/validation/ValidatorI.java (87%) rename {models/sitemap/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/sitemap}/model/XmlSitemap.java (84%) rename {models/sitemap/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/sitemap}/model/XmlSitemapIndex.java (70%) rename {models/sitemap/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/sitemap}/model/XmlSitemapSet.java (67%) rename {models/sitemap/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/sitemap}/model/XmlUrl.java (77%) rename {models/sitemap/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/sitemap}/model/XmlUrlSet.java (62%) rename {models/sitemap/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/sitemap}/model/package-info.java (82%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrDeserializer.java (97%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrSerializer.java (97%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/SolrConfig.java (97%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/model/SolrSample.java (96%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/model/field/FilterCriteriaBuilder.java (92%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAccessionField.java (92%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAttributeValueField.java (90%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAuthenticationField.java (95%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleDateField.java (91%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleExternalReferenceDataField.java (90%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleField.java (94%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleInverseRelationField.java (90%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleNameField.java (92%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleRelationField.java (90%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/FacetFetchStrategy.java (95%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/RegularFacetFetchStrategy.java (95%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/repo/CursorArrayList.java (97%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepository.java (98%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryCustom.java (97%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryImpl.java (97%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/service/SampleToSolrSampleConverter.java (89%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFacetService.java (95%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFieldService.java (96%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java (95%) rename {models/solr => core}/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java (96%) rename {utils/client => core}/src/main/java/uk/ac/ebi/biosamples/utils/ClientUtils.java (97%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/utils/LinkUtils.java (97%) rename {commons => core}/src/main/java/uk/ac/ebi/biosamples/utils/json/JsonFragmenter.java (97%) rename {utils/ols/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/utils}/ols/OlsProcessor.java (97%) rename {utils/ols/src/main/java/uk/ac/ebi/biosamples => core/src/main/java/uk/ac/ebi/biosamples/utils}/ols/OlsResult.java (93%) rename {commons => core}/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetriever.java (96%) rename {commons => core}/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketAttribute.java (96%) rename {commons => core}/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConversionHelper.java (96%) rename {commons => core}/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverter.java (97%) rename {utils/thread/src/main/java/uk/ac/ebi/biosamples/utils => core/src/main/java/uk/ac/ebi/biosamples/utils/thread}/AdaptiveThreadPoolExecutor.java (96%) rename {utils/thread/src/main/java/uk/ac/ebi/biosamples/utils => core/src/main/java/uk/ac/ebi/biosamples/utils/thread}/ThreadUtils.java (95%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/utils/upload/Characteristics.java (97%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/utils/upload/FileUploadUtils.java (96%) rename {utils/webapp => core}/src/main/java/uk/ac/ebi/biosamples/utils/upload/ValidationResult.java (96%) rename {models/curami => core}/src/main/resources/abbreviations.csv (86%) rename {models/curami => core}/src/main/resources/attributes.csv (95%) rename {models/curami => core}/src/main/resources/curations.csv (96%) rename {utils/webapp => core}/src/main/resources/public.explore.der (100%) rename {utils/webapp => core}/src/main/resources/public.old.explore.der (100%) rename {utils/webapp => core}/src/main/resources/public.prod.der (100%) rename {models/core/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/core}/model/AttributeTest.java (93%) rename {models/core/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/core}/model/ContactComparisonTest.java (96%) rename {models/core/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/core}/model/CurationTest.java (95%) rename {models/core/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/core}/model/FilterSerializationTest.java (95%) rename {models/core/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/core}/model/RelationshipValidationTests.java (91%) rename {models/core/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/core}/model/SampleBuilderTest.java (93%) rename {models/core/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/core}/model/SampleTaxIdTest.java (96%) rename {models/core/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/core}/model/SerializationTest.java (95%) rename {models/core/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/core}/service/HttpOlsUrlResolutionServiceTest.java (84%) rename {models/jsonld/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/jsonld}/model/BioschemasContextTest.java (96%) rename {models/jsonld/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/jsonld}/service/ContextDeserializerTest.java (94%) rename {models/jsonld/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/jsonld}/service/ContextSerializerTest.java (92%) rename {models/jsonld/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/jsonld}/service/SampleToJsonLDSampleRecordConverterTest.java (91%) rename {models/mongo/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/mongo}/model/MongoSerializationTest.java (94%) rename {models/mongo/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/mongo}/service/CurationReadServiceTest.java (97%) rename models/neo4j/src/test/java/uk/ac/ebi/biosamples/AppTest.java => core/src/test/java/uk/ac/ebi/biosamples/solr/SolrSampleTestConfiguration.java (68%) create mode 100644 core/src/test/java/uk/ac/ebi/biosamples/solr/SolrServiceTests.java rename {utils/webapp => core}/src/test/java/uk/ac/ebi/biosamples/utils/LinkUtilsTest.java (97%) rename {utils/ols/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/utils}/ols/OlsProcessorTest.java (95%) rename {utils/ols/src/test/java/uk/ac/ebi/biosamples => core/src/test/java/uk/ac/ebi/biosamples/utils}/ols/TestApplication.java (94%) rename {commons => core}/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetrieverTest.java (97%) rename {commons => core}/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverterTest.java (96%) rename {utils/validation => core}/src/test/java/uk/ac/ebi/biosamples/validation/TestApplication.java (97%) rename {utils/ols/src/test/resources => core/src/test}/logback-test.xml (98%) rename {models/mongo => core}/src/test/resources/TEST1.json (94%) rename {models/mongo => core}/src/test/resources/TEST2.json (95%) rename {models/core/src/test/resources => core/src/test/resources/core}/TEST1.json (94%) rename {models/core/src/test/resources => core/src/test/resources/core}/TEST1.xml (97%) rename {models/core/src/test/resources => core/src/test/resources/core}/TEST2.json (95%) rename {models/core/src/test/resources => core/src/test/resources/core}/curation.json (93%) rename {utils/validation/src/test/resources/examples => core/src/test/resources}/ols-responses/FBcv_0003016.json (97%) rename {utils/ols/src/test/resources/examples => core/src/test/resources}/ols-responses/NCBITaxon_3702.json (97%) rename {utils/ols/src/test/resources/examples => core/src/test/resources}/ols-responses/NCIT_C14207.json (97%) rename {utils/ols/src/test/resources/examples => core/src/test/resources}/ols-responses/PATO_0000384.json (97%) rename {utils/ols/src/test/resources/examples => core/src/test/resources}/ols-responses/invalid-term.json (94%) rename {commons => core}/src/test/resources/phenopacket/basicPhenopacketJson.json (95%) rename {commons => core}/src/test/resources/phenopacket/biosample1.json (95%) rename {commons => core}/src/test/resources/phenopacket/biosample3.json (95%) rename {commons => core}/src/test/resources/phenopacket/biosamples2.json (95%) rename {commons => core}/src/test/resources/phenopacket/diseaseTestPhenopacket.json (95%) rename {commons => core}/src/test/resources/phenopacket/individualMappingTestPhenopacket.json (95%) rename {commons => core}/src/test/resources/phenopacket/phenopacket1.json (95%) rename {commons => core}/src/test/resources/phenopacket/phenopacket2.json (95%) rename {commons => core}/src/test/resources/phenopacket/phenopacket3.json (95%) rename {commons => core}/src/test/resources/phenopacket/phenopacket_1.json (96%) rename {commons => core}/src/test/resources/phenopacket/phenotypeTestPhenopacket.json (95%) delete mode 100644 messaging/pom.xml delete mode 100644 models/README.md delete mode 100644 models/core/pom.xml delete mode 100644 models/core/src/main/xsd/BioSD/BioSDSchema.xsd delete mode 100644 models/core/src/main/xsd/ResultQuerySample/ResultQuerySampleSchema.xsd delete mode 100644 models/core/src/main/xsd/ResultQuerySampleGroup/ResultQuerySampleGroupSchema.xsd delete mode 100644 models/curami/pom.xml delete mode 100644 models/jsonld/pom.xml delete mode 100644 models/mongo/pom.xml delete mode 100644 models/neo4j/pom.xml delete mode 100644 models/pom.xml delete mode 100644 models/security/pom.xml delete mode 100644 models/security/src/main/resources/abbreviations.csv delete mode 100644 models/security/src/main/resources/attributes.csv delete mode 100644 models/security/src/main/resources/curations.csv delete mode 100644 models/sitemap/pom.xml delete mode 100644 models/solr/pom.xml delete mode 100644 models/solr/src/test/uk/ac/ebi/biosamples/solr/SolrSampleTestConfiguration.java delete mode 100644 models/solr/src/test/uk/ac/ebi/biosamples/solr/SolrServiceTests.java create mode 100644 pipelines/chain/src/main/java/uk/ac/ebi/biosamples/auth/services/SamplesCrawlerAuthChangeHandler.java rename {utils/pipeline => pipelines/common}/pom.xml (71%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/PipelineApplicationRunner.java (92%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/PipelineFutureCallback.java (93%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/PipelineResult.java (97%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/PipelineSampleCallable.java (94%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/PipelinesHelper.java (97%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java (96%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/configuration/ExclusionConfiguration.java (97%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/model/AccessionFtpUrlPair.java (96%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/model/PipelineName.java (96%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/service/AmrDataLoaderService.java (95%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/service/BioSampleConverter.java (96%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/service/EnaConfig.java (97%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionConstants.java (98%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionService.java (92%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleXmlEnhancer.java (96%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/service/EnaXmlUtil.java (97%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/service/EraProDao.java (97%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/service/EraproSample.java (94%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/service/SampleCallbackResult.java (96%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/service/TaxonomyService.java (97%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/service/XmlUtils.java (96%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUniqueIdentifierGenerator.java (97%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java (94%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/utils/XmlFragmenter.java (100%) rename {utils/pipeline => pipelines/common}/src/main/java/uk/ac/ebi/biosamples/utils/XmlPathBuilder.java (100%) rename {utils/pipeline => pipelines/common}/src/test/java/uk/ac/ebi/biosamples/utils/XmlPathBuilderTest.java (82%) delete mode 100644 scripts/data/samples/sample_ERC000022.json delete mode 100644 scripts/data/samples/sample_ERC000023.json delete mode 100644 scripts/load_test_bulk_sample_submission.py delete mode 100644 scripts/requirements.txt delete mode 100644 utils/client/pom.xml delete mode 100644 utils/ols/pom.xml delete mode 100644 utils/ols/src/test/resources/examples/ols-responses/FBcv_0003016.json delete mode 100644 utils/pom.xml delete mode 100644 utils/thread/pom.xml delete mode 100644 utils/validation/pom.xml delete mode 100644 utils/validation/src/test/resources/examples/ols-responses/NCBITaxon_3702.json delete mode 100644 utils/validation/src/test/resources/examples/ols-responses/NCIT_C14207.json delete mode 100644 utils/validation/src/test/resources/examples/ols-responses/PATO_0000384.json delete mode 100644 utils/validation/src/test/resources/examples/ols-responses/invalid-term.json delete mode 100644 utils/validation/src/test/resources/logback-test.xml delete mode 100644 utils/webapp/pom.xml create mode 100644 webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/service/SampleService.java rename {utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/security => webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/service}/WebinAuthenticationService.java (94%) create mode 100644 webapps/core/Dockerfile create mode 100644 webapps/core/core-dependencies-local-only.yaml create mode 100644 webapps/core/core-deployment.yaml create mode 100644 webapps/core/core-service.yaml delete mode 100644 webapps/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java rename {utils/webapp => webapps/core}/src/main/java/uk/ac/ebi/biosamples/service/SampleService.java (96%) create mode 100644 webapps/core/src/main/java/uk/ac/ebi/biosamples/service/WebinAuthenticationService.java diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cd8a905a7..b4d9d0904 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,13 @@ services: variables: DEPLOY_PATH: ~/gitlab + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + DOCKER_TAG: $CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA + DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE:$DOCKER_TAG + DOCKER_PULL_SECRET: docker-registry-secret + APP_NAME: $CI_PROJECT_NAME + before_script: - echo $CI_BUILD_REF @@ -11,33 +18,71 @@ before_script: - apk update && apk add git stages: + - build - package - config - deploy -maven-package: +maven-package-webapps-core: image: ${CI_REGISTRY_IMAGE}/eclipse-temurin:17-jdk - stage: package + stage: build script: - - './mvnw -q deploy -P embl-ebi -s ci_settings.xml -DskipTests -Dmaven.source.skip=true' - - mkdir deployment - - cp webapps/core/target/webapps-core-*.war deployment/webapps-core.war - - cp webapps/core-v2/target/webapps-core-v2*.jar deployment/webapps-core-v2.jar - - cp agents/solr/target/agents-solr-*.jar deployment/agents-solr.jar - - cp agents/uploadworkers/target/agents-uploadworkers-*.jar deployment/agents-uploadworkers.jar - - cp pipelines/curation/target/pipelines-curation-*.jar deployment/pipelines-curation.jar - # - cp pipelines/ena/target/pipelines-ena-*.jar deployment/pipelines-ena.jar - # - cp pipelines/ncbi-ena-link/target/pipelines-ncbi-ena-link-*.jar deployment/pipelines-ncbi-ena-link.jar - - cp pipelines/sample-release/target/pipelines-sample-release-*.jar deployment/pipelines-sample-release.jar - - cp pipelines/sample-post-release-action/target/pipelines-sample-post-release-action*.jar deployment/pipelines-sample-post-release-action.jar - - cp pipelines/ncbi/target/pipelines-ncbi-*.jar deployment/pipelines-ncbi.jar - - cp pipelines/reindex/target/pipelines-reindex-*.jar deployment/pipelines-reindex.jar - - cp pipelines/sample-transformation-dtol/target/pipelines-sample-transformation-dtol-*.jar deployment/pipelines-sample-transformation-dtol.jar + - './mvnw -q deploy -pl webapps/core -am -P embl-ebi -s ci_settings.xml -DskipTests -Dmaven.source.skip=true' artifacts: paths: - - deployment + - webapps/core/target/webapps-core-*.war + + # maven-package: + # image: ${CI_REGISTRY_IMAGE}/eclipse-temurin:17-jdk + # stage: build + # script: + # - './mvnw -q deploy -P embl-ebi -s ci_settings.xml -DskipTests -Dmaven.source.skip=true' + # - mkdir deployment + # - cp webapps/core/target/webapps-core-*.war deployment/webapps-core.war + # - cp webapps/core-v2/target/webapps-core-v2*.jar deployment/webapps-core-v2.jar + # - cp agents/solr/target/agents-solr-*.jar deployment/agents-solr.jar + # - cp agents/uploadworkers/target/agents-uploadworkers-*.jar deployment/agents-uploadworkers.jar + # - cp pipelines/curation/target/pipelines-curation-*.jar deployment/pipelines-curation.jar + # - cp pipelines/ena/target/pipelines-ena-*.jar deployment/pipelines-ena.jar + # - cp pipelines/ncbi-ena-link/target/pipelines-ncbi-ena-link-*.jar deployment/pipelines-ncbi-ena-link.jar + # - cp pipelines/sample-release/target/pipelines-sample-release-*.jar deployment/pipelines-sample-release.jar + # - cp pipelines/sample-post-release-action/target/pipelines-sample-post-release-action*.jar deployment/pipelines-sample-post-release-action.jar + # - cp pipelines/ncbi/target/pipelines-ncbi-*.jar deployment/pipelines-ncbi.jar + # - cp pipelines/reindex/target/pipelines-reindex-*.jar deployment/pipelines-reindex.jar + # - cp pipelines/sample-transformation-dtol/target/pipelines-sample-transformation-dtol-*.jar deployment/pipelines-sample-transformation-dtol.jar + # artifacts: + # paths: + # - deployment # when: manual +build_docker_image: + stage: package + image: docker:stable + services: + - docker:stable-dind + before_script: + - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY" + script: + - docker build --build-arg DOCKER_REGISTRY=${CI_REGISTRY_IMAGE} -t $DOCKER_IMAGE_NAME -f webapps/core/Dockerfile . + - docker push $DOCKER_IMAGE_NAME + after_script: + - docker logout ${CI_REGISTRY_IMAGE} + +deploy-dev-bsd-v1: + image: dtzar/helm-kubectl:3.11.0 + stage: deploy + script: + - cd webapps/core + - sed -i "s|%DOCKER_IMAGE%|$DOCKER_IMAGE_NAME|g" core-deployment.yaml + - kubectl config set-cluster bsd-cluster --server="${K8_HL_SERVER}" + - kubectl config set clusters.bsd-cluster.certificate-authority-data ${K8_HL_CERTIFICATE_AUTHORITY_DATA} + - kubectl config set-credentials bsd-user --token="${K8_HL_CREDENTIALS}" + - kubectl config set-context bsd-context --cluster=bsd-cluster --user=bsd-user + - kubectl config use-context bsd-context + - kubectl apply -f core-deployment.yaml --namespace=biosamples-dev + - kubectl apply -f core-service.yaml --namespace=biosamples-dev + when: manual + clone-config-dev: stage: config script: @@ -127,7 +172,7 @@ deploy-production: parallel: matrix: - BSD_NODE_NAME: bsd_prod - BSD_HOST_NAME: [wp-p2m-40, wp-p2m-41, wp-p1m-40, wp-p1m-41] + BSD_HOST_NAME: [ wp-p2m-40, wp-p2m-41, wp-p1m-40, wp-p1m-41 ] environment: name: prod-${BSD_HOST_NAME} url: http://${BSD_HOST_NAME}:8081/biosamples/ @@ -144,7 +189,7 @@ deploy-production-solr: parallel: matrix: - BSD_NODE_NAME: bsd_prod - BSD_SOLR_HOST_NAME: [wp-p1m-42, wp-p2m-42] + BSD_SOLR_HOST_NAME: [ wp-p1m-42, wp-p2m-42 ] environment: name: prod-solr-${BSD_HOST_NAME} url: http://${BSD_SOLR_HOST_NAME}.ebi.ac.uk:8983/solr diff --git a/RELEASE.adoc b/RELEASE.adoc index 5e521f01e..4b69922de 100644 --- a/RELEASE.adoc +++ b/RELEASE.adoc @@ -2,6 +2,12 @@ :toc: auto This pages contains links to release notes for BioSamples for version 4.0.0 and higher. This release represents a comprehensive overhaul and therefore previous release notes are no longer applicable. + +[[v5.3.12]] +== v5.3.12 Release notes +* EBI Survey banner +* Website main navigation menu: Bring "Upload" menu under "Submit" menu + [[v5.3.11]] == v5.3.11 Release notes * Fixed ASCII docs diff --git a/agents/pom.xml b/agents/pom.xml index ebae5ca28..0fb28d424 100644 --- a/agents/pom.xml +++ b/agents/pom.xml @@ -9,7 +9,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT diff --git a/agents/solr/pom.xml b/agents/solr/pom.xml index 529b785a8..a0e3442f4 100644 --- a/agents/solr/pom.xml +++ b/agents/solr/pom.xml @@ -7,7 +7,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ @@ -15,22 +15,12 @@ uk.ac.ebi.biosamples properties - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples - models-solr - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - messaging - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-ols - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT @@ -38,6 +28,10 @@ spring-hateoas 1.3.4 + + org.springframework.boot + spring-boot-starter-web + com.github.ben-manes.caffeine caffeine diff --git a/agents/solr/src/main/java/uk/ac/ebi/biosamples/solr/MessageHandlerSolr.java b/agents/solr/src/main/java/uk/ac/ebi/biosamples/solr/MessageHandlerSolr.java index fb785dd50..444a54bd0 100644 --- a/agents/solr/src/main/java/uk/ac/ebi/biosamples/solr/MessageHandlerSolr.java +++ b/agents/solr/src/main/java/uk/ac/ebi/biosamples/solr/MessageHandlerSolr.java @@ -19,10 +19,10 @@ import org.slf4j.LoggerFactory; import org.springframework.amqp.rabbit.annotation.RabbitListener; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.MessageContent; -import uk.ac.ebi.biosamples.Messaging; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.messaging.MessagingConstants; +import uk.ac.ebi.biosamples.messaging.model.MessageContent; import uk.ac.ebi.biosamples.solr.model.SolrSample; import uk.ac.ebi.biosamples.solr.repo.SolrSampleRepository; import uk.ac.ebi.biosamples.solr.service.SampleToSolrSampleConverter; @@ -51,14 +51,14 @@ public MessageHandlerSolr( } @RabbitListener( - queues = Messaging.INDEXING_QUEUE, + queues = MessagingConstants.INDEXING_QUEUE, containerFactory = "biosamplesAgentSolrContainerFactory") public void handleIndexing(final MessageContent messageContent) { handle(messageContent); } @RabbitListener( - queues = Messaging.REINDEXING_QUEUE, + queues = MessagingConstants.REINDEXING_QUEUE, containerFactory = "biosamplesAgentSolrContainerFactory") public void handleReindexing(final MessageContent messageContent) { handle(messageContent); diff --git a/agents/solr/src/main/java/uk/ac/ebi/biosamples/solr/SolrRunner.java b/agents/solr/src/main/java/uk/ac/ebi/biosamples/solr/SolrRunner.java index f9ea5182f..9e449c844 100644 --- a/agents/solr/src/main/java/uk/ac/ebi/biosamples/solr/SolrRunner.java +++ b/agents/solr/src/main/java/uk/ac/ebi/biosamples/solr/SolrRunner.java @@ -17,8 +17,8 @@ import org.springframework.boot.ApplicationRunner; import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.MessageUtils; -import uk.ac.ebi.biosamples.Messaging; +import uk.ac.ebi.biosamples.messaging.MessagingConstants; +import uk.ac.ebi.biosamples.messaging.service.MessageUtils; @Component public class SolrRunner implements ApplicationRunner { @@ -36,9 +36,9 @@ public void run(final ApplicationArguments args) throws Exception { || messageCount == null || messageCount > 0) { Thread.sleep(1000); - messageCount = messageUtils.getQueueCount(Messaging.INDEXING_QUEUE); + messageCount = messageUtils.getQueueCount(MessagingConstants.INDEXING_QUEUE); - log.trace("Messages remaining in " + Messaging.INDEXING_QUEUE + " " + messageCount); + log.trace("Messages remaining in " + MessagingConstants.INDEXING_QUEUE + " " + messageCount); } } } diff --git a/agents/solr/src/test/java/uk/ac/ebi/biosamples/solr/MessageHandlerSolrTest.java b/agents/solr/src/test/java/uk/ac/ebi/biosamples/solr/MessageHandlerSolrTest.java index a7d3a0169..943ad5081 100644 --- a/agents/solr/src/test/java/uk/ac/ebi/biosamples/solr/MessageHandlerSolrTest.java +++ b/agents/solr/src/test/java/uk/ac/ebi/biosamples/solr/MessageHandlerSolrTest.java @@ -25,9 +25,9 @@ import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.web.client.RestTemplate; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SampleStatus; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SampleStatus; public class MessageHandlerSolrTest { diff --git a/agents/uploadworkers/pom.xml b/agents/uploadworkers/pom.xml index 0239e9c70..1d0068a67 100644 --- a/agents/uploadworkers/pom.xml +++ b/agents/uploadworkers/pom.xml @@ -1,22 +1,38 @@ - -4.0.0 + + 4.0.0 agents-uploadworkers jar uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ + + uk.ac.ebi.biosamples biosamples-spring-boot-starter - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT + + uk.ac.ebi.biosamples + properties + 5.3.13-SNAPSHOT + + + uk.ac.ebi.biosamples + core + 5.3.13-SNAPSHOT + + + org.springframework.boot spring-boot-starter-web @@ -30,31 +46,14 @@ spring-hateoas 1.3.4 - - uk.ac.ebi.biosamples - properties - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-webapp - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - messaging - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - models-mongo - 5.3.12-SNAPSHOT - + + org.apache.commons commons-csv 1.8 + @@ -72,4 +71,5 @@ + diff --git a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/runner/FileUploadMessageQueueRunner.java b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/runner/FileUploadMessageQueueRunner.java index 5b95fcce6..51fb88693 100644 --- a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/runner/FileUploadMessageQueueRunner.java +++ b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/runner/FileUploadMessageQueueRunner.java @@ -16,8 +16,8 @@ import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.MessageUtils; -import uk.ac.ebi.biosamples.Messaging; +import uk.ac.ebi.biosamples.messaging.MessagingConstants; +import uk.ac.ebi.biosamples.messaging.service.MessageUtils; @Component public class FileUploadMessageQueueRunner implements ApplicationRunner { @@ -30,9 +30,9 @@ public void run(final ApplicationArguments args) { while (true) { log.trace( "Messages remaining in " - + Messaging.UPLOAD_QUEUE + + MessagingConstants.UPLOAD_QUEUE + " " - + messageUtils.getQueueCount(Messaging.UPLOAD_QUEUE)); + + messageUtils.getQueueCount(MessagingConstants.UPLOAD_QUEUE)); } } } diff --git a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java index c529e08e2..aa59fb079 100644 --- a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java +++ b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java @@ -27,11 +27,11 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.Messaging; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; +import uk.ac.ebi.biosamples.messaging.MessagingConstants; import uk.ac.ebi.biosamples.model.SubmissionFile; import uk.ac.ebi.biosamples.mongo.model.MongoFileUpload; import uk.ac.ebi.biosamples.mongo.repository.MongoFileUploadRepository; @@ -56,7 +56,7 @@ public class FileUploadSubmissionService { @Autowired MongoSampleRepository mongoSampleRepository; @RabbitListener( - queues = Messaging.UPLOAD_QUEUE, + queues = MessagingConstants.UPLOAD_QUEUE, containerFactory = "biosamplesFileUploadSubmissionContainerFactory") public void receiveMessageFromBioSamplesFileUploaderQueue(final String mongoFileId) { handleMessage(mongoFileId); diff --git a/client/client/pom.xml b/client/client/pom.xml index 973d45797..3e5091f2b 100644 --- a/client/client/pom.xml +++ b/client/client/pom.xml @@ -21,15 +21,15 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ uk.ac.ebi.biosamples - models-core - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT org.springframework.hateoas diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/BioSamplesClient.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/BioSamplesClient.java index 40b3c978d..7815e1237 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/BioSamplesClient.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/BioSamplesClient.java @@ -32,13 +32,10 @@ import org.springframework.web.util.UriComponentsBuilder; import uk.ac.ebi.biosamples.client.service.*; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.CurationLink; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SubmissionReceipt; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.model.structured.StructuredData; -import uk.ac.ebi.biosamples.service.SampleValidator; +import uk.ac.ebi.biosamples.core.model.*; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; +import uk.ac.ebi.biosamples.core.service.SampleValidator; /** * This is the primary class for interacting with BioSamples. @@ -101,15 +98,13 @@ public BioSamplesClient( sampleRetrievalService = new SampleRetrievalService(restOperations, traverson); samplePageRetrievalService = new SamplePageRetrievalService(restOperations, traverson); - sampleCursorRetrievalService = - new SampleCursorRetrievalService( - restOperations, traverson, clientProperties.getBiosamplesClientPagesize()); + sampleCursorRetrievalService = new SampleCursorRetrievalService(restOperations, traverson); sampleSubmissionService = new SampleSubmissionService(restOperations, traverson); sampleSubmissionServiceV2 = new SampleSubmissionServiceV2(restOperations, uriV2); sampleRetrievalServiceV2 = new SampleRetrievalServiceV2(restOperations, uriV2); curationRetrievalService = new CurationRetrievalService( - restOperations, traverson, clientProperties.getBiosamplesClientPagesize()); + restOperations, traverson, clientProperties.getBiosamplesClientMaxPages()); /*TODO: In CurationSubmissionService and StructuredDataSubmissionService webin auth is handled more elegantly, replicate it in all other services*/ curationSubmissionService = new CurationSubmissionService(restOperations, traverson); structuredDataSubmissionService = diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/BioSamplesWebinGetClient.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/BioSamplesWebinGetClient.java index b0507d953..e09211266 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/BioSamplesWebinGetClient.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/BioSamplesWebinGetClient.java @@ -34,8 +34,8 @@ import uk.ac.ebi.biosamples.client.service.SampleRetrievalServiceV2; import uk.ac.ebi.biosamples.client.service.WebinAuthClientService; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.service.SampleValidator; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.service.SampleValidator; public class BioSamplesWebinGetClient implements AutoCloseable { private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/CurationRetrievalService.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/CurationRetrievalService.java index 1c5db6bdf..bfc20a09e 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/CurationRetrievalService.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/CurationRetrievalService.java @@ -21,8 +21,8 @@ import org.springframework.util.MultiValueMap; import org.springframework.web.client.RestOperations; import uk.ac.ebi.biosamples.client.utils.IterableResourceFetchAll; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.CurationLink; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.CurationLink; public class CurationRetrievalService { private final Traverson traverson; @@ -43,17 +43,13 @@ public Iterable> fetchAll() { } public Iterable> fetchAll(final String jwt) { - final MultiValueMap params = new LinkedMultiValueMap<>(); - - params.add("size", Integer.toString(pageSize)); - return new IterableResourceFetchAll<>( executor, traverson, restOperations, new ParameterizedTypeReference>>() {}, jwt, - params, + null, "curations"); } diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/CurationSubmissionService.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/CurationSubmissionService.java index e3090d311..47f31efd1 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/CurationSubmissionService.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/CurationSubmissionService.java @@ -24,7 +24,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestOperations; -import uk.ac.ebi.biosamples.model.CurationLink; +import uk.ac.ebi.biosamples.core.model.CurationLink; public class CurationSubmissionService { private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleCursorRetrievalService.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleCursorRetrievalService.java index 627ad47bc..ef524e767 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleCursorRetrievalService.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleCursorRetrievalService.java @@ -23,8 +23,8 @@ import org.springframework.util.MultiValueMap; import org.springframework.web.client.RestOperations; import uk.ac.ebi.biosamples.client.utils.IterableResourceFetchAll; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.Filter; public class SampleCursorRetrievalService { private static final ParameterizedTypeReference>> @@ -33,14 +33,12 @@ public class SampleCursorRetrievalService { private final Traverson traverson; private final ExecutorService executor; private final RestOperations restOperations; - private final int pageSize; public SampleCursorRetrievalService( - final RestOperations restOperations, final Traverson traverson, final int pageSize) { + final RestOperations restOperations, final Traverson traverson) { this.restOperations = restOperations; this.traverson = traverson; this.executor = Executors.newSingleThreadExecutor(); - this.pageSize = pageSize; } public Iterable> fetchAll( @@ -64,17 +62,14 @@ public Iterable> fetchAll( final String jwt, final boolean addCurations) { MultiValueMap params = new LinkedMultiValueMap<>(); + params.add("text", text); for (final Filter filter : filterCollection) { params.add("filter", filter.getSerialization()); } - params.add("size", Integer.toString(pageSize)); - - if (!addCurations) { - params.add("applyCurations", "false"); - } + params.add("applyCurations", String.valueOf(addCurations)); params = encodePlusInQueryParameters(params); diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SamplePageRetrievalService.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SamplePageRetrievalService.java index cc80d5815..5d9297eb7 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SamplePageRetrievalService.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SamplePageRetrievalService.java @@ -29,8 +29,8 @@ import org.springframework.util.MultiValueMap; import org.springframework.web.client.RestOperations; import org.springframework.web.util.UriComponentsBuilder; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.Filter; public class SamplePageRetrievalService { private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleRetrievalService.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleRetrievalService.java index 6f89701a0..ff4146743 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleRetrievalService.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleRetrievalService.java @@ -24,7 +24,7 @@ import org.springframework.web.client.HttpStatusCodeException; import org.springframework.web.client.RestOperations; import org.springframework.web.util.UriComponentsBuilder; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; public class SampleRetrievalService { /*TODO: check if private sample fetch work - jwt not used*/ diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleRetrievalServiceV2.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleRetrievalServiceV2.java index aef17078f..f81b9f126 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleRetrievalServiceV2.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleRetrievalServiceV2.java @@ -23,7 +23,7 @@ import org.springframework.web.client.HttpStatusCodeException; import org.springframework.web.client.RestOperations; import org.springframework.web.util.UriComponentsBuilder; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; public class SampleRetrievalServiceV2 { private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleSubmissionService.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleSubmissionService.java index f8fc2c9e4..fa33e979b 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleSubmissionService.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleSubmissionService.java @@ -28,7 +28,7 @@ import org.springframework.web.client.RestClientResponseException; import org.springframework.web.client.RestOperations; import org.springframework.web.util.UriComponentsBuilder; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; public class SampleSubmissionService { private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleSubmissionServiceV2.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleSubmissionServiceV2.java index 101ab6710..179049aa7 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleSubmissionServiceV2.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/SampleSubmissionServiceV2.java @@ -25,8 +25,8 @@ import org.springframework.web.client.RestClientResponseException; import org.springframework.web.client.RestOperations; import org.springframework.web.util.UriComponentsBuilder; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SubmissionReceipt; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SubmissionReceipt; public class SampleSubmissionServiceV2 { private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/StructuredDataSubmissionService.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/StructuredDataSubmissionService.java index 0d3260108..cec256b02 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/StructuredDataSubmissionService.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/service/StructuredDataSubmissionService.java @@ -23,7 +23,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestOperations; -import uk.ac.ebi.biosamples.model.structured.StructuredData; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; public class StructuredDataSubmissionService { private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/ClientProperties.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/ClientProperties.java index 096ff238f..840e904c1 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/ClientProperties.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/ClientProperties.java @@ -28,7 +28,7 @@ public class ClientProperties { private URI biosamplesClientUriV2; @Value("${biosamples.client.pagesize:499}") - private int biosamplesClientPagesize; + private int biosamplesClientMaxPages; // in milliseconds @Value("${biosamples.client.timeout:60000}") diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/IterableResourceFetchAll.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/IterableResourceFetchAll.java index 83f1b7aa6..ec99ea5b2 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/IterableResourceFetchAll.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/IterableResourceFetchAll.java @@ -13,12 +13,14 @@ import java.net.URI; import java.util.Arrays; import java.util.Iterator; +import java.util.List; import java.util.NoSuchElementException; import java.util.Objects; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.core.ParameterizedTypeReference; @@ -36,6 +38,7 @@ import org.springframework.web.util.UriComponentsBuilder; import org.springframework.web.util.UriTemplate; +@Slf4j public class IterableResourceFetchAll implements Iterable> { private final Traverson traverson; private final RestOperations restOperations; @@ -82,8 +85,9 @@ public IterableResourceFetchAll( @Override public Iterator> iterator() { - + boolean applyCurations = true; TraversalBuilder traversonBuilder = null; + for (final Hop hop : hops) { if (traversonBuilder == null) { traversonBuilder = traverson.follow(hop); @@ -92,29 +96,49 @@ public Iterator> iterator() { } } - // get the first page - final URI uri = - UriComponentsBuilder.fromHttpUrl(traversonBuilder.asLink().getHref()) - .queryParams(params) - .build() - .toUri(); + final String href = traversonBuilder.asLink().getHref(); + // Remove the existing 'applyCurations' parameter while preserving encoding + final UriComponentsBuilder uriBuilder = UriComponentsBuilder.fromHttpUrl(href); - final RequestEntity requestEntity = - IteratorResourceFetchAll.NextPageCallable.buildRequestEntity(jwt, uri); + uriBuilder.replaceQueryParam("applyCurations"); // removes all instances + + // Add query parameters while preserving their encoding + if (params != null) { + for (final String key : params.keySet()) { + final List values = params.get(key); + + if (values != null) { + for (final String value : values) { + uriBuilder.queryParam(key, value); + } + } + } + } + // Build the URI with encoding disabled to preserve existing encoding + final URI finalUri = uriBuilder.build(false).toUri(); + final RequestEntity requestEntity = + IteratorResourceFetchAll.NextPageCallable.buildRequestEntity(jwt, finalUri); final ResponseEntity>> responseEntity = restOperations.exchange(requestEntity, parameterizedTypeReference); + + if (params != null + && params.get("applyCurations") != null + && Objects.equals(params.getFirst("applyCurations"), "false")) { + applyCurations = false; + } + return new IteratorResourceFetchAll<>( Objects.requireNonNull(responseEntity.getBody()), restOperations, parameterizedTypeReference, executor, - jwt); + jwt, + applyCurations); } private static class IteratorResourceFetchAll implements Iterator> { private final Logger log = LoggerFactory.getLogger(getClass()); - private final RestOperations restOperations; private final ExecutorService executor; private final ParameterizedTypeReference>> parameterizedTypeReference; @@ -122,37 +146,48 @@ private static class IteratorResourceFetchAll implements Iterator> pageIterator; private Future>> nextPageFuture; private final String jwt; + private final boolean applyCurations; IteratorResourceFetchAll( final PagedModel> page, final RestOperations restOperations, final ParameterizedTypeReference>> parameterizedTypeReference, final ExecutorService executor, - final String jwt) { - + final String jwt, + final boolean applyCurations) { this.page = page; - pageIterator = page.iterator(); this.restOperations = restOperations; this.executor = executor; this.parameterizedTypeReference = parameterizedTypeReference; this.jwt = jwt; + this.applyCurations = applyCurations; + + pageIterator = page.iterator(); } @Override public synchronized boolean hasNext() { // pre-emptively grab the next page as a future if (nextPageFuture == null && page.hasLink(IanaLinkRelations.NEXT)) { - final Link nextLink = page.getLink(IanaLinkRelations.NEXT).get(); - final URI uri; + URI uri; + if (nextLink.isTemplated()) { final UriTemplate uriTemplate = new UriTemplate(nextLink.getHref()); + uri = uriTemplate.expand(); } else { uri = URI.create(nextLink.getHref()); } - log.trace("getting next page uri " + uri); + + uri = + UriComponentsBuilder.fromUri(uri) + .queryParam("applyCurations", applyCurations) + .build(true) + .toUri(); + + log.trace("Getting next page uri " + uri); nextPageFuture = executor.submit( @@ -171,8 +206,10 @@ public synchronized boolean hasNext() { throw new RuntimeException(e); } pageIterator = page.iterator(); + return hasNext(); } + return false; } @@ -191,6 +228,7 @@ public EntityModel next() { } catch (final InterruptedException | ExecutionException e) { throw new RuntimeException(e); } + if (pageIterator.hasNext()) { return pageIterator.next(); } @@ -228,6 +266,7 @@ public PagedModel> call() { private static RequestEntity buildRequestEntity(final String jwt, final URI uri) { final MultiValueMap headers = new LinkedMultiValueMap<>(); + headers.add(HttpHeaders.CONTENT_TYPE, MediaTypes.HAL_JSON.toString()); if (jwt != null) { diff --git a/client/client/src/test/java/uk/ac/ebi/biosamples/model/ResourceSerializationTest.java b/client/client/src/test/java/uk/ac/ebi/biosamples/model/ResourceSerializationTest.java index 9fc33a956..924a65c57 100644 --- a/client/client/src/test/java/uk/ac/ebi/biosamples/model/ResourceSerializationTest.java +++ b/client/client/src/test/java/uk/ac/ebi/biosamples/model/ResourceSerializationTest.java @@ -36,6 +36,7 @@ import org.springframework.hateoas.EntityModel; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import uk.ac.ebi.biosamples.core.model.*; @RunWith(SpringRunner.class) @JsonTest diff --git a/client/pom.xml b/client/pom.xml index 6eee481bc..a8a37ad7e 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -14,7 +14,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../ diff --git a/client/starter/pom.xml b/client/starter/pom.xml index ab270b0cb..3873616fd 100644 --- a/client/starter/pom.xml +++ b/client/starter/pom.xml @@ -4,8 +4,7 @@ uk.ac.ebi.biosamples biosamples-spring-boot-starter - 5.3.12-SNAPSHOT - + 5.3.13-SNAPSHOT jar @@ -17,7 +16,7 @@ uk.ac.ebi.biosamples client - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT org.springframework.boot @@ -89,6 +88,11 @@ + + org.apache.maven.plugins + maven-deploy-plugin + 3.1.1 + com.diffplug.spotless spotless-maven-plugin diff --git a/client/starter/src/main/java/uk/ac/ebi/biosamples/client/BioSamplesAutoConfiguration.java b/client/starter/src/main/java/uk/ac/ebi/biosamples/client/BioSamplesAutoConfiguration.java index c5206559b..4198edf14 100644 --- a/client/starter/src/main/java/uk/ac/ebi/biosamples/client/BioSamplesAutoConfiguration.java +++ b/client/starter/src/main/java/uk/ac/ebi/biosamples/client/BioSamplesAutoConfiguration.java @@ -49,8 +49,8 @@ import uk.ac.ebi.biosamples.client.service.ClientService; import uk.ac.ebi.biosamples.client.service.WebinAuthClientService; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.service.AttributeValidator; -import uk.ac.ebi.biosamples.service.SampleValidator; +import uk.ac.ebi.biosamples.core.service.AttributeValidator; +import uk.ac.ebi.biosamples.core.service.SampleValidator; @Configuration @AutoConfigureAfter(WebClientAutoConfiguration.class) diff --git a/commons/pom.xml b/commons/pom.xml deleted file mode 100644 index edc41d35e..000000000 --- a/commons/pom.xml +++ /dev/null @@ -1,55 +0,0 @@ - -4.0.0 - -commons -commons - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - - - - 17 - 17 - 17 - - - - - org.springframework - spring-webmvc - - - uk.ac.ebi.biosamples - models-mongo - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - models-core - 5.3.12-SNAPSHOT - - - com.fasterxml.jackson.core - jackson-annotations - 2.12.5 - - - com.google.protobuf - protobuf-java - 3.16.1 - - - com.google.protobuf - protobuf-java-util - 3.8.0 - - - org.phenopackets - phenopacket-schema - 1.0.0 - - - diff --git a/core/pom.xml b/core/pom.xml new file mode 100644 index 000000000..769e2ab8c --- /dev/null +++ b/core/pom.xml @@ -0,0 +1,183 @@ + + + + 4.0.0 + + + uk.ac.ebi.biosamples + biosamples + 5.3.13-SNAPSHOT + + + core + + + 17 + 17 + UTF-8 + + + + + + + uk.ac.ebi.biosamples + properties + 5.3.13-SNAPSHOT + + + uk.ac.ebi.ena.taxonomy + sub-taxonomy-client + 2.2.8 + + + + + org.springframework + spring-web + + + org.springframework.boot + spring-boot-starter-amqp + + + org.springframework.boot + spring-boot-starter-cache + + + org.springframework.boot + spring-boot-starter-data-mongodb + + + org.springframework.boot + spring-boot-starter-test + 1.5.15.RELEASE + test + + + org.springframework.data + spring-data-solr + 4.1.0.RELEASE + + + org.springframework.hateoas + spring-hateoas + 1.3.4 + + + org.springframework.security.oauth + spring-security-oauth2 + 2.5.0.RELEASE + + + + + com.fasterxml.jackson.core + jackson-annotations + 2.12.5 + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + + + com.fasterxml.jackson.module + jackson-module-jaxb-annotations + + + org.codehaus.woodstox + woodstox-core-asl + 4.1.4 + + + + + com.google.protobuf + protobuf-java + 3.16.1 + + + com.google.protobuf + protobuf-java-util + 3.8.0 + + + org.phenopackets + phenopacket-schema + 1.0.0 + + + + + com.auth0 + java-jwt + 4.0.0 + + + + + com.github.ben-manes.caffeine + caffeine + + + org.apache.commons + commons-collections4 + 4.1 + + + org.apache.commons + commons-csv + 1.8 + + + org.apache.commons + commons-text + 1.8 + + + org.neo4j.driver + neo4j-java-driver + 4.0.0 + + + + + com.google.code + sitemapgen4j + 1.0.1 + + + + + javax.servlet + javax.servlet-api + + + javax.validation + validation-api + + + + + + + + + xml-apis + xml-apis + 1.4.01 + + + + + diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/AccessControlControllerAdvice.java b/core/src/main/java/uk/ac/ebi/biosamples/advice/AccessControlControllerAdvice.java similarity index 94% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/AccessControlControllerAdvice.java rename to core/src/main/java/uk/ac/ebi/biosamples/advice/AccessControlControllerAdvice.java index 5cc4fba6f..0e20b81d9 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/AccessControlControllerAdvice.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/advice/AccessControlControllerAdvice.java @@ -1,29 +1,29 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.advice; - -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.ControllerAdvice; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.context.request.WebRequest; -import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; - -@ControllerAdvice -public class AccessControlControllerAdvice extends ResponseEntityExceptionHandler { - @ExceptionHandler(value = {GlobalExceptions.AccessControlException.class}) - protected ResponseEntity handleConflict(RuntimeException e, WebRequest request) { - return handleExceptionInternal( - e, e.getMessage(), new HttpHeaders(), HttpStatus.UNAUTHORIZED, request); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.advice; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; + +@ControllerAdvice +public class AccessControlControllerAdvice extends ResponseEntityExceptionHandler { + @ExceptionHandler(value = {GlobalExceptions.AccessControlException.class}) + protected ResponseEntity handleConflict(RuntimeException e, WebRequest request) { + return handleExceptionInternal( + e, e.getMessage(), new HttpHeaders(), HttpStatus.UNAUTHORIZED, request); + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/ModelAttributeControllerAdvice.java b/core/src/main/java/uk/ac/ebi/biosamples/advice/ModelAttributeControllerAdvice.java similarity index 97% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/ModelAttributeControllerAdvice.java rename to core/src/main/java/uk/ac/ebi/biosamples/advice/ModelAttributeControllerAdvice.java index 226bc672e..65d5b66ed 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/ModelAttributeControllerAdvice.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/advice/ModelAttributeControllerAdvice.java @@ -1,30 +1,30 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.advice; - -import java.net.URI; -import org.springframework.web.bind.annotation.ControllerAdvice; -import org.springframework.web.bind.annotation.ModelAttribute; -import uk.ac.ebi.biosamples.BioSamplesProperties; - -@ControllerAdvice -public class ModelAttributeControllerAdvice { - private final BioSamplesProperties bioSamplesProperties; - - public ModelAttributeControllerAdvice(final BioSamplesProperties bioSamplesProperties) { - this.bioSamplesProperties = bioSamplesProperties; - } - - @ModelAttribute("usiUrl") - public URI addLinkUsi() { - return bioSamplesProperties.getUsiCoreUri(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.advice; + +import java.net.URI; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ModelAttribute; +import uk.ac.ebi.biosamples.BioSamplesProperties; + +@ControllerAdvice +public class ModelAttributeControllerAdvice { + private final BioSamplesProperties bioSamplesProperties; + + public ModelAttributeControllerAdvice(final BioSamplesProperties bioSamplesProperties) { + this.bioSamplesProperties = bioSamplesProperties; + } + + @ModelAttribute("usiUrl") + public URI addLinkUsi() { + return bioSamplesProperties.getUsiCoreUri(); + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/SampleConversionControllerAdvice.java b/core/src/main/java/uk/ac/ebi/biosamples/advice/SampleConversionControllerAdvice.java similarity index 94% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/SampleConversionControllerAdvice.java rename to core/src/main/java/uk/ac/ebi/biosamples/advice/SampleConversionControllerAdvice.java index 6599ec0dc..f5970ab99 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/SampleConversionControllerAdvice.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/advice/SampleConversionControllerAdvice.java @@ -1,29 +1,29 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.advice; - -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.ControllerAdvice; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.context.request.WebRequest; -import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; - -@ControllerAdvice -public class SampleConversionControllerAdvice extends ResponseEntityExceptionHandler { - @ExceptionHandler(GlobalExceptions.SampleConversionException.class) - protected ResponseEntity handleConflict(RuntimeException e, WebRequest request) { - return handleExceptionInternal( - e, e.getMessage(), new HttpHeaders(), HttpStatus.BAD_REQUEST, request); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.advice; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; + +@ControllerAdvice +public class SampleConversionControllerAdvice extends ResponseEntityExceptionHandler { + @ExceptionHandler(GlobalExceptions.SampleConversionException.class) + protected ResponseEntity handleConflict(RuntimeException e, WebRequest request) { + return handleExceptionInternal( + e, e.getMessage(), new HttpHeaders(), HttpStatus.BAD_REQUEST, request); + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotAccessibleControllerAdvice.java b/core/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotAccessibleControllerAdvice.java similarity index 94% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotAccessibleControllerAdvice.java rename to core/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotAccessibleControllerAdvice.java index 2392306f8..8ba751750 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotAccessibleControllerAdvice.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotAccessibleControllerAdvice.java @@ -1,28 +1,28 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.advice; - -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.ControllerAdvice; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.context.request.WebRequest; -import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; - -@ControllerAdvice -public class SampleNotAccessibleControllerAdvice extends ResponseEntityExceptionHandler { - @ExceptionHandler(value = {GlobalExceptions.SampleNotAccessibleAdviceException.class}) - protected ResponseEntity handleConflict(RuntimeException ex, WebRequest request) { - return handleExceptionInternal(ex, null, new HttpHeaders(), HttpStatus.FORBIDDEN, request); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.advice; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; + +@ControllerAdvice +public class SampleNotAccessibleControllerAdvice extends ResponseEntityExceptionHandler { + @ExceptionHandler(value = {GlobalExceptions.SampleNotAccessibleAdviceException.class}) + protected ResponseEntity handleConflict(RuntimeException ex, WebRequest request) { + return handleExceptionInternal(ex, null, new HttpHeaders(), HttpStatus.FORBIDDEN, request); + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotFoundControllerAdvice.java b/core/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotFoundControllerAdvice.java similarity index 94% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotFoundControllerAdvice.java rename to core/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotFoundControllerAdvice.java index 7a19b4e2e..ae824006f 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotFoundControllerAdvice.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/advice/SampleNotFoundControllerAdvice.java @@ -1,28 +1,28 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.advice; - -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.ControllerAdvice; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.context.request.WebRequest; -import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; - -@ControllerAdvice -public class SampleNotFoundControllerAdvice extends ResponseEntityExceptionHandler { - @ExceptionHandler(value = {GlobalExceptions.SampleNotFoundException.class}) - protected ResponseEntity handleConflict(RuntimeException ex, WebRequest request) { - return handleExceptionInternal(ex, null, new HttpHeaders(), HttpStatus.NOT_FOUND, request); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.advice; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; + +@ControllerAdvice +public class SampleNotFoundControllerAdvice extends ResponseEntityExceptionHandler { + @ExceptionHandler(value = {GlobalExceptions.SampleNotFoundException.class}) + protected ResponseEntity handleConflict(RuntimeException ex, WebRequest request) { + return handleExceptionInternal(ex, null, new HttpHeaders(), HttpStatus.NOT_FOUND, request); + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/SchemaValidationControllerAdvice.java b/core/src/main/java/uk/ac/ebi/biosamples/advice/SchemaValidationControllerAdvice.java similarity index 94% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/SchemaValidationControllerAdvice.java rename to core/src/main/java/uk/ac/ebi/biosamples/advice/SchemaValidationControllerAdvice.java index 4772d0f8f..a1552371a 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/advice/SchemaValidationControllerAdvice.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/advice/SchemaValidationControllerAdvice.java @@ -1,29 +1,29 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.advice; - -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.ControllerAdvice; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.context.request.WebRequest; -import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; - -@ControllerAdvice -public class SchemaValidationControllerAdvice extends ResponseEntityExceptionHandler { - @ExceptionHandler(value = {GlobalExceptions.SchemaValidationException.class}) - protected ResponseEntity handleConflict(RuntimeException e, WebRequest request) { - return handleExceptionInternal( - e, e.getMessage(), new HttpHeaders(), HttpStatus.BAD_REQUEST, request); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.advice; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; + +@ControllerAdvice +public class SchemaValidationControllerAdvice extends ResponseEntityExceptionHandler { + @ExceptionHandler(value = {GlobalExceptions.SchemaValidationException.class}) + protected ResponseEntity handleConflict(RuntimeException e, WebRequest request) { + return handleExceptionInternal( + e, e.getMessage(), new HttpHeaders(), HttpStatus.BAD_REQUEST, request); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/ModelConfig.java b/core/src/main/java/uk/ac/ebi/biosamples/core/ModelConfig.java similarity index 96% rename from models/core/src/main/java/uk/ac/ebi/biosamples/ModelConfig.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/ModelConfig.java index 2369c9e36..3469b0a57 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/ModelConfig.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/ModelConfig.java @@ -1,34 +1,34 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples; - -import org.springframework.context.annotation.Configuration; - -@Configuration -public class ModelConfig { - - // @Bean - // @Order(1) - // public Jackson2ObjectMapperBuilderCustomizer customizer() { - // return (Jackson2ObjectMapperBuilder jackson2ObjectMapperBuilder) -> { - // Map, JsonSerializer> serializers = new HashMap<>(); - // Map, JsonDeserializer> deserializers = new HashMap<>(); - // - // serializers.put(Sample.class, new CustomSampleSerializer()); - // serializers.put(Instant.class, new CustomInstantSerializer()); - // deserializers.put(Sample.class, new CustomSampleDeserializer()); - // deserializers.put(Sample.class, new CustomSampleDeserializer()); - // - // jackson2ObjectMapperBuilder.serializersByType(serializers); - // jackson2ObjectMapperBuilder.deserializersByType(deserializers); - // }; - // } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core; + +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ModelConfig { + + // @Bean + // @Order(1) + // public Jackson2ObjectMapperBuilderCustomizer customizer() { + // return (Jackson2ObjectMapperBuilder jackson2ObjectMapperBuilder) -> { + // Map, JsonSerializer> serializers = new HashMap<>(); + // Map, JsonDeserializer> deserializers = new HashMap<>(); + // + // serializers.put(Sample.class, new CustomSampleSerializer()); + // serializers.put(Instant.class, new CustomInstantSerializer()); + // deserializers.put(Sample.class, new CustomSampleDeserializer()); + // deserializers.put(Sample.class, new CustomSampleDeserializer()); + // + // jackson2ObjectMapperBuilder.serializersByType(serializers); + // jackson2ObjectMapperBuilder.deserializersByType(deserializers); + // }; + // } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Accession.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Accession.java similarity index 91% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/Accession.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/Accession.java index 6f0b3c55d..6b4343316 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Accession.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Accession.java @@ -1,48 +1,48 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import lombok.Data; -import uk.ac.ebi.biosamples.service.AccessionSerializer; - -@Data -@JsonInclude(JsonInclude.Include.NON_EMPTY) -@JsonSerialize(using = AccessionSerializer.class) -public class Accession implements Comparable { - @JsonProperty("id") - protected String id; - - private Accession(final String id) { - this.id = id; - } - - @Override - public int compareTo(final Accession other) { - if (other == null) { - return 1; - } - - if (!id.equals(other.id)) { - return id.compareTo(other.id); - } - - return 0; - } - - @JsonCreator - public static Accession build(@JsonProperty("id") final String accession) { - return new Accession(accession); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import lombok.Data; +import uk.ac.ebi.biosamples.core.service.AccessionSerializer; + +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@JsonSerialize(using = AccessionSerializer.class) +public class Accession implements Comparable { + @JsonProperty("id") + protected String id; + + private Accession(final String id) { + this.id = id; + } + + @Override + public int compareTo(final Accession other) { + if (other == null) { + return 1; + } + + if (!id.equals(other.id)) { + return id.compareTo(other.id); + } + + return 0; + } + + @JsonCreator + public static Accession build(@JsonProperty("id") final String accession) { + return new Accession(accession); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/AccessionType.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/AccessionType.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/AccessionType.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/AccessionType.java index bcbc78af3..58f091617 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/AccessionType.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/AccessionType.java @@ -1,33 +1,33 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import lombok.Getter; - -@Getter -public enum AccessionType { - ANY("SAM[END][AG]?[0-9]+"), - ANY_GROUP("SAMEG[0-9]+"), - ANY_SAMPLE("SAM[END][A]?[0-9]+"), - NCBI_SAMPLE("SAMN[0-9]+"), - EBI_SAMPLE("SAME[AG]?[0-9]+"), - DDBJ_SAMPLE("SAMD[0-9]+"); - - private final String accessionRegex; - - AccessionType(String regex) { - this.accessionRegex = regex; - } - - public boolean matches(String accession) { - return accession.matches(this.accessionRegex); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import lombok.Getter; + +@Getter +public enum AccessionType { + ANY("SAM[END][AG]?[0-9]+"), + ANY_GROUP("SAMEG[0-9]+"), + ANY_SAMPLE("SAM[END][A]?[0-9]+"), + NCBI_SAMPLE("SAMN[0-9]+"), + EBI_SAMPLE("SAME[AG]?[0-9]+"), + DDBJ_SAMPLE("SAMD[0-9]+"); + + private final String accessionRegex; + + AccessionType(String regex) { + this.accessionRegex = regex; + } + + public boolean matches(String accession) { + return accession.matches(this.accessionRegex); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Attribute.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Attribute.java similarity index 95% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/Attribute.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/Attribute.java index 4413b4814..a83f05bf7 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Attribute.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Attribute.java @@ -1,224 +1,224 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.Lists; -import java.util.*; -import lombok.Data; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@Data -@JsonInclude(JsonInclude.Include.NON_EMPTY) -public class Attribute implements Comparable { - // TODO: This needs to be public static otherwise spring-data-mongo goes crazy - public static Logger log = LoggerFactory.getLogger(Attribute.class); - - @JsonProperty("type") - private String type; - - @JsonProperty("value") - private String value; - - @JsonProperty("iri") - private SortedSet iri; - - @JsonProperty("unit") - private String unit; - - @JsonProperty("tag") - private String tag; - - private Attribute() {} - - @Override - public int compareTo(final Attribute other) { - if (other == null) { - return 1; - } - - int comparison = nullSafeStringComparison(type, other.type); - - if (comparison != 0) { - return comparison; - } - - comparison = nullSafeStringComparison(value, other.value); - - if (comparison != 0) { - return comparison; - } - - comparison = nullSafeStringComparison(tag, other.tag); - - if (comparison != 0) { - return comparison; - } - - if (iri == null && other.iri != null) { - return -1; - } - - if (iri != null && other.iri == null) { - return 1; - } - - assert iri != null; - - if (!iri.equals(other.iri)) { - if (iri.size() < other.iri.size()) { - return -1; - } else if (iri.size() > other.iri.size()) { - return 1; - } else { - final Iterator thisIt = iri.iterator(); - final Iterator otherIt = other.iri.iterator(); - - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - - if (val != 0) { - return val; - } - } - } - } - - return nullSafeStringComparison(unit, other.unit); - } - - private int nullSafeStringComparison(final String one, final String two) { - if (one == null && two != null) { - return -1; - } - - if (one != null && two == null) { - return 1; - } - - if (one != null && !one.equals(two)) { - return one.compareTo(two); - } - - return 0; - } - - public static Attribute build(final String type, final String value) { - return build(type, value, null, Lists.newArrayList(), null); - } - - public static Attribute build( - final String type, final String value, String iri, final String unit) { - if (iri == null) { - iri = ""; - } - - return build(type, value, null, Lists.newArrayList(iri), unit); - } - - public static Attribute build( - final String type, final String value, final String tag, String iri, final String unit) { - if (iri == null) { - iri = ""; - } - - return build(type, value, tag, Lists.newArrayList(iri), unit); - } - - @JsonCreator - public static Attribute build( - @JsonProperty("type") String type, - @JsonProperty("value") String value, - @JsonProperty("tag") String tag, - @JsonProperty("iri") Collection iri, - @JsonProperty("unit") String unit) { - // check for nulls - if (type == null) { - throw new IllegalArgumentException("type must not be null"); - } - - if (value == null) { - value = ""; - } - - if (iri == null) { - iri = Lists.newArrayList(); - } - // cleanup inputs - type = type.trim(); - value = value.trim(); - - if (tag != null) { - tag = tag.trim(); - } - - if (unit != null) { - unit = unit.trim(); - } - // create output - final Attribute attr = new Attribute(); - - attr.type = type; - attr.value = value; - attr.tag = tag; - attr.iri = new TreeSet<>(); - - for (final String iriOne : iri) { - if (iriOne != null) { - attr.iri.add(iriOne); - } - } - - attr.unit = unit; - return attr; - } - - public static class Builder { - private final String type; - private final String value; - private final List iris = new ArrayList<>(); - private String unit; - private String tag; - - public Builder(final String type, final String value, final String tag) { - this.type = type; - this.value = value; - this.tag = tag; - } - - public Builder(final String type, final String value) { - this.type = type; - this.value = value; - } - - public Builder withIri(final String iri) { - iris.add(iri); - return this; - } - - public Builder withUnit(final String unit) { - this.unit = unit; - return this; - } - - public Builder withTag(final String tag) { - this.tag = tag; - return this; - } - - public Attribute build() { - return Attribute.build(type, value, tag, iris, unit); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Lists; +import java.util.*; +import lombok.Data; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class Attribute implements Comparable { + // TODO: This needs to be public static otherwise spring-data-mongo goes crazy + public static Logger log = LoggerFactory.getLogger(Attribute.class); + + @JsonProperty("type") + private String type; + + @JsonProperty("value") + private String value; + + @JsonProperty("iri") + private SortedSet iri; + + @JsonProperty("unit") + private String unit; + + @JsonProperty("tag") + private String tag; + + private Attribute() {} + + @Override + public int compareTo(final Attribute other) { + if (other == null) { + return 1; + } + + int comparison = nullSafeStringComparison(type, other.type); + + if (comparison != 0) { + return comparison; + } + + comparison = nullSafeStringComparison(value, other.value); + + if (comparison != 0) { + return comparison; + } + + comparison = nullSafeStringComparison(tag, other.tag); + + if (comparison != 0) { + return comparison; + } + + if (iri == null && other.iri != null) { + return -1; + } + + if (iri != null && other.iri == null) { + return 1; + } + + assert iri != null; + + if (!iri.equals(other.iri)) { + if (iri.size() < other.iri.size()) { + return -1; + } else if (iri.size() > other.iri.size()) { + return 1; + } else { + final Iterator thisIt = iri.iterator(); + final Iterator otherIt = other.iri.iterator(); + + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + + if (val != 0) { + return val; + } + } + } + } + + return nullSafeStringComparison(unit, other.unit); + } + + private int nullSafeStringComparison(final String one, final String two) { + if (one == null && two != null) { + return -1; + } + + if (one != null && two == null) { + return 1; + } + + if (one != null && !one.equals(two)) { + return one.compareTo(two); + } + + return 0; + } + + public static Attribute build(final String type, final String value) { + return build(type, value, null, Lists.newArrayList(), null); + } + + public static Attribute build( + final String type, final String value, String iri, final String unit) { + if (iri == null) { + iri = ""; + } + + return build(type, value, null, Lists.newArrayList(iri), unit); + } + + public static Attribute build( + final String type, final String value, final String tag, String iri, final String unit) { + if (iri == null) { + iri = ""; + } + + return build(type, value, tag, Lists.newArrayList(iri), unit); + } + + @JsonCreator + public static Attribute build( + @JsonProperty("type") String type, + @JsonProperty("value") String value, + @JsonProperty("tag") String tag, + @JsonProperty("iri") Collection iri, + @JsonProperty("unit") String unit) { + // check for nulls + if (type == null) { + throw new IllegalArgumentException("type must not be null"); + } + + if (value == null) { + value = ""; + } + + if (iri == null) { + iri = Lists.newArrayList(); + } + // cleanup inputs + type = type.trim(); + value = value.trim(); + + if (tag != null) { + tag = tag.trim(); + } + + if (unit != null) { + unit = unit.trim(); + } + // create output + final Attribute attr = new Attribute(); + + attr.type = type; + attr.value = value; + attr.tag = tag; + attr.iri = new TreeSet<>(); + + for (final String iriOne : iri) { + if (iriOne != null) { + attr.iri.add(iriOne); + } + } + + attr.unit = unit; + return attr; + } + + public static class Builder { + private final String type; + private final String value; + private final List iris = new ArrayList<>(); + private String unit; + private String tag; + + public Builder(final String type, final String value, final String tag) { + this.type = type; + this.value = value; + this.tag = tag; + } + + public Builder(final String type, final String value) { + this.type = type; + this.value = value; + } + + public Builder withIri(final String iri) { + iris.add(iri); + return this; + } + + public Builder withUnit(final String unit) { + this.unit = unit; + return this; + } + + public Builder withTag(final String tag) { + this.tag = tag; + return this; + } + + public Attribute build() { + return Attribute.build(type, value, tag, iris, unit); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Certificate.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Certificate.java similarity index 94% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/Certificate.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/Certificate.java index 9ee6c3eb4..aef672279 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Certificate.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Certificate.java @@ -1,99 +1,99 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.*; - -@Data -public class Certificate implements Comparable { - @JsonProperty("name") - private String name; - - @JsonProperty("version") - private String version; - - @JsonProperty("fileName") - private String fileName; - - public Certificate(final String name, final String version, final String fileName) { - this.name = name; - this.version = version; - this.fileName = fileName; - } - - public Certificate() {} - - @Override - public int compareTo(final Certificate cert) { - if (cert == null) { - return 1; - } - - int comparison = nullSafeStringComparison(name, cert.name); - - if (comparison != 0) { - return comparison; - } - - comparison = nullSafeStringComparison(version, cert.version); - if (comparison != 0) { - return comparison; - } - - return 0; - } - - private int nullSafeStringComparison(final String one, final String two) { - - if (one == null && two != null) { - return -1; - } - if (one != null && two == null) { - return 1; - } - if (one != null && !one.equals(two)) { - return one.compareTo(two); - } - - return 0; - } - - @JsonCreator - public static Certificate build( - @JsonProperty("name") String name, - @JsonProperty("version") String version, - @JsonProperty("fileName") final String fileName) { - // check for nulls - if (name == null) { - throw new IllegalArgumentException("Certificate name must not be null"); - } - - if (version == null) { - version = ""; - } - - if (fileName == null) { - throw new IllegalArgumentException("Certificate file name must not be null"); - } - - name = name.trim(); - version = version.trim(); - - final Certificate cert = new Certificate(); - cert.name = name; - cert.fileName = fileName; - cert.version = version; - - return cert; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.*; + +@Data +public class Certificate implements Comparable { + @JsonProperty("name") + private String name; + + @JsonProperty("version") + private String version; + + @JsonProperty("fileName") + private String fileName; + + public Certificate(final String name, final String version, final String fileName) { + this.name = name; + this.version = version; + this.fileName = fileName; + } + + public Certificate() {} + + @Override + public int compareTo(final Certificate cert) { + if (cert == null) { + return 1; + } + + int comparison = nullSafeStringComparison(name, cert.name); + + if (comparison != 0) { + return comparison; + } + + comparison = nullSafeStringComparison(version, cert.version); + if (comparison != 0) { + return comparison; + } + + return 0; + } + + private int nullSafeStringComparison(final String one, final String two) { + + if (one == null && two != null) { + return -1; + } + if (one != null && two == null) { + return 1; + } + if (one != null && !one.equals(two)) { + return one.compareTo(two); + } + + return 0; + } + + @JsonCreator + public static Certificate build( + @JsonProperty("name") String name, + @JsonProperty("version") String version, + @JsonProperty("fileName") final String fileName) { + // check for nulls + if (name == null) { + throw new IllegalArgumentException("Certificate name must not be null"); + } + + if (version == null) { + version = ""; + } + + if (fileName == null) { + throw new IllegalArgumentException("Certificate file name must not be null"); + } + + name = name.trim(); + version = version.trim(); + + final Certificate cert = new Certificate(); + cert.name = name; + cert.fileName = fileName; + cert.version = version; + + return cert; + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Contact.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Contact.java similarity index 95% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/Contact.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/Contact.java index 8b40e70bf..a99e6e26b 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Contact.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Contact.java @@ -1,295 +1,295 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import java.util.Objects; -import org.springframework.util.StringUtils; - -@JsonInclude(JsonInclude.Include.NON_EMPTY) -@JsonDeserialize(builder = Contact.Builder.class) -public class Contact implements Comparable { - private final String firstName; - private final String lastName; - private final String midInitials; - private final String role; - private final String email; - private final String affiliation; - private final String name; - private final String url; - - private Contact( - final String firstName, - final String lastName, - final String midInitials, - final String name, - final String role, - final String email, - final String affiliation, - final String url) { - this.firstName = firstName; - this.lastName = lastName; - this.midInitials = midInitials; - this.name = name; - this.role = role; - this.email = email; - this.affiliation = affiliation; - this.url = url; - } - - @JsonProperty("FirstName") - public String getFirstName() { - return firstName; - } - - @JsonProperty("LastName") - public String getLastName() { - return lastName; - } - - @JsonProperty("MidInitials") - public String getMidInitials() { - return midInitials; - } - - @JsonProperty("Name") - public String getName() { - return name; - } - - @JsonProperty("Role") - public String getRole() { - return role; - } - - @JsonProperty("E-mail") - public String getEmail() { - return email; - } - - // @JsonIgnore - @JsonProperty("Affiliation") - public String getAffiliation() { - return affiliation; - } - - // @JsonIgnore - @JsonProperty("URL") - public String getUrl() { - return url; - } - - @Override - public boolean equals(final Object o) { - - if (o == this) { - return true; - } - if (!(o instanceof Contact)) { - return false; - } - final Contact other = (Contact) o; - return Objects.equals(firstName, other.firstName) - && Objects.equals(lastName, other.lastName) - && Objects.equals(midInitials, other.midInitials) - && Objects.equals(name, other.name) - && Objects.equals(role, other.role) - && Objects.equals(email, other.email) - && Objects.equals(url, other.url) - && Objects.equals(affiliation, other.affiliation); - } - - @Override - public int hashCode() { - return Objects.hash(firstName, lastName, midInitials, name, role, email, affiliation, url); - } - - @Override - public String toString() { - return "Contact{" - + "firstName='" - + firstName - + '\'' - + ", lastName='" - + lastName - + '\'' - + ", midInitials='" - + midInitials - + '\'' - + ", role='" - + role - + '\'' - + ", email='" - + email - + '\'' - + ", affiliation='" - + affiliation - + '\'' - + ", name='" - + name - + '\'' - + ", url='" - + url - + '\'' - + '}'; - } - - @Override - public int compareTo(final Contact other) { - if (other == null) { - return 1; - } - - int comparisonResult = nullSafeStringComparison(firstName, other.firstName); - if (comparisonResult != 0) { - return comparisonResult; - } - - comparisonResult = nullSafeStringComparison(lastName, other.lastName); - if (comparisonResult != 0) { - return comparisonResult; - } - - comparisonResult = nullSafeStringComparison(midInitials, other.midInitials); - if (comparisonResult != 0) { - return comparisonResult; - } - - comparisonResult = nullSafeStringComparison(name, other.name); - if (comparisonResult != 0) { - return comparisonResult; - } - - comparisonResult = nullSafeStringComparison(role, other.role); - if (comparisonResult != 0) { - return comparisonResult; - } - - comparisonResult = nullSafeStringComparison(email, other.email); - if (comparisonResult != 0) { - return comparisonResult; - } - - comparisonResult = nullSafeStringComparison(affiliation, other.affiliation); - return comparisonResult; - } - - private int nullSafeStringComparison(final String first, final String other) { - if (first == null && other == null) { - return 0; - } - if (first == null) { - return -1; - } - if (other == null) { - return 1; - } - return first.compareTo(other); - } - - // @JsonCreator - // public static Contact build(@JsonProperty("Name") String name, - // @JsonProperty("Affiliation") String affiliation, - // @JsonProperty("URL") String url) { - // return new Contact(name, affiliation,url); - // } - public static class Builder { - private String firstName; - private String lastName; - private String midInitials; - private String role; - private String email; - private String url; - private String affiliation; - private String name; - - @JsonCreator - public Builder() {} - - @JsonProperty("FirstName") - public Builder firstName(final String firstName) { - this.firstName = firstName; - return this; - } - - @JsonProperty("LastName") - public Builder lastName(final String lastName) { - this.lastName = lastName; - return this; - } - - @JsonProperty("MidInitials") - public Builder midInitials(final String midInitials) { - this.midInitials = midInitials; - return this; - } - - @JsonProperty("Role") - public Builder role(final String role) { - this.role = role; - return this; - } - - @JsonProperty("E-mail") - public Builder email(final String email) { - this.email = email; - return this; - } - - @JsonProperty("URL") - public Builder url(final String url) { - this.url = url; - return this; - } - - @JsonProperty("Affiliation") - public Builder affiliation(final String affiliation) { - this.affiliation = affiliation; - return this; - } - - @JsonProperty("Name") - public Builder name(final String name) { - this.name = name; - return this; - } - - public boolean isNotEmpty() { - // only check fields that could be meaningful alone - return StringUtils.hasText(firstName) - || StringUtils.hasText(lastName) - || StringUtils.hasText(email) - || StringUtils.hasText(name) - || StringUtils.hasText(url); - } - - private String composedName() { - final String nullSafeFirstName = firstName == null ? "" : firstName; - final String nullSafeLastName = lastName == null ? "" : lastName; - final String fullName = (nullSafeFirstName + " " + nullSafeLastName).trim(); - - if (fullName.isEmpty()) { - return null; - } - return fullName; - } - - public Contact build() { - if (name == null) { - name = composedName(); - } - return new Contact(firstName, lastName, midInitials, name, role, email, affiliation, url); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.util.Objects; +import org.springframework.util.StringUtils; + +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@JsonDeserialize(builder = Contact.Builder.class) +public class Contact implements Comparable { + private final String firstName; + private final String lastName; + private final String midInitials; + private final String role; + private final String email; + private final String affiliation; + private final String name; + private final String url; + + private Contact( + final String firstName, + final String lastName, + final String midInitials, + final String name, + final String role, + final String email, + final String affiliation, + final String url) { + this.firstName = firstName; + this.lastName = lastName; + this.midInitials = midInitials; + this.name = name; + this.role = role; + this.email = email; + this.affiliation = affiliation; + this.url = url; + } + + @JsonProperty("FirstName") + public String getFirstName() { + return firstName; + } + + @JsonProperty("LastName") + public String getLastName() { + return lastName; + } + + @JsonProperty("MidInitials") + public String getMidInitials() { + return midInitials; + } + + @JsonProperty("Name") + public String getName() { + return name; + } + + @JsonProperty("Role") + public String getRole() { + return role; + } + + @JsonProperty("E-mail") + public String getEmail() { + return email; + } + + // @JsonIgnore + @JsonProperty("Affiliation") + public String getAffiliation() { + return affiliation; + } + + // @JsonIgnore + @JsonProperty("URL") + public String getUrl() { + return url; + } + + @Override + public boolean equals(final Object o) { + + if (o == this) { + return true; + } + if (!(o instanceof Contact)) { + return false; + } + final Contact other = (Contact) o; + return Objects.equals(firstName, other.firstName) + && Objects.equals(lastName, other.lastName) + && Objects.equals(midInitials, other.midInitials) + && Objects.equals(name, other.name) + && Objects.equals(role, other.role) + && Objects.equals(email, other.email) + && Objects.equals(url, other.url) + && Objects.equals(affiliation, other.affiliation); + } + + @Override + public int hashCode() { + return Objects.hash(firstName, lastName, midInitials, name, role, email, affiliation, url); + } + + @Override + public String toString() { + return "Contact{" + + "firstName='" + + firstName + + '\'' + + ", lastName='" + + lastName + + '\'' + + ", midInitials='" + + midInitials + + '\'' + + ", role='" + + role + + '\'' + + ", email='" + + email + + '\'' + + ", affiliation='" + + affiliation + + '\'' + + ", name='" + + name + + '\'' + + ", url='" + + url + + '\'' + + '}'; + } + + @Override + public int compareTo(final Contact other) { + if (other == null) { + return 1; + } + + int comparisonResult = nullSafeStringComparison(firstName, other.firstName); + if (comparisonResult != 0) { + return comparisonResult; + } + + comparisonResult = nullSafeStringComparison(lastName, other.lastName); + if (comparisonResult != 0) { + return comparisonResult; + } + + comparisonResult = nullSafeStringComparison(midInitials, other.midInitials); + if (comparisonResult != 0) { + return comparisonResult; + } + + comparisonResult = nullSafeStringComparison(name, other.name); + if (comparisonResult != 0) { + return comparisonResult; + } + + comparisonResult = nullSafeStringComparison(role, other.role); + if (comparisonResult != 0) { + return comparisonResult; + } + + comparisonResult = nullSafeStringComparison(email, other.email); + if (comparisonResult != 0) { + return comparisonResult; + } + + comparisonResult = nullSafeStringComparison(affiliation, other.affiliation); + return comparisonResult; + } + + private int nullSafeStringComparison(final String first, final String other) { + if (first == null && other == null) { + return 0; + } + if (first == null) { + return -1; + } + if (other == null) { + return 1; + } + return first.compareTo(other); + } + + // @JsonCreator + // public static Contact build(@JsonProperty("Name") String name, + // @JsonProperty("Affiliation") String affiliation, + // @JsonProperty("URL") String url) { + // return new Contact(name, affiliation,url); + // } + public static class Builder { + private String firstName; + private String lastName; + private String midInitials; + private String role; + private String email; + private String url; + private String affiliation; + private String name; + + @JsonCreator + public Builder() {} + + @JsonProperty("FirstName") + public Builder firstName(final String firstName) { + this.firstName = firstName; + return this; + } + + @JsonProperty("LastName") + public Builder lastName(final String lastName) { + this.lastName = lastName; + return this; + } + + @JsonProperty("MidInitials") + public Builder midInitials(final String midInitials) { + this.midInitials = midInitials; + return this; + } + + @JsonProperty("Role") + public Builder role(final String role) { + this.role = role; + return this; + } + + @JsonProperty("E-mail") + public Builder email(final String email) { + this.email = email; + return this; + } + + @JsonProperty("URL") + public Builder url(final String url) { + this.url = url; + return this; + } + + @JsonProperty("Affiliation") + public Builder affiliation(final String affiliation) { + this.affiliation = affiliation; + return this; + } + + @JsonProperty("Name") + public Builder name(final String name) { + this.name = name; + return this; + } + + public boolean isNotEmpty() { + // only check fields that could be meaningful alone + return StringUtils.hasText(firstName) + || StringUtils.hasText(lastName) + || StringUtils.hasText(email) + || StringUtils.hasText(name) + || StringUtils.hasText(url); + } + + private String composedName() { + final String nullSafeFirstName = firstName == null ? "" : firstName; + final String nullSafeLastName = lastName == null ? "" : lastName; + final String fullName = (nullSafeFirstName + " " + nullSafeLastName).trim(); + + if (fullName.isEmpty()) { + return null; + } + return fullName; + } + + public Contact build() { + if (name == null) { + name = composedName(); + } + return new Contact(firstName, lastName, midInitials, name, role, email, affiliation, url); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Curation.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Curation.java similarity index 97% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/Curation.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/Curation.java index fed1f1103..e669c24fa 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Curation.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Curation.java @@ -1,332 +1,332 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.hash.Hasher; -import com.google.common.hash.Hashing; -import java.util.*; -import lombok.Data; - -@Data -public class Curation implements Comparable { - @JsonProperty("attributesPre") - private final SortedSet attributesPre; - - @JsonProperty("attributesPost") - private final SortedSet attributesPost; - - private final SortedSet externalPre; - private final SortedSet externalPost; - - @JsonProperty("relationshipsPre") - private final SortedSet relationshipsPre; - - @JsonProperty("relationshipsPost") - private final SortedSet relationshipsPost; - - private final String hash; - - private Curation( - final Collection attributesPre, - final Collection attributesPost, - final Collection externalPre, - final Collection externalPost, - final Collection relationshipsPre, - final Collection relationshipsPost, - final String hash) { - this.attributesPre = Collections.unmodifiableSortedSet(new TreeSet<>(attributesPre)); - this.attributesPost = Collections.unmodifiableSortedSet(new TreeSet<>(attributesPost)); - this.externalPre = Collections.unmodifiableSortedSet(new TreeSet<>(externalPre)); - this.externalPost = Collections.unmodifiableSortedSet(new TreeSet<>(externalPost)); - this.relationshipsPre = - relationshipsPre != null - ? Collections.unmodifiableSortedSet(new TreeSet<>(relationshipsPre)) - : Collections.unmodifiableSortedSet(new TreeSet<>()); - this.relationshipsPost = - relationshipsPost != null - ? Collections.unmodifiableSortedSet(new TreeSet<>(relationshipsPost)) - : Collections.unmodifiableSortedSet(new TreeSet<>()); - this.hash = hash; - } - - @JsonProperty("externalReferencesPre") - public SortedSet getExternalReferencesPre() { - return externalPre; - } - - @JsonProperty("externalReferencesPost") - public SortedSet getExternalReferencesPost() { - return externalPost; - } - - @Override - public int compareTo(final Curation other) { - if (other == null) { - return 1; - } - - if (!attributesPre.equals(other.attributesPre)) { - if (attributesPre.size() < other.attributesPre.size()) { - return -1; - } else if (attributesPre.size() > other.attributesPre.size()) { - return 1; - } else { - final Iterator thisIt = attributesPre.iterator(); - final Iterator otherIt = other.attributesPre.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - if (!attributesPost.equals(other.attributesPost)) { - if (attributesPost.size() < other.attributesPost.size()) { - return -1; - } else if (attributesPost.size() > other.attributesPost.size()) { - return 1; - } else { - final Iterator thisIt = attributesPost.iterator(); - final Iterator otherIt = other.attributesPost.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - - if (!externalPre.equals(other.externalPre)) { - if (externalPre.size() < other.externalPre.size()) { - return -1; - } else if (externalPre.size() > other.externalPre.size()) { - return 1; - } else { - final Iterator thisIt = externalPre.iterator(); - final Iterator otherIt = other.externalPre.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - if (!externalPost.equals(other.externalPost)) { - if (externalPost.size() < other.externalPost.size()) { - return -1; - } else if (externalPost.size() > other.externalPost.size()) { - return 1; - } else { - final Iterator thisIt = externalPost.iterator(); - final Iterator otherIt = other.externalPost.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - - if (!relationshipsPre.equals(other.relationshipsPre)) { - if (relationshipsPre.size() < other.relationshipsPre.size()) { - return -1; - } else if (relationshipsPre.size() > other.relationshipsPre.size()) { - return 1; - } else { - final Iterator thisIt = relationshipsPre.iterator(); - final Iterator otherIt = other.relationshipsPre.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - if (!relationshipsPost.equals(other.relationshipsPost)) { - if (relationshipsPost.size() < other.relationshipsPost.size()) { - return -1; - } else if (relationshipsPost.size() > other.relationshipsPost.size()) { - return 1; - } else { - final Iterator thisIt = relationshipsPost.iterator(); - final Iterator otherIt = other.relationshipsPost.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - return 0; - } - - @JsonCreator - public static Curation build( - @JsonProperty("attributesPre") final Collection attributesPre, - @JsonProperty("attributesPost") final Collection attributesPost, - @JsonProperty("externalReferencesPre") final Collection externalPre, - @JsonProperty("externalReferencesPost") final Collection externalPost, - @JsonProperty("relationshipsPre") final Collection relationshipsPre, - @JsonProperty("relationshipsPost") final Collection relationshipsPost) { - - SortedSet sortedPreAttributes = new TreeSet<>(); - SortedSet sortedPostAttributes = new TreeSet<>(); - SortedSet sortedPreExternal = new TreeSet<>(); - SortedSet sortedPostExternal = new TreeSet<>(); - SortedSet sortedPreRelationships = new TreeSet<>(); - SortedSet sortedPostRelationships = new TreeSet<>(); - - if (attributesPre != null) { - sortedPreAttributes.addAll(attributesPre); - } - if (attributesPost != null) { - sortedPostAttributes.addAll(attributesPost); - } - if (externalPre != null) { - sortedPreExternal.addAll(externalPre); - } - if (externalPost != null) { - sortedPostExternal.addAll(externalPost); - } - if (relationshipsPre != null) { - sortedPreRelationships.addAll(relationshipsPre); - } - if (relationshipsPost != null) { - sortedPostRelationships.addAll(relationshipsPost); - } - - sortedPreAttributes = Collections.unmodifiableSortedSet(sortedPreAttributes); - sortedPostAttributes = Collections.unmodifiableSortedSet(sortedPostAttributes); - sortedPreExternal = Collections.unmodifiableSortedSet(sortedPreExternal); - sortedPostExternal = Collections.unmodifiableSortedSet(sortedPostExternal); - sortedPreRelationships = Collections.unmodifiableSortedSet(sortedPreRelationships); - sortedPostRelationships = Collections.unmodifiableSortedSet(sortedPostRelationships); - - final Hasher hasher = Hashing.sha256().newHasher(); - - for (final Attribute a : sortedPreAttributes) { - hasher.putUnencodedChars(a.getType()); - hasher.putUnencodedChars(a.getValue()); - - /*Consider tag if present*/ - if (a.getTag() != null) { - hasher.putUnencodedChars(a.getTag()); - } - if (a.getUnit() != null) { - hasher.putUnencodedChars(a.getUnit()); - } - for (final String iri : a.getIri()) { - hasher.putUnencodedChars(iri); - } - } - - for (final Attribute a : sortedPostAttributes) { - hasher.putUnencodedChars(a.getType()); - hasher.putUnencodedChars(a.getValue()); - /*Consider tag if present*/ - if (a.getTag() != null) { - hasher.putUnencodedChars(a.getTag()); - } - if (a.getUnit() != null) { - hasher.putUnencodedChars(a.getUnit()); - } - for (final String iri : a.getIri()) { - hasher.putUnencodedChars(iri); - } - } - - for (final ExternalReference a : sortedPreExternal) { - hasher.putUnencodedChars(a.getUrl()); - for (final String s : a.getDuo()) { - hasher.putUnencodedChars(s); - } - } - - for (final ExternalReference a : sortedPostExternal) { - hasher.putUnencodedChars(a.getUrl()); - for (final String s : a.getDuo()) { - hasher.putUnencodedChars(s); - } - } - - for (final Relationship a : sortedPreRelationships) { - hasher.putUnencodedChars(a.getSource()); - hasher.putUnencodedChars(a.getTarget()); - hasher.putUnencodedChars(a.getType()); - } - - for (final Relationship a : sortedPostRelationships) { - hasher.putUnencodedChars(a.getSource()); - hasher.putUnencodedChars(a.getTarget()); - hasher.putUnencodedChars(a.getType()); - } - - final String hash = hasher.hash().toString(); - - return new Curation( - sortedPreAttributes, - sortedPostAttributes, - sortedPreExternal, - sortedPostExternal, - sortedPreRelationships, - sortedPostRelationships, - hash); - } - - // @JsonCreator - // public static Curation build(@JsonProperty("attributesPre") Collection - // attributesPre, - // @JsonProperty("attributesPost") Collection attributesPost, - // @JsonProperty("externalReferencesPre") Collection externalPre, - // @JsonProperty("externalReferencesPost") Collection externalPost) { - // - // return build(attributesPre, attributesPost, externalPre, externalPost, null, null); - // } - - public static Curation build( - final Collection attributesPre, - final Collection attributesPost, - final Collection externalPre, - final Collection externalPost) { - - return build(attributesPre, attributesPost, externalPre, externalPost, null, null); - } - - public static Curation build( - final Collection attributesPre, final Collection attributesPost) { - return build(attributesPre, attributesPost, null, null); - } - - public static Curation build(final Attribute attributePre, final Attribute attributePost) { - - if (attributePre == null && attributePost == null) { - throw new IllegalArgumentException("Must specify pre and/or post attribute"); - } else if (attributePre == null) { - // insertion curation - return build(null, Collections.singleton(attributePost), null, null); - } else if (attributePost == null) { - // deletion curation - return build(Collections.singleton(attributePre), null, null, null); - } else { - // one-to-one edit curation - return build( - Collections.singleton(attributePre), Collections.singleton(attributePost), null, null); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import java.util.*; +import lombok.Data; + +@Data +public class Curation implements Comparable { + @JsonProperty("attributesPre") + private final SortedSet attributesPre; + + @JsonProperty("attributesPost") + private final SortedSet attributesPost; + + private final SortedSet externalPre; + private final SortedSet externalPost; + + @JsonProperty("relationshipsPre") + private final SortedSet relationshipsPre; + + @JsonProperty("relationshipsPost") + private final SortedSet relationshipsPost; + + private final String hash; + + private Curation( + final Collection attributesPre, + final Collection attributesPost, + final Collection externalPre, + final Collection externalPost, + final Collection relationshipsPre, + final Collection relationshipsPost, + final String hash) { + this.attributesPre = Collections.unmodifiableSortedSet(new TreeSet<>(attributesPre)); + this.attributesPost = Collections.unmodifiableSortedSet(new TreeSet<>(attributesPost)); + this.externalPre = Collections.unmodifiableSortedSet(new TreeSet<>(externalPre)); + this.externalPost = Collections.unmodifiableSortedSet(new TreeSet<>(externalPost)); + this.relationshipsPre = + relationshipsPre != null + ? Collections.unmodifiableSortedSet(new TreeSet<>(relationshipsPre)) + : Collections.unmodifiableSortedSet(new TreeSet<>()); + this.relationshipsPost = + relationshipsPost != null + ? Collections.unmodifiableSortedSet(new TreeSet<>(relationshipsPost)) + : Collections.unmodifiableSortedSet(new TreeSet<>()); + this.hash = hash; + } + + @JsonProperty("externalReferencesPre") + public SortedSet getExternalReferencesPre() { + return externalPre; + } + + @JsonProperty("externalReferencesPost") + public SortedSet getExternalReferencesPost() { + return externalPost; + } + + @Override + public int compareTo(final Curation other) { + if (other == null) { + return 1; + } + + if (!attributesPre.equals(other.attributesPre)) { + if (attributesPre.size() < other.attributesPre.size()) { + return -1; + } else if (attributesPre.size() > other.attributesPre.size()) { + return 1; + } else { + final Iterator thisIt = attributesPre.iterator(); + final Iterator otherIt = other.attributesPre.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + if (!attributesPost.equals(other.attributesPost)) { + if (attributesPost.size() < other.attributesPost.size()) { + return -1; + } else if (attributesPost.size() > other.attributesPost.size()) { + return 1; + } else { + final Iterator thisIt = attributesPost.iterator(); + final Iterator otherIt = other.attributesPost.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + + if (!externalPre.equals(other.externalPre)) { + if (externalPre.size() < other.externalPre.size()) { + return -1; + } else if (externalPre.size() > other.externalPre.size()) { + return 1; + } else { + final Iterator thisIt = externalPre.iterator(); + final Iterator otherIt = other.externalPre.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + if (!externalPost.equals(other.externalPost)) { + if (externalPost.size() < other.externalPost.size()) { + return -1; + } else if (externalPost.size() > other.externalPost.size()) { + return 1; + } else { + final Iterator thisIt = externalPost.iterator(); + final Iterator otherIt = other.externalPost.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + + if (!relationshipsPre.equals(other.relationshipsPre)) { + if (relationshipsPre.size() < other.relationshipsPre.size()) { + return -1; + } else if (relationshipsPre.size() > other.relationshipsPre.size()) { + return 1; + } else { + final Iterator thisIt = relationshipsPre.iterator(); + final Iterator otherIt = other.relationshipsPre.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + if (!relationshipsPost.equals(other.relationshipsPost)) { + if (relationshipsPost.size() < other.relationshipsPost.size()) { + return -1; + } else if (relationshipsPost.size() > other.relationshipsPost.size()) { + return 1; + } else { + final Iterator thisIt = relationshipsPost.iterator(); + final Iterator otherIt = other.relationshipsPost.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + return 0; + } + + @JsonCreator + public static Curation build( + @JsonProperty("attributesPre") final Collection attributesPre, + @JsonProperty("attributesPost") final Collection attributesPost, + @JsonProperty("externalReferencesPre") final Collection externalPre, + @JsonProperty("externalReferencesPost") final Collection externalPost, + @JsonProperty("relationshipsPre") final Collection relationshipsPre, + @JsonProperty("relationshipsPost") final Collection relationshipsPost) { + + SortedSet sortedPreAttributes = new TreeSet<>(); + SortedSet sortedPostAttributes = new TreeSet<>(); + SortedSet sortedPreExternal = new TreeSet<>(); + SortedSet sortedPostExternal = new TreeSet<>(); + SortedSet sortedPreRelationships = new TreeSet<>(); + SortedSet sortedPostRelationships = new TreeSet<>(); + + if (attributesPre != null) { + sortedPreAttributes.addAll(attributesPre); + } + if (attributesPost != null) { + sortedPostAttributes.addAll(attributesPost); + } + if (externalPre != null) { + sortedPreExternal.addAll(externalPre); + } + if (externalPost != null) { + sortedPostExternal.addAll(externalPost); + } + if (relationshipsPre != null) { + sortedPreRelationships.addAll(relationshipsPre); + } + if (relationshipsPost != null) { + sortedPostRelationships.addAll(relationshipsPost); + } + + sortedPreAttributes = Collections.unmodifiableSortedSet(sortedPreAttributes); + sortedPostAttributes = Collections.unmodifiableSortedSet(sortedPostAttributes); + sortedPreExternal = Collections.unmodifiableSortedSet(sortedPreExternal); + sortedPostExternal = Collections.unmodifiableSortedSet(sortedPostExternal); + sortedPreRelationships = Collections.unmodifiableSortedSet(sortedPreRelationships); + sortedPostRelationships = Collections.unmodifiableSortedSet(sortedPostRelationships); + + final Hasher hasher = Hashing.sha256().newHasher(); + + for (final Attribute a : sortedPreAttributes) { + hasher.putUnencodedChars(a.getType()); + hasher.putUnencodedChars(a.getValue()); + + /*Consider tag if present*/ + if (a.getTag() != null) { + hasher.putUnencodedChars(a.getTag()); + } + if (a.getUnit() != null) { + hasher.putUnencodedChars(a.getUnit()); + } + for (final String iri : a.getIri()) { + hasher.putUnencodedChars(iri); + } + } + + for (final Attribute a : sortedPostAttributes) { + hasher.putUnencodedChars(a.getType()); + hasher.putUnencodedChars(a.getValue()); + /*Consider tag if present*/ + if (a.getTag() != null) { + hasher.putUnencodedChars(a.getTag()); + } + if (a.getUnit() != null) { + hasher.putUnencodedChars(a.getUnit()); + } + for (final String iri : a.getIri()) { + hasher.putUnencodedChars(iri); + } + } + + for (final ExternalReference a : sortedPreExternal) { + hasher.putUnencodedChars(a.getUrl()); + for (final String s : a.getDuo()) { + hasher.putUnencodedChars(s); + } + } + + for (final ExternalReference a : sortedPostExternal) { + hasher.putUnencodedChars(a.getUrl()); + for (final String s : a.getDuo()) { + hasher.putUnencodedChars(s); + } + } + + for (final Relationship a : sortedPreRelationships) { + hasher.putUnencodedChars(a.getSource()); + hasher.putUnencodedChars(a.getTarget()); + hasher.putUnencodedChars(a.getType()); + } + + for (final Relationship a : sortedPostRelationships) { + hasher.putUnencodedChars(a.getSource()); + hasher.putUnencodedChars(a.getTarget()); + hasher.putUnencodedChars(a.getType()); + } + + final String hash = hasher.hash().toString(); + + return new Curation( + sortedPreAttributes, + sortedPostAttributes, + sortedPreExternal, + sortedPostExternal, + sortedPreRelationships, + sortedPostRelationships, + hash); + } + + // @JsonCreator + // public static Curation build(@JsonProperty("attributesPre") Collection + // attributesPre, + // @JsonProperty("attributesPost") Collection attributesPost, + // @JsonProperty("externalReferencesPre") Collection externalPre, + // @JsonProperty("externalReferencesPost") Collection externalPost) { + // + // return build(attributesPre, attributesPost, externalPre, externalPost, null, null); + // } + + public static Curation build( + final Collection attributesPre, + final Collection attributesPost, + final Collection externalPre, + final Collection externalPost) { + + return build(attributesPre, attributesPost, externalPre, externalPost, null, null); + } + + public static Curation build( + final Collection attributesPre, final Collection attributesPost) { + return build(attributesPre, attributesPost, null, null); + } + + public static Curation build(final Attribute attributePre, final Attribute attributePost) { + + if (attributePre == null && attributePost == null) { + throw new IllegalArgumentException("Must specify pre and/or post attribute"); + } else if (attributePre == null) { + // insertion curation + return build(null, Collections.singleton(attributePost), null, null); + } else if (attributePost == null) { + // deletion curation + return build(Collections.singleton(attributePre), null, null, null); + } else { + // one-to-one edit curation + return build( + Collections.singleton(attributePre), Collections.singleton(attributePost), null, null); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/CurationLink.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/CurationLink.java similarity index 92% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/CurationLink.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/CurationLink.java index 589301682..3607ce5bb 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/CurationLink.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/CurationLink.java @@ -1,93 +1,93 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.google.common.hash.Hashing; -import java.time.Instant; -import lombok.Data; -import uk.ac.ebi.biosamples.service.CustomInstantDeserializer; -import uk.ac.ebi.biosamples.service.CustomInstantSerializer; - -@Data -public class CurationLink implements Comparable { - private final Curation curation; - private final String sample; - private final String webinSubmissionAccountId; - private final String hash; - private String domain; - - @JsonSerialize(using = CustomInstantSerializer.class) - protected final Instant created; - - private CurationLink( - final String sample, - final String domain, - final String webinSubmissionAccountId, - final Curation curation, - final String hash, - final Instant created) { - this.sample = sample; - this.domain = domain; - this.webinSubmissionAccountId = webinSubmissionAccountId; - this.curation = curation; - this.hash = hash; - this.created = created; - } - - @Override - public int compareTo(final CurationLink other) { - if (other == null) { - return 1; - } - - if (!domain.equals(other.domain)) { - return domain.compareTo(other.domain); - } - - if (!webinSubmissionAccountId.equals(other.webinSubmissionAccountId)) { - return webinSubmissionAccountId.compareTo(other.webinSubmissionAccountId); - } - - if (!sample.equals(other.sample)) { - return sample.compareTo(other.sample); - } - - if (!curation.equals(other.curation)) { - return curation.compareTo(other.curation); - } - - return 0; - } - - // Used for deserialization (JSON -> Java) - @JsonCreator - public static CurationLink build( - @JsonProperty("sample") final String sample, - @JsonProperty("curation") final Curation curation, - @JsonProperty("domain") final String domain, - @JsonProperty("webinSubmissionAccountId") final String webinSubmissionAccountId, - @JsonProperty("created") @JsonDeserialize(using = CustomInstantDeserializer.class) - final Instant created) { - final String hash = - Hashing.sha256() - .newHasher() - .putUnencodedChars(curation.getHash()) - .putUnencodedChars(sample) - .hash() - .toString(); - - return new CurationLink(sample, domain, webinSubmissionAccountId, curation, hash, created); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.hash.Hashing; +import java.time.Instant; +import lombok.Data; +import uk.ac.ebi.biosamples.core.service.CustomInstantDeserializer; +import uk.ac.ebi.biosamples.core.service.CustomInstantSerializer; + +@Data +public class CurationLink implements Comparable { + private final Curation curation; + private final String sample; + private final String webinSubmissionAccountId; + private final String hash; + private String domain; + + @JsonSerialize(using = CustomInstantSerializer.class) + protected final Instant created; + + private CurationLink( + final String sample, + final String domain, + final String webinSubmissionAccountId, + final Curation curation, + final String hash, + final Instant created) { + this.sample = sample; + this.domain = domain; + this.webinSubmissionAccountId = webinSubmissionAccountId; + this.curation = curation; + this.hash = hash; + this.created = created; + } + + @Override + public int compareTo(final CurationLink other) { + if (other == null) { + return 1; + } + + if (!domain.equals(other.domain)) { + return domain.compareTo(other.domain); + } + + if (!webinSubmissionAccountId.equals(other.webinSubmissionAccountId)) { + return webinSubmissionAccountId.compareTo(other.webinSubmissionAccountId); + } + + if (!sample.equals(other.sample)) { + return sample.compareTo(other.sample); + } + + if (!curation.equals(other.curation)) { + return curation.compareTo(other.curation); + } + + return 0; + } + + // Used for deserialization (JSON -> Java) + @JsonCreator + public static CurationLink build( + @JsonProperty("sample") final String sample, + @JsonProperty("curation") final Curation curation, + @JsonProperty("domain") final String domain, + @JsonProperty("webinSubmissionAccountId") final String webinSubmissionAccountId, + @JsonProperty("created") @JsonDeserialize(using = CustomInstantDeserializer.class) + final Instant created) { + final String hash = + Hashing.sha256() + .newHasher() + .putUnencodedChars(curation.getHash()) + .putUnencodedChars(sample) + .hash() + .toString(); + + return new CurationLink(sample, domain, webinSubmissionAccountId, curation, hash, created); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/CurationRule.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/CurationRule.java similarity index 95% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/CurationRule.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/CurationRule.java index 38b892efd..1b017b405 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/CurationRule.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/CurationRule.java @@ -1,74 +1,74 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Objects; -import lombok.Getter; - -@Getter -public class CurationRule implements Comparable { - private final String attributePre; - private final String attributePost; - - private CurationRule(final String attributePre, final String attributePost) { - this.attributePre = attributePre; - this.attributePost = attributePost; - } - - @Override - public boolean equals(final Object o) { - if (o == this) { - return true; - } - if (!(o instanceof CurationRule)) { - return false; - } - final CurationRule other = (CurationRule) o; - return Objects.equals( - attributePre, other.attributePre); // Attribute pre should be unique to pick a rule - } - - @Override - public int hashCode() { - return Objects.hash(attributePre); - } - - @Override - public int compareTo(final CurationRule other) { - if (other == null) { - return 1; - } else if (attributePre.equals(other.attributePre)) { - return 0; - } else { - return attributePre.compareTo(other.attributePre); - } - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - sb.append("CurationRule("); - sb.append(attributePre); - sb.append(","); - sb.append(attributePost); - sb.append(")"); - return sb.toString(); - } - - @JsonCreator - public static CurationRule build( - @JsonProperty("attributePre") final String attributePre, - @JsonProperty("attributePost") final String attributePost) { - return new CurationRule(attributePre, attributePost); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Objects; +import lombok.Getter; + +@Getter +public class CurationRule implements Comparable { + private final String attributePre; + private final String attributePost; + + private CurationRule(final String attributePre, final String attributePost) { + this.attributePre = attributePre; + this.attributePost = attributePost; + } + + @Override + public boolean equals(final Object o) { + if (o == this) { + return true; + } + if (!(o instanceof CurationRule)) { + return false; + } + final CurationRule other = (CurationRule) o; + return Objects.equals( + attributePre, other.attributePre); // Attribute pre should be unique to pick a rule + } + + @Override + public int hashCode() { + return Objects.hash(attributePre); + } + + @Override + public int compareTo(final CurationRule other) { + if (other == null) { + return 1; + } else if (attributePre.equals(other.attributePre)) { + return 0; + } else { + return attributePre.compareTo(other.attributePre); + } + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("CurationRule("); + sb.append(attributePre); + sb.append(","); + sb.append(attributePost); + sb.append(")"); + return sb.toString(); + } + + @JsonCreator + public static CurationRule build( + @JsonProperty("attributePre") final String attributePre, + @JsonProperty("attributePost") final String attributePost) { + return new CurationRule(attributePre, attributePost); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/ExternalReference.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/ExternalReference.java similarity index 96% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/ExternalReference.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/ExternalReference.java index 16307cf60..4985f9c58 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/ExternalReference.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/ExternalReference.java @@ -1,116 +1,116 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.hash.Hasher; -import com.google.common.hash.Hashing; -import java.util.*; -import lombok.Data; -import org.springframework.web.util.UriComponents; -import org.springframework.web.util.UriComponentsBuilder; - -@Data -@JsonInclude(JsonInclude.Include.NON_NULL) -public class ExternalReference implements Comparable { - private final String url; - @JsonIgnore private final String hash; - private final SortedSet duo; - - private ExternalReference(final String url, final String hash, final SortedSet duo) { - this.url = url; - this.hash = hash; - this.duo = duo; - } - - @Override - public int compareTo(final ExternalReference other) { - if (other == null) { - return 1; - } - - if (!url.equals(other.url)) { - return url.compareTo(other.url); - } - - if (duo == other.duo) { - return 0; - } else if (other.duo == null) { - return 1; - } else if (duo == null) { - return -1; - } - - if (!duo.equals(other.duo)) { - if (duo.size() < other.duo.size()) { - return -1; - } else if (duo.size() > other.duo.size()) { - return 1; - } else { - final Iterator thisIt = duo.iterator(); - final Iterator otherIt = other.duo.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - return 0; - } - - public static ExternalReference build(String url, SortedSet duo) { - final UriComponentsBuilder uriComponentsBuilder = UriComponentsBuilder.fromUriString(url); - final UriComponents uriComponents = uriComponentsBuilder.build().normalize(); - - url = uriComponents.toUriString(); - - final Hasher hasher = - Hashing.sha256() - .newHasher() - .putUnencodedChars( - Objects.nonNull(uriComponents.getScheme()) ? uriComponents.getScheme() : "") - .putUnencodedChars( - Objects.nonNull(uriComponents.getSchemeSpecificPart()) - ? uriComponents.getSchemeSpecificPart() - : "") - .putUnencodedChars( - Objects.nonNull(uriComponents.getUserInfo()) ? uriComponents.getUserInfo() : "") - .putUnencodedChars( - Objects.nonNull(uriComponents.getHost()) ? uriComponents.getHost() : "") - .putInt(Objects.nonNull(uriComponents.getPort()) ? uriComponents.getPort() : 0) - .putUnencodedChars( - Objects.nonNull(uriComponents.getPath()) ? uriComponents.getPath() : "") - .putUnencodedChars( - Objects.nonNull(uriComponents.getQuery()) ? uriComponents.getQuery() : "") - .putUnencodedChars( - Objects.nonNull(uriComponents.getFragment()) ? uriComponents.getFragment() : ""); - - if (duo != null) { - for (final String s : duo) { - hasher.putUnencodedChars(s); - } - } else { - duo = Collections.emptySortedSet(); - } - - return new ExternalReference(url, hasher.hash().toString(), duo); - } - - @JsonCreator - public static ExternalReference build(@JsonProperty("url") final String url) { - return build(url, new TreeSet<>()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import java.util.*; +import lombok.Data; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ExternalReference implements Comparable { + private final String url; + @JsonIgnore private final String hash; + private final SortedSet duo; + + private ExternalReference(final String url, final String hash, final SortedSet duo) { + this.url = url; + this.hash = hash; + this.duo = duo; + } + + @Override + public int compareTo(final ExternalReference other) { + if (other == null) { + return 1; + } + + if (!url.equals(other.url)) { + return url.compareTo(other.url); + } + + if (duo == other.duo) { + return 0; + } else if (other.duo == null) { + return 1; + } else if (duo == null) { + return -1; + } + + if (!duo.equals(other.duo)) { + if (duo.size() < other.duo.size()) { + return -1; + } else if (duo.size() > other.duo.size()) { + return 1; + } else { + final Iterator thisIt = duo.iterator(); + final Iterator otherIt = other.duo.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + return 0; + } + + public static ExternalReference build(String url, SortedSet duo) { + final UriComponentsBuilder uriComponentsBuilder = UriComponentsBuilder.fromUriString(url); + final UriComponents uriComponents = uriComponentsBuilder.build().normalize(); + + url = uriComponents.toUriString(); + + final Hasher hasher = + Hashing.sha256() + .newHasher() + .putUnencodedChars( + Objects.nonNull(uriComponents.getScheme()) ? uriComponents.getScheme() : "") + .putUnencodedChars( + Objects.nonNull(uriComponents.getSchemeSpecificPart()) + ? uriComponents.getSchemeSpecificPart() + : "") + .putUnencodedChars( + Objects.nonNull(uriComponents.getUserInfo()) ? uriComponents.getUserInfo() : "") + .putUnencodedChars( + Objects.nonNull(uriComponents.getHost()) ? uriComponents.getHost() : "") + .putInt(Objects.nonNull(uriComponents.getPort()) ? uriComponents.getPort() : 0) + .putUnencodedChars( + Objects.nonNull(uriComponents.getPath()) ? uriComponents.getPath() : "") + .putUnencodedChars( + Objects.nonNull(uriComponents.getQuery()) ? uriComponents.getQuery() : "") + .putUnencodedChars( + Objects.nonNull(uriComponents.getFragment()) ? uriComponents.getFragment() : ""); + + if (duo != null) { + for (final String s : duo) { + hasher.putUnencodedChars(s); + } + } else { + duo = Collections.emptySortedSet(); + } + + return new ExternalReference(url, hasher.hash().toString(), duo); + } + + @JsonCreator + public static ExternalReference build(@JsonProperty("url") final String url) { + return build(url, new TreeSet<>()); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Organization.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Organization.java similarity index 95% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/Organization.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/Organization.java index aba65b151..3f944179e 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Organization.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Organization.java @@ -1,211 +1,211 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import java.util.Objects; -import org.springframework.util.StringUtils; - -@JsonInclude(JsonInclude.Include.NON_EMPTY) -@JsonDeserialize(builder = Organization.Builder.class) -public class Organization implements Comparable { - private final String name; - private final String role; - private final String address; - private final String email; - private final String url; - - private Organization( - final String name, - final String role, - final String email, - final String url, - final String address) { - this.name = name; - this.role = role; - this.email = email; - this.url = url; - this.address = address; - } - - @JsonProperty("Name") - public String getName() { - return name; - } - - @JsonProperty("Role") - public String getRole() { - return role; - } - - @JsonProperty("E-mail") - public String getEmail() { - return email; - } - - @JsonProperty("URL") - public String getUrl() { - return url; - } - - @JsonProperty("Address") - public String getAddress() { - return address; - } - - @Override - public boolean equals(final Object o) { - - if (o == this) { - return true; - } - if (!(o instanceof Organization)) { - return false; - } - final Organization other = (Organization) o; - return Objects.equals(name, other.name) - && Objects.equals(role, other.role) - && Objects.equals(email, other.email) - && Objects.equals(url, other.url) - && Objects.equals(address, other.address); - } - - @Override - public int hashCode() { - return Objects.hash(name, role, email, url, address); - } - - @Override - public int compareTo(final Organization other) { - if (other == null) { - return 1; - } - - int comparisonResult = nullSafeStringComparison(name, other.name); - if (comparisonResult != 0) { - return comparisonResult; - } - - comparisonResult = nullSafeStringComparison(address, other.address); - if (comparisonResult != 0) { - return comparisonResult; - } - - comparisonResult = nullSafeStringComparison(role, other.role); - if (comparisonResult != 0) { - return comparisonResult; - } - - comparisonResult = nullSafeStringComparison(email, other.email); - if (comparisonResult != 0) { - return comparisonResult; - } - - return nullSafeStringComparison(url, other.url); - } - - @Override - public String toString() { - return "Organization{" - + "name='" - + name - + '\'' - + ", role='" - + role - + '\'' - + ", address='" - + address - + '\'' - + ", email='" - + email - + '\'' - + ", url='" - + url - + '\'' - + '}'; - } - - private int nullSafeStringComparison(final String first, final String other) { - if (first == null && other == null) { - return 0; - } - if (first == null) { - return -1; - } - if (other == null) { - return 1; - } - return first.compareTo(other); - } - - // @JsonCreator - // public static Organization build(@JsonProperty("Name") String name, - // @JsonProperty("Role") String role, - // @JsonProperty("E-mail") String email, - // @JsonProperty("URL") String url) { - // return new Organization(name, role,email,url); - // } - public static class Builder { - private String name; - private String url; - private String email; - private String address; - private String role; - - @JsonCreator - public Builder() {} - - @JsonProperty("Name") - public Builder name(final String name) { - this.name = name; - return this; - } - - @JsonProperty("URL") - public Builder url(final String url) { - this.url = url; - return this; - } - - @JsonProperty("E-mail") - public Builder email(final String email) { - this.email = email; - return this; - } - - @JsonProperty("Address") - public Builder address(final String address) { - this.address = address; - return this; - } - - @JsonProperty("Role") - public Builder role(final String role) { - this.role = role; - return this; - } - - public boolean isNotEmpty() { - // only check fields that could be meaningful alone - return StringUtils.hasText(name) - || StringUtils.hasText(url) - || StringUtils.hasText(email) - || StringUtils.hasText(address); - } - - public Organization build() { - return new Organization(name, role, email, url, address); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.util.Objects; +import org.springframework.util.StringUtils; + +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@JsonDeserialize(builder = Organization.Builder.class) +public class Organization implements Comparable { + private final String name; + private final String role; + private final String address; + private final String email; + private final String url; + + private Organization( + final String name, + final String role, + final String email, + final String url, + final String address) { + this.name = name; + this.role = role; + this.email = email; + this.url = url; + this.address = address; + } + + @JsonProperty("Name") + public String getName() { + return name; + } + + @JsonProperty("Role") + public String getRole() { + return role; + } + + @JsonProperty("E-mail") + public String getEmail() { + return email; + } + + @JsonProperty("URL") + public String getUrl() { + return url; + } + + @JsonProperty("Address") + public String getAddress() { + return address; + } + + @Override + public boolean equals(final Object o) { + + if (o == this) { + return true; + } + if (!(o instanceof Organization)) { + return false; + } + final Organization other = (Organization) o; + return Objects.equals(name, other.name) + && Objects.equals(role, other.role) + && Objects.equals(email, other.email) + && Objects.equals(url, other.url) + && Objects.equals(address, other.address); + } + + @Override + public int hashCode() { + return Objects.hash(name, role, email, url, address); + } + + @Override + public int compareTo(final Organization other) { + if (other == null) { + return 1; + } + + int comparisonResult = nullSafeStringComparison(name, other.name); + if (comparisonResult != 0) { + return comparisonResult; + } + + comparisonResult = nullSafeStringComparison(address, other.address); + if (comparisonResult != 0) { + return comparisonResult; + } + + comparisonResult = nullSafeStringComparison(role, other.role); + if (comparisonResult != 0) { + return comparisonResult; + } + + comparisonResult = nullSafeStringComparison(email, other.email); + if (comparisonResult != 0) { + return comparisonResult; + } + + return nullSafeStringComparison(url, other.url); + } + + @Override + public String toString() { + return "Organization{" + + "name='" + + name + + '\'' + + ", role='" + + role + + '\'' + + ", address='" + + address + + '\'' + + ", email='" + + email + + '\'' + + ", url='" + + url + + '\'' + + '}'; + } + + private int nullSafeStringComparison(final String first, final String other) { + if (first == null && other == null) { + return 0; + } + if (first == null) { + return -1; + } + if (other == null) { + return 1; + } + return first.compareTo(other); + } + + // @JsonCreator + // public static Organization build(@JsonProperty("Name") String name, + // @JsonProperty("Role") String role, + // @JsonProperty("E-mail") String email, + // @JsonProperty("URL") String url) { + // return new Organization(name, role,email,url); + // } + public static class Builder { + private String name; + private String url; + private String email; + private String address; + private String role; + + @JsonCreator + public Builder() {} + + @JsonProperty("Name") + public Builder name(final String name) { + this.name = name; + return this; + } + + @JsonProperty("URL") + public Builder url(final String url) { + this.url = url; + return this; + } + + @JsonProperty("E-mail") + public Builder email(final String email) { + this.email = email; + return this; + } + + @JsonProperty("Address") + public Builder address(final String address) { + this.address = address; + return this; + } + + @JsonProperty("Role") + public Builder role(final String role) { + this.role = role; + return this; + } + + public boolean isNotEmpty() { + // only check fields that could be meaningful alone + return StringUtils.hasText(name) + || StringUtils.hasText(url) + || StringUtils.hasText(email) + || StringUtils.hasText(address); + } + + public Organization build() { + return new Organization(name, role, email, url, address); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/PipelineAnalytics.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/PipelineAnalytics.java similarity index 92% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/PipelineAnalytics.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/PipelineAnalytics.java index afb5ba91f..db4a284d1 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/PipelineAnalytics.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/PipelineAnalytics.java @@ -1,83 +1,83 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import java.time.Instant; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Collection; -import lombok.Getter; -import uk.ac.ebi.biosamples.model.filter.DateRangeFilter; -import uk.ac.ebi.biosamples.model.filter.Filter; - -@Getter -public class PipelineAnalytics { - private String name; - private Instant startTime; - private Instant endTime; - private String dateRange; - private long processedRecords; - private long modifiedRecords; - - public PipelineAnalytics( - final String name, - final Instant startTime, - final Instant endTime, - final long processedRecords, - final long modifiedRecords) { - this.name = name; - this.startTime = startTime; - this.endTime = endTime; - this.processedRecords = processedRecords; - this.modifiedRecords = modifiedRecords; - } - - public void setStartTime(final Instant startTime) { - this.startTime = startTime; - } - - public void setEndTime(final Instant endTime) { - this.endTime = endTime; - } - - public void setDateRange(final String dateRange) { - this.dateRange = dateRange; - } - - public void setDateRange(final Collection filters) { - final DateRangeFilter dateRangeFilter = - filters.stream() - .filter(f -> f instanceof DateRangeFilter) - .map(DateRangeFilter.class::cast) - .findFirst() - .orElse(null); - if (dateRangeFilter != null && dateRangeFilter.getContent().isPresent()) { - final DateTimeFormatter dateTimeFormatter = - DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault()); - dateRange = - dateTimeFormatter.format(dateRangeFilter.getContent().get().getFrom()) - + " : " - + dateTimeFormatter.format(dateRangeFilter.getContent().get().getUntil()); - } - } - - public void setName(final String name) { - this.name = name; - } - - public void setProcessedRecords(final long processedRecords) { - this.processedRecords = processedRecords; - } - - public void setModifiedRecords(final long modifiedRecords) { - this.modifiedRecords = modifiedRecords; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Collection; +import lombok.Getter; +import uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; + +@Getter +public class PipelineAnalytics { + private String name; + private Instant startTime; + private Instant endTime; + private String dateRange; + private long processedRecords; + private long modifiedRecords; + + public PipelineAnalytics( + final String name, + final Instant startTime, + final Instant endTime, + final long processedRecords, + final long modifiedRecords) { + this.name = name; + this.startTime = startTime; + this.endTime = endTime; + this.processedRecords = processedRecords; + this.modifiedRecords = modifiedRecords; + } + + public void setStartTime(final Instant startTime) { + this.startTime = startTime; + } + + public void setEndTime(final Instant endTime) { + this.endTime = endTime; + } + + public void setDateRange(final String dateRange) { + this.dateRange = dateRange; + } + + public void setDateRange(final Collection filters) { + final DateRangeFilter dateRangeFilter = + filters.stream() + .filter(f -> f instanceof DateRangeFilter) + .map(DateRangeFilter.class::cast) + .findFirst() + .orElse(null); + if (dateRangeFilter != null && dateRangeFilter.getContent().isPresent()) { + final DateTimeFormatter dateTimeFormatter = + DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault()); + dateRange = + dateTimeFormatter.format(dateRangeFilter.getContent().get().getFrom()) + + " : " + + dateTimeFormatter.format(dateRangeFilter.getContent().get().getUntil()); + } + } + + public void setName(final String name) { + this.name = name; + } + + public void setProcessedRecords(final long processedRecords) { + this.processedRecords = processedRecords; + } + + public void setModifiedRecords(final long modifiedRecords) { + this.modifiedRecords = modifiedRecords; + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Publication.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Publication.java similarity index 95% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/Publication.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/Publication.java index 231e6d3ea..6e555c47f 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Publication.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Publication.java @@ -1,119 +1,119 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import java.util.Objects; - -@JsonInclude(JsonInclude.Include.NON_EMPTY) -@JsonDeserialize(builder = Publication.Builder.class) -public class Publication implements Comparable { - private final String doi; - private final String pubmed_id; - - private Publication(final String doi, final String pubmed_id) { - this.doi = doi; - this.pubmed_id = pubmed_id; - } - - @JsonProperty("doi") - public String getDoi() { - return doi; - } - - @JsonProperty("pubmed_id") - public String getPubMedId() { - return pubmed_id; - } - - @Override - public boolean equals(final Object o) { - - if (o == this) { - return true; - } - if (!(o instanceof Publication)) { - return false; - } - final Publication other = (Publication) o; - return Objects.equals(doi, other.doi) && Objects.equals(pubmed_id, other.pubmed_id); - } - - @Override - public int hashCode() { - return Objects.hash(doi, pubmed_id); - } - - @Override - public int compareTo(final Publication other) { - if (other == null) { - return 1; - } - - final int comparisonResult = nullSafeStringComparison(doi, other.doi); - if (comparisonResult != 0) { - return comparisonResult; - } - - return nullSafeStringComparison(pubmed_id, other.pubmed_id); - } - - @Override - public String toString() { - return "Publication{" + "doi='" + doi + '\'' + ", pubmed_id='" + pubmed_id + '\'' + '}'; - } - - private int nullSafeStringComparison(final String first, final String other) { - if (first == null && other == null) { - return 0; - } - if (first == null) { - return -1; - } - if (other == null) { - return 1; - } - return first.compareTo(other); - } - - // @JsonCreator - // public static Publication build(@JsonProperty("doi") String doi, - // @JsonProperty("pubmed_id") String pubmedId) { - // return new Publication(doi, pubmedId); - // } - - public static class Builder { - private String doi; - private String pubmed_id; - - @JsonCreator - public Builder() {} - - @JsonProperty("doi") - public Builder doi(final String doi) { - this.doi = doi; - return this; - } - - @JsonProperty("pubmed_id") - public Builder pubmed_id(final String pubmed_id) { - this.pubmed_id = pubmed_id; - return this; - } - - public Publication build() { - return new Publication(doi, pubmed_id); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.util.Objects; + +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@JsonDeserialize(builder = Publication.Builder.class) +public class Publication implements Comparable { + private final String doi; + private final String pubmed_id; + + private Publication(final String doi, final String pubmed_id) { + this.doi = doi; + this.pubmed_id = pubmed_id; + } + + @JsonProperty("doi") + public String getDoi() { + return doi; + } + + @JsonProperty("pubmed_id") + public String getPubMedId() { + return pubmed_id; + } + + @Override + public boolean equals(final Object o) { + + if (o == this) { + return true; + } + if (!(o instanceof Publication)) { + return false; + } + final Publication other = (Publication) o; + return Objects.equals(doi, other.doi) && Objects.equals(pubmed_id, other.pubmed_id); + } + + @Override + public int hashCode() { + return Objects.hash(doi, pubmed_id); + } + + @Override + public int compareTo(final Publication other) { + if (other == null) { + return 1; + } + + final int comparisonResult = nullSafeStringComparison(doi, other.doi); + if (comparisonResult != 0) { + return comparisonResult; + } + + return nullSafeStringComparison(pubmed_id, other.pubmed_id); + } + + @Override + public String toString() { + return "Publication{" + "doi='" + doi + '\'' + ", pubmed_id='" + pubmed_id + '\'' + '}'; + } + + private int nullSafeStringComparison(final String first, final String other) { + if (first == null && other == null) { + return 0; + } + if (first == null) { + return -1; + } + if (other == null) { + return 1; + } + return first.compareTo(other); + } + + // @JsonCreator + // public static Publication build(@JsonProperty("doi") String doi, + // @JsonProperty("pubmed_id") String pubmedId) { + // return new Publication(doi, pubmedId); + // } + + public static class Builder { + private String doi; + private String pubmed_id; + + @JsonCreator + public Builder() {} + + @JsonProperty("doi") + public Builder doi(final String doi) { + this.doi = doi; + return this; + } + + @JsonProperty("pubmed_id") + public Builder pubmed_id(final String pubmed_id) { + this.pubmed_id = pubmed_id; + return this; + } + + public Publication build() { + return new Publication(doi, pubmed_id); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Relationship.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Relationship.java similarity index 95% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/Relationship.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/Relationship.java index 1ad4c97b3..53eb2d2eb 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Relationship.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Relationship.java @@ -1,129 +1,129 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Objects; -import lombok.Getter; - -@Getter -public class Relationship implements Comparable { - private final String type; - private final String target; - private final String source; - - private Relationship(final String type, final String target, final String source) { - - this.type = type; - this.target = target; - this.source = source; - } - - @Override - public boolean equals(final Object o) { - - if (o == this) { - return true; - } - if (!(o instanceof Relationship)) { - return false; - } - final Relationship other = (Relationship) o; - return Objects.equals(type, other.type) - && Objects.equals(target, other.target) - && Objects.equals(source, other.source); - } - - @Override - public int hashCode() { - return Objects.hash(type, target, source); - } - - @Override - public int compareTo(final Relationship other) { - if (other == null) { - return 1; - } - if (!Objects.equals(type, other.type)) { - return type.compareTo(other.type); - } - - if (!Objects.equals(target, other.target)) { - return target.compareTo(other.target); - } - - if (!Objects.equals(source, other.source)) { - if (source == null) { - return 1; - } else if (other.source == null) { - return -1; - } else { - return source.compareTo(other.source); - } - } - return 0; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - sb.append("Relationships("); - sb.append(source); - sb.append(","); - sb.append(type); - sb.append(","); - sb.append(target); - sb.append(")"); - return sb.toString(); - } - - @JsonCreator - public static Relationship build( - @JsonProperty("source") final String source, - @JsonProperty("type") final String type, - @JsonProperty("target") final String target) { - if (type == null || type.trim().isEmpty()) { - throw new IllegalArgumentException("type cannot be empty"); - } - if (target == null || target.trim().isEmpty()) { - throw new IllegalArgumentException("target cannot be empty"); - } - return new Relationship(type, target, source); - } - - public static class Builder { - private String source = null; - private String target = null; - private String type = null; - - public Builder() {} - - public Builder withSource(final String source) { - this.source = source; - return this; - } - - public Builder withTarget(final String target) { - this.target = target; - return this; - } - - public Builder withType(final String type) { - this.type = type; - return this; - } - - public Relationship build() { - return Relationship.build(source, type, target); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Objects; +import lombok.Getter; + +@Getter +public class Relationship implements Comparable { + private final String type; + private final String target; + private final String source; + + private Relationship(final String type, final String target, final String source) { + + this.type = type; + this.target = target; + this.source = source; + } + + @Override + public boolean equals(final Object o) { + + if (o == this) { + return true; + } + if (!(o instanceof Relationship)) { + return false; + } + final Relationship other = (Relationship) o; + return Objects.equals(type, other.type) + && Objects.equals(target, other.target) + && Objects.equals(source, other.source); + } + + @Override + public int hashCode() { + return Objects.hash(type, target, source); + } + + @Override + public int compareTo(final Relationship other) { + if (other == null) { + return 1; + } + if (!Objects.equals(type, other.type)) { + return type.compareTo(other.type); + } + + if (!Objects.equals(target, other.target)) { + return target.compareTo(other.target); + } + + if (!Objects.equals(source, other.source)) { + if (source == null) { + return 1; + } else if (other.source == null) { + return -1; + } else { + return source.compareTo(other.source); + } + } + return 0; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("Relationships("); + sb.append(source); + sb.append(","); + sb.append(type); + sb.append(","); + sb.append(target); + sb.append(")"); + return sb.toString(); + } + + @JsonCreator + public static Relationship build( + @JsonProperty("source") final String source, + @JsonProperty("type") final String type, + @JsonProperty("target") final String target) { + if (type == null || type.trim().isEmpty()) { + throw new IllegalArgumentException("type cannot be empty"); + } + if (target == null || target.trim().isEmpty()) { + throw new IllegalArgumentException("target cannot be empty"); + } + return new Relationship(type, target, source); + } + + public static class Builder { + private String source = null; + private String target = null; + private String type = null; + + public Builder() {} + + public Builder withSource(final String source) { + this.source = source; + return this; + } + + public Builder withTarget(final String target) { + this.target = target; + return this; + } + + public Builder withType(final String type) { + this.type = type; + return this; + } + + public Relationship build() { + return Relationship.build(source, type, target); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/RelationshipType.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/RelationshipType.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/RelationshipType.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/RelationshipType.java index 615325acf..6f451cdd5 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/RelationshipType.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/RelationshipType.java @@ -1,32 +1,32 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -public enum RelationshipType { - DERIVED_FROM, - SAME_AS, - CHILD_OF, - HAS_MEMBER, - EXTERNAL_REFERENCE, - OTHER, - ANY; - - public static RelationshipType getType(String relationshipTypeString) { - RelationshipType type; - try { - type = RelationshipType.valueOf(relationshipTypeString.replace(" ", "_").toUpperCase()); - } catch (IllegalArgumentException e) { - type = RelationshipType.OTHER; - } - - return type; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +public enum RelationshipType { + DERIVED_FROM, + SAME_AS, + CHILD_OF, + HAS_MEMBER, + EXTERNAL_REFERENCE, + OTHER, + ANY; + + public static RelationshipType getType(String relationshipTypeString) { + RelationshipType type; + try { + type = RelationshipType.valueOf(relationshipTypeString.replace(" ", "_").toUpperCase()); + } catch (IllegalArgumentException e) { + type = RelationshipType.OTHER; + } + + return type; + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Sample.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Sample.java similarity index 95% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/Sample.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/Sample.java index c7121564e..b87c23c80 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/Sample.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/Sample.java @@ -1,1074 +1,1074 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE; -import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME; - -import com.fasterxml.jackson.annotation.*; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import java.time.*; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.time.temporal.TemporalAccessor; -import java.util.*; -import lombok.Getter; -import lombok.ToString; -import uk.ac.ebi.biosamples.model.structured.AbstractData; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; -import uk.ac.ebi.biosamples.service.CharacteristicDeserializer; -import uk.ac.ebi.biosamples.service.CharacteristicSerializer; -import uk.ac.ebi.biosamples.service.CustomInstantDeserializer; -import uk.ac.ebi.biosamples.service.CustomInstantSerializer; - -@Getter -@ToString -@JsonInclude(JsonInclude.Include.NON_EMPTY) -@JsonPropertyOrder({ - "name", - "accession", - "sraAccession", - "domain", - "webinSubmissionAccountId", - "taxId", - "status", - "release", - "update", - "submitted", - "characteristics", - "relationships", - "externalReferences", - "releaseDate", - "updateDate", - "submittedDate", - "submittedVia" -}) -public class Sample implements Comparable { - @JsonProperty("accession") - protected String accession; - - @JsonProperty("sraAccession") - protected String sraAccession; - - @JsonProperty("name") - protected String name; - - @JsonProperty("domain") - protected String domain; - - @JsonProperty("webinSubmissionAccountId") - protected String webinSubmissionAccountId; - - protected Long taxId; - protected SampleStatus status; - - @JsonSerialize(using = CustomInstantSerializer.class) - protected Instant release; - - @JsonSerialize(using = CustomInstantSerializer.class) - protected Instant update; - - @JsonSerialize(using = CustomInstantSerializer.class) - protected Instant create; - - @JsonSerialize(using = CustomInstantSerializer.class) - protected Instant submitted; - - @JsonSerialize(using = CustomInstantSerializer.class) - @JsonIgnore - protected Instant reviewed; - - @JsonIgnore protected SortedSet attributes; - - @JsonProperty("data") - protected SortedSet data; - - @JsonProperty("structuredData") - protected Set structuredData; - - @JsonProperty("relationships") - protected SortedSet relationships; - - @JsonProperty("externalReferences") - protected SortedSet externalReferences; - - @JsonProperty("organization") - protected SortedSet organizations; - - @JsonProperty("contact") - protected SortedSet contacts; - - @JsonProperty("publications") - protected SortedSet publications; - - @JsonProperty("certificates") - protected SortedSet certificates; - - @JsonProperty("submittedVia") - private SubmittedViaType submittedVia; - - protected Sample() {} - - @JsonSerialize(using = CustomInstantSerializer.class) - @JsonIgnore - public Instant getReviewed() { - return reviewed; - } - - @JsonIgnore - public boolean hasAccession() { - return accession != null && !accession.trim().isEmpty(); - } - - @JsonIgnore - public boolean hasSraAccession() { - return sraAccession != null && !sraAccession.trim().isEmpty(); - } - - @JsonProperty(value = "releaseDate", access = JsonProperty.Access.READ_ONLY) - @JsonIgnore - public String getReleaseDate() { - return release != null - ? ZonedDateTime.ofInstant(release, ZoneOffset.UTC).format(ISO_LOCAL_DATE) - : null; - } - - @JsonProperty(value = "updateDate", access = JsonProperty.Access.READ_ONLY) - @JsonIgnore - public String getUpdateDate() { - return update != null - ? ZonedDateTime.ofInstant(update, ZoneOffset.UTC).format(ISO_LOCAL_DATE) - : null; - } - - @JsonProperty(value = "submittedDate", access = JsonProperty.Access.READ_ONLY) - @JsonIgnore - public String getSubmittedDate() { - return submitted != null - ? ZonedDateTime.ofInstant(submitted, ZoneOffset.UTC).format(ISO_LOCAL_DATE) - : null; - } - - @JsonProperty(value = "reviewedDate", access = JsonProperty.Access.READ_ONLY) - @JsonIgnore - public String getReviewedDate() { - return reviewed != null - ? ZonedDateTime.ofInstant(reviewed, ZoneOffset.UTC).format(ISO_LOCAL_DATE) - : null; - } - - @JsonProperty(value = "taxId") - public Long getTaxId() { - if (taxId != null) { - return taxId; - } - - Optional taxon = Optional.empty(); - - for (final Attribute attribute : attributes) { - if ("organism".equalsIgnoreCase(attribute.getType()) && !attribute.getIri().isEmpty()) { - taxon = - attribute.getIri().stream() - .map(this::extractTaxIdFromIri) - .filter(i -> i > 0) - .findFirst(); - break; - } - } - - return taxon.orElse(null); - } - - @JsonProperty(value = "status") - public SampleStatus getStatus() { - if (status != null) { - return status; - } - - return null; - } - - private long extractTaxIdFromIri(final String iri) { - if (iri.isEmpty()) { - return 0; - } - final String[] segments = iri.split("NCBITaxon_"); - try { - return Integer.parseInt(segments[segments.length - 1]); - } catch (final NumberFormatException e) { - return 0; - } - } - - // DO NOT specify the JSON property value manually, must be autoinferred or errors - @JsonSerialize(using = CharacteristicSerializer.class) - public SortedSet getCharacteristics() { - return attributes; - } - - @Override - public boolean equals(final Object o) { - if (o == this) { - return true; - } - - if (!(o instanceof Sample)) { - return false; - } - - final Sample other = (Sample) o; - - // dont use update date for comparisons, too volatile. SubmittedVia doesnt contain information - // for comparison - - return Objects.equals(name, other.name) - && Objects.equals(accession, other.accession) - && Objects.equals(domain, other.domain) - && Objects.equals(webinSubmissionAccountId, other.webinSubmissionAccountId) - && Objects.equals(taxId, other.taxId) - && Objects.equals(status, other.status) - && Objects.equals(release, other.release) - && Objects.equals(attributes, other.attributes) - && Objects.equals(data, other.data) - && Objects.equals(relationships, other.relationships) - && Objects.equals(externalReferences, other.externalReferences) - && Objects.equals(organizations, other.organizations) - && Objects.equals(contacts, other.contacts) - && Objects.equals(publications, other.publications); - } - - @Override - public int hashCode() { - // don't put update date in the hash because it's not in comparison - return Objects.hash( - name, - accession, - taxId, - status, - release, - attributes, - data, - relationships, - externalReferences, - organizations, - publications); - } - - @Override - public int compareTo(final Sample other) { - if (other == null) { - return 1; - } - - if (!accession.equals(other.accession)) { - return accession.compareTo(other.accession); - } - - if (!sraAccession.equals(other.sraAccession)) { - return sraAccession.compareTo(other.sraAccession); - } - - if (!name.equals(other.name)) { - return name.compareTo(other.name); - } - - if (!taxId.equals(other.taxId)) { - return taxId.compareTo(other.taxId); - } - - if (!status.equals(other.status)) { - return status.compareTo(other.status); - } - - if (!release.equals(other.release)) { - return release.compareTo(other.release); - } - - if (!attributes.equals(other.attributes)) { - if (attributes.size() < other.attributes.size()) { - return -1; - } else if (attributes.size() > other.attributes.size()) { - return 1; - } else { - final Iterator thisIt = attributes.iterator(); - final Iterator otherIt = other.attributes.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - if (!relationships.equals(other.relationships)) { - if (relationships.size() < other.relationships.size()) { - return -1; - } else if (relationships.size() > other.relationships.size()) { - return 1; - } else { - final Iterator thisIt = relationships.iterator(); - final Iterator otherIt = other.relationships.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - if (!externalReferences.equals(other.externalReferences)) { - if (externalReferences.size() < other.externalReferences.size()) { - return -1; - } else if (externalReferences.size() > other.externalReferences.size()) { - return 1; - } else { - final Iterator thisIt = externalReferences.iterator(); - final Iterator otherIt = other.externalReferences.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - if (!organizations.equals(other.organizations)) { - if (organizations.size() < other.organizations.size()) { - return -1; - } else if (organizations.size() > other.organizations.size()) { - return 1; - } else { - final Iterator thisIt = organizations.iterator(); - final Iterator otherIt = other.organizations.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - if (!contacts.equals(other.contacts)) { - if (contacts.size() < other.contacts.size()) { - return -1; - } else if (contacts.size() > other.contacts.size()) { - return 1; - } else { - final Iterator thisIt = contacts.iterator(); - final Iterator otherIt = other.contacts.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - if (!publications.equals(other.publications)) { - if (publications.size() < other.publications.size()) { - return -1; - } else if (publications.size() > other.publications.size()) { - return 1; - } else { - final Iterator thisIt = publications.iterator(); - final Iterator otherIt = other.publications.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - return 0; - } - - public static Sample build( - final String name, - final String accession, - final String sraAccession, - final String domain, - final String webinSubmissionAccountId, - final Long taxId, - final SampleStatus status, - final Instant release, - final Instant update, - final Instant create, - final Instant submitted, - final Instant reviewed, - final Set attributes, - final Set relationships, - final Set externalReferences) { - return build( - name, - accession, - sraAccession, - domain, - webinSubmissionAccountId, - taxId, - status, - release, - update, - create, - submitted, - reviewed, - attributes, - null, - null, - relationships, - externalReferences, - null, - null, - null, - null, - null); - } - - public static Sample build( - final String name, - final String accession, - final String sraAccession, - final String domain, - final String webinSubmissionAccountId, - final Long taxId, - final SampleStatus status, - final Instant release, - final Instant update, - final Instant create, - final Instant submitted, - final Instant reviewed, - final Set attributes, - final Set relationships, - final Set externalReferences, - final SubmittedViaType submittedVia) { - return build( - name, - accession, - sraAccession, - domain, - webinSubmissionAccountId, - taxId, - status, - release, - update, - create, - submitted, - reviewed, - attributes, - null, - null, - relationships, - externalReferences, - null, - null, - null, - null, - submittedVia); - } - - // Used for deserializtion (JSON -> Java) - @JsonCreator - public static Sample build( - @JsonProperty("name") final String name, - @JsonProperty("accession") final String accession, - @JsonProperty("sraAccession") final String sraAccession, - @JsonProperty("domain") final String domain, - @JsonProperty("webinSubmissionAccountId") final String webinSubmissionAccountId, - @JsonProperty("taxId") final Long taxId, - @JsonProperty("status") final SampleStatus status, - @JsonProperty("release") @JsonDeserialize(using = CustomInstantDeserializer.class) - final Instant release, - @JsonProperty("update") @JsonDeserialize(using = CustomInstantDeserializer.class) - final Instant update, - @JsonProperty("create") @JsonDeserialize(using = CustomInstantDeserializer.class) - final Instant create, - @JsonProperty("submitted") @JsonDeserialize(using = CustomInstantDeserializer.class) - final Instant submitted, - @JsonProperty("reviewed") @JsonDeserialize(using = CustomInstantDeserializer.class) - final Instant reviewed, - @JsonProperty("characteristics") @JsonDeserialize(using = CharacteristicDeserializer.class) - final Collection attributes, - @JsonProperty("data") final Collection data, - @JsonProperty("structuredData") final Collection structuredData, - @JsonProperty("relationships") final Collection relationships, - @JsonProperty("externalReferences") final Collection externalReferences, - @JsonProperty("organization") final Collection organizations, - @JsonProperty("contact") final Collection contacts, - @JsonProperty("publications") final Collection publications, - @JsonProperty("certificates") final Collection certificates, - @JsonProperty("submittedVia") final SubmittedViaType submittedVia) { - - final Sample sample = new Sample(); - - if (accession != null) { - sample.accession = accession.trim(); - } - - if (sraAccession != null) { - sample.sraAccession = sraAccession.trim(); - } - - if (name == null) { - throw new IllegalArgumentException("Sample name must be provided"); - } - - sample.name = name.trim(); - - if (domain != null) { - sample.domain = domain.trim(); - } - - if (webinSubmissionAccountId != null) { - sample.webinSubmissionAccountId = webinSubmissionAccountId.trim(); - } - - if (taxId != null) { - sample.taxId = taxId; - } - - // Instead of validation failure, if null, set it to now - sample.update = update == null ? Instant.now() : update; - - sample.create = create == null ? sample.update : create; - - sample.submitted = submitted; - - sample.reviewed = reviewed; - - // Validation moved to a later stage, to capture the error (SampleService.store()) - sample.release = release; - - if (status != null) { - sample.status = status; - } else { - if (sample.release != null) { - sample.status = - sample.release.isAfter(Instant.now()) ? SampleStatus.PRIVATE : SampleStatus.PUBLIC; - } else { - sample.status = SampleStatus.PRIVATE; - } - } - - sample.attributes = new TreeSet<>(); - - if (attributes != null) { - sample.attributes.addAll(attributes); - } - - sample.relationships = new TreeSet<>(); - - if (relationships != null) { - sample.relationships.addAll(relationships); - } - - sample.externalReferences = new TreeSet<>(); - - if (externalReferences != null) { - sample.externalReferences.addAll(externalReferences); - } - - sample.organizations = new TreeSet<>(); - - if (organizations != null) { - sample.organizations.addAll(organizations); - } - - sample.contacts = new TreeSet<>(); - - if (contacts != null) { - sample.contacts.addAll(contacts); - } - - sample.publications = new TreeSet<>(); - - if (publications != null) { - sample.publications.addAll(publications); - } - - sample.certificates = new TreeSet<>(); - - if (certificates != null) { - sample.certificates.addAll(certificates); - } - - sample.data = new TreeSet<>(); - - if (data != null) { - sample.data.addAll(data); - } - - sample.structuredData = new HashSet<>(); - - if (structuredData != null) { - sample.structuredData.addAll(structuredData); - } - - sample.submittedVia = submittedVia != null ? submittedVia : SubmittedViaType.JSON_API; - - return sample; - } - - public static class Builder { - protected String name; - protected String accession = null; - protected String sraAccession = null; - protected String domain = null; - protected String webinSubmissionAccountId = null; - protected Long taxId = null; - protected SampleStatus status = null; - protected Instant release = Instant.now(); - protected Instant update = Instant.now(); - protected Instant create = Instant.now(); - protected Instant submitted = Instant.now(); - protected Instant reviewed; - SubmittedViaType submittedVia; - protected SortedSet attributes = new TreeSet<>(); - protected SortedSet relationships = new TreeSet<>(); - protected SortedSet externalReferences = new TreeSet<>(); - protected SortedSet organizations = new TreeSet<>(); - protected SortedSet contacts = new TreeSet<>(); - protected SortedSet publications = new TreeSet<>(); - protected SortedSet certificates = new TreeSet<>(); - protected Set data = new TreeSet<>(); - protected Set structuredData = new HashSet<>(); - - public Builder(final String name, final String accession) { - this.name = name; - this.accession = accession; - } - - public Builder(final String name, final String accession, final String sraAccession) { - this.name = name; - this.accession = accession; - this.sraAccession = sraAccession; - } - - public Builder( - final String name, - final String accession, - final String sraAccession, - final SampleStatus status) { - this.name = name; - this.accession = accession; - this.sraAccession = sraAccession; - this.status = status; - } - - public Builder(final String name, final String accession, final SampleStatus status) { - this.name = name; - this.accession = accession; - this.status = status; - } - - public Builder(final String name) { - this.name = name; - } - - public Builder(final String name, final SampleStatus status) { - this.name = name; - this.status = status; - } - - public Builder withAccession(final String accession) { - this.accession = accession; - return this; - } - - public Builder withSraAccession(final String sraAccession) { - this.sraAccession = sraAccession; - return this; - } - - public Builder withDomain(final String domain) { - this.domain = domain; - return this; - } - - public Builder withWebinSubmissionAccountId(final String webinSubmissionAccountId) { - this.webinSubmissionAccountId = webinSubmissionAccountId; - return this; - } - - public Builder withTaxId(final Long taxId) { - this.taxId = taxId; - return this; - } - - public Builder withStatus(final SampleStatus status) { - this.status = status; - return this; - } - - public Builder withRelease(final String release) { - this.release = Objects.requireNonNull(parseDateTime(release)).toInstant(); - return this; - } - - public Builder withRelease(final Instant release) { - this.release = release; - return this; - } - - public Builder withUpdate(final Instant update) { - this.update = update; - return this; - } - - public Builder withUpdate(final String update) { - this.update = Objects.requireNonNull(parseDateTime(update)).toInstant(); - return this; - } - - public Builder withCreate(final Instant create) { - this.create = create; - return this; - } - - public Builder withCreate(final String create) { - this.create = Objects.requireNonNull(parseDateTime(create)).toInstant(); - return this; - } - - public Builder withSubmitted(final Instant submitted) { - this.submitted = submitted; - return this; - } - - public Builder withSubmitted(final String submitted) { - this.submitted = Objects.requireNonNull(parseDateTime(submitted)).toInstant(); - return this; - } - - public Builder withNoSubmitted() { - submitted = null; - return this; - } - - public Builder withReviewed(final Instant reviewed) { - this.reviewed = reviewed; - return this; - } - - public Builder withReviewed(final String reviewed) { - this.reviewed = Objects.requireNonNull(parseDateTime(reviewed)).toInstant(); - return this; - } - - public Builder withNoReviewed() { - reviewed = null; - return this; - } - - public Builder withSubmittedVia(final SubmittedViaType submittedVia) { - this.submittedVia = submittedVia; - return this; - } - - /** Replace builder's attributes with the provided attribute collection */ - public Builder withAttributes(final Collection attributes) { - this.attributes = new TreeSet<>(attributes); - return this; - } - - public Builder addAttribute(final Attribute attribute) { - attributes.add(attribute); - return this; - } - - public Builder addAllAttributes(final Collection attributes) { - this.attributes.addAll(attributes); - return this; - } - - /** Replace builder structuredData with the provided structuredData collection */ - public Builder withData(final Collection data) { - if (data != null) { - this.data = new TreeSet<>(data); - } else { - this.data = new TreeSet<>(); - } - - return this; - } - - public Builder withStructuredData(final Set structuredData) { - this.structuredData = structuredData != null ? structuredData : new HashSet<>(); - - return this; - } - - public Builder addData(final AbstractData data) { - this.data.add(data); - return this; - } - - public Builder addAllData(final Collection data) { - this.data.addAll(data); - return this; - } - - /** Replace builder's relationships with the provided relationships collection */ - public Builder withRelationships(final Collection relationships) { - if (relationships != null) { - this.relationships = new TreeSet<>(relationships); - } - return this; - } - - public Builder addRelationship(final Relationship relationship) { - relationships.add(relationship); - return this; - } - - public Builder addAllRelationships(final Collection relationships) { - this.relationships.addAll(relationships); - return this; - } - - /** Replace builder's externalReferences with the provided external references collection */ - public Builder withExternalReferences(final Collection externalReferences) { - if (externalReferences != null && !externalReferences.isEmpty()) { - this.externalReferences = new TreeSet<>(externalReferences); - } - return this; - } - - public Builder addExternalReference(final ExternalReference externalReference) { - externalReferences.add(externalReference); - return this; - } - - public Builder addAllExternalReferences( - final Collection externalReferences) { - this.externalReferences.addAll(externalReferences); - return this; - } - - /** Replace builder's organisations with the provided organisation collection */ - public Builder withOrganizations(final Collection organizations) { - if (organizations != null && !organizations.isEmpty()) { - this.organizations = new TreeSet<>(organizations); - } - return this; - } - - public Builder addOrganization(final Organization organization) { - organizations.add(organization); - return this; - } - - public Builder allAllOrganizations(final Collection organizations) { - this.organizations.addAll(organizations); - return this; - } - - /** Replace builder's contacts with the provided contact collection */ - public Builder withContacts(final Collection contacts) { - if (contacts != null && !contacts.isEmpty()) { - this.contacts = new TreeSet<>(contacts); - } - return this; - } - - public Builder addContact(final Contact contact) { - contacts.add(contact); - return this; - } - - public Builder addAllContacts(final Collection contacts) { - this.contacts.addAll(contacts); - return this; - } - - /** Replace the publications with the provided collections */ - public Builder withPublications(final Collection publications) { - if (publications != null && !publications.isEmpty()) { - this.publications = new TreeSet<>(publications); - } - return this; - } - - /** Add a publication to the list of builder publications */ - public Builder addPublication(final Publication publication) { - publications.add(publication); - return this; - } - - /** Add all publications in the provided collection to the builder publications */ - public Builder addAllPublications(final Collection publications) { - this.publications.addAll(publications); - return this; - } - - public Builder withCertificates(final Collection certificates) { - if (certificates != null && !certificates.isEmpty()) { - this.certificates = new TreeSet<>(certificates); - } - return this; - } - - public Builder addAllCertificates(final Collection certificates) { - this.certificates.addAll(certificates); - return this; - } - - // Clean accession field - public Builder withNoAccession() { - accession = null; - return this; - } - - // Clean domain field - public Builder withNoDomain() { - domain = null; - return this; - } - - // Clean webin account id field - public Builder withNoWebinSubmissionAccountId() { - webinSubmissionAccountId = null; - return this; - } - - // Clean collection fields - public Builder withNoAttributes() { - attributes = new TreeSet<>(); - return this; - } - - public Builder withNoRelationships() { - relationships = new TreeSet<>(); - return this; - } - - public Builder withNoData() { - data = new TreeSet<>(); - return this; - } - - public Builder withNoStructuredData() { - structuredData = new HashSet<>(); - return this; - } - - public Builder withNoExternalReferences() { - externalReferences = new TreeSet<>(); - return this; - } - - public Builder withNoContacts() { - contacts = new TreeSet<>(); - return this; - } - - public Builder withNoOrganisations() { - organizations = new TreeSet<>(); - return this; - } - - public Builder withNoPublications() { - publications = new TreeSet<>(); - return this; - } - - public Sample build() { - return Sample.build( - name, - accession, - sraAccession, - domain, - webinSubmissionAccountId, - taxId, - status, - release, - update, - create, - submitted, - reviewed, - attributes, - data, - structuredData, - relationships, - externalReferences, - organizations, - contacts, - publications, - certificates, - submittedVia); - } - - private ZonedDateTime parseDateTime(final String datetimeString) { - if (datetimeString.isEmpty()) { - return null; - } - final TemporalAccessor temporalAccessor = - getFormatter() - .parseBest(datetimeString, ZonedDateTime::from, LocalDateTime::from, LocalDate::from); - if (temporalAccessor instanceof ZonedDateTime) { - return (ZonedDateTime) temporalAccessor; - } else if (temporalAccessor instanceof LocalDateTime) { - return ((LocalDateTime) temporalAccessor).atZone(ZoneId.of("UTC")); - } else { - return ((LocalDate) temporalAccessor).atStartOfDay(ZoneId.of("UTC")); - } - } - - /** - * Return a Builder produced extracting informations from the sample - * - * @param sample the sample to use as reference - * @return the Builder - */ - public static Builder fromSample(final Sample sample) { - return new Builder(sample.getName(), sample.getAccession(), sample.getSraAccession()) - .withDomain(sample.getDomain()) - .withWebinSubmissionAccountId(sample.getWebinSubmissionAccountId()) - .withTaxId(sample.getTaxId()) - .withStatus(sample.getStatus()) - .withRelease(sample.getRelease()) - .withUpdate(sample.getUpdate()) - .withCreate(sample.getCreate()) - .withSubmitted(sample.getSubmitted()) - .withReviewed(sample.getReviewed()) - .withAttributes(sample.getAttributes()) - .withData(sample.getData()) - .withStructuredData(sample.getStructuredData()) - .withRelationships(sample.getRelationships()) - .withExternalReferences(sample.getExternalReferences()) - .withOrganizations(sample.getOrganizations()) - .withPublications(sample.getPublications()) - .withCertificates(sample.getCertificates()) - .withContacts(sample.getContacts()) - .withSubmittedVia(sample.getSubmittedVia()); - } - - private DateTimeFormatter getFormatter() { - return new DateTimeFormatterBuilder() - .parseCaseInsensitive() - .append(ISO_LOCAL_DATE) - .optionalStart() // time made optional - .appendLiteral('T') - .append(ISO_LOCAL_TIME) - .optionalStart() // zone and offset made optional - .appendOffsetId() - .optionalStart() - .appendLiteral('[') - .parseCaseSensitive() - .appendZoneRegionId() - .appendLiteral(']') - .optionalEnd() - .optionalEnd() - .optionalEnd() - .toFormatter(); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE; +import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME; + +import com.fasterxml.jackson.annotation.*; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.TemporalAccessor; +import java.util.*; +import lombok.Getter; +import lombok.ToString; +import uk.ac.ebi.biosamples.core.model.structured.AbstractData; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; +import uk.ac.ebi.biosamples.core.service.CharacteristicDeserializer; +import uk.ac.ebi.biosamples.core.service.CharacteristicSerializer; +import uk.ac.ebi.biosamples.core.service.CustomInstantDeserializer; +import uk.ac.ebi.biosamples.core.service.CustomInstantSerializer; + +@Getter +@ToString +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@JsonPropertyOrder({ + "name", + "accession", + "sraAccession", + "domain", + "webinSubmissionAccountId", + "taxId", + "status", + "release", + "update", + "submitted", + "characteristics", + "relationships", + "externalReferences", + "releaseDate", + "updateDate", + "submittedDate", + "submittedVia" +}) +public class Sample implements Comparable { + @JsonProperty("accession") + protected String accession; + + @JsonProperty("sraAccession") + protected String sraAccession; + + @JsonProperty("name") + protected String name; + + @JsonProperty("domain") + protected String domain; + + @JsonProperty("webinSubmissionAccountId") + protected String webinSubmissionAccountId; + + protected Long taxId; + protected SampleStatus status; + + @JsonSerialize(using = CustomInstantSerializer.class) + protected Instant release; + + @JsonSerialize(using = CustomInstantSerializer.class) + protected Instant update; + + @JsonSerialize(using = CustomInstantSerializer.class) + protected Instant create; + + @JsonSerialize(using = CustomInstantSerializer.class) + protected Instant submitted; + + @JsonSerialize(using = CustomInstantSerializer.class) + @JsonIgnore + protected Instant reviewed; + + @JsonIgnore protected SortedSet attributes; + + @JsonProperty("data") + protected SortedSet data; + + @JsonProperty("structuredData") + protected Set structuredData; + + @JsonProperty("relationships") + protected SortedSet relationships; + + @JsonProperty("externalReferences") + protected SortedSet externalReferences; + + @JsonProperty("organization") + protected SortedSet organizations; + + @JsonProperty("contact") + protected SortedSet contacts; + + @JsonProperty("publications") + protected SortedSet publications; + + @JsonProperty("certificates") + protected SortedSet certificates; + + @JsonProperty("submittedVia") + private SubmittedViaType submittedVia; + + protected Sample() {} + + @JsonSerialize(using = CustomInstantSerializer.class) + @JsonIgnore + public Instant getReviewed() { + return reviewed; + } + + @JsonIgnore + public boolean hasAccession() { + return accession != null && !accession.trim().isEmpty(); + } + + @JsonIgnore + public boolean hasSraAccession() { + return sraAccession != null && !sraAccession.trim().isEmpty(); + } + + @JsonProperty(value = "releaseDate", access = JsonProperty.Access.READ_ONLY) + @JsonIgnore + public String getReleaseDate() { + return release != null + ? ZonedDateTime.ofInstant(release, ZoneOffset.UTC).format(ISO_LOCAL_DATE) + : null; + } + + @JsonProperty(value = "updateDate", access = JsonProperty.Access.READ_ONLY) + @JsonIgnore + public String getUpdateDate() { + return update != null + ? ZonedDateTime.ofInstant(update, ZoneOffset.UTC).format(ISO_LOCAL_DATE) + : null; + } + + @JsonProperty(value = "submittedDate", access = JsonProperty.Access.READ_ONLY) + @JsonIgnore + public String getSubmittedDate() { + return submitted != null + ? ZonedDateTime.ofInstant(submitted, ZoneOffset.UTC).format(ISO_LOCAL_DATE) + : null; + } + + @JsonProperty(value = "reviewedDate", access = JsonProperty.Access.READ_ONLY) + @JsonIgnore + public String getReviewedDate() { + return reviewed != null + ? ZonedDateTime.ofInstant(reviewed, ZoneOffset.UTC).format(ISO_LOCAL_DATE) + : null; + } + + @JsonProperty(value = "taxId") + public Long getTaxId() { + if (taxId != null) { + return taxId; + } + + Optional taxon = Optional.empty(); + + for (final Attribute attribute : attributes) { + if ("organism".equalsIgnoreCase(attribute.getType()) && !attribute.getIri().isEmpty()) { + taxon = + attribute.getIri().stream() + .map(this::extractTaxIdFromIri) + .filter(i -> i > 0) + .findFirst(); + break; + } + } + + return taxon.orElse(null); + } + + @JsonProperty(value = "status") + public SampleStatus getStatus() { + if (status != null) { + return status; + } + + return null; + } + + private long extractTaxIdFromIri(final String iri) { + if (iri.isEmpty()) { + return 0; + } + final String[] segments = iri.split("NCBITaxon_"); + try { + return Integer.parseInt(segments[segments.length - 1]); + } catch (final NumberFormatException e) { + return 0; + } + } + + // DO NOT specify the JSON property value manually, must be autoinferred or errors + @JsonSerialize(using = CharacteristicSerializer.class) + public SortedSet getCharacteristics() { + return attributes; + } + + @Override + public boolean equals(final Object o) { + if (o == this) { + return true; + } + + if (!(o instanceof Sample)) { + return false; + } + + final Sample other = (Sample) o; + + // dont use update date for comparisons, too volatile. SubmittedVia doesnt contain information + // for comparison + + return Objects.equals(name, other.name) + && Objects.equals(accession, other.accession) + && Objects.equals(domain, other.domain) + && Objects.equals(webinSubmissionAccountId, other.webinSubmissionAccountId) + && Objects.equals(taxId, other.taxId) + && Objects.equals(status, other.status) + && Objects.equals(release, other.release) + && Objects.equals(attributes, other.attributes) + && Objects.equals(data, other.data) + && Objects.equals(relationships, other.relationships) + && Objects.equals(externalReferences, other.externalReferences) + && Objects.equals(organizations, other.organizations) + && Objects.equals(contacts, other.contacts) + && Objects.equals(publications, other.publications); + } + + @Override + public int hashCode() { + // don't put update date in the hash because it's not in comparison + return Objects.hash( + name, + accession, + taxId, + status, + release, + attributes, + data, + relationships, + externalReferences, + organizations, + publications); + } + + @Override + public int compareTo(final Sample other) { + if (other == null) { + return 1; + } + + if (!accession.equals(other.accession)) { + return accession.compareTo(other.accession); + } + + if (!sraAccession.equals(other.sraAccession)) { + return sraAccession.compareTo(other.sraAccession); + } + + if (!name.equals(other.name)) { + return name.compareTo(other.name); + } + + if (!taxId.equals(other.taxId)) { + return taxId.compareTo(other.taxId); + } + + if (!status.equals(other.status)) { + return status.compareTo(other.status); + } + + if (!release.equals(other.release)) { + return release.compareTo(other.release); + } + + if (!attributes.equals(other.attributes)) { + if (attributes.size() < other.attributes.size()) { + return -1; + } else if (attributes.size() > other.attributes.size()) { + return 1; + } else { + final Iterator thisIt = attributes.iterator(); + final Iterator otherIt = other.attributes.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + if (!relationships.equals(other.relationships)) { + if (relationships.size() < other.relationships.size()) { + return -1; + } else if (relationships.size() > other.relationships.size()) { + return 1; + } else { + final Iterator thisIt = relationships.iterator(); + final Iterator otherIt = other.relationships.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + if (!externalReferences.equals(other.externalReferences)) { + if (externalReferences.size() < other.externalReferences.size()) { + return -1; + } else if (externalReferences.size() > other.externalReferences.size()) { + return 1; + } else { + final Iterator thisIt = externalReferences.iterator(); + final Iterator otherIt = other.externalReferences.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + if (!organizations.equals(other.organizations)) { + if (organizations.size() < other.organizations.size()) { + return -1; + } else if (organizations.size() > other.organizations.size()) { + return 1; + } else { + final Iterator thisIt = organizations.iterator(); + final Iterator otherIt = other.organizations.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + if (!contacts.equals(other.contacts)) { + if (contacts.size() < other.contacts.size()) { + return -1; + } else if (contacts.size() > other.contacts.size()) { + return 1; + } else { + final Iterator thisIt = contacts.iterator(); + final Iterator otherIt = other.contacts.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + if (!publications.equals(other.publications)) { + if (publications.size() < other.publications.size()) { + return -1; + } else if (publications.size() > other.publications.size()) { + return 1; + } else { + final Iterator thisIt = publications.iterator(); + final Iterator otherIt = other.publications.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + return 0; + } + + public static Sample build( + final String name, + final String accession, + final String sraAccession, + final String domain, + final String webinSubmissionAccountId, + final Long taxId, + final SampleStatus status, + final Instant release, + final Instant update, + final Instant create, + final Instant submitted, + final Instant reviewed, + final Set attributes, + final Set relationships, + final Set externalReferences) { + return build( + name, + accession, + sraAccession, + domain, + webinSubmissionAccountId, + taxId, + status, + release, + update, + create, + submitted, + reviewed, + attributes, + null, + null, + relationships, + externalReferences, + null, + null, + null, + null, + null); + } + + public static Sample build( + final String name, + final String accession, + final String sraAccession, + final String domain, + final String webinSubmissionAccountId, + final Long taxId, + final SampleStatus status, + final Instant release, + final Instant update, + final Instant create, + final Instant submitted, + final Instant reviewed, + final Set attributes, + final Set relationships, + final Set externalReferences, + final SubmittedViaType submittedVia) { + return build( + name, + accession, + sraAccession, + domain, + webinSubmissionAccountId, + taxId, + status, + release, + update, + create, + submitted, + reviewed, + attributes, + null, + null, + relationships, + externalReferences, + null, + null, + null, + null, + submittedVia); + } + + // Used for deserializtion (JSON -> Java) + @JsonCreator + public static Sample build( + @JsonProperty("name") final String name, + @JsonProperty("accession") final String accession, + @JsonProperty("sraAccession") final String sraAccession, + @JsonProperty("domain") final String domain, + @JsonProperty("webinSubmissionAccountId") final String webinSubmissionAccountId, + @JsonProperty("taxId") final Long taxId, + @JsonProperty("status") final SampleStatus status, + @JsonProperty("release") @JsonDeserialize(using = CustomInstantDeserializer.class) + final Instant release, + @JsonProperty("update") @JsonDeserialize(using = CustomInstantDeserializer.class) + final Instant update, + @JsonProperty("create") @JsonDeserialize(using = CustomInstantDeserializer.class) + final Instant create, + @JsonProperty("submitted") @JsonDeserialize(using = CustomInstantDeserializer.class) + final Instant submitted, + @JsonProperty("reviewed") @JsonDeserialize(using = CustomInstantDeserializer.class) + final Instant reviewed, + @JsonProperty("characteristics") @JsonDeserialize(using = CharacteristicDeserializer.class) + final Collection attributes, + @JsonProperty("data") final Collection data, + @JsonProperty("structuredData") final Collection structuredData, + @JsonProperty("relationships") final Collection relationships, + @JsonProperty("externalReferences") final Collection externalReferences, + @JsonProperty("organization") final Collection organizations, + @JsonProperty("contact") final Collection contacts, + @JsonProperty("publications") final Collection publications, + @JsonProperty("certificates") final Collection certificates, + @JsonProperty("submittedVia") final SubmittedViaType submittedVia) { + + final Sample sample = new Sample(); + + if (accession != null) { + sample.accession = accession.trim(); + } + + if (sraAccession != null) { + sample.sraAccession = sraAccession.trim(); + } + + if (name == null) { + throw new IllegalArgumentException("Sample name must be provided"); + } + + sample.name = name.trim(); + + if (domain != null) { + sample.domain = domain.trim(); + } + + if (webinSubmissionAccountId != null) { + sample.webinSubmissionAccountId = webinSubmissionAccountId.trim(); + } + + if (taxId != null) { + sample.taxId = taxId; + } + + // Instead of validation failure, if null, set it to now + sample.update = update == null ? Instant.now() : update; + + sample.create = create == null ? sample.update : create; + + sample.submitted = submitted; + + sample.reviewed = reviewed; + + // Validation moved to a later stage, to capture the error (SampleService.store()) + sample.release = release; + + if (status != null) { + sample.status = status; + } else { + if (sample.release != null) { + sample.status = + sample.release.isAfter(Instant.now()) ? SampleStatus.PRIVATE : SampleStatus.PUBLIC; + } else { + sample.status = SampleStatus.PRIVATE; + } + } + + sample.attributes = new TreeSet<>(); + + if (attributes != null) { + sample.attributes.addAll(attributes); + } + + sample.relationships = new TreeSet<>(); + + if (relationships != null) { + sample.relationships.addAll(relationships); + } + + sample.externalReferences = new TreeSet<>(); + + if (externalReferences != null) { + sample.externalReferences.addAll(externalReferences); + } + + sample.organizations = new TreeSet<>(); + + if (organizations != null) { + sample.organizations.addAll(organizations); + } + + sample.contacts = new TreeSet<>(); + + if (contacts != null) { + sample.contacts.addAll(contacts); + } + + sample.publications = new TreeSet<>(); + + if (publications != null) { + sample.publications.addAll(publications); + } + + sample.certificates = new TreeSet<>(); + + if (certificates != null) { + sample.certificates.addAll(certificates); + } + + sample.data = new TreeSet<>(); + + if (data != null) { + sample.data.addAll(data); + } + + sample.structuredData = new HashSet<>(); + + if (structuredData != null) { + sample.structuredData.addAll(structuredData); + } + + sample.submittedVia = submittedVia != null ? submittedVia : SubmittedViaType.JSON_API; + + return sample; + } + + public static class Builder { + protected String name; + protected String accession = null; + protected String sraAccession = null; + protected String domain = null; + protected String webinSubmissionAccountId = null; + protected Long taxId = null; + protected SampleStatus status = null; + protected Instant release = Instant.now(); + protected Instant update = Instant.now(); + protected Instant create = Instant.now(); + protected Instant submitted = Instant.now(); + protected Instant reviewed; + SubmittedViaType submittedVia; + protected SortedSet attributes = new TreeSet<>(); + protected SortedSet relationships = new TreeSet<>(); + protected SortedSet externalReferences = new TreeSet<>(); + protected SortedSet organizations = new TreeSet<>(); + protected SortedSet contacts = new TreeSet<>(); + protected SortedSet publications = new TreeSet<>(); + protected SortedSet certificates = new TreeSet<>(); + protected Set data = new TreeSet<>(); + protected Set structuredData = new HashSet<>(); + + public Builder(final String name, final String accession) { + this.name = name; + this.accession = accession; + } + + public Builder(final String name, final String accession, final String sraAccession) { + this.name = name; + this.accession = accession; + this.sraAccession = sraAccession; + } + + public Builder( + final String name, + final String accession, + final String sraAccession, + final SampleStatus status) { + this.name = name; + this.accession = accession; + this.sraAccession = sraAccession; + this.status = status; + } + + public Builder(final String name, final String accession, final SampleStatus status) { + this.name = name; + this.accession = accession; + this.status = status; + } + + public Builder(final String name) { + this.name = name; + } + + public Builder(final String name, final SampleStatus status) { + this.name = name; + this.status = status; + } + + public Builder withAccession(final String accession) { + this.accession = accession; + return this; + } + + public Builder withSraAccession(final String sraAccession) { + this.sraAccession = sraAccession; + return this; + } + + public Builder withDomain(final String domain) { + this.domain = domain; + return this; + } + + public Builder withWebinSubmissionAccountId(final String webinSubmissionAccountId) { + this.webinSubmissionAccountId = webinSubmissionAccountId; + return this; + } + + public Builder withTaxId(final Long taxId) { + this.taxId = taxId; + return this; + } + + public Builder withStatus(final SampleStatus status) { + this.status = status; + return this; + } + + public Builder withRelease(final String release) { + this.release = Objects.requireNonNull(parseDateTime(release)).toInstant(); + return this; + } + + public Builder withRelease(final Instant release) { + this.release = release; + return this; + } + + public Builder withUpdate(final Instant update) { + this.update = update; + return this; + } + + public Builder withUpdate(final String update) { + this.update = Objects.requireNonNull(parseDateTime(update)).toInstant(); + return this; + } + + public Builder withCreate(final Instant create) { + this.create = create; + return this; + } + + public Builder withCreate(final String create) { + this.create = Objects.requireNonNull(parseDateTime(create)).toInstant(); + return this; + } + + public Builder withSubmitted(final Instant submitted) { + this.submitted = submitted; + return this; + } + + public Builder withSubmitted(final String submitted) { + this.submitted = Objects.requireNonNull(parseDateTime(submitted)).toInstant(); + return this; + } + + public Builder withNoSubmitted() { + submitted = null; + return this; + } + + public Builder withReviewed(final Instant reviewed) { + this.reviewed = reviewed; + return this; + } + + public Builder withReviewed(final String reviewed) { + this.reviewed = Objects.requireNonNull(parseDateTime(reviewed)).toInstant(); + return this; + } + + public Builder withNoReviewed() { + reviewed = null; + return this; + } + + public Builder withSubmittedVia(final SubmittedViaType submittedVia) { + this.submittedVia = submittedVia; + return this; + } + + /** Replace builder's attributes with the provided attribute collection */ + public Builder withAttributes(final Collection attributes) { + this.attributes = new TreeSet<>(attributes); + return this; + } + + public Builder addAttribute(final Attribute attribute) { + attributes.add(attribute); + return this; + } + + public Builder addAllAttributes(final Collection attributes) { + this.attributes.addAll(attributes); + return this; + } + + /** Replace builder structuredData with the provided structuredData collection */ + public Builder withData(final Collection data) { + if (data != null) { + this.data = new TreeSet<>(data); + } else { + this.data = new TreeSet<>(); + } + + return this; + } + + public Builder withStructuredData(final Set structuredData) { + this.structuredData = structuredData != null ? structuredData : new HashSet<>(); + + return this; + } + + public Builder addData(final AbstractData data) { + this.data.add(data); + return this; + } + + public Builder addAllData(final Collection data) { + this.data.addAll(data); + return this; + } + + /** Replace builder's relationships with the provided relationships collection */ + public Builder withRelationships(final Collection relationships) { + if (relationships != null) { + this.relationships = new TreeSet<>(relationships); + } + return this; + } + + public Builder addRelationship(final Relationship relationship) { + relationships.add(relationship); + return this; + } + + public Builder addAllRelationships(final Collection relationships) { + this.relationships.addAll(relationships); + return this; + } + + /** Replace builder's externalReferences with the provided external references collection */ + public Builder withExternalReferences(final Collection externalReferences) { + if (externalReferences != null && !externalReferences.isEmpty()) { + this.externalReferences = new TreeSet<>(externalReferences); + } + return this; + } + + public Builder addExternalReference(final ExternalReference externalReference) { + externalReferences.add(externalReference); + return this; + } + + public Builder addAllExternalReferences( + final Collection externalReferences) { + this.externalReferences.addAll(externalReferences); + return this; + } + + /** Replace builder's organisations with the provided organisation collection */ + public Builder withOrganizations(final Collection organizations) { + if (organizations != null && !organizations.isEmpty()) { + this.organizations = new TreeSet<>(organizations); + } + return this; + } + + public Builder addOrganization(final Organization organization) { + organizations.add(organization); + return this; + } + + public Builder allAllOrganizations(final Collection organizations) { + this.organizations.addAll(organizations); + return this; + } + + /** Replace builder's contacts with the provided contact collection */ + public Builder withContacts(final Collection contacts) { + if (contacts != null && !contacts.isEmpty()) { + this.contacts = new TreeSet<>(contacts); + } + return this; + } + + public Builder addContact(final Contact contact) { + contacts.add(contact); + return this; + } + + public Builder addAllContacts(final Collection contacts) { + this.contacts.addAll(contacts); + return this; + } + + /** Replace the publications with the provided collections */ + public Builder withPublications(final Collection publications) { + if (publications != null && !publications.isEmpty()) { + this.publications = new TreeSet<>(publications); + } + return this; + } + + /** Add a publication to the list of builder publications */ + public Builder addPublication(final Publication publication) { + publications.add(publication); + return this; + } + + /** Add all publications in the provided collection to the builder publications */ + public Builder addAllPublications(final Collection publications) { + this.publications.addAll(publications); + return this; + } + + public Builder withCertificates(final Collection certificates) { + if (certificates != null && !certificates.isEmpty()) { + this.certificates = new TreeSet<>(certificates); + } + return this; + } + + public Builder addAllCertificates(final Collection certificates) { + this.certificates.addAll(certificates); + return this; + } + + // Clean accession field + public Builder withNoAccession() { + accession = null; + return this; + } + + // Clean domain field + public Builder withNoDomain() { + domain = null; + return this; + } + + // Clean webin account id field + public Builder withNoWebinSubmissionAccountId() { + webinSubmissionAccountId = null; + return this; + } + + // Clean collection fields + public Builder withNoAttributes() { + attributes = new TreeSet<>(); + return this; + } + + public Builder withNoRelationships() { + relationships = new TreeSet<>(); + return this; + } + + public Builder withNoData() { + data = new TreeSet<>(); + return this; + } + + public Builder withNoStructuredData() { + structuredData = new HashSet<>(); + return this; + } + + public Builder withNoExternalReferences() { + externalReferences = new TreeSet<>(); + return this; + } + + public Builder withNoContacts() { + contacts = new TreeSet<>(); + return this; + } + + public Builder withNoOrganisations() { + organizations = new TreeSet<>(); + return this; + } + + public Builder withNoPublications() { + publications = new TreeSet<>(); + return this; + } + + public Sample build() { + return Sample.build( + name, + accession, + sraAccession, + domain, + webinSubmissionAccountId, + taxId, + status, + release, + update, + create, + submitted, + reviewed, + attributes, + data, + structuredData, + relationships, + externalReferences, + organizations, + contacts, + publications, + certificates, + submittedVia); + } + + private ZonedDateTime parseDateTime(final String datetimeString) { + if (datetimeString.isEmpty()) { + return null; + } + final TemporalAccessor temporalAccessor = + getFormatter() + .parseBest(datetimeString, ZonedDateTime::from, LocalDateTime::from, LocalDate::from); + if (temporalAccessor instanceof ZonedDateTime) { + return (ZonedDateTime) temporalAccessor; + } else if (temporalAccessor instanceof LocalDateTime) { + return ((LocalDateTime) temporalAccessor).atZone(ZoneId.of("UTC")); + } else { + return ((LocalDate) temporalAccessor).atStartOfDay(ZoneId.of("UTC")); + } + } + + /** + * Return a Builder produced extracting informations from the sample + * + * @param sample the sample to use as reference + * @return the Builder + */ + public static Builder fromSample(final Sample sample) { + return new Builder(sample.getName(), sample.getAccession(), sample.getSraAccession()) + .withDomain(sample.getDomain()) + .withWebinSubmissionAccountId(sample.getWebinSubmissionAccountId()) + .withTaxId(sample.getTaxId()) + .withStatus(sample.getStatus()) + .withRelease(sample.getRelease()) + .withUpdate(sample.getUpdate()) + .withCreate(sample.getCreate()) + .withSubmitted(sample.getSubmitted()) + .withReviewed(sample.getReviewed()) + .withAttributes(sample.getAttributes()) + .withData(sample.getData()) + .withStructuredData(sample.getStructuredData()) + .withRelationships(sample.getRelationships()) + .withExternalReferences(sample.getExternalReferences()) + .withOrganizations(sample.getOrganizations()) + .withPublications(sample.getPublications()) + .withCertificates(sample.getCertificates()) + .withContacts(sample.getContacts()) + .withSubmittedVia(sample.getSubmittedVia()); + } + + private DateTimeFormatter getFormatter() { + return new DateTimeFormatterBuilder() + .parseCaseInsensitive() + .append(ISO_LOCAL_DATE) + .optionalStart() // time made optional + .appendLiteral('T') + .append(ISO_LOCAL_TIME) + .optionalStart() // zone and offset made optional + .appendOffsetId() + .optionalStart() + .appendLiteral('[') + .parseCaseSensitive() + .appendZoneRegionId() + .appendLiteral(']') + .optionalEnd() + .optionalEnd() + .optionalEnd() + .toFormatter(); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/SampleAnalytics.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/SampleAnalytics.java similarity index 92% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/SampleAnalytics.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/SampleAnalytics.java index f34022046..368a3d07f 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/SampleAnalytics.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/SampleAnalytics.java @@ -1,83 +1,83 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import lombok.Getter; -import uk.ac.ebi.biosamples.model.filter.DateRangeFilter; -import uk.ac.ebi.biosamples.model.filter.Filter; - -@Getter -public class SampleAnalytics { - private long totalRecords; - private String dateRange; - private long processedRecords; - protected Map center; - protected Map channel; - protected Map> facets; - - public SampleAnalytics() { - center = new HashMap<>(); - channel = new HashMap<>(); - facets = new HashMap<>(); - } - - public void setTotalRecords(final long totalRecords) { - this.totalRecords = totalRecords; - } - - public void setDateRange(final String dateRange) { - this.dateRange = dateRange; - } - - public void setDateRange(final Collection filters) { - final DateRangeFilter dateRangeFilter = - filters.stream() - .filter(f -> f instanceof DateRangeFilter) - .map(DateRangeFilter.class::cast) - .findFirst() - .orElse(null); - if (dateRangeFilter != null && dateRangeFilter.getContent().isPresent()) { - final DateTimeFormatter dateTimeFormatter = - DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault()); - dateRange = - dateTimeFormatter.format(dateRangeFilter.getContent().get().getFrom()) - + " : " - + dateTimeFormatter.format(dateRangeFilter.getContent().get().getUntil()); - } - } - - public void setProcessedRecords(final long processedRecords) { - this.processedRecords = processedRecords; - } - - public void addToCenter(final String centerName) { - if (center.containsKey(centerName)) { - center.put(centerName, center.get(centerName) + 1); - } else { - center.put(centerName, 1L); - } - } - - public void addToChannel(final String accessionPrefix) { - if (channel.containsKey(accessionPrefix)) { - channel.put(accessionPrefix, channel.get(accessionPrefix) + 1); - } else { - channel.put(accessionPrefix, 1L); - } - } - - public void addToFacets() {} -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import lombok.Getter; +import uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; + +@Getter +public class SampleAnalytics { + private long totalRecords; + private String dateRange; + private long processedRecords; + protected Map center; + protected Map channel; + protected Map> facets; + + public SampleAnalytics() { + center = new HashMap<>(); + channel = new HashMap<>(); + facets = new HashMap<>(); + } + + public void setTotalRecords(final long totalRecords) { + this.totalRecords = totalRecords; + } + + public void setDateRange(final String dateRange) { + this.dateRange = dateRange; + } + + public void setDateRange(final Collection filters) { + final DateRangeFilter dateRangeFilter = + filters.stream() + .filter(f -> f instanceof DateRangeFilter) + .map(DateRangeFilter.class::cast) + .findFirst() + .orElse(null); + if (dateRangeFilter != null && dateRangeFilter.getContent().isPresent()) { + final DateTimeFormatter dateTimeFormatter = + DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault()); + dateRange = + dateTimeFormatter.format(dateRangeFilter.getContent().get().getFrom()) + + " : " + + dateTimeFormatter.format(dateRangeFilter.getContent().get().getUntil()); + } + } + + public void setProcessedRecords(final long processedRecords) { + this.processedRecords = processedRecords; + } + + public void addToCenter(final String centerName) { + if (center.containsKey(centerName)) { + center.put(centerName, center.get(centerName) + 1); + } else { + center.put(centerName, 1L); + } + } + + public void addToChannel(final String accessionPrefix) { + if (channel.containsKey(accessionPrefix)) { + channel.put(accessionPrefix, channel.get(accessionPrefix) + 1); + } else { + channel.put(accessionPrefix, 1L); + } + } + + public void addToFacets() {} +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/SampleCompareResult.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/SampleCompareResult.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/SampleCompareResult.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/SampleCompareResult.java index 7925c1a28..bd722430a 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/SampleCompareResult.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/SampleCompareResult.java @@ -1,29 +1,29 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -public class SampleCompareResult { - private final Sample sample; - private final boolean taxIdChanged; - - public SampleCompareResult(Sample sample, boolean taxIdChanged) { - this.sample = sample; - this.taxIdChanged = taxIdChanged; - } - - public Sample getSample() { - return sample; - } - - public boolean isTaxIdChanged() { - return taxIdChanged; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +public class SampleCompareResult { + private final Sample sample; + private final boolean taxIdChanged; + + public SampleCompareResult(Sample sample, boolean taxIdChanged) { + this.sample = sample; + this.taxIdChanged = taxIdChanged; + } + + public Sample getSample() { + return sample; + } + + public boolean isTaxIdChanged() { + return taxIdChanged; + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/SampleFacetValue.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/SampleFacetValue.java similarity index 94% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/SampleFacetValue.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/SampleFacetValue.java index 7ebd9e5b8..5ab5d6372 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/SampleFacetValue.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/SampleFacetValue.java @@ -1,38 +1,38 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -public class SampleFacetValue implements Comparable { - public final String label; - public final long count; - - public SampleFacetValue(String label, long count) { - this.label = label; - this.count = count; - } - - @Override - public int compareTo(SampleFacetValue o) { - return Long.compare(this.count, o.count); - } - - @JsonCreator - public static SampleFacetValue build( - @JsonProperty("label") String label, @JsonProperty("count") long count) { - if (label == null || label.trim().length() == 0) { - throw new IllegalArgumentException("label must not be blank"); - } - return new SampleFacetValue(label.trim(), count); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class SampleFacetValue implements Comparable { + public final String label; + public final long count; + + public SampleFacetValue(String label, long count) { + this.label = label; + this.count = count; + } + + @Override + public int compareTo(SampleFacetValue o) { + return Long.compare(this.count, o.count); + } + + @JsonCreator + public static SampleFacetValue build( + @JsonProperty("label") String label, @JsonProperty("count") long count) { + if (label == null || label.trim().length() == 0) { + throw new IllegalArgumentException("label must not be blank"); + } + return new SampleFacetValue(label.trim(), count); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/SampleStatus.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/SampleStatus.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/SampleStatus.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/SampleStatus.java index e51260fbb..8461eb29b 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/SampleStatus.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/SampleStatus.java @@ -1,27 +1,27 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import java.util.Arrays; -import java.util.List; - -public enum SampleStatus { - DRAFT, - PRIVATE, - PUBLIC, - CANCELLED, - SUPPRESSED, - KILLED; - - public static List getSearchHiddenStatuses() { - return Arrays.asList(SUPPRESSED.name(), KILLED.name()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import java.util.Arrays; +import java.util.List; + +public enum SampleStatus { + DRAFT, + PRIVATE, + PUBLIC, + CANCELLED, + SUPPRESSED, + KILLED; + + public static List getSearchHiddenStatuses() { + return Arrays.asList(SUPPRESSED.name(), KILLED.name()); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/SubmissionReceipt.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/SubmissionReceipt.java similarity index 96% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/SubmissionReceipt.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/SubmissionReceipt.java index 409816b58..181e20463 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/SubmissionReceipt.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/SubmissionReceipt.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.model; +package uk.ac.ebi.biosamples.core.model; import java.util.List; import lombok.AllArgsConstructor; diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/SubmittedViaType.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/SubmittedViaType.java similarity index 92% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/SubmittedViaType.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/SubmittedViaType.java index fee0ebd15..f6377bfd1 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/SubmittedViaType.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/SubmittedViaType.java @@ -1,21 +1,21 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -public enum SubmittedViaType { - JSON_API, - FILE_UPLOADER, - PIPELINE_IMPORT, - WEBIN_SERVICES, - FILE_UPLOADER_PLACEHOLDER, - USI, - SAMPLETAB -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +public enum SubmittedViaType { + JSON_API, + FILE_UPLOADER, + PIPELINE_IMPORT, + WEBIN_SERVICES, + FILE_UPLOADER_PLACEHOLDER, + USI, + SAMPLETAB +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/AttributeFacet.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/AttributeFacet.java similarity index 89% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/AttributeFacet.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/AttributeFacet.java index 655c0db48..8909c661f 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/AttributeFacet.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/AttributeFacet.java @@ -1,109 +1,109 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import java.util.Optional; -import org.springframework.hateoas.server.core.Relation; -import uk.ac.ebi.biosamples.model.facet.content.FacetContent; -import uk.ac.ebi.biosamples.model.facet.content.LabelCountListContent; -import uk.ac.ebi.biosamples.model.filter.FilterType; - -@Relation(collectionRelation = "facets") -@JsonDeserialize(builder = AttributeFacet.Builder.class) -public class AttributeFacet implements Facet { - private final String facetLabel; - private final Long facetCount; - private final LabelCountListContent content; - - private AttributeFacet( - final String facetLabel, final Long facetCount, final LabelCountListContent content) { - this.facetLabel = facetLabel; - this.facetCount = facetCount; - this.content = content; - } - - @Override - public FacetType getType() { - return FacetType.ATTRIBUTE_FACET; - } - - @Override - public Optional getAssociatedFilterType() { - return Optional.of(FilterType.ATTRIBUTE_FILTER); - } - - @Override - public String getLabel() { - return facetLabel; - } - - @Override - public Long getCount() { - return facetCount; - } - - @Override - public LabelCountListContent getContent() { - return content; - } - - @Override - public String getContentSerializableFilter(final String label) { - return label; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - sb.append("AttributeFacet("); - sb.append(facetLabel); - sb.append(","); - sb.append(facetCount); - sb.append(","); - sb.append(content); - sb.append(")"); - return sb.toString(); - } - - public static class Builder implements Facet.Builder { - - private final String field; - private final Long count; - private LabelCountListContent content = null; - - @JsonCreator - public Builder( - @JsonProperty("label") final String field, @JsonProperty("count") final Long count) { - this.field = field; - this.count = count; - } - - @JsonProperty - @Override - public Builder withContent(final FacetContent content) { - - if (!(content instanceof LabelCountListContent)) { - throw new RuntimeException("Content not compatible with an attribute facet"); - } - - this.content = (LabelCountListContent) content; - return this; - } - - @Override - public Facet build() { - return new AttributeFacet(field, count, content); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.facet; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.util.Optional; +import org.springframework.hateoas.server.core.Relation; +import uk.ac.ebi.biosamples.core.model.facet.content.FacetContent; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; +import uk.ac.ebi.biosamples.core.model.filter.FilterType; + +@Relation(collectionRelation = "facets") +@JsonDeserialize(builder = AttributeFacet.Builder.class) +public class AttributeFacet implements Facet { + private final String facetLabel; + private final Long facetCount; + private final LabelCountListContent content; + + private AttributeFacet( + final String facetLabel, final Long facetCount, final LabelCountListContent content) { + this.facetLabel = facetLabel; + this.facetCount = facetCount; + this.content = content; + } + + @Override + public FacetType getType() { + return FacetType.ATTRIBUTE_FACET; + } + + @Override + public Optional getAssociatedFilterType() { + return Optional.of(FilterType.ATTRIBUTE_FILTER); + } + + @Override + public String getLabel() { + return facetLabel; + } + + @Override + public Long getCount() { + return facetCount; + } + + @Override + public LabelCountListContent getContent() { + return content; + } + + @Override + public String getContentSerializableFilter(final String label) { + return label; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("AttributeFacet("); + sb.append(facetLabel); + sb.append(","); + sb.append(facetCount); + sb.append(","); + sb.append(content); + sb.append(")"); + return sb.toString(); + } + + public static class Builder implements Facet.Builder { + + private final String field; + private final Long count; + private LabelCountListContent content = null; + + @JsonCreator + public Builder( + @JsonProperty("label") final String field, @JsonProperty("count") final Long count) { + this.field = field; + this.count = count; + } + + @JsonProperty + @Override + public Builder withContent(final FacetContent content) { + + if (!(content instanceof LabelCountListContent)) { + throw new RuntimeException("Content not compatible with an attribute facet"); + } + + this.content = (LabelCountListContent) content; + return this; + } + + @Override + public Facet build() { + return new AttributeFacet(field, count, content); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/DateRangeFacet.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/DateRangeFacet.java similarity index 90% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/DateRangeFacet.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/DateRangeFacet.java index 276ad0017..51fc8aa56 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/DateRangeFacet.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/DateRangeFacet.java @@ -1,133 +1,133 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import org.springframework.hateoas.server.core.Relation; -import uk.ac.ebi.biosamples.model.facet.content.FacetContent; -import uk.ac.ebi.biosamples.model.facet.content.LabelCountEntry; -import uk.ac.ebi.biosamples.model.facet.content.LabelCountListContent; -import uk.ac.ebi.biosamples.model.filter.FilterType; - -@Relation(collectionRelation = "facets") -@JsonDeserialize(builder = DateRangeFacet.Builder.class) -public class DateRangeFacet implements Facet { - private final String facetLabel; - private final Long facetCount; - private final LabelCountListContent content; - - private DateRangeFacet( - final String facetLabel, final Long facetCount, final LabelCountListContent content) { - this.facetLabel = facetLabel; - this.facetCount = facetCount; - this.content = content; - } - - @Override - public FacetType getType() { - return FacetType.DATE_RANGE_FACET; - } - - @Override - public Optional getAssociatedFilterType() { - return Optional.of(FilterType.DATE_FILTER); - } - - @Override - public String getLabel() { - return facetLabel; - } - - @Override - public Long getCount() { - return facetCount; - } - - @Override - public LabelCountListContent getContent() { - return content; - } - - @Override - public String getContentSerializableFilter(final String label) { - final String[] range = label.split(" to "); - return "from=" + range[0] + "until=" + range[1]; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - sb.append("DateRangeFacet("); - sb.append(facetLabel); - sb.append(","); - sb.append(facetCount); - sb.append(","); - sb.append(content); - sb.append(")"); - return sb.toString(); - } - - public static class Builder implements Facet.Builder { - - private final String field; - private final Long count; - private LabelCountListContent content = null; - - @JsonCreator - public Builder( - @JsonProperty("label") final String field, @JsonProperty("count") final Long count) { - this.field = field; - this.count = count; - } - - @JsonProperty - @Override - public Builder withContent(final FacetContent content) { - - if (!(content instanceof LabelCountListContent)) { - throw new RuntimeException("Content not compatible with an attribute facet"); - } - - final LabelCountListContent tempContent = (LabelCountListContent) content; - final List contentList = new ArrayList<>(); - for (int i = 0; i < tempContent.size(); i++) { - final LabelCountEntry entry = tempContent.get(i); - contentList.add( - LabelCountEntry.build(parseLabelToDateRange(entry.getLabel()), entry.getCount())); - } - - this.content = new LabelCountListContent(contentList); - return this; - } - - private String parseLabelToDateRange(final String label) { - final String dateLabel = label.substring(0, 10); - final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - final LocalDate start = LocalDate.parse(dateLabel, formatter); - final LocalDate end = start.plusYears(1); - // LocalDateTime dateTime = LocalDateTime.parse(dateLabel, formatter); - - return start + " to " + end; - } - - @Override - public Facet build() { - return new DateRangeFacet(field, count, content); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.facet; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.springframework.hateoas.server.core.Relation; +import uk.ac.ebi.biosamples.core.model.facet.content.FacetContent; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountEntry; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; +import uk.ac.ebi.biosamples.core.model.filter.FilterType; + +@Relation(collectionRelation = "facets") +@JsonDeserialize(builder = DateRangeFacet.Builder.class) +public class DateRangeFacet implements Facet { + private final String facetLabel; + private final Long facetCount; + private final LabelCountListContent content; + + private DateRangeFacet( + final String facetLabel, final Long facetCount, final LabelCountListContent content) { + this.facetLabel = facetLabel; + this.facetCount = facetCount; + this.content = content; + } + + @Override + public FacetType getType() { + return FacetType.DATE_RANGE_FACET; + } + + @Override + public Optional getAssociatedFilterType() { + return Optional.of(FilterType.DATE_FILTER); + } + + @Override + public String getLabel() { + return facetLabel; + } + + @Override + public Long getCount() { + return facetCount; + } + + @Override + public LabelCountListContent getContent() { + return content; + } + + @Override + public String getContentSerializableFilter(final String label) { + final String[] range = label.split(" to "); + return "from=" + range[0] + "until=" + range[1]; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("DateRangeFacet("); + sb.append(facetLabel); + sb.append(","); + sb.append(facetCount); + sb.append(","); + sb.append(content); + sb.append(")"); + return sb.toString(); + } + + public static class Builder implements Facet.Builder { + + private final String field; + private final Long count; + private LabelCountListContent content = null; + + @JsonCreator + public Builder( + @JsonProperty("label") final String field, @JsonProperty("count") final Long count) { + this.field = field; + this.count = count; + } + + @JsonProperty + @Override + public Builder withContent(final FacetContent content) { + + if (!(content instanceof LabelCountListContent)) { + throw new RuntimeException("Content not compatible with an attribute facet"); + } + + final LabelCountListContent tempContent = (LabelCountListContent) content; + final List contentList = new ArrayList<>(); + for (int i = 0; i < tempContent.size(); i++) { + final LabelCountEntry entry = tempContent.get(i); + contentList.add( + LabelCountEntry.build(parseLabelToDateRange(entry.getLabel()), entry.getCount())); + } + + this.content = new LabelCountListContent(contentList); + return this; + } + + private String parseLabelToDateRange(final String label) { + final String dateLabel = label.substring(0, 10); + final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + final LocalDate start = LocalDate.parse(dateLabel, formatter); + final LocalDate end = start.plusYears(1); + // LocalDateTime dateTime = LocalDateTime.parse(dateLabel, formatter); + + return start + " to " + end; + } + + @Override + public Facet build() { + return new DateRangeFacet(field, count, content); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/ExternalReferenceDataFacet.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/ExternalReferenceDataFacet.java similarity index 89% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/ExternalReferenceDataFacet.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/ExternalReferenceDataFacet.java index 261e58caa..a574cf9aa 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/ExternalReferenceDataFacet.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/ExternalReferenceDataFacet.java @@ -1,107 +1,107 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import java.util.Optional; -import uk.ac.ebi.biosamples.model.facet.content.FacetContent; -import uk.ac.ebi.biosamples.model.facet.content.LabelCountListContent; -import uk.ac.ebi.biosamples.model.filter.FilterType; - -@JsonDeserialize(builder = AttributeFacet.Builder.class) -public class ExternalReferenceDataFacet implements Facet { - private final String facetLabel; - private final Long facetCount; - private final LabelCountListContent content; - - private ExternalReferenceDataFacet( - final String facetLabel, final Long facetCount, final LabelCountListContent content) { - this.facetLabel = facetLabel; - this.facetCount = facetCount; - this.content = content; - } - - @Override - public FacetType getType() { - return FacetType.EXTERNAL_REFERENCE_DATA_FACET; - } - - @Override - public Optional getAssociatedFilterType() { - return Optional.of(FilterType.EXTERNAL_REFERENCE_DATA_FILTER); - } - - @Override - public String getLabel() { - return facetLabel; - } - - @Override - public Long getCount() { - return facetCount; - } - - @Override - public LabelCountListContent getContent() { - return content; - } - - @Override - public String getContentSerializableFilter(final String label) { - return label; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - sb.append("ExternalReferenceDataFacet("); - sb.append(facetLabel); - sb.append(","); - sb.append(facetCount); - sb.append(","); - sb.append(content); - sb.append(")"); - return sb.toString(); - } - - public static class Builder implements Facet.Builder { - - private final String field; - private final Long count; - private LabelCountListContent content = null; - - @JsonCreator - public Builder( - @JsonProperty("label") final String field, @JsonProperty("count") final Long count) { - this.field = field; - this.count = count; - } - - @JsonProperty - @Override - public Builder withContent(final FacetContent content) { - - if (!(content instanceof LabelCountListContent)) { - throw new RuntimeException("Content not compatible with an attribute facet"); - } - - this.content = (LabelCountListContent) content; - return this; - } - - @Override - public Facet build() { - return new ExternalReferenceDataFacet(field, count, content); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.facet; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.util.Optional; +import uk.ac.ebi.biosamples.core.model.facet.content.FacetContent; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; +import uk.ac.ebi.biosamples.core.model.filter.FilterType; + +@JsonDeserialize(builder = AttributeFacet.Builder.class) +public class ExternalReferenceDataFacet implements Facet { + private final String facetLabel; + private final Long facetCount; + private final LabelCountListContent content; + + private ExternalReferenceDataFacet( + final String facetLabel, final Long facetCount, final LabelCountListContent content) { + this.facetLabel = facetLabel; + this.facetCount = facetCount; + this.content = content; + } + + @Override + public FacetType getType() { + return FacetType.EXTERNAL_REFERENCE_DATA_FACET; + } + + @Override + public Optional getAssociatedFilterType() { + return Optional.of(FilterType.EXTERNAL_REFERENCE_DATA_FILTER); + } + + @Override + public String getLabel() { + return facetLabel; + } + + @Override + public Long getCount() { + return facetCount; + } + + @Override + public LabelCountListContent getContent() { + return content; + } + + @Override + public String getContentSerializableFilter(final String label) { + return label; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("ExternalReferenceDataFacet("); + sb.append(facetLabel); + sb.append(","); + sb.append(facetCount); + sb.append(","); + sb.append(content); + sb.append(")"); + return sb.toString(); + } + + public static class Builder implements Facet.Builder { + + private final String field; + private final Long count; + private LabelCountListContent content = null; + + @JsonCreator + public Builder( + @JsonProperty("label") final String field, @JsonProperty("count") final Long count) { + this.field = field; + this.count = count; + } + + @JsonProperty + @Override + public Builder withContent(final FacetContent content) { + + if (!(content instanceof LabelCountListContent)) { + throw new RuntimeException("Content not compatible with an attribute facet"); + } + + this.content = (LabelCountListContent) content; + return this; + } + + @Override + public Facet build() { + return new ExternalReferenceDataFacet(field, count, content); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/Facet.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/Facet.java similarity index 90% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/Facet.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/Facet.java index 764b9b0b5..c1010b428 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/Facet.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/Facet.java @@ -1,66 +1,66 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet; - -import com.fasterxml.jackson.annotation.*; -import java.util.Optional; -import uk.ac.ebi.biosamples.model.facet.content.FacetContent; -import uk.ac.ebi.biosamples.model.filter.FilterType; - -// TODO constant this -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.EXISTING_PROPERTY, - property = "type") -@JsonSubTypes({ - @JsonSubTypes.Type(value = AttributeFacet.class, name = "attribute"), - @JsonSubTypes.Type(value = RelationFacet.class, name = "relation"), - @JsonSubTypes.Type(value = InverseRelationFacet.class, name = "inverse relation"), - @JsonSubTypes.Type(value = ExternalReferenceDataFacet.class, name = "external reference data") -}) -@JsonPropertyOrder(value = {"type", "label", "count", "content"}) -public interface Facet extends Comparable { - - @JsonProperty("type") - public FacetType getType(); - - public String getLabel(); - - public Long getCount(); - - public FacetContent getContent(); - - // @JsonIgnore - // public default Optional getAssociatedFilterType() { - // return FacetFilterFieldType.getFilterForFacet(this.getType()); - // } - - @JsonIgnore - public Optional getAssociatedFilterType(); - - @JsonIgnore - public String getContentSerializableFilter(String label); - - /* - * Builder interface to build Facets - */ - public interface Builder { - Facet build(); - - Builder withContent(FacetContent content); - } - - @Override - default int compareTo(final Facet otherFacet) { - return FacetHelper.compareFacets(getLabel(), otherFacet.getLabel()); - // return Long.compare(this.getCount(), otherFacet.getCount()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.facet; + +import com.fasterxml.jackson.annotation.*; +import java.util.Optional; +import uk.ac.ebi.biosamples.core.model.facet.content.FacetContent; +import uk.ac.ebi.biosamples.core.model.filter.FilterType; + +// TODO constant this +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.EXISTING_PROPERTY, + property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = AttributeFacet.class, name = "attribute"), + @JsonSubTypes.Type(value = RelationFacet.class, name = "relation"), + @JsonSubTypes.Type(value = InverseRelationFacet.class, name = "inverse relation"), + @JsonSubTypes.Type(value = ExternalReferenceDataFacet.class, name = "external reference data") +}) +@JsonPropertyOrder(value = {"type", "label", "count", "content"}) +public interface Facet extends Comparable { + + @JsonProperty("type") + public FacetType getType(); + + public String getLabel(); + + public Long getCount(); + + public FacetContent getContent(); + + // @JsonIgnore + // public default Optional getAssociatedFilterType() { + // return FacetFilterFieldType.getFilterForFacet(this.getType()); + // } + + @JsonIgnore + public Optional getAssociatedFilterType(); + + @JsonIgnore + public String getContentSerializableFilter(String label); + + /* + * Builder interface to build Facets + */ + public interface Builder { + Facet build(); + + Builder withContent(FacetContent content); + } + + @Override + default int compareTo(final Facet otherFacet) { + return FacetHelper.compareFacets(getLabel(), otherFacet.getLabel()); + // return Long.compare(this.getCount(), otherFacet.getCount()); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/FacetHelper.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/FacetHelper.java similarity index 96% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/FacetHelper.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/FacetHelper.java index 9d16ffa6c..e3063c928 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/FacetHelper.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/FacetHelper.java @@ -1,117 +1,117 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet; - -import java.util.ArrayList; -import java.util.List; - -public class FacetHelper { - private FacetHelper() { - // hide constructor - } - - public static final List FACETING_FIELDS; - public static final List - RANGE_FACETING_FIELDS; // we are only supporting date range facets now - public static final List IGNORE_FACETING_FIELDS; - - static { - FACETING_FIELDS = new ArrayList<>(); - FACETING_FIELDS.add("organism"); - FACETING_FIELDS.add("external reference"); - FACETING_FIELDS.add("sex"); - FACETING_FIELDS.add("tissue"); - FACETING_FIELDS.add("strain"); - FACETING_FIELDS.add("organism part"); - FACETING_FIELDS.add("cell type"); - FACETING_FIELDS.add("isolate"); - FACETING_FIELDS.add("sample type"); - FACETING_FIELDS.add("genotype"); - FACETING_FIELDS.add("isolation source"); - FACETING_FIELDS.add("histological type"); - FACETING_FIELDS.add("age"); - FACETING_FIELDS.add("host"); - FACETING_FIELDS.add("latitude and longitude"); - FACETING_FIELDS.add("environmental medium"); - FACETING_FIELDS.add("biomaterial provider"); - FACETING_FIELDS.add("development stage"); - FACETING_FIELDS.add("investigation type"); - FACETING_FIELDS.add("disease state"); - FACETING_FIELDS.add("cell line"); - FACETING_FIELDS.add("treatment"); - FACETING_FIELDS.add("depth"); - FACETING_FIELDS.add("host sex"); - FACETING_FIELDS.add("cultivar"); - FACETING_FIELDS.add("elevation"); - FACETING_FIELDS.add("host disease"); - FACETING_FIELDS.add("developmental stage"); - FACETING_FIELDS.add("disease"); - FACETING_FIELDS.add("host age"); - FACETING_FIELDS.add("phenotype"); - FACETING_FIELDS.add("breed"); - FACETING_FIELDS.add("collection date"); - FACETING_FIELDS.add("geographic location"); - FACETING_FIELDS.add("data use conditions"); - } - - static { - RANGE_FACETING_FIELDS = new ArrayList<>(); - RANGE_FACETING_FIELDS.add("release"); - } - - static { - IGNORE_FACETING_FIELDS = new ArrayList<>(); - IGNORE_FACETING_FIELDS.add("description"); - IGNORE_FACETING_FIELDS.add("title"); - IGNORE_FACETING_FIELDS.add("NCBI submission model"); - IGNORE_FACETING_FIELDS.add("External Id"); - IGNORE_FACETING_FIELDS.add("INSDC last update"); - IGNORE_FACETING_FIELDS.add("INSDC first public"); - IGNORE_FACETING_FIELDS.add("INSDC status"); - IGNORE_FACETING_FIELDS.add("INSDC center name"); - IGNORE_FACETING_FIELDS.add("INSDC center alias"); - IGNORE_FACETING_FIELDS.add("description title"); - IGNORE_FACETING_FIELDS.add("gap subject id"); - IGNORE_FACETING_FIELDS.add("gap accession"); - IGNORE_FACETING_FIELDS.add("gap sample id"); - IGNORE_FACETING_FIELDS.add("external ID"); - IGNORE_FACETING_FIELDS.add("submitted subject id"); - IGNORE_FACETING_FIELDS.add("submitted sample id"); - IGNORE_FACETING_FIELDS.add("submitter handle"); - IGNORE_FACETING_FIELDS.add("biospecimen repository sample id"); - } - - public static String getEncodingSuffix(String attribute) { - String suffix = ""; - if (FACETING_FIELDS.contains(attribute)) { - suffix = "_av_ss"; - } else if (RANGE_FACETING_FIELDS.contains(attribute)) { - suffix = "_dt"; - } - return suffix; - } - - public static String getEncodingSuffixForFacetingFields() { - return "_av_ss"; - } - - public static int compareFacets(String f1, String f2) { - if (!FACETING_FIELDS.contains(f1) && !FACETING_FIELDS.contains(f2)) { - return 0; - } else if (!FACETING_FIELDS.contains(f1)) { - return -1; - } else if (!FACETING_FIELDS.contains(f2)) { - return 1; - } - - return FACETING_FIELDS.indexOf(f2) - FACETING_FIELDS.indexOf(f1); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.facet; + +import java.util.ArrayList; +import java.util.List; + +public class FacetHelper { + private FacetHelper() { + // hide constructor + } + + public static final List FACETING_FIELDS; + public static final List + RANGE_FACETING_FIELDS; // we are only supporting date range facets now + public static final List IGNORE_FACETING_FIELDS; + + static { + FACETING_FIELDS = new ArrayList<>(); + FACETING_FIELDS.add("organism"); + FACETING_FIELDS.add("external reference"); + FACETING_FIELDS.add("sex"); + FACETING_FIELDS.add("tissue"); + FACETING_FIELDS.add("strain"); + FACETING_FIELDS.add("organism part"); + FACETING_FIELDS.add("cell type"); + FACETING_FIELDS.add("isolate"); + FACETING_FIELDS.add("sample type"); + FACETING_FIELDS.add("genotype"); + FACETING_FIELDS.add("isolation source"); + FACETING_FIELDS.add("histological type"); + FACETING_FIELDS.add("age"); + FACETING_FIELDS.add("host"); + FACETING_FIELDS.add("latitude and longitude"); + FACETING_FIELDS.add("environmental medium"); + FACETING_FIELDS.add("biomaterial provider"); + FACETING_FIELDS.add("development stage"); + FACETING_FIELDS.add("investigation type"); + FACETING_FIELDS.add("disease state"); + FACETING_FIELDS.add("cell line"); + FACETING_FIELDS.add("treatment"); + FACETING_FIELDS.add("depth"); + FACETING_FIELDS.add("host sex"); + FACETING_FIELDS.add("cultivar"); + FACETING_FIELDS.add("elevation"); + FACETING_FIELDS.add("host disease"); + FACETING_FIELDS.add("developmental stage"); + FACETING_FIELDS.add("disease"); + FACETING_FIELDS.add("host age"); + FACETING_FIELDS.add("phenotype"); + FACETING_FIELDS.add("breed"); + FACETING_FIELDS.add("collection date"); + FACETING_FIELDS.add("geographic location"); + FACETING_FIELDS.add("data use conditions"); + } + + static { + RANGE_FACETING_FIELDS = new ArrayList<>(); + RANGE_FACETING_FIELDS.add("release"); + } + + static { + IGNORE_FACETING_FIELDS = new ArrayList<>(); + IGNORE_FACETING_FIELDS.add("description"); + IGNORE_FACETING_FIELDS.add("title"); + IGNORE_FACETING_FIELDS.add("NCBI submission model"); + IGNORE_FACETING_FIELDS.add("External Id"); + IGNORE_FACETING_FIELDS.add("INSDC last update"); + IGNORE_FACETING_FIELDS.add("INSDC first public"); + IGNORE_FACETING_FIELDS.add("INSDC status"); + IGNORE_FACETING_FIELDS.add("INSDC center name"); + IGNORE_FACETING_FIELDS.add("INSDC center alias"); + IGNORE_FACETING_FIELDS.add("description title"); + IGNORE_FACETING_FIELDS.add("gap subject id"); + IGNORE_FACETING_FIELDS.add("gap accession"); + IGNORE_FACETING_FIELDS.add("gap sample id"); + IGNORE_FACETING_FIELDS.add("external ID"); + IGNORE_FACETING_FIELDS.add("submitted subject id"); + IGNORE_FACETING_FIELDS.add("submitted sample id"); + IGNORE_FACETING_FIELDS.add("submitter handle"); + IGNORE_FACETING_FIELDS.add("biospecimen repository sample id"); + } + + public static String getEncodingSuffix(String attribute) { + String suffix = ""; + if (FACETING_FIELDS.contains(attribute)) { + suffix = "_av_ss"; + } else if (RANGE_FACETING_FIELDS.contains(attribute)) { + suffix = "_dt"; + } + return suffix; + } + + public static String getEncodingSuffixForFacetingFields() { + return "_av_ss"; + } + + public static int compareFacets(String f1, String f2) { + if (!FACETING_FIELDS.contains(f1) && !FACETING_FIELDS.contains(f2)) { + return 0; + } else if (!FACETING_FIELDS.contains(f1)) { + return -1; + } else if (!FACETING_FIELDS.contains(f2)) { + return 1; + } + + return FACETING_FIELDS.indexOf(f2) - FACETING_FIELDS.indexOf(f1); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/FacetType.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/FacetType.java similarity index 95% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/FacetType.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/FacetType.java index 9e050d819..45c2ad4e2 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/FacetType.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/FacetType.java @@ -1,50 +1,50 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet; - -import com.fasterxml.jackson.annotation.JsonValue; -import java.lang.reflect.InvocationTargetException; - -public enum FacetType { - ATTRIBUTE_FACET("attribute", AttributeFacet.Builder.class), - DATE_RANGE_FACET("date range", DateRangeFacet.Builder.class), - INVERSE_RELATION_FACET("inverse relation", InverseRelationFacet.Builder.class), - RELATION_FACET("relation", RelationFacet.Builder.class), - EXTERNAL_REFERENCE_DATA_FACET( - "external reference data", ExternalReferenceDataFacet.Builder.class), - NO_TYPE("", null); - - private final String name; - private final Class associatedClass; - - FacetType(final String name, final Class associatedClass) { - this.name = name; - this.associatedClass = associatedClass; - } - - public Facet.Builder getBuilderForLabelAndCount(final String facetLabel, final Long facetCount) { - try { - return associatedClass - .getConstructor(String.class, Long.class) - .newInstance(facetLabel, facetCount); - } catch (final NoSuchMethodException - | IllegalAccessException - | InstantiationException - | InvocationTargetException e) { - throw new RuntimeException("FacetType " + this + " does not provide a proper builder"); - } - } - - @JsonValue - public String getFacetName() { - return name.toLowerCase(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.facet; + +import com.fasterxml.jackson.annotation.JsonValue; +import java.lang.reflect.InvocationTargetException; + +public enum FacetType { + ATTRIBUTE_FACET("attribute", AttributeFacet.Builder.class), + DATE_RANGE_FACET("date range", DateRangeFacet.Builder.class), + INVERSE_RELATION_FACET("inverse relation", InverseRelationFacet.Builder.class), + RELATION_FACET("relation", RelationFacet.Builder.class), + EXTERNAL_REFERENCE_DATA_FACET( + "external reference data", ExternalReferenceDataFacet.Builder.class), + NO_TYPE("", null); + + private final String name; + private final Class associatedClass; + + FacetType(final String name, final Class associatedClass) { + this.name = name; + this.associatedClass = associatedClass; + } + + public Facet.Builder getBuilderForLabelAndCount(final String facetLabel, final Long facetCount) { + try { + return associatedClass + .getConstructor(String.class, Long.class) + .newInstance(facetLabel, facetCount); + } catch (final NoSuchMethodException + | IllegalAccessException + | InstantiationException + | InvocationTargetException e) { + throw new RuntimeException("FacetType " + this + " does not provide a proper builder"); + } + } + + @JsonValue + public String getFacetName() { + return name.toLowerCase(); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/InverseRelationFacet.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/InverseRelationFacet.java similarity index 89% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/InverseRelationFacet.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/InverseRelationFacet.java index c4637aa12..778739b40 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/InverseRelationFacet.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/InverseRelationFacet.java @@ -1,108 +1,108 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import java.util.Optional; -import org.springframework.hateoas.server.core.Relation; -import uk.ac.ebi.biosamples.model.facet.content.FacetContent; -import uk.ac.ebi.biosamples.model.facet.content.LabelCountListContent; -import uk.ac.ebi.biosamples.model.filter.FilterType; - -@Relation(collectionRelation = "facets") -@JsonDeserialize(builder = InverseRelationFacet.Builder.class) -public class InverseRelationFacet implements Facet { - private final String facetLabel; - private final Long facetCount; - private final LabelCountListContent content; - - private InverseRelationFacet( - final String facetLabel, final Long facetCount, final LabelCountListContent content) { - this.facetLabel = facetLabel; - this.facetCount = facetCount; - this.content = content; - } - - @Override - public FacetType getType() { - return FacetType.INVERSE_RELATION_FACET; - } - - @Override - public Optional getAssociatedFilterType() { - return Optional.of(FilterType.INVERSE_RELATION_FILTER); - } - - @Override - public String getLabel() { - return facetLabel; - } - - @Override - public Long getCount() { - return facetCount; - } - - @Override - public LabelCountListContent getContent() { - return content; - } - - @Override - public String getContentSerializableFilter(final String label) { - return label; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - sb.append("InverseRelationFacet("); - sb.append(facetLabel); - sb.append(","); - sb.append(facetCount); - sb.append(","); - sb.append(content); - sb.append(")"); - return sb.toString(); - } - - public static class Builder implements Facet.Builder { - - private final String label; - private final Long count; - private LabelCountListContent content = null; - - @JsonCreator - public Builder( - @JsonProperty("label") final String label, @JsonProperty("count") final Long count) { - this.label = label; - this.count = count; - } - - @JsonProperty - @Override - public Builder withContent(final FacetContent content) { - if (!(content instanceof LabelCountListContent)) { - throw new RuntimeException("Content not compatible with an inverse relation facet"); - } - - this.content = (LabelCountListContent) content; - return this; - } - - @Override - public Facet build() { - return new InverseRelationFacet(label, count, content); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.facet; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.util.Optional; +import org.springframework.hateoas.server.core.Relation; +import uk.ac.ebi.biosamples.core.model.facet.content.FacetContent; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; +import uk.ac.ebi.biosamples.core.model.filter.FilterType; + +@Relation(collectionRelation = "facets") +@JsonDeserialize(builder = InverseRelationFacet.Builder.class) +public class InverseRelationFacet implements Facet { + private final String facetLabel; + private final Long facetCount; + private final LabelCountListContent content; + + private InverseRelationFacet( + final String facetLabel, final Long facetCount, final LabelCountListContent content) { + this.facetLabel = facetLabel; + this.facetCount = facetCount; + this.content = content; + } + + @Override + public FacetType getType() { + return FacetType.INVERSE_RELATION_FACET; + } + + @Override + public Optional getAssociatedFilterType() { + return Optional.of(FilterType.INVERSE_RELATION_FILTER); + } + + @Override + public String getLabel() { + return facetLabel; + } + + @Override + public Long getCount() { + return facetCount; + } + + @Override + public LabelCountListContent getContent() { + return content; + } + + @Override + public String getContentSerializableFilter(final String label) { + return label; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("InverseRelationFacet("); + sb.append(facetLabel); + sb.append(","); + sb.append(facetCount); + sb.append(","); + sb.append(content); + sb.append(")"); + return sb.toString(); + } + + public static class Builder implements Facet.Builder { + + private final String label; + private final Long count; + private LabelCountListContent content = null; + + @JsonCreator + public Builder( + @JsonProperty("label") final String label, @JsonProperty("count") final Long count) { + this.label = label; + this.count = count; + } + + @JsonProperty + @Override + public Builder withContent(final FacetContent content) { + if (!(content instanceof LabelCountListContent)) { + throw new RuntimeException("Content not compatible with an inverse relation facet"); + } + + this.content = (LabelCountListContent) content; + return this; + } + + @Override + public Facet build() { + return new InverseRelationFacet(label, count, content); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/RelationFacet.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/RelationFacet.java similarity index 89% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/RelationFacet.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/RelationFacet.java index 0227fe1a8..fce4be5ac 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/RelationFacet.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/RelationFacet.java @@ -1,108 +1,108 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import java.util.Optional; -import org.springframework.hateoas.server.core.Relation; -import uk.ac.ebi.biosamples.model.facet.content.FacetContent; -import uk.ac.ebi.biosamples.model.facet.content.LabelCountListContent; -import uk.ac.ebi.biosamples.model.filter.FilterType; - -@Relation(collectionRelation = "facets") -@JsonDeserialize(builder = RelationFacet.Builder.class) -public class RelationFacet implements Facet { - private final String facetLabel; - private final Long facetCount; - private final LabelCountListContent content; - - private RelationFacet( - final String facetLabel, final Long facetCount, final LabelCountListContent content) { - this.facetLabel = facetLabel; - this.facetCount = facetCount; - this.content = content; - } - - @Override - public FacetType getType() { - return FacetType.RELATION_FACET; - } - - @Override - public Optional getAssociatedFilterType() { - return Optional.of(FilterType.RELATION_FILER); - } - - @Override - public String getLabel() { - return facetLabel; - } - - @Override - public Long getCount() { - return facetCount; - } - - @Override - public LabelCountListContent getContent() { - return content; - } - - @Override - public String getContentSerializableFilter(final String label) { - return label; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - sb.append("RelationFacet("); - sb.append(facetLabel); - sb.append(","); - sb.append(facetCount); - sb.append(","); - sb.append(content); - sb.append(")"); - return sb.toString(); - } - - public static class Builder implements Facet.Builder { - - private final String label; - private final Long count; - private LabelCountListContent content = null; - - @JsonCreator - public Builder( - @JsonProperty("label") final String label, @JsonProperty("count") final Long count) { - this.label = label; - this.count = count; - } - - @JsonProperty - @Override - public Builder withContent(final FacetContent content) { - if (!(content instanceof LabelCountListContent)) { - throw new RuntimeException("Content not compatible with a relation facet"); - } - - this.content = (LabelCountListContent) content; - return this; - } - - @Override - public Facet build() { - return new RelationFacet(label, count, content); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.facet; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.util.Optional; +import org.springframework.hateoas.server.core.Relation; +import uk.ac.ebi.biosamples.core.model.facet.content.FacetContent; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; +import uk.ac.ebi.biosamples.core.model.filter.FilterType; + +@Relation(collectionRelation = "facets") +@JsonDeserialize(builder = RelationFacet.Builder.class) +public class RelationFacet implements Facet { + private final String facetLabel; + private final Long facetCount; + private final LabelCountListContent content; + + private RelationFacet( + final String facetLabel, final Long facetCount, final LabelCountListContent content) { + this.facetLabel = facetLabel; + this.facetCount = facetCount; + this.content = content; + } + + @Override + public FacetType getType() { + return FacetType.RELATION_FACET; + } + + @Override + public Optional getAssociatedFilterType() { + return Optional.of(FilterType.RELATION_FILER); + } + + @Override + public String getLabel() { + return facetLabel; + } + + @Override + public Long getCount() { + return facetCount; + } + + @Override + public LabelCountListContent getContent() { + return content; + } + + @Override + public String getContentSerializableFilter(final String label) { + return label; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("RelationFacet("); + sb.append(facetLabel); + sb.append(","); + sb.append(facetCount); + sb.append(","); + sb.append(content); + sb.append(")"); + return sb.toString(); + } + + public static class Builder implements Facet.Builder { + + private final String label; + private final Long count; + private LabelCountListContent content = null; + + @JsonCreator + public Builder( + @JsonProperty("label") final String label, @JsonProperty("count") final Long count) { + this.label = label; + this.count = count; + } + + @JsonProperty + @Override + public Builder withContent(final FacetContent content) { + if (!(content instanceof LabelCountListContent)) { + throw new RuntimeException("Content not compatible with a relation facet"); + } + + this.content = (LabelCountListContent) content; + return this; + } + + @Override + public Facet build() { + return new RelationFacet(label, count, content); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/FacetContent.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/FacetContent.java similarity index 84% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/FacetContent.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/FacetContent.java index 0d4514098..fa00904d5 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/FacetContent.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/FacetContent.java @@ -1,16 +1,17 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet.content; - -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; - -@JsonDeserialize(using = FacetContentDeserializer.class) -public interface FacetContent {} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.facet.content; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import uk.ac.ebi.biosamples.core.service.facet.content.FacetContentDeserializer; + +@JsonDeserialize(using = FacetContentDeserializer.class) +public interface FacetContent {} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/LabelCountEntry.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/LabelCountEntry.java similarity index 94% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/LabelCountEntry.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/LabelCountEntry.java index 2d0c9362d..04936275d 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/LabelCountEntry.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/LabelCountEntry.java @@ -1,74 +1,74 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet.content; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Arrays; -import java.util.Map; - -public class LabelCountEntry implements Comparable { - private final String label; - private final long count; - - private LabelCountEntry(final String label, final long count) { - this.label = label; - this.count = count; - } - - public String getLabel() { - return label; - } - - public long getCount() { - return count; - } - - @Override - public int compareTo(final LabelCountEntry o) { - return Long.compare(count, o.count); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - sb.append("LabelCountEntry("); - sb.append(label); - sb.append(","); - sb.append(count); - sb.append(")"); - return sb.toString(); - } - - @JsonCreator - public static LabelCountEntry build( - @JsonProperty("label") final String label, @JsonProperty("count") final long count) { - if (label == null || label.trim().isEmpty()) { - throw new IllegalArgumentException("label must not be blank"); - } - - return new LabelCountEntry(label.trim(), count); - } - - @JsonCreator - public static LabelCountEntry build(final Map entryMap) { - if (isValidLabelCount(entryMap)) { - return new LabelCountEntry(entryMap.get("label"), Long.parseLong(entryMap.get("count"))); - } - - throw new RuntimeException( - "Provided object is not suitable to be converted to LabelCountEntry"); - } - - private static Boolean isValidLabelCount(final Map content) { - return content.keySet().containsAll(Arrays.asList("label", "count")); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.facet.content; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Arrays; +import java.util.Map; + +public class LabelCountEntry implements Comparable { + private final String label; + private final long count; + + private LabelCountEntry(final String label, final long count) { + this.label = label; + this.count = count; + } + + public String getLabel() { + return label; + } + + public long getCount() { + return count; + } + + @Override + public int compareTo(final LabelCountEntry o) { + return Long.compare(count, o.count); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("LabelCountEntry("); + sb.append(label); + sb.append(","); + sb.append(count); + sb.append(")"); + return sb.toString(); + } + + @JsonCreator + public static LabelCountEntry build( + @JsonProperty("label") final String label, @JsonProperty("count") final long count) { + if (label == null || label.trim().isEmpty()) { + throw new IllegalArgumentException("label must not be blank"); + } + + return new LabelCountEntry(label.trim(), count); + } + + @JsonCreator + public static LabelCountEntry build(final Map entryMap) { + if (isValidLabelCount(entryMap)) { + return new LabelCountEntry(entryMap.get("label"), Long.parseLong(entryMap.get("count"))); + } + + throw new RuntimeException( + "Provided object is not suitable to be converted to LabelCountEntry"); + } + + private static Boolean isValidLabelCount(final Map content) { + return content.keySet().containsAll(Arrays.asList("label", "count")); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/LabelCountListContent.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/LabelCountListContent.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/LabelCountListContent.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/LabelCountListContent.java index 002c8fb9d..9a07fc2b5 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/LabelCountListContent.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/LabelCountListContent.java @@ -1,45 +1,45 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet.content; - -import com.fasterxml.jackson.annotation.JsonCreator; -import java.util.AbstractList; -import java.util.List; - -public class LabelCountListContent extends AbstractList implements FacetContent { - private final List labelCountEntryList; - - @JsonCreator - public LabelCountListContent(final List labelCountEntryList) { - this.labelCountEntryList = labelCountEntryList; - } - - @Override - public LabelCountEntry get(final int index) { - return labelCountEntryList.get(index); - } - - @Override - public int size() { - return labelCountEntryList.size(); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - - sb.append("LabelCountListContent("); - sb.append(labelCountEntryList); - sb.append(")"); - - return sb.toString(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.facet.content; + +import com.fasterxml.jackson.annotation.JsonCreator; +import java.util.AbstractList; +import java.util.List; + +public class LabelCountListContent extends AbstractList implements FacetContent { + private final List labelCountEntryList; + + @JsonCreator + public LabelCountListContent(final List labelCountEntryList) { + this.labelCountEntryList = labelCountEntryList; + } + + @Override + public LabelCountEntry get(final int index) { + return labelCountEntryList.get(index); + } + + @Override + public int size() { + return labelCountEntryList.size(); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + + sb.append("LabelCountListContent("); + sb.append(labelCountEntryList); + sb.append(")"); + + return sb.toString(); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/RangeCountEntry.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/RangeCountEntry.java similarity index 95% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/RangeCountEntry.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/RangeCountEntry.java index 74e52e414..8cc468781 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/RangeCountEntry.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/RangeCountEntry.java @@ -1,93 +1,93 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet.content; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Arrays; -import java.util.Map; - -public class RangeCountEntry implements Comparable { - private final String startLabel; - private final String endLabel; - private final long count; - - private RangeCountEntry(final String startLabel, final String endLabel, final long count) { - this.startLabel = startLabel; - this.endLabel = endLabel; - this.count = count; - } - - public String getStartLabel() { - return startLabel; - } - - public String getEndLabel() { - return endLabel; - } - - public long getCount() { - return count; - } - - public String getLabel() { - return startLabel + " TO " + endLabel; - } - - @Override - public int compareTo(final RangeCountEntry o) { - return Long.compare(count, o.count); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - - sb.append("LabelCountEntry("); - sb.append(getLabel()); - sb.append(","); - sb.append(count); - sb.append(")"); - - return sb.toString(); - } - - @JsonCreator - public static RangeCountEntry build( - @JsonProperty("label") final String startLabel, - @JsonProperty("label") final String endLabel, - @JsonProperty("count") final long count) { - if (startLabel == null - || startLabel.trim().isEmpty() - || endLabel == null - || endLabel.trim().isEmpty()) { - throw new IllegalArgumentException("start/end label must not be blank"); - } - - return new RangeCountEntry(startLabel.trim(), endLabel.trim(), count); - } - - @JsonCreator - public static RangeCountEntry build(final Map entryMap) { - if (isValidLabelCount(entryMap)) { - return new RangeCountEntry( - entryMap.get("startLabel"), - entryMap.get("endLabel"), - Long.parseLong(entryMap.get("count"))); - } - throw new RuntimeException( - "Provided object is not suitable to be converted to RangeCountEntry"); - } - - private static boolean isValidLabelCount(final Map content) { - return content.keySet().containsAll(Arrays.asList("startLabel", "endLabel", "count")); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.facet.content; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Arrays; +import java.util.Map; + +public class RangeCountEntry implements Comparable { + private final String startLabel; + private final String endLabel; + private final long count; + + private RangeCountEntry(final String startLabel, final String endLabel, final long count) { + this.startLabel = startLabel; + this.endLabel = endLabel; + this.count = count; + } + + public String getStartLabel() { + return startLabel; + } + + public String getEndLabel() { + return endLabel; + } + + public long getCount() { + return count; + } + + public String getLabel() { + return startLabel + " TO " + endLabel; + } + + @Override + public int compareTo(final RangeCountEntry o) { + return Long.compare(count, o.count); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + + sb.append("LabelCountEntry("); + sb.append(getLabel()); + sb.append(","); + sb.append(count); + sb.append(")"); + + return sb.toString(); + } + + @JsonCreator + public static RangeCountEntry build( + @JsonProperty("label") final String startLabel, + @JsonProperty("label") final String endLabel, + @JsonProperty("count") final long count) { + if (startLabel == null + || startLabel.trim().isEmpty() + || endLabel == null + || endLabel.trim().isEmpty()) { + throw new IllegalArgumentException("start/end label must not be blank"); + } + + return new RangeCountEntry(startLabel.trim(), endLabel.trim(), count); + } + + @JsonCreator + public static RangeCountEntry build(final Map entryMap) { + if (isValidLabelCount(entryMap)) { + return new RangeCountEntry( + entryMap.get("startLabel"), + entryMap.get("endLabel"), + Long.parseLong(entryMap.get("count"))); + } + throw new RuntimeException( + "Provided object is not suitable to be converted to RangeCountEntry"); + } + + private static boolean isValidLabelCount(final Map content) { + return content.keySet().containsAll(Arrays.asList("startLabel", "endLabel", "count")); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/RangeCountListContent.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/RangeCountListContent.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/RangeCountListContent.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/RangeCountListContent.java index a9cbc5552..ac73d9e90 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/RangeCountListContent.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/facet/content/RangeCountListContent.java @@ -1,45 +1,45 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet.content; - -import com.fasterxml.jackson.annotation.JsonCreator; -import java.util.AbstractList; -import java.util.List; - -public class RangeCountListContent extends AbstractList implements FacetContent { - private final List rangeCountEntryList; - - @JsonCreator - public RangeCountListContent(final List rangeCountEntryList) { - this.rangeCountEntryList = rangeCountEntryList; - } - - @Override - public RangeCountEntry get(final int index) { - return rangeCountEntryList.get(index); - } - - @Override - public int size() { - return rangeCountEntryList.size(); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - - sb.append("RangeCountListContent("); - sb.append(rangeCountEntryList); - sb.append(")"); - - return sb.toString(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.facet.content; + +import com.fasterxml.jackson.annotation.JsonCreator; +import java.util.AbstractList; +import java.util.List; + +public class RangeCountListContent extends AbstractList implements FacetContent { + private final List rangeCountEntryList; + + @JsonCreator + public RangeCountListContent(final List rangeCountEntryList) { + this.rangeCountEntryList = rangeCountEntryList; + } + + @Override + public RangeCountEntry get(final int index) { + return rangeCountEntryList.get(index); + } + + @Override + public int size() { + return rangeCountEntryList.size(); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + + sb.append("RangeCountListContent("); + sb.append(rangeCountEntryList); + sb.append(")"); + + return sb.toString(); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/AccessionFilter.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/AccessionFilter.java similarity index 92% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/AccessionFilter.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/AccessionFilter.java index 3339e1570..97e18b2c2 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/AccessionFilter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/AccessionFilter.java @@ -1,85 +1,85 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.filter; - -import java.util.Objects; -import java.util.Optional; -import uk.ac.ebi.biosamples.model.facet.FacetType; - -public class AccessionFilter implements Filter { - - private final String accessionPattern; - - private AccessionFilter(final String accessionPattern) { - this.accessionPattern = accessionPattern; - } - - @Override - public FilterType getType() { - return FilterType.ACCESSION_FILTER; - } - - @Override - public String getLabel() { - return "id"; - } - - @Override - public Optional getContent() { - return Optional.of(accessionPattern); - } - - @Override - public FacetType getAssociatedFacetType() { - return FacetType.NO_TYPE; - } - - @Override - public String getSerialization() { - return getType().getSerialization() + ":" + accessionPattern; - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof AccessionFilter)) { - return false; - } - final AccessionFilter other = (AccessionFilter) obj; - return Objects.equals(other.getContent().orElse(null), getContent().orElse(null)); - } - - @Override - public int hashCode() { - return Objects.hash(getContent().orElse(null)); - } - - public static class Builder implements Filter.Builder { - - private final String pattern; - - public Builder(final String pattern) { - this.pattern = pattern; - } - - @Override - public Filter build() { - return new AccessionFilter(pattern); - } - - @Override - public Filter.Builder parseContent(final String filterSerialized) { - return new Builder(filterSerialized); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.filter; + +import java.util.Objects; +import java.util.Optional; +import uk.ac.ebi.biosamples.core.model.facet.FacetType; + +public class AccessionFilter implements Filter { + + private final String accessionPattern; + + private AccessionFilter(final String accessionPattern) { + this.accessionPattern = accessionPattern; + } + + @Override + public FilterType getType() { + return FilterType.ACCESSION_FILTER; + } + + @Override + public String getLabel() { + return "id"; + } + + @Override + public Optional getContent() { + return Optional.of(accessionPattern); + } + + @Override + public FacetType getAssociatedFacetType() { + return FacetType.NO_TYPE; + } + + @Override + public String getSerialization() { + return getType().getSerialization() + ":" + accessionPattern; + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof AccessionFilter)) { + return false; + } + final AccessionFilter other = (AccessionFilter) obj; + return Objects.equals(other.getContent().orElse(null), getContent().orElse(null)); + } + + @Override + public int hashCode() { + return Objects.hash(getContent().orElse(null)); + } + + public static class Builder implements Filter.Builder { + + private final String pattern; + + public Builder(final String pattern) { + this.pattern = pattern; + } + + @Override + public Filter build() { + return new AccessionFilter(pattern); + } + + @Override + public Filter.Builder parseContent(final String filterSerialized) { + return new Builder(filterSerialized); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/AttributeFilter.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/AttributeFilter.java similarity index 92% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/AttributeFilter.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/AttributeFilter.java index 53e5786aa..a2998928b 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/AttributeFilter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/AttributeFilter.java @@ -1,95 +1,95 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.filter; - -import java.util.Objects; -import java.util.Optional; -import uk.ac.ebi.biosamples.model.facet.FacetType; - -public class AttributeFilter implements Filter { - - private final String label; - private final String value; - - private AttributeFilter(final String label, final String value) { - this.label = label; - this.value = value; - } - - @Override - public FilterType getType() { - return FilterType.ATTRIBUTE_FILTER; - } - - @Override - public String getLabel() { - return label; - } - - @Override - public Optional getContent() { - return Optional.ofNullable(value); - } - - @Override - public FacetType getAssociatedFacetType() { - return FacetType.ATTRIBUTE_FACET; - } - - @Override - public String getSerialization() { - final StringBuilder serialization = - new StringBuilder(getType().getSerialization()).append(":").append(label); - getContent().ifPresent(value -> serialization.append(":").append(value)); - return serialization.toString(); - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof AttributeFilter)) { - return false; - } - final AttributeFilter other = (AttributeFilter) obj; - return Objects.equals(other.label, label) && Objects.equals(other.value, value); - } - - @Override - public int hashCode() { - return Objects.hash(label, value); - } - - public static class Builder implements Filter.Builder { - private String value; - private final String label; - - public Builder(final String label) { - this.label = label; - } - - public Builder withValue(final String value) { - this.value = value; - return this; - } - - @Override - public AttributeFilter build() { - return new AttributeFilter(label, value); - } - - @Override - public Builder parseContent(final String filterValue) { - return withValue(filterValue); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.filter; + +import java.util.Objects; +import java.util.Optional; +import uk.ac.ebi.biosamples.core.model.facet.FacetType; + +public class AttributeFilter implements Filter { + + private final String label; + private final String value; + + private AttributeFilter(final String label, final String value) { + this.label = label; + this.value = value; + } + + @Override + public FilterType getType() { + return FilterType.ATTRIBUTE_FILTER; + } + + @Override + public String getLabel() { + return label; + } + + @Override + public Optional getContent() { + return Optional.ofNullable(value); + } + + @Override + public FacetType getAssociatedFacetType() { + return FacetType.ATTRIBUTE_FACET; + } + + @Override + public String getSerialization() { + final StringBuilder serialization = + new StringBuilder(getType().getSerialization()).append(":").append(label); + getContent().ifPresent(value -> serialization.append(":").append(value)); + return serialization.toString(); + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof AttributeFilter)) { + return false; + } + final AttributeFilter other = (AttributeFilter) obj; + return Objects.equals(other.label, label) && Objects.equals(other.value, value); + } + + @Override + public int hashCode() { + return Objects.hash(label, value); + } + + public static class Builder implements Filter.Builder { + private String value; + private final String label; + + public Builder(final String label) { + this.label = label; + } + + public Builder withValue(final String value) { + this.value = value; + return this; + } + + @Override + public AttributeFilter build() { + return new AttributeFilter(label, value); + } + + @Override + public Builder parseContent(final String filterValue) { + return withValue(filterValue); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/AuthenticationFilter.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/AuthenticationFilter.java similarity index 95% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/AuthenticationFilter.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/AuthenticationFilter.java index 51bbfec8c..507a9adfb 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/AuthenticationFilter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/AuthenticationFilter.java @@ -8,11 +8,11 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.model.filter; +package uk.ac.ebi.biosamples.core.model.filter; import java.util.Objects; import java.util.Optional; -import uk.ac.ebi.biosamples.model.facet.FacetType; +import uk.ac.ebi.biosamples.core.model.facet.FacetType; public class AuthenticationFilter implements Filter { private final String authInfo; diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/DateRangeFilter.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/DateRangeFilter.java similarity index 95% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/DateRangeFilter.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/DateRangeFilter.java index fd85a5d2f..f9a7aa620 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/DateRangeFilter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/DateRangeFilter.java @@ -1,271 +1,271 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.filter; - -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.Objects; -import java.util.Optional; -import uk.ac.ebi.biosamples.model.facet.FacetType; - -public class DateRangeFilter implements Filter { - - private final Optional dateRange; - private final String label; - - DateRangeFilter(final String label, final DateRange dateRange) { - this.label = label; - this.dateRange = Optional.ofNullable(dateRange); - } - - @Override - public FilterType getType() { - return FilterType.DATE_FILTER; - } - - @Override - public String getLabel() { - return label; - } - - @Override - public FacetType getAssociatedFacetType() { - return FacetType.NO_TYPE; - } - - @Override - public Optional getContent() { - return dateRange; - } - - @Override - public String getSerialization() { - final StringBuilder serializationBuilder = - new StringBuilder(getType().getSerialization()).append(":").append(getLabel()); - - getContent() - .ifPresent( - dateRange -> { - serializationBuilder.append(":"); - - if (!dateRange.isFromMinDate()) { - serializationBuilder - .append("from=") - .append( - DateTimeFormatter.ISO_LOCAL_DATE_TIME.format( - LocalDateTime.ofInstant(dateRange.getFrom(), ZoneOffset.UTC))); - } - - if (!dateRange.isUntilMaxDate()) { - serializationBuilder - .append("until=") - .append( - DateTimeFormatter.ISO_LOCAL_DATE_TIME.format( - LocalDateTime.ofInstant(dateRange.getUntil(), ZoneOffset.UTC))); - } - }); - return serializationBuilder.toString(); - } - - @Override - public int hashCode() { - return Objects.hash(getLabel(), getContent()); - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof DateRangeFilter)) { - return false; - } - final DateRangeFilter other = (DateRangeFilter) obj; - return Objects.equals(getLabel(), other.getLabel()) - && Objects.equals(getContent(), other.getContent()); - } - - public static class DateRangeFilterBuilder implements Filter.Builder { - private final String label; - - private Instant from = null; - private Instant until = null; - - public DateRangeFilterBuilder(final String label) { - this.label = label; - } - - public DateRangeFilterBuilder from(final Instant from) { - this.from = from; - return this; - } - - public DateRangeFilterBuilder from(final String value) { - // quick exit - if (value == null || value.trim().length() == 0) { - return this; - } - - try { - from = - LocalDateTime.parse(value, DateTimeFormatter.ISO_LOCAL_DATE_TIME) - .toInstant(ZoneOffset.UTC); - } catch (final DateTimeParseException e) { - // retry for just the date - from = - LocalDate.parse(value, DateTimeFormatter.ISO_LOCAL_DATE) - .atStartOfDay() - .toInstant(ZoneOffset.UTC); - } - return this; - } - - public DateRangeFilterBuilder until(final Instant until) { - this.until = until; - return this; - } - - public DateRangeFilterBuilder until(final String value) { - // quick exit - if (value == null || value.trim().length() == 0) { - return this; - } - - try { - until = - LocalDateTime.parse(value, DateTimeFormatter.ISO_LOCAL_DATE_TIME) - .toInstant(ZoneOffset.UTC); - } catch (final DateTimeParseException e) { - // retry for just the date - until = - LocalDate.parse(value, DateTimeFormatter.ISO_LOCAL_DATE) - .plusDays(1) - .atStartOfDay() - .toInstant(ZoneOffset.UTC); - // LocalDateTime.ofInstant(this.until, ZoneOffset.UTC).plusDays(1).minusN - } - return this; - } - - @Override - public DateRangeFilter build() { - if (from == null && until == null) { - return new DateRangeFilter(label, null); - } - return new DateRangeFilter(label, new DateRange(from, until)); - } - - @Override - public DateRangeFilterBuilder parseContent(final String filterValue) { - final String fromValue = extractFromFieldFromString(filterValue); - final String untilValue = extractToFieldFromString(filterValue); - return from(fromValue).until(untilValue); - } - - private String extractFromFieldFromString(final String dateRangeString) { - final int fromIndex = dateRangeString.indexOf(getFromFieldPrefix()); - final int toIndex = dateRangeString.indexOf(getToFieldPrefix()); - if (fromIndex == -1) { - return ""; - } else { - if (toIndex < fromIndex) { - return dateRangeString.substring(fromIndex + getFromFieldPrefix().length()); - } else { - return dateRangeString.substring(fromIndex + getFromFieldPrefix().length(), toIndex); - } - } - } - - private String extractToFieldFromString(final String dateRangeString) { - final int fromIndex = dateRangeString.indexOf(getFromFieldPrefix()); - final int toIndex = dateRangeString.indexOf(getToFieldPrefix()); - if (toIndex == -1) { - return ""; - } else { - if (toIndex < fromIndex) { - return dateRangeString.substring(toIndex + getToFieldPrefix().length(), fromIndex); - } else { - return dateRangeString.substring(toIndex + getToFieldPrefix().length()); - } - } - } - - private String getFromFieldPrefix() { - return "from="; - } - - private String getToFieldPrefix() { - return "until="; - } - } - - public static class DateRange { - private final Instant from; - private final Instant until; - private static final Instant min = LocalDateTime.MIN.atZone(ZoneOffset.UTC).toInstant(); - private static final Instant max = LocalDateTime.MAX.atZone(ZoneOffset.UTC).toInstant(); - - private DateRange(final Instant from, final Instant until) { - if (from == null) { - this.from = min; - } else { - this.from = from; - } - - if (until == null) { - this.until = max; - } else { - this.until = until; - } - } - - public Instant getFrom() { - return from; - } - - public Instant getUntil() { - return until; - } - - public boolean isFromMinDate() { - return getFrom().equals(min); - } - - public boolean isUntilMaxDate() { - return getUntil().equals(max); - } - - public static DateRange any() { - return new DateRange(min, max); - } - - @Override - public int hashCode() { - return Objects.hash(from, until); - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof DateRange)) { - return false; - } - final DateRange other = (DateRange) obj; - return Objects.equals(from, other.from) && Objects.equals(until, other.until); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.filter; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Objects; +import java.util.Optional; +import uk.ac.ebi.biosamples.core.model.facet.FacetType; + +public class DateRangeFilter implements Filter { + + private final Optional dateRange; + private final String label; + + DateRangeFilter(final String label, final DateRange dateRange) { + this.label = label; + this.dateRange = Optional.ofNullable(dateRange); + } + + @Override + public FilterType getType() { + return FilterType.DATE_FILTER; + } + + @Override + public String getLabel() { + return label; + } + + @Override + public FacetType getAssociatedFacetType() { + return FacetType.NO_TYPE; + } + + @Override + public Optional getContent() { + return dateRange; + } + + @Override + public String getSerialization() { + final StringBuilder serializationBuilder = + new StringBuilder(getType().getSerialization()).append(":").append(getLabel()); + + getContent() + .ifPresent( + dateRange -> { + serializationBuilder.append(":"); + + if (!dateRange.isFromMinDate()) { + serializationBuilder + .append("from=") + .append( + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format( + LocalDateTime.ofInstant(dateRange.getFrom(), ZoneOffset.UTC))); + } + + if (!dateRange.isUntilMaxDate()) { + serializationBuilder + .append("until=") + .append( + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format( + LocalDateTime.ofInstant(dateRange.getUntil(), ZoneOffset.UTC))); + } + }); + return serializationBuilder.toString(); + } + + @Override + public int hashCode() { + return Objects.hash(getLabel(), getContent()); + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof DateRangeFilter)) { + return false; + } + final DateRangeFilter other = (DateRangeFilter) obj; + return Objects.equals(getLabel(), other.getLabel()) + && Objects.equals(getContent(), other.getContent()); + } + + public static class DateRangeFilterBuilder implements Filter.Builder { + private final String label; + + private Instant from = null; + private Instant until = null; + + public DateRangeFilterBuilder(final String label) { + this.label = label; + } + + public DateRangeFilterBuilder from(final Instant from) { + this.from = from; + return this; + } + + public DateRangeFilterBuilder from(final String value) { + // quick exit + if (value == null || value.trim().length() == 0) { + return this; + } + + try { + from = + LocalDateTime.parse(value, DateTimeFormatter.ISO_LOCAL_DATE_TIME) + .toInstant(ZoneOffset.UTC); + } catch (final DateTimeParseException e) { + // retry for just the date + from = + LocalDate.parse(value, DateTimeFormatter.ISO_LOCAL_DATE) + .atStartOfDay() + .toInstant(ZoneOffset.UTC); + } + return this; + } + + public DateRangeFilterBuilder until(final Instant until) { + this.until = until; + return this; + } + + public DateRangeFilterBuilder until(final String value) { + // quick exit + if (value == null || value.trim().length() == 0) { + return this; + } + + try { + until = + LocalDateTime.parse(value, DateTimeFormatter.ISO_LOCAL_DATE_TIME) + .toInstant(ZoneOffset.UTC); + } catch (final DateTimeParseException e) { + // retry for just the date + until = + LocalDate.parse(value, DateTimeFormatter.ISO_LOCAL_DATE) + .plusDays(1) + .atStartOfDay() + .toInstant(ZoneOffset.UTC); + // LocalDateTime.ofInstant(this.until, ZoneOffset.UTC).plusDays(1).minusN + } + return this; + } + + @Override + public DateRangeFilter build() { + if (from == null && until == null) { + return new DateRangeFilter(label, null); + } + return new DateRangeFilter(label, new DateRange(from, until)); + } + + @Override + public DateRangeFilterBuilder parseContent(final String filterValue) { + final String fromValue = extractFromFieldFromString(filterValue); + final String untilValue = extractToFieldFromString(filterValue); + return from(fromValue).until(untilValue); + } + + private String extractFromFieldFromString(final String dateRangeString) { + final int fromIndex = dateRangeString.indexOf(getFromFieldPrefix()); + final int toIndex = dateRangeString.indexOf(getToFieldPrefix()); + if (fromIndex == -1) { + return ""; + } else { + if (toIndex < fromIndex) { + return dateRangeString.substring(fromIndex + getFromFieldPrefix().length()); + } else { + return dateRangeString.substring(fromIndex + getFromFieldPrefix().length(), toIndex); + } + } + } + + private String extractToFieldFromString(final String dateRangeString) { + final int fromIndex = dateRangeString.indexOf(getFromFieldPrefix()); + final int toIndex = dateRangeString.indexOf(getToFieldPrefix()); + if (toIndex == -1) { + return ""; + } else { + if (toIndex < fromIndex) { + return dateRangeString.substring(toIndex + getToFieldPrefix().length(), fromIndex); + } else { + return dateRangeString.substring(toIndex + getToFieldPrefix().length()); + } + } + } + + private String getFromFieldPrefix() { + return "from="; + } + + private String getToFieldPrefix() { + return "until="; + } + } + + public static class DateRange { + private final Instant from; + private final Instant until; + private static final Instant min = LocalDateTime.MIN.atZone(ZoneOffset.UTC).toInstant(); + private static final Instant max = LocalDateTime.MAX.atZone(ZoneOffset.UTC).toInstant(); + + private DateRange(final Instant from, final Instant until) { + if (from == null) { + this.from = min; + } else { + this.from = from; + } + + if (until == null) { + this.until = max; + } else { + this.until = until; + } + } + + public Instant getFrom() { + return from; + } + + public Instant getUntil() { + return until; + } + + public boolean isFromMinDate() { + return getFrom().equals(min); + } + + public boolean isUntilMaxDate() { + return getUntil().equals(max); + } + + public static DateRange any() { + return new DateRange(min, max); + } + + @Override + public int hashCode() { + return Objects.hash(from, until); + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof DateRange)) { + return false; + } + final DateRange other = (DateRange) obj; + return Objects.equals(from, other.from) && Objects.equals(until, other.until); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/ExternalReferenceDataFilter.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/ExternalReferenceDataFilter.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/ExternalReferenceDataFilter.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/ExternalReferenceDataFilter.java index b2a6b258c..a27afd235 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/ExternalReferenceDataFilter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/ExternalReferenceDataFilter.java @@ -1,95 +1,95 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.filter; - -import java.util.Objects; -import java.util.Optional; -import uk.ac.ebi.biosamples.model.facet.FacetType; - -public class ExternalReferenceDataFilter implements Filter { - private final String label; - private final String value; - - private ExternalReferenceDataFilter(final String label, final String value) { - this.label = label; - this.value = value; - } - - @Override - public FilterType getType() { - return FilterType.EXTERNAL_REFERENCE_DATA_FILTER; - } - - @Override - public String getLabel() { - return label; - } - - @Override - public Optional getContent() { - return Optional.ofNullable(value); - } - - @Override - public FacetType getAssociatedFacetType() { - return FacetType.EXTERNAL_REFERENCE_DATA_FACET; - } - - @Override - public String getSerialization() { - final StringBuilder serialization = - new StringBuilder(getType().getSerialization()).append(":").append(label); - getContent().ifPresent(value -> serialization.append(":").append(value)); - return serialization.toString(); - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof ExternalReferenceDataFilter)) { - return false; - } - final ExternalReferenceDataFilter other = (ExternalReferenceDataFilter) obj; - return Objects.equals(other.label, label) - && Objects.equals(other.getContent().orElse(null), getContent().orElse(null)); - } - - @Override - public int hashCode() { - return Objects.hash(label, getContent().orElse(null)); - } - - public static class Builder implements Filter.Builder { - private String value; - private final String label; - - public Builder(final String label) { - this.label = label; - } - - public Builder withValue(final String value) { - this.value = value; - return this; - } - - @Override - public ExternalReferenceDataFilter build() { - return new ExternalReferenceDataFilter(label, value); - } - - @Override - public Builder parseContent(final String filterValue) { - return withValue(filterValue); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.filter; + +import java.util.Objects; +import java.util.Optional; +import uk.ac.ebi.biosamples.core.model.facet.FacetType; + +public class ExternalReferenceDataFilter implements Filter { + private final String label; + private final String value; + + private ExternalReferenceDataFilter(final String label, final String value) { + this.label = label; + this.value = value; + } + + @Override + public FilterType getType() { + return FilterType.EXTERNAL_REFERENCE_DATA_FILTER; + } + + @Override + public String getLabel() { + return label; + } + + @Override + public Optional getContent() { + return Optional.ofNullable(value); + } + + @Override + public FacetType getAssociatedFacetType() { + return FacetType.EXTERNAL_REFERENCE_DATA_FACET; + } + + @Override + public String getSerialization() { + final StringBuilder serialization = + new StringBuilder(getType().getSerialization()).append(":").append(label); + getContent().ifPresent(value -> serialization.append(":").append(value)); + return serialization.toString(); + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof ExternalReferenceDataFilter)) { + return false; + } + final ExternalReferenceDataFilter other = (ExternalReferenceDataFilter) obj; + return Objects.equals(other.label, label) + && Objects.equals(other.getContent().orElse(null), getContent().orElse(null)); + } + + @Override + public int hashCode() { + return Objects.hash(label, getContent().orElse(null)); + } + + public static class Builder implements Filter.Builder { + private String value; + private final String label; + + public Builder(final String label) { + this.label = label; + } + + public Builder withValue(final String value) { + this.value = value; + return this; + } + + @Override + public ExternalReferenceDataFilter build() { + return new ExternalReferenceDataFilter(label, value); + } + + @Override + public Builder parseContent(final String filterValue) { + return withValue(filterValue); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/Filter.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/Filter.java similarity index 91% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/Filter.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/Filter.java index ab29e7920..e2bc58c02 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/Filter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/Filter.java @@ -1,59 +1,59 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.filter; - -import java.util.Optional; -import uk.ac.ebi.biosamples.model.facet.FacetType; - -public interface Filter { - public FilterType getType(); - - /** - * The label the filter is targeting - * - * @return - */ - public String getLabel(); - - /** - * Return the optional content of the filter. The content is optional because can be used also to - * filter for samples having a specific characteristic - * - * @return filter specific value, if available - */ - public Optional getContent(); - - /** - * Generate the serialized version of the filter usable through the web interface - * - * @return string representing the filter value - */ - public String getSerialization(); - - /** - * Get the facet associated to the filter, if any is available - * - * @return optional facet type - */ - public FacetType getAssociatedFacetType(); - - interface Builder { - Filter build(); - - /** - * Create a builder starting from a filter serialization - * - * @param filterSerialized string representing a filter - * @return a Builder to compose the filter - */ - public Filter.Builder parseContent(String filterSerialized); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.filter; + +import java.util.Optional; +import uk.ac.ebi.biosamples.core.model.facet.FacetType; + +public interface Filter { + public FilterType getType(); + + /** + * The label the filter is targeting + * + * @return + */ + public String getLabel(); + + /** + * Return the optional content of the filter. The content is optional because can be used also to + * filter for samples having a specific characteristic + * + * @return filter specific value, if available + */ + public Optional getContent(); + + /** + * Generate the serialized version of the filter usable through the web interface + * + * @return string representing the filter value + */ + public String getSerialization(); + + /** + * Get the facet associated to the filter, if any is available + * + * @return optional facet type + */ + public FacetType getAssociatedFacetType(); + + interface Builder { + Filter build(); + + /** + * Create a builder starting from a filter serialization + * + * @param filterSerialized string representing a filter + * @return a Builder to compose the filter + */ + public Filter.Builder parseContent(String filterSerialized); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/FilterType.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/FilterType.java similarity index 96% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/FilterType.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/FilterType.java index b68737069..4a5b0b942 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/FilterType.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/FilterType.java @@ -1,73 +1,73 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.filter; - -import java.lang.reflect.InvocationTargetException; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public enum FilterType { - ATTRIBUTE_FILTER("attr", AttributeFilter.Builder.class), - NAME_FILTER("name", NameFilter.Builder.class), - RELATION_FILER("rel", RelationFilter.Builder.class), - INVERSE_RELATION_FILTER("rrel", InverseRelationFilter.Builder.class), - DOMAIN_FILTER("dom", AuthenticationFilter.Builder.class), - WEBINID_FILTER("webinId", AuthenticationFilter.Builder.class), - DATE_FILTER("dt", DateRangeFilter.DateRangeFilterBuilder.class), - EXTERNAL_REFERENCE_DATA_FILTER("extd", ExternalReferenceDataFilter.Builder.class), - ACCESSION_FILTER("acc", AccessionFilter.Builder.class); - - private static final List filterTypesByLength; - - static { - filterTypesByLength = - Stream.of(values()) - .sorted( - Comparator.comparingInt((FilterType f) -> f.getSerialization().length()) - .reversed() - .thenComparing((FilterType::getSerialization))) - .collect(Collectors.toList()); - } - - String serialization; - Class associatedBuilder; - - FilterType(final String serialization, final Class associatedBuilder) { - this.serialization = serialization; - this.associatedBuilder = associatedBuilder; - } - - public String getSerialization() { - return this.serialization; - } - - public Filter.Builder getBuilderForLabel(final String label) { - try { - return this.associatedBuilder.getConstructor(String.class).newInstance(label); - } catch (final NoSuchMethodException - | IllegalAccessException - | InstantiationException - | InvocationTargetException e) { - throw new RuntimeException("FilterType " + this + " does not provide a proper builder"); - } - } - - public static FilterType ofFilterString(final String filterString) { - for (final FilterType type : filterTypesByLength) { - if (filterString.startsWith(type.getSerialization())) { - return type; - } - } - throw new IllegalArgumentException("Cannot infer filter type from string " + filterString); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.filter; + +import java.lang.reflect.InvocationTargetException; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public enum FilterType { + ATTRIBUTE_FILTER("attr", AttributeFilter.Builder.class), + NAME_FILTER("name", NameFilter.Builder.class), + RELATION_FILER("rel", RelationFilter.Builder.class), + INVERSE_RELATION_FILTER("rrel", InverseRelationFilter.Builder.class), + DOMAIN_FILTER("dom", AuthenticationFilter.Builder.class), + WEBINID_FILTER("webinId", AuthenticationFilter.Builder.class), + DATE_FILTER("dt", DateRangeFilter.DateRangeFilterBuilder.class), + EXTERNAL_REFERENCE_DATA_FILTER("extd", ExternalReferenceDataFilter.Builder.class), + ACCESSION_FILTER("acc", AccessionFilter.Builder.class); + + private static final List filterTypesByLength; + + static { + filterTypesByLength = + Stream.of(values()) + .sorted( + Comparator.comparingInt((FilterType f) -> f.getSerialization().length()) + .reversed() + .thenComparing((FilterType::getSerialization))) + .collect(Collectors.toList()); + } + + String serialization; + Class associatedBuilder; + + FilterType(final String serialization, final Class associatedBuilder) { + this.serialization = serialization; + this.associatedBuilder = associatedBuilder; + } + + public String getSerialization() { + return this.serialization; + } + + public Filter.Builder getBuilderForLabel(final String label) { + try { + return this.associatedBuilder.getConstructor(String.class).newInstance(label); + } catch (final NoSuchMethodException + | IllegalAccessException + | InstantiationException + | InvocationTargetException e) { + throw new RuntimeException("FilterType " + this + " does not provide a proper builder"); + } + } + + public static FilterType ofFilterString(final String filterString) { + for (final FilterType type : filterTypesByLength) { + if (filterString.startsWith(type.getSerialization())) { + return type; + } + } + throw new IllegalArgumentException("Cannot infer filter type from string " + filterString); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/InverseRelationFilter.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/InverseRelationFilter.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/InverseRelationFilter.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/InverseRelationFilter.java index 06afd3113..312c1ca75 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/InverseRelationFilter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/InverseRelationFilter.java @@ -1,96 +1,96 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.filter; - -import java.util.Objects; -import java.util.Optional; -import uk.ac.ebi.biosamples.model.facet.FacetType; - -public class InverseRelationFilter implements Filter { - - private final String label; - private final String value; - - private InverseRelationFilter(final String label, final String value) { - this.label = label; - this.value = value; - } - - @Override - public FilterType getType() { - return FilterType.INVERSE_RELATION_FILTER; - } - - @Override - public String getLabel() { - return label; - } - - @Override - public Optional getContent() { - return Optional.ofNullable(value); - } - - @Override - public FacetType getAssociatedFacetType() { - return FacetType.INVERSE_RELATION_FACET; - } - - @Override - public String getSerialization() { - final StringBuilder serializationBuilder = - new StringBuilder(getType().getSerialization()).append(":").append(getLabel()); - getContent().ifPresent(content -> serializationBuilder.append(":").append(content)); - return serializationBuilder.toString(); - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof InverseRelationFilter)) { - return false; - } - final InverseRelationFilter other = (InverseRelationFilter) obj; - return Objects.equals(other.getLabel(), getLabel()) - && Objects.equals(other.getContent().orElse(null), getContent().orElse(null)); - } - - @Override - public int hashCode() { - return Objects.hash(getLabel(), getContent().orElse(null)); - } - - public static class Builder implements Filter.Builder { - private String value; - private final String label; - - public Builder(final String label) { - this.label = label; - } - - public Builder withValue(final String value) { - this.value = value; - return this; - } - - @Override - public InverseRelationFilter build() { - return new InverseRelationFilter(label, value); - } - - @Override - public Builder parseContent(final String filterValue) { - return withValue(filterValue); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.filter; + +import java.util.Objects; +import java.util.Optional; +import uk.ac.ebi.biosamples.core.model.facet.FacetType; + +public class InverseRelationFilter implements Filter { + + private final String label; + private final String value; + + private InverseRelationFilter(final String label, final String value) { + this.label = label; + this.value = value; + } + + @Override + public FilterType getType() { + return FilterType.INVERSE_RELATION_FILTER; + } + + @Override + public String getLabel() { + return label; + } + + @Override + public Optional getContent() { + return Optional.ofNullable(value); + } + + @Override + public FacetType getAssociatedFacetType() { + return FacetType.INVERSE_RELATION_FACET; + } + + @Override + public String getSerialization() { + final StringBuilder serializationBuilder = + new StringBuilder(getType().getSerialization()).append(":").append(getLabel()); + getContent().ifPresent(content -> serializationBuilder.append(":").append(content)); + return serializationBuilder.toString(); + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof InverseRelationFilter)) { + return false; + } + final InverseRelationFilter other = (InverseRelationFilter) obj; + return Objects.equals(other.getLabel(), getLabel()) + && Objects.equals(other.getContent().orElse(null), getContent().orElse(null)); + } + + @Override + public int hashCode() { + return Objects.hash(getLabel(), getContent().orElse(null)); + } + + public static class Builder implements Filter.Builder { + private String value; + private final String label; + + public Builder(final String label) { + this.label = label; + } + + public Builder withValue(final String value) { + this.value = value; + return this; + } + + @Override + public InverseRelationFilter build() { + return new InverseRelationFilter(label, value); + } + + @Override + public Builder parseContent(final String filterValue) { + return withValue(filterValue); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/NameFilter.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/NameFilter.java similarity index 91% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/NameFilter.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/NameFilter.java index 7c7fa7152..f5bf97629 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/NameFilter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/NameFilter.java @@ -1,85 +1,85 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.filter; - -import java.util.Objects; -import java.util.Optional; -import uk.ac.ebi.biosamples.model.facet.FacetType; - -public class NameFilter implements Filter { - - private final String name; - - private NameFilter(final String name) { - this.name = name; - } - - @Override - public FilterType getType() { - return FilterType.NAME_FILTER; - } - - @Override - public String getLabel() { - return "name"; - } - - @Override - public Optional getContent() { - return Optional.of(name); - } - - @Override - public FacetType getAssociatedFacetType() { - return FacetType.NO_TYPE; - } - - @Override - public String getSerialization() { - return getType().getSerialization() + ":" + name; - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof NameFilter)) { - return false; - } - final NameFilter other = (NameFilter) obj; - return Objects.equals(other.getContent().orElse(null), getContent().orElse(null)); - } - - @Override - public int hashCode() { - return Objects.hash(getContent().orElse(null)); - } - - public static class Builder implements Filter.Builder { - - private final String name; - - public Builder(final String name) { - this.name = name; - } - - @Override - public Filter build() { - return new NameFilter(name); - } - - @Override - public Filter.Builder parseContent(final String filterSerialized) { - return new Builder(filterSerialized); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.filter; + +import java.util.Objects; +import java.util.Optional; +import uk.ac.ebi.biosamples.core.model.facet.FacetType; + +public class NameFilter implements Filter { + + private final String name; + + private NameFilter(final String name) { + this.name = name; + } + + @Override + public FilterType getType() { + return FilterType.NAME_FILTER; + } + + @Override + public String getLabel() { + return "name"; + } + + @Override + public Optional getContent() { + return Optional.of(name); + } + + @Override + public FacetType getAssociatedFacetType() { + return FacetType.NO_TYPE; + } + + @Override + public String getSerialization() { + return getType().getSerialization() + ":" + name; + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof NameFilter)) { + return false; + } + final NameFilter other = (NameFilter) obj; + return Objects.equals(other.getContent().orElse(null), getContent().orElse(null)); + } + + @Override + public int hashCode() { + return Objects.hash(getContent().orElse(null)); + } + + public static class Builder implements Filter.Builder { + + private final String name; + + public Builder(final String name) { + this.name = name; + } + + @Override + public Filter build() { + return new NameFilter(name); + } + + @Override + public Filter.Builder parseContent(final String filterSerialized) { + return new Builder(filterSerialized); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/RelationFilter.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/RelationFilter.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/RelationFilter.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/RelationFilter.java index 06e101095..f9349d2c6 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/filter/RelationFilter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/filter/RelationFilter.java @@ -1,96 +1,96 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.filter; - -import java.util.Objects; -import java.util.Optional; -import uk.ac.ebi.biosamples.model.facet.FacetType; - -public class RelationFilter implements Filter { - - private final String label; - private final String value; - - RelationFilter(final String label, final String value) { - this.label = label; - this.value = value; - } - - @Override - public FilterType getType() { - return FilterType.RELATION_FILER; - } - - @Override - public String getLabel() { - return label; - } - - @Override - public FacetType getAssociatedFacetType() { - return FacetType.RELATION_FACET; - } - - @Override - public Optional getContent() { - return Optional.ofNullable(value); - } - - @Override - public String getSerialization() { - final StringBuilder serializationBuilder = - new StringBuilder(getType().getSerialization()).append(":").append(getLabel()); - getContent().ifPresent(content -> serializationBuilder.append(":").append(content)); - return serializationBuilder.toString(); - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof RelationFilter)) { - return false; - } - final RelationFilter other = (RelationFilter) obj; - return Objects.equals(other.getLabel(), getLabel()) - && Objects.equals(other.getContent().orElse(null), getContent().orElse(null)); - } - - @Override - public int hashCode() { - return Objects.hash(getLabel(), getContent().orElse(null)); - } - - public static class Builder implements Filter.Builder { - private String value; - private final String label; - - public Builder(final String label) { - this.label = label; - } - - public Builder withValue(final String value) { - this.value = value; - return this; - } - - @Override - public RelationFilter build() { - return new RelationFilter(label, value); - } - - @Override - public Builder parseContent(final String filterValue) { - return withValue(filterValue); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model.filter; + +import java.util.Objects; +import java.util.Optional; +import uk.ac.ebi.biosamples.core.model.facet.FacetType; + +public class RelationFilter implements Filter { + + private final String label; + private final String value; + + RelationFilter(final String label, final String value) { + this.label = label; + this.value = value; + } + + @Override + public FilterType getType() { + return FilterType.RELATION_FILER; + } + + @Override + public String getLabel() { + return label; + } + + @Override + public FacetType getAssociatedFacetType() { + return FacetType.RELATION_FACET; + } + + @Override + public Optional getContent() { + return Optional.ofNullable(value); + } + + @Override + public String getSerialization() { + final StringBuilder serializationBuilder = + new StringBuilder(getType().getSerialization()).append(":").append(getLabel()); + getContent().ifPresent(content -> serializationBuilder.append(":").append(content)); + return serializationBuilder.toString(); + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof RelationFilter)) { + return false; + } + final RelationFilter other = (RelationFilter) obj; + return Objects.equals(other.getLabel(), getLabel()) + && Objects.equals(other.getContent().orElse(null), getContent().orElse(null)); + } + + @Override + public int hashCode() { + return Objects.hash(getLabel(), getContent().orElse(null)); + } + + public static class Builder implements Filter.Builder { + private String value; + private final String label; + + public Builder(final String label) { + this.label = label; + } + + public Builder withValue(final String value) { + this.value = value; + return this; + } + + @Override + public RelationFilter build() { + return new RelationFilter(label, value); + } + + @Override + public Builder parseContent(final String filterValue) { + return withValue(filterValue); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/AbstractData.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/AbstractData.java similarity index 92% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/AbstractData.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/AbstractData.java index 0e8598f8a..07b809baa 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/AbstractData.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/AbstractData.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.model.structured; +package uk.ac.ebi.biosamples.core.model.structured; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -19,7 +19,7 @@ import java.util.Map; import lombok.Data; import lombok.NoArgsConstructor; -import uk.ac.ebi.biosamples.service.structured.AbstractDataDeserializer; +import uk.ac.ebi.biosamples.core.service.structured.AbstractDataDeserializer; @Data @NoArgsConstructor diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredCell.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredCell.java similarity index 96% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredCell.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredCell.java index 65392b89b..f993a99e2 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredCell.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredCell.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.model.structured; +package uk.ac.ebi.biosamples.core.model.structured; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.*; diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredData.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredData.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredData.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredData.java index 76a307e34..127d60efe 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredData.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredData.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.model.structured; +package uk.ac.ebi.biosamples.core.model.structured; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -16,8 +16,8 @@ import java.util.Objects; import java.util.Set; import lombok.Getter; -import uk.ac.ebi.biosamples.service.CustomInstantDeserializer; -import uk.ac.ebi.biosamples.service.CustomInstantSerializer; +import uk.ac.ebi.biosamples.core.service.CustomInstantDeserializer; +import uk.ac.ebi.biosamples.core.service.CustomInstantSerializer; @Getter public class StructuredData { diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredDataEntry.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredDataEntry.java similarity index 94% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredDataEntry.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredDataEntry.java index 71032dabc..1b6df82f3 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredDataEntry.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredDataEntry.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.model.structured; +package uk.ac.ebi.biosamples.core.model.structured; import lombok.Getter; diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredDataTable.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredDataTable.java similarity index 98% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredDataTable.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredDataTable.java index a634450b0..4524179c0 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredDataTable.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredDataTable.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.model.structured; +package uk.ac.ebi.biosamples.core.model.structured; import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.*; diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredDataType.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredDataType.java similarity index 96% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredDataType.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredDataType.java index 98685c853..4a77b330f 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredDataType.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredDataType.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.model.structured; +package uk.ac.ebi.biosamples.core.model.structured; import java.util.Arrays; import java.util.Collections; diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredEntry.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredEntry.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredEntry.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredEntry.java index f93a496e5..8df42dfae 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredEntry.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredEntry.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.model.structured; +package uk.ac.ebi.biosamples.core.model.structured; import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.Map; diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredTable.java b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredTable.java similarity index 98% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredTable.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredTable.java index ac0124804..460185ecd 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/structured/StructuredTable.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/model/structured/StructuredTable.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.model.structured; +package uk.ac.ebi.biosamples.core.model.structured; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionSerializer.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/AccessionSerializer.java similarity index 90% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionSerializer.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/AccessionSerializer.java index 934cc4761..5fc85b7c1 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionSerializer.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/AccessionSerializer.java @@ -1,30 +1,30 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import java.io.IOException; -import uk.ac.ebi.biosamples.model.Accession; - -public class AccessionSerializer extends StdSerializer { - protected AccessionSerializer() { - super(Accession.class); - } - - @Override - public void serialize( - final Accession accession, final JsonGenerator gen, final SerializerProvider provider) - throws IOException { - gen.writeString(accession.getId()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import uk.ac.ebi.biosamples.core.model.Accession; + +public class AccessionSerializer extends StdSerializer { + protected AccessionSerializer() { + super(Accession.class); + } + + @Override + public void serialize( + final Accession accession, final JsonGenerator gen, final SerializerProvider provider) + throws IOException { + gen.writeString(accession.getId()); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/AttributeValidator.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/AttributeValidator.java similarity index 91% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/AttributeValidator.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/AttributeValidator.java index 36a410ddd..842e633e0 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/AttributeValidator.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/AttributeValidator.java @@ -1,36 +1,36 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.util.Collection; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Attribute; - -@Service -public class AttributeValidator { - - public void validate(final Attribute attribute, final Collection errors) { - /* - if (attribute.getType().length() > 255) { - errors.add(attribute+" type too long"); - } - if (attribute.getValue().length() > 255) { - errors.add(attribute+" value too long"); - } - if (attribute.getIri() != null && attribute.getIri().length() > 255) { - errors.add(attribute+" iri too long"); - } - if (attribute.getUnit() != null && attribute.getUnit().length() > 255) { - errors.add(attribute+" unit too long"); - } - */ - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import java.util.Collection; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Attribute; + +@Service +public class AttributeValidator { + + public void validate(final Attribute attribute, final Collection errors) { + /* + if (attribute.getType().length() > 255) { + errors.add(attribute+" type too long"); + } + if (attribute.getValue().length() > 255) { + errors.add(attribute+" value too long"); + } + if (attribute.getIri() != null && attribute.getIri().length() > 255) { + errors.add(attribute+" iri too long"); + } + if (attribute.getUnit() != null && attribute.getUnit().length() > 255) { + errors.add(attribute+" unit too long"); + } + */ + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/CharacteristicDeserializer.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/CharacteristicDeserializer.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/CharacteristicDeserializer.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/CharacteristicDeserializer.java index 829f34db7..ebecb1a4d 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/CharacteristicDeserializer.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/CharacteristicDeserializer.java @@ -1,83 +1,83 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.SortedSet; -import java.util.TreeSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import uk.ac.ebi.biosamples.model.Attribute; - -/* -"characteristics": { - "material": [ - { - "text": "specimen from organism", - "ontologyTerms": [ - "http://purl.obolibrary.org/obo/OBI_0001479" - ] - } - ], - "specimenCollectionDate": [ - { - "text": "2013-05", - "unit": "YYYY-MM" - } - ], - */ -public class CharacteristicDeserializer extends StdDeserializer { - private final Logger log = LoggerFactory.getLogger(getClass()); - - public CharacteristicDeserializer() { - this(SortedSet.class); - } - - private CharacteristicDeserializer(final Class t) { - super(t); - } - - @Override - public SortedSet deserialize(final JsonParser p, final DeserializationContext ctxt) - throws IOException { - final SortedSet attributes = new TreeSet<>(); - final Map> characteristics = - p.readValueAs(new TypeReference>>() {}); - - for (final String type : characteristics.keySet()) { - for (final LegacyAttribute legacy : characteristics.get(type)) { - attributes.add( - Attribute.build(type, legacy.text, legacy.tag, legacy.ontologyTerms, legacy.unit)); - } - } - - return attributes; - } - - /** - * This internal class is used for easier deserializtion with jackson. The fields match the field - * names in the JSON and therefore should not be changed! - * - * @author faulcon - */ - private static class LegacyAttribute { - public String text; - public List ontologyTerms; - public String tag; - public String unit; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import uk.ac.ebi.biosamples.core.model.Attribute; + +/* +"characteristics": { + "material": [ + { + "text": "specimen from organism", + "ontologyTerms": [ + "http://purl.obolibrary.org/obo/OBI_0001479" + ] + } + ], + "specimenCollectionDate": [ + { + "text": "2013-05", + "unit": "YYYY-MM" + } + ], + */ +public class CharacteristicDeserializer extends StdDeserializer { + private final Logger log = LoggerFactory.getLogger(getClass()); + + public CharacteristicDeserializer() { + this(SortedSet.class); + } + + private CharacteristicDeserializer(final Class t) { + super(t); + } + + @Override + public SortedSet deserialize(final JsonParser p, final DeserializationContext ctxt) + throws IOException { + final SortedSet attributes = new TreeSet<>(); + final Map> characteristics = + p.readValueAs(new TypeReference>>() {}); + + for (final String type : characteristics.keySet()) { + for (final LegacyAttribute legacy : characteristics.get(type)) { + attributes.add( + Attribute.build(type, legacy.text, legacy.tag, legacy.ontologyTerms, legacy.unit)); + } + } + + return attributes; + } + + /** + * This internal class is used for easier deserializtion with jackson. The fields match the field + * names in the JSON and therefore should not be changed! + * + * @author faulcon + */ + private static class LegacyAttribute { + public String text; + public List ontologyTerms; + public String tag; + public String unit; + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/CharacteristicSerializer.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/CharacteristicSerializer.java similarity index 94% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/CharacteristicSerializer.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/CharacteristicSerializer.java index 6592b6371..49a2038c7 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/CharacteristicSerializer.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/CharacteristicSerializer.java @@ -1,115 +1,115 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import java.io.IOException; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import uk.ac.ebi.biosamples.model.Attribute; - -/* - -"characteristics": { - "material": [ - { - "text": "specimen from organism", - "ontologyTerms": [ - "http://purl.obolibrary.org/obo/OBI_0001479" - ] - } - ], - "specimenCollectionDate": [ - { - "text": "2013-05", - "unit": "YYYY-MM" - } - ], - */ -public class CharacteristicSerializer extends StdSerializer { - private final Logger log = LoggerFactory.getLogger(getClass()); - - public CharacteristicSerializer() { - this(SortedSet.class); - } - - private CharacteristicSerializer(final Class t) { - super(t); - } - - @Override - public void serialize( - final SortedSet attributesRaw, final JsonGenerator gen, final SerializerProvider arg2) - throws IOException { - final SortedSet attributes = (SortedSet) attributesRaw; - - gen.writeStartObject(); - - final SortedMap> attributeMap = - new TreeMap<>(); - - if (attributes != null && !attributes.isEmpty()) { - for (final Attribute attribute : attributes) { - attributeMap - .computeIfAbsent(attribute.getType(), k -> new ArrayListValuedHashMap<>()) - .put( - attribute.getValue(), - Attribute.build( - attribute.getType(), - attribute.getValue(), - attribute.getTag(), - attribute.getIri(), - attribute.getUnit())); - } - - for (final String type : attributeMap.keySet()) { - gen.writeArrayFieldStart(type); - - for (final String value : attributeMap.get(type).keySet()) { - for (final Attribute attr : attributeMap.get(type).get(value)) { - gen.writeStartObject(); - gen.writeStringField("text", value); - - if (attr.getIri() != null && !attr.getIri().isEmpty()) { - gen.writeArrayFieldStart("ontologyTerms"); - - for (final String iri : attr.getIri()) { - gen.writeString(iri); - } - - gen.writeEndArray(); - } - - if (attr.getUnit() != null) { - gen.writeStringField("unit", attr.getUnit()); - } - - if (attr.getTag() != null) { - gen.writeStringField("tag", attr.getTag()); - } - - gen.writeEndObject(); - } - } - - gen.writeEndArray(); - } - } - - gen.writeEndObject(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import uk.ac.ebi.biosamples.core.model.Attribute; + +/* + +"characteristics": { + "material": [ + { + "text": "specimen from organism", + "ontologyTerms": [ + "http://purl.obolibrary.org/obo/OBI_0001479" + ] + } + ], + "specimenCollectionDate": [ + { + "text": "2013-05", + "unit": "YYYY-MM" + } + ], + */ +public class CharacteristicSerializer extends StdSerializer { + private final Logger log = LoggerFactory.getLogger(getClass()); + + public CharacteristicSerializer() { + this(SortedSet.class); + } + + private CharacteristicSerializer(final Class t) { + super(t); + } + + @Override + public void serialize( + final SortedSet attributesRaw, final JsonGenerator gen, final SerializerProvider arg2) + throws IOException { + final SortedSet attributes = (SortedSet) attributesRaw; + + gen.writeStartObject(); + + final SortedMap> attributeMap = + new TreeMap<>(); + + if (attributes != null && !attributes.isEmpty()) { + for (final Attribute attribute : attributes) { + attributeMap + .computeIfAbsent(attribute.getType(), k -> new ArrayListValuedHashMap<>()) + .put( + attribute.getValue(), + Attribute.build( + attribute.getType(), + attribute.getValue(), + attribute.getTag(), + attribute.getIri(), + attribute.getUnit())); + } + + for (final String type : attributeMap.keySet()) { + gen.writeArrayFieldStart(type); + + for (final String value : attributeMap.get(type).keySet()) { + for (final Attribute attr : attributeMap.get(type).get(value)) { + gen.writeStartObject(); + gen.writeStringField("text", value); + + if (attr.getIri() != null && !attr.getIri().isEmpty()) { + gen.writeArrayFieldStart("ontologyTerms"); + + for (final String iri : attr.getIri()) { + gen.writeString(iri); + } + + gen.writeEndArray(); + } + + if (attr.getUnit() != null) { + gen.writeStringField("unit", attr.getUnit()); + } + + if (attr.getTag() != null) { + gen.writeStringField("tag", attr.getTag()); + } + + gen.writeEndObject(); + } + } + + gen.writeEndArray(); + } + } + + gen.writeEndObject(); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/CurationApplicationService.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/CurationApplicationService.java similarity index 92% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/CurationApplicationService.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/CurationApplicationService.java index f67b4f7cf..4742a0906 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/CurationApplicationService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/CurationApplicationService.java @@ -1,118 +1,118 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.util.Collection; -import java.util.Iterator; -import java.util.SortedSet; -import java.util.TreeSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.ExternalReference; -import uk.ac.ebi.biosamples.model.Sample; - -@Service -public class CurationApplicationService { - private final Logger log = LoggerFactory.getLogger(getClass()); - - public Sample applyCurationToSample(final Sample sample, final Curation curation) { - log.trace("Applying curation " + curation + " to sample " + sample); - - final SortedSet attributes = new TreeSet<>(sample.getAttributes()); - final SortedSet externalReferences = - new TreeSet<>(sample.getExternalReferences()); - - // remove pre-curation things - for (final Attribute attribute : curation.getAttributesPre()) { - if (!attributes.contains(attribute)) { - throw new IllegalArgumentException( - "Attempting to apply curation " + curation + " to sample " + sample); - } - - attributes.remove(attribute); - } - for (final ExternalReference externalReference : curation.getExternalReferencesPre()) { - if (!externalReferences.contains(externalReference)) { - throw new IllegalArgumentException( - "Attempting to apply curation " + curation + " to sample " + sample); - } - - externalReferences.remove(externalReference); - } - // add post-curation things - for (final Attribute attribute : curation.getAttributesPost()) { - if (attributes.contains(attribute)) { - throw new IllegalArgumentException( - "Attempting to apply curation " + curation + " to sample " + sample); - } - - attributes.add(attribute); - } - for (final ExternalReference externalReference : curation.getExternalReferencesPost()) { - if (externalReferences.contains(externalReference)) { - throw new IllegalArgumentException( - "Attempting to apply curation " + curation + " to sample " + sample); - } - - externalReferences.add(externalReference); - } - - return Sample.build( - sample.getName(), - sample.getAccession(), - sample.getSraAccession(), - sample.getDomain(), - sample.getWebinSubmissionAccountId(), - sample.getTaxId(), - sample.getStatus(), - sample.getRelease(), - sample.getUpdate(), - sample.getCreate(), - sample.getSubmitted(), - sample.getReviewed(), - attributes, - sample.getData(), - sample.getStructuredData(), - sample.getRelationships(), - externalReferences, - sample.getOrganizations(), - sample.getContacts(), - sample.getPublications(), - sample.getCertificates(), - sample.getSubmittedVia()); - } - - public Sample applyAllCurationToSample(Sample sample, final Collection curations) { - boolean curationApplied = true; - - while (curationApplied && !curations.isEmpty()) { - final Iterator it = curations.iterator(); - curationApplied = false; - - while (it.hasNext()) { - final Curation curation = it.next(); - - try { - sample = applyCurationToSample(sample, curation); - it.remove(); - curationApplied = true; - } catch (final IllegalArgumentException e) { - // do nothing, will try again next loop - } - } - } - - return sample; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import java.util.Collection; +import java.util.Iterator; +import java.util.SortedSet; +import java.util.TreeSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.ExternalReference; +import uk.ac.ebi.biosamples.core.model.Sample; + +@Service +public class CurationApplicationService { + private final Logger log = LoggerFactory.getLogger(getClass()); + + public Sample applyCurationToSample(final Sample sample, final Curation curation) { + log.trace("Applying curation " + curation + " to sample " + sample); + + final SortedSet attributes = new TreeSet<>(sample.getAttributes()); + final SortedSet externalReferences = + new TreeSet<>(sample.getExternalReferences()); + + // remove pre-curation things + for (final Attribute attribute : curation.getAttributesPre()) { + if (!attributes.contains(attribute)) { + throw new IllegalArgumentException( + "Attempting to apply curation " + curation + " to sample " + sample); + } + + attributes.remove(attribute); + } + for (final ExternalReference externalReference : curation.getExternalReferencesPre()) { + if (!externalReferences.contains(externalReference)) { + throw new IllegalArgumentException( + "Attempting to apply curation " + curation + " to sample " + sample); + } + + externalReferences.remove(externalReference); + } + // add post-curation things + for (final Attribute attribute : curation.getAttributesPost()) { + if (attributes.contains(attribute)) { + throw new IllegalArgumentException( + "Attempting to apply curation " + curation + " to sample " + sample); + } + + attributes.add(attribute); + } + for (final ExternalReference externalReference : curation.getExternalReferencesPost()) { + if (externalReferences.contains(externalReference)) { + throw new IllegalArgumentException( + "Attempting to apply curation " + curation + " to sample " + sample); + } + + externalReferences.add(externalReference); + } + + return Sample.build( + sample.getName(), + sample.getAccession(), + sample.getSraAccession(), + sample.getDomain(), + sample.getWebinSubmissionAccountId(), + sample.getTaxId(), + sample.getStatus(), + sample.getRelease(), + sample.getUpdate(), + sample.getCreate(), + sample.getSubmitted(), + sample.getReviewed(), + attributes, + sample.getData(), + sample.getStructuredData(), + sample.getRelationships(), + externalReferences, + sample.getOrganizations(), + sample.getContacts(), + sample.getPublications(), + sample.getCertificates(), + sample.getSubmittedVia()); + } + + public Sample applyAllCurationToSample(Sample sample, final Collection curations) { + boolean curationApplied = true; + + while (curationApplied && !curations.isEmpty()) { + final Iterator it = curations.iterator(); + curationApplied = false; + + while (it.hasNext()) { + final Curation curation = it.next(); + + try { + sample = applyCurationToSample(sample, curation); + it.remove(); + curationApplied = true; + } catch (final IllegalArgumentException e) { + // do nothing, will try again next loop + } + } + } + + return sample; + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/CustomInstantDeserializer.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/CustomInstantDeserializer.java similarity index 94% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/CustomInstantDeserializer.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/CustomInstantDeserializer.java index b8a705349..8598e008c 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/CustomInstantDeserializer.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/CustomInstantDeserializer.java @@ -1,45 +1,45 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import java.io.IOException; -import java.time.Instant; -import java.time.format.DateTimeParseException; - -public class CustomInstantDeserializer extends StdDeserializer { - - public CustomInstantDeserializer() { - this(null); - } - - private CustomInstantDeserializer(final Class t) { - super(t); - } - - @Override - public Instant deserialize(final JsonParser jsonparser, final DeserializationContext ctxt) - throws IOException, JsonProcessingException { - String date = jsonparser.getText(); - // TODO remove this hack - if (!date.endsWith("Z")) { - date = date + "Z"; - } - try { - return Instant.parse(date); - } catch (final DateTimeParseException e) { - throw new RuntimeException(e); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import java.io.IOException; +import java.time.Instant; +import java.time.format.DateTimeParseException; + +public class CustomInstantDeserializer extends StdDeserializer { + + public CustomInstantDeserializer() { + this(null); + } + + private CustomInstantDeserializer(final Class t) { + super(t); + } + + @Override + public Instant deserialize(final JsonParser jsonparser, final DeserializationContext ctxt) + throws IOException, JsonProcessingException { + String date = jsonparser.getText(); + // TODO remove this hack + if (!date.endsWith("Z")) { + date = date + "Z"; + } + try { + return Instant.parse(date); + } catch (final DateTimeParseException e) { + throw new RuntimeException(e); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/CustomInstantSerializer.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/CustomInstantSerializer.java similarity index 94% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/CustomInstantSerializer.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/CustomInstantSerializer.java index a7c31b6e7..358dd8cb7 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/CustomInstantSerializer.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/CustomInstantSerializer.java @@ -1,35 +1,35 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import java.io.IOException; -import java.time.Instant; -import java.time.format.DateTimeFormatter; - -public class CustomInstantSerializer extends StdSerializer { - - public CustomInstantSerializer() { - this(null); - } - - private CustomInstantSerializer(final Class t) { - super(t); - } - - @Override - public void serialize(final Instant value, final JsonGenerator gen, final SerializerProvider arg2) - throws IOException { - gen.writeString(DateTimeFormatter.ISO_INSTANT.format(value)); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.time.Instant; +import java.time.format.DateTimeFormatter; + +public class CustomInstantSerializer extends StdSerializer { + + public CustomInstantSerializer() { + this(null); + } + + private CustomInstantSerializer(final Class t) { + super(t); + } + + @Override + public void serialize(final Instant value, final JsonGenerator gen, final SerializerProvider arg2) + throws IOException { + gen.writeString(DateTimeFormatter.ISO_INSTANT.format(value)); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/ExternalReferenceService.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/ExternalReferenceService.java similarity index 96% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/ExternalReferenceService.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/ExternalReferenceService.java index e1927953a..3ae34c646 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/ExternalReferenceService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/ExternalReferenceService.java @@ -1,104 +1,104 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.ExternalReference; - -@Service -public class ExternalReferenceService { - private final Map> ontologyMap; - - public ExternalReferenceService() { - ontologyMap = new HashMap<>(); - populateOntologies(ontologyMap); - } - - public String getNickname(final ExternalReference externalReference) { - return ExternalReferenceUtils.getNickname(externalReference); - } - - public Optional getDataId(final ExternalReference externalReference) { - return ExternalReferenceUtils.getDataId(externalReference); - } - - public String getDuoUrl(final String duoCode) { - return ExternalReferenceUtils.getDuoUrl(duoCode); - } - - public String getOntologyTitle(final String ontologyId) { - return ontologyMap.containsKey(ontologyId) ? ontologyMap.get(ontologyId).get("title") : ""; - } - - public String getOntologyDescription(final String ontologyId) { - return ontologyMap.containsKey(ontologyId) ? ontologyMap.get(ontologyId).get("body") : ""; - } - - private void populateOntologies(final Map> ontologyMap) { - Map ontology = new HashMap<>(); - ontology.put("title", "Data use permission"); - ontology.put( - "body", - "A data item that is used to indicate consent permissions for datasets and/or materials, and relates to the purposes for which datasets and/or material might be removed, stored or used"); - ontologyMap.put("DUO:0000001", ontology); - ontology = new HashMap<>(); - ontology.put("title", "No restriction"); - ontology.put("body", "This data use permission indicates there is no restriction on use."); - ontologyMap.put("DUO:0000004", ontology); - ontology = new HashMap<>(); - ontology.put("title", "General research use and clinical care (obsolete)"); - ontology.put( - "body", - "This data use limitation indicates that use is allowed for health/medical/biomedical purposes and other biological research, including the study of population origins or ancestry"); - ontologyMap.put("DUO:0000005", ontology); - ontology = new HashMap<>(); - ontology.put("title", "Health/Medical/Biomedical research and clinical care"); - ontology.put( - "body", - "Use of the data is limited to health/medical/biomedical purposes; does not include the study of populations origin or ancestry"); - ontologyMap.put("DUO:0000006", ontology); - ontology = new HashMap<>(); - ontology.put("title", "Disease specific research"); - ontology.put( - "body", - "This term should be coupled with a term describing a disease from an ontology to specify the disease the restriction applies to"); - ontologyMap.put("DUO:0000007", ontology); - ontology = new HashMap<>(); - ontology.put("title", "Research use only (obsolete)"); - ontology.put( - "body", "This data use limitation indicates that use is limited to research purposes"); - ontologyMap.put("DUO:0000014", ontology); - ontology = new HashMap<>(); - ontology.put("title", "Publication required"); - ontology.put( - "body", - "This data use modifier indicates that requestor agrees to make results of studies using the data available to the larger scientific community"); - ontologyMap.put("DUO:0000018", ontology); - ontology = new HashMap<>(); - ontology.put("title", "Not-for-profit use only"); - ontology.put("body", "Use of the data is limited to not-for-profit organizations"); - ontologyMap.put("DUO:0000019", ontology); - ontology = new HashMap<>(); - ontology.put("title", "User specific restriction"); - ontology.put( - "body", "This data use modifier indicates that use is limited to use by approved users."); - ontologyMap.put("DUO:0000026", ontology); - ontology = new HashMap<>(); - ontology.put("title", "Institution specific restriction"); - ontology.put( - "body", - "This data use modifier indicates that use is limited to use within an approved institution"); - ontologyMap.put("DUO:0000028", ontology); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.ExternalReference; + +@Service +public class ExternalReferenceService { + private final Map> ontologyMap; + + public ExternalReferenceService() { + ontologyMap = new HashMap<>(); + populateOntologies(ontologyMap); + } + + public String getNickname(final ExternalReference externalReference) { + return ExternalReferenceUtils.getNickname(externalReference); + } + + public Optional getDataId(final ExternalReference externalReference) { + return ExternalReferenceUtils.getDataId(externalReference); + } + + public String getDuoUrl(final String duoCode) { + return ExternalReferenceUtils.getDuoUrl(duoCode); + } + + public String getOntologyTitle(final String ontologyId) { + return ontologyMap.containsKey(ontologyId) ? ontologyMap.get(ontologyId).get("title") : ""; + } + + public String getOntologyDescription(final String ontologyId) { + return ontologyMap.containsKey(ontologyId) ? ontologyMap.get(ontologyId).get("body") : ""; + } + + private void populateOntologies(final Map> ontologyMap) { + Map ontology = new HashMap<>(); + ontology.put("title", "Data use permission"); + ontology.put( + "body", + "A data item that is used to indicate consent permissions for datasets and/or materials, and relates to the purposes for which datasets and/or material might be removed, stored or used"); + ontologyMap.put("DUO:0000001", ontology); + ontology = new HashMap<>(); + ontology.put("title", "No restriction"); + ontology.put("body", "This data use permission indicates there is no restriction on use."); + ontologyMap.put("DUO:0000004", ontology); + ontology = new HashMap<>(); + ontology.put("title", "General research use and clinical care (obsolete)"); + ontology.put( + "body", + "This data use limitation indicates that use is allowed for health/medical/biomedical purposes and other biological research, including the study of population origins or ancestry"); + ontologyMap.put("DUO:0000005", ontology); + ontology = new HashMap<>(); + ontology.put("title", "Health/Medical/Biomedical research and clinical care"); + ontology.put( + "body", + "Use of the data is limited to health/medical/biomedical purposes; does not include the study of populations origin or ancestry"); + ontologyMap.put("DUO:0000006", ontology); + ontology = new HashMap<>(); + ontology.put("title", "Disease specific research"); + ontology.put( + "body", + "This term should be coupled with a term describing a disease from an ontology to specify the disease the restriction applies to"); + ontologyMap.put("DUO:0000007", ontology); + ontology = new HashMap<>(); + ontology.put("title", "Research use only (obsolete)"); + ontology.put( + "body", "This data use limitation indicates that use is limited to research purposes"); + ontologyMap.put("DUO:0000014", ontology); + ontology = new HashMap<>(); + ontology.put("title", "Publication required"); + ontology.put( + "body", + "This data use modifier indicates that requestor agrees to make results of studies using the data available to the larger scientific community"); + ontologyMap.put("DUO:0000018", ontology); + ontology = new HashMap<>(); + ontology.put("title", "Not-for-profit use only"); + ontology.put("body", "Use of the data is limited to not-for-profit organizations"); + ontologyMap.put("DUO:0000019", ontology); + ontology = new HashMap<>(); + ontology.put("title", "User specific restriction"); + ontology.put( + "body", "This data use modifier indicates that use is limited to use by approved users."); + ontologyMap.put("DUO:0000026", ontology); + ontology = new HashMap<>(); + ontology.put("title", "Institution specific restriction"); + ontology.put( + "body", + "This data use modifier indicates that use is limited to use within an approved institution"); + ontologyMap.put("DUO:0000028", ontology); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/ExternalReferenceUtils.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/ExternalReferenceUtils.java similarity index 95% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/ExternalReferenceUtils.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/ExternalReferenceUtils.java index d5c546bbf..d03284962 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/ExternalReferenceUtils.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/ExternalReferenceUtils.java @@ -1,89 +1,89 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.util.Optional; -import org.springframework.web.util.UriComponents; -import org.springframework.web.util.UriComponentsBuilder; -import uk.ac.ebi.biosamples.model.ExternalReference; - -public class ExternalReferenceUtils { - private ExternalReferenceUtils() { - // private - } - - private static final String ENA_BASE_URL_FRAGMENT = "www.ebi.ac.uk/ena"; - private static final String ARRAYEXPRESS_BASE_URL_FRAGMENT = "www.ebi.ac.uk/arrayexpress"; - private static final String HPSCREG_URL_FRAGMENT = "hpscreg.eu/"; - private static final String DBGAP_BASE_URL_FRAGMENT = "ncbi.nlm.nih.gov/projects/gap"; - private static final String EGA_DATASET_BASE_URL_FRAGMENT = "ega-archive.org/datasets"; - private static final String EGA_SAMPLE_BASE_URL_FRAGMENT = "ega-archive.org/metadata"; - private static final String EGA_STUDY_BASE_URL_FRAGMENT = "ega-archive.org/studies"; - private static final String BIOSTUDIES_BASE_URL_FRAGMENT = "ebi.ac.uk/biostudies"; - private static final String EVA_BASE_URL_FRAGMENT = "ebi.ac.uk/eva"; - private static final String DUO_BASE_URL = "http://purl.obolibrary.org/obo/"; - - public static String getNickname(final ExternalReference externalReference) { - if (externalReference.getUrl().contains(ENA_BASE_URL_FRAGMENT)) { - return "ENA"; - } else if (externalReference.getUrl().contains(ARRAYEXPRESS_BASE_URL_FRAGMENT)) { - return "ArrayExpress"; - } else if (externalReference.getUrl().contains(HPSCREG_URL_FRAGMENT)) { - return "hPSCreg"; - } else if (externalReference.getUrl().contains(DBGAP_BASE_URL_FRAGMENT)) { - return "dbGaP"; - } else if (externalReference.getUrl().contains(EGA_DATASET_BASE_URL_FRAGMENT)) { - return "EGA Dataset"; - } else if (externalReference.getUrl().contains(EGA_SAMPLE_BASE_URL_FRAGMENT)) { - return "EGA Sample"; - } else if (externalReference.getUrl().contains(EGA_STUDY_BASE_URL_FRAGMENT)) { - return "EGA Study"; - } else if (externalReference.getUrl().contains(BIOSTUDIES_BASE_URL_FRAGMENT)) { - return "BioStudies"; - } else if (externalReference.getUrl().contains(EVA_BASE_URL_FRAGMENT)) { - return "EVA"; - } else { - return "other"; - } - } - - public static Optional getDataId(final ExternalReference externalReference) { - final String nickname = getNickname(externalReference); - if ("ENA".equals(nickname) - || "ArrayExpress".equals(nickname) - || "hPSCreg".equals(nickname) - || "EGA Dataset".equals(nickname) - || "EGA Sample".equals(nickname) - || "EGA Study".equals(nickname) - || "BioStudies".equals(nickname)) { - final UriComponents uriComponents = - UriComponentsBuilder.fromHttpUrl(externalReference.getUrl()).build(); - final String lastPathSegment = - uriComponents.getPathSegments().get(uriComponents.getPathSegments().size() - 1); - return Optional.of(lastPathSegment); - } else if ("dbGaP".equals(nickname)) { - final UriComponents uriComponents = - UriComponentsBuilder.fromHttpUrl(externalReference.getUrl()).build(); - final String studyId = uriComponents.getQueryParams().getFirst("study_id"); - return Optional.of(studyId); - } else if ("EVA".equals(nickname)) { - final UriComponents uriComponents = - UriComponentsBuilder.fromHttpUrl(externalReference.getUrl()).build(); - final String evaStudyId = uriComponents.getQueryParams().getFirst("eva-study"); - return Optional.of(evaStudyId); - } - return Optional.empty(); - } - - static String getDuoUrl(final String duoCode) { - return DUO_BASE_URL + duoCode.replace(":", "_"); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import java.util.Optional; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; +import uk.ac.ebi.biosamples.core.model.ExternalReference; + +public class ExternalReferenceUtils { + private ExternalReferenceUtils() { + // private + } + + private static final String ENA_BASE_URL_FRAGMENT = "www.ebi.ac.uk/ena"; + private static final String ARRAYEXPRESS_BASE_URL_FRAGMENT = "www.ebi.ac.uk/arrayexpress"; + private static final String HPSCREG_URL_FRAGMENT = "hpscreg.eu/"; + private static final String DBGAP_BASE_URL_FRAGMENT = "ncbi.nlm.nih.gov/projects/gap"; + private static final String EGA_DATASET_BASE_URL_FRAGMENT = "ega-archive.org/datasets"; + private static final String EGA_SAMPLE_BASE_URL_FRAGMENT = "ega-archive.org/metadata"; + private static final String EGA_STUDY_BASE_URL_FRAGMENT = "ega-archive.org/studies"; + private static final String BIOSTUDIES_BASE_URL_FRAGMENT = "ebi.ac.uk/biostudies"; + private static final String EVA_BASE_URL_FRAGMENT = "ebi.ac.uk/eva"; + private static final String DUO_BASE_URL = "http://purl.obolibrary.org/obo/"; + + public static String getNickname(final ExternalReference externalReference) { + if (externalReference.getUrl().contains(ENA_BASE_URL_FRAGMENT)) { + return "ENA"; + } else if (externalReference.getUrl().contains(ARRAYEXPRESS_BASE_URL_FRAGMENT)) { + return "ArrayExpress"; + } else if (externalReference.getUrl().contains(HPSCREG_URL_FRAGMENT)) { + return "hPSCreg"; + } else if (externalReference.getUrl().contains(DBGAP_BASE_URL_FRAGMENT)) { + return "dbGaP"; + } else if (externalReference.getUrl().contains(EGA_DATASET_BASE_URL_FRAGMENT)) { + return "EGA Dataset"; + } else if (externalReference.getUrl().contains(EGA_SAMPLE_BASE_URL_FRAGMENT)) { + return "EGA Sample"; + } else if (externalReference.getUrl().contains(EGA_STUDY_BASE_URL_FRAGMENT)) { + return "EGA Study"; + } else if (externalReference.getUrl().contains(BIOSTUDIES_BASE_URL_FRAGMENT)) { + return "BioStudies"; + } else if (externalReference.getUrl().contains(EVA_BASE_URL_FRAGMENT)) { + return "EVA"; + } else { + return "other"; + } + } + + public static Optional getDataId(final ExternalReference externalReference) { + final String nickname = getNickname(externalReference); + if ("ENA".equals(nickname) + || "ArrayExpress".equals(nickname) + || "hPSCreg".equals(nickname) + || "EGA Dataset".equals(nickname) + || "EGA Sample".equals(nickname) + || "EGA Study".equals(nickname) + || "BioStudies".equals(nickname)) { + final UriComponents uriComponents = + UriComponentsBuilder.fromHttpUrl(externalReference.getUrl()).build(); + final String lastPathSegment = + uriComponents.getPathSegments().get(uriComponents.getPathSegments().size() - 1); + return Optional.of(lastPathSegment); + } else if ("dbGaP".equals(nickname)) { + final UriComponents uriComponents = + UriComponentsBuilder.fromHttpUrl(externalReference.getUrl()).build(); + final String studyId = uriComponents.getQueryParams().getFirst("study_id"); + return Optional.of(studyId); + } else if ("EVA".equals(nickname)) { + final UriComponents uriComponents = + UriComponentsBuilder.fromHttpUrl(externalReference.getUrl()).build(); + final String evaStudyId = uriComponents.getQueryParams().getFirst("eva-study"); + return Optional.of(evaStudyId); + } + return Optional.empty(); + } + + static String getDuoUrl(final String duoCode) { + return DUO_BASE_URL + duoCode.replace(":", "_"); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/FileDownloadSerializer.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/FileDownloadSerializer.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/FileDownloadSerializer.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/FileDownloadSerializer.java index 44801b827..83379949a 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/FileDownloadSerializer.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/FileDownloadSerializer.java @@ -1,108 +1,108 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import java.io.IOException; -import uk.ac.ebi.biosamples.model.Sample; - -public interface FileDownloadSerializer { - - static FileDownloadSerializer getSerializerFor(final String format) { - final FileDownloadSerializer serializer; - if ("txt".equalsIgnoreCase(format)) { - serializer = new FileDownloadAccessionsSerializer(); - } else if ("xml".equalsIgnoreCase(format)) { - serializer = new FileDownloadXmlSerializer(); - } else { - serializer = new FileDownloadJsonSerializer(); - } - return serializer; - } - - String asString(Sample sample) throws IOException; - - String startOfFile(); - - String endOfFile(); - - String delimiter(); - - class FileDownloadJsonSerializer implements FileDownloadSerializer { - private final ObjectMapper objectMapper = - new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); - - @Override - public String asString(final Sample sample) throws IOException { - return objectMapper.writeValueAsString(sample); - } - - @Override - public String startOfFile() { - return "["; - } - - @Override - public String endOfFile() { - return "]"; - } - - @Override - public String delimiter() { - return "," + System.lineSeparator(); - } - } - - class FileDownloadXmlSerializer implements FileDownloadSerializer { - @Override - public String asString(final Sample sample) { - throw new RuntimeException("XML not a supported format"); - } - - @Override - public String startOfFile() { - return "" + System.lineSeparator(); - } - - @Override - public String endOfFile() { - return ""; - } - - @Override - public String delimiter() { - return System.lineSeparator(); - } - } - - class FileDownloadAccessionsSerializer implements FileDownloadSerializer { - @Override - public String asString(final Sample sample) { - return sample.getAccession(); - } - - @Override - public String startOfFile() { - return ""; - } - - @Override - public String endOfFile() { - return ""; - } - - @Override - public String delimiter() { - return System.lineSeparator(); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import java.io.IOException; +import uk.ac.ebi.biosamples.core.model.Sample; + +public interface FileDownloadSerializer { + + static FileDownloadSerializer getSerializerFor(final String format) { + final FileDownloadSerializer serializer; + if ("txt".equalsIgnoreCase(format)) { + serializer = new FileDownloadAccessionsSerializer(); + } else if ("xml".equalsIgnoreCase(format)) { + serializer = new FileDownloadXmlSerializer(); + } else { + serializer = new FileDownloadJsonSerializer(); + } + return serializer; + } + + String asString(Sample sample) throws IOException; + + String startOfFile(); + + String endOfFile(); + + String delimiter(); + + class FileDownloadJsonSerializer implements FileDownloadSerializer { + private final ObjectMapper objectMapper = + new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); + + @Override + public String asString(final Sample sample) throws IOException { + return objectMapper.writeValueAsString(sample); + } + + @Override + public String startOfFile() { + return "["; + } + + @Override + public String endOfFile() { + return "]"; + } + + @Override + public String delimiter() { + return "," + System.lineSeparator(); + } + } + + class FileDownloadXmlSerializer implements FileDownloadSerializer { + @Override + public String asString(final Sample sample) { + throw new RuntimeException("XML not a supported format"); + } + + @Override + public String startOfFile() { + return "" + System.lineSeparator(); + } + + @Override + public String endOfFile() { + return ""; + } + + @Override + public String delimiter() { + return System.lineSeparator(); + } + } + + class FileDownloadAccessionsSerializer implements FileDownloadSerializer { + @Override + public String asString(final Sample sample) { + return sample.getAccession(); + } + + @Override + public String startOfFile() { + return ""; + } + + @Override + public String endOfFile() { + return ""; + } + + @Override + public String delimiter() { + return System.lineSeparator(); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/FilterBuilder.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/FilterBuilder.java similarity index 94% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/FilterBuilder.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/FilterBuilder.java index 57ff24bb6..33fe4df30 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/FilterBuilder.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/FilterBuilder.java @@ -1,86 +1,86 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.filter.*; - -@Service -public class FilterBuilder { - public AttributeFilter.Builder onAttribute(final String label) { - return new AttributeFilter.Builder(label); - } - - public RelationFilter.Builder onRelation(final String label) { - return new RelationFilter.Builder(label); - } - - public InverseRelationFilter.Builder onInverseRelation(final String label) { - return new InverseRelationFilter.Builder(label); - } - - public DateRangeFilter.DateRangeFilterBuilder onDate(final String fieldLabel) { - return new DateRangeFilter.DateRangeFilterBuilder(fieldLabel); - } - - public DateRangeFilter.DateRangeFilterBuilder onReleaseDate() { - return new DateRangeFilter.DateRangeFilterBuilder("release"); - } - - public DateRangeFilter.DateRangeFilterBuilder onUpdateDate() { - return new DateRangeFilter.DateRangeFilterBuilder("update"); - } - - public AuthenticationFilter.Builder onAuthInfo(final String domain) { - return new AuthenticationFilter.Builder(domain); - } - - public NameFilter.Builder onName(final String name) { - return new NameFilter.Builder(name); - } - - public AccessionFilter.Builder onAccession(final String accession) { - return new AccessionFilter.Builder(accession); - } - - public ExternalReferenceDataFilter.Builder onDataFromExternalReference( - final String extReference) { - return new ExternalReferenceDataFilter.Builder(extReference); - } - - public Filter buildFromString(final String serializedFilter) { - final FilterType filterType = FilterType.ofFilterString(serializedFilter); - final List filterParts = filterParts(serializedFilter); - - if (filterParts.size() > 2) { - return filterType - .getBuilderForLabel(filterParts.get(1)) - .parseContent(filterParts.get(2)) - .build(); - } else { - return filterType.getBuilderForLabel(filterParts.get(1)).build(); - } - } - - private List filterParts(final String filterLabelAndValue) { - // TODO hack, need to be improved - return Arrays.stream(filterLabelAndValue.split("(? s.replace("\\:", ":")) - .collect(Collectors.toList()); - } - - public static FilterBuilder create() { - return new FilterBuilder(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.filter.*; + +@Service +public class FilterBuilder { + public AttributeFilter.Builder onAttribute(final String label) { + return new AttributeFilter.Builder(label); + } + + public RelationFilter.Builder onRelation(final String label) { + return new RelationFilter.Builder(label); + } + + public InverseRelationFilter.Builder onInverseRelation(final String label) { + return new InverseRelationFilter.Builder(label); + } + + public DateRangeFilter.DateRangeFilterBuilder onDate(final String fieldLabel) { + return new DateRangeFilter.DateRangeFilterBuilder(fieldLabel); + } + + public DateRangeFilter.DateRangeFilterBuilder onReleaseDate() { + return new DateRangeFilter.DateRangeFilterBuilder("release"); + } + + public DateRangeFilter.DateRangeFilterBuilder onUpdateDate() { + return new DateRangeFilter.DateRangeFilterBuilder("update"); + } + + public AuthenticationFilter.Builder onAuthInfo(final String domain) { + return new AuthenticationFilter.Builder(domain); + } + + public NameFilter.Builder onName(final String name) { + return new NameFilter.Builder(name); + } + + public AccessionFilter.Builder onAccession(final String accession) { + return new AccessionFilter.Builder(accession); + } + + public ExternalReferenceDataFilter.Builder onDataFromExternalReference( + final String extReference) { + return new ExternalReferenceDataFilter.Builder(extReference); + } + + public Filter buildFromString(final String serializedFilter) { + final FilterType filterType = FilterType.ofFilterString(serializedFilter); + final List filterParts = filterParts(serializedFilter); + + if (filterParts.size() > 2) { + return filterType + .getBuilderForLabel(filterParts.get(1)) + .parseContent(filterParts.get(2)) + .build(); + } else { + return filterType.getBuilderForLabel(filterParts.get(1)).build(); + } + } + + private List filterParts(final String filterLabelAndValue) { + // TODO hack, need to be improved + return Arrays.stream(filterLabelAndValue.split("(? s.replace("\\:", ":")) + .collect(Collectors.toList()); + } + + public static FilterBuilder create() { + return new FilterBuilder(); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/HttpOlsUrlResolutionService.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/HttpOlsUrlResolutionService.java similarity index 95% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/HttpOlsUrlResolutionService.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/HttpOlsUrlResolutionService.java index 49d5bf742..3958e0b87 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/HttpOlsUrlResolutionService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/HttpOlsUrlResolutionService.java @@ -1,94 +1,94 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.util.SortedSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; -import org.springframework.web.util.UriComponents; -import org.springframework.web.util.UriComponentsBuilder; - -@Service -public class HttpOlsUrlResolutionService { - private static final String OLS_PREFIX = "https://www.ebi.ac.uk/ols4?termId="; - private static final String OBO_URL_SEGMENT = "purl.obolibrary.org/obo"; - private static final String EBI_URL_SEGMENT = "www.ebi.ac.uk"; - public static Logger log = LoggerFactory.getLogger(HttpOlsUrlResolutionService.class); - - public HttpOlsUrlResolutionService() {} - - public static boolean isCurie(String term) { - String[] segments; - if (term.contains(":")) { - segments = term.split(":"); - } else if (term.contains("_")) { - segments = term.split("_"); - } else { - return false; - } - - return segments.length == 2 && segments[0].matches("[a-zA-Z]+") && segments[1].matches("\\d+"); - } - - private static String formatCurie(String displayIri) { - return displayIri.trim().replace("_", ":"); - } - - /** - * This returns a string representation of the URL to lookup the associated ontology term iri in - * EBI OLS. - * - * @return url representation of the IRI - */ - // @Cacheable(value = "iri") - public String getIriOls(final SortedSet iri) { - if (iri == null || iri.size() == 0) { - return null; - } - - final String displayIri = iri.first(); - - if (isCurie(displayIri)) { - return formatCurie(displayIri); - } - - // check this is a sane iri - try { - final UriComponents iriComponents = - UriComponentsBuilder.fromUriString(displayIri).build(true); - - if (iriComponents.getScheme() == null - || iriComponents.getHost() == null - || iriComponents.getPath() == null) { - // incomplete iri (e.g. 9606, EFO_12345) don't bother to check - return null; - } - - String[] segments = displayIri.split("/"); - String curie = segments[segments.length - 1].replace("_", ":"); - - if (checkUrlForPattern(displayIri)) { - return OLS_PREFIX + curie; - } else { - return displayIri; - } - } catch (final Exception e) { - // FIXME: Can't use a non static logger here because - log.error("An error occurred while trying to build OLS iri for " + displayIri, e); - return null; - } - } - - private boolean checkUrlForPattern(final String displayIri) { - return displayIri.contains(OBO_URL_SEGMENT) || displayIri.contains(EBI_URL_SEGMENT); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import java.util.SortedSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; + +@Service +public class HttpOlsUrlResolutionService { + private static final String OLS_PREFIX = "https://www.ebi.ac.uk/ols4?termId="; + private static final String OBO_URL_SEGMENT = "purl.obolibrary.org/obo"; + private static final String EBI_URL_SEGMENT = "www.ebi.ac.uk"; + public static Logger log = LoggerFactory.getLogger(HttpOlsUrlResolutionService.class); + + public HttpOlsUrlResolutionService() {} + + public static boolean isCurie(String term) { + String[] segments; + if (term.contains(":")) { + segments = term.split(":"); + } else if (term.contains("_")) { + segments = term.split("_"); + } else { + return false; + } + + return segments.length == 2 && segments[0].matches("[a-zA-Z]+") && segments[1].matches("\\d+"); + } + + private static String formatCurie(String displayIri) { + return displayIri.trim().replace("_", ":"); + } + + /** + * This returns a string representation of the URL to lookup the associated ontology term iri in + * EBI OLS. + * + * @return url representation of the IRI + */ + // @Cacheable(value = "iri") + public String getIriOls(final SortedSet iri) { + if (iri == null || iri.size() == 0) { + return null; + } + + final String displayIri = iri.first(); + + if (isCurie(displayIri)) { + return formatCurie(displayIri); + } + + // check this is a sane iri + try { + final UriComponents iriComponents = + UriComponentsBuilder.fromUriString(displayIri).build(true); + + if (iriComponents.getScheme() == null + || iriComponents.getHost() == null + || iriComponents.getPath() == null) { + // incomplete iri (e.g. 9606, EFO_12345) don't bother to check + return null; + } + + String[] segments = displayIri.split("/"); + String curie = segments[segments.length - 1].replace("_", ":"); + + if (checkUrlForPattern(displayIri)) { + return OLS_PREFIX + curie; + } else { + return displayIri; + } + } catch (final Exception e) { + // FIXME: Can't use a non static logger here because + log.error("An error occurred while trying to build OLS iri for " + displayIri, e); + return null; + } + } + + private boolean checkUrlForPattern(final String displayIri) { + return displayIri.contains(OBO_URL_SEGMENT) || displayIri.contains(EBI_URL_SEGMENT); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/RelationshipValidator.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/RelationshipValidator.java similarity index 92% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/RelationshipValidator.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/RelationshipValidator.java index 74c962644..44c7c6e29 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/RelationshipValidator.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/RelationshipValidator.java @@ -1,64 +1,64 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.util.ArrayList; -import java.util.Collection; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.AccessionType; -import uk.ac.ebi.biosamples.model.Relationship; - -@Service -public class RelationshipValidator { - - public Collection validate(final Relationship rel) { - return validate(rel, new ArrayList<>()); - } - - public Collection validate(final Relationship rel, final String accession) { - // TODO validate that relationships have this sample as the source - final Collection errors = validate(rel, new ArrayList<>()); - return validateSourceAccession(accession, rel, errors); - } - - public Collection validate(final Relationship rel, final Collection errors) { - if (rel.getSource() == null || rel.getSource().isEmpty()) { - // errors.add("Source of a relationship must be non empty");//todo re-enable - // after - // samepletab deprecation - } else if (!AccessionType.ANY.matches(rel.getSource())) { - errors.add( - "Source of a relationship must be an accession but was \"" + rel.getSource() + "\""); - } - - if (rel.getTarget() == null || rel.getTarget().isEmpty()) { - // errors.add("Target of a relationship must be non empty");//todo re-enable - // after - // samepletab deprecation - } else if (!AccessionType.ANY.matches(rel.getTarget())) { - errors.add( - "Target of a relationship must be an accession but was \"" + rel.getTarget() + "\""); - } - - return errors; - } - - private Collection validateSourceAccession( - final String accession, final Relationship rel, final Collection errors) { - if (accession != null && !accession.equals(rel.getSource())) { - // errors.add("Source of the relationship must equal to the sample - // accession");// - // todo enable after fixing ENA import pipeline - } - - return errors; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import java.util.ArrayList; +import java.util.Collection; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.AccessionType; +import uk.ac.ebi.biosamples.core.model.Relationship; + +@Service +public class RelationshipValidator { + + public Collection validate(final Relationship rel) { + return validate(rel, new ArrayList<>()); + } + + public Collection validate(final Relationship rel, final String accession) { + // TODO validate that relationships have this sample as the source + final Collection errors = validate(rel, new ArrayList<>()); + return validateSourceAccession(accession, rel, errors); + } + + public Collection validate(final Relationship rel, final Collection errors) { + if (rel.getSource() == null || rel.getSource().isEmpty()) { + // errors.add("Source of a relationship must be non empty");//todo re-enable + // after + // samepletab deprecation + } else if (!AccessionType.ANY.matches(rel.getSource())) { + errors.add( + "Source of a relationship must be an accession but was \"" + rel.getSource() + "\""); + } + + if (rel.getTarget() == null || rel.getTarget().isEmpty()) { + // errors.add("Target of a relationship must be non empty");//todo re-enable + // after + // samepletab deprecation + } else if (!AccessionType.ANY.matches(rel.getTarget())) { + errors.add( + "Target of a relationship must be an accession but was \"" + rel.getTarget() + "\""); + } + + return errors; + } + + private Collection validateSourceAccession( + final String accession, final Relationship rel, final Collection errors) { + if (accession != null && !accession.equals(rel.getSource())) { + // errors.add("Source of the relationship must equal to the sample + // accession");// + // todo enable after fixing ENA import pipeline + } + + return errors; + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/SampleRelationshipUtils.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/SampleRelationshipUtils.java similarity index 91% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/SampleRelationshipUtils.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/SampleRelationshipUtils.java index d8e1c8373..507c6d641 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/SampleRelationshipUtils.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/SampleRelationshipUtils.java @@ -1,48 +1,48 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.stream.Collectors; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; - -public class SampleRelationshipUtils { - - /** - * Given a sample, get the collection of relationships where this sample is the source. - * - *

Sample will be the source if it matches the accession, or if it is blank if the sample has - * no accession. - */ - public static SortedSet getOutgoingRelationships(final Sample sample) { - final SortedSet relationships = new TreeSet<>(); - for (final Relationship relationship : sample.getRelationships()) { - if (!sample.hasAccession() - && (relationship.getSource() == null || relationship.getSource().trim().length() == 0)) { - relationships.add(relationship); - } else if (relationship.getSource() != null - && relationship.getSource().equals(sample.getAccession())) { - relationships.add(relationship); - } - } - - return relationships; - } - - /** Given a sample, get the collection of relationships where this sample is the target. */ - public static SortedSet getIncomingRelationships(final Sample sample) { - return sample.getRelationships().stream() - .filter(rel -> rel.getTarget().equals(sample.getAccession())) - .collect(Collectors.toCollection(TreeSet::new)); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.stream.Collectors; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Sample; + +public class SampleRelationshipUtils { + + /** + * Given a sample, get the collection of relationships where this sample is the source. + * + *

Sample will be the source if it matches the accession, or if it is blank if the sample has + * no accession. + */ + public static SortedSet getOutgoingRelationships(final Sample sample) { + final SortedSet relationships = new TreeSet<>(); + for (final Relationship relationship : sample.getRelationships()) { + if (!sample.hasAccession() + && (relationship.getSource() == null || relationship.getSource().trim().length() == 0)) { + relationships.add(relationship); + } else if (relationship.getSource() != null + && relationship.getSource().equals(sample.getAccession())) { + relationships.add(relationship); + } + } + + return relationships; + } + + /** Given a sample, get the collection of relationships where this sample is the target. */ + public static SortedSet getIncomingRelationships(final Sample sample) { + return sample.getRelationships().stream() + .filter(rel -> rel.getTarget().equals(sample.getAccession())) + .collect(Collectors.toCollection(TreeSet::new)); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/SampleUtils.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/SampleUtils.java similarity index 94% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/SampleUtils.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/SampleUtils.java index be9c20172..1a7d764f3 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/SampleUtils.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/SampleUtils.java @@ -1,120 +1,120 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; - -@Service -public class SampleUtils { - - public String getAccessionPattern() { - return "SAM[END][AG]?[0-9]+"; - } - - /** - * Get an optional list of attributes matching the provided type without case sensitivity - * - * @param sample the sample to check - * @param attributeType the type to use as filter - * @return an optional list of attributes - */ - public Optional> getAttributesWithType( - final Sample sample, final String attributeType) { - return getAttributesWithType(sample, attributeType, false); - } - - /** - * Get an optional list of attributes matching the provided type - * - * @param sample the sample to check - * @param attributeType the type to use as filter - * @param useCaseSensitivity if the match should be case sensitive - * @return an optional list of attributes - */ - private Optional> getAttributesWithType( - final Sample sample, final String attributeType, final boolean useCaseSensitivity) { - return Optional.ofNullable( - sample.getCharacteristics().stream() - .filter( - attr -> - useCaseSensitivity - ? attr.getType().equals(attributeType) - : attr.getType().equalsIgnoreCase(attributeType)) - .collect(Collectors.toList())); - } - - /** - * Get an optional list of attributes matching the provided type regular expression - * - * @param sample the sample to check - * @param typeRegex the regular expression to match the type - * @return an optional list of attributes - */ - public Optional> getAttributesWithTypeMatching( - final Sample sample, final String typeRegex) { - return Optional.ofNullable( - sample.getCharacteristics().stream() - .filter(attr -> attr.getType().matches(typeRegex)) - .collect(Collectors.toList())); - } - - /** - * Get an optional list of attributes matching the provided value - * - * @param sample the sample to check - * @param attributeValue the attribute to use as filter - * @param useCaseSensitivity if the match should be case sensitive - * @return an optional list of attributes - */ - private Optional> getAttributesWithValue( - final Sample sample, final String attributeValue, final boolean useCaseSensitivity) { - return Optional.ofNullable( - sample.getCharacteristics().stream() - .filter( - attr -> - useCaseSensitivity - ? attr.getValue().equals(attributeValue) - : attr.getValue().equalsIgnoreCase(attributeValue)) - .collect(Collectors.toList())); - } - - /** - * Get an optional list of attributes matching the provided value without case sensitivity - * - * @param sample the sample to check - * @param attributeValue the attribute to use as filter - * @return an optional list of attributes - */ - public Optional> getAttributesWithValue( - final Sample sample, final String attributeValue) { - return getAttributesWithValue(sample, attributeValue, false); - } - - /** - * Get an optional list of attributes matching the provided value regular expression - * - * @param sample the sample to check - * @param valueRegex the attribute value regular expression - * @return an optional list of attributes - */ - public Optional> getAttributesWithValueMatching( - final Sample sample, final String valueRegex) { - return Optional.ofNullable( - sample.getCharacteristics().stream() - .filter(attr -> attr.getValue().matches(valueRegex)) - .collect(Collectors.toList())); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; + +@Service +public class SampleUtils { + + public String getAccessionPattern() { + return "SAM[END][AG]?[0-9]+"; + } + + /** + * Get an optional list of attributes matching the provided type without case sensitivity + * + * @param sample the sample to check + * @param attributeType the type to use as filter + * @return an optional list of attributes + */ + public Optional> getAttributesWithType( + final Sample sample, final String attributeType) { + return getAttributesWithType(sample, attributeType, false); + } + + /** + * Get an optional list of attributes matching the provided type + * + * @param sample the sample to check + * @param attributeType the type to use as filter + * @param useCaseSensitivity if the match should be case sensitive + * @return an optional list of attributes + */ + private Optional> getAttributesWithType( + final Sample sample, final String attributeType, final boolean useCaseSensitivity) { + return Optional.ofNullable( + sample.getCharacteristics().stream() + .filter( + attr -> + useCaseSensitivity + ? attr.getType().equals(attributeType) + : attr.getType().equalsIgnoreCase(attributeType)) + .collect(Collectors.toList())); + } + + /** + * Get an optional list of attributes matching the provided type regular expression + * + * @param sample the sample to check + * @param typeRegex the regular expression to match the type + * @return an optional list of attributes + */ + public Optional> getAttributesWithTypeMatching( + final Sample sample, final String typeRegex) { + return Optional.ofNullable( + sample.getCharacteristics().stream() + .filter(attr -> attr.getType().matches(typeRegex)) + .collect(Collectors.toList())); + } + + /** + * Get an optional list of attributes matching the provided value + * + * @param sample the sample to check + * @param attributeValue the attribute to use as filter + * @param useCaseSensitivity if the match should be case sensitive + * @return an optional list of attributes + */ + private Optional> getAttributesWithValue( + final Sample sample, final String attributeValue, final boolean useCaseSensitivity) { + return Optional.ofNullable( + sample.getCharacteristics().stream() + .filter( + attr -> + useCaseSensitivity + ? attr.getValue().equals(attributeValue) + : attr.getValue().equalsIgnoreCase(attributeValue)) + .collect(Collectors.toList())); + } + + /** + * Get an optional list of attributes matching the provided value without case sensitivity + * + * @param sample the sample to check + * @param attributeValue the attribute to use as filter + * @return an optional list of attributes + */ + public Optional> getAttributesWithValue( + final Sample sample, final String attributeValue) { + return getAttributesWithValue(sample, attributeValue, false); + } + + /** + * Get an optional list of attributes matching the provided value regular expression + * + * @param sample the sample to check + * @param valueRegex the attribute value regular expression + * @return an optional list of attributes + */ + public Optional> getAttributesWithValueMatching( + final Sample sample, final String valueRegex) { + return Optional.ofNullable( + sample.getCharacteristics().stream() + .filter(attr -> attr.getValue().matches(valueRegex)) + .collect(Collectors.toList())); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/SampleValidator.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/SampleValidator.java similarity index 90% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/SampleValidator.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/SampleValidator.java index 744c31cf9..328f8cf48 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/SampleValidator.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/SampleValidator.java @@ -1,82 +1,82 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; - -@Service -public class SampleValidator { - private final AttributeValidator attributeValidator; - private final RelationshipValidator relationshipValidator; - - public SampleValidator(final AttributeValidator attributeValidator) { - this.attributeValidator = attributeValidator; - this.relationshipValidator = new RelationshipValidator(); - } - - public Collection validate(final Sample sample) { - final Collection errors = new ArrayList<>(); - - validate(sample, errors); - - return errors; - } - - public List validate(final Map sampleAsMap) { - final List errors = new ArrayList<>(); - - if (sampleAsMap.get("release") == null) { - errors.add("Must provide release date in format YYYY-MM-DDTHH:MM:SS"); - } - - if (sampleAsMap.get("name") == null) { - errors.add("Must provide name"); - } - - final ObjectMapper mapper = new ObjectMapper(); - - try { - final Sample sample = mapper.convertValue(sampleAsMap, Sample.class); - validate(sample, errors); - } catch (final IllegalArgumentException e) { - errors.add(e.getMessage()); - } - - return errors; - } - - public void validate(final Sample sample, final Collection errors) { - if (sample.getRelease() == null) { - errors.add("Must provide release date in format YYYY-MM-DDTHH:MM:SS"); - } - - if (sample.getName() == null) { - errors.add("Must provide name"); - } - - // TODO more validation - for (final Attribute attribute : sample.getAttributes()) { - attributeValidator.validate(attribute, errors); - } - - for (final Relationship rel : sample.getRelationships()) { - errors.addAll(relationshipValidator.validate(rel, sample.getAccession())); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Sample; + +@Service +public class SampleValidator { + private final AttributeValidator attributeValidator; + private final RelationshipValidator relationshipValidator; + + public SampleValidator(final AttributeValidator attributeValidator) { + this.attributeValidator = attributeValidator; + this.relationshipValidator = new RelationshipValidator(); + } + + public Collection validate(final Sample sample) { + final Collection errors = new ArrayList<>(); + + validate(sample, errors); + + return errors; + } + + public List validate(final Map sampleAsMap) { + final List errors = new ArrayList<>(); + + if (sampleAsMap.get("release") == null) { + errors.add("Must provide release date in format YYYY-MM-DDTHH:MM:SS"); + } + + if (sampleAsMap.get("name") == null) { + errors.add("Must provide name"); + } + + final ObjectMapper mapper = new ObjectMapper(); + + try { + final Sample sample = mapper.convertValue(sampleAsMap, Sample.class); + validate(sample, errors); + } catch (final IllegalArgumentException e) { + errors.add(e.getMessage()); + } + + return errors; + } + + public void validate(final Sample sample, final Collection errors) { + if (sample.getRelease() == null) { + errors.add("Must provide release date in format YYYY-MM-DDTHH:MM:SS"); + } + + if (sample.getName() == null) { + errors.add("Must provide name"); + } + + // TODO more validation + for (final Attribute attribute : sample.getAttributes()) { + attributeValidator.validate(attribute, errors); + } + + for (final Relationship rel : sample.getRelationships()) { + errors.addAll(relationshipValidator.validate(rel, sample.getAccession())); + } + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/FacetContentDeserializer.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/facet/content/FacetContentDeserializer.java similarity index 88% rename from models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/FacetContentDeserializer.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/facet/content/FacetContentDeserializer.java index 6e724e6f4..fb8d9d7b4 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/model/facet/content/FacetContentDeserializer.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/facet/content/FacetContentDeserializer.java @@ -1,52 +1,55 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.facet.content; - -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonNode; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class FacetContentDeserializer extends JsonDeserializer { - @Override - public FacetContent deserialize(final JsonParser p, final DeserializationContext ctxt) - throws IOException { - final JsonNode root = p.readValueAsTree(); - if (root.isArray()) { - // If the json is an array, I'm assuming it to be a LabelCountListContent facet content - return parseLabelCountListContent(root); - } - - throw new JsonParseException(p, "Unable to parse facet content"); - } - - /** - * Deserialize the Json to a LabelCountListContent - * - * @param jsonArray the Json Array representing LabelCountListContent - * @return LabelCountListContent - */ - private LabelCountListContent parseLabelCountListContent(final JsonNode jsonArray) { - final List labelCountList = new ArrayList<>(); - for (final JsonNode labelCountJsonEntry : jsonArray) { - final String label = labelCountJsonEntry.get("label").asText(); - final Long count = labelCountJsonEntry.get("count").asLong(); - - labelCountList.add(LabelCountEntry.build(label, count)); - } - - return new LabelCountListContent(labelCountList); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.service.facet.content; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import uk.ac.ebi.biosamples.core.model.facet.content.FacetContent; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountEntry; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; + +public class FacetContentDeserializer extends JsonDeserializer { + @Override + public FacetContent deserialize(final JsonParser p, final DeserializationContext ctxt) + throws IOException { + final JsonNode root = p.readValueAsTree(); + if (root.isArray()) { + // If the json is an array, I'm assuming it to be a LabelCountListContent facet content + return parseLabelCountListContent(root); + } + + throw new JsonParseException(p, "Unable to parse facet content"); + } + + /** + * Deserialize the Json to a LabelCountListContent + * + * @param jsonArray the Json Array representing LabelCountListContent + * @return LabelCountListContent + */ + private LabelCountListContent parseLabelCountListContent(final JsonNode jsonArray) { + final List labelCountList = new ArrayList<>(); + for (final JsonNode labelCountJsonEntry : jsonArray) { + final String label = labelCountJsonEntry.get("label").asText(); + final Long count = labelCountJsonEntry.get("count").asLong(); + + labelCountList.add(LabelCountEntry.build(label, count)); + } + + return new LabelCountListContent(labelCountList); + } +} diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/structured/AbstractDataDeserializer.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/structured/AbstractDataDeserializer.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/structured/AbstractDataDeserializer.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/structured/AbstractDataDeserializer.java index 6ef1a12bd..8f87d6186 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/structured/AbstractDataDeserializer.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/structured/AbstractDataDeserializer.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.service.structured; +package uk.ac.ebi.biosamples.core.service.structured; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; @@ -18,7 +18,7 @@ import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import uk.ac.ebi.biosamples.model.structured.AbstractData; +import uk.ac.ebi.biosamples.core.model.structured.AbstractData; public class AbstractDataDeserializer extends StdDeserializer { private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/models/core/src/main/java/uk/ac/ebi/biosamples/service/structured/AbstractDataSerializer.java b/core/src/main/java/uk/ac/ebi/biosamples/core/service/structured/AbstractDataSerializer.java similarity index 93% rename from models/core/src/main/java/uk/ac/ebi/biosamples/service/structured/AbstractDataSerializer.java rename to core/src/main/java/uk/ac/ebi/biosamples/core/service/structured/AbstractDataSerializer.java index 787403fa7..cb07310d0 100644 --- a/models/core/src/main/java/uk/ac/ebi/biosamples/service/structured/AbstractDataSerializer.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/core/service/structured/AbstractDataSerializer.java @@ -8,14 +8,14 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.service.structured; +package uk.ac.ebi.biosamples.core.service.structured; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.ser.std.StdSerializer; import java.io.IOException; import java.util.Set; -import uk.ac.ebi.biosamples.model.structured.AbstractData; +import uk.ac.ebi.biosamples.core.model.structured.AbstractData; public class AbstractDataSerializer extends StdSerializer { diff --git a/models/curami/src/main/java/uk/ac/ebi/biosamples/model/AttributeRecommendation.java b/core/src/main/java/uk/ac/ebi/biosamples/curami/model/AttributeRecommendation.java similarity index 95% rename from models/curami/src/main/java/uk/ac/ebi/biosamples/model/AttributeRecommendation.java rename to core/src/main/java/uk/ac/ebi/biosamples/curami/model/AttributeRecommendation.java index 7fd41998a..6e9db960d 100644 --- a/models/curami/src/main/java/uk/ac/ebi/biosamples/model/AttributeRecommendation.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/curami/model/AttributeRecommendation.java @@ -1,76 +1,76 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import java.util.Objects; - -public class AttributeRecommendation implements Comparable { - private final String attribute; - private final String recommendation; - - private AttributeRecommendation(String attribute, String recommendations) { - this.attribute = attribute; - this.recommendation = recommendations; - } - - public String getAttribute() { - return attribute; - } - - public String getRecommendation() { - return recommendation; - } - - @Override - public String toString() { - return attribute + "->" + recommendation; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o instanceof AttributeRecommendation) { - AttributeRecommendation other = (AttributeRecommendation) o; - return Objects.equals(this.getAttribute(), other.getAttribute()) - && Objects.equals(this.getRecommendation(), other.getRecommendation()); - } - return false; - } - - @Override - public int hashCode() { - return Objects.hash(attribute); - } - - @Override - public int compareTo(AttributeRecommendation other) { - return this.attribute.compareTo(other.attribute); - } - - public static class Builder { - private String attribute; - private String recommendation; - - public Builder withAttribute(String attribute) { - this.attribute = attribute; - return this; - } - - public Builder withRecommendation(String recommendations) { - this.recommendation = recommendations; - return this; - } - - public AttributeRecommendation build() { - return new AttributeRecommendation(attribute, recommendation); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.curami.model; + +import java.util.Objects; + +public class AttributeRecommendation implements Comparable { + private final String attribute; + private final String recommendation; + + private AttributeRecommendation(String attribute, String recommendations) { + this.attribute = attribute; + this.recommendation = recommendations; + } + + public String getAttribute() { + return attribute; + } + + public String getRecommendation() { + return recommendation; + } + + @Override + public String toString() { + return attribute + "->" + recommendation; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o instanceof AttributeRecommendation) { + AttributeRecommendation other = (AttributeRecommendation) o; + return Objects.equals(this.getAttribute(), other.getAttribute()) + && Objects.equals(this.getRecommendation(), other.getRecommendation()); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(attribute); + } + + @Override + public int compareTo(AttributeRecommendation other) { + return this.attribute.compareTo(other.attribute); + } + + public static class Builder { + private String attribute; + private String recommendation; + + public Builder withAttribute(String attribute) { + this.attribute = attribute; + return this; + } + + public Builder withRecommendation(String recommendations) { + this.recommendation = recommendations; + return this; + } + + public AttributeRecommendation build() { + return new AttributeRecommendation(attribute, recommendation); + } + } +} diff --git a/models/curami/src/main/java/uk/ac/ebi/biosamples/model/CuramiRecommendation.java b/core/src/main/java/uk/ac/ebi/biosamples/curami/model/CuramiRecommendation.java similarity index 95% rename from models/curami/src/main/java/uk/ac/ebi/biosamples/model/CuramiRecommendation.java rename to core/src/main/java/uk/ac/ebi/biosamples/curami/model/CuramiRecommendation.java index 8f85aa3a1..a38e8fb1d 100644 --- a/models/curami/src/main/java/uk/ac/ebi/biosamples/model/CuramiRecommendation.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/curami/model/CuramiRecommendation.java @@ -1,93 +1,93 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.SortedSet; -import java.util.TreeSet; - -public class CuramiRecommendation { - @JsonProperty("SampleQuality") - private int quality; - - @JsonProperty("knownAttributes") - private SortedSet knownAttributes; - - @JsonProperty("AttributeRecommendations") - private SortedSet attributeRecommendations; - - @JsonProperty("unknownAttributes") - private SortedSet unknownAttributes; - - private CuramiRecommendation( - final int quality, - final SortedSet knownAttributes, - final SortedSet attributeRecommendations, - final SortedSet unknownAttributes) { - this.quality = quality; - this.knownAttributes = knownAttributes; - this.attributeRecommendations = attributeRecommendations; - this.unknownAttributes = unknownAttributes; - } - - public int getQuality() { - return quality; - } - - public SortedSet getKnownAttributes() { - return knownAttributes; - } - - public SortedSet getAttributeRecommendations() { - return attributeRecommendations; - } - - public SortedSet getUnknownAttributes() { - return unknownAttributes; - } - - public static class Builder { - private int quality; - private SortedSet goodAttributes; - private SortedSet badAttributes; - private SortedSet missingAttributes; - - public Builder() { - goodAttributes = new TreeSet<>(); - badAttributes = new TreeSet<>(); - missingAttributes = new TreeSet<>(); - } - - public Builder withQuality(final int quality) { - this.quality = Math.min(Math.max(quality, 0), 100); - return this; - } - - public Builder withGoodAttributes(final SortedSet goodAttributes) { - this.goodAttributes = goodAttributes; - return this; - } - - public Builder withBadAttributes(final SortedSet badAttributes) { - this.badAttributes = badAttributes; - return this; - } - - public Builder withMissingAttributes(final SortedSet missingAttributes) { - this.missingAttributes = missingAttributes; - return this; - } - - public CuramiRecommendation build() { - return new CuramiRecommendation(quality, goodAttributes, badAttributes, missingAttributes); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.curami.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.SortedSet; +import java.util.TreeSet; + +public class CuramiRecommendation { + @JsonProperty("SampleQuality") + private int quality; + + @JsonProperty("knownAttributes") + private SortedSet knownAttributes; + + @JsonProperty("AttributeRecommendations") + private SortedSet attributeRecommendations; + + @JsonProperty("unknownAttributes") + private SortedSet unknownAttributes; + + private CuramiRecommendation( + final int quality, + final SortedSet knownAttributes, + final SortedSet attributeRecommendations, + final SortedSet unknownAttributes) { + this.quality = quality; + this.knownAttributes = knownAttributes; + this.attributeRecommendations = attributeRecommendations; + this.unknownAttributes = unknownAttributes; + } + + public int getQuality() { + return quality; + } + + public SortedSet getKnownAttributes() { + return knownAttributes; + } + + public SortedSet getAttributeRecommendations() { + return attributeRecommendations; + } + + public SortedSet getUnknownAttributes() { + return unknownAttributes; + } + + public static class Builder { + private int quality; + private SortedSet goodAttributes; + private SortedSet badAttributes; + private SortedSet missingAttributes; + + public Builder() { + goodAttributes = new TreeSet<>(); + badAttributes = new TreeSet<>(); + missingAttributes = new TreeSet<>(); + } + + public Builder withQuality(final int quality) { + this.quality = Math.min(Math.max(quality, 0), 100); + return this; + } + + public Builder withGoodAttributes(final SortedSet goodAttributes) { + this.goodAttributes = goodAttributes; + return this; + } + + public Builder withBadAttributes(final SortedSet badAttributes) { + this.badAttributes = badAttributes; + return this; + } + + public Builder withMissingAttributes(final SortedSet missingAttributes) { + this.missingAttributes = missingAttributes; + return this; + } + + public CuramiRecommendation build() { + return new CuramiRecommendation(quality, goodAttributes, badAttributes, missingAttributes); + } + } +} diff --git a/models/curami/src/main/java/uk/ac/ebi/biosamples/model/SampleRecommendation.java b/core/src/main/java/uk/ac/ebi/biosamples/curami/model/SampleRecommendation.java similarity index 91% rename from models/curami/src/main/java/uk/ac/ebi/biosamples/model/SampleRecommendation.java rename to core/src/main/java/uk/ac/ebi/biosamples/curami/model/SampleRecommendation.java index c2857a74a..e3b3ea794 100644 --- a/models/curami/src/main/java/uk/ac/ebi/biosamples/model/SampleRecommendation.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/curami/model/SampleRecommendation.java @@ -1,29 +1,31 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -public class SampleRecommendation { - private CuramiRecommendation recommendations; - private Sample sample; - - public SampleRecommendation(CuramiRecommendation recommendations, Sample sample) { - this.recommendations = recommendations; - this.sample = sample; - } - - public CuramiRecommendation getRecommendations() { - return recommendations; - } - - public Sample getSample() { - return sample; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.curami.model; + +import uk.ac.ebi.biosamples.core.model.Sample; + +public class SampleRecommendation { + private CuramiRecommendation recommendations; + private Sample sample; + + public SampleRecommendation(CuramiRecommendation recommendations, Sample sample) { + this.recommendations = recommendations; + this.sample = sample; + } + + public CuramiRecommendation getRecommendations() { + return recommendations; + } + + public Sample getSample() { + return sample; + } +} diff --git a/models/curami/src/main/java/uk/ac/ebi/biosamples/service/CuramiUtils.java b/core/src/main/java/uk/ac/ebi/biosamples/curami/service/CuramiUtils.java similarity index 89% rename from models/curami/src/main/java/uk/ac/ebi/biosamples/service/CuramiUtils.java rename to core/src/main/java/uk/ac/ebi/biosamples/curami/service/CuramiUtils.java index b0fdc5bb7..ede9151a7 100644 --- a/models/curami/src/main/java/uk/ac/ebi/biosamples/service/CuramiUtils.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/curami/service/CuramiUtils.java @@ -1,56 +1,56 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.util.*; -import java.util.stream.Collectors; -import org.apache.commons.text.similarity.JaroWinklerSimilarity; - -public class CuramiUtils { - private static final double SIMILARITY_THRESHOLD = 0.8; - - private static double getSimilarityScore(final String s1, final String s2) { - return new JaroWinklerSimilarity().apply(s1, s2); - } - - public static List getSimilarAttributes( - final String attribute, final SortedSet attributes) { - return attributes.stream() - .filter(a -> CuramiUtils.getSimilarityScore(a, attribute) > 0.8) - .collect(Collectors.toList()); - } - - static Optional getMostSimilarAttribute( - final String attribute, final SortedSet attributes) { - double similarity = 0; - Optional similarAttribute = Optional.empty(); - - for (final String a : attributes) { - final double score = CuramiUtils.getSimilarityScore(a, attribute); - if (score > SIMILARITY_THRESHOLD && score > similarity) { - similarity = score; - similarAttribute = Optional.of(a); - } - } - - return similarAttribute; - } - - static String normaliseAttribute(final String attribute, final Set abbreviations) { - final String normalisedAttribute = attribute.replace("_", " "); - final String[] words = normalisedAttribute.split(" "); - final StringJoiner attributeJoiner = new StringJoiner(" "); - for (final String word : words) { - attributeJoiner.add(abbreviations.contains(word) ? word : word.toLowerCase()); - } - return attributeJoiner.toString(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.curami.service; + +import java.util.*; +import java.util.stream.Collectors; +import org.apache.commons.text.similarity.JaroWinklerSimilarity; + +public class CuramiUtils { + private static final double SIMILARITY_THRESHOLD = 0.8; + + private static double getSimilarityScore(final String s1, final String s2) { + return new JaroWinklerSimilarity().apply(s1, s2); + } + + public static List getSimilarAttributes( + final String attribute, final SortedSet attributes) { + return attributes.stream() + .filter(a -> CuramiUtils.getSimilarityScore(a, attribute) > 0.8) + .collect(Collectors.toList()); + } + + public static Optional getMostSimilarAttribute( + final String attribute, final SortedSet attributes) { + double similarity = 0; + Optional similarAttribute = Optional.empty(); + + for (final String a : attributes) { + final double score = CuramiUtils.getSimilarityScore(a, attribute); + if (score > SIMILARITY_THRESHOLD && score > similarity) { + similarity = score; + similarAttribute = Optional.of(a); + } + } + + return similarAttribute; + } + + public static String normaliseAttribute(final String attribute, final Set abbreviations) { + final String normalisedAttribute = attribute.replace("_", " "); + final String[] words = normalisedAttribute.split(" "); + final StringJoiner attributeJoiner = new StringJoiner(" "); + for (final String word : words) { + attributeJoiner.add(abbreviations.contains(word) ? word : word.toLowerCase()); + } + return attributeJoiner.toString(); + } +} diff --git a/models/curami/src/main/java/uk/ac/ebi/biosamples/service/DataLoader.java b/core/src/main/java/uk/ac/ebi/biosamples/curami/service/DataLoader.java similarity index 92% rename from models/curami/src/main/java/uk/ac/ebi/biosamples/service/DataLoader.java rename to core/src/main/java/uk/ac/ebi/biosamples/curami/service/DataLoader.java index a0a754aaf..adfeaeadd 100644 --- a/models/curami/src/main/java/uk/ac/ebi/biosamples/service/DataLoader.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/curami/service/DataLoader.java @@ -1,96 +1,96 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.*; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.core.io.ClassPathResource; - -public class DataLoader { - private static final Logger LOG = LoggerFactory.getLogger(DataLoader.class); - - private static final SortedSet POPULAR_ATTRIBUTES = new TreeSet<>(); - private static final Set ABBREVIATIONS = new HashSet<>(); - private static final Map CURATIONS = new HashMap<>(); - - SortedSet getPopularAttributes() { - return POPULAR_ATTRIBUTES; - } - - Set getAbbreviations() { - return ABBREVIATIONS; - } - - public Map getCurations() { - return CURATIONS; - } - - void loadDataFromClassPathResource() { - loadPopularAttributes("attributes.csv"); - loadAbbreviations("abbreviations.csv"); - loadCurations("curations.csv"); - } - - private void loadPopularAttributes(final String filePath) { - try { - final Reader in = - new BufferedReader( - new InputStreamReader(new ClassPathResource(filePath).getInputStream())); - final Iterable records = - CSVFormat.DEFAULT.withHeader("ATTRIBUTE", "COUNT").parse(in); - for (final CSVRecord record : records) { - final String attribute = record.get("ATTRIBUTE"); - POPULAR_ATTRIBUTES.add(attribute); - } - } catch (final IOException e) { - LOG.error("Failed to load CSV file at: " + filePath, e); - } - } - - private void loadAbbreviations(final String filePath) { - try { - final Reader in = - new BufferedReader( - new InputStreamReader(new ClassPathResource(filePath).getInputStream())); - final Iterable records = CSVFormat.DEFAULT.withHeader("ATTRIBUTE").parse(in); - for (final CSVRecord record : records) { - final String attribute = record.get("ATTRIBUTE"); - ABBREVIATIONS.add(attribute); - } - } catch (final IOException e) { - LOG.error("Failed to load CSV file at: " + filePath, e); - } - } - - private void loadCurations(final String filePath) { - try { - final Reader in = - new BufferedReader( - new InputStreamReader(new ClassPathResource(filePath).getInputStream())); - final Iterable records = - CSVFormat.DEFAULT.withHeader("ATTRIBUTE", "CURATION").parse(in); - for (final CSVRecord record : records) { - final String attribute = record.get("ATTRIBUTE"); - final String curation = record.get("CURATION"); - CURATIONS.put(attribute, curation); - } - } catch (final IOException e) { - LOG.error("Failed to load CSV file at: " + filePath, e); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.curami.service; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.*; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.ClassPathResource; + +public class DataLoader { + private static final Logger LOG = LoggerFactory.getLogger(DataLoader.class); + + private static final SortedSet POPULAR_ATTRIBUTES = new TreeSet<>(); + private static final Set ABBREVIATIONS = new HashSet<>(); + private static final Map CURATIONS = new HashMap<>(); + + public SortedSet getPopularAttributes() { + return POPULAR_ATTRIBUTES; + } + + public Set getAbbreviations() { + return ABBREVIATIONS; + } + + public Map getCurations() { + return CURATIONS; + } + + public void loadDataFromClassPathResource() { + loadPopularAttributes("attributes.csv"); + loadAbbreviations("abbreviations.csv"); + loadCurations("curations.csv"); + } + + private void loadPopularAttributes(final String filePath) { + try { + final Reader in = + new BufferedReader( + new InputStreamReader(new ClassPathResource(filePath).getInputStream())); + final Iterable records = + CSVFormat.DEFAULT.withHeader("ATTRIBUTE", "COUNT").parse(in); + for (final CSVRecord record : records) { + final String attribute = record.get("ATTRIBUTE"); + POPULAR_ATTRIBUTES.add(attribute); + } + } catch (final IOException e) { + LOG.error("Failed to load CSV file at: " + filePath, e); + } + } + + private void loadAbbreviations(final String filePath) { + try { + final Reader in = + new BufferedReader( + new InputStreamReader(new ClassPathResource(filePath).getInputStream())); + final Iterable records = CSVFormat.DEFAULT.withHeader("ATTRIBUTE").parse(in); + for (final CSVRecord record : records) { + final String attribute = record.get("ATTRIBUTE"); + ABBREVIATIONS.add(attribute); + } + } catch (final IOException e) { + LOG.error("Failed to load CSV file at: " + filePath, e); + } + } + + private void loadCurations(final String filePath) { + try { + final Reader in = + new BufferedReader( + new InputStreamReader(new ClassPathResource(filePath).getInputStream())); + final Iterable records = + CSVFormat.DEFAULT.withHeader("ATTRIBUTE", "CURATION").parse(in); + for (final CSVRecord record : records) { + final String attribute = record.get("ATTRIBUTE"); + final String curation = record.get("CURATION"); + CURATIONS.put(attribute, curation); + } + } catch (final IOException e) { + LOG.error("Failed to load CSV file at: " + filePath, e); + } + } +} diff --git a/commons/src/main/java/uk/ac/ebi/biosamples/exceptions/GlobalExceptions.java b/core/src/main/java/uk/ac/ebi/biosamples/exception/GlobalExceptions.java similarity index 97% rename from commons/src/main/java/uk/ac/ebi/biosamples/exceptions/GlobalExceptions.java rename to core/src/main/java/uk/ac/ebi/biosamples/exception/GlobalExceptions.java index 18b2aed73..33d23e6b4 100644 --- a/commons/src/main/java/uk/ac/ebi/biosamples/exceptions/GlobalExceptions.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/exception/GlobalExceptions.java @@ -1,168 +1,168 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.exceptions; - -import java.io.Serial; -import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.ResponseStatus; - -/* -All BioSamples exceptions are declared in this class - */ -public class GlobalExceptions { - @ResponseStatus( - value = HttpStatus.BAD_REQUEST, - reason = - "New sample submission should not contain an accession or a SRA accession, these are auto-generated by BioSamples database") - public static class SampleWithAccessionSubmissionException extends RuntimeException {} - - @ResponseStatus( - value = HttpStatus.BAD_REQUEST, - reason = "Bulk fetch request parameter accessions is null or empty") - public static class BulkFetchInvalidRequestException extends RuntimeException {} - - @ResponseStatus( - value = HttpStatus.BAD_REQUEST, - reason = "Sample accession must match URL accession") // 400 - public static class SampleAccessionMismatchException extends RuntimeException {} - - @ResponseStatus(value = HttpStatus.BAD_REQUEST, reason = "Sample accession does not exist") // 400 - public static class SampleAccessionDoesNotExistException extends RuntimeException {} - - public static class SampleNotFoundException extends RuntimeException { - @Serial private static final long serialVersionUID = 1376682660925892995L; - } - - public static class SampleValidationException extends RuntimeException { - public SampleValidationException(final String message) { - super(message); - } - } - - public static class SchemaValidationException extends RuntimeException { - public SchemaValidationException(final String message, final Exception e) { - super(message, e); - } - - public SchemaValidationException(final String message) { - super(message); - } - } - - @ResponseStatus( - value = HttpStatus.FORBIDDEN, - reason = - "This sample is private and not available for browsing. If you think this is an error and/or you should have access please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk") - public static class SampleNotAccessibleException extends RuntimeException {} - - @ResponseStatus( - value = HttpStatus.FORBIDDEN, - reason = - "This sample is not permitted to be updated. Please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk for more information") - public static class InvalidSubmissionSourceException extends RuntimeException {} - - @ResponseStatus( - value = HttpStatus.FORBIDDEN, - reason = - "This sample is permitted to be updated only by the original submitter. Please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk for more information") - public static class NonSubmitterUpdateAttemptException extends RuntimeException {} - - public static class SampleNotAccessibleAdviceException extends RuntimeException { - @Serial private static final long serialVersionUID = -6250819256457895445L; - } - - @ResponseStatus(value = HttpStatus.FORBIDDEN) - public static class AccessControlException extends RuntimeException { - public AccessControlException() { - super(); - } - - public AccessControlException(final String message) { - super(message); - } - } - - @ResponseStatus(HttpStatus.BAD_REQUEST) - public static class SampleMandatoryFieldsMissingException extends RuntimeException { - @Serial private static final long serialVersionUID = -7937033504537036300L; - - public SampleMandatoryFieldsMissingException(final String message) { - super(message); - } - } - - @ResponseStatus( - value = HttpStatus.FORBIDDEN, - reason = - "You don't have access to the sample structured data. If you think this is an error and/or you should have access please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk") // 403 - public static class StructuredDataNotAccessibleException extends RuntimeException {} - - @ResponseStatus(value = HttpStatus.BAD_REQUEST, reason = "Sample domain mismatch") // 400 - public static class SampleDomainMismatchException extends RuntimeException {} - - @ResponseStatus( - value = HttpStatus.BAD_REQUEST, - reason = - "Sample submitted via V2 submission endpoints shouldn't contain relationships, please use the traditional endpoint instead") // 400 - public static class SampleWithRelationshipSubmissionExceptionV2 extends RuntimeException {} - - @ResponseStatus(value = HttpStatus.UNAUTHORIZED, reason = "Unauthorized submitter") - public static class WebinUserLoginUnauthorizedException extends RuntimeException {} - - @ResponseStatus( - value = HttpStatus.BAD_REQUEST, - reason = "Structured data must have a Webin Account ID") // 400 - public static class StructuredDataWebinIdMissingException extends RuntimeException {} - - @ResponseStatus(value = HttpStatus.BAD_REQUEST) - public static class UploadInvalidException extends RuntimeException { - public UploadInvalidException(final String collect) { - super(collect); - } - } - - @ResponseStatus( - value = HttpStatus.BAD_REQUEST, - reason = - "Validation of taxonomy failed against the ENA taxonomy service. The Organism attribute is either invalid or not submittable") - public static class ENATaxonUnresolvedException extends RuntimeException {} - - @ResponseStatus( - value = HttpStatus.BAD_REQUEST, - reason = "Sample must match URL or be omitted") // 400 - public static class SampleNotMatchException extends RuntimeException {} - - public static class SampleConversionException extends RuntimeException { - public SampleConversionException(final String message) { - super(message); - } - } - - @ResponseStatus( - value = HttpStatus.BAD_REQUEST, - reason = - "BioSamples pagination using 'page' and 'size' parameters are limited due to performance reasons. " - + "(max page = 500 and max page size = 200). Please use 'cursor' and 'size' for pagination and crawling. " - + "You can find more information about using cursor in our documentation") - public static class PaginationException extends RuntimeException {} - - @ResponseStatus( - value = HttpStatus.BAD_REQUEST, - reason = - "Sample is invalid. Please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk for more information.") - public static class InvalidSampleException extends RuntimeException {} - - @ResponseStatus( - value = HttpStatus.BAD_REQUEST, - reason = - "SRA accession cannot be changed while sample updates. Please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk for more information.") - public static class ChangedSRAAccessionException extends RuntimeException {} -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.exception; + +import java.io.Serial; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +/* +All BioSamples exceptions are declared in this class + */ +public class GlobalExceptions { + @ResponseStatus( + value = HttpStatus.BAD_REQUEST, + reason = + "New sample submission should not contain an accession or a SRA accession, these are auto-generated by BioSamples database") + public static class SampleWithAccessionSubmissionException extends RuntimeException {} + + @ResponseStatus( + value = HttpStatus.BAD_REQUEST, + reason = "Bulk fetch request parameter accessions is null or empty") + public static class BulkFetchInvalidRequestException extends RuntimeException {} + + @ResponseStatus( + value = HttpStatus.BAD_REQUEST, + reason = "Sample accession must match URL accession") // 400 + public static class SampleAccessionMismatchException extends RuntimeException {} + + @ResponseStatus(value = HttpStatus.BAD_REQUEST, reason = "Sample accession does not exist") // 400 + public static class SampleAccessionDoesNotExistException extends RuntimeException {} + + public static class SampleNotFoundException extends RuntimeException { + @Serial private static final long serialVersionUID = 1376682660925892995L; + } + + public static class SampleValidationException extends RuntimeException { + public SampleValidationException(final String message) { + super(message); + } + } + + public static class SchemaValidationException extends RuntimeException { + public SchemaValidationException(final String message, final Exception e) { + super(message, e); + } + + public SchemaValidationException(final String message) { + super(message); + } + } + + @ResponseStatus( + value = HttpStatus.FORBIDDEN, + reason = + "This sample is private and not available for browsing. If you think this is an error and/or you should have access please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk") + public static class SampleNotAccessibleException extends RuntimeException {} + + @ResponseStatus( + value = HttpStatus.FORBIDDEN, + reason = + "This sample is not permitted to be updated. Please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk for more information") + public static class InvalidSubmissionSourceException extends RuntimeException {} + + @ResponseStatus( + value = HttpStatus.FORBIDDEN, + reason = + "This sample is permitted to be updated only by the original submitter. Please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk for more information") + public static class NonSubmitterUpdateAttemptException extends RuntimeException {} + + public static class SampleNotAccessibleAdviceException extends RuntimeException { + @Serial private static final long serialVersionUID = -6250819256457895445L; + } + + @ResponseStatus(value = HttpStatus.FORBIDDEN) + public static class AccessControlException extends RuntimeException { + public AccessControlException() { + super(); + } + + public AccessControlException(final String message) { + super(message); + } + } + + @ResponseStatus(HttpStatus.BAD_REQUEST) + public static class SampleMandatoryFieldsMissingException extends RuntimeException { + @Serial private static final long serialVersionUID = -7937033504537036300L; + + public SampleMandatoryFieldsMissingException(final String message) { + super(message); + } + } + + @ResponseStatus( + value = HttpStatus.FORBIDDEN, + reason = + "You don't have access to the sample structured data. If you think this is an error and/or you should have access please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk") // 403 + public static class StructuredDataNotAccessibleException extends RuntimeException {} + + @ResponseStatus(value = HttpStatus.BAD_REQUEST, reason = "Sample domain mismatch") // 400 + public static class SampleDomainMismatchException extends RuntimeException {} + + @ResponseStatus( + value = HttpStatus.BAD_REQUEST, + reason = + "Sample submitted via V2 submission endpoints shouldn't contain relationships, please use the traditional endpoint instead") // 400 + public static class SampleWithRelationshipSubmissionExceptionV2 extends RuntimeException {} + + @ResponseStatus(value = HttpStatus.UNAUTHORIZED, reason = "Unauthorized submitter") + public static class WebinUserLoginUnauthorizedException extends RuntimeException {} + + @ResponseStatus( + value = HttpStatus.BAD_REQUEST, + reason = "Structured data must have a Webin Account ID") // 400 + public static class StructuredDataWebinIdMissingException extends RuntimeException {} + + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + public static class UploadInvalidException extends RuntimeException { + public UploadInvalidException(final String collect) { + super(collect); + } + } + + @ResponseStatus( + value = HttpStatus.BAD_REQUEST, + reason = + "Validation of taxonomy failed against the ENA taxonomy service. The Organism attribute is either invalid or not submittable") + public static class ENATaxonUnresolvedException extends RuntimeException {} + + @ResponseStatus( + value = HttpStatus.BAD_REQUEST, + reason = "Sample must match URL or be omitted") // 400 + public static class SampleNotMatchException extends RuntimeException {} + + public static class SampleConversionException extends RuntimeException { + public SampleConversionException(final String message) { + super(message); + } + } + + @ResponseStatus( + value = HttpStatus.BAD_REQUEST, + reason = + "BioSamples pagination using 'page' and 'size' parameters are limited due to performance reasons. " + + "(max page = 500 and max page size = 200). Please use 'cursor' and 'size' for pagination and crawling. " + + "You can find more information about using cursor in our documentation") + public static class PaginationException extends RuntimeException {} + + @ResponseStatus( + value = HttpStatus.BAD_REQUEST, + reason = + "Sample is invalid. Please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk for more information.") + public static class InvalidSampleException extends RuntimeException {} + + @ResponseStatus( + value = HttpStatus.BAD_REQUEST, + reason = + "SRA accession cannot be changed while sample updates. Please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk for more information.") + public static class ChangedSRAAccessionException extends RuntimeException {} +} diff --git a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/BioSchemasContext.java b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/BioSchemasContext.java similarity index 88% rename from models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/BioSchemasContext.java rename to core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/BioSchemasContext.java index 87e7743ff..41f94a12e 100644 --- a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/BioSchemasContext.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/BioSchemasContext.java @@ -1,46 +1,46 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import java.net.URI; -import java.util.HashMap; -import java.util.Map; -import uk.ac.ebi.biosamples.service.ContextDeserializer; -import uk.ac.ebi.biosamples.service.ContextSerializer; - -@JsonSerialize(using = ContextSerializer.class) -@JsonDeserialize(using = ContextDeserializer.class) -public class BioSchemasContext { - - private final URI schemaOrgContext; - private final Map otherContexts; - - public BioSchemasContext() { - schemaOrgContext = URI.create("http://schema.org"); - otherContexts = new HashMap<>(); - otherContexts.put("OBI", URI.create("http://purl.obolibrary.org/obo/OBI_")); - otherContexts.put("biosample", URI.create("http://identifiers.org/biosample/")); - } - - public URI getSchemaOrgContext() { - return schemaOrgContext; - } - - public Map getOtherContexts() { - return otherContexts; - } - - public void addOtherContexts(final String name, final URI id) { - otherContexts.put(name, id); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.model; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import uk.ac.ebi.biosamples.jsonld.service.ContextDeserializer; +import uk.ac.ebi.biosamples.jsonld.service.ContextSerializer; + +@JsonSerialize(using = ContextSerializer.class) +@JsonDeserialize(using = ContextDeserializer.class) +public class BioSchemasContext { + + private final URI schemaOrgContext; + private final Map otherContexts; + + public BioSchemasContext() { + schemaOrgContext = URI.create("http://schema.org"); + otherContexts = new HashMap<>(); + otherContexts.put("OBI", URI.create("http://purl.obolibrary.org/obo/OBI_")); + otherContexts.put("biosample", URI.create("http://identifiers.org/biosample/")); + } + + public URI getSchemaOrgContext() { + return schemaOrgContext; + } + + public Map getOtherContexts() { + return otherContexts; + } + + public void addOtherContexts(final String name, final URI id) { + otherContexts.put(name, id); + } +} diff --git a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/BioschemasObject.java b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/BioschemasObject.java similarity index 92% rename from models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/BioschemasObject.java rename to core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/BioschemasObject.java index c5c66121d..0d30a008c 100644 --- a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/BioschemasObject.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/BioschemasObject.java @@ -1,13 +1,13 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -public interface BioschemasObject {} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.model; + +public interface BioschemasObject {} diff --git a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDDataCatalog.java b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDDataCatalog.java similarity index 96% rename from models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDDataCatalog.java rename to core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDDataCatalog.java index 6947706c9..31c0ef51a 100644 --- a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDDataCatalog.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDDataCatalog.java @@ -1,145 +1,145 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonPropertyOrder( - value = { - "@context", - "@type", - "description", - "keywords", - "name", - "url", - "url", - "publication", - "provider", - "sourceOrganization" - }) -public class JsonLDDataCatalog implements BioschemasObject { - - @JsonProperty("@context") - private final String context = "http://schema.org"; - - @JsonProperty("@type") - private final String type = "DataCatalog"; - - @JsonProperty("description") - private final String description; - - @JsonProperty("keywords") - private final String keywords; - - @JsonProperty("provider") - private final Map provider; - - @JsonProperty("name") - private final String name; - - private String url; - - private List> dataset; - - @JsonProperty("publication") - private final List> publication; - - @JsonProperty("sourceOrganization") - private final Map sourceOrganization; - - public JsonLDDataCatalog() { - description = - "BioSamples stores and supplies descriptions and metadata about biological samples " - + "used in research and development by academia and industry. " - + "Samples are either 'reference' samples (e.g. from 1000 Genomes, HipSci, FAANG) " - + "or have been used in an assay database such as the European Nucleotide Archive (ENA) or ArrayExpress."; - keywords = "samples, sample metadata"; - provider = getBiosamplesProvider(); - name = "BioSamples database"; - url = "https://www.ebi.ac.uk/biosamples"; - dataset = getDefaultBioSamplesDataset(); - publication = getBioSamplesPublication(); - sourceOrganization = getBioSamplesSourceOrganization(); - } - - private Map getBioSamplesSourceOrganization() { - final Map sourceOrganization = new HashMap<>(); - sourceOrganization.put("@type", "Organization"); - sourceOrganization.put("name", "The European Bioinformatics Institute (EMBL-EBI)"); - sourceOrganization.put("url", "https://www.ebi.ac.uk/"); - return sourceOrganization; - } - - private List> getBioSamplesPublication() { - final List> publication = new ArrayList<>(); - final Map publicationEntry = new HashMap<>(); - publicationEntry.put("@type", "PublicationEvent"); - publicationEntry.put( - "name", "Updates to BioSamples database at European Bioinformatics Institute"); - publicationEntry.put("url", "http://identifiers.org/pubmed:24265224"); - publication.add(publicationEntry); - return publication; - } - - private List> getDefaultBioSamplesDataset() { - final List> dataset = new ArrayList<>(); - final Map datasetEntry = new HashMap<>(); - datasetEntry.put("@type", "Dataset"); - datasetEntry.put("@id", "https://www.ebi.ac.uk/biosamples/samples"); - dataset.add(datasetEntry); - return dataset; - } - - private Map getBiosamplesProvider() { - final Map provider = new HashMap<>(); - provider.put("@type", "Organization"); - provider.put("name", "BioSamples"); - provider.put("email", "biosamples@ebi.ac.uk"); - return provider; - } - - @JsonProperty("provider") - private Map getProvider() { - return provider; - } - - public List> getDataset() { - return dataset; - } - - @JsonProperty("dataset") - public JsonLDDataCatalog datasetUrl(final String datasetUrl) { - List> allDatasets = getDefaultBioSamplesDataset(); - final Map dataset = allDatasets.get(0); - dataset.put("@id", datasetUrl); - allDatasets = new ArrayList<>(); - allDatasets.add(dataset); - this.dataset = allDatasets; - return this; - } - - @JsonProperty("url") - public String getUrl() { - return url; - } - - public JsonLDDataCatalog url(final String url) { - this.url = url; - return this; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder( + value = { + "@context", + "@type", + "description", + "keywords", + "name", + "url", + "url", + "publication", + "provider", + "sourceOrganization" + }) +public class JsonLDDataCatalog implements BioschemasObject { + + @JsonProperty("@context") + private final String context = "http://schema.org"; + + @JsonProperty("@type") + private final String type = "DataCatalog"; + + @JsonProperty("description") + private final String description; + + @JsonProperty("keywords") + private final String keywords; + + @JsonProperty("provider") + private final Map provider; + + @JsonProperty("name") + private final String name; + + private String url; + + private List> dataset; + + @JsonProperty("publication") + private final List> publication; + + @JsonProperty("sourceOrganization") + private final Map sourceOrganization; + + public JsonLDDataCatalog() { + description = + "BioSamples stores and supplies descriptions and metadata about biological samples " + + "used in research and development by academia and industry. " + + "Samples are either 'reference' samples (e.g. from 1000 Genomes, HipSci, FAANG) " + + "or have been used in an assay database such as the European Nucleotide Archive (ENA) or ArrayExpress."; + keywords = "samples, sample metadata"; + provider = getBiosamplesProvider(); + name = "BioSamples database"; + url = "https://www.ebi.ac.uk/biosamples"; + dataset = getDefaultBioSamplesDataset(); + publication = getBioSamplesPublication(); + sourceOrganization = getBioSamplesSourceOrganization(); + } + + private Map getBioSamplesSourceOrganization() { + final Map sourceOrganization = new HashMap<>(); + sourceOrganization.put("@type", "Organization"); + sourceOrganization.put("name", "The European Bioinformatics Institute (EMBL-EBI)"); + sourceOrganization.put("url", "https://www.ebi.ac.uk/"); + return sourceOrganization; + } + + private List> getBioSamplesPublication() { + final List> publication = new ArrayList<>(); + final Map publicationEntry = new HashMap<>(); + publicationEntry.put("@type", "PublicationEvent"); + publicationEntry.put( + "name", "Updates to BioSamples database at European Bioinformatics Institute"); + publicationEntry.put("url", "http://identifiers.org/pubmed:24265224"); + publication.add(publicationEntry); + return publication; + } + + private List> getDefaultBioSamplesDataset() { + final List> dataset = new ArrayList<>(); + final Map datasetEntry = new HashMap<>(); + datasetEntry.put("@type", "Dataset"); + datasetEntry.put("@id", "https://www.ebi.ac.uk/biosamples/samples"); + dataset.add(datasetEntry); + return dataset; + } + + private Map getBiosamplesProvider() { + final Map provider = new HashMap<>(); + provider.put("@type", "Organization"); + provider.put("name", "BioSamples"); + provider.put("email", "biosamples@ebi.ac.uk"); + return provider; + } + + @JsonProperty("provider") + private Map getProvider() { + return provider; + } + + public List> getDataset() { + return dataset; + } + + @JsonProperty("dataset") + public JsonLDDataCatalog datasetUrl(final String datasetUrl) { + List> allDatasets = getDefaultBioSamplesDataset(); + final Map dataset = allDatasets.get(0); + dataset.put("@id", datasetUrl); + allDatasets = new ArrayList<>(); + allDatasets.add(dataset); + this.dataset = allDatasets; + return this; + } + + @JsonProperty("url") + public String getUrl() { + return url; + } + + public JsonLDDataCatalog url(final String url) { + this.url = url; + return this; + } +} diff --git a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDDataRecord.java b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDDataRecord.java similarity index 95% rename from models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDDataRecord.java rename to core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDDataRecord.java index be33bc62c..fb53c8c82 100644 --- a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDDataRecord.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDDataRecord.java @@ -1,176 +1,176 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.*; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.HashMap; -import java.util.Map; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonPropertyOrder( - value = { - "@context", - "@type", - "@id", - "identifier", - "dateCreated", - "dateModified", - "dateReleased", - "dateSubmitted", - "mainEntity", - "isPartOf" - }) -public class JsonLDDataRecord implements BioschemasObject { - - private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ISO_OFFSET_DATE_TIME; - - @JsonProperty("@id") - private String id; - - @JsonProperty("@context") - private final BioSchemasContext context = new BioSchemasContext(); - - @JsonProperty("@type") - private final String type = "DataRecord"; - - @JsonProperty("identifier") - private String identifier; - - @JsonProperty("mainEntity") - private JsonLDSample mainEntity; - - private ZonedDateTime dateCreated; - - private ZonedDateTime dateReleased; - - private ZonedDateTime dateSubmitted; - - @JsonProperty("dateModified") - private ZonedDateTime dateModified; - - private final Map partOf = getDatasetPartOf(); - - public String getId() { - return id; - } - - public BioSchemasContext getContext() { - return context; - } - - public String getIdentifier() { - return identifier; - } - - public JsonLDDataRecord identifier(final String identifier) { - this.identifier = identifier; - id = identifier; - return this; - } - - public JsonLDSample getMainEntity() { - return mainEntity; - } - - public JsonLDDataRecord mainEntity(final JsonLDSample mainEntity) { - this.mainEntity = mainEntity; - return this; - } - - // public ZonedDateTime getDateModified() { - // return dateModified; - // } - - // public ZonedDateTime getDateCreated() { - // return dateCreated; - // } - - @JsonGetter("dateModified") - public String getDateModified() { - return dateTimeFormatter.format(dateModified); - } - - @JsonGetter("dateCreated") - public String getDateCreated() { - return dateTimeFormatter.format(dateCreated); - } - - @JsonGetter("dateSubmitted") - public String getDateSubmitted() { - return dateSubmitted != null ? dateTimeFormatter.format(dateSubmitted) : null; - } - - @JsonGetter("dateReleased") - public String getDateReleased() { - return dateTimeFormatter.format(dateReleased); - } - - @JsonSetter("dateCreated") - public JsonLDDataRecord dateCreated(final String dateCreated) { - this.dateCreated = - LocalDateTime.parse(dateCreated, dateTimeFormatter).atZone(ZoneId.systemDefault()); - return this; - } - - @JsonSetter("dateModified") - public JsonLDDataRecord dateModified(final String dateModified) { - this.dateModified = - LocalDateTime.parse(dateModified, dateTimeFormatter).atZone(ZoneId.systemDefault()); - return this; - } - - @JsonSetter("dateSubmitted") - public JsonLDDataRecord dateSubmitted(final String dateSubmitted) { - this.dateSubmitted = - LocalDateTime.parse(dateSubmitted, dateTimeFormatter).atZone(ZoneId.systemDefault()); - return this; - } - - @JsonSetter("dateReleased") - public JsonLDDataRecord dateReleased(final String dateReleased) { - this.dateReleased = - LocalDateTime.parse(dateReleased, dateTimeFormatter).atZone(ZoneId.systemDefault()); - return this; - } - - public JsonLDDataRecord dateModified(final ZonedDateTime dateModified) { - this.dateModified = dateModified; - return this; - } - - public JsonLDDataRecord dateCreated(final ZonedDateTime dateCreated) { - this.dateCreated = dateCreated; - return this; - } - - public JsonLDDataRecord dateSubmitted(final ZonedDateTime dateSubmitted) { - this.dateSubmitted = dateSubmitted; - return this; - } - - public JsonLDDataRecord dateReleased(final ZonedDateTime dateReleased) { - this.dateReleased = dateReleased; - return this; - } - - @JsonProperty("isPartOf") - public Map getDatasetPartOf() { - final Map datasetPartOf = new HashMap<>(); - datasetPartOf.put("@type", "Dataset"); - // TODO Use relative application url and not hard-coded one - datasetPartOf.put("@id", "https://www.ebi.ac.uk/biosamples/samples"); - return datasetPartOf; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.model; + +import com.fasterxml.jackson.annotation.*; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder( + value = { + "@context", + "@type", + "@id", + "identifier", + "dateCreated", + "dateModified", + "dateReleased", + "dateSubmitted", + "mainEntity", + "isPartOf" + }) +public class JsonLDDataRecord implements BioschemasObject { + + private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ISO_OFFSET_DATE_TIME; + + @JsonProperty("@id") + private String id; + + @JsonProperty("@context") + private final BioSchemasContext context = new BioSchemasContext(); + + @JsonProperty("@type") + private final String type = "DataRecord"; + + @JsonProperty("identifier") + private String identifier; + + @JsonProperty("mainEntity") + private JsonLDSample mainEntity; + + private ZonedDateTime dateCreated; + + private ZonedDateTime dateReleased; + + private ZonedDateTime dateSubmitted; + + @JsonProperty("dateModified") + private ZonedDateTime dateModified; + + private final Map partOf = getDatasetPartOf(); + + public String getId() { + return id; + } + + public BioSchemasContext getContext() { + return context; + } + + public String getIdentifier() { + return identifier; + } + + public JsonLDDataRecord identifier(final String identifier) { + this.identifier = identifier; + id = identifier; + return this; + } + + public JsonLDSample getMainEntity() { + return mainEntity; + } + + public JsonLDDataRecord mainEntity(final JsonLDSample mainEntity) { + this.mainEntity = mainEntity; + return this; + } + + // public ZonedDateTime getDateModified() { + // return dateModified; + // } + + // public ZonedDateTime getDateCreated() { + // return dateCreated; + // } + + @JsonGetter("dateModified") + public String getDateModified() { + return dateTimeFormatter.format(dateModified); + } + + @JsonGetter("dateCreated") + public String getDateCreated() { + return dateTimeFormatter.format(dateCreated); + } + + @JsonGetter("dateSubmitted") + public String getDateSubmitted() { + return dateSubmitted != null ? dateTimeFormatter.format(dateSubmitted) : null; + } + + @JsonGetter("dateReleased") + public String getDateReleased() { + return dateTimeFormatter.format(dateReleased); + } + + @JsonSetter("dateCreated") + public JsonLDDataRecord dateCreated(final String dateCreated) { + this.dateCreated = + LocalDateTime.parse(dateCreated, dateTimeFormatter).atZone(ZoneId.systemDefault()); + return this; + } + + @JsonSetter("dateModified") + public JsonLDDataRecord dateModified(final String dateModified) { + this.dateModified = + LocalDateTime.parse(dateModified, dateTimeFormatter).atZone(ZoneId.systemDefault()); + return this; + } + + @JsonSetter("dateSubmitted") + public JsonLDDataRecord dateSubmitted(final String dateSubmitted) { + this.dateSubmitted = + LocalDateTime.parse(dateSubmitted, dateTimeFormatter).atZone(ZoneId.systemDefault()); + return this; + } + + @JsonSetter("dateReleased") + public JsonLDDataRecord dateReleased(final String dateReleased) { + this.dateReleased = + LocalDateTime.parse(dateReleased, dateTimeFormatter).atZone(ZoneId.systemDefault()); + return this; + } + + public JsonLDDataRecord dateModified(final ZonedDateTime dateModified) { + this.dateModified = dateModified; + return this; + } + + public JsonLDDataRecord dateCreated(final ZonedDateTime dateCreated) { + this.dateCreated = dateCreated; + return this; + } + + public JsonLDDataRecord dateSubmitted(final ZonedDateTime dateSubmitted) { + this.dateSubmitted = dateSubmitted; + return this; + } + + public JsonLDDataRecord dateReleased(final ZonedDateTime dateReleased) { + this.dateReleased = dateReleased; + return this; + } + + @JsonProperty("isPartOf") + public Map getDatasetPartOf() { + final Map datasetPartOf = new HashMap<>(); + datasetPartOf.put("@type", "Dataset"); + // TODO Use relative application url and not hard-coded one + datasetPartOf.put("@id", "https://www.ebi.ac.uk/biosamples/samples"); + return datasetPartOf; + } +} diff --git a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDDataset.java b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDDataset.java similarity index 96% rename from models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDDataset.java rename to core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDDataset.java index c3f2a3cbb..d60d727b1 100644 --- a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDDataset.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDDataset.java @@ -1,69 +1,69 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import java.util.HashMap; -import java.util.Map; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonPropertyOrder( - value = {"@context", "@type", "name", "description", "url", "includedInDataCatalog"}) -public class JsonLDDataset implements BioschemasObject { - - @JsonProperty("@context") - private final String context = "http://schema.org"; - - @JsonProperty("@type") - private final String type = "Dataset"; - - @JsonProperty("name") - private final String name = "EMBL-EBI BioSamples"; - - @JsonProperty("description") - private final String description = - "The BioSamples database at EMBL-EBI provides a central hub for sample metadata storage and linkage to other EMBL-EBI resources. BioSamples contains just over 5 million samples in 2018. Fast, reciprocal data exchange is fully established between sister Biosample databases and other INSDC partners, enabling a worldwide common representation and centralisation of sample metadata. BioSamples plays a vital role in data coordination, acting as the sample metadata repository for many biomedical projects including the Functional Annotation of Animal Genomes (FAANG), the European Bank for induced pluripotent Stem Cells (EBiSC) and the {Human Induced Pluripotent Stem Cell Initiative (HipSci). Data growth is driven by the ever more important role BioSamples is playing as an ELIXIR data deposition database and as the EMBL-EBI hub for sample metadata. BioSamples is now the destination for samples from ELIXIR-EXCELERATE use case projects ranging from plant phenotyping to marine metagenomics, as well as several other community efforts including the Global Alliance for Genomics and Health (GA4GH) and the ELIXIR Bioschemas project."; - - private String url = "https://www.ebi.ac.uk/biosamples/samples"; - - private Map dataCatalog = getDefaultDataCatalog(); - - private Map getDefaultDataCatalog() { - final Map dataCatalog = new HashMap<>(); - dataCatalog.put("@type", "DataCatalog"); - dataCatalog.put("@id", "https://www.ebi.ac.uk/biosamples"); - return dataCatalog; - } - - @JsonProperty("url") - public String getUrl() { - return url; - } - - public JsonLDDataset datasetUrl(final String url) { - this.url = url; - return this; - } - - @JsonProperty("includedInDataCatalog") - public Map getDataCatalog() { - return dataCatalog; - } - - public JsonLDDataset dataCatalogUrl(final String dataCatalogUrl) { - final Map dataCatalog = getDefaultDataCatalog(); - dataCatalog.put("@id", dataCatalogUrl); - this.dataCatalog = dataCatalog; - return this; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import java.util.HashMap; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder( + value = {"@context", "@type", "name", "description", "url", "includedInDataCatalog"}) +public class JsonLDDataset implements BioschemasObject { + + @JsonProperty("@context") + private final String context = "http://schema.org"; + + @JsonProperty("@type") + private final String type = "Dataset"; + + @JsonProperty("name") + private final String name = "EMBL-EBI BioSamples"; + + @JsonProperty("description") + private final String description = + "The BioSamples database at EMBL-EBI provides a central hub for sample metadata storage and linkage to other EMBL-EBI resources. BioSamples contains just over 5 million samples in 2018. Fast, reciprocal data exchange is fully established between sister Biosample databases and other INSDC partners, enabling a worldwide common representation and centralisation of sample metadata. BioSamples plays a vital role in data coordination, acting as the sample metadata repository for many biomedical projects including the Functional Annotation of Animal Genomes (FAANG), the European Bank for induced pluripotent Stem Cells (EBiSC) and the {Human Induced Pluripotent Stem Cell Initiative (HipSci). Data growth is driven by the ever more important role BioSamples is playing as an ELIXIR data deposition database and as the EMBL-EBI hub for sample metadata. BioSamples is now the destination for samples from ELIXIR-EXCELERATE use case projects ranging from plant phenotyping to marine metagenomics, as well as several other community efforts including the Global Alliance for Genomics and Health (GA4GH) and the ELIXIR Bioschemas project."; + + private String url = "https://www.ebi.ac.uk/biosamples/samples"; + + private Map dataCatalog = getDefaultDataCatalog(); + + private Map getDefaultDataCatalog() { + final Map dataCatalog = new HashMap<>(); + dataCatalog.put("@type", "DataCatalog"); + dataCatalog.put("@id", "https://www.ebi.ac.uk/biosamples"); + return dataCatalog; + } + + @JsonProperty("url") + public String getUrl() { + return url; + } + + public JsonLDDataset datasetUrl(final String url) { + this.url = url; + return this; + } + + @JsonProperty("includedInDataCatalog") + public Map getDataCatalog() { + return dataCatalog; + } + + public JsonLDDataset dataCatalogUrl(final String dataCatalogUrl) { + final Map dataCatalog = getDefaultDataCatalog(); + dataCatalog.put("@id", dataCatalogUrl); + this.dataCatalog = dataCatalog; + return this; + } +} diff --git a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDDefinedTerm.java b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDDefinedTerm.java similarity index 94% rename from models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDDefinedTerm.java rename to core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDDefinedTerm.java index 9aca6acb3..1c36255a9 100644 --- a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDDefinedTerm.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDDefinedTerm.java @@ -1,62 +1,62 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - -@JsonPropertyOrder({"@id", "@type", "name", "inDefinedTermSet", "termCode"}) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class JsonLDDefinedTerm implements BioschemasObject { - - @JsonProperty("@type") - private final String type = "DefinedTerm"; - - @JsonProperty("@id") - private String id; - - private String name; - private String inDefinedTermSet; - private String termCode; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getName() { - return name; - } - - public String getInDefinedTermSet() { - return inDefinedTermSet; - } - - public String getTermCode() { - return termCode; - } - - public void setTermCode(String termCode) { - this.termCode = termCode; - } - - public void setName(String name) { - this.name = name; - } - - public void setInDefinedTermSet(String inDefinedTermSet) { - this.inDefinedTermSet = inDefinedTermSet; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonPropertyOrder({"@id", "@type", "name", "inDefinedTermSet", "termCode"}) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class JsonLDDefinedTerm implements BioschemasObject { + + @JsonProperty("@type") + private final String type = "DefinedTerm"; + + @JsonProperty("@id") + private String id; + + private String name; + private String inDefinedTermSet; + private String termCode; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public String getInDefinedTermSet() { + return inDefinedTermSet; + } + + public String getTermCode() { + return termCode; + } + + public void setTermCode(String termCode) { + this.termCode = termCode; + } + + public void setName(String name) { + this.name = name; + } + + public void setInDefinedTermSet(String inDefinedTermSet) { + this.inDefinedTermSet = inDefinedTermSet; + } +} diff --git a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDPropertyValue.java b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDPropertyValue.java similarity index 95% rename from models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDPropertyValue.java rename to core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDPropertyValue.java index b4bd13f97..500f52244 100644 --- a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDPropertyValue.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDPropertyValue.java @@ -1,85 +1,85 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import java.util.List; - -/** - * Object to represent the ld+json version of the @see Property Value in schema.org - */ -@JsonPropertyOrder({"@type", "name", "value", "valueReference"}) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class JsonLDPropertyValue implements BioschemasObject { - - @JsonProperty("@type") - private final String type = "PropertyValue"; - - private String name; - private String value; - private String unitText; - private String unitCode; - - // @JsonProperty("valueReference") - // private List valueReference; - - private List valueReference; - private List propertyId; - - public String getType() { - return type; - } - - public String getValue() { - return value; - } - - public void setName(final String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public void setValue(final String value) { - this.value = value; - } - - public List getValueReference() { - return valueReference; - } - - public void setValueReference(final List valueReference) { - this.valueReference = valueReference; - } - - public String getUnitCode() { - return unitCode; - } - - public JsonLDPropertyValue unitCode(final String unitCode) { - this.unitCode = unitCode; - return this; - } - - public String getUnitText() { - return unitText; - } - - public JsonLDPropertyValue unitText(final String unitText) { - this.unitText = unitText; - return this; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import java.util.List; + +/** + * Object to represent the ld+json version of the @see Property Value in schema.org + */ +@JsonPropertyOrder({"@type", "name", "value", "valueReference"}) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class JsonLDPropertyValue implements BioschemasObject { + + @JsonProperty("@type") + private final String type = "PropertyValue"; + + private String name; + private String value; + private String unitText; + private String unitCode; + + // @JsonProperty("valueReference") + // private List valueReference; + + private List valueReference; + private List propertyId; + + public String getType() { + return type; + } + + public String getValue() { + return value; + } + + public void setName(final String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setValue(final String value) { + this.value = value; + } + + public List getValueReference() { + return valueReference; + } + + public void setValueReference(final List valueReference) { + this.valueReference = valueReference; + } + + public String getUnitCode() { + return unitCode; + } + + public JsonLDPropertyValue unitCode(final String unitCode) { + this.unitCode = unitCode; + return this; + } + + public String getUnitText() { + return unitText; + } + + public JsonLDPropertyValue unitText(final String unitText) { + this.unitText = unitText; + return this; + } +} diff --git a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDSample.java b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDSample.java similarity index 95% rename from models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDSample.java rename to core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDSample.java index 3a94c9adf..021586246 100644 --- a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDSample.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDSample.java @@ -1,143 +1,143 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import java.net.URI; -import java.util.List; - -/** Object representing BioSchema Sample entity */ -@JsonPropertyOrder({ - "@id", - "@context", - "@type", - "additionalType", - "identifier", - "name", - "description", - "url", - "subjectOf", - "additionalProperty" -}) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class JsonLDSample implements BioschemasObject { - - private final URI sampleOntologyURI = URI.create("http://purl.obolibrary.org/obo/OBI_0000747"); - - // @JsonProperty("@context") - // private final BioSchemasContext sampleContext = new BioSchemasContext(); - - @JsonProperty("@type") - private final String[] type = {"Sample", "OBI:0000747"}; - - // private final String additionalType = - // "http://www.ontobee.org/ontology/OBI?iri=http://purl.obolibrary.org/obo/OBI_0000747"; - private String id; - private String sameAs; - private String[] identifiers; - private String name; - private String description; - private String url; - // private final URI additionalType = - // URI.create("http://purl.obolibrary.org/obo/OBI_0000747"); - - private List subjectOf; - - @JsonProperty("additionalProperty") - private List additionalProperties; - - // @JsonIgnore - // public BioSchemasContext getContext() { - // return sampleContext; - // } - - public String[] getType() { - return type; - } - - // public String getAdditionalType() { - // return additionalType; - // } - - @JsonProperty("@id") - public String getId() { - return id; - } - - public void setId(final String id) { - this.id = id; - } - - @JsonProperty("sameAs") - public String getSameAs() { - return sameAs; - } - - public void setSameAs(final String sameAs) { - this.sameAs = sameAs; - } - - @JsonProperty("identifier") - public String[] getIdentifiers() { - return identifiers; - } - - public void setIdentifiers(final String[] identifiers) { - this.identifiers = identifiers; - } - - public String getName() { - return name; - } - - public void setName(final String name) { - this.name = name; - } - - public String getDescription() { - return description; - } - - public void setDescription(final String description) { - this.description = description; - } - - public String getUrl() { - return url; - } - - public void setUrl(final String url) { - this.url = url; - } - - public List getSubjectOf() { - return subjectOf; - } - - public void setSubjectOf(final List subjectOf) { - this.subjectOf = subjectOf; - } - - public List getAdditionalProperties() { - return additionalProperties; - } - - public void setAdditionalProperties(final List additionalProperties) { - this.additionalProperties = additionalProperties; - } - - // public URI getAdditionalType() { - // return additionalType; - // } - -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import java.net.URI; +import java.util.List; + +/** Object representing BioSchema Sample entity */ +@JsonPropertyOrder({ + "@id", + "@context", + "@type", + "additionalType", + "identifier", + "name", + "description", + "url", + "subjectOf", + "additionalProperty" +}) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class JsonLDSample implements BioschemasObject { + + private final URI sampleOntologyURI = URI.create("http://purl.obolibrary.org/obo/OBI_0000747"); + + // @JsonProperty("@context") + // private final BioSchemasContext sampleContext = new BioSchemasContext(); + + @JsonProperty("@type") + private final String[] type = {"Sample", "OBI:0000747"}; + + // private final String additionalType = + // "http://www.ontobee.org/ontology/OBI?iri=http://purl.obolibrary.org/obo/OBI_0000747"; + private String id; + private String sameAs; + private String[] identifiers; + private String name; + private String description; + private String url; + // private final URI additionalType = + // URI.create("http://purl.obolibrary.org/obo/OBI_0000747"); + + private List subjectOf; + + @JsonProperty("additionalProperty") + private List additionalProperties; + + // @JsonIgnore + // public BioSchemasContext getContext() { + // return sampleContext; + // } + + public String[] getType() { + return type; + } + + // public String getAdditionalType() { + // return additionalType; + // } + + @JsonProperty("@id") + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + @JsonProperty("sameAs") + public String getSameAs() { + return sameAs; + } + + public void setSameAs(final String sameAs) { + this.sameAs = sameAs; + } + + @JsonProperty("identifier") + public String[] getIdentifiers() { + return identifiers; + } + + public void setIdentifiers(final String[] identifiers) { + this.identifiers = identifiers; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getUrl() { + return url; + } + + public void setUrl(final String url) { + this.url = url; + } + + public List getSubjectOf() { + return subjectOf; + } + + public void setSubjectOf(final List subjectOf) { + this.subjectOf = subjectOf; + } + + public List getAdditionalProperties() { + return additionalProperties; + } + + public void setAdditionalProperties(final List additionalProperties) { + this.additionalProperties = additionalProperties; + } + + // public URI getAdditionalType() { + // return additionalType; + // } + +} diff --git a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDStructuredValue.java b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDStructuredValue.java similarity index 94% rename from models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDStructuredValue.java rename to core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDStructuredValue.java index 3122241f2..526f42e05 100644 --- a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/model/JsonLDStructuredValue.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/model/JsonLDStructuredValue.java @@ -1,47 +1,47 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - -@JsonPropertyOrder({"@type", "name", "url", "identifier"}) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class JsonLDStructuredValue implements BioschemasObject { - - @JsonProperty("@type") - private final String type = "StructuredValue"; - - private String name; - private String url; - private String identifier; - - public String getName() { - return name; - } - - public String getUrl() { - return url; - } - - public void setIdentifier(String identifier) { - this.identifier = identifier; - } - - public void setName(String name) { - this.name = name; - } - - public void setUrl(String url) { - this.url = url; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonPropertyOrder({"@type", "name", "url", "identifier"}) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class JsonLDStructuredValue implements BioschemasObject { + + @JsonProperty("@type") + private final String type = "StructuredValue"; + + private String name; + private String url; + private String identifier; + + public String getName() { + return name; + } + + public String getUrl() { + return url; + } + + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + + public void setName(String name) { + this.name = name; + } + + public void setUrl(String url) { + this.url = url; + } +} diff --git a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/service/ContextDeserializer.java b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/service/ContextDeserializer.java similarity index 94% rename from models/jsonld/src/main/java/uk/ac/ebi/biosamples/service/ContextDeserializer.java rename to core/src/main/java/uk/ac/ebi/biosamples/jsonld/service/ContextDeserializer.java index 1d40b6668..88453e7a9 100644 --- a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/service/ContextDeserializer.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/service/ContextDeserializer.java @@ -1,60 +1,60 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import java.io.IOException; -import java.net.URI; -import uk.ac.ebi.biosamples.model.BioSchemasContext; - -public class ContextDeserializer extends StdDeserializer { - - protected ContextDeserializer() { - super(BioSchemasContext.class); - } - - @Override - public BioSchemasContext deserialize( - final JsonParser jsonParser, final DeserializationContext deserializationContext) - throws IOException { - final BioSchemasContext context = new BioSchemasContext(); - - JsonToken currentToken = jsonParser.getCurrentToken(); - if (currentToken.equals(JsonToken.START_ARRAY)) { - while (jsonParser.hasCurrentToken() && !currentToken.equals(JsonToken.END_ARRAY)) { - currentToken = jsonParser.nextToken(); - - if (currentToken.equals(JsonToken.VALUE_STRING) - && !jsonParser.getValueAsString().equals("http://schema.org")) { - // This is not the link to schema.org we expect to see - throw new JsonParseException( - jsonParser, "BioSchemasContext should contain a single schema.org entry string"); - } else if (currentToken.equals(JsonToken.START_OBJECT)) { - while (jsonParser.hasCurrentToken() && !currentToken.equals(JsonToken.END_OBJECT)) { - currentToken = jsonParser.nextToken(); - if (currentToken.equals(JsonToken.FIELD_NAME)) { - context.addOtherContexts( - jsonParser.getValueAsString(), URI.create(jsonParser.nextTextValue())); - } - } - } - } - } else { - throw new JsonParseException(jsonParser, "BioSchemasContext should be an array"); - } - - return context; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.service; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import java.io.IOException; +import java.net.URI; +import uk.ac.ebi.biosamples.jsonld.model.BioSchemasContext; + +public class ContextDeserializer extends StdDeserializer { + + protected ContextDeserializer() { + super(BioSchemasContext.class); + } + + @Override + public BioSchemasContext deserialize( + final JsonParser jsonParser, final DeserializationContext deserializationContext) + throws IOException { + final BioSchemasContext context = new BioSchemasContext(); + + JsonToken currentToken = jsonParser.getCurrentToken(); + if (currentToken.equals(JsonToken.START_ARRAY)) { + while (jsonParser.hasCurrentToken() && !currentToken.equals(JsonToken.END_ARRAY)) { + currentToken = jsonParser.nextToken(); + + if (currentToken.equals(JsonToken.VALUE_STRING) + && !jsonParser.getValueAsString().equals("http://schema.org")) { + // This is not the link to schema.org we expect to see + throw new JsonParseException( + jsonParser, "BioSchemasContext should contain a single schema.org entry string"); + } else if (currentToken.equals(JsonToken.START_OBJECT)) { + while (jsonParser.hasCurrentToken() && !currentToken.equals(JsonToken.END_OBJECT)) { + currentToken = jsonParser.nextToken(); + if (currentToken.equals(JsonToken.FIELD_NAME)) { + context.addOtherContexts( + jsonParser.getValueAsString(), URI.create(jsonParser.nextTextValue())); + } + } + } + } + } else { + throw new JsonParseException(jsonParser, "BioSchemasContext should be an array"); + } + + return context; + } +} diff --git a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/service/ContextSerializer.java b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/service/ContextSerializer.java similarity index 92% rename from models/jsonld/src/main/java/uk/ac/ebi/biosamples/service/ContextSerializer.java rename to core/src/main/java/uk/ac/ebi/biosamples/jsonld/service/ContextSerializer.java index c0ca99fed..834dc4b94 100644 --- a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/service/ContextSerializer.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/service/ContextSerializer.java @@ -1,49 +1,49 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import java.io.IOException; -import uk.ac.ebi.biosamples.model.BioSchemasContext; - -public class ContextSerializer extends StdSerializer { - - public ContextSerializer() { - super(BioSchemasContext.class); - } - - @Override - public void serialize( - final BioSchemasContext bioSchemasContext, - final JsonGenerator jsonGenerator, - final SerializerProvider serializerProvider) - throws IOException { - - // Write the @base field -> Not sure why this is need, but following UNIPROT convention - // jsonGenerator.writeStartObject(); - // jsonGenerator.writeStringField("@base", bioSchemasContext.getBaseContext().toString()); - // jsonGenerator.writeEndObject(); - - jsonGenerator.writeStartArray(); - - // Write the schema.org base namespace - jsonGenerator.writeString(bioSchemasContext.getSchemaOrgContext().toString()); - - // Write all the other contexts - if (!bioSchemasContext.getOtherContexts().isEmpty()) { - jsonGenerator.writeObject(bioSchemasContext.getOtherContexts()); - } - - jsonGenerator.writeEndArray(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.service; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import uk.ac.ebi.biosamples.jsonld.model.BioSchemasContext; + +public class ContextSerializer extends StdSerializer { + + public ContextSerializer() { + super(BioSchemasContext.class); + } + + @Override + public void serialize( + final BioSchemasContext bioSchemasContext, + final JsonGenerator jsonGenerator, + final SerializerProvider serializerProvider) + throws IOException { + + // Write the @base field -> Not sure why this is need, but following UNIPROT convention + // jsonGenerator.writeStartObject(); + // jsonGenerator.writeStringField("@base", bioSchemasContext.getBaseContext().toString()); + // jsonGenerator.writeEndObject(); + + jsonGenerator.writeStartArray(); + + // Write the schema.org base namespace + jsonGenerator.writeString(bioSchemasContext.getSchemaOrgContext().toString()); + + // Write all the other contexts + if (!bioSchemasContext.getOtherContexts().isEmpty()) { + jsonGenerator.writeObject(bioSchemasContext.getOtherContexts()); + } + + jsonGenerator.writeEndArray(); + } +} diff --git a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/service/SampleToJsonLDSampleRecordConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/service/SampleToJsonLDSampleRecordConverter.java similarity index 91% rename from models/jsonld/src/main/java/uk/ac/ebi/biosamples/service/SampleToJsonLDSampleRecordConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/jsonld/service/SampleToJsonLDSampleRecordConverter.java index 623e13898..c348c3e94 100644 --- a/models/jsonld/src/main/java/uk/ac/ebi/biosamples/service/SampleToJsonLDSampleRecordConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/jsonld/service/SampleToJsonLDSampleRecordConverter.java @@ -1,118 +1,124 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; -import org.springframework.core.convert.converter.Converter; -import uk.ac.ebi.biosamples.model.*; - -public class SampleToJsonLDSampleRecordConverter implements Converter { - - @Override - public JsonLDDataRecord convert(final Sample sample) { - - final JsonLDDataRecord sampleRecord = new JsonLDDataRecord(); - - sampleRecord.dateCreated(sample.getCreate().atZone(ZoneId.of("UTC"))); - sampleRecord.dateReleased(sample.getRelease().atZone(ZoneId.of("UTC"))); - sampleRecord.dateModified(sample.getUpdate().atZone(ZoneId.of("UTC"))); - - if (sample.getSubmitted() != null) { - sampleRecord.dateSubmitted(sample.getSubmitted().atZone(ZoneId.of("UTC"))); - } - - final JsonLDSample jsonLD = new JsonLDSample(); - final String[] identifiers = {getBioSamplesIdentifierDotOrg(sample.getAccession())}; - jsonLD.setSameAs(getBioSamplesIdentifierDotOrgLink(sample.getAccession())); - jsonLD.setIdentifiers(identifiers); - jsonLD.setName(sample.getName()); - - final List jsonLDAttributeList = getAttributeList(sample); - if (!jsonLDAttributeList.isEmpty()) { - final Optional optionalDescription = - jsonLDAttributeList.stream() - .filter(attr -> attr.getName().equalsIgnoreCase("description")) - .findFirst(); - if (optionalDescription.isPresent()) { - final JsonLDPropertyValue description = optionalDescription.get(); - jsonLD.setDescription(description.getValue()); - jsonLDAttributeList.remove(description); - } - jsonLD.setAdditionalProperties(jsonLDAttributeList); - } - - final List datasets = getDatasets(sample); - if (!datasets.isEmpty()) { - jsonLD.setSubjectOf(datasets); - } - - sampleRecord.identifier(getBioSamplesIdentifierDotOrg(sample.getAccession())); - sampleRecord.mainEntity(jsonLD); - - return sampleRecord; - } - - private List getAttributeList(final Sample sample) { - final Iterator attributesIterator = sample.getAttributes().iterator(); - final List jsonLDAttributeList = new ArrayList<>(); - while (attributesIterator.hasNext()) { - final Attribute attr = attributesIterator.next(); - final JsonLDPropertyValue pv = new JsonLDPropertyValue(); - pv.setName(attr.getType()); - pv.setValue(attr.getValue()); - if (!attr.getIri().isEmpty()) { - // this only puts the first IRI in - // JsonLDMedicalCode medicalCode = new JsonLDMedicalCode(); - // medicalCode.setTermCode(attr.getIri().iterator().next()); - - // Assuming that if the iri is not starting with a http[s] is - // probably a CURIE - final List valueReferences = new ArrayList<>(); - for (final String iri : attr.getIri()) { - final JsonLDDefinedTerm valueReference = new JsonLDDefinedTerm(); - if (iri.matches("^https?://.*")) { - valueReference.setId(iri); - } else { - valueReference.setTermCode(iri); - } - valueReferences.add(valueReference); - } - pv.setValueReference(valueReferences); - } - jsonLDAttributeList.add(pv); - } - return jsonLDAttributeList; - } - - private List getDatasets(final Sample sample) { - final Iterator externalRefsIterator = - sample.getExternalReferences().iterator(); - final List datasets = new ArrayList<>(); - while (externalRefsIterator.hasNext()) { - final ExternalReference externalReference = externalRefsIterator.next(); - datasets.add(externalReference.getUrl()); - } - return datasets; - } - - private String getBioSamplesIdentifierDotOrg(final String accession) { - return "biosample:" + accession; - } - - // TODO change identifiers - private String getBioSamplesIdentifierDotOrgLink(final String accession) { - return "http://identifiers.org/biosample/" + accession; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.service; + +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import org.springframework.core.convert.converter.Converter; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.ExternalReference; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.jsonld.model.JsonLDDataRecord; +import uk.ac.ebi.biosamples.jsonld.model.JsonLDDefinedTerm; +import uk.ac.ebi.biosamples.jsonld.model.JsonLDPropertyValue; +import uk.ac.ebi.biosamples.jsonld.model.JsonLDSample; + +public class SampleToJsonLDSampleRecordConverter implements Converter { + + @Override + public JsonLDDataRecord convert(final Sample sample) { + + final JsonLDDataRecord sampleRecord = new JsonLDDataRecord(); + + sampleRecord.dateCreated(sample.getCreate().atZone(ZoneId.of("UTC"))); + sampleRecord.dateReleased(sample.getRelease().atZone(ZoneId.of("UTC"))); + sampleRecord.dateModified(sample.getUpdate().atZone(ZoneId.of("UTC"))); + + if (sample.getSubmitted() != null) { + sampleRecord.dateSubmitted(sample.getSubmitted().atZone(ZoneId.of("UTC"))); + } + + final JsonLDSample jsonLD = new JsonLDSample(); + final String[] identifiers = {getBioSamplesIdentifierDotOrg(sample.getAccession())}; + jsonLD.setSameAs(getBioSamplesIdentifierDotOrgLink(sample.getAccession())); + jsonLD.setIdentifiers(identifiers); + jsonLD.setName(sample.getName()); + + final List jsonLDAttributeList = getAttributeList(sample); + if (!jsonLDAttributeList.isEmpty()) { + final Optional optionalDescription = + jsonLDAttributeList.stream() + .filter(attr -> attr.getName().equalsIgnoreCase("description")) + .findFirst(); + if (optionalDescription.isPresent()) { + final JsonLDPropertyValue description = optionalDescription.get(); + jsonLD.setDescription(description.getValue()); + jsonLDAttributeList.remove(description); + } + jsonLD.setAdditionalProperties(jsonLDAttributeList); + } + + final List datasets = getDatasets(sample); + if (!datasets.isEmpty()) { + jsonLD.setSubjectOf(datasets); + } + + sampleRecord.identifier(getBioSamplesIdentifierDotOrg(sample.getAccession())); + sampleRecord.mainEntity(jsonLD); + + return sampleRecord; + } + + private List getAttributeList(final Sample sample) { + final Iterator attributesIterator = sample.getAttributes().iterator(); + final List jsonLDAttributeList = new ArrayList<>(); + while (attributesIterator.hasNext()) { + final Attribute attr = attributesIterator.next(); + final JsonLDPropertyValue pv = new JsonLDPropertyValue(); + pv.setName(attr.getType()); + pv.setValue(attr.getValue()); + if (!attr.getIri().isEmpty()) { + // this only puts the first IRI in + // JsonLDMedicalCode medicalCode = new JsonLDMedicalCode(); + // medicalCode.setTermCode(attr.getIri().iterator().next()); + + // Assuming that if the iri is not starting with a http[s] is + // probably a CURIE + final List valueReferences = new ArrayList<>(); + for (final String iri : attr.getIri()) { + final JsonLDDefinedTerm valueReference = new JsonLDDefinedTerm(); + if (iri.matches("^https?://.*")) { + valueReference.setId(iri); + } else { + valueReference.setTermCode(iri); + } + valueReferences.add(valueReference); + } + pv.setValueReference(valueReferences); + } + jsonLDAttributeList.add(pv); + } + return jsonLDAttributeList; + } + + private List getDatasets(final Sample sample) { + final Iterator externalRefsIterator = + sample.getExternalReferences().iterator(); + final List datasets = new ArrayList<>(); + while (externalRefsIterator.hasNext()) { + final ExternalReference externalReference = externalRefsIterator.next(); + datasets.add(externalReference.getUrl()); + } + return datasets; + } + + private String getBioSamplesIdentifierDotOrg(final String accession) { + return "biosample:" + accession; + } + + // TODO change identifiers + private String getBioSamplesIdentifierDotOrgLink(final String accession) { + return "http://identifiers.org/biosample/" + accession; + } +} diff --git a/messaging/src/main/java/uk/ac/ebi/biosamples/Messaging.java b/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java similarity index 86% rename from messaging/src/main/java/uk/ac/ebi/biosamples/Messaging.java rename to core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java index 77689af98..315c3e48e 100644 --- a/messaging/src/main/java/uk/ac/ebi/biosamples/Messaging.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java @@ -1,20 +1,20 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples; - -public class Messaging { - public static final String INDEXING_EXCHANGE = "biosamples.forindexing.solr"; - public static final String INDEXING_QUEUE = "biosamples.tobeindexed.solr"; - static final String REINDEXING_EXCHANGE = "biosamples.reindex.solr"; - public static final String REINDEXING_QUEUE = "biosamples.reindex.solr"; - public static final String UPLOAD_QUEUE = "biosamples.uploaded.files"; - public static final String UPLOAD_EXCHANGE = "biosamples.uploaded.files.exchange"; -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.messaging; + +public class MessagingConstants { + public static final String INDEXING_EXCHANGE = "biosamples.forindexing.solr"; + public static final String INDEXING_QUEUE = "biosamples.tobeindexed.solr"; + public static final String REINDEXING_EXCHANGE = "biosamples.reindex.solr"; + public static final String REINDEXING_QUEUE = "biosamples.reindex.solr"; + public static final String UPLOAD_QUEUE = "biosamples.uploaded.files"; + public static final String UPLOAD_EXCHANGE = "biosamples.uploaded.files.exchange"; +} diff --git a/messaging/src/main/java/uk/ac/ebi/biosamples/MessageConfig.java b/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java similarity index 73% rename from messaging/src/main/java/uk/ac/ebi/biosamples/MessageConfig.java rename to core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java index cb371e5e3..e9d592c41 100644 --- a/messaging/src/main/java/uk/ac/ebi/biosamples/MessageConfig.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java @@ -1,86 +1,91 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples; - -import org.springframework.amqp.core.*; -import org.springframework.amqp.rabbit.annotation.EnableRabbit; -import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter; -import org.springframework.amqp.support.converter.MessageConverter; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -@EnableRabbit -public class MessageConfig { - // declare queues - @Bean(name = "indexingQueue") - public Queue indexingQueue() { - return QueueBuilder.durable(Messaging.INDEXING_QUEUE).build(); - } - - @Bean(name = "reindexingQueue") - public Queue reindexingQueue() { - return QueueBuilder.durable(Messaging.REINDEXING_QUEUE).build(); - } - - @Bean(name = "uploaderQueue") - public Queue uploaderQueue() { - return QueueBuilder.durable(Messaging.UPLOAD_QUEUE).build(); - } - - // declare exchanges - @Bean(name = "indexingExchange") - public Exchange indexingExchange() { - return ExchangeBuilder.directExchange(Messaging.INDEXING_EXCHANGE).durable(true).build(); - } - - @Bean(name = "reindexingExchange") - public Exchange reindexingExchange() { - return ExchangeBuilder.directExchange(Messaging.REINDEXING_EXCHANGE).durable(true).build(); - } - - @Bean(name = "uploadExchange") - public Exchange uploadExchange() { - return ExchangeBuilder.fanoutExchange(Messaging.UPLOAD_EXCHANGE).durable(true).build(); - } - - // bind queues to exchanges - @Bean(name = "indexBinding") - public Binding indexBinding() { - return BindingBuilder.bind(indexingQueue()) - .to(indexingExchange()) - .with(Messaging.INDEXING_QUEUE) - .noargs(); - } - - @Bean(name = "reindexingBinding") - public Binding reindexBinding() { - return BindingBuilder.bind(reindexingQueue()) - .to(reindexingExchange()) - .with(Messaging.REINDEXING_QUEUE) - .noargs(); - } - - @Bean(name = "uploaderBindings") - public Binding uploadBinding() { - return BindingBuilder.bind(uploaderQueue()) - .to(uploadExchange()) - .with(Messaging.UPLOAD_QUEUE) - .noargs(); - } - - // enable messaging in json - // note that this class is not the same as the http MessageConverter class - @Bean - public MessageConverter getJackson2MessageConverter() { - return new Jackson2JsonMessageConverter(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.messaging.config; + +import org.springframework.amqp.core.*; +import org.springframework.amqp.rabbit.annotation.EnableRabbit; +import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter; +import org.springframework.amqp.support.converter.MessageConverter; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.biosamples.messaging.MessagingConstants; + +@Configuration +@EnableRabbit +public class MessageConfig { + // declare queues + @Bean(name = "indexingQueue") + public Queue indexingQueue() { + return QueueBuilder.durable(MessagingConstants.INDEXING_QUEUE).build(); + } + + @Bean(name = "reindexingQueue") + public Queue reindexingQueue() { + return QueueBuilder.durable(MessagingConstants.REINDEXING_QUEUE).build(); + } + + @Bean(name = "uploaderQueue") + public Queue uploaderQueue() { + return QueueBuilder.durable(MessagingConstants.UPLOAD_QUEUE).build(); + } + + // declare exchanges + @Bean(name = "indexingExchange") + public Exchange indexingExchange() { + return ExchangeBuilder.directExchange(MessagingConstants.INDEXING_EXCHANGE) + .durable(true) + .build(); + } + + @Bean(name = "reindexingExchange") + public Exchange reindexingExchange() { + return ExchangeBuilder.directExchange(MessagingConstants.REINDEXING_EXCHANGE) + .durable(true) + .build(); + } + + @Bean(name = "uploadExchange") + public Exchange uploadExchange() { + return ExchangeBuilder.fanoutExchange(MessagingConstants.UPLOAD_EXCHANGE).durable(true).build(); + } + + // bind queues to exchanges + @Bean(name = "indexBinding") + public Binding indexBinding() { + return BindingBuilder.bind(indexingQueue()) + .to(indexingExchange()) + .with(MessagingConstants.INDEXING_QUEUE) + .noargs(); + } + + @Bean(name = "reindexingBinding") + public Binding reindexBinding() { + return BindingBuilder.bind(reindexingQueue()) + .to(reindexingExchange()) + .with(MessagingConstants.REINDEXING_QUEUE) + .noargs(); + } + + @Bean(name = "uploaderBindings") + public Binding uploadBinding() { + return BindingBuilder.bind(uploaderQueue()) + .to(uploadExchange()) + .with(MessagingConstants.UPLOAD_QUEUE) + .noargs(); + } + + // enable messaging in json + // note that this class is not the same as the http MessageConverter class + @Bean + public MessageConverter getJackson2MessageConverter() { + return new Jackson2JsonMessageConverter(); + } +} diff --git a/messaging/src/main/java/uk/ac/ebi/biosamples/MessageContent.java b/core/src/main/java/uk/ac/ebi/biosamples/messaging/model/MessageContent.java similarity index 91% rename from messaging/src/main/java/uk/ac/ebi/biosamples/MessageContent.java rename to core/src/main/java/uk/ac/ebi/biosamples/messaging/model/MessageContent.java index 8b1bb2ede..4770eff2b 100644 --- a/messaging/src/main/java/uk/ac/ebi/biosamples/MessageContent.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/messaging/model/MessageContent.java @@ -1,61 +1,61 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.List; -import lombok.Getter; -import uk.ac.ebi.biosamples.model.CurationLink; -import uk.ac.ebi.biosamples.model.Sample; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@Getter -public class MessageContent { - private final Sample sample; - private final CurationLink curationLink; - private final List related; - private final boolean delete; - private final String creationTime; - - private MessageContent( - final Sample sample, - final CurationLink curationLink, - final List related, - final boolean delete) { - this.sample = sample; - this.curationLink = curationLink; - this.related = related; - this.delete = delete; - - creationTime = ZonedDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT); - } - - @Override - public String toString() { - final String sb = - "MessageContent(" + sample + "," + curationLink + "," + related + "," + delete + ")"; - return sb; - } - - @JsonCreator - public static MessageContent build( - @JsonProperty("sample") final Sample sample, - @JsonProperty("curationLink") final CurationLink curationLink, - @JsonProperty("related") final List related, - @JsonProperty("delete") final boolean delete) { - return new MessageContent(sample, curationLink, related, delete); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.messaging.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import lombok.Getter; +import uk.ac.ebi.biosamples.core.model.CurationLink; +import uk.ac.ebi.biosamples.core.model.Sample; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@Getter +public class MessageContent { + private final Sample sample; + private final CurationLink curationLink; + private final List related; + private final boolean delete; + private final String creationTime; + + private MessageContent( + final Sample sample, + final CurationLink curationLink, + final List related, + final boolean delete) { + this.sample = sample; + this.curationLink = curationLink; + this.related = related; + this.delete = delete; + + creationTime = ZonedDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT); + } + + @Override + public String toString() { + final String sb = + "MessageContent(" + sample + "," + curationLink + "," + related + "," + delete + ")"; + return sb; + } + + @JsonCreator + public static MessageContent build( + @JsonProperty("sample") final Sample sample, + @JsonProperty("curationLink") final CurationLink curationLink, + @JsonProperty("related") final List related, + @JsonProperty("delete") final boolean delete) { + return new MessageContent(sample, curationLink, related, delete); + } +} diff --git a/messaging/src/main/java/uk/ac/ebi/biosamples/MessageUtils.java b/core/src/main/java/uk/ac/ebi/biosamples/messaging/service/MessageUtils.java similarity index 95% rename from messaging/src/main/java/uk/ac/ebi/biosamples/MessageUtils.java rename to core/src/main/java/uk/ac/ebi/biosamples/messaging/service/MessageUtils.java index 432165bc5..e96636a02 100644 --- a/messaging/src/main/java/uk/ac/ebi/biosamples/MessageUtils.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/messaging/service/MessageUtils.java @@ -1,31 +1,31 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -@Component -public class MessageUtils { - @Autowired private RabbitTemplate rabbitTemplate; - private final Logger log = LoggerFactory.getLogger(getClass()); - - public long getQueueCount(final String queue) { - final long count = rabbitTemplate.execute(channel -> channel.messageCount(queue)); - - log.trace("Number of messages in " + queue + " = " + count); - - return count; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.messaging.service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class MessageUtils { + @Autowired private RabbitTemplate rabbitTemplate; + private final Logger log = LoggerFactory.getLogger(getClass()); + + public long getQueueCount(final String queue) { + final long count = rabbitTemplate.execute(channel -> channel.messageCount(queue)); + + log.trace("Number of messages in " + queue + " = " + count); + + return count; + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/MongoProperties.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/MongoProperties.java similarity index 97% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/MongoProperties.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/MongoProperties.java index 4354c19a6..979e5a1ae 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/MongoProperties.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/MongoProperties.java @@ -1,22 +1,22 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Data -@Component -public class MongoProperties { - @Value("${biosamples.mongo.sample.writeConcern:1}") - private String sampleWriteConcern; -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class MongoProperties { + @Value("${biosamples.mongo.sample.writeConcern:1}") + private String sampleWriteConcern; +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/MongoConfig.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/config/MongoConfig.java similarity index 92% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/MongoConfig.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/config/MongoConfig.java index 63d9889da..afd0541cc 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/MongoConfig.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/config/MongoConfig.java @@ -1,44 +1,44 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.convert.MongoConverter; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; -import uk.ac.ebi.biosamples.mongo.service.CustomWriteConcernResolver; - -@Configuration -@EnableMongoRepositories(basePackageClasses = MongoConfig.class) -public class MongoConfig { - @Value("${spring.data.mongodb.uri:mongodb://localhost:27017/biosamples}") - private String mongoDbUrl; - - @Bean - public MongoTemplate mongoTemplate( - final MongoDatabaseFactory mongoDatabaseFactory, - final MongoConverter mongoConverter, - final CustomWriteConcernResolver customWriteConcernResolver) { - final MongoTemplate ops = new MongoTemplate(mongoDatabaseFactory, mongoConverter); - ops.setWriteConcernResolver(customWriteConcernResolver); - - return ops; - } - - @Bean - public MongoDatabaseFactory mongoDatabaseFactory() { - return new SimpleMongoClientDatabaseFactory(mongoDbUrl); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.mongodb.MongoDatabaseFactory; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.convert.MongoConverter; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import uk.ac.ebi.biosamples.mongo.service.CustomWriteConcernResolver; + +@Configuration +@EnableMongoRepositories(basePackages = "uk.ac.ebi.biosamples.mongo.repository") +public class MongoConfig { + @Value("${spring.data.mongodb.uri:mongodb://localhost:27017/biosamples}") + private String mongoDbUrl; + + @Bean + public MongoTemplate mongoTemplate( + final MongoDatabaseFactory mongoDatabaseFactory, + final MongoConverter mongoConverter, + final CustomWriteConcernResolver customWriteConcernResolver) { + final MongoTemplate ops = new MongoTemplate(mongoDatabaseFactory, mongoConverter); + ops.setWriteConcernResolver(customWriteConcernResolver); + + return ops; + } + + @Bean + public MongoDatabaseFactory mongoDatabaseFactory() { + return new SimpleMongoClientDatabaseFactory(mongoDbUrl); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoAnalytics.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoAnalytics.java similarity index 92% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoAnalytics.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoAnalytics.java index dde525073..4c140485a 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoAnalytics.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoAnalytics.java @@ -1,81 +1,81 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.model; - -import java.util.ArrayList; -import java.util.List; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; -import org.springframework.data.mongodb.core.mapping.Field; -import uk.ac.ebi.biosamples.model.PipelineAnalytics; -import uk.ac.ebi.biosamples.model.SampleAnalytics; - -@Document -public class MongoAnalytics { - @Id protected String collectionDate; - private final int year; - private final int month; - private final int day; - - protected List pipelines; - - @Field("samples") - private SampleAnalytics sampleAnalytics; - - public MongoAnalytics( - final String collectionDate, final int year, final int month, final int day) { - this.collectionDate = collectionDate; - pipelines = new ArrayList<>(); - this.year = year; - this.month = month; - this.day = day; - } - - public String getCollectionDate() { - return collectionDate; - } - - public void setCollectionDate(final String collectionDate) { - this.collectionDate = collectionDate; - } - - public List getPipelineAnalytics() { - return pipelines; - } - - public void addPipelineAnalytics(final PipelineAnalytics pipelineAnalytics) { - pipelines.add(pipelineAnalytics); - } - - public SampleAnalytics getSampleAnalytics() { - return sampleAnalytics; - } - - public void setSampleAnalytics(final SampleAnalytics sampleAnalytics) { - this.sampleAnalytics = sampleAnalytics; - } - - public int getYear() { - return year; - } - - public int getMonth() { - return month; - } - - public int getDay() { - return day; - } - - public List getPipelines() { - return pipelines; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.model; + +import java.util.ArrayList; +import java.util.List; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.data.mongodb.core.mapping.Field; +import uk.ac.ebi.biosamples.core.model.PipelineAnalytics; +import uk.ac.ebi.biosamples.core.model.SampleAnalytics; + +@Document +public class MongoAnalytics { + @Id protected String collectionDate; + private final int year; + private final int month; + private final int day; + + protected List pipelines; + + @Field("samples") + private SampleAnalytics sampleAnalytics; + + public MongoAnalytics( + final String collectionDate, final int year, final int month, final int day) { + this.collectionDate = collectionDate; + pipelines = new ArrayList<>(); + this.year = year; + this.month = month; + this.day = day; + } + + public String getCollectionDate() { + return collectionDate; + } + + public void setCollectionDate(final String collectionDate) { + this.collectionDate = collectionDate; + } + + public List getPipelineAnalytics() { + return pipelines; + } + + public void addPipelineAnalytics(final PipelineAnalytics pipelineAnalytics) { + pipelines.add(pipelineAnalytics); + } + + public SampleAnalytics getSampleAnalytics() { + return sampleAnalytics; + } + + public void setSampleAnalytics(final SampleAnalytics sampleAnalytics) { + this.sampleAnalytics = sampleAnalytics; + } + + public int getYear() { + return year; + } + + public int getMonth() { + return month; + } + + public int getDay() { + return day; + } + + public List getPipelines() { + return pipelines; + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoAuthChangeRecord.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoAuthChangeRecord.java similarity index 100% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoAuthChangeRecord.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoAuthChangeRecord.java diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCertificate.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCertificate.java similarity index 94% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCertificate.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCertificate.java index 3d1ad6c84..81faf2e15 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCertificate.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCertificate.java @@ -1,145 +1,145 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.model; - -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Objects; -import uk.ac.ebi.biosamples.model.Certificate; - -public class MongoCertificate implements Comparable { - private String name; - private String version; - private String fileName; - - public MongoCertificate(final String name, final String version, final String fileName) { - this.name = name; - this.version = version; - this.fileName = fileName; - } - - private MongoCertificate() {} - - @JsonProperty("name") - public String getName() { - return name; - } - - public void setName(final String name) { - this.name = name; - } - - @JsonProperty("version") - public String getVersion() { - return version; - } - - public void setVersion(final String version) { - this.version = version; - } - - @JsonProperty("fileName") - public String getFileName() { - return fileName; - } - - public void setFileName(final String fileName) { - this.fileName = fileName; - } - - @Override - public String toString() { - return "Certificate{" - + "name='" - + name - + '\'' - + ", version='" - + version - + '\'' - + ", fileName='" - + fileName - + '\'' - + '}'; - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (!(o instanceof Certificate)) { - return false; - } - final Certificate that = (Certificate) o; - return Objects.equals(getName(), that.getName()) - && Objects.equals(getVersion(), that.getVersion()) - && Objects.equals(getFileName(), that.getFileName()); - } - - @Override - public int hashCode() { - return Objects.hash(getName(), getVersion(), getFileName()); - } - - @Override - public int compareTo(final MongoCertificate cert) { - if (cert == null) { - return 1; - } - - int comparison = nullSafeStringComparison(name, cert.name); - - if (comparison != 0) { - return comparison; - } - - comparison = nullSafeStringComparison(version, cert.version); - return comparison; - } - - private int nullSafeStringComparison(final String one, final String two) { - if (one == null && two != null) { - return -1; - } - if (one != null && two == null) { - return 1; - } - if (one != null && !one.equals(two)) { - return one.compareTo(two); - } - - return 0; - } - - public static MongoCertificate build(String name, String version, final String fileName) { - // check for nulls - if (name == null) { - throw new IllegalArgumentException("Certificate name must not be null"); - } - - if (version == null) { - version = ""; - } - - if (fileName == null) { - throw new IllegalArgumentException("Certificate file name must not be null"); - } - - name = name.trim(); - version = version.trim(); - - final MongoCertificate cert = new MongoCertificate(); - cert.name = name; - cert.fileName = fileName; - cert.version = version; - - return cert; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Objects; +import uk.ac.ebi.biosamples.core.model.Certificate; + +public class MongoCertificate implements Comparable { + private String name; + private String version; + private String fileName; + + public MongoCertificate(final String name, final String version, final String fileName) { + this.name = name; + this.version = version; + this.fileName = fileName; + } + + private MongoCertificate() {} + + @JsonProperty("name") + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + @JsonProperty("version") + public String getVersion() { + return version; + } + + public void setVersion(final String version) { + this.version = version; + } + + @JsonProperty("fileName") + public String getFileName() { + return fileName; + } + + public void setFileName(final String fileName) { + this.fileName = fileName; + } + + @Override + public String toString() { + return "Certificate{" + + "name='" + + name + + '\'' + + ", version='" + + version + + '\'' + + ", fileName='" + + fileName + + '\'' + + '}'; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Certificate)) { + return false; + } + final Certificate that = (Certificate) o; + return Objects.equals(getName(), that.getName()) + && Objects.equals(getVersion(), that.getVersion()) + && Objects.equals(getFileName(), that.getFileName()); + } + + @Override + public int hashCode() { + return Objects.hash(getName(), getVersion(), getFileName()); + } + + @Override + public int compareTo(final MongoCertificate cert) { + if (cert == null) { + return 1; + } + + int comparison = nullSafeStringComparison(name, cert.name); + + if (comparison != 0) { + return comparison; + } + + comparison = nullSafeStringComparison(version, cert.version); + return comparison; + } + + private int nullSafeStringComparison(final String one, final String two) { + if (one == null && two != null) { + return -1; + } + if (one != null && two == null) { + return 1; + } + if (one != null && !one.equals(two)) { + return one.compareTo(two); + } + + return 0; + } + + public static MongoCertificate build(String name, String version, final String fileName) { + // check for nulls + if (name == null) { + throw new IllegalArgumentException("Certificate name must not be null"); + } + + if (version == null) { + version = ""; + } + + if (fileName == null) { + throw new IllegalArgumentException("Certificate file name must not be null"); + } + + name = name.trim(); + version = version.trim(); + + final MongoCertificate cert = new MongoCertificate(); + cert.name = name; + cert.fileName = fileName; + cert.version = version; + + return cert; + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCuration.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCuration.java similarity index 96% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCuration.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCuration.java index 027e41c61..68c691063 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCuration.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCuration.java @@ -1,348 +1,348 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.hash.Hasher; -import com.google.common.hash.Hashing; -import java.util.*; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.ExternalReference; -import uk.ac.ebi.biosamples.model.Relationship; - -@Document -public class MongoCuration implements Comparable { - - private final SortedSet attributesPre; - private final SortedSet attributesPost; - - private final SortedSet externalPre; - private final SortedSet externaPost; - - private final SortedSet relationshipsPre; - private final SortedSet relationshipsPost; - - @Id private String hash; - - private MongoCuration( - final Collection attributesPre, - final Collection attributesPost, - final Collection externalPre, - final Collection externaPost, - final Collection relationshipsPre, - final Collection relationshipsPost, - final String hash) { - this.attributesPre = Collections.unmodifiableSortedSet(new TreeSet<>(attributesPre)); - this.attributesPost = Collections.unmodifiableSortedSet(new TreeSet<>(attributesPost)); - this.externalPre = Collections.unmodifiableSortedSet(new TreeSet<>(externalPre)); - this.externaPost = Collections.unmodifiableSortedSet(new TreeSet<>(externaPost)); - this.relationshipsPre = - relationshipsPre != null - ? Collections.unmodifiableSortedSet(new TreeSet<>(relationshipsPre)) - : Collections.unmodifiableSortedSet(new TreeSet<>()); - this.relationshipsPost = - relationshipsPost != null - ? Collections.unmodifiableSortedSet(new TreeSet<>(relationshipsPost)) - : Collections.unmodifiableSortedSet(new TreeSet<>()); - this.hash = hash; - } - - @JsonProperty("attributesPre") - public SortedSet getAttributesPre() { - return attributesPre; - } - - @JsonProperty("attributesPost") - public SortedSet getAttributesPost() { - return attributesPost; - } - - @JsonProperty("externalReferencesPre") - public SortedSet getExternalReferencesPre() { - return externalPre; - } - - @JsonProperty("externalReferencesPost") - public SortedSet getExternalReferencesPost() { - return externaPost; - } - - @JsonProperty("relationshipsPre") - public SortedSet getRelationshipsPre() { - return relationshipsPre; - } - - @JsonProperty("relationshipsPost") - public SortedSet getRelationshipsPost() { - return relationshipsPost; - } - - public String getHash() { - return hash; - } - - @Override - public int hashCode() { - return Objects.hash(hash); - } - - @Override - public boolean equals(final Object o) { - if (o == this) { - return true; - } - if (!(o instanceof MongoCuration)) { - return false; - } - final MongoCuration other = (MongoCuration) o; - return Objects.equals(hash, other.hash) - && Objects.equals(attributesPre, other.attributesPre) - && Objects.equals(attributesPost, other.attributesPost) - && Objects.equals(externalPre, other.externalPre) - && Objects.equals(externaPost, other.externaPost) - && Objects.equals(relationshipsPre, other.relationshipsPre) - && Objects.equals(relationshipsPost, other.relationshipsPost); - } - - @Override - public int compareTo(final MongoCuration other) { - if (other == null) { - return 1; - } - - if (!attributesPre.equals(other.attributesPre)) { - if (attributesPre.size() < other.attributesPre.size()) { - return -1; - } else if (attributesPre.size() > other.attributesPre.size()) { - return 1; - } else { - final Iterator thisIt = attributesPre.iterator(); - final Iterator otherIt = other.attributesPre.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - if (!attributesPost.equals(other.attributesPost)) { - if (attributesPost.size() < other.attributesPost.size()) { - return -1; - } else if (attributesPost.size() > other.attributesPost.size()) { - return 1; - } else { - final Iterator thisIt = attributesPost.iterator(); - final Iterator otherIt = other.attributesPost.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - - if (!externalPre.equals(other.externalPre)) { - if (externalPre.size() < other.externalPre.size()) { - return -1; - } else if (externalPre.size() > other.externalPre.size()) { - return 1; - } else { - final Iterator thisIt = externalPre.iterator(); - final Iterator otherIt = other.externalPre.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - if (!externaPost.equals(other.externaPost)) { - if (externaPost.size() < other.externaPost.size()) { - return -1; - } else if (externaPost.size() > other.externaPost.size()) { - return 1; - } else { - final Iterator thisIt = externaPost.iterator(); - final Iterator otherIt = other.externaPost.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - - if (!relationshipsPre.equals(other.relationshipsPre)) { - if (relationshipsPre.size() < other.relationshipsPre.size()) { - return -1; - } else if (relationshipsPre.size() > other.relationshipsPre.size()) { - return 1; - } else { - final Iterator thisIt = relationshipsPre.iterator(); - final Iterator otherIt = other.relationshipsPre.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - if (!relationshipsPost.equals(other.relationshipsPost)) { - if (relationshipsPost.size() < other.relationshipsPost.size()) { - return -1; - } else if (relationshipsPost.size() > other.relationshipsPost.size()) { - return 1; - } else { - final Iterator thisIt = relationshipsPost.iterator(); - final Iterator otherIt = other.relationshipsPost.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - return 0; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - sb.append("MongoCuration("); - sb.append(attributesPre); - sb.append(","); - sb.append(attributesPost); - sb.append(","); - sb.append(externalPre); - sb.append(","); - sb.append(externaPost); - sb.append(","); - sb.append(relationshipsPre); - sb.append(","); - sb.append(relationshipsPost); - sb.append(")"); - return sb.toString(); - } - - @JsonCreator - public static MongoCuration build( - @JsonProperty("attributesPre") final Collection attributesPre, - @JsonProperty("attributesPost") final Collection attributesPost, - @JsonProperty("externalReferencesPre") final Collection externalPre, - @JsonProperty("externalReferencesPost") final Collection externaPost, - @JsonProperty("relationshipsPre") final Collection relationshipsPre, - @JsonProperty("relationshipsPost") final Collection relationshipsPost) { - - SortedSet sortedPreAttributes = new TreeSet<>(); - SortedSet sortedPostAttributes = new TreeSet<>(); - SortedSet sortedPreExternal = new TreeSet<>(); - SortedSet sortedPostExternal = new TreeSet<>(); - SortedSet sortedPreRelationships = new TreeSet<>(); - SortedSet sortedPostRelationships = new TreeSet<>(); - - if (attributesPre != null) { - sortedPreAttributes.addAll(attributesPre); - } - if (attributesPost != null) { - sortedPostAttributes.addAll(attributesPost); - } - if (externalPre != null) { - sortedPreExternal.addAll(externalPre); - } - if (externaPost != null) { - sortedPostExternal.addAll(externaPost); - } - if (relationshipsPre != null) { - sortedPreRelationships.addAll(relationshipsPre); - } - if (relationshipsPost != null) { - sortedPostRelationships.addAll(relationshipsPost); - } - - sortedPreAttributes = Collections.unmodifiableSortedSet(sortedPreAttributes); - sortedPostAttributes = Collections.unmodifiableSortedSet(sortedPostAttributes); - sortedPreExternal = Collections.unmodifiableSortedSet(sortedPreExternal); - sortedPostExternal = Collections.unmodifiableSortedSet(sortedPostExternal); - sortedPreRelationships = Collections.unmodifiableSortedSet(sortedPreRelationships); - sortedPostRelationships = Collections.unmodifiableSortedSet(sortedPostRelationships); - - final Hasher hasher = Hashing.sha256().newHasher(); - for (final Attribute a : sortedPreAttributes) { - hasher.putUnencodedChars(a.getType()); - hasher.putUnencodedChars(a.getValue()); - if (a.getUnit() != null) { - hasher.putUnencodedChars(a.getUnit()); - } - if (a.getIri() != null) { - for (final String iri : a.getIri()) { - hasher.putUnencodedChars(iri); - } - } - } - for (final Attribute a : sortedPostAttributes) { - hasher.putUnencodedChars(a.getType()); - hasher.putUnencodedChars(a.getValue()); - if (a.getUnit() != null) { - hasher.putUnencodedChars(a.getUnit()); - } - if (a.getIri() != null) { - for (final String iri : a.getIri()) { - hasher.putUnencodedChars(iri); - } - } - } - for (final ExternalReference a : sortedPreExternal) { - hasher.putUnencodedChars(a.getUrl()); - if (a.getDuo() != null) { - for (final String duo : a.getDuo()) { - hasher.putUnencodedChars(duo); - } - } - } - for (final ExternalReference a : sortedPostExternal) { - hasher.putUnencodedChars(a.getUrl()); - if (a.getDuo() != null) { - for (final String duo : a.getDuo()) { - hasher.putUnencodedChars(duo); - } - } - } - for (final Relationship a : sortedPreRelationships) { - hasher.putUnencodedChars(a.getSource()); - hasher.putUnencodedChars(a.getTarget()); - hasher.putUnencodedChars(a.getType()); - } - for (final Relationship a : sortedPostRelationships) { - hasher.putUnencodedChars(a.getSource()); - hasher.putUnencodedChars(a.getTarget()); - hasher.putUnencodedChars(a.getType()); - } - final String hash = hasher.hash().toString(); - - return new MongoCuration( - sortedPreAttributes, - sortedPostAttributes, - sortedPreExternal, - sortedPostExternal, - sortedPreRelationships, - sortedPostRelationships, - hash); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import java.util.*; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.ExternalReference; +import uk.ac.ebi.biosamples.core.model.Relationship; + +@Document +public class MongoCuration implements Comparable { + + private final SortedSet attributesPre; + private final SortedSet attributesPost; + + private final SortedSet externalPre; + private final SortedSet externaPost; + + private final SortedSet relationshipsPre; + private final SortedSet relationshipsPost; + + @Id private String hash; + + private MongoCuration( + final Collection attributesPre, + final Collection attributesPost, + final Collection externalPre, + final Collection externaPost, + final Collection relationshipsPre, + final Collection relationshipsPost, + final String hash) { + this.attributesPre = Collections.unmodifiableSortedSet(new TreeSet<>(attributesPre)); + this.attributesPost = Collections.unmodifiableSortedSet(new TreeSet<>(attributesPost)); + this.externalPre = Collections.unmodifiableSortedSet(new TreeSet<>(externalPre)); + this.externaPost = Collections.unmodifiableSortedSet(new TreeSet<>(externaPost)); + this.relationshipsPre = + relationshipsPre != null + ? Collections.unmodifiableSortedSet(new TreeSet<>(relationshipsPre)) + : Collections.unmodifiableSortedSet(new TreeSet<>()); + this.relationshipsPost = + relationshipsPost != null + ? Collections.unmodifiableSortedSet(new TreeSet<>(relationshipsPost)) + : Collections.unmodifiableSortedSet(new TreeSet<>()); + this.hash = hash; + } + + @JsonProperty("attributesPre") + public SortedSet getAttributesPre() { + return attributesPre; + } + + @JsonProperty("attributesPost") + public SortedSet getAttributesPost() { + return attributesPost; + } + + @JsonProperty("externalReferencesPre") + public SortedSet getExternalReferencesPre() { + return externalPre; + } + + @JsonProperty("externalReferencesPost") + public SortedSet getExternalReferencesPost() { + return externaPost; + } + + @JsonProperty("relationshipsPre") + public SortedSet getRelationshipsPre() { + return relationshipsPre; + } + + @JsonProperty("relationshipsPost") + public SortedSet getRelationshipsPost() { + return relationshipsPost; + } + + public String getHash() { + return hash; + } + + @Override + public int hashCode() { + return Objects.hash(hash); + } + + @Override + public boolean equals(final Object o) { + if (o == this) { + return true; + } + if (!(o instanceof MongoCuration)) { + return false; + } + final MongoCuration other = (MongoCuration) o; + return Objects.equals(hash, other.hash) + && Objects.equals(attributesPre, other.attributesPre) + && Objects.equals(attributesPost, other.attributesPost) + && Objects.equals(externalPre, other.externalPre) + && Objects.equals(externaPost, other.externaPost) + && Objects.equals(relationshipsPre, other.relationshipsPre) + && Objects.equals(relationshipsPost, other.relationshipsPost); + } + + @Override + public int compareTo(final MongoCuration other) { + if (other == null) { + return 1; + } + + if (!attributesPre.equals(other.attributesPre)) { + if (attributesPre.size() < other.attributesPre.size()) { + return -1; + } else if (attributesPre.size() > other.attributesPre.size()) { + return 1; + } else { + final Iterator thisIt = attributesPre.iterator(); + final Iterator otherIt = other.attributesPre.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + if (!attributesPost.equals(other.attributesPost)) { + if (attributesPost.size() < other.attributesPost.size()) { + return -1; + } else if (attributesPost.size() > other.attributesPost.size()) { + return 1; + } else { + final Iterator thisIt = attributesPost.iterator(); + final Iterator otherIt = other.attributesPost.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + + if (!externalPre.equals(other.externalPre)) { + if (externalPre.size() < other.externalPre.size()) { + return -1; + } else if (externalPre.size() > other.externalPre.size()) { + return 1; + } else { + final Iterator thisIt = externalPre.iterator(); + final Iterator otherIt = other.externalPre.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + if (!externaPost.equals(other.externaPost)) { + if (externaPost.size() < other.externaPost.size()) { + return -1; + } else if (externaPost.size() > other.externaPost.size()) { + return 1; + } else { + final Iterator thisIt = externaPost.iterator(); + final Iterator otherIt = other.externaPost.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + + if (!relationshipsPre.equals(other.relationshipsPre)) { + if (relationshipsPre.size() < other.relationshipsPre.size()) { + return -1; + } else if (relationshipsPre.size() > other.relationshipsPre.size()) { + return 1; + } else { + final Iterator thisIt = relationshipsPre.iterator(); + final Iterator otherIt = other.relationshipsPre.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + if (!relationshipsPost.equals(other.relationshipsPost)) { + if (relationshipsPost.size() < other.relationshipsPost.size()) { + return -1; + } else if (relationshipsPost.size() > other.relationshipsPost.size()) { + return 1; + } else { + final Iterator thisIt = relationshipsPost.iterator(); + final Iterator otherIt = other.relationshipsPost.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + return 0; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("MongoCuration("); + sb.append(attributesPre); + sb.append(","); + sb.append(attributesPost); + sb.append(","); + sb.append(externalPre); + sb.append(","); + sb.append(externaPost); + sb.append(","); + sb.append(relationshipsPre); + sb.append(","); + sb.append(relationshipsPost); + sb.append(")"); + return sb.toString(); + } + + @JsonCreator + public static MongoCuration build( + @JsonProperty("attributesPre") final Collection attributesPre, + @JsonProperty("attributesPost") final Collection attributesPost, + @JsonProperty("externalReferencesPre") final Collection externalPre, + @JsonProperty("externalReferencesPost") final Collection externaPost, + @JsonProperty("relationshipsPre") final Collection relationshipsPre, + @JsonProperty("relationshipsPost") final Collection relationshipsPost) { + + SortedSet sortedPreAttributes = new TreeSet<>(); + SortedSet sortedPostAttributes = new TreeSet<>(); + SortedSet sortedPreExternal = new TreeSet<>(); + SortedSet sortedPostExternal = new TreeSet<>(); + SortedSet sortedPreRelationships = new TreeSet<>(); + SortedSet sortedPostRelationships = new TreeSet<>(); + + if (attributesPre != null) { + sortedPreAttributes.addAll(attributesPre); + } + if (attributesPost != null) { + sortedPostAttributes.addAll(attributesPost); + } + if (externalPre != null) { + sortedPreExternal.addAll(externalPre); + } + if (externaPost != null) { + sortedPostExternal.addAll(externaPost); + } + if (relationshipsPre != null) { + sortedPreRelationships.addAll(relationshipsPre); + } + if (relationshipsPost != null) { + sortedPostRelationships.addAll(relationshipsPost); + } + + sortedPreAttributes = Collections.unmodifiableSortedSet(sortedPreAttributes); + sortedPostAttributes = Collections.unmodifiableSortedSet(sortedPostAttributes); + sortedPreExternal = Collections.unmodifiableSortedSet(sortedPreExternal); + sortedPostExternal = Collections.unmodifiableSortedSet(sortedPostExternal); + sortedPreRelationships = Collections.unmodifiableSortedSet(sortedPreRelationships); + sortedPostRelationships = Collections.unmodifiableSortedSet(sortedPostRelationships); + + final Hasher hasher = Hashing.sha256().newHasher(); + for (final Attribute a : sortedPreAttributes) { + hasher.putUnencodedChars(a.getType()); + hasher.putUnencodedChars(a.getValue()); + if (a.getUnit() != null) { + hasher.putUnencodedChars(a.getUnit()); + } + if (a.getIri() != null) { + for (final String iri : a.getIri()) { + hasher.putUnencodedChars(iri); + } + } + } + for (final Attribute a : sortedPostAttributes) { + hasher.putUnencodedChars(a.getType()); + hasher.putUnencodedChars(a.getValue()); + if (a.getUnit() != null) { + hasher.putUnencodedChars(a.getUnit()); + } + if (a.getIri() != null) { + for (final String iri : a.getIri()) { + hasher.putUnencodedChars(iri); + } + } + } + for (final ExternalReference a : sortedPreExternal) { + hasher.putUnencodedChars(a.getUrl()); + if (a.getDuo() != null) { + for (final String duo : a.getDuo()) { + hasher.putUnencodedChars(duo); + } + } + } + for (final ExternalReference a : sortedPostExternal) { + hasher.putUnencodedChars(a.getUrl()); + if (a.getDuo() != null) { + for (final String duo : a.getDuo()) { + hasher.putUnencodedChars(duo); + } + } + } + for (final Relationship a : sortedPreRelationships) { + hasher.putUnencodedChars(a.getSource()); + hasher.putUnencodedChars(a.getTarget()); + hasher.putUnencodedChars(a.getType()); + } + for (final Relationship a : sortedPostRelationships) { + hasher.putUnencodedChars(a.getSource()); + hasher.putUnencodedChars(a.getTarget()); + hasher.putUnencodedChars(a.getType()); + } + final String hash = hasher.hash().toString(); + + return new MongoCuration( + sortedPreAttributes, + sortedPostAttributes, + sortedPreExternal, + sortedPostExternal, + sortedPreRelationships, + sortedPostRelationships, + hash); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationLink.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationLink.java similarity index 91% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationLink.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationLink.java index 1d5258bcf..9195fd7a4 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationLink.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationLink.java @@ -1,144 +1,144 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.google.common.hash.Hashing; -import java.time.Instant; -import java.util.Objects; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.Document; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.CurationLink; -import uk.ac.ebi.biosamples.service.CustomInstantDeserializer; -import uk.ac.ebi.biosamples.service.CustomInstantSerializer; - -@Document -public class MongoCurationLink implements Comparable { - @Id private final String hash; - - @Indexed(background = true) - private final String sample; - - private final String webinSubmissionAccountId; - - @Indexed(background = true) - protected final Instant created; - - private final Curation curation; - private String domain; - - private MongoCurationLink( - final String sample, - final String domain, - final String webinSubmissionAccountId, - final Curation curation, - final String hash, - final Instant created) { - this.sample = sample; - this.domain = domain; - this.webinSubmissionAccountId = webinSubmissionAccountId; - this.curation = curation; - this.hash = hash; - this.created = created; - } - - public String getSample() { - return sample; - } - - public Curation getCuration() { - return curation; - } - - public String getDomain() { - return domain; - } - - public String getWebinSubmissionAccountId() { - return webinSubmissionAccountId; - } - - public String getHash() { - return hash; - } - - @JsonSerialize(using = CustomInstantSerializer.class) - public Instant getCreated() { - return created; - } - - @Override - public boolean equals(final Object o) { - if (o == this) { - return true; - } - if (!(o instanceof CurationLink)) { - return false; - } - final MongoCurationLink other = (MongoCurationLink) o; - - return Objects.equals(curation, other.curation) && Objects.equals(sample, other.sample); - } - - @Override - public int hashCode() { - return Objects.hash(sample, domain, webinSubmissionAccountId, curation); - } - - @Override - public int compareTo(final MongoCurationLink other) { - if (other == null) { - return 1; - } - - if (!sample.equals(other.sample)) { - return sample.compareTo(other.sample); - } - - if (!curation.equals(other.curation)) { - return curation.compareTo(other.curation); - } - - return 0; - } - - @Override - public String toString() { - final String sb = "MongoCurationLink(" + sample + "," + curation + ")"; - - return sb; - } - - // Used for deserializtion (JSON -> Java) - @JsonCreator - public static MongoCurationLink build( - @JsonProperty("sample") final String sample, - @JsonProperty("curation") final Curation curation, - @JsonProperty("domain") final String domain, - @JsonProperty("webinSubmissionAccountId") final String webinSubmissionAccountId, - @JsonProperty("created") @JsonDeserialize(using = CustomInstantDeserializer.class) - final Instant created) { - final String hash = - Hashing.sha256() - .newHasher() - .putUnencodedChars(curation.getHash()) - .putUnencodedChars(sample) - .hash() - .toString(); - - return new MongoCurationLink(sample, domain, webinSubmissionAccountId, curation, hash, created); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.hash.Hashing; +import java.time.Instant; +import java.util.Objects; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.CurationLink; +import uk.ac.ebi.biosamples.core.service.CustomInstantDeserializer; +import uk.ac.ebi.biosamples.core.service.CustomInstantSerializer; + +@Document +public class MongoCurationLink implements Comparable { + @Id private final String hash; + + @Indexed(background = true) + private final String sample; + + private final String webinSubmissionAccountId; + + @Indexed(background = true) + protected final Instant created; + + private final Curation curation; + private String domain; + + private MongoCurationLink( + final String sample, + final String domain, + final String webinSubmissionAccountId, + final Curation curation, + final String hash, + final Instant created) { + this.sample = sample; + this.domain = domain; + this.webinSubmissionAccountId = webinSubmissionAccountId; + this.curation = curation; + this.hash = hash; + this.created = created; + } + + public String getSample() { + return sample; + } + + public Curation getCuration() { + return curation; + } + + public String getDomain() { + return domain; + } + + public String getWebinSubmissionAccountId() { + return webinSubmissionAccountId; + } + + public String getHash() { + return hash; + } + + @JsonSerialize(using = CustomInstantSerializer.class) + public Instant getCreated() { + return created; + } + + @Override + public boolean equals(final Object o) { + if (o == this) { + return true; + } + if (!(o instanceof CurationLink)) { + return false; + } + final MongoCurationLink other = (MongoCurationLink) o; + + return Objects.equals(curation, other.curation) && Objects.equals(sample, other.sample); + } + + @Override + public int hashCode() { + return Objects.hash(sample, domain, webinSubmissionAccountId, curation); + } + + @Override + public int compareTo(final MongoCurationLink other) { + if (other == null) { + return 1; + } + + if (!sample.equals(other.sample)) { + return sample.compareTo(other.sample); + } + + if (!curation.equals(other.curation)) { + return curation.compareTo(other.curation); + } + + return 0; + } + + @Override + public String toString() { + final String sb = "MongoCurationLink(" + sample + "," + curation + ")"; + + return sb; + } + + // Used for deserializtion (JSON -> Java) + @JsonCreator + public static MongoCurationLink build( + @JsonProperty("sample") final String sample, + @JsonProperty("curation") final Curation curation, + @JsonProperty("domain") final String domain, + @JsonProperty("webinSubmissionAccountId") final String webinSubmissionAccountId, + @JsonProperty("created") @JsonDeserialize(using = CustomInstantDeserializer.class) + final Instant created) { + final String hash = + Hashing.sha256() + .newHasher() + .putUnencodedChars(curation.getHash()) + .putUnencodedChars(sample) + .hash() + .toString(); + + return new MongoCurationLink(sample, domain, webinSubmissionAccountId, curation, hash, created); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationRule.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationRule.java similarity index 93% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationRule.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationRule.java index c9fd5de45..5976b2c0e 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationRule.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoCurationRule.java @@ -1,107 +1,107 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import java.time.Instant; -import java.util.Objects; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; -import uk.ac.ebi.biosamples.service.CustomInstantDeserializer; -import uk.ac.ebi.biosamples.service.CustomInstantSerializer; - -@Document -public class MongoCurationRule implements Comparable { - - @Id private String id; - private final String attributePre; - private final String attributePost; - - @JsonSerialize(using = CustomInstantSerializer.class) - @JsonDeserialize(using = CustomInstantDeserializer.class) - private Instant created; - - private MongoCurationRule(final String attributePre, final String attributePost) { - this.attributePre = attributePre; - this.attributePost = attributePost; - id = attributePre; - created = Instant.now(); - } - - public String getId() { - return id; - } - - public String getAttributePre() { - return attributePre; - } - - public String getAttributePost() { - return attributePost; - } - - public Instant getCreated() { - return created; - } - - @Override - public int hashCode() { - return Objects.hash(attributePre); - } - - @Override - public boolean equals(final Object o) { - if (o == this) { - return true; - } - if (!(o instanceof MongoCurationRule)) { - return false; - } - final MongoCurationRule other = (MongoCurationRule) o; - return Objects.equals(attributePre, other.attributePre) - && Objects.equals(attributePost, other.attributePost) - && Objects.equals(created, other.created); - } - - @Override - public int compareTo(final MongoCurationRule other) { - if (other == null) { - return 1; - } - - if (!attributePre.equals(other.attributePre)) { - return attributePre.compareTo(other.attributePre); - } else if (!attributePost.equals(other.attributePost)) { - return attributePost.compareTo(other.attributePost); - } else if (!created.equals(other.created)) { - return created.compareTo(other.created); - } - - return 0; - } - - @Override - public String toString() { - final String sb = - "MongoCurationRule(" + attributePre + "," + attributePost + "," + created + ")"; - return sb; - } - - @JsonCreator - public static MongoCurationRule build( - @JsonProperty("attributePre") final String attributePre, - @JsonProperty("attributePost") final String attributePost) { - return new MongoCurationRule(attributePre, attributePost); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.time.Instant; +import java.util.Objects; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; +import uk.ac.ebi.biosamples.core.service.CustomInstantDeserializer; +import uk.ac.ebi.biosamples.core.service.CustomInstantSerializer; + +@Document +public class MongoCurationRule implements Comparable { + + @Id private String id; + private final String attributePre; + private final String attributePost; + + @JsonSerialize(using = CustomInstantSerializer.class) + @JsonDeserialize(using = CustomInstantDeserializer.class) + private Instant created; + + private MongoCurationRule(final String attributePre, final String attributePost) { + this.attributePre = attributePre; + this.attributePost = attributePost; + id = attributePre; + created = Instant.now(); + } + + public String getId() { + return id; + } + + public String getAttributePre() { + return attributePre; + } + + public String getAttributePost() { + return attributePost; + } + + public Instant getCreated() { + return created; + } + + @Override + public int hashCode() { + return Objects.hash(attributePre); + } + + @Override + public boolean equals(final Object o) { + if (o == this) { + return true; + } + if (!(o instanceof MongoCurationRule)) { + return false; + } + final MongoCurationRule other = (MongoCurationRule) o; + return Objects.equals(attributePre, other.attributePre) + && Objects.equals(attributePost, other.attributePost) + && Objects.equals(created, other.created); + } + + @Override + public int compareTo(final MongoCurationRule other) { + if (other == null) { + return 1; + } + + if (!attributePre.equals(other.attributePre)) { + return attributePre.compareTo(other.attributePre); + } else if (!attributePost.equals(other.attributePost)) { + return attributePost.compareTo(other.attributePost); + } else if (!created.equals(other.created)) { + return created.compareTo(other.created); + } + + return 0; + } + + @Override + public String toString() { + final String sb = + "MongoCurationRule(" + attributePre + "," + attributePost + "," + created + ")"; + return sb; + } + + @JsonCreator + public static MongoCurationRule build( + @JsonProperty("attributePre") final String attributePre, + @JsonProperty("attributePost") final String attributePost) { + return new MongoCurationRule(attributePre, attributePost); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoExternalReference.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoExternalReference.java similarity index 96% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoExternalReference.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoExternalReference.java index d0399f94b..a7550444a 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoExternalReference.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoExternalReference.java @@ -1,150 +1,150 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.hash.Hasher; -import com.google.common.hash.Hashing; -import java.util.Iterator; -import java.util.Objects; -import java.util.SortedSet; -import java.util.TreeSet; -import org.springframework.web.util.UriComponents; -import org.springframework.web.util.UriComponentsBuilder; - -public class MongoExternalReference implements Comparable { - private final String url; - private final String hash; - private final SortedSet duo; - - private MongoExternalReference(final String url, final String hash, final SortedSet duo) { - this.url = url; - this.hash = hash; - this.duo = duo; - } - - public String getUrl() { - return url; - } - - @JsonIgnore - public String getHash() { - return hash; - } - - public SortedSet getDuo() { - return duo; - } - - @Override - public boolean equals(final Object o) { - if (o == this) { - return true; - } - if (!(o instanceof MongoExternalReference)) { - return false; - } - final MongoExternalReference other = (MongoExternalReference) o; - return Objects.equals(url, other.url) && Objects.equals(duo, other.duo); - } - - @Override - public int hashCode() { - return Objects.hash(hash); - } - - @Override - public int compareTo(final MongoExternalReference other) { - if (other == null) { - return 1; - } - - if (!url.equals(other.url)) { - return url.compareTo(other.url); - } - - if (duo == other.duo) { - return 0; - } else if (other.duo == null) { - return 1; - } else if (duo == null) { - return -1; - } - - if (!duo.equals(other.duo)) { - if (duo.size() < other.duo.size()) { - return -1; - } else if (duo.size() > other.duo.size()) { - return 1; - } else { - final Iterator thisIt = duo.iterator(); - final Iterator otherIt = other.duo.iterator(); - while (thisIt.hasNext() && otherIt.hasNext()) { - final int val = thisIt.next().compareTo(otherIt.next()); - if (val != 0) { - return val; - } - } - } - } - return 0; - } - - @Override - public String toString() { - final String sb = "ExternalReference(" + url + "," + duo + ")"; - return sb; - } - - public static MongoExternalReference build(String url, final SortedSet duo) { - final UriComponentsBuilder uriComponentsBuilder = UriComponentsBuilder.fromUriString(url); - final UriComponents uriComponents = uriComponentsBuilder.build().normalize(); - - url = uriComponents.toUriString(); - - uriComponents.getPort(); - final Hasher hasher = - Hashing.sha256() - .newHasher() - .putUnencodedChars( - Objects.nonNull(uriComponents.getScheme()) ? uriComponents.getScheme() : "") - .putUnencodedChars( - Objects.nonNull(uriComponents.getSchemeSpecificPart()) - ? uriComponents.getSchemeSpecificPart() - : "") - .putUnencodedChars( - Objects.nonNull(uriComponents.getUserInfo()) ? uriComponents.getUserInfo() : "") - .putUnencodedChars( - Objects.nonNull(uriComponents.getHost()) ? uriComponents.getHost() : "") - .putInt(uriComponents.getPort()) - .putUnencodedChars( - Objects.nonNull(uriComponents.getPath()) ? uriComponents.getPath() : "") - .putUnencodedChars( - Objects.nonNull(uriComponents.getQuery()) ? uriComponents.getQuery() : "") - .putUnencodedChars( - Objects.nonNull(uriComponents.getFragment()) ? uriComponents.getFragment() : ""); - - if (duo != null) { - for (final String s : duo) { - hasher.putUnencodedChars(s); - } - } - - return new MongoExternalReference(url, hasher.hash().toString(), duo); - } - - @JsonCreator - public static MongoExternalReference build(@JsonProperty("url") final String url) { - return build(url, new TreeSet<>()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import java.util.Iterator; +import java.util.Objects; +import java.util.SortedSet; +import java.util.TreeSet; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; + +public class MongoExternalReference implements Comparable { + private final String url; + private final String hash; + private final SortedSet duo; + + private MongoExternalReference(final String url, final String hash, final SortedSet duo) { + this.url = url; + this.hash = hash; + this.duo = duo; + } + + public String getUrl() { + return url; + } + + @JsonIgnore + public String getHash() { + return hash; + } + + public SortedSet getDuo() { + return duo; + } + + @Override + public boolean equals(final Object o) { + if (o == this) { + return true; + } + if (!(o instanceof MongoExternalReference)) { + return false; + } + final MongoExternalReference other = (MongoExternalReference) o; + return Objects.equals(url, other.url) && Objects.equals(duo, other.duo); + } + + @Override + public int hashCode() { + return Objects.hash(hash); + } + + @Override + public int compareTo(final MongoExternalReference other) { + if (other == null) { + return 1; + } + + if (!url.equals(other.url)) { + return url.compareTo(other.url); + } + + if (duo == other.duo) { + return 0; + } else if (other.duo == null) { + return 1; + } else if (duo == null) { + return -1; + } + + if (!duo.equals(other.duo)) { + if (duo.size() < other.duo.size()) { + return -1; + } else if (duo.size() > other.duo.size()) { + return 1; + } else { + final Iterator thisIt = duo.iterator(); + final Iterator otherIt = other.duo.iterator(); + while (thisIt.hasNext() && otherIt.hasNext()) { + final int val = thisIt.next().compareTo(otherIt.next()); + if (val != 0) { + return val; + } + } + } + } + return 0; + } + + @Override + public String toString() { + final String sb = "ExternalReference(" + url + "," + duo + ")"; + return sb; + } + + public static MongoExternalReference build(String url, final SortedSet duo) { + final UriComponentsBuilder uriComponentsBuilder = UriComponentsBuilder.fromUriString(url); + final UriComponents uriComponents = uriComponentsBuilder.build().normalize(); + + url = uriComponents.toUriString(); + + uriComponents.getPort(); + final Hasher hasher = + Hashing.sha256() + .newHasher() + .putUnencodedChars( + Objects.nonNull(uriComponents.getScheme()) ? uriComponents.getScheme() : "") + .putUnencodedChars( + Objects.nonNull(uriComponents.getSchemeSpecificPart()) + ? uriComponents.getSchemeSpecificPart() + : "") + .putUnencodedChars( + Objects.nonNull(uriComponents.getUserInfo()) ? uriComponents.getUserInfo() : "") + .putUnencodedChars( + Objects.nonNull(uriComponents.getHost()) ? uriComponents.getHost() : "") + .putInt(uriComponents.getPort()) + .putUnencodedChars( + Objects.nonNull(uriComponents.getPath()) ? uriComponents.getPath() : "") + .putUnencodedChars( + Objects.nonNull(uriComponents.getQuery()) ? uriComponents.getQuery() : "") + .putUnencodedChars( + Objects.nonNull(uriComponents.getFragment()) ? uriComponents.getFragment() : ""); + + if (duo != null) { + for (final String s : duo) { + hasher.putUnencodedChars(s); + } + } + + return new MongoExternalReference(url, hasher.hash().toString(), duo); + } + + @JsonCreator + public static MongoExternalReference build(@JsonProperty("url") final String url) { + return build(url, new TreeSet<>()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoFileUpload.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoFileUpload.java similarity index 97% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoFileUpload.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoFileUpload.java index 497c0cef8..f61fa66af 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoFileUpload.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoFileUpload.java @@ -1,90 +1,90 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.model; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import java.util.List; -import java.util.Objects; -import lombok.Getter; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.Document; -import uk.ac.ebi.biosamples.mongo.util.BioSamplesFileUploadSubmissionStatus; -import uk.ac.ebi.biosamples.mongo.util.SampleNameAccessionPair; - -@Document(collection = "mongoFileUpload") -@Getter -public class MongoFileUpload { - @Id @JsonIgnore @Indexed private final String submissionId; - private final BioSamplesFileUploadSubmissionStatus submissionStatus; - private final String submissionDate; - private final String lastUpdateDate; - private final String submitterDetails; - private final String checklist; - private final List sampleNameAccessionPairs; - @JsonIgnore private final boolean isWebin; - - @JsonInclude(JsonInclude.Include.NON_NULL) - private final String validationMessage; - - public MongoFileUpload( - final String submissionId, - final BioSamplesFileUploadSubmissionStatus submissionStatus, - final String submissionDate, - final String lastUpdateDate, - final String submitterDetails, - final String checklist, - final boolean isWebin, - final List sampleNameAccessionPairs, - final String validationMessage) { - this.submissionId = submissionId; - this.submissionStatus = submissionStatus; - this.submissionDate = submissionDate; - this.lastUpdateDate = lastUpdateDate; - this.submitterDetails = submitterDetails; - this.checklist = checklist; - this.isWebin = isWebin; - this.sampleNameAccessionPairs = sampleNameAccessionPairs; - this.validationMessage = validationMessage; - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (!(o instanceof MongoFileUpload)) { - return false; - } - final MongoFileUpload that = (MongoFileUpload) o; - - return isWebin() == that.isWebin() - && Objects.equals(getSubmissionId(), that.getSubmissionId()) - && getSubmissionStatus() == that.getSubmissionStatus() - && Objects.equals(getSubmitterDetails(), that.getSubmitterDetails()) - && Objects.equals(getChecklist(), that.getChecklist()) - && Objects.equals(getSampleNameAccessionPairs(), that.getSampleNameAccessionPairs()) - && Objects.equals(getValidationMessage(), that.getValidationMessage()); - } - - @Override - public int hashCode() { - return Objects.hash( - getSubmissionId(), - getSubmissionStatus(), - getSubmitterDetails(), - getChecklist(), - isWebin(), - getSampleNameAccessionPairs(), - getValidationMessage()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import java.util.List; +import java.util.Objects; +import lombok.Getter; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; +import uk.ac.ebi.biosamples.mongo.util.BioSamplesFileUploadSubmissionStatus; +import uk.ac.ebi.biosamples.mongo.util.SampleNameAccessionPair; + +@Document(collection = "mongoFileUpload") +@Getter +public class MongoFileUpload { + @Id @JsonIgnore @Indexed private final String submissionId; + private final BioSamplesFileUploadSubmissionStatus submissionStatus; + private final String submissionDate; + private final String lastUpdateDate; + private final String submitterDetails; + private final String checklist; + private final List sampleNameAccessionPairs; + @JsonIgnore private final boolean isWebin; + + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String validationMessage; + + public MongoFileUpload( + final String submissionId, + final BioSamplesFileUploadSubmissionStatus submissionStatus, + final String submissionDate, + final String lastUpdateDate, + final String submitterDetails, + final String checklist, + final boolean isWebin, + final List sampleNameAccessionPairs, + final String validationMessage) { + this.submissionId = submissionId; + this.submissionStatus = submissionStatus; + this.submissionDate = submissionDate; + this.lastUpdateDate = lastUpdateDate; + this.submitterDetails = submitterDetails; + this.checklist = checklist; + this.isWebin = isWebin; + this.sampleNameAccessionPairs = sampleNameAccessionPairs; + this.validationMessage = validationMessage; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MongoFileUpload)) { + return false; + } + final MongoFileUpload that = (MongoFileUpload) o; + + return isWebin() == that.isWebin() + && Objects.equals(getSubmissionId(), that.getSubmissionId()) + && getSubmissionStatus() == that.getSubmissionStatus() + && Objects.equals(getSubmitterDetails(), that.getSubmitterDetails()) + && Objects.equals(getChecklist(), that.getChecklist()) + && Objects.equals(getSampleNameAccessionPairs(), that.getSampleNameAccessionPairs()) + && Objects.equals(getValidationMessage(), that.getValidationMessage()); + } + + @Override + public int hashCode() { + return Objects.hash( + getSubmissionId(), + getSubmissionStatus(), + getSubmitterDetails(), + getChecklist(), + isWebin(), + getSampleNameAccessionPairs(), + getValidationMessage()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoPipeline.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoPipeline.java similarity index 97% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoPipeline.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoPipeline.java index 6a24d6715..ae4e16686 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoPipeline.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoPipeline.java @@ -1,80 +1,80 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.model; - -import java.util.Date; -import java.util.Objects; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.Document; -import uk.ac.ebi.biosamples.mongo.util.PipelineCompletionStatus; - -@Document(collection = "pipelineRunStatus") -public class MongoPipeline { - @Id @Indexed private final String pipelineUniqueIdentifier; - private final Date pipelineRunDate; - private final String pipelineName; - private final PipelineCompletionStatus pipelineCompletionStatus; - private final String failedSamples; - private final String exceptionCause; - - public MongoPipeline( - final String pipelineUniqueIdentifier, - final Date pipelineRunDate, - final String pipelineName, - final PipelineCompletionStatus pipelineCompletionStatus, - final String failedSamples, - final String exceptionCause) { - this.pipelineUniqueIdentifier = pipelineUniqueIdentifier; - this.pipelineRunDate = pipelineRunDate; - this.pipelineName = pipelineName; - this.pipelineCompletionStatus = pipelineCompletionStatus; - this.failedSamples = failedSamples; - this.exceptionCause = exceptionCause; - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (!(o instanceof MongoPipeline)) { - return false; - } - final MongoPipeline that = (MongoPipeline) o; - return Objects.equals(pipelineUniqueIdentifier, that.pipelineUniqueIdentifier) - && Objects.equals(pipelineRunDate, that.pipelineRunDate) - && Objects.equals(pipelineName, that.pipelineName) - && pipelineCompletionStatus == that.pipelineCompletionStatus; - } - - @Override - public int hashCode() { - return Objects.hash( - pipelineUniqueIdentifier, pipelineRunDate, pipelineName, pipelineCompletionStatus); - } - - @Override - public String toString() { - return "MongoPipeline{" - + "pipelineUniqueIdentifier='" - + pipelineUniqueIdentifier - + '\'' - + ", pipelineRunDate=" - + pipelineRunDate - + ", pipelineName='" - + pipelineName - + '\'' - + ", pipelineCompletionStatus=" - + pipelineCompletionStatus - + '}'; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.model; + +import java.util.Date; +import java.util.Objects; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; +import uk.ac.ebi.biosamples.mongo.util.PipelineCompletionStatus; + +@Document(collection = "pipelineRunStatus") +public class MongoPipeline { + @Id @Indexed private final String pipelineUniqueIdentifier; + private final Date pipelineRunDate; + private final String pipelineName; + private final PipelineCompletionStatus pipelineCompletionStatus; + private final String failedSamples; + private final String exceptionCause; + + public MongoPipeline( + final String pipelineUniqueIdentifier, + final Date pipelineRunDate, + final String pipelineName, + final PipelineCompletionStatus pipelineCompletionStatus, + final String failedSamples, + final String exceptionCause) { + this.pipelineUniqueIdentifier = pipelineUniqueIdentifier; + this.pipelineRunDate = pipelineRunDate; + this.pipelineName = pipelineName; + this.pipelineCompletionStatus = pipelineCompletionStatus; + this.failedSamples = failedSamples; + this.exceptionCause = exceptionCause; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MongoPipeline)) { + return false; + } + final MongoPipeline that = (MongoPipeline) o; + return Objects.equals(pipelineUniqueIdentifier, that.pipelineUniqueIdentifier) + && Objects.equals(pipelineRunDate, that.pipelineRunDate) + && Objects.equals(pipelineName, that.pipelineName) + && pipelineCompletionStatus == that.pipelineCompletionStatus; + } + + @Override + public int hashCode() { + return Objects.hash( + pipelineUniqueIdentifier, pipelineRunDate, pipelineName, pipelineCompletionStatus); + } + + @Override + public String toString() { + return "MongoPipeline{" + + "pipelineUniqueIdentifier='" + + pipelineUniqueIdentifier + + '\'' + + ", pipelineRunDate=" + + pipelineRunDate + + ", pipelineName='" + + pipelineName + + '\'' + + ", pipelineCompletionStatus=" + + pipelineCompletionStatus + + '}'; + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoRelationship.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoRelationship.java similarity index 96% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoRelationship.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoRelationship.java index 51c62b222..e6dfa021e 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoRelationship.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoRelationship.java @@ -1,130 +1,130 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.hash.Hasher; -import com.google.common.hash.Hashing; -import java.util.Objects; -import org.springframework.data.mongodb.core.index.Indexed; - -public class MongoRelationship implements Comparable { - - private final String hash; - - private final String type; - - @Indexed(background = true, sparse = true) - private final String target; - - private final String source; - - private MongoRelationship( - final String type, final String target, final String source, final String hash) { - this.type = type; - this.target = target; - this.source = source; - this.hash = hash; - } - - public String getType() { - return type; - } - - public String getTarget() { - return target; - } - - public String getSource() { - return source; - } - - public String getHash() { - return hash; - } - - @Override - public boolean equals(final Object o) { - - if (o == this) { - return true; - } - if (!(o instanceof MongoRelationship)) { - return false; - } - final MongoRelationship other = (MongoRelationship) o; - return Objects.equals(type, other.type) - && Objects.equals(target, other.target) - && Objects.equals(source, other.source); - } - - @Override - public int hashCode() { - return Objects.hash(type, target, source); - } - - @Override - public int compareTo(final MongoRelationship other) { - if (other == null) { - return 1; - } - - if (!Objects.equals(type, other.type)) { - return type.compareTo(other.type); - } - - if (!Objects.equals(target, other.target)) { - return target.compareTo(other.target); - } - - if (!Objects.equals(source, other.source)) { - if (source == null) { - return 1; - } else if (other.source == null) { - return -1; - } else { - return source.compareTo(other.source); - } - } - return 0; - } - - @Override - public String toString() { - final String sb = "Relationships(" + source + "," + type + "," + target + ")"; - return sb; - } - - @JsonCreator - public static MongoRelationship build( - @JsonProperty("source") final String source, - @JsonProperty("type") final String type, - @JsonProperty("target") final String target) { - if (type == null || type.trim().length() == 0) { - throw new IllegalArgumentException("type cannot be empty"); - } - if (target == null || target.trim().length() == 0) { - throw new IllegalArgumentException("target cannot be empty"); - } - - final Hasher hasher = Hashing.sha256().newHasher(); - hasher.putUnencodedChars(type); - hasher.putUnencodedChars(target); - if (source != null) { - hasher.putUnencodedChars(source); - } - - final String hash = hasher.hash().toString(); - - return new MongoRelationship(type, target, source, hash); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import java.util.Objects; +import org.springframework.data.mongodb.core.index.Indexed; + +public class MongoRelationship implements Comparable { + + private final String hash; + + private final String type; + + @Indexed(background = true, sparse = true) + private final String target; + + private final String source; + + private MongoRelationship( + final String type, final String target, final String source, final String hash) { + this.type = type; + this.target = target; + this.source = source; + this.hash = hash; + } + + public String getType() { + return type; + } + + public String getTarget() { + return target; + } + + public String getSource() { + return source; + } + + public String getHash() { + return hash; + } + + @Override + public boolean equals(final Object o) { + + if (o == this) { + return true; + } + if (!(o instanceof MongoRelationship)) { + return false; + } + final MongoRelationship other = (MongoRelationship) o; + return Objects.equals(type, other.type) + && Objects.equals(target, other.target) + && Objects.equals(source, other.source); + } + + @Override + public int hashCode() { + return Objects.hash(type, target, source); + } + + @Override + public int compareTo(final MongoRelationship other) { + if (other == null) { + return 1; + } + + if (!Objects.equals(type, other.type)) { + return type.compareTo(other.type); + } + + if (!Objects.equals(target, other.target)) { + return target.compareTo(other.target); + } + + if (!Objects.equals(source, other.source)) { + if (source == null) { + return 1; + } else if (other.source == null) { + return -1; + } else { + return source.compareTo(other.source); + } + } + return 0; + } + + @Override + public String toString() { + final String sb = "Relationships(" + source + "," + type + "," + target + ")"; + return sb; + } + + @JsonCreator + public static MongoRelationship build( + @JsonProperty("source") final String source, + @JsonProperty("type") final String type, + @JsonProperty("target") final String target) { + if (type == null || type.trim().length() == 0) { + throw new IllegalArgumentException("type cannot be empty"); + } + if (target == null || target.trim().length() == 0) { + throw new IllegalArgumentException("target cannot be empty"); + } + + final Hasher hasher = Hashing.sha256().newHasher(); + hasher.putUnencodedChars(type); + hasher.putUnencodedChars(target); + if (source != null) { + hasher.putUnencodedChars(source); + } + + final String hash = hasher.hash().toString(); + + return new MongoRelationship(type, target, source, hash); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSample.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSample.java similarity index 94% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSample.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSample.java index 10b572cff..7719f33e9 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSample.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSample.java @@ -1,225 +1,225 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import java.time.Instant; -import java.util.HashSet; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import lombok.Data; -import org.springframework.data.annotation.Id; -import org.springframework.data.annotation.LastModifiedDate; -import org.springframework.data.annotation.Transient; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.Document; -import org.springframework.data.mongodb.core.mapping.Field; -import org.springframework.data.mongodb.core.mapping.FieldType; -import uk.ac.ebi.biosamples.model.*; -import uk.ac.ebi.biosamples.model.structured.AbstractData; -import uk.ac.ebi.biosamples.service.CustomInstantDeserializer; -import uk.ac.ebi.biosamples.service.CustomInstantSerializer; - -@Data -@JsonInclude(JsonInclude.Include.NON_NULL) -@Document -/*@CompoundIndexes({ - @CompoundIndex( - name = "sra_accession_index", - def = "{'attributes.value': 1}", - partialFilter = "{'attributes.type': 'SRA accession'}", - background = true) -})*/ -public class MongoSample { - @Transient public static final String SEQUENCE_NAME = "accession_sequence"; - @Transient public static final String SRA_SEQUENCE_NAME = "sra_accession_sequence"; - - @Id protected String accession; - - @Field(targetType = FieldType.STRING, name = "sraAccession") - @Indexed(background = true) - private String sraAccession; - - @Indexed(background = true) - private String accessionPrefix; - - @Indexed(background = true) - private Integer accessionNumber; - - protected String name; - - /** This is the unique permanent ID of the AAP domain/team that owns this sample. */ - @Indexed(background = true) - protected String domain; - - @Indexed(background = true) - protected String webinSubmissionAccountId; - - @Indexed(background = true) - protected Long taxId; - - protected SampleStatus status; - - @JsonSerialize(using = CustomInstantSerializer.class) - @JsonDeserialize(using = CustomInstantDeserializer.class) - @Indexed(background = true) - protected Instant release; - - @JsonSerialize(using = CustomInstantSerializer.class) - @JsonDeserialize(using = CustomInstantDeserializer.class) - @LastModifiedDate - @Indexed(background = true) - protected Instant update; - - @JsonSerialize(using = CustomInstantSerializer.class) - @JsonDeserialize(using = CustomInstantDeserializer.class) - protected Instant create; - - @JsonSerialize(using = CustomInstantSerializer.class) - @JsonDeserialize(using = CustomInstantDeserializer.class) - protected Instant submitted; - - @JsonSerialize(using = CustomInstantSerializer.class) - @JsonDeserialize(using = CustomInstantDeserializer.class) - protected Instant reviewed; - - protected SortedSet attributes; - protected SortedSet relationships; - protected SortedSet externalReferences; - private SortedSet organizations; - protected SortedSet contacts; - protected SortedSet publications; - protected SortedSet certificates; - - @Transient protected Set data; - private SubmittedViaType submittedVia; - - @JsonIgnore - public boolean hasAccession() { - if (accession != null && !accession.trim().isEmpty()) { - return true; - } else { - return false; - } - } - - @JsonCreator - public static MongoSample build( - @JsonProperty("name") final String name, - @JsonProperty("accession") final String accession, - @JsonProperty("sraAccession") final String sraAccession, - @JsonProperty("domain") final String domain, - @JsonProperty("webinSubmissionAccountId") final String webinSubmissionAccountId, - @JsonProperty("taxId") final Long taxId, - @JsonProperty("status") final SampleStatus status, - @JsonProperty("release") final Instant release, - @JsonProperty("update") final Instant update, - @JsonProperty("create") final Instant create, - @JsonProperty("submitted") final Instant submitted, - @JsonProperty("reviewed") final Instant reviewed, - @JsonProperty("attributes") final Set attributes, - @JsonProperty("data") final Set structuredData, - @JsonProperty("relationships") final Set relationships, - @JsonProperty("externalReferences") - final SortedSet externalReferences, - @JsonProperty("organizations") final SortedSet organizations, - @JsonProperty("contacts") final SortedSet contacts, - @JsonProperty("publications") final SortedSet publications, - @JsonProperty("certificates") final SortedSet certificates, - @JsonProperty("submittedVia") final SubmittedViaType submittedVia) { - - final MongoSample mongoSample = new MongoSample(); - - mongoSample.accession = accession; - mongoSample.sraAccession = sraAccession; - mongoSample.name = name; - mongoSample.domain = domain; - mongoSample.webinSubmissionAccountId = webinSubmissionAccountId; - mongoSample.taxId = taxId; - mongoSample.status = status; - mongoSample.release = release; - mongoSample.update = update; - mongoSample.create = create; - mongoSample.submitted = submitted; - mongoSample.reviewed = reviewed; - mongoSample.attributes = new TreeSet<>(); - - if (attributes != null && !attributes.isEmpty()) { - mongoSample.attributes.addAll(attributes); - } - - mongoSample.data = new HashSet<>(); - - if (structuredData != null && !structuredData.isEmpty()) { - mongoSample.data.addAll(structuredData); - } - - mongoSample.relationships = new TreeSet<>(); - - if (relationships != null && !relationships.isEmpty()) { - mongoSample.relationships.addAll(relationships); - } - - mongoSample.externalReferences = new TreeSet<>(); - - if (externalReferences != null && !externalReferences.isEmpty()) { - mongoSample.externalReferences.addAll(externalReferences); - } - - mongoSample.organizations = new TreeSet<>(); - - if (organizations != null && !organizations.isEmpty()) { - mongoSample.organizations.addAll(organizations); - } - - mongoSample.contacts = new TreeSet<>(); - - if (contacts != null && !contacts.isEmpty()) { - mongoSample.contacts.addAll(contacts); - } - - mongoSample.publications = new TreeSet<>(); - - if (publications != null && !publications.isEmpty()) { - mongoSample.publications.addAll(publications); - } - - mongoSample.certificates = new TreeSet<>(); - - if (certificates != null && !certificates.isEmpty()) { - mongoSample.certificates.addAll(certificates); - } - - mongoSample.submittedVia = submittedVia; - - // split accession into prefix & number, if possible - final Pattern r = Pattern.compile("^(\\D+)(\\d+)$"); - - if (accession != null) { - final Matcher m = r.matcher(accession); - - if (m.matches() && m.groupCount() == 2) { - mongoSample.accessionPrefix = m.group(1); - mongoSample.accessionNumber = Integer.parseInt(m.group(2)); - } - } - - return mongoSample; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.time.Instant; +import java.util.HashSet; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import lombok.Data; +import org.springframework.data.annotation.Id; +import org.springframework.data.annotation.LastModifiedDate; +import org.springframework.data.annotation.Transient; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.data.mongodb.core.mapping.Field; +import org.springframework.data.mongodb.core.mapping.FieldType; +import uk.ac.ebi.biosamples.core.model.*; +import uk.ac.ebi.biosamples.core.model.structured.AbstractData; +import uk.ac.ebi.biosamples.core.service.CustomInstantDeserializer; +import uk.ac.ebi.biosamples.core.service.CustomInstantSerializer; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +@Document +/*@CompoundIndexes({ + @CompoundIndex( + name = "sra_accession_index", + def = "{'attributes.value': 1}", + partialFilter = "{'attributes.type': 'SRA accession'}", + background = true) +})*/ +public class MongoSample { + @Transient public static final String SEQUENCE_NAME = "accession_sequence"; + @Transient public static final String SRA_SEQUENCE_NAME = "sra_accession_sequence"; + + @Id protected String accession; + + @Field(targetType = FieldType.STRING, name = "sraAccession") + @Indexed(background = true) + private String sraAccession; + + @Indexed(background = true) + private String accessionPrefix; + + @Indexed(background = true) + private Integer accessionNumber; + + protected String name; + + /** This is the unique permanent ID of the AAP domain/team that owns this sample. */ + @Indexed(background = true) + protected String domain; + + @Indexed(background = true) + protected String webinSubmissionAccountId; + + @Indexed(background = true) + protected Long taxId; + + protected SampleStatus status; + + @JsonSerialize(using = CustomInstantSerializer.class) + @JsonDeserialize(using = CustomInstantDeserializer.class) + @Indexed(background = true) + protected Instant release; + + @JsonSerialize(using = CustomInstantSerializer.class) + @JsonDeserialize(using = CustomInstantDeserializer.class) + @LastModifiedDate + @Indexed(background = true) + protected Instant update; + + @JsonSerialize(using = CustomInstantSerializer.class) + @JsonDeserialize(using = CustomInstantDeserializer.class) + protected Instant create; + + @JsonSerialize(using = CustomInstantSerializer.class) + @JsonDeserialize(using = CustomInstantDeserializer.class) + protected Instant submitted; + + @JsonSerialize(using = CustomInstantSerializer.class) + @JsonDeserialize(using = CustomInstantDeserializer.class) + protected Instant reviewed; + + protected SortedSet attributes; + protected SortedSet relationships; + protected SortedSet externalReferences; + private SortedSet organizations; + protected SortedSet contacts; + protected SortedSet publications; + protected SortedSet certificates; + + @Transient protected Set data; + private SubmittedViaType submittedVia; + + @JsonIgnore + public boolean hasAccession() { + if (accession != null && !accession.trim().isEmpty()) { + return true; + } else { + return false; + } + } + + @JsonCreator + public static MongoSample build( + @JsonProperty("name") final String name, + @JsonProperty("accession") final String accession, + @JsonProperty("sraAccession") final String sraAccession, + @JsonProperty("domain") final String domain, + @JsonProperty("webinSubmissionAccountId") final String webinSubmissionAccountId, + @JsonProperty("taxId") final Long taxId, + @JsonProperty("status") final SampleStatus status, + @JsonProperty("release") final Instant release, + @JsonProperty("update") final Instant update, + @JsonProperty("create") final Instant create, + @JsonProperty("submitted") final Instant submitted, + @JsonProperty("reviewed") final Instant reviewed, + @JsonProperty("attributes") final Set attributes, + @JsonProperty("data") final Set structuredData, + @JsonProperty("relationships") final Set relationships, + @JsonProperty("externalReferences") + final SortedSet externalReferences, + @JsonProperty("organizations") final SortedSet organizations, + @JsonProperty("contacts") final SortedSet contacts, + @JsonProperty("publications") final SortedSet publications, + @JsonProperty("certificates") final SortedSet certificates, + @JsonProperty("submittedVia") final SubmittedViaType submittedVia) { + + final MongoSample mongoSample = new MongoSample(); + + mongoSample.accession = accession; + mongoSample.sraAccession = sraAccession; + mongoSample.name = name; + mongoSample.domain = domain; + mongoSample.webinSubmissionAccountId = webinSubmissionAccountId; + mongoSample.taxId = taxId; + mongoSample.status = status; + mongoSample.release = release; + mongoSample.update = update; + mongoSample.create = create; + mongoSample.submitted = submitted; + mongoSample.reviewed = reviewed; + mongoSample.attributes = new TreeSet<>(); + + if (attributes != null && !attributes.isEmpty()) { + mongoSample.attributes.addAll(attributes); + } + + mongoSample.data = new HashSet<>(); + + if (structuredData != null && !structuredData.isEmpty()) { + mongoSample.data.addAll(structuredData); + } + + mongoSample.relationships = new TreeSet<>(); + + if (relationships != null && !relationships.isEmpty()) { + mongoSample.relationships.addAll(relationships); + } + + mongoSample.externalReferences = new TreeSet<>(); + + if (externalReferences != null && !externalReferences.isEmpty()) { + mongoSample.externalReferences.addAll(externalReferences); + } + + mongoSample.organizations = new TreeSet<>(); + + if (organizations != null && !organizations.isEmpty()) { + mongoSample.organizations.addAll(organizations); + } + + mongoSample.contacts = new TreeSet<>(); + + if (contacts != null && !contacts.isEmpty()) { + mongoSample.contacts.addAll(contacts); + } + + mongoSample.publications = new TreeSet<>(); + + if (publications != null && !publications.isEmpty()) { + mongoSample.publications.addAll(publications); + } + + mongoSample.certificates = new TreeSet<>(); + + if (certificates != null && !certificates.isEmpty()) { + mongoSample.certificates.addAll(certificates); + } + + mongoSample.submittedVia = submittedVia; + + // split accession into prefix & number, if possible + final Pattern r = Pattern.compile("^(\\D+)(\\d+)$"); + + if (accession != null) { + final Matcher m = r.matcher(accession); + + if (m.matches() && m.groupCount() == 2) { + mongoSample.accessionPrefix = m.group(1); + mongoSample.accessionNumber = Integer.parseInt(m.group(2)); + } + } + + return mongoSample; + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSampleMessage.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSampleMessage.java similarity index 96% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSampleMessage.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSampleMessage.java index 32b25fad6..422e0696b 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSampleMessage.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSampleMessage.java @@ -1,75 +1,75 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.model; - -import java.time.Instant; -import java.util.Objects; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.Document; - -@Document(collection = "mongoSampleMessage") -public class MongoSampleMessage { - @Id @Indexed private final String biosampleAccession; - private final Instant updateDate; - private final Long taxId; - - public MongoSampleMessage( - final String biosampleAccession, final Instant updateDate, final Long taxId) { - this.biosampleAccession = biosampleAccession; - this.updateDate = updateDate; - this.taxId = taxId; - } - - private String getBiosampleAccession() { - return biosampleAccession; - } - - public Long getTaxId() { - return taxId; - } - - public Instant getUpdateDate() { - return updateDate; - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (!(o instanceof MongoSampleMessage)) { - return false; - } - final MongoSampleMessage that = (MongoSampleMessage) o; - return Objects.equals(getBiosampleAccession(), that.getBiosampleAccession()) - && Objects.equals(getUpdateDate(), that.getUpdateDate()) - && Objects.equals(getTaxId(), that.getTaxId()); - } - - @Override - public int hashCode() { - return Objects.hash(getBiosampleAccession(), getUpdateDate(), getTaxId()); - } - - @Override - public String toString() { - return "MongoSampleMessage{" - + "biosampleAccession='" - + biosampleAccession - + '\'' - + ", updateDate=" - + updateDate - + ", taxId=" - + taxId - + '}'; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.model; + +import java.time.Instant; +import java.util.Objects; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; + +@Document(collection = "mongoSampleMessage") +public class MongoSampleMessage { + @Id @Indexed private final String biosampleAccession; + private final Instant updateDate; + private final Long taxId; + + public MongoSampleMessage( + final String biosampleAccession, final Instant updateDate, final Long taxId) { + this.biosampleAccession = biosampleAccession; + this.updateDate = updateDate; + this.taxId = taxId; + } + + private String getBiosampleAccession() { + return biosampleAccession; + } + + public Long getTaxId() { + return taxId; + } + + public Instant getUpdateDate() { + return updateDate; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MongoSampleMessage)) { + return false; + } + final MongoSampleMessage that = (MongoSampleMessage) o; + return Objects.equals(getBiosampleAccession(), that.getBiosampleAccession()) + && Objects.equals(getUpdateDate(), that.getUpdateDate()) + && Objects.equals(getTaxId(), that.getTaxId()); + } + + @Override + public int hashCode() { + return Objects.hash(getBiosampleAccession(), getUpdateDate(), getTaxId()); + } + + @Override + public String toString() { + return "MongoSampleMessage{" + + "biosampleAccession='" + + biosampleAccession + + '\'' + + ", updateDate=" + + updateDate + + ", taxId=" + + taxId + + '}'; + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSequence.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSequence.java similarity index 96% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSequence.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSequence.java index ebf970b52..72fc1b518 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSequence.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoSequence.java @@ -1,36 +1,36 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.model; - -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; - -@Document(collection = "mongoSequence") -public class MongoSequence { - @Id private String id; - private long seq; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public long getSeq() { - return seq; - } - - public void setSeq(long seq) { - this.seq = seq; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.model; + +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +@Document(collection = "mongoSequence") +public class MongoSequence { + @Id private String id; + private long seq; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public long getSeq() { + return seq; + } + + public void setSeq(long seq) { + this.seq = seq; + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoStructuredData.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoStructuredData.java similarity index 89% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoStructuredData.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoStructuredData.java index f0e41f7e1..9ff618a4f 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoStructuredData.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/model/MongoStructuredData.java @@ -1,81 +1,81 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.model; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import java.time.Instant; -import java.util.Set; -import org.springframework.data.annotation.Id; -import org.springframework.data.annotation.LastModifiedDate; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.Document; -import uk.ac.ebi.biosamples.model.structured.StructuredData; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; -import uk.ac.ebi.biosamples.service.CustomInstantDeserializer; -import uk.ac.ebi.biosamples.service.CustomInstantSerializer; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@Document -public class MongoStructuredData { - @Id protected String accession; - - @JsonSerialize(using = CustomInstantSerializer.class) - @JsonDeserialize(using = CustomInstantDeserializer.class) - @LastModifiedDate - @Indexed(background = true) - protected Instant update; - - @JsonSerialize(using = CustomInstantSerializer.class) - @JsonDeserialize(using = CustomInstantDeserializer.class) - protected Instant create; - - protected Set data; - - public String getAccession() { - return accession; - } - - public Instant getUpdate() { - return update; - } - - public Instant getCreate() { - return create; - } - - public Set getData() { - return data; - } - - public static MongoStructuredData build( - final String accession, - final Instant update, - final Instant create, - final Set data) { - final MongoStructuredData mongoStructuredData = new MongoStructuredData(); - mongoStructuredData.accession = accession; - mongoStructuredData.update = update; - mongoStructuredData.create = create; - mongoStructuredData.data = data; - return mongoStructuredData; - } - - public static MongoStructuredData build(final StructuredData structuredData) { - final MongoStructuredData mongoStructuredData = new MongoStructuredData(); - mongoStructuredData.accession = structuredData.getAccession(); - mongoStructuredData.update = structuredData.getUpdate(); - mongoStructuredData.create = structuredData.getCreate(); - mongoStructuredData.data = structuredData.getData(); - return mongoStructuredData; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.time.Instant; +import java.util.Set; +import org.springframework.data.annotation.Id; +import org.springframework.data.annotation.LastModifiedDate; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; +import uk.ac.ebi.biosamples.core.service.CustomInstantDeserializer; +import uk.ac.ebi.biosamples.core.service.CustomInstantSerializer; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@Document +public class MongoStructuredData { + @Id protected String accession; + + @JsonSerialize(using = CustomInstantSerializer.class) + @JsonDeserialize(using = CustomInstantDeserializer.class) + @LastModifiedDate + @Indexed(background = true) + protected Instant update; + + @JsonSerialize(using = CustomInstantSerializer.class) + @JsonDeserialize(using = CustomInstantDeserializer.class) + protected Instant create; + + protected Set data; + + public String getAccession() { + return accession; + } + + public Instant getUpdate() { + return update; + } + + public Instant getCreate() { + return create; + } + + public Set getData() { + return data; + } + + public static MongoStructuredData build( + final String accession, + final Instant update, + final Instant create, + final Set data) { + final MongoStructuredData mongoStructuredData = new MongoStructuredData(); + mongoStructuredData.accession = accession; + mongoStructuredData.update = update; + mongoStructuredData.create = create; + mongoStructuredData.data = data; + return mongoStructuredData; + } + + public static MongoStructuredData build(final StructuredData structuredData) { + final MongoStructuredData mongoStructuredData = new MongoStructuredData(); + mongoStructuredData.accession = structuredData.getAccession(); + mongoStructuredData.update = structuredData.getUpdate(); + mongoStructuredData.create = structuredData.getCreate(); + mongoStructuredData.data = structuredData.getData(); + return mongoStructuredData; + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoAnalyticsRepository.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoAnalyticsRepository.java similarity index 100% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoAnalyticsRepository.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoAnalyticsRepository.java diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoAuthChangeRepository.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoAuthChangeRepository.java similarity index 100% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoAuthChangeRepository.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoAuthChangeRepository.java diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationLinkRepository.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationLinkRepository.java similarity index 100% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationLinkRepository.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationLinkRepository.java diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationRepository.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationRepository.java similarity index 100% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationRepository.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationRepository.java diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationRuleRepository.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationRuleRepository.java similarity index 100% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationRuleRepository.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoCurationRuleRepository.java diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoFileUploadRepository.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoFileUploadRepository.java similarity index 100% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoFileUploadRepository.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoFileUploadRepository.java diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoPipelineRepository.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoPipelineRepository.java similarity index 100% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoPipelineRepository.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoPipelineRepository.java diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleMessageRepository.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleMessageRepository.java similarity index 100% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleMessageRepository.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleMessageRepository.java diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepository.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepository.java similarity index 100% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepository.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepository.java diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepositoryCustom.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepositoryCustom.java similarity index 100% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepositoryCustom.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepositoryCustom.java diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepositoryImpl.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepositoryImpl.java similarity index 100% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepositoryImpl.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoSampleRepositoryImpl.java diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoStructuredDataRepository.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoStructuredDataRepository.java similarity index 100% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoStructuredDataRepository.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/repository/MongoStructuredDataRepository.java diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/AnalyticsService.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/AnalyticsService.java similarity index 95% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/AnalyticsService.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/AnalyticsService.java index a403079d1..0ff0b5e5e 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/AnalyticsService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/AnalyticsService.java @@ -1,130 +1,130 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import java.time.*; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.PipelineAnalytics; -import uk.ac.ebi.biosamples.model.SampleAnalytics; -import uk.ac.ebi.biosamples.mongo.model.MongoAnalytics; -import uk.ac.ebi.biosamples.mongo.repository.MongoAnalyticsRepository; - -@Service -public class AnalyticsService { - private static final Logger LOG = LoggerFactory.getLogger(AnalyticsService.class); - private final MongoAnalyticsRepository analyticsRepository; - private final DateTimeFormatter dateTimeFormatter; - - public AnalyticsService(final MongoAnalyticsRepository analyticsRepository) { - this.analyticsRepository = analyticsRepository; - dateTimeFormatter = DateTimeFormatter.ofPattern("yyyyMMdd"); - } - - public MongoAnalytics getAnalytics(final LocalDate date) { - final String dateString = dateTimeFormatter.format(date); - final Optional byId = analyticsRepository.findById(dateString); - - return byId.isPresent() ? byId.get() : null; - } - - public List getAnalytics(final LocalDate start, final LocalDate end) { - final String startString = dateTimeFormatter.format(start); - final String endString = dateTimeFormatter.format(end); - - return analyticsRepository.findMongoAnalyticsByIdBetween(startString, endString); - } - - public MongoAnalytics getLatestAnalytics() { - return analyticsRepository.findFirstByOrderByCollectionDateDesc(); - } - - public void mergeSampleAnalytics(final Instant runTime, final SampleAnalytics sampleAnalytics) { - final String pipelineRunDate = getApproximateRunDateAsString(runTime); - LOG.info("Saving sample types for date: {}", pipelineRunDate); - - final LocalDate localDate = runTime.atZone(ZoneId.systemDefault()).toLocalDate(); - // LocalDate localDate = LocalDate.ofInstant(runTime, ZoneId.systemDefault()); - final Optional byId = analyticsRepository.findById(pipelineRunDate); - MongoAnalytics analyticsRecord = byId.isPresent() ? byId.get() : null; - - if (analyticsRecord == null) { - analyticsRecord = - new MongoAnalytics( - pipelineRunDate, - localDate.getYear(), - localDate.getMonthValue(), - localDate.getDayOfMonth()); - } else { - final SampleAnalytics oldSampleAnalytics = analyticsRecord.getSampleAnalytics(); - if (oldSampleAnalytics != null) { - sampleAnalytics.getCenter().putAll(oldSampleAnalytics.getCenter()); - sampleAnalytics.getChannel().putAll(oldSampleAnalytics.getChannel()); - sampleAnalytics.setProcessedRecords(oldSampleAnalytics.getProcessedRecords()); - sampleAnalytics.setDateRange(oldSampleAnalytics.getDateRange()); - } - } - - analyticsRecord.setSampleAnalytics(sampleAnalytics); - analyticsRepository.save(analyticsRecord); - } - - public void persistSampleAnalytics(final Instant runTime, final SampleAnalytics sampleAnalytics) { - final String pipelineRunDate = getApproximateRunDateAsString(runTime); - LOG.info("Saving sample types for date: {}", pipelineRunDate); - - final LocalDate localDate = runTime.atZone(ZoneId.systemDefault()).toLocalDate(); - final Optional byId = analyticsRepository.findById(pipelineRunDate); - MongoAnalytics analyticsRecord = byId.isPresent() ? byId.get() : null; - - if (analyticsRecord == null) { - analyticsRecord = - new MongoAnalytics( - pipelineRunDate, - localDate.getYear(), - localDate.getMonthValue(), - localDate.getDayOfMonth()); - } - analyticsRecord.setSampleAnalytics(sampleAnalytics); - analyticsRepository.save(analyticsRecord); - } - - public void persistPipelineAnalytics(final PipelineAnalytics pipelineAnalytics) { - final String pipelineRunDate = getApproximateRunDateAsString(pipelineAnalytics.getStartTime()); - LOG.info("Saving {} analytics for date: {}", pipelineAnalytics.getName(), pipelineRunDate); - - final LocalDate localDate = - pipelineAnalytics.getStartTime().atZone(ZoneId.systemDefault()).toLocalDate(); - final Optional byId = analyticsRepository.findById(pipelineRunDate); - MongoAnalytics analyticsRecord = byId.isPresent() ? byId.get() : null; - - if (analyticsRecord == null) { - analyticsRecord = - new MongoAnalytics( - pipelineRunDate, - localDate.getYear(), - localDate.getMonthValue(), - localDate.getDayOfMonth()); - } - analyticsRecord.addPipelineAnalytics(pipelineAnalytics); - analyticsRepository.save(analyticsRecord); - } - - private String getApproximateRunDateAsString(final Instant date) { - final LocalDateTime adjustedDate = LocalDateTime.ofInstant(date, ZoneOffset.UTC).plusHours(3); - - return adjustedDate.format(dateTimeFormatter); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.PipelineAnalytics; +import uk.ac.ebi.biosamples.core.model.SampleAnalytics; +import uk.ac.ebi.biosamples.mongo.model.MongoAnalytics; +import uk.ac.ebi.biosamples.mongo.repository.MongoAnalyticsRepository; + +@Service +public class AnalyticsService { + private static final Logger LOG = LoggerFactory.getLogger(AnalyticsService.class); + private final MongoAnalyticsRepository analyticsRepository; + private final DateTimeFormatter dateTimeFormatter; + + public AnalyticsService(final MongoAnalyticsRepository analyticsRepository) { + this.analyticsRepository = analyticsRepository; + dateTimeFormatter = DateTimeFormatter.ofPattern("yyyyMMdd"); + } + + public MongoAnalytics getAnalytics(final LocalDate date) { + final String dateString = dateTimeFormatter.format(date); + final Optional byId = analyticsRepository.findById(dateString); + + return byId.isPresent() ? byId.get() : null; + } + + public List getAnalytics(final LocalDate start, final LocalDate end) { + final String startString = dateTimeFormatter.format(start); + final String endString = dateTimeFormatter.format(end); + + return analyticsRepository.findMongoAnalyticsByIdBetween(startString, endString); + } + + public MongoAnalytics getLatestAnalytics() { + return analyticsRepository.findFirstByOrderByCollectionDateDesc(); + } + + public void mergeSampleAnalytics(final Instant runTime, final SampleAnalytics sampleAnalytics) { + final String pipelineRunDate = getApproximateRunDateAsString(runTime); + LOG.info("Saving sample types for date: {}", pipelineRunDate); + + final LocalDate localDate = runTime.atZone(ZoneId.systemDefault()).toLocalDate(); + // LocalDate localDate = LocalDate.ofInstant(runTime, ZoneId.systemDefault()); + final Optional byId = analyticsRepository.findById(pipelineRunDate); + MongoAnalytics analyticsRecord = byId.isPresent() ? byId.get() : null; + + if (analyticsRecord == null) { + analyticsRecord = + new MongoAnalytics( + pipelineRunDate, + localDate.getYear(), + localDate.getMonthValue(), + localDate.getDayOfMonth()); + } else { + final SampleAnalytics oldSampleAnalytics = analyticsRecord.getSampleAnalytics(); + if (oldSampleAnalytics != null) { + sampleAnalytics.getCenter().putAll(oldSampleAnalytics.getCenter()); + sampleAnalytics.getChannel().putAll(oldSampleAnalytics.getChannel()); + sampleAnalytics.setProcessedRecords(oldSampleAnalytics.getProcessedRecords()); + sampleAnalytics.setDateRange(oldSampleAnalytics.getDateRange()); + } + } + + analyticsRecord.setSampleAnalytics(sampleAnalytics); + analyticsRepository.save(analyticsRecord); + } + + public void persistSampleAnalytics(final Instant runTime, final SampleAnalytics sampleAnalytics) { + final String pipelineRunDate = getApproximateRunDateAsString(runTime); + LOG.info("Saving sample types for date: {}", pipelineRunDate); + + final LocalDate localDate = runTime.atZone(ZoneId.systemDefault()).toLocalDate(); + final Optional byId = analyticsRepository.findById(pipelineRunDate); + MongoAnalytics analyticsRecord = byId.isPresent() ? byId.get() : null; + + if (analyticsRecord == null) { + analyticsRecord = + new MongoAnalytics( + pipelineRunDate, + localDate.getYear(), + localDate.getMonthValue(), + localDate.getDayOfMonth()); + } + analyticsRecord.setSampleAnalytics(sampleAnalytics); + analyticsRepository.save(analyticsRecord); + } + + public void persistPipelineAnalytics(final PipelineAnalytics pipelineAnalytics) { + final String pipelineRunDate = getApproximateRunDateAsString(pipelineAnalytics.getStartTime()); + LOG.info("Saving {} analytics for date: {}", pipelineAnalytics.getName(), pipelineRunDate); + + final LocalDate localDate = + pipelineAnalytics.getStartTime().atZone(ZoneId.systemDefault()).toLocalDate(); + final Optional byId = analyticsRepository.findById(pipelineRunDate); + MongoAnalytics analyticsRecord = byId.isPresent() ? byId.get() : null; + + if (analyticsRecord == null) { + analyticsRecord = + new MongoAnalytics( + pipelineRunDate, + localDate.getYear(), + localDate.getMonthValue(), + localDate.getDayOfMonth()); + } + analyticsRecord.addPipelineAnalytics(pipelineAnalytics); + analyticsRepository.save(analyticsRecord); + } + + private String getApproximateRunDateAsString(final Instant date) { + final LocalDateTime adjustedDate = LocalDateTime.ofInstant(date, ZoneOffset.UTC).plusHours(3); + + return adjustedDate.format(dateTimeFormatter); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CertificateToMongoCertificateConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CertificateToMongoCertificateConverter.java similarity index 93% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CertificateToMongoCertificateConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CertificateToMongoCertificateConverter.java index 6508b545d..8acb350ef 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CertificateToMongoCertificateConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CertificateToMongoCertificateConverter.java @@ -1,25 +1,25 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Certificate; -import uk.ac.ebi.biosamples.mongo.model.MongoCertificate; - -@Service -public class CertificateToMongoCertificateConverter - implements Converter { - @Override - public MongoCertificate convert(Certificate cert) { - return MongoCertificate.build(cert.getName(), cert.getVersion(), cert.getFileName()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Certificate; +import uk.ac.ebi.biosamples.mongo.model.MongoCertificate; + +@Service +public class CertificateToMongoCertificateConverter + implements Converter { + @Override + public MongoCertificate convert(Certificate cert) { + return MongoCertificate.build(cert.getName(), cert.getVersion(), cert.getFileName()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationLinkToMongoCurationLinkConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationLinkToMongoCurationLinkConverter.java similarity index 94% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationLinkToMongoCurationLinkConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationLinkToMongoCurationLinkConverter.java index 6ac0284ca..174598b05 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationLinkToMongoCurationLinkConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationLinkToMongoCurationLinkConverter.java @@ -1,31 +1,31 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.CurationLink; -import uk.ac.ebi.biosamples.mongo.model.MongoCurationLink; - -@Service -public class CurationLinkToMongoCurationLinkConverter - implements Converter { - - @Override - public MongoCurationLink convert(CurationLink curationLink) { - return MongoCurationLink.build( - curationLink.getSample(), - curationLink.getCuration(), - curationLink.getDomain(), - curationLink.getWebinSubmissionAccountId(), - curationLink.getCreated()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.CurationLink; +import uk.ac.ebi.biosamples.mongo.model.MongoCurationLink; + +@Service +public class CurationLinkToMongoCurationLinkConverter + implements Converter { + + @Override + public MongoCurationLink convert(CurationLink curationLink) { + return MongoCurationLink.build( + curationLink.getSample(), + curationLink.getCuration(), + curationLink.getDomain(), + curationLink.getWebinSubmissionAccountId(), + curationLink.getCreated()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationReadService.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationReadService.java similarity index 96% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationReadService.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationReadService.java index 84a6eaea0..7f747890c 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationReadService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationReadService.java @@ -1,191 +1,191 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import java.time.Instant; -import java.util.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Sort; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.*; -import uk.ac.ebi.biosamples.mongo.model.MongoCuration; -import uk.ac.ebi.biosamples.mongo.model.MongoCurationLink; -import uk.ac.ebi.biosamples.mongo.repository.MongoCurationLinkRepository; -import uk.ac.ebi.biosamples.mongo.repository.MongoCurationRepository; - -@Service -public class CurationReadService { - private final Logger log = LoggerFactory.getLogger(getClass()); - @Autowired private MongoCurationRepository mongoCurationRepository; - @Autowired private MongoCurationLinkRepository mongoCurationLinkRepository; - - // TODO use a ConversionService to manage all these - @Autowired - private MongoCurationLinkToCurationLinkConverter mongoCurationLinkToCurationLinkConverter; - - @Autowired private MongoCurationToCurationConverter mongoCurationToCurationConverter; - - public Page getPage(final Pageable pageable) { - final Page pageNeoCuration = mongoCurationRepository.findAll(pageable); - return pageNeoCuration.map(mongoCurationToCurationConverter); - } - - public Curation getCuration(final String hash) { - final Optional byId = mongoCurationRepository.findById(hash); - final MongoCuration neoCuration = byId.orElse(null); - - if (neoCuration == null) { - return null; - } else { - return mongoCurationToCurationConverter.apply(neoCuration); - } - } - - public Page getCurationLinksForSample( - final String accession, final Pageable pageable) { - final Page pageNeoCurationLink = - mongoCurationLinkRepository.findBySample(accession, pageable); - - // convert them into a state to return - return pageNeoCurationLink.map(mongoCurationLinkToCurationLinkConverter); - } - - public CurationLink getCurationLink(final String hash) { - final Optional byId = mongoCurationLinkRepository.findById(hash); - final MongoCurationLink mongoCurationLink = byId.orElse(null); - - assert mongoCurationLink != null; - return mongoCurationLinkToCurationLinkConverter.apply(mongoCurationLink); - } - - /** - * This applies a given curation link to a sample and returns a new sample. - * - *

This needs a curation link rather than a curation object because the samples update date may - * be modified if the curation link is newer. - */ - public Sample applyCurationLinkToSample(final Sample sample, final CurationLink curationLink) { - log.trace("Applying curation " + curationLink + " to sample " + sample.getAccession()); - - final Curation curation = curationLink.getCuration(); - final SortedSet attributes = new TreeSet<>(sample.getAttributes()); - final SortedSet externalReferences = - new TreeSet<>(sample.getExternalReferences()); - final SortedSet relationships = new TreeSet<>(sample.getRelationships()); - - // remove pre-curation things - for (final Attribute attribute : curation.getAttributesPre()) { - if (!attributes.contains(attribute)) { - throw new IllegalArgumentException( - "Failed to apply curation " + curation + " to sample " + sample); - } - - attributes.remove(attribute); - } - for (final ExternalReference externalReference : curation.getExternalReferencesPre()) { - if (!externalReferences.contains(externalReference)) { - throw new IllegalArgumentException( - "Failed to apply curation " + curation + " to sample " + sample); - } - - externalReferences.remove(externalReference); - } - for (final Relationship relationship : curation.getRelationshipsPre()) { - if (!relationships.contains(relationship)) { - throw new IllegalArgumentException( - "Failed to apply curation " + curation + " to sample " + sample); - } - - relationships.remove(relationship); - } - // add post-curation things - for (final Attribute attribute : curation.getAttributesPost()) { - if (attributes.contains(attribute)) { - throw new IllegalArgumentException( - "Failed to apply curation " + curation + " to sample " + sample); - } - - attributes.add(attribute); - } - for (final ExternalReference externalReference : curation.getExternalReferencesPost()) { - if (externalReferences.contains(externalReference)) { - throw new IllegalArgumentException( - "Failed to apply curation " + curation + " to sample " + sample); - } - - externalReferences.add(externalReference); - } - for (final Relationship relationship : curation.getRelationshipsPost()) { - if (relationships.contains(relationship)) { - throw new IllegalArgumentException( - "Failed to apply curation " + curation + " to sample " + sample); - } - - relationships.add(relationship); - } - - // update the sample's reviewed date - Instant reviewed = curationLink.getCreated(); - - if (reviewed != null) { - final Instant update = sample.getUpdate(); - - if (update.isAfter(reviewed)) { - reviewed = update; - } - } - - return Sample.Builder.fromSample(sample) - .withReviewed(reviewed) - .withAttributes(attributes) - .withExternalReferences(externalReferences) - .withRelationships(relationships) - .build(); - } - - public Sample applyAllCurationToSample(Sample sample) { - // Try to apply curations in the order of creation date. - // Because of the index in creation date mongo returns in that order - final Set curationLinks = new LinkedHashSet<>(); - int pageNo = 0; - Page page; - - do { - final Pageable pageable = PageRequest.of(pageNo, 1000, Sort.Direction.ASC, "created"); - page = getCurationLinksForSample(sample.getAccession(), pageable); - - for (final CurationLink curationLink : page) { - curationLinks.add(curationLink); - } - - pageNo += 1; - } while (pageNo < page.getTotalPages()); - - for (final CurationLink curation : curationLinks) { - try { - sample = applyCurationLinkToSample(sample, curation); - } catch (final IllegalArgumentException e) { - log.trace(e.getMessage()); - } - } - - if (sample.getReviewed() == null) { - sample = Sample.Builder.fromSample(sample).withReviewed(sample.getUpdate()).build(); - } - - return sample; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import java.time.Instant; +import java.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.*; +import uk.ac.ebi.biosamples.mongo.model.MongoCuration; +import uk.ac.ebi.biosamples.mongo.model.MongoCurationLink; +import uk.ac.ebi.biosamples.mongo.repository.MongoCurationLinkRepository; +import uk.ac.ebi.biosamples.mongo.repository.MongoCurationRepository; + +@Service +public class CurationReadService { + private final Logger log = LoggerFactory.getLogger(getClass()); + @Autowired private MongoCurationRepository mongoCurationRepository; + @Autowired private MongoCurationLinkRepository mongoCurationLinkRepository; + + // TODO use a ConversionService to manage all these + @Autowired + private MongoCurationLinkToCurationLinkConverter mongoCurationLinkToCurationLinkConverter; + + @Autowired private MongoCurationToCurationConverter mongoCurationToCurationConverter; + + public Page getPage(final Pageable pageable) { + final Page pageNeoCuration = mongoCurationRepository.findAll(pageable); + return pageNeoCuration.map(mongoCurationToCurationConverter); + } + + public Curation getCuration(final String hash) { + final Optional byId = mongoCurationRepository.findById(hash); + final MongoCuration neoCuration = byId.orElse(null); + + if (neoCuration == null) { + return null; + } else { + return mongoCurationToCurationConverter.apply(neoCuration); + } + } + + public Page getCurationLinksForSample( + final String accession, final Pageable pageable) { + final Page pageNeoCurationLink = + mongoCurationLinkRepository.findBySample(accession, pageable); + + // convert them into a state to return + return pageNeoCurationLink.map(mongoCurationLinkToCurationLinkConverter); + } + + public CurationLink getCurationLink(final String hash) { + final Optional byId = mongoCurationLinkRepository.findById(hash); + final MongoCurationLink mongoCurationLink = byId.orElse(null); + + assert mongoCurationLink != null; + return mongoCurationLinkToCurationLinkConverter.apply(mongoCurationLink); + } + + /** + * This applies a given curation link to a sample and returns a new sample. + * + *

This needs a curation link rather than a curation object because the samples update date may + * be modified if the curation link is newer. + */ + public Sample applyCurationLinkToSample(final Sample sample, final CurationLink curationLink) { + log.trace("Applying curation " + curationLink + " to sample " + sample.getAccession()); + + final Curation curation = curationLink.getCuration(); + final SortedSet attributes = new TreeSet<>(sample.getAttributes()); + final SortedSet externalReferences = + new TreeSet<>(sample.getExternalReferences()); + final SortedSet relationships = new TreeSet<>(sample.getRelationships()); + + // remove pre-curation things + for (final Attribute attribute : curation.getAttributesPre()) { + if (!attributes.contains(attribute)) { + throw new IllegalArgumentException( + "Failed to apply curation " + curation + " to sample " + sample); + } + + attributes.remove(attribute); + } + for (final ExternalReference externalReference : curation.getExternalReferencesPre()) { + if (!externalReferences.contains(externalReference)) { + throw new IllegalArgumentException( + "Failed to apply curation " + curation + " to sample " + sample); + } + + externalReferences.remove(externalReference); + } + for (final Relationship relationship : curation.getRelationshipsPre()) { + if (!relationships.contains(relationship)) { + throw new IllegalArgumentException( + "Failed to apply curation " + curation + " to sample " + sample); + } + + relationships.remove(relationship); + } + // add post-curation things + for (final Attribute attribute : curation.getAttributesPost()) { + if (attributes.contains(attribute)) { + throw new IllegalArgumentException( + "Failed to apply curation " + curation + " to sample " + sample); + } + + attributes.add(attribute); + } + for (final ExternalReference externalReference : curation.getExternalReferencesPost()) { + if (externalReferences.contains(externalReference)) { + throw new IllegalArgumentException( + "Failed to apply curation " + curation + " to sample " + sample); + } + + externalReferences.add(externalReference); + } + for (final Relationship relationship : curation.getRelationshipsPost()) { + if (relationships.contains(relationship)) { + throw new IllegalArgumentException( + "Failed to apply curation " + curation + " to sample " + sample); + } + + relationships.add(relationship); + } + + // update the sample's reviewed date + Instant reviewed = curationLink.getCreated(); + + if (reviewed != null) { + final Instant update = sample.getUpdate(); + + if (update.isAfter(reviewed)) { + reviewed = update; + } + } + + return Sample.Builder.fromSample(sample) + .withReviewed(reviewed) + .withAttributes(attributes) + .withExternalReferences(externalReferences) + .withRelationships(relationships) + .build(); + } + + public Sample applyAllCurationToSample(Sample sample) { + // Try to apply curations in the order of creation date. + // Because of the index in creation date mongo returns in that order + final Set curationLinks = new LinkedHashSet<>(); + int pageNo = 0; + Page page; + + do { + final Pageable pageable = PageRequest.of(pageNo, 1000, Sort.Direction.ASC, "created"); + page = getCurationLinksForSample(sample.getAccession(), pageable); + + for (final CurationLink curationLink : page) { + curationLinks.add(curationLink); + } + + pageNo += 1; + } while (pageNo < page.getTotalPages()); + + for (final CurationLink curation : curationLinks) { + try { + sample = applyCurationLinkToSample(sample, curation); + } catch (final IllegalArgumentException e) { + log.trace(e.getMessage()); + } + } + + if (sample.getReviewed() == null) { + sample = Sample.Builder.fromSample(sample).withReviewed(sample.getUpdate()).build(); + } + + return sample; + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationToMongoCurationConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationToMongoCurationConverter.java similarity index 94% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationToMongoCurationConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationToMongoCurationConverter.java index f6c441cb8..9279200b9 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationToMongoCurationConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CurationToMongoCurationConverter.java @@ -1,33 +1,33 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.mongo.model.MongoCuration; - -@Service -@ConfigurationPropertiesBinding -public class CurationToMongoCurationConverter implements Converter { - - @Override - public MongoCuration convert(Curation curation) { - return MongoCuration.build( - curation.getAttributesPre(), - curation.getAttributesPost(), - curation.getExternalReferencesPre(), - curation.getExternalReferencesPost(), - curation.getRelationshipsPre(), - curation.getRelationshipsPost()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.mongo.model.MongoCuration; + +@Service +@ConfigurationPropertiesBinding +public class CurationToMongoCurationConverter implements Converter { + + @Override + public MongoCuration convert(Curation curation) { + return MongoCuration.build( + curation.getAttributesPre(), + curation.getAttributesPost(), + curation.getExternalReferencesPre(), + curation.getExternalReferencesPost(), + curation.getRelationshipsPre(), + curation.getRelationshipsPost()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CustomWriteConcernResolver.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CustomWriteConcernResolver.java similarity index 97% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CustomWriteConcernResolver.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CustomWriteConcernResolver.java index 2a531318a..de4debe7f 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/CustomWriteConcernResolver.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/CustomWriteConcernResolver.java @@ -1,47 +1,47 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import com.mongodb.WriteConcern; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.mongodb.core.MongoAction; -import org.springframework.data.mongodb.core.WriteConcernResolver; -import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.mongo.MongoProperties; -import uk.ac.ebi.biosamples.mongo.model.MongoSample; - -// TODO wire this into config -@Component -public class CustomWriteConcernResolver implements WriteConcernResolver { - - private final Logger log = LoggerFactory.getLogger(getClass()); - - @Autowired private MongoProperties mongoProperties; - - @Override - public WriteConcern resolve(final MongoAction action) { - log.trace("Resolving mongoAction " + action); - - if (MongoSample.class.isAssignableFrom(action.getEntityType())) { - final String sampleWriteConcern = mongoProperties.getSampleWriteConcern(); - - if (sampleWriteConcern.matches("[0-9]+")) { - return new WriteConcern(Integer.parseInt(sampleWriteConcern)); - } else { - return new WriteConcern(sampleWriteConcern); - } - } - - return action.getDefaultWriteConcern(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import com.mongodb.WriteConcern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoAction; +import org.springframework.data.mongodb.core.WriteConcernResolver; +import org.springframework.stereotype.Component; +import uk.ac.ebi.biosamples.mongo.MongoProperties; +import uk.ac.ebi.biosamples.mongo.model.MongoSample; + +// TODO wire this into config +@Component +public class CustomWriteConcernResolver implements WriteConcernResolver { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + @Autowired private MongoProperties mongoProperties; + + @Override + public WriteConcern resolve(final MongoAction action) { + log.trace("Resolving mongoAction " + action); + + if (MongoSample.class.isAssignableFrom(action.getEntityType())) { + final String sampleWriteConcern = mongoProperties.getSampleWriteConcern(); + + if (sampleWriteConcern.matches("[0-9]+")) { + return new WriteConcern(Integer.parseInt(sampleWriteConcern)); + } else { + return new WriteConcern(sampleWriteConcern); + } + } + + return action.getDefaultWriteConcern(); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/ExternalReferenceToMongoExternalReferenceConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/ExternalReferenceToMongoExternalReferenceConverter.java similarity index 93% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/ExternalReferenceToMongoExternalReferenceConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/ExternalReferenceToMongoExternalReferenceConverter.java index b6d0679d4..93ba803ee 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/ExternalReferenceToMongoExternalReferenceConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/ExternalReferenceToMongoExternalReferenceConverter.java @@ -1,28 +1,28 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.ExternalReference; -import uk.ac.ebi.biosamples.mongo.model.MongoExternalReference; - -@Service -@ConfigurationPropertiesBinding -public class ExternalReferenceToMongoExternalReferenceConverter - implements Converter { - - @Override - public MongoExternalReference convert(ExternalReference externalReference) { - return MongoExternalReference.build(externalReference.getUrl(), externalReference.getDuo()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.ExternalReference; +import uk.ac.ebi.biosamples.mongo.model.MongoExternalReference; + +@Service +@ConfigurationPropertiesBinding +public class ExternalReferenceToMongoExternalReferenceConverter + implements Converter { + + @Override + public MongoExternalReference convert(ExternalReference externalReference) { + return MongoExternalReference.build(externalReference.getUrl(), externalReference.getDuo()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoAccessionService.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoAccessionService.java similarity index 96% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoAccessionService.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoAccessionService.java index 62030f459..43b6641f9 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoAccessionService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoAccessionService.java @@ -1,207 +1,207 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import static org.springframework.data.mongodb.core.FindAndModifyOptions.options; -import static org.springframework.data.mongodb.core.query.Criteria.where; -import static org.springframework.data.mongodb.core.query.Query.query; - -import java.util.Objects; -import java.util.SortedSet; -import java.util.TreeSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.mongodb.core.MongoOperations; -import org.springframework.data.mongodb.core.query.Update; -import uk.ac.ebi.biosamples.BioSamplesConstants; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; -import uk.ac.ebi.biosamples.mongo.model.MongoSample; -import uk.ac.ebi.biosamples.mongo.model.MongoSequence; -import uk.ac.ebi.biosamples.mongo.repository.MongoSampleRepository; - -// this needs to be the spring exception, not the mongo one -public class MongoAccessionService { - private final Logger log = LoggerFactory.getLogger(getClass()); - private final MongoSampleRepository mongoSampleRepository; - private final SampleToMongoSampleConverter sampleToMongoSampleConverter; - private final MongoSampleToSampleConverter mongoSampleToSampleConverter; - private final MongoOperations mongoOperations; - - public MongoAccessionService( - final MongoSampleRepository mongoSampleRepository, - final SampleToMongoSampleConverter sampleToMongoSampleConverter, - final MongoSampleToSampleConverter mongoSampleToSampleConverter, - final MongoOperations mongoOperations) { - this.mongoSampleRepository = mongoSampleRepository; - this.sampleToMongoSampleConverter = sampleToMongoSampleConverter; - this.mongoSampleToSampleConverter = mongoSampleToSampleConverter; - this.mongoOperations = mongoOperations; - } - - public Sample generateAccession(final Sample sample, final boolean generateSAMEAndSRAAccession) { - MongoSample mongoSample = sampleToMongoSampleConverter.convert(sample); - - mongoSample = accessionAndInsert(mongoSample, generateSAMEAndSRAAccession); - - return mongoSampleToSampleConverter.apply(mongoSample); - } - - private MongoSample accessionAndInsert( - MongoSample sample, final boolean generateSAMEAndSRAAccession) { - log.trace("Generating a new accession"); - - final MongoSample originalSample = sample; - // inspired by Counter collection + Optimistic Loops of - // https://docs.mongodb.com/v3.0/tutorial/create-an-auto-incrementing-field/ - - boolean success = false; - int numRetry = 0; - - while (!success) { - // TODO add a timeout here - try { - sample = prepare(sample, generateUniqueAccessions(generateSAMEAndSRAAccession)); - } catch (final Exception e) { - throw new RuntimeException(e); - } - - try { - sample = mongoSampleRepository.insertNew(sample); - success = true; - } catch (final Exception e) { - if (++numRetry == BioSamplesConstants.MAX_RETRIES) { - throw new RuntimeException( - "Cannot generate a new BioSample accession. please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk"); - } - - sample = originalSample; - } - } - - if (generateSAMEAndSRAAccession) { - log.info("Generated BioSample and SRA accession " + sample); - } else { - log.info("Generated BioSample accession " + sample); - } - - return sample; - } - - private MongoSample prepare(MongoSample sample, final Accessions accessions) { - final SortedSet relationships = sample.getRelationships(); - final SortedSet newRelationships = new TreeSet<>(); - for (final MongoRelationship relationship : relationships) { - // this relationship could not specify a source because the sample is un-accessioned - // now we are assigning an accession, set the source to the accession - if (relationship.getSource() == null || relationship.getSource().trim().isEmpty()) { - newRelationships.add( - MongoRelationship.build( - accessions.accession, relationship.getType(), relationship.getTarget())); - } else { - newRelationships.add(relationship); - } - } - - if (accessions.sraAccession != null) { - sample - .getAttributes() - .add(Attribute.build(BioSamplesConstants.SRA_ACCESSION, accessions.sraAccession)); - } - - sample = - MongoSample.build( - sample.getName(), - accessions.accession, - accessions.sraAccession != null ? accessions.sraAccession : sample.getSraAccession(), - sample.getDomain(), - sample.getWebinSubmissionAccountId(), - sample.getTaxId(), - sample.getStatus(), - sample.getRelease(), - sample.getUpdate(), - sample.getCreate(), - sample.getSubmitted(), - sample.getReviewed(), - sample.getAttributes(), - sample.getData(), - newRelationships, - sample.getExternalReferences(), - sample.getOrganizations(), - sample.getContacts(), - sample.getPublications(), - sample.getCertificates(), - sample.getSubmittedVia()); - - return sample; - } - - private Accessions generateUniqueAccessions(final boolean generateSAMEAndSRAAccession) { - final MongoSequence accessionSeq = - mongoOperations.findAndModify( - query(where("_id").is(MongoSample.SEQUENCE_NAME)), - new Update().inc("seq", 1), - options().returnNew(true).upsert(true), - MongoSequence.class); - - if (generateSAMEAndSRAAccession) { - final MongoSequence sraAccessionSeq = - mongoOperations.findAndModify( - query(where("_id").is(MongoSample.SRA_SEQUENCE_NAME)), - new Update().inc("seq", 1), - options().returnNew(true).upsert(true), - MongoSequence.class); - - if (!Objects.isNull(accessionSeq) && !Objects.isNull(sraAccessionSeq)) { - return new Accessions( - BioSamplesConstants.ACCESSION_PREFIX + accessionSeq.getSeq(), - BioSamplesConstants.SRA_ACCESSION_PREFIX + sraAccessionSeq.getSeq()); - } else { - throw new RuntimeException( - "Cannot generate a new BioSample and SRA accession. please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk"); - } - } else { - if (!Objects.isNull(accessionSeq)) { - return new Accessions(BioSamplesConstants.ACCESSION_PREFIX + accessionSeq.getSeq(), null); - } else { - throw new RuntimeException( - "Cannot generate a new BioSample accession. please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk"); - } - } - } - - public String generateOneSRAAccession() { - final MongoSequence sraAccessionSeq = - mongoOperations.findAndModify( - query(where("_id").is(MongoSample.SRA_SEQUENCE_NAME)), - new Update().inc("seq", 1), - options().returnNew(true).upsert(true), - MongoSequence.class); - - if (!Objects.isNull(sraAccessionSeq)) { - return BioSamplesConstants.SRA_ACCESSION_PREFIX + sraAccessionSeq.getSeq(); - } else { - throw new RuntimeException( - "Cannot generate a new SRA accession number. please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk"); - } - } - - private static class Accessions { - private final String accession; - private final String sraAccession; - - private Accessions(final String accession, final String sraAccession) { - this.accession = accession; - this.sraAccession = sraAccession; - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import static org.springframework.data.mongodb.core.FindAndModifyOptions.options; +import static org.springframework.data.mongodb.core.query.Criteria.where; +import static org.springframework.data.mongodb.core.query.Query.query; + +import java.util.Objects; +import java.util.SortedSet; +import java.util.TreeSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.mongodb.core.MongoOperations; +import org.springframework.data.mongodb.core.query.Update; +import uk.ac.ebi.biosamples.BioSamplesConstants; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; +import uk.ac.ebi.biosamples.mongo.model.MongoSample; +import uk.ac.ebi.biosamples.mongo.model.MongoSequence; +import uk.ac.ebi.biosamples.mongo.repository.MongoSampleRepository; + +// this needs to be the spring exception, not the mongo one +public class MongoAccessionService { + private final Logger log = LoggerFactory.getLogger(getClass()); + private final MongoSampleRepository mongoSampleRepository; + private final SampleToMongoSampleConverter sampleToMongoSampleConverter; + private final MongoSampleToSampleConverter mongoSampleToSampleConverter; + private final MongoOperations mongoOperations; + + public MongoAccessionService( + final MongoSampleRepository mongoSampleRepository, + final SampleToMongoSampleConverter sampleToMongoSampleConverter, + final MongoSampleToSampleConverter mongoSampleToSampleConverter, + final MongoOperations mongoOperations) { + this.mongoSampleRepository = mongoSampleRepository; + this.sampleToMongoSampleConverter = sampleToMongoSampleConverter; + this.mongoSampleToSampleConverter = mongoSampleToSampleConverter; + this.mongoOperations = mongoOperations; + } + + public Sample generateAccession(final Sample sample, final boolean generateSAMEAndSRAAccession) { + MongoSample mongoSample = sampleToMongoSampleConverter.convert(sample); + + mongoSample = accessionAndInsert(mongoSample, generateSAMEAndSRAAccession); + + return mongoSampleToSampleConverter.apply(mongoSample); + } + + private MongoSample accessionAndInsert( + MongoSample sample, final boolean generateSAMEAndSRAAccession) { + log.trace("Generating a new accession"); + + final MongoSample originalSample = sample; + // inspired by Counter collection + Optimistic Loops of + // https://docs.mongodb.com/v3.0/tutorial/create-an-auto-incrementing-field/ + + boolean success = false; + int numRetry = 0; + + while (!success) { + // TODO add a timeout here + try { + sample = prepare(sample, generateUniqueAccessions(generateSAMEAndSRAAccession)); + } catch (final Exception e) { + throw new RuntimeException(e); + } + + try { + sample = mongoSampleRepository.insertNew(sample); + success = true; + } catch (final Exception e) { + if (++numRetry == BioSamplesConstants.MAX_RETRIES) { + throw new RuntimeException( + "Cannot generate a new BioSample accession. please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk"); + } + + sample = originalSample; + } + } + + if (generateSAMEAndSRAAccession) { + log.info("Generated BioSample and SRA accession " + sample); + } else { + log.info("Generated BioSample accession " + sample); + } + + return sample; + } + + private MongoSample prepare(MongoSample sample, final Accessions accessions) { + final SortedSet relationships = sample.getRelationships(); + final SortedSet newRelationships = new TreeSet<>(); + for (final MongoRelationship relationship : relationships) { + // this relationship could not specify a source because the sample is un-accessioned + // now we are assigning an accession, set the source to the accession + if (relationship.getSource() == null || relationship.getSource().trim().isEmpty()) { + newRelationships.add( + MongoRelationship.build( + accessions.accession, relationship.getType(), relationship.getTarget())); + } else { + newRelationships.add(relationship); + } + } + + if (accessions.sraAccession != null) { + sample + .getAttributes() + .add(Attribute.build(BioSamplesConstants.SRA_ACCESSION, accessions.sraAccession)); + } + + sample = + MongoSample.build( + sample.getName(), + accessions.accession, + accessions.sraAccession != null ? accessions.sraAccession : sample.getSraAccession(), + sample.getDomain(), + sample.getWebinSubmissionAccountId(), + sample.getTaxId(), + sample.getStatus(), + sample.getRelease(), + sample.getUpdate(), + sample.getCreate(), + sample.getSubmitted(), + sample.getReviewed(), + sample.getAttributes(), + sample.getData(), + newRelationships, + sample.getExternalReferences(), + sample.getOrganizations(), + sample.getContacts(), + sample.getPublications(), + sample.getCertificates(), + sample.getSubmittedVia()); + + return sample; + } + + private Accessions generateUniqueAccessions(final boolean generateSAMEAndSRAAccession) { + final MongoSequence accessionSeq = + mongoOperations.findAndModify( + query(where("_id").is(MongoSample.SEQUENCE_NAME)), + new Update().inc("seq", 1), + options().returnNew(true).upsert(true), + MongoSequence.class); + + if (generateSAMEAndSRAAccession) { + final MongoSequence sraAccessionSeq = + mongoOperations.findAndModify( + query(where("_id").is(MongoSample.SRA_SEQUENCE_NAME)), + new Update().inc("seq", 1), + options().returnNew(true).upsert(true), + MongoSequence.class); + + if (!Objects.isNull(accessionSeq) && !Objects.isNull(sraAccessionSeq)) { + return new Accessions( + BioSamplesConstants.ACCESSION_PREFIX + accessionSeq.getSeq(), + BioSamplesConstants.SRA_ACCESSION_PREFIX + sraAccessionSeq.getSeq()); + } else { + throw new RuntimeException( + "Cannot generate a new BioSample and SRA accession. please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk"); + } + } else { + if (!Objects.isNull(accessionSeq)) { + return new Accessions(BioSamplesConstants.ACCESSION_PREFIX + accessionSeq.getSeq(), null); + } else { + throw new RuntimeException( + "Cannot generate a new BioSample accession. please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk"); + } + } + } + + public String generateOneSRAAccession() { + final MongoSequence sraAccessionSeq = + mongoOperations.findAndModify( + query(where("_id").is(MongoSample.SRA_SEQUENCE_NAME)), + new Update().inc("seq", 1), + options().returnNew(true).upsert(true), + MongoSequence.class); + + if (!Objects.isNull(sraAccessionSeq)) { + return BioSamplesConstants.SRA_ACCESSION_PREFIX + sraAccessionSeq.getSeq(); + } else { + throw new RuntimeException( + "Cannot generate a new SRA accession number. please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk"); + } + } + + private static class Accessions { + private final String accession; + private final String sraAccession; + + private Accessions(final String accession, final String sraAccession) { + this.accession = accession; + this.sraAccession = sraAccession; + } + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCertificateToCertificateConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCertificateToCertificateConverter.java similarity index 94% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCertificateToCertificateConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCertificateToCertificateConverter.java index ea3c8bd1d..b856e061a 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCertificateToCertificateConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCertificateToCertificateConverter.java @@ -1,28 +1,28 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Certificate; -import uk.ac.ebi.biosamples.mongo.model.MongoCertificate; - -@Service -@ConfigurationPropertiesBinding -public class MongoCertificateToCertificateConverter - implements Converter { - @Override - public Certificate convert(MongoCertificate mongoCertificate) { - return Certificate.build( - mongoCertificate.getName(), mongoCertificate.getVersion(), mongoCertificate.getFileName()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Certificate; +import uk.ac.ebi.biosamples.mongo.model.MongoCertificate; + +@Service +@ConfigurationPropertiesBinding +public class MongoCertificateToCertificateConverter + implements Converter { + @Override + public Certificate convert(MongoCertificate mongoCertificate) { + return Certificate.build( + mongoCertificate.getName(), mongoCertificate.getVersion(), mongoCertificate.getFileName()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationLinkToCurationLinkConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationLinkToCurationLinkConverter.java similarity index 94% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationLinkToCurationLinkConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationLinkToCurationLinkConverter.java index a287b2877..07cd18aee 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationLinkToCurationLinkConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationLinkToCurationLinkConverter.java @@ -1,31 +1,31 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import java.util.function.Function; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.CurationLink; -import uk.ac.ebi.biosamples.mongo.model.MongoCurationLink; - -@Service -public class MongoCurationLinkToCurationLinkConverter - implements Function { - - @Override - public CurationLink apply(final MongoCurationLink mongoCurationLink) { - return CurationLink.build( - mongoCurationLink.getSample(), - mongoCurationLink.getCuration(), - mongoCurationLink.getDomain(), - mongoCurationLink.getWebinSubmissionAccountId(), - mongoCurationLink.getCreated()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import java.util.function.Function; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.CurationLink; +import uk.ac.ebi.biosamples.mongo.model.MongoCurationLink; + +@Service +public class MongoCurationLinkToCurationLinkConverter + implements Function { + + @Override + public CurationLink apply(final MongoCurationLink mongoCurationLink) { + return CurationLink.build( + mongoCurationLink.getSample(), + mongoCurationLink.getCuration(), + mongoCurationLink.getDomain(), + mongoCurationLink.getWebinSubmissionAccountId(), + mongoCurationLink.getCreated()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationToCurationConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationToCurationConverter.java similarity index 94% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationToCurationConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationToCurationConverter.java index 573813fae..3d6284d84 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationToCurationConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoCurationToCurationConverter.java @@ -1,33 +1,33 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import java.util.function.Function; -import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.mongo.model.MongoCuration; - -@Service -@ConfigurationPropertiesBinding -public class MongoCurationToCurationConverter implements Function { - - @Override - public Curation apply(final MongoCuration mongoCuration) { - return Curation.build( - mongoCuration.getAttributesPre(), - mongoCuration.getAttributesPost(), - mongoCuration.getExternalReferencesPre(), - mongoCuration.getExternalReferencesPost(), - mongoCuration.getRelationshipsPre(), - mongoCuration.getRelationshipsPost()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import java.util.function.Function; +import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.mongo.model.MongoCuration; + +@Service +@ConfigurationPropertiesBinding +public class MongoCurationToCurationConverter implements Function { + + @Override + public Curation apply(final MongoCuration mongoCuration) { + return Curation.build( + mongoCuration.getAttributesPre(), + mongoCuration.getAttributesPost(), + mongoCuration.getExternalReferencesPre(), + mongoCuration.getExternalReferencesPost(), + mongoCuration.getRelationshipsPre(), + mongoCuration.getRelationshipsPost()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoExternalReferenceToExternalReferenceConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoExternalReferenceToExternalReferenceConverter.java similarity index 94% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoExternalReferenceToExternalReferenceConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoExternalReferenceToExternalReferenceConverter.java index f0d968c1c..f0ad98eaf 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoExternalReferenceToExternalReferenceConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoExternalReferenceToExternalReferenceConverter.java @@ -1,27 +1,27 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.ExternalReference; -import uk.ac.ebi.biosamples.mongo.model.MongoExternalReference; - -@Service -@ConfigurationPropertiesBinding -public class MongoExternalReferenceToExternalReferenceConverter - implements Converter { - @Override - public ExternalReference convert(MongoExternalReference externalReference) { - return ExternalReference.build(externalReference.getUrl(), externalReference.getDuo()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.ExternalReference; +import uk.ac.ebi.biosamples.mongo.model.MongoExternalReference; + +@Service +@ConfigurationPropertiesBinding +public class MongoExternalReferenceToExternalReferenceConverter + implements Converter { + @Override + public ExternalReference convert(MongoExternalReference externalReference) { + return ExternalReference.build(externalReference.getUrl(), externalReference.getDuo()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoInverseRelationshipService.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoInverseRelationshipService.java similarity index 97% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoInverseRelationshipService.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoInverseRelationshipService.java index 5b36f4a06..6ea0eb12b 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoInverseRelationshipService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoInverseRelationshipService.java @@ -1,64 +1,64 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import java.util.ArrayList; -import java.util.List; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.BasicQuery; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; -import uk.ac.ebi.biosamples.mongo.model.MongoSample; - -@Service -public class MongoInverseRelationshipService { - - private final MongoTemplate mongoTemplate; - - public MongoInverseRelationshipService(final MongoTemplate mongoTemplate) { - this.mongoTemplate = mongoTemplate; - } - - public MongoSample addInverseRelationships(final MongoSample mongoSample) { - final String accession = mongoSample.getAccession(); - if (accession == null) { - return mongoSample; - } - - final Query query = - new BasicQuery("{'relationships.target':'" + accession + "'}", "{'relationships.$':1}"); - for (final MongoSample other : mongoTemplate.find(query, MongoSample.class)) { - for (final MongoRelationship relationship : other.getRelationships()) { - if (relationship.getTarget().equals(accession)) { - mongoSample.getRelationships().add(relationship); - } - } - } - - return mongoSample; - } - - public List getInverseRelationshipsTargets(final String accession) { - final List relTargetAccessionList = new ArrayList<>(); - final Query query = - new BasicQuery("{'relationships.target':'" + accession + "'}", "{'relationships.$':1}"); - for (final MongoSample other : mongoTemplate.find(query, MongoSample.class)) { - for (final MongoRelationship relationship : other.getRelationships()) { - if (relationship.getTarget().equals(accession)) { - relTargetAccessionList.add(relationship.getSource()); - } - } - } - - return relTargetAccessionList; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import java.util.ArrayList; +import java.util.List; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.BasicQuery; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; +import uk.ac.ebi.biosamples.mongo.model.MongoSample; + +@Service +public class MongoInverseRelationshipService { + + private final MongoTemplate mongoTemplate; + + public MongoInverseRelationshipService(final MongoTemplate mongoTemplate) { + this.mongoTemplate = mongoTemplate; + } + + public MongoSample addInverseRelationships(final MongoSample mongoSample) { + final String accession = mongoSample.getAccession(); + if (accession == null) { + return mongoSample; + } + + final Query query = + new BasicQuery("{'relationships.target':'" + accession + "'}", "{'relationships.$':1}"); + for (final MongoSample other : mongoTemplate.find(query, MongoSample.class)) { + for (final MongoRelationship relationship : other.getRelationships()) { + if (relationship.getTarget().equals(accession)) { + mongoSample.getRelationships().add(relationship); + } + } + } + + return mongoSample; + } + + public List getInverseRelationshipsTargets(final String accession) { + final List relTargetAccessionList = new ArrayList<>(); + final Query query = + new BasicQuery("{'relationships.target':'" + accession + "'}", "{'relationships.$':1}"); + for (final MongoSample other : mongoTemplate.find(query, MongoSample.class)) { + for (final MongoRelationship relationship : other.getRelationships()) { + if (relationship.getTarget().equals(accession)) { + relTargetAccessionList.add(relationship.getSource()); + } + } + } + + return relTargetAccessionList; + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoRelationshipToRelationshipConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoRelationshipToRelationshipConverter.java similarity index 93% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoRelationshipToRelationshipConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoRelationshipToRelationshipConverter.java index 197b3af94..9969b23ec 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoRelationshipToRelationshipConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoRelationshipToRelationshipConverter.java @@ -1,26 +1,26 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; - -@Service -public class MongoRelationshipToRelationshipConverter - implements Converter { - @Override - public Relationship convert(MongoRelationship relationship) { - return Relationship.build( - relationship.getSource(), relationship.getType(), relationship.getTarget()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; + +@Service +public class MongoRelationshipToRelationshipConverter + implements Converter { + @Override + public Relationship convert(MongoRelationship relationship) { + return Relationship.build( + relationship.getSource(), relationship.getType(), relationship.getTarget()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoSampleToSampleConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoSampleToSampleConverter.java similarity index 94% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoSampleToSampleConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoSampleToSampleConverter.java index 6aa94143f..857fb4fac 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoSampleToSampleConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoSampleToSampleConverter.java @@ -1,147 +1,147 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import java.time.Instant; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.function.Function; -import lombok.extern.slf4j.Slf4j; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Certificate; -import uk.ac.ebi.biosamples.model.ExternalReference; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.mongo.model.MongoCertificate; -import uk.ac.ebi.biosamples.mongo.model.MongoExternalReference; -import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; -import uk.ac.ebi.biosamples.mongo.model.MongoSample; - -@Service -@Slf4j -public class MongoSampleToSampleConverter implements Function { - @Autowired - private MongoExternalReferenceToExternalReferenceConverter - mongoExternalReferenceToExternalReferenceConverter; - - @Autowired - private MongoRelationshipToRelationshipConverter mongoRelationshipToRelationshipConverter; - - @Autowired private MongoCertificateToCertificateConverter mongoCertificateToCertificateConverter; - private static final Logger LOGGER = LoggerFactory.getLogger(MongoSampleToSampleConverter.class); - - @Override - public Sample apply(final MongoSample mongoSample) { - final Sample convertedSample; - final SortedSet externalReferences = new TreeSet<>(); - - if (mongoSample.getExternalReferences() != null - && !mongoSample.getExternalReferences().isEmpty()) { - for (final MongoExternalReference mongoExternalReference : - mongoSample.getExternalReferences()) { - if (mongoExternalReference != null) { - externalReferences.add( - mongoExternalReferenceToExternalReferenceConverter.convert(mongoExternalReference)); - } - } - } - - final SortedSet relationships = new TreeSet<>(); - - if (mongoSample.getRelationships() != null && !mongoSample.getRelationships().isEmpty()) { - for (final MongoRelationship mongoRelationship : mongoSample.getRelationships()) { - if (mongoRelationship != null) { - relationships.add(mongoRelationshipToRelationshipConverter.convert(mongoRelationship)); - } - } - } - - final SortedSet certificates = new TreeSet<>(); - - if (mongoSample.getCertificates() != null && !mongoSample.getCertificates().isEmpty()) { - for (final MongoCertificate certificate : mongoSample.getCertificates()) { - if (certificate != null) { - certificates.add(mongoCertificateToCertificateConverter.convert(certificate)); - } - } - } - - // when we convert to a Sample then the MongoSample *must* have a domain or a Webin ID - if (mongoSample.getDomain() == null && mongoSample.getWebinSubmissionAccountId() == null) { - LOGGER.warn( - String.format( - "Sample %s does not have a domain or a WEBIN submission account ID", - mongoSample.getAccession())); - throw new RuntimeException("Sample does not have domain or a WEBIN submission account ID"); - } - - final Instant submitted = mongoSample.getSubmitted(); - - if (submitted == null) { - convertedSample = - new Sample.Builder( - mongoSample.getName(), mongoSample.getAccession(), mongoSample.getSraAccession()) - .withDomain(mongoSample.getDomain()) - .withTaxId(mongoSample.getTaxId()) - .withStatus(mongoSample.getStatus()) - .withWebinSubmissionAccountId(mongoSample.getWebinSubmissionAccountId()) - .withRelease(mongoSample.getRelease()) - .withUpdate(mongoSample.getUpdate()) - .withCreate(mongoSample.getCreate()) - .withNoSubmitted() - .withAttributes(mongoSample.getAttributes()) - .withRelationships(relationships) - .withData(mongoSample.getData()) - .withExternalReferences(externalReferences) - .withOrganizations(mongoSample.getOrganizations()) - .withContacts(mongoSample.getContacts()) - .withPublications(mongoSample.getPublications()) - .withCertificates(certificates) - .withSubmittedVia(mongoSample.getSubmittedVia()) - .build(); - } else { - convertedSample = - new Sample.Builder( - mongoSample.getName(), mongoSample.getAccession(), mongoSample.getSraAccession()) - .withDomain(mongoSample.getDomain()) - .withTaxId(mongoSample.getTaxId()) - .withStatus(mongoSample.getStatus()) - .withWebinSubmissionAccountId(mongoSample.getWebinSubmissionAccountId()) - .withRelease(mongoSample.getRelease()) - .withUpdate(mongoSample.getUpdate()) - .withCreate(mongoSample.getCreate()) - .withSubmitted(mongoSample.getSubmitted()) - .withAttributes(mongoSample.getAttributes()) - .withRelationships(relationships) - .withData(mongoSample.getData()) - .withExternalReferences(externalReferences) - .withOrganizations(mongoSample.getOrganizations()) - .withContacts(mongoSample.getContacts()) - .withPublications(mongoSample.getPublications()) - .withCertificates(certificates) - .withSubmittedVia(mongoSample.getSubmittedVia()) - .build(); - } - - final Instant reviewed = mongoSample.getReviewed(); - - if (reviewed == null) { - return Sample.Builder.fromSample(convertedSample).withNoReviewed().build(); - } else { - return Sample.Builder.fromSample(convertedSample) - .withReviewed(mongoSample.getReviewed()) - .build(); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import java.time.Instant; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.function.Function; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Certificate; +import uk.ac.ebi.biosamples.core.model.ExternalReference; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.mongo.model.MongoCertificate; +import uk.ac.ebi.biosamples.mongo.model.MongoExternalReference; +import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; +import uk.ac.ebi.biosamples.mongo.model.MongoSample; + +@Service +@Slf4j +public class MongoSampleToSampleConverter implements Function { + @Autowired + private MongoExternalReferenceToExternalReferenceConverter + mongoExternalReferenceToExternalReferenceConverter; + + @Autowired + private MongoRelationshipToRelationshipConverter mongoRelationshipToRelationshipConverter; + + @Autowired private MongoCertificateToCertificateConverter mongoCertificateToCertificateConverter; + private static final Logger LOGGER = LoggerFactory.getLogger(MongoSampleToSampleConverter.class); + + @Override + public Sample apply(final MongoSample mongoSample) { + final Sample convertedSample; + final SortedSet externalReferences = new TreeSet<>(); + + if (mongoSample.getExternalReferences() != null + && !mongoSample.getExternalReferences().isEmpty()) { + for (final MongoExternalReference mongoExternalReference : + mongoSample.getExternalReferences()) { + if (mongoExternalReference != null) { + externalReferences.add( + mongoExternalReferenceToExternalReferenceConverter.convert(mongoExternalReference)); + } + } + } + + final SortedSet relationships = new TreeSet<>(); + + if (mongoSample.getRelationships() != null && !mongoSample.getRelationships().isEmpty()) { + for (final MongoRelationship mongoRelationship : mongoSample.getRelationships()) { + if (mongoRelationship != null) { + relationships.add(mongoRelationshipToRelationshipConverter.convert(mongoRelationship)); + } + } + } + + final SortedSet certificates = new TreeSet<>(); + + if (mongoSample.getCertificates() != null && !mongoSample.getCertificates().isEmpty()) { + for (final MongoCertificate certificate : mongoSample.getCertificates()) { + if (certificate != null) { + certificates.add(mongoCertificateToCertificateConverter.convert(certificate)); + } + } + } + + // when we convert to a Sample then the MongoSample *must* have a domain or a Webin ID + if (mongoSample.getDomain() == null && mongoSample.getWebinSubmissionAccountId() == null) { + LOGGER.warn( + String.format( + "Sample %s does not have a domain or a WEBIN submission account ID", + mongoSample.getAccession())); + throw new RuntimeException("Sample does not have domain or a WEBIN submission account ID"); + } + + final Instant submitted = mongoSample.getSubmitted(); + + if (submitted == null) { + convertedSample = + new Sample.Builder( + mongoSample.getName(), mongoSample.getAccession(), mongoSample.getSraAccession()) + .withDomain(mongoSample.getDomain()) + .withTaxId(mongoSample.getTaxId()) + .withStatus(mongoSample.getStatus()) + .withWebinSubmissionAccountId(mongoSample.getWebinSubmissionAccountId()) + .withRelease(mongoSample.getRelease()) + .withUpdate(mongoSample.getUpdate()) + .withCreate(mongoSample.getCreate()) + .withNoSubmitted() + .withAttributes(mongoSample.getAttributes()) + .withRelationships(relationships) + .withData(mongoSample.getData()) + .withExternalReferences(externalReferences) + .withOrganizations(mongoSample.getOrganizations()) + .withContacts(mongoSample.getContacts()) + .withPublications(mongoSample.getPublications()) + .withCertificates(certificates) + .withSubmittedVia(mongoSample.getSubmittedVia()) + .build(); + } else { + convertedSample = + new Sample.Builder( + mongoSample.getName(), mongoSample.getAccession(), mongoSample.getSraAccession()) + .withDomain(mongoSample.getDomain()) + .withTaxId(mongoSample.getTaxId()) + .withStatus(mongoSample.getStatus()) + .withWebinSubmissionAccountId(mongoSample.getWebinSubmissionAccountId()) + .withRelease(mongoSample.getRelease()) + .withUpdate(mongoSample.getUpdate()) + .withCreate(mongoSample.getCreate()) + .withSubmitted(mongoSample.getSubmitted()) + .withAttributes(mongoSample.getAttributes()) + .withRelationships(relationships) + .withData(mongoSample.getData()) + .withExternalReferences(externalReferences) + .withOrganizations(mongoSample.getOrganizations()) + .withContacts(mongoSample.getContacts()) + .withPublications(mongoSample.getPublications()) + .withCertificates(certificates) + .withSubmittedVia(mongoSample.getSubmittedVia()) + .build(); + } + + final Instant reviewed = mongoSample.getReviewed(); + + if (reviewed == null) { + return Sample.Builder.fromSample(convertedSample).withNoReviewed().build(); + } else { + return Sample.Builder.fromSample(convertedSample) + .withReviewed(mongoSample.getReviewed()) + .build(); + } + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoStructuredDataToStructuredDataConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoStructuredDataToStructuredDataConverter.java similarity index 93% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoStructuredDataToStructuredDataConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoStructuredDataToStructuredDataConverter.java index 80020a5df..d50d0128f 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoStructuredDataToStructuredDataConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/MongoStructuredDataToStructuredDataConverter.java @@ -1,29 +1,29 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.structured.StructuredData; -import uk.ac.ebi.biosamples.mongo.model.MongoStructuredData; - -@Service -public class MongoStructuredDataToStructuredDataConverter - implements Converter { - @Override - public StructuredData convert(MongoStructuredData mongoStructuredData) { - return StructuredData.build( - mongoStructuredData.getAccession(), - mongoStructuredData.getCreate(), - mongoStructuredData.getUpdate(), - mongoStructuredData.getData()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; +import uk.ac.ebi.biosamples.mongo.model.MongoStructuredData; + +@Service +public class MongoStructuredDataToStructuredDataConverter + implements Converter { + @Override + public StructuredData convert(MongoStructuredData mongoStructuredData) { + return StructuredData.build( + mongoStructuredData.getAccession(), + mongoStructuredData.getCreate(), + mongoStructuredData.getUpdate(), + mongoStructuredData.getData()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/RelationshipToMongoRelationshipConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/RelationshipToMongoRelationshipConverter.java similarity index 93% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/RelationshipToMongoRelationshipConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/RelationshipToMongoRelationshipConverter.java index 97339d6b1..7a59d3b9d 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/RelationshipToMongoRelationshipConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/RelationshipToMongoRelationshipConverter.java @@ -1,26 +1,26 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; - -@Service -public class RelationshipToMongoRelationshipConverter - implements Converter { - @Override - public MongoRelationship convert(Relationship relationship) { - return MongoRelationship.build( - relationship.getSource(), relationship.getType(), relationship.getTarget()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; + +@Service +public class RelationshipToMongoRelationshipConverter + implements Converter { + @Override + public MongoRelationship convert(Relationship relationship) { + return MongoRelationship.build( + relationship.getSource(), relationship.getType(), relationship.getTarget()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleReadService.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleReadService.java similarity index 95% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleReadService.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleReadService.java index 2a34a3d53..06ef2a42f 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleReadService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleReadService.java @@ -1,159 +1,159 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicReference; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.BioSamplesConstants; -import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.structured.StructuredData; -import uk.ac.ebi.biosamples.mongo.model.MongoSample; -import uk.ac.ebi.biosamples.mongo.model.MongoStructuredData; -import uk.ac.ebi.biosamples.mongo.repository.MongoSampleRepository; -import uk.ac.ebi.biosamples.mongo.repository.MongoStructuredDataRepository; -import uk.ac.ebi.biosamples.utils.AdaptiveThreadPoolExecutor; - -/** - * Service layer business logic for centralising repository access and conversions between different - * controller. Use this instead of linking to repositories directly. - * - * @author faulcon - */ -@Service -public class SampleReadService { - private static final Logger LOGGER = LoggerFactory.getLogger(SampleReadService.class); - private final MongoSampleRepository mongoSampleRepository; - private final MongoSampleToSampleConverter mongoSampleToSampleConverter; - private final CurationReadService curationReadService; - private final MongoInverseRelationshipService mongoInverseRelationshipService; - private final MongoStructuredDataRepository mongoStructuredDataRepository; - private final MongoStructuredDataToStructuredDataConverter - mongoStructuredDataToStructuredDataConverter; - private final ExecutorService executorService; - - public SampleReadService( - final MongoSampleRepository mongoSampleRepository, - final MongoSampleToSampleConverter mongoSampleToSampleConverter, - final CurationReadService curationReadService, - final MongoInverseRelationshipService mongoInverseRelationshipService, - final MongoStructuredDataRepository mongoStructuredDataRepository, - final MongoStructuredDataToStructuredDataConverter - mongoStructuredDataToStructuredDataConverter, - final BioSamplesProperties bioSamplesProperties) { - this.mongoSampleRepository = mongoSampleRepository; - this.mongoSampleToSampleConverter = mongoSampleToSampleConverter; - this.curationReadService = curationReadService; - this.mongoInverseRelationshipService = mongoInverseRelationshipService; - this.mongoStructuredDataRepository = mongoStructuredDataRepository; - this.mongoStructuredDataToStructuredDataConverter = - mongoStructuredDataToStructuredDataConverter; - executorService = - AdaptiveThreadPoolExecutor.create( - 10000, - 1000, - false, - bioSamplesProperties.getBiosamplesCorePageThreadCount(), - bioSamplesProperties.getBiosamplesCorePageThreadCountMax()); - } - - public Optional fetch(final String accession, final boolean applyCurations) { - final MongoSample mongoSample = getMongoSample(accession); - - if (mongoSample == null) { - LOGGER.warn(String.format("Failed to retrieve sample with accession %s", accession)); - - return Optional.empty(); - } - - final MongoSample updatedMongoSample = - mongoInverseRelationshipService.addInverseRelationships(mongoSample); - final AtomicReference sample = - new AtomicReference<>(mongoSampleToSampleConverter.apply(updatedMongoSample)); - - if (applyCurations) { - sample.set(curationReadService.applyAllCurationToSample(sample.get())); - } - - final Optional mongoStructuredData = - mongoStructuredDataRepository.findById(accession); - - mongoStructuredData.ifPresent( - data -> { - final StructuredData structuredData = - mongoStructuredDataToStructuredDataConverter.convert(data); - sample.set( - Sample.Builder.fromSample(sample.get()) - .withStructuredData(structuredData.getData()) - .build()); - }); - - return Optional.of(sample.get()); - } - - private MongoSample getMongoSample(final String accession) { - if (startsWithSraPrefix(accession)) { - final List samples = /*mongoSampleRepository.findBySraAccession(accession);*/ - mongoSampleRepository.findUsingSraAccessionIndex(accession); - - /*if (samples == null || samples.isEmpty()) { - samples = mongoSampleRepository.findUsingSraAccessionIndex(accession); - }*/ - - return samples.isEmpty() ? null : samples.get(0); - } else { - return mongoSampleRepository.findById(accession).orElse(null); - } - } - - private boolean startsWithSraPrefix(final String accession) { - return Arrays.stream(BioSamplesConstants.sraSampleAccessionPrefixesString) - .anyMatch(accession::startsWith); - } - - public Future> fetchAsync(final String accession, final boolean applyCurations) { - return executorService.submit(new FetchCallable(accession, this, applyCurations)); - } - - private static class FetchCallable implements Callable> { - private final SampleReadService sampleReadService; - private final String accession; - private final boolean applyCurations; - - FetchCallable( - final String accession, - final SampleReadService sampleReadService, - final boolean applyCurations) { - this.accession = accession; - this.sampleReadService = sampleReadService; - this.applyCurations = applyCurations; - } - - @Override - public Optional call() { - final Optional opt = sampleReadService.fetch(accession, applyCurations); - - if (!opt.isPresent()) { - LOGGER.warn(String.format("failed to retrieve sample with accession %s", accession)); - } - - return opt; - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.BioSamplesConstants; +import uk.ac.ebi.biosamples.BioSamplesProperties; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; +import uk.ac.ebi.biosamples.mongo.model.MongoSample; +import uk.ac.ebi.biosamples.mongo.model.MongoStructuredData; +import uk.ac.ebi.biosamples.mongo.repository.MongoSampleRepository; +import uk.ac.ebi.biosamples.mongo.repository.MongoStructuredDataRepository; +import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; + +/** + * Service layer business logic for centralising repository access and conversions between different + * controller. Use this instead of linking to repositories directly. + * + * @author faulcon + */ +@Service +public class SampleReadService { + private static final Logger LOGGER = LoggerFactory.getLogger(SampleReadService.class); + private final MongoSampleRepository mongoSampleRepository; + private final MongoSampleToSampleConverter mongoSampleToSampleConverter; + private final CurationReadService curationReadService; + private final MongoInverseRelationshipService mongoInverseRelationshipService; + private final MongoStructuredDataRepository mongoStructuredDataRepository; + private final MongoStructuredDataToStructuredDataConverter + mongoStructuredDataToStructuredDataConverter; + private final ExecutorService executorService; + + public SampleReadService( + final MongoSampleRepository mongoSampleRepository, + final MongoSampleToSampleConverter mongoSampleToSampleConverter, + final CurationReadService curationReadService, + final MongoInverseRelationshipService mongoInverseRelationshipService, + final MongoStructuredDataRepository mongoStructuredDataRepository, + final MongoStructuredDataToStructuredDataConverter + mongoStructuredDataToStructuredDataConverter, + final BioSamplesProperties bioSamplesProperties) { + this.mongoSampleRepository = mongoSampleRepository; + this.mongoSampleToSampleConverter = mongoSampleToSampleConverter; + this.curationReadService = curationReadService; + this.mongoInverseRelationshipService = mongoInverseRelationshipService; + this.mongoStructuredDataRepository = mongoStructuredDataRepository; + this.mongoStructuredDataToStructuredDataConverter = + mongoStructuredDataToStructuredDataConverter; + executorService = + AdaptiveThreadPoolExecutor.create( + 10000, + 1000, + false, + bioSamplesProperties.getBiosamplesCorePageThreadCount(), + bioSamplesProperties.getBiosamplesCorePageThreadCountMax()); + } + + public Optional fetch(final String accession, final boolean applyCurations) { + final MongoSample mongoSample = getMongoSample(accession); + + if (mongoSample == null) { + LOGGER.warn(String.format("Failed to retrieve sample with accession %s", accession)); + + return Optional.empty(); + } + + final MongoSample updatedMongoSample = + mongoInverseRelationshipService.addInverseRelationships(mongoSample); + final AtomicReference sample = + new AtomicReference<>(mongoSampleToSampleConverter.apply(updatedMongoSample)); + + if (applyCurations) { + sample.set(curationReadService.applyAllCurationToSample(sample.get())); + } + + final Optional mongoStructuredData = + mongoStructuredDataRepository.findById(accession); + + mongoStructuredData.ifPresent( + data -> { + final StructuredData structuredData = + mongoStructuredDataToStructuredDataConverter.convert(data); + sample.set( + Sample.Builder.fromSample(sample.get()) + .withStructuredData(structuredData.getData()) + .build()); + }); + + return Optional.of(sample.get()); + } + + private MongoSample getMongoSample(final String accession) { + if (startsWithSraPrefix(accession)) { + final List samples = /*mongoSampleRepository.findBySraAccession(accession);*/ + mongoSampleRepository.findUsingSraAccessionIndex(accession); + + /*if (samples == null || samples.isEmpty()) { + samples = mongoSampleRepository.findUsingSraAccessionIndex(accession); + }*/ + + return samples.isEmpty() ? null : samples.get(0); + } else { + return mongoSampleRepository.findById(accession).orElse(null); + } + } + + private boolean startsWithSraPrefix(final String accession) { + return Arrays.stream(BioSamplesConstants.sraSampleAccessionPrefixesString) + .anyMatch(accession::startsWith); + } + + public Future> fetchAsync(final String accession, final boolean applyCurations) { + return executorService.submit(new FetchCallable(accession, this, applyCurations)); + } + + private static class FetchCallable implements Callable> { + private final SampleReadService sampleReadService; + private final String accession; + private final boolean applyCurations; + + FetchCallable( + final String accession, + final SampleReadService sampleReadService, + final boolean applyCurations) { + this.accession = accession; + this.sampleReadService = sampleReadService; + this.applyCurations = applyCurations; + } + + @Override + public Optional call() { + final Optional opt = sampleReadService.fetch(accession, applyCurations); + + if (!opt.isPresent()) { + LOGGER.warn(String.format("failed to retrieve sample with accession %s", accession)); + } + + return opt; + } + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleToMongoSampleConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleToMongoSampleConverter.java similarity index 92% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleToMongoSampleConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleToMongoSampleConverter.java index bf8084c6e..8027fdead 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleToMongoSampleConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/SampleToMongoSampleConverter.java @@ -1,85 +1,85 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import java.util.SortedSet; -import java.util.TreeSet; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Certificate; -import uk.ac.ebi.biosamples.model.ExternalReference; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.mongo.model.MongoCertificate; -import uk.ac.ebi.biosamples.mongo.model.MongoExternalReference; -import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; -import uk.ac.ebi.biosamples.mongo.model.MongoSample; - -@Service -public class SampleToMongoSampleConverter implements Converter { - @Autowired - private ExternalReferenceToMongoExternalReferenceConverter - externalReferenceToMongoExternalReferenceConverter; - - @Autowired - private RelationshipToMongoRelationshipConverter relationshipToMongoRelationshipConverter; - - @Autowired private CertificateToMongoCertificateConverter certificateToMongoCertificateConverter; - - @Override - public MongoSample convert(final Sample sample) { - final SortedSet externalReferences = new TreeSet<>(); - for (final ExternalReference mongoExternalReference : sample.getExternalReferences()) { - externalReferences.add( - externalReferenceToMongoExternalReferenceConverter.convert(mongoExternalReference)); - } - - final SortedSet relationships = new TreeSet<>(); - for (final Relationship relationship : sample.getRelationships()) { - relationships.add(relationshipToMongoRelationshipConverter.convert(relationship)); - } - - final SortedSet certificates = new TreeSet<>(); - - for (final Certificate certificate : sample.getCertificates()) { - certificates.add(certificateToMongoCertificateConverter.convert(certificate)); - } - - // when we convert to a MongoSample then the Sample *must* have a domain or a Webin ID - if (sample.getDomain() == null && sample.getWebinSubmissionAccountId() == null) { - throw new RuntimeException("Sample does not have domain or a WEBIN submission account ID"); - } - - return MongoSample.build( - sample.getName(), - sample.getAccession(), - sample.getSraAccession(), - sample.getDomain(), - sample.getWebinSubmissionAccountId(), - sample.getTaxId(), - sample.getStatus(), - sample.getRelease(), - sample.getUpdate(), - sample.getCreate(), - sample.getSubmitted(), - sample.getReviewed(), - sample.getCharacteristics(), - sample.getData(), - relationships, - externalReferences, - sample.getOrganizations(), - sample.getContacts(), - sample.getPublications(), - certificates, - sample.getSubmittedVia()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import java.util.SortedSet; +import java.util.TreeSet; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Certificate; +import uk.ac.ebi.biosamples.core.model.ExternalReference; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.mongo.model.MongoCertificate; +import uk.ac.ebi.biosamples.mongo.model.MongoExternalReference; +import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; +import uk.ac.ebi.biosamples.mongo.model.MongoSample; + +@Service +public class SampleToMongoSampleConverter implements Converter { + @Autowired + private ExternalReferenceToMongoExternalReferenceConverter + externalReferenceToMongoExternalReferenceConverter; + + @Autowired + private RelationshipToMongoRelationshipConverter relationshipToMongoRelationshipConverter; + + @Autowired private CertificateToMongoCertificateConverter certificateToMongoCertificateConverter; + + @Override + public MongoSample convert(final Sample sample) { + final SortedSet externalReferences = new TreeSet<>(); + for (final ExternalReference mongoExternalReference : sample.getExternalReferences()) { + externalReferences.add( + externalReferenceToMongoExternalReferenceConverter.convert(mongoExternalReference)); + } + + final SortedSet relationships = new TreeSet<>(); + for (final Relationship relationship : sample.getRelationships()) { + relationships.add(relationshipToMongoRelationshipConverter.convert(relationship)); + } + + final SortedSet certificates = new TreeSet<>(); + + for (final Certificate certificate : sample.getCertificates()) { + certificates.add(certificateToMongoCertificateConverter.convert(certificate)); + } + + // when we convert to a MongoSample then the Sample *must* have a domain or a Webin ID + if (sample.getDomain() == null && sample.getWebinSubmissionAccountId() == null) { + throw new RuntimeException("Sample does not have domain or a WEBIN submission account ID"); + } + + return MongoSample.build( + sample.getName(), + sample.getAccession(), + sample.getSraAccession(), + sample.getDomain(), + sample.getWebinSubmissionAccountId(), + sample.getTaxId(), + sample.getStatus(), + sample.getRelease(), + sample.getUpdate(), + sample.getCreate(), + sample.getSubmitted(), + sample.getReviewed(), + sample.getCharacteristics(), + sample.getData(), + relationships, + externalReferences, + sample.getOrganizations(), + sample.getContacts(), + sample.getPublications(), + certificates, + sample.getSubmittedVia()); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/StructuredDataToMongoStructuredDataConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/StructuredDataToMongoStructuredDataConverter.java similarity index 92% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/StructuredDataToMongoStructuredDataConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/service/StructuredDataToMongoStructuredDataConverter.java index cc738865e..3e5dba3d4 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/service/StructuredDataToMongoStructuredDataConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/service/StructuredDataToMongoStructuredDataConverter.java @@ -1,26 +1,26 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.service; - -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.structured.StructuredData; -import uk.ac.ebi.biosamples.mongo.model.MongoStructuredData; - -@Service -public class StructuredDataToMongoStructuredDataConverter - implements Converter { - - @Override - public MongoStructuredData convert(StructuredData structuredData) { - return MongoStructuredData.build(structuredData); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.service; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; +import uk.ac.ebi.biosamples.mongo.model.MongoStructuredData; + +@Service +public class StructuredDataToMongoStructuredDataConverter + implements Converter { + + @Override + public MongoStructuredData convert(StructuredData structuredData) { + return MongoStructuredData.build(structuredData); + } +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/util/BioSamplesFileUploadSubmissionStatus.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/util/BioSamplesFileUploadSubmissionStatus.java similarity index 97% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/util/BioSamplesFileUploadSubmissionStatus.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/util/BioSamplesFileUploadSubmissionStatus.java index 1cdc9581f..fb1c38f21 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/util/BioSamplesFileUploadSubmissionStatus.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/util/BioSamplesFileUploadSubmissionStatus.java @@ -1,19 +1,19 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.util; - -public enum BioSamplesFileUploadSubmissionStatus { - ACTIVE, - FAILED, - COMPLETED, - NOT_FOUND, - COMPLETED_WITH_ERRORS -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.util; + +public enum BioSamplesFileUploadSubmissionStatus { + ACTIVE, + FAILED, + COMPLETED, + NOT_FOUND, + COMPLETED_WITH_ERRORS +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/util/PipelineCompletionStatus.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/util/PipelineCompletionStatus.java similarity index 97% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/util/PipelineCompletionStatus.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/util/PipelineCompletionStatus.java index 65a74160c..56484c45d 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/util/PipelineCompletionStatus.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/util/PipelineCompletionStatus.java @@ -1,16 +1,16 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.util; - -public enum PipelineCompletionStatus { - COMPLETED, - FAILED -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.util; + +public enum PipelineCompletionStatus { + COMPLETED, + FAILED +} diff --git a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/util/SampleNameAccessionPair.java b/core/src/main/java/uk/ac/ebi/biosamples/mongo/util/SampleNameAccessionPair.java similarity index 97% rename from models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/util/SampleNameAccessionPair.java rename to core/src/main/java/uk/ac/ebi/biosamples/mongo/util/SampleNameAccessionPair.java index 5d2a53917..8ee1aa619 100644 --- a/models/mongo/src/main/java/uk/ac/ebi/biosamples/mongo/util/SampleNameAccessionPair.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/mongo/util/SampleNameAccessionPair.java @@ -1,37 +1,37 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.mongo.util; - -public class SampleNameAccessionPair { - private String sampleName; - private String sampleAccession; - - public SampleNameAccessionPair(final String sampleName, final String sampleAccession) { - this.sampleName = sampleName; - this.sampleAccession = sampleAccession; - } - - public void setSampleName(String sampleName) { - this.sampleName = sampleName; - } - - public void setSampleAccession(String sampleAccession) { - this.sampleAccession = sampleAccession; - } - - public String getSampleName() { - return sampleName; - } - - public String getSampleAccession() { - return sampleAccession; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.mongo.util; + +public class SampleNameAccessionPair { + private String sampleName; + private String sampleAccession; + + public SampleNameAccessionPair(final String sampleName, final String sampleAccession) { + this.sampleName = sampleName; + this.sampleAccession = sampleAccession; + } + + public void setSampleName(String sampleName) { + this.sampleName = sampleName; + } + + public void setSampleAccession(String sampleAccession) { + this.sampleAccession = sampleAccession; + } + + public String getSampleName() { + return sampleName; + } + + public String getSampleAccession() { + return sampleAccession; + } +} diff --git a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/NeoProperties.java b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/NeoProperties.java similarity index 97% rename from models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/NeoProperties.java rename to core/src/main/java/uk/ac/ebi/biosamples/neo4j/NeoProperties.java index 1e8054d5a..487c534df 100644 --- a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/NeoProperties.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/NeoProperties.java @@ -1,28 +1,28 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.neo4j; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Data -@Component -public class NeoProperties { - @Value("${biosamples.neo.url:neo4j://localhost:7687}") - private String neoUrl; - - @Value("${biosamples.neo.username:neo4j}") - private String neoUsername; - - @Value("${biosamples.neo.password:neo5j}") - private String neoPassword; -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.neo4j; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class NeoProperties { + @Value("${biosamples.neo.url:neo4j://localhost:7687}") + private String neoUrl; + + @Value("${biosamples.neo.username:neo4j}") + private String neoUsername; + + @Value("${biosamples.neo.password:neo5j}") + private String neoPassword; +} diff --git a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/CypherQuery.java b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/CypherQuery.java similarity index 96% rename from models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/CypherQuery.java rename to core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/CypherQuery.java index cbb25e49c..ce32f3a0c 100644 --- a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/CypherQuery.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/CypherQuery.java @@ -1,35 +1,35 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.neo4j.model; - -import java.util.List; -import java.util.Map; - -public class CypherQuery { - private String query; - private List> response; - - public String getQuery() { - return query; - } - - public void setQuery(String query) { - this.query = query; - } - - public List> getResponse() { - return response; - } - - public void setResponse(List> response) { - this.response = response; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.neo4j.model; + +import java.util.List; +import java.util.Map; + +public class CypherQuery { + private String query; + private List> response; + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public List> getResponse() { + return response; + } + + public void setResponse(List> response) { + this.response = response; + } +} diff --git a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphLink.java b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphLink.java similarity index 94% rename from models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphLink.java rename to core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphLink.java index 151029875..8dc947f1a 100644 --- a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphLink.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphLink.java @@ -1,78 +1,78 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.neo4j.model; - -import java.util.Objects; -import uk.ac.ebi.biosamples.model.RelationshipType; - -public class GraphLink implements Comparable { - private RelationshipType type; - private String startNode; - private String endNode; - - public RelationshipType getType() { - return type; - } - - public void setType(final RelationshipType type) { - this.type = type; - } - - private String getStartNode() { - return startNode; - } - - public void setStartNode(final String startNode) { - this.startNode = startNode; - } - - private String getEndNode() { - return endNode; - } - - public void setEndNode(final String endNode) { - this.endNode = endNode; - } - - public String getQueryString(final String relName) { - final String rel = (type == RelationshipType.ANY) ? relName : relName + ":" + type; - return "(" + startNode + ")-[" + rel + "]->(" + endNode + ") "; - } - - @Override - public boolean equals(final Object other) { - if (!(other instanceof GraphLink)) { - return false; - } - final GraphLink otherLink = (GraphLink) other; - return type.equals(otherLink.getType()) - && startNode.equals(otherLink.getStartNode()) - && endNode.equals(otherLink.getEndNode()); - } - - @Override - public int hashCode() { - return Objects.hash(type, startNode, endNode); - } - - @Override - public int compareTo(final GraphLink other) { - if (!type.equals(other.getType())) { - return type.compareTo(other.getType()); - } - - if (!startNode.equals(other.getStartNode())) { - return startNode.compareTo(other.getStartNode()); - } - - return endNode.compareTo(other.getEndNode()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.neo4j.model; + +import java.util.Objects; +import uk.ac.ebi.biosamples.core.model.RelationshipType; + +public class GraphLink implements Comparable { + private RelationshipType type; + private String startNode; + private String endNode; + + public RelationshipType getType() { + return type; + } + + public void setType(final RelationshipType type) { + this.type = type; + } + + private String getStartNode() { + return startNode; + } + + public void setStartNode(final String startNode) { + this.startNode = startNode; + } + + private String getEndNode() { + return endNode; + } + + public void setEndNode(final String endNode) { + this.endNode = endNode; + } + + public String getQueryString(final String relName) { + final String rel = (type == RelationshipType.ANY) ? relName : relName + ":" + type; + return "(" + startNode + ")-[" + rel + "]->(" + endNode + ") "; + } + + @Override + public boolean equals(final Object other) { + if (!(other instanceof GraphLink)) { + return false; + } + final GraphLink otherLink = (GraphLink) other; + return type.equals(otherLink.getType()) + && startNode.equals(otherLink.getStartNode()) + && endNode.equals(otherLink.getEndNode()); + } + + @Override + public int hashCode() { + return Objects.hash(type, startNode, endNode); + } + + @Override + public int compareTo(final GraphLink other) { + if (!type.equals(other.getType())) { + return type.compareTo(other.getType()); + } + + if (!startNode.equals(other.getStartNode())) { + return startNode.compareTo(other.getStartNode()); + } + + return endNode.compareTo(other.getEndNode()); + } +} diff --git a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphNode.java b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphNode.java similarity index 96% rename from models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphNode.java rename to core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphNode.java index ff2382b13..92a414237 100644 --- a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphNode.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphNode.java @@ -1,96 +1,96 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.neo4j.model; - -import java.util.Map; -import java.util.Objects; -import java.util.StringJoiner; - -public class GraphNode implements Comparable { - private String id; - private String type; - private Map attributes; - - public GraphNode() { - // default constructor - } - - public GraphNode(Map nodeAsMap) { - type = String.valueOf(nodeAsMap.get("type")); - attributes = (Map) nodeAsMap.get("attributes"); - id = attributes.get("accession"); - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public Map getAttributes() { - return attributes; - } - - public void setAttributes(Map attributes) { - this.attributes = attributes; - } - - public String getQueryString() { - String queryString; - if (attributes != null) { - StringJoiner joiner = new StringJoiner(","); - for (Map.Entry e : attributes.entrySet()) { - joiner.add( - e.getKey().replaceAll("\\s+", "").toLowerCase() - + ":'" - + e.getValue().toLowerCase() - + "'"); - } - queryString = ":" + type + "{" + joiner.toString() + "}"; - } else { - queryString = ":" + type; - } - return queryString; - } - - @Override - public boolean equals(Object other) { - if (other instanceof GraphNode && this.type.equalsIgnoreCase(((GraphNode) other).getType())) { - return this.id.equals(((GraphNode) other).id); - } - - return false; - } - - @Override - public int hashCode() { - return Objects.hash(type.toUpperCase(), id); - } - - @Override - public int compareTo(GraphNode other) { - if (this.type.equalsIgnoreCase(other.getType())) { - return this.id.compareTo(other.getId()); - } else { - return this.type.compareTo(other.getType()); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.neo4j.model; + +import java.util.Map; +import java.util.Objects; +import java.util.StringJoiner; + +public class GraphNode implements Comparable { + private String id; + private String type; + private Map attributes; + + public GraphNode() { + // default constructor + } + + public GraphNode(Map nodeAsMap) { + type = String.valueOf(nodeAsMap.get("type")); + attributes = (Map) nodeAsMap.get("attributes"); + id = attributes.get("accession"); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Map getAttributes() { + return attributes; + } + + public void setAttributes(Map attributes) { + this.attributes = attributes; + } + + public String getQueryString() { + String queryString; + if (attributes != null) { + StringJoiner joiner = new StringJoiner(","); + for (Map.Entry e : attributes.entrySet()) { + joiner.add( + e.getKey().replaceAll("\\s+", "").toLowerCase() + + ":'" + + e.getValue().toLowerCase() + + "'"); + } + queryString = ":" + type + "{" + joiner.toString() + "}"; + } else { + queryString = ":" + type; + } + return queryString; + } + + @Override + public boolean equals(Object other) { + if (other instanceof GraphNode && this.type.equalsIgnoreCase(((GraphNode) other).getType())) { + return this.id.equals(((GraphNode) other).id); + } + + return false; + } + + @Override + public int hashCode() { + return Objects.hash(type.toUpperCase(), id); + } + + @Override + public int compareTo(GraphNode other) { + if (this.type.equalsIgnoreCase(other.getType())) { + return this.id.compareTo(other.getId()); + } else { + return this.type.compareTo(other.getType()); + } + } +} diff --git a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphRelationship.java b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphRelationship.java similarity index 92% rename from models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphRelationship.java rename to core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphRelationship.java index 5e5fcad60..fa2cef6fb 100644 --- a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphRelationship.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphRelationship.java @@ -1,53 +1,53 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.neo4j.model; - -import uk.ac.ebi.biosamples.model.RelationshipType; - -public class GraphRelationship { - private RelationshipType type; - private GraphNode startNode; - private GraphNode endNode; - - public RelationshipType getType() { - return type; - } - - public void setType(RelationshipType type) { - this.type = type; - } - - public GraphNode getStartNode() { - return startNode; - } - - public void setStartNode(GraphNode startNode) { - this.startNode = startNode; - } - - public GraphNode getEndNode() { - return endNode; - } - - public void setEndNode(GraphNode endNode) { - this.endNode = endNode; - } - - public String getQueryString() { - return "(a" - + startNode.getQueryString() - + ")-[r:" - + type - + "]-(b" - + endNode.getQueryString() - + ")"; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.neo4j.model; + +import uk.ac.ebi.biosamples.core.model.RelationshipType; + +public class GraphRelationship { + private RelationshipType type; + private GraphNode startNode; + private GraphNode endNode; + + public RelationshipType getType() { + return type; + } + + public void setType(RelationshipType type) { + this.type = type; + } + + public GraphNode getStartNode() { + return startNode; + } + + public void setStartNode(GraphNode startNode) { + this.startNode = startNode; + } + + public GraphNode getEndNode() { + return endNode; + } + + public void setEndNode(GraphNode endNode) { + this.endNode = endNode; + } + + public String getQueryString() { + return "(a" + + startNode.getQueryString() + + ")-[r:" + + type + + "]-(b" + + endNode.getQueryString() + + ")"; + } +} diff --git a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphSearchQuery.java b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphSearchQuery.java similarity index 96% rename from models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphSearchQuery.java rename to core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphSearchQuery.java index cc61c6f90..d66e4a07a 100644 --- a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphSearchQuery.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/GraphSearchQuery.java @@ -1,61 +1,61 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.neo4j.model; - -import java.util.Set; - -public class GraphSearchQuery { - private Set nodes; - private Set links; - private int page; - private int size; - private int totalElements; - - public Set getNodes() { - return nodes; - } - - public void setNodes(Set nodes) { - this.nodes = nodes; - } - - public Set getLinks() { - return links; - } - - public void setLinks(Set links) { - this.links = links; - } - - public int getPage() { - return page; - } - - public void setPage(int page) { - this.page = page; - } - - public int getSize() { - return size; - } - - public void setSize(int size) { - this.size = size; - } - - public int getTotalElements() { - return totalElements; - } - - public void setTotalElements(int totalElements) { - this.totalElements = totalElements; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.neo4j.model; + +import java.util.Set; + +public class GraphSearchQuery { + private Set nodes; + private Set links; + private int page; + private int size; + private int totalElements; + + public Set getNodes() { + return nodes; + } + + public void setNodes(Set nodes) { + this.nodes = nodes; + } + + public Set getLinks() { + return links; + } + + public void setLinks(Set links) { + this.links = links; + } + + public int getPage() { + return page; + } + + public void setPage(int page) { + this.page = page; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public int getTotalElements() { + return totalElements; + } + + public void setTotalElements(int totalElements) { + this.totalElements = totalElements; + } +} diff --git a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoExternalEntity.java b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoExternalEntity.java similarity index 90% rename from models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoExternalEntity.java rename to core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoExternalEntity.java index 46de68928..3f9a49caa 100644 --- a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoExternalEntity.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoExternalEntity.java @@ -1,47 +1,47 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.neo4j.model; - -import uk.ac.ebi.biosamples.model.ExternalReference; -import uk.ac.ebi.biosamples.service.ExternalReferenceUtils; - -public class NeoExternalEntity { - private String url; - private String archive; - private String ref; - - private NeoExternalEntity(String url, String archive, String ref) { - this.url = url; - this.archive = archive; - this.ref = ref; - } - - public String getUrl() { - return url; - } - - public String getArchive() { - return archive; - } - - public String getRef() { - return ref; - } - - public static NeoExternalEntity build(ExternalReference reference) { - String externalRef = ExternalReferenceUtils.getNickname(reference).toLowerCase(); - return new NeoExternalEntity( - reference.getUrl(), - externalRef.startsWith("ega") ? "ega" : externalRef, - ExternalReferenceUtils.getDataId(reference) - .orElse(String.valueOf(Math.abs(reference.getUrl().hashCode())))); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.neo4j.model; + +import uk.ac.ebi.biosamples.core.model.ExternalReference; +import uk.ac.ebi.biosamples.core.service.ExternalReferenceUtils; + +public class NeoExternalEntity { + private String url; + private String archive; + private String ref; + + private NeoExternalEntity(String url, String archive, String ref) { + this.url = url; + this.archive = archive; + this.ref = ref; + } + + public String getUrl() { + return url; + } + + public String getArchive() { + return archive; + } + + public String getRef() { + return ref; + } + + public static NeoExternalEntity build(ExternalReference reference) { + String externalRef = ExternalReferenceUtils.getNickname(reference).toLowerCase(); + return new NeoExternalEntity( + reference.getUrl(), + externalRef.startsWith("ega") ? "ega" : externalRef, + ExternalReferenceUtils.getDataId(reference) + .orElse(String.valueOf(Math.abs(reference.getUrl().hashCode())))); + } +} diff --git a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoRelationship.java b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoRelationship.java similarity index 90% rename from models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoRelationship.java rename to core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoRelationship.java index 73dc8b8f0..afecb4790 100644 --- a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoRelationship.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoRelationship.java @@ -1,45 +1,45 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.neo4j.model; - -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.RelationshipType; - -public class NeoRelationship { - private RelationshipType type; - private String source; - private String target; - - private NeoRelationship(RelationshipType type, String source, String target) { - this.type = type; - this.source = source; - this.target = target; - } - - public RelationshipType getType() { - return type; - } - - public String getSource() { - return source; - } - - public String getTarget() { - return target; - } - - public static NeoRelationship build(Relationship relationship) { - return new NeoRelationship( - RelationshipType.getType(relationship.getType()), - relationship.getSource(), - relationship.getTarget()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.neo4j.model; + +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.RelationshipType; + +public class NeoRelationship { + private RelationshipType type; + private String source; + private String target; + + private NeoRelationship(RelationshipType type, String source, String target) { + this.type = type; + this.source = source; + this.target = target; + } + + public RelationshipType getType() { + return type; + } + + public String getSource() { + return source; + } + + public String getTarget() { + return target; + } + + public static NeoRelationship build(Relationship relationship) { + return new NeoRelationship( + RelationshipType.getType(relationship.getType()), + relationship.getSource(), + relationship.getTarget()); + } +} diff --git a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoSample.java b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoSample.java similarity index 92% rename from models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoSample.java rename to core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoSample.java index 1af5e10ef..5fbb1d55e 100644 --- a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoSample.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/model/NeoSample.java @@ -1,128 +1,125 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.neo4j.model; - -import java.util.ArrayList; -import java.util.List; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.ExternalReference; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; - -public class NeoSample { - private final String accession; - private String name; - private String organism; - private String taxId; - private String sex; - private String cellType; - private String material; - private String project; - private String cellLine; - private String organismPart; - - private final List relationships; - private final List externalRefs; - - private NeoSample(final String accession) { - this.accession = accession; - relationships = new ArrayList<>(); - externalRefs = new ArrayList<>(); - } - - public String getAccession() { - return accession; - } - - public String getName() { - return name; - } - - public String getOrganism() { - return organism; - } - - public String getTaxId() { - return taxId; - } - - public String getSex() { - return sex; - } - - public String getCellType() { - return cellType; - } - - public String getMaterial() { - return material; - } - - public String getProject() { - return project; - } - - public String getCellLine() { - return cellLine; - } - - public String getOrganismPart() { - return organismPart; - } - - public List getRelationships() { - return relationships; - } - - public List getExternalRefs() { - return externalRefs; - } - - public static NeoSample build(final Sample sample) { - final NeoSample neoSample = new NeoSample(sample.getAccession()); - neoSample.name = sample.getName(); - neoSample.taxId = String.valueOf(sample.getTaxId()); - - for (final Attribute attribute : sample.getAttributes()) { - if ("organism".equalsIgnoreCase(attribute.getType())) { - neoSample.organism = attribute.getValue().toLowerCase(); - } else if ("sex".equalsIgnoreCase(attribute.getType())) { - neoSample.sex = attribute.getValue().toLowerCase(); - } else if ("cellType".equalsIgnoreCase(attribute.getType().replaceAll("\\s+", ""))) { - neoSample.cellType = attribute.getValue().toLowerCase(); - } else if ("material".equalsIgnoreCase(attribute.getType())) { - neoSample.material = attribute.getValue().toLowerCase(); - } else if ("project".equalsIgnoreCase(attribute.getType())) { - neoSample.project = attribute.getValue().toLowerCase(); - } else if ("cellLine".equalsIgnoreCase(attribute.getType().replaceAll("\\s+", ""))) { - neoSample.cellLine = attribute.getValue().toLowerCase(); - } else if ("organismPart".equalsIgnoreCase(attribute.getType().replaceAll("\\s+", ""))) { - neoSample.organismPart = attribute.getValue().toLowerCase(); - } - } - - for (final Relationship relationship : sample.getRelationships()) { - if (relationship.getSource().equals(neoSample.accession)) { - neoSample.relationships.add(NeoRelationship.build(relationship)); - } - } - - for (final ExternalReference ref : sample.getExternalReferences()) { - neoSample.externalRefs.add(NeoExternalEntity.build(ref)); - } - - return neoSample; - } - - public static NeoSample build(final String accession) { - return new NeoSample(accession); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.neo4j.model; + +import java.util.ArrayList; +import java.util.List; +import uk.ac.ebi.biosamples.core.model.*; + +public class NeoSample { + private final String accession; + private String name; + private String organism; + private String taxId; + private String sex; + private String cellType; + private String material; + private String project; + private String cellLine; + private String organismPart; + + private final List relationships; + private final List externalRefs; + + private NeoSample(final String accession) { + this.accession = accession; + relationships = new ArrayList<>(); + externalRefs = new ArrayList<>(); + } + + public String getAccession() { + return accession; + } + + public String getName() { + return name; + } + + public String getOrganism() { + return organism; + } + + public String getTaxId() { + return taxId; + } + + public String getSex() { + return sex; + } + + public String getCellType() { + return cellType; + } + + public String getMaterial() { + return material; + } + + public String getProject() { + return project; + } + + public String getCellLine() { + return cellLine; + } + + public String getOrganismPart() { + return organismPart; + } + + public List getRelationships() { + return relationships; + } + + public List getExternalRefs() { + return externalRefs; + } + + public static NeoSample build(final Sample sample) { + final NeoSample neoSample = new NeoSample(sample.getAccession()); + neoSample.name = sample.getName(); + neoSample.taxId = String.valueOf(sample.getTaxId()); + + for (final Attribute attribute : sample.getAttributes()) { + if ("organism".equalsIgnoreCase(attribute.getType())) { + neoSample.organism = attribute.getValue().toLowerCase(); + } else if ("sex".equalsIgnoreCase(attribute.getType())) { + neoSample.sex = attribute.getValue().toLowerCase(); + } else if ("cellType".equalsIgnoreCase(attribute.getType().replaceAll("\\s+", ""))) { + neoSample.cellType = attribute.getValue().toLowerCase(); + } else if ("material".equalsIgnoreCase(attribute.getType())) { + neoSample.material = attribute.getValue().toLowerCase(); + } else if ("project".equalsIgnoreCase(attribute.getType())) { + neoSample.project = attribute.getValue().toLowerCase(); + } else if ("cellLine".equalsIgnoreCase(attribute.getType().replaceAll("\\s+", ""))) { + neoSample.cellLine = attribute.getValue().toLowerCase(); + } else if ("organismPart".equalsIgnoreCase(attribute.getType().replaceAll("\\s+", ""))) { + neoSample.organismPart = attribute.getValue().toLowerCase(); + } + } + + for (final Relationship relationship : sample.getRelationships()) { + if (relationship.getSource().equals(neoSample.accession)) { + neoSample.relationships.add(NeoRelationship.build(relationship)); + } + } + + for (final ExternalReference ref : sample.getExternalReferences()) { + neoSample.externalRefs.add(NeoExternalEntity.build(ref)); + } + + return neoSample; + } + + public static NeoSample build(final String accession) { + return new NeoSample(accession); + } +} diff --git a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/repo/NeoSampleRepository.java b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/repo/NeoSampleRepository.java similarity index 96% rename from models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/repo/NeoSampleRepository.java rename to core/src/main/java/uk/ac/ebi/biosamples/neo4j/repo/NeoSampleRepository.java index 6dbad00e1..40a8754c6 100644 --- a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/repo/NeoSampleRepository.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/repo/NeoSampleRepository.java @@ -1,262 +1,262 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.neo4j.repo; - -import java.util.*; -import org.neo4j.driver.*; -import org.neo4j.driver.types.Node; -import org.neo4j.driver.types.Relationship; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.RelationshipType; -import uk.ac.ebi.biosamples.neo4j.NeoProperties; -import uk.ac.ebi.biosamples.neo4j.model.*; - -@Component -public class NeoSampleRepository implements AutoCloseable { - private static final Logger LOG = LoggerFactory.getLogger(NeoSampleRepository.class); - - private final Driver driver; - - public NeoSampleRepository(final NeoProperties neoProperties) { - driver = - GraphDatabase.driver( - neoProperties.getNeoUrl(), - AuthTokens.basic(neoProperties.getNeoUsername(), neoProperties.getNeoPassword())); - } - - @Override - public void close() throws Exception { - driver.close(); - } - - public List> executeCypher(final String cypherQuery) { - List> resultList; - try (final Session session = driver.session()) { - final Result result = session.run(cypherQuery); - resultList = result.list(r -> r.asMap(NeoSampleRepository::convert)); - } catch (final Exception e) { - resultList = new ArrayList<>(); - } - - return resultList; - } - - private static Object convert(final Value value) { - switch (value.type().name()) { - case "PATH": - return value.asList(NeoSampleRepository::convert); - case "NODE": - case "RELATIONSHIP": - return value.asMap(); - } - return value.asObject(); - } - - public GraphSearchQuery graphSearch( - final GraphSearchQuery searchQuery, final int limit, final int page) { - final int skip = (page - 1) * limit; - final StringBuilder query = new StringBuilder(); - final StringJoiner idJoiner = new StringJoiner(","); - for (final GraphNode node : searchQuery.getNodes()) { - query.append("MATCH (").append(node.getId()).append(node.getQueryString()).append(") "); - idJoiner.add(node.getId()); - } - - int relCount = 0; - for (final GraphLink link : searchQuery.getLinks()) { - relCount++; - final String relName = "r" + relCount; - query.append("MATCH ").append(link.getQueryString(relName)); - idJoiner.add(relName); - } - - if (query.length() == 0) { - query.append("MATCH(a1:Sample) "); - idJoiner.add("a1"); - } - - final StringBuilder countQuery = new StringBuilder(query.toString()).append(" RETURN COUNT(*)"); - query.append(" RETURN ").append(idJoiner.toString()); - query - .append(" ORDER BY ") - .append(idJoiner.toString().contains("a1") ? "a1" : "a2") - .append(".accession SKIP ") - .append(skip) - .append(" LIMIT ") - .append(limit); - - final GraphSearchQuery response = new GraphSearchQuery(); - response.setPage(page); - response.setSize(limit); - try (final Session session = driver.session()) { - LOG.info("Graph query: {}", query); - final Result countResult = session.run(countQuery.toString()); - final int totalElements = countResult.single().get(0).asInt(); - response.setTotalElements(totalElements); - - final Result result = session.run(query.toString()); - final Set responseNodes = new HashSet<>(); - final Set responseLinks = new HashSet<>(); - response.setNodes(responseNodes); - response.setLinks(responseLinks); - - while (result.hasNext()) { - final org.neo4j.driver.Record record = result.next(); - for (final Value value : record.values()) { - addToResponse(value, responseNodes, responseLinks); - } - } - } catch (final Exception e) { - LOG.error("Failed to load graph search results", e); - } - - return response; - } - - private void addToResponse( - final Value value, final Set responseNodes, final Set responseLinks) { - switch (value.type().name()) { - case "PATH": - // todo handle PATH type - LOG.warn("not handled yet"); - break; - case "NODE": - final Node internalNode = value.asNode(); - final GraphNode node = new GraphNode(); - node.setType(internalNode.labels().iterator().next()); - node.setAttributes((Map) internalNode.asMap()); - node.setId(String.valueOf(internalNode.id())); - responseNodes.add(node); - break; - case "RELATIONSHIP": - final Relationship internalRel = value.asRelationship(); - final GraphLink link = new GraphLink(); - link.setType(RelationshipType.getType(internalRel.type())); - link.setStartNode(String.valueOf(internalRel.startNodeId())); - link.setEndNode(String.valueOf(internalRel.endNodeId())); - responseLinks.add(link); - break; - default: - LOG.warn("Invalid neo4j value type: {}", value.type().name()); - break; - } - } - - /** ********************************************************************* */ - public void loadSample(final NeoSample sample) { - try (final Session session = driver.session()) { - createSample(session, sample); - - for (final NeoRelationship relationship : sample.getRelationships()) { - createSampleRelationship(session, relationship); - } - - for (final NeoExternalEntity ref : sample.getExternalRefs()) { - createExternalRelationship(session, sample.getAccession(), ref); - } - } - } - - private void createSample(final Session session, final NeoSample sample) { - String query = - "MERGE (a:Sample{accession:$accession}) " + "SET a.name = $name, a.taxid = $taxid"; - final Map sampleBasicInfoMap = new HashMap<>(); - sampleBasicInfoMap.put("accession", sample.getAccession()); - sampleBasicInfoMap.put("name", sample.getName()); - sampleBasicInfoMap.put("taxid", sample.getTaxId()); - - final Map params = new HashMap<>(sampleBasicInfoMap); - - if (sample.getOrganism() != null) { - query = query + ", a.organism = $organism"; - params.put("organism", sample.getOrganism()); - } - - if (sample.getSex() != null) { - query = query + ", a.sex = $sex"; - params.put("sex", sample.getSex()); - } - - if (sample.getCellType() != null) { - query = query + ", a.celltype = $cellType"; - params.put("cellType", sample.getCellType()); - } - - if (sample.getMaterial() != null) { - query = query + ", a.material = $material"; - params.put("material", sample.getMaterial()); - } - - if (sample.getProject() != null) { - query = query + ", a.project = $project"; - params.put("project", sample.getProject()); - } - - if (sample.getCellLine() != null) { - query = query + ", a.cellline = $cellLine"; - params.put("cellLine", sample.getCellLine()); - } - - if (sample.getOrganismPart() != null) { - query = query + ", a.organismpart = $organismPart"; - params.put("organismPart", sample.getOrganismPart()); - } - - query = query + " RETURN a.accession"; - - session.run(query, params); - } - - private void createSampleRelationship(final Session session, final NeoRelationship relationship) { - final String query = - "MERGE (a:Sample {accession:$fromAccession}) " - + "MERGE (b:Sample {accession:$toAccession}) " - + "MERGE (a)-[r:" - + relationship.getType() - + "]->(b)"; - final Map params = new HashMap<>(); - params.put("fromAccession", relationship.getSource()); - params.put("toAccession", relationship.getTarget()); - - session.run(query, params); - } - - private void createExternalRelationship( - final Session session, final String accession, final NeoExternalEntity externalEntity) { - final String query = - "MERGE (a:Sample {accession:$accession}) " - + "MERGE (b:ExternalEntity {url:$url}) " - + "SET b.archive = $archive, b.ref = $ref " - + "MERGE (a)-[r:EXTERNAL_REFERENCE]->(b)"; - final Map params = new HashMap<>(); - params.put("accession", accession); - params.put("url", externalEntity.getUrl()); - params.put("archive", externalEntity.getArchive()); - params.put("ref", externalEntity.getRef()); - - session.run(query, params); - } - - public void createExternalEntity( - final Session session, final String archive, final String externalRef, final String url) { - final String query = - "MERGE (a:ExternalEntity{url:$url}) " - + "SET a.archive = $archive, a.externalRef = $externalRef"; - final Map params = new HashMap<>(); - params.put("url", url); - params.put("archive", archive); - params.put("externalRef", externalRef); - - session.run(query, params); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.neo4j.repo; + +import java.util.*; +import org.neo4j.driver.*; +import org.neo4j.driver.types.Node; +import org.neo4j.driver.types.Relationship; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import uk.ac.ebi.biosamples.core.model.RelationshipType; +import uk.ac.ebi.biosamples.neo4j.NeoProperties; +import uk.ac.ebi.biosamples.neo4j.model.*; + +@Component +public class NeoSampleRepository implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(NeoSampleRepository.class); + + private final Driver driver; + + public NeoSampleRepository(final NeoProperties neoProperties) { + driver = + GraphDatabase.driver( + neoProperties.getNeoUrl(), + AuthTokens.basic(neoProperties.getNeoUsername(), neoProperties.getNeoPassword())); + } + + @Override + public void close() throws Exception { + driver.close(); + } + + public List> executeCypher(final String cypherQuery) { + List> resultList; + try (final Session session = driver.session()) { + final Result result = session.run(cypherQuery); + resultList = result.list(r -> r.asMap(NeoSampleRepository::convert)); + } catch (final Exception e) { + resultList = new ArrayList<>(); + } + + return resultList; + } + + private static Object convert(final Value value) { + switch (value.type().name()) { + case "PATH": + return value.asList(NeoSampleRepository::convert); + case "NODE": + case "RELATIONSHIP": + return value.asMap(); + } + return value.asObject(); + } + + public GraphSearchQuery graphSearch( + final GraphSearchQuery searchQuery, final int limit, final int page) { + final int skip = (page - 1) * limit; + final StringBuilder query = new StringBuilder(); + final StringJoiner idJoiner = new StringJoiner(","); + for (final GraphNode node : searchQuery.getNodes()) { + query.append("MATCH (").append(node.getId()).append(node.getQueryString()).append(") "); + idJoiner.add(node.getId()); + } + + int relCount = 0; + for (final GraphLink link : searchQuery.getLinks()) { + relCount++; + final String relName = "r" + relCount; + query.append("MATCH ").append(link.getQueryString(relName)); + idJoiner.add(relName); + } + + if (query.length() == 0) { + query.append("MATCH(a1:Sample) "); + idJoiner.add("a1"); + } + + final StringBuilder countQuery = new StringBuilder(query.toString()).append(" RETURN COUNT(*)"); + query.append(" RETURN ").append(idJoiner.toString()); + query + .append(" ORDER BY ") + .append(idJoiner.toString().contains("a1") ? "a1" : "a2") + .append(".accession SKIP ") + .append(skip) + .append(" LIMIT ") + .append(limit); + + final GraphSearchQuery response = new GraphSearchQuery(); + response.setPage(page); + response.setSize(limit); + try (final Session session = driver.session()) { + LOG.info("Graph query: {}", query); + final Result countResult = session.run(countQuery.toString()); + final int totalElements = countResult.single().get(0).asInt(); + response.setTotalElements(totalElements); + + final Result result = session.run(query.toString()); + final Set responseNodes = new HashSet<>(); + final Set responseLinks = new HashSet<>(); + response.setNodes(responseNodes); + response.setLinks(responseLinks); + + while (result.hasNext()) { + final org.neo4j.driver.Record record = result.next(); + for (final Value value : record.values()) { + addToResponse(value, responseNodes, responseLinks); + } + } + } catch (final Exception e) { + LOG.error("Failed to load graph search results", e); + } + + return response; + } + + private void addToResponse( + final Value value, final Set responseNodes, final Set responseLinks) { + switch (value.type().name()) { + case "PATH": + // todo handle PATH type + LOG.warn("not handled yet"); + break; + case "NODE": + final Node internalNode = value.asNode(); + final GraphNode node = new GraphNode(); + node.setType(internalNode.labels().iterator().next()); + node.setAttributes((Map) internalNode.asMap()); + node.setId(String.valueOf(internalNode.id())); + responseNodes.add(node); + break; + case "RELATIONSHIP": + final Relationship internalRel = value.asRelationship(); + final GraphLink link = new GraphLink(); + link.setType(RelationshipType.getType(internalRel.type())); + link.setStartNode(String.valueOf(internalRel.startNodeId())); + link.setEndNode(String.valueOf(internalRel.endNodeId())); + responseLinks.add(link); + break; + default: + LOG.warn("Invalid neo4j value type: {}", value.type().name()); + break; + } + } + + /** ********************************************************************* */ + public void loadSample(final NeoSample sample) { + try (final Session session = driver.session()) { + createSample(session, sample); + + for (final NeoRelationship relationship : sample.getRelationships()) { + createSampleRelationship(session, relationship); + } + + for (final NeoExternalEntity ref : sample.getExternalRefs()) { + createExternalRelationship(session, sample.getAccession(), ref); + } + } + } + + private void createSample(final Session session, final NeoSample sample) { + String query = + "MERGE (a:Sample{accession:$accession}) " + "SET a.name = $name, a.taxid = $taxid"; + final Map sampleBasicInfoMap = new HashMap<>(); + sampleBasicInfoMap.put("accession", sample.getAccession()); + sampleBasicInfoMap.put("name", sample.getName()); + sampleBasicInfoMap.put("taxid", sample.getTaxId()); + + final Map params = new HashMap<>(sampleBasicInfoMap); + + if (sample.getOrganism() != null) { + query = query + ", a.organism = $organism"; + params.put("organism", sample.getOrganism()); + } + + if (sample.getSex() != null) { + query = query + ", a.sex = $sex"; + params.put("sex", sample.getSex()); + } + + if (sample.getCellType() != null) { + query = query + ", a.celltype = $cellType"; + params.put("cellType", sample.getCellType()); + } + + if (sample.getMaterial() != null) { + query = query + ", a.material = $material"; + params.put("material", sample.getMaterial()); + } + + if (sample.getProject() != null) { + query = query + ", a.project = $project"; + params.put("project", sample.getProject()); + } + + if (sample.getCellLine() != null) { + query = query + ", a.cellline = $cellLine"; + params.put("cellLine", sample.getCellLine()); + } + + if (sample.getOrganismPart() != null) { + query = query + ", a.organismpart = $organismPart"; + params.put("organismPart", sample.getOrganismPart()); + } + + query = query + " RETURN a.accession"; + + session.run(query, params); + } + + private void createSampleRelationship(final Session session, final NeoRelationship relationship) { + final String query = + "MERGE (a:Sample {accession:$fromAccession}) " + + "MERGE (b:Sample {accession:$toAccession}) " + + "MERGE (a)-[r:" + + relationship.getType() + + "]->(b)"; + final Map params = new HashMap<>(); + params.put("fromAccession", relationship.getSource()); + params.put("toAccession", relationship.getTarget()); + + session.run(query, params); + } + + private void createExternalRelationship( + final Session session, final String accession, final NeoExternalEntity externalEntity) { + final String query = + "MERGE (a:Sample {accession:$accession}) " + + "MERGE (b:ExternalEntity {url:$url}) " + + "SET b.archive = $archive, b.ref = $ref " + + "MERGE (a)-[r:EXTERNAL_REFERENCE]->(b)"; + final Map params = new HashMap<>(); + params.put("accession", accession); + params.put("url", externalEntity.getUrl()); + params.put("archive", externalEntity.getArchive()); + params.put("ref", externalEntity.getRef()); + + session.run(query, params); + } + + public void createExternalEntity( + final Session session, final String archive, final String externalRef, final String url) { + final String query = + "MERGE (a:ExternalEntity{url:$url}) " + + "SET a.archive = $archive, a.externalRef = $externalRef"; + final Map params = new HashMap<>(); + params.put("url", url); + params.put("archive", archive); + params.put("externalRef", externalRef); + + session.run(query, params); + } +} diff --git a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoRelationshipService.java b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoRelationshipService.java similarity index 98% rename from models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoRelationshipService.java rename to core/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoRelationshipService.java index 2eaffd39c..d5f07d6e3 100644 --- a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoRelationshipService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoRelationshipService.java @@ -1,13 +1,13 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.neo4j.service; - -public class NeoRelationshipService {} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.neo4j.service; + +public class NeoRelationshipService {} diff --git a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoSampleConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoSampleConverter.java similarity index 92% rename from models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoSampleConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoSampleConverter.java index 7bb7984a8..952eb390e 100644 --- a/models/neo4j/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoSampleConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/neo4j/service/NeoSampleConverter.java @@ -1,21 +1,21 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.neo4j.service; - -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.neo4j.model.NeoSample; - -public class NeoSampleConverter { - - public static Sample fromNeoSampleToSample(NeoSample neoSample) { - return null; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.neo4j.service; + +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.neo4j.model.NeoSample; + +public class NeoSampleConverter { + + public static Sample fromNeoSampleToSample(NeoSample neoSample) { + return null; + } +} diff --git a/models/security/src/main/java/uk/ac/ebi/biosamples/model/auth/AuthRealm.java b/core/src/main/java/uk/ac/ebi/biosamples/security/model/AuthRealm.java similarity index 91% rename from models/security/src/main/java/uk/ac/ebi/biosamples/model/auth/AuthRealm.java rename to core/src/main/java/uk/ac/ebi/biosamples/security/model/AuthRealm.java index 19e47ab43..afe95efa0 100644 --- a/models/security/src/main/java/uk/ac/ebi/biosamples/model/auth/AuthRealm.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/security/model/AuthRealm.java @@ -1,16 +1,16 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.auth; - -public enum AuthRealm { - ENA, - EGA -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.security.model; + +public enum AuthRealm { + ENA, + EGA +} diff --git a/models/security/src/main/java/uk/ac/ebi/biosamples/model/auth/AuthRequestWebin.java b/core/src/main/java/uk/ac/ebi/biosamples/security/model/AuthRequestWebin.java similarity index 93% rename from models/security/src/main/java/uk/ac/ebi/biosamples/model/auth/AuthRequestWebin.java rename to core/src/main/java/uk/ac/ebi/biosamples/security/model/AuthRequestWebin.java index d587bccf1..c8e984d05 100644 --- a/models/security/src/main/java/uk/ac/ebi/biosamples/model/auth/AuthRequestWebin.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/security/model/AuthRequestWebin.java @@ -1,31 +1,31 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.auth; - -import java.io.Serial; -import java.io.Serializable; -import java.util.List; -import lombok.*; - -@Data -public class AuthRequestWebin implements Serializable { - @Serial private static final long serialVersionUID = 1L; - private List authRealms; - private String password; - private String username; - - public AuthRequestWebin( - final String username, final String password, final List authRealms) { - this.username = username; - this.password = password; - this.authRealms = authRealms; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.security.model; + +import java.io.Serial; +import java.io.Serializable; +import java.util.List; +import lombok.*; + +@Data +public class AuthRequestWebin implements Serializable { + @Serial private static final long serialVersionUID = 1L; + private List authRealms; + private String password; + private String username; + + public AuthRequestWebin( + final String username, final String password, final List authRealms) { + this.username = username; + this.password = password; + this.authRealms = authRealms; + } +} diff --git a/models/security/src/main/java/uk/ac/ebi/biosamples/model/AuthToken.java b/core/src/main/java/uk/ac/ebi/biosamples/security/model/AuthToken.java similarity index 89% rename from models/security/src/main/java/uk/ac/ebi/biosamples/model/AuthToken.java rename to core/src/main/java/uk/ac/ebi/biosamples/security/model/AuthToken.java index 132d5361d..096d3b7a2 100644 --- a/models/security/src/main/java/uk/ac/ebi/biosamples/model/AuthToken.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/security/model/AuthToken.java @@ -1,48 +1,47 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import java.util.List; -import uk.ac.ebi.biosamples.model.auth.AuthorizationProvider; - -public class AuthToken { - private final String algorithm; - private final AuthorizationProvider authority; - private final String user; - private final List roles; - - public AuthToken( - final String algorithm, - final AuthorizationProvider authority, - final String user, - final List roles) { - this.algorithm = algorithm; - this.authority = authority; - this.user = user; - this.roles = roles; - } - - public String getAlgorithm() { - return algorithm; - } - - public AuthorizationProvider getAuthority() { - return authority; - } - - public String getUser() { - return user; - } - - public List getRoles() { - return roles; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.security.model; + +import java.util.List; + +public class AuthToken { + private final String algorithm; + private final AuthorizationProvider authority; + private final String user; + private final List roles; + + public AuthToken( + final String algorithm, + final AuthorizationProvider authority, + final String user, + final List roles) { + this.algorithm = algorithm; + this.authority = authority; + this.user = user; + this.roles = roles; + } + + public String getAlgorithm() { + return algorithm; + } + + public AuthorizationProvider getAuthority() { + return authority; + } + + public String getUser() { + return user; + } + + public List getRoles() { + return roles; + } +} diff --git a/models/security/src/main/java/uk/ac/ebi/biosamples/model/auth/AuthorizationProvider.java b/core/src/main/java/uk/ac/ebi/biosamples/security/model/AuthorizationProvider.java similarity index 92% rename from models/security/src/main/java/uk/ac/ebi/biosamples/model/auth/AuthorizationProvider.java rename to core/src/main/java/uk/ac/ebi/biosamples/security/model/AuthorizationProvider.java index 93523a4ef..441b64600 100644 --- a/models/security/src/main/java/uk/ac/ebi/biosamples/model/auth/AuthorizationProvider.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/security/model/AuthorizationProvider.java @@ -1,15 +1,15 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model.auth; - -public enum AuthorizationProvider { - WEBIN; -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.security.model; + +public enum AuthorizationProvider { + WEBIN; +} diff --git a/models/security/src/main/java/uk/ac/ebi/biosamples/model/auth/FileUploaderAuthRequest.java b/core/src/main/java/uk/ac/ebi/biosamples/security/model/FileUploaderAuthRequest.java similarity index 94% rename from models/security/src/main/java/uk/ac/ebi/biosamples/model/auth/FileUploaderAuthRequest.java rename to core/src/main/java/uk/ac/ebi/biosamples/security/model/FileUploaderAuthRequest.java index 9ac9ecf70..5d418b652 100644 --- a/models/security/src/main/java/uk/ac/ebi/biosamples/model/auth/FileUploaderAuthRequest.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/security/model/FileUploaderAuthRequest.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.model.auth; +package uk.ac.ebi.biosamples.security.model; import java.io.Serializable; import lombok.*; diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/UserAuthentication.java b/core/src/main/java/uk/ac/ebi/biosamples/security/model/UserAuthentication.java similarity index 94% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/UserAuthentication.java rename to core/src/main/java/uk/ac/ebi/biosamples/security/model/UserAuthentication.java index 5989c78bc..8345cce41 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/UserAuthentication.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/security/model/UserAuthentication.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.security; +package uk.ac.ebi.biosamples.security.model; import java.util.Collection; import org.springframework.security.core.Authentication; @@ -20,7 +20,7 @@ public class UserAuthentication implements Authentication { private User user; private boolean authenticated = true; - UserAuthentication(User user) { + public UserAuthentication(User user) { this.user = user; } diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/security/AccessControlService.java b/core/src/main/java/uk/ac/ebi/biosamples/security/service/AccessControlService.java similarity index 92% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/security/AccessControlService.java rename to core/src/main/java/uk/ac/ebi/biosamples/security/service/AccessControlService.java index 1d9496955..430d3b200 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/security/AccessControlService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/security/service/AccessControlService.java @@ -1,94 +1,94 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service.security; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.interfaces.DecodedJWT; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.util.*; -import org.springframework.http.HttpStatus; -import org.springframework.stereotype.Service; -import org.springframework.web.server.ResponseStatusException; -import uk.ac.ebi.biosamples.model.AuthToken; -import uk.ac.ebi.biosamples.model.auth.AuthorizationProvider; - -@Service -public class AccessControlService { - private final ObjectMapper objectMapper; - - public AccessControlService(final ObjectMapper objectMapper) { - this.objectMapper = objectMapper; - } - - public Optional extractToken(String token) { - try { - if (token == null || token.isEmpty()) { - return Optional.empty(); - } - - token = token.startsWith("Bearer ") ? token.split("Bearer ")[1] : token; - - final String[] chunks = token.split("\\."); - final Base64.Decoder decoder = Base64.getDecoder(); - final String header = new String(decoder.decode(chunks[0])); - final String payload = new String(decoder.decode(chunks[1])); - - if (!verifySignature()) { - throw new ResponseStatusException(HttpStatus.UNAUTHORIZED); - } - - final AuthToken authToken; - - try { - final String algorithm = objectMapper.readTree(header).get("alg").asText(); - final AuthorizationProvider authority; - final String user; - final List roles; - final JsonNode node = objectMapper.readTree(payload); - - verifyValidity(token); - - authority = AuthorizationProvider.WEBIN; - user = node.get("principle").asText(); - roles = objectMapper.convertValue(node.get("role"), new TypeReference<>() {}); - - authToken = new AuthToken(algorithm, authority, user, roles); - } catch (final IOException e) { - throw new ResponseStatusException( - HttpStatus.UNAUTHORIZED, "Invalid authentication details provided", e); - } - - return Optional.of(authToken); - } catch (final Exception e) { - throw new ResponseStatusException( - HttpStatus.UNAUTHORIZED, "Invalid authentication details provided", e); - } - } - - private void verifyValidity(final String token) { - final DecodedJWT jwt = JWT.decode(token); - - if (jwt.getExpiresAt().before(new Date())) { - throw new ResponseStatusException(HttpStatus.UNAUTHORIZED); - } - } - - private boolean verifySignature() { - return true; - } - - public String getUser(final AuthToken token) { - return token.getUser(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.security.service; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.*; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; +import uk.ac.ebi.biosamples.security.model.AuthToken; +import uk.ac.ebi.biosamples.security.model.AuthorizationProvider; + +@Service +public class AccessControlService { + private final ObjectMapper objectMapper; + + public AccessControlService(final ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + public Optional extractToken(String token) { + try { + if (token == null || token.isEmpty()) { + return Optional.empty(); + } + + token = token.startsWith("Bearer ") ? token.split("Bearer ")[1] : token; + + final String[] chunks = token.split("\\."); + final Base64.Decoder decoder = Base64.getDecoder(); + final String header = new String(decoder.decode(chunks[0])); + final String payload = new String(decoder.decode(chunks[1])); + + if (!verifySignature()) { + throw new ResponseStatusException(HttpStatus.UNAUTHORIZED); + } + + final AuthToken authToken; + + try { + final String algorithm = objectMapper.readTree(header).get("alg").asText(); + final AuthorizationProvider authority; + final String user; + final List roles; + final JsonNode node = objectMapper.readTree(payload); + + verifyValidity(token); + + authority = AuthorizationProvider.WEBIN; + user = node.get("principle").asText(); + roles = objectMapper.convertValue(node.get("role"), new TypeReference<>() {}); + + authToken = new AuthToken(algorithm, authority, user, roles); + } catch (final IOException e) { + throw new ResponseStatusException( + HttpStatus.UNAUTHORIZED, "Invalid authentication details provided", e); + } + + return Optional.of(authToken); + } catch (final Exception e) { + throw new ResponseStatusException( + HttpStatus.UNAUTHORIZED, "Invalid authentication details provided", e); + } + } + + private void verifyValidity(final String token) { + final DecodedJWT jwt = JWT.decode(token); + + if (jwt.getExpiresAt().before(new Date())) { + throw new ResponseStatusException(HttpStatus.UNAUTHORIZED); + } + } + + private boolean verifySignature() { + return true; + } + + public String getUser(final AuthToken token) { + return token.getUser(); + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/security/BioSamplesCrossSourceIngestAccessControlService.java b/core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesCrossSourceIngestAccessControlService.java similarity index 91% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/security/BioSamplesCrossSourceIngestAccessControlService.java rename to core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesCrossSourceIngestAccessControlService.java index e633923a2..9ef41034f 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/security/BioSamplesCrossSourceIngestAccessControlService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesCrossSourceIngestAccessControlService.java @@ -1,85 +1,83 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service.security; - -import static uk.ac.ebi.biosamples.BioSamplesConstants.*; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SubmittedViaType; - -@Service -public class BioSamplesCrossSourceIngestAccessControlService { - private final Logger log = LoggerFactory.getLogger(getClass()); - private static final String ENA_CHECKLIST = "ENA-CHECKLIST"; - - @Autowired BioSamplesProperties bioSamplesProperties; - - public void isOriginalSubmitterInSampleMetadata( - final String webinIdInOldSample, final Sample newSample) { - log.info("Super user and file upload submission"); - - // file uploader submission access protection - if (webinIdInOldSample != null - && !webinIdInOldSample.equals(newSample.getWebinSubmissionAccountId())) { - throw new GlobalExceptions.NonSubmitterUpdateAttemptException(); - } - } - - public void accessControlWebinSourcedSampleByCheckingEnaChecklistAttribute( - final Sample oldSample, final Sample newSample) { - /* - Old sample has ENA-CHECKLIST attribute, hence it can be concluded that it is imported from ENA - New sample has ENA-CHECKLIST attribute, means its updated by ENA pipeline or WEBIN, allow further computation - New sample doesn't have ENA-CHECKLIST attribute, means it's not updated by ENA pipeline or WEBIN, don't allow further computation and throw exception - */ - if (oldSample.getAttributes().stream() - .anyMatch(attribute -> attribute.getType().equalsIgnoreCase(ENA_CHECKLIST))) { - if (newSample.getAttributes().stream() - .noneMatch(attribute -> attribute.getType().equalsIgnoreCase(ENA_CHECKLIST))) { - throw new GlobalExceptions.InvalidSubmissionSourceException(); - } - } - } - - public void accessControlWebinSourcedSampleByCheckingSubmittedViaType( - final Sample oldSample, final Sample newSample) { - if (oldSample.getSubmittedVia() == SubmittedViaType.WEBIN_SERVICES) { - if (newSample.getSubmittedVia() != SubmittedViaType.WEBIN_SERVICES) { - throw new GlobalExceptions.InvalidSubmissionSourceException(); - } - } - } - - public void accessControlPipelineImportedSamples(final Sample oldSample, final Sample newSample) { - if (oldSample.getSubmittedVia() - == SubmittedViaType.PIPELINE_IMPORT) { // pipeline imports access protection - if (newSample.getSubmittedVia() != SubmittedViaType.PIPELINE_IMPORT) { - throw new GlobalExceptions.InvalidSubmissionSourceException(); - } - } - } - - public void validateFileUploaderSampleUpdateHasExistingAccession(final Sample newSample) { - if (newSample.getSubmittedVia() == SubmittedViaType.FILE_UPLOADER) { - log.error( - "Not permitted to update sample not in database using the file uploader, accession: {}", - newSample.getAccession()); - - throw new GlobalExceptions.SampleAccessionDoesNotExistException(); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.security.service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.BioSamplesProperties; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SubmittedViaType; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; + +@Service +public class BioSamplesCrossSourceIngestAccessControlService { + private final Logger log = LoggerFactory.getLogger(getClass()); + private static final String ENA_CHECKLIST = "ENA-CHECKLIST"; + + @Autowired BioSamplesProperties bioSamplesProperties; + + public void isOriginalSubmitterInSampleMetadata( + final String webinIdInOldSample, final Sample newSample) { + log.info("Super user and file upload submission"); + + // file uploader submission access protection + if (webinIdInOldSample != null + && !webinIdInOldSample.equals(newSample.getWebinSubmissionAccountId())) { + throw new GlobalExceptions.NonSubmitterUpdateAttemptException(); + } + } + + public void accessControlWebinSourcedSampleByCheckingEnaChecklistAttribute( + final Sample oldSample, final Sample newSample) { + /* + Old sample has ENA-CHECKLIST attribute, hence it can be concluded that it is imported from ENA + New sample has ENA-CHECKLIST attribute, means its updated by ENA pipeline or WEBIN, allow further computation + New sample doesn't have ENA-CHECKLIST attribute, means it's not updated by ENA pipeline or WEBIN, don't allow further computation and throw exception + */ + if (oldSample.getAttributes().stream() + .anyMatch(attribute -> attribute.getType().equalsIgnoreCase(ENA_CHECKLIST))) { + if (newSample.getAttributes().stream() + .noneMatch(attribute -> attribute.getType().equalsIgnoreCase(ENA_CHECKLIST))) { + throw new GlobalExceptions.InvalidSubmissionSourceException(); + } + } + } + + public void accessControlWebinSourcedSampleByCheckingSubmittedViaType( + final Sample oldSample, final Sample newSample) { + if (oldSample.getSubmittedVia() == SubmittedViaType.WEBIN_SERVICES) { + if (newSample.getSubmittedVia() != SubmittedViaType.WEBIN_SERVICES) { + throw new GlobalExceptions.InvalidSubmissionSourceException(); + } + } + } + + public void accessControlPipelineImportedSamples(final Sample oldSample, final Sample newSample) { + if (oldSample.getSubmittedVia() + == SubmittedViaType.PIPELINE_IMPORT) { // pipeline imports access protection + if (newSample.getSubmittedVia() != SubmittedViaType.PIPELINE_IMPORT) { + throw new GlobalExceptions.InvalidSubmissionSourceException(); + } + } + } + + public void validateFileUploaderSampleUpdateHasExistingAccession(final Sample newSample) { + if (newSample.getSubmittedVia() == SubmittedViaType.FILE_UPLOADER) { + log.error( + "Not permitted to update sample not in database using the file uploader, accession: {}", + newSample.getAccession()); + + throw new GlobalExceptions.SampleAccessionDoesNotExistException(); + } + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/BioSamplesTokenAuthenticationFilter.java b/core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesTokenAuthenticationFilter.java similarity index 98% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/BioSamplesTokenAuthenticationFilter.java rename to core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesTokenAuthenticationFilter.java index cd90dd452..b6a00af51 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/BioSamplesTokenAuthenticationFilter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesTokenAuthenticationFilter.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.security; +package uk.ac.ebi.biosamples.security.service; import java.io.IOException; import javax.servlet.FilterChain; diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/BioSamplesTokenAuthenticationService.java b/core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesTokenAuthenticationService.java similarity index 93% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/BioSamplesTokenAuthenticationService.java rename to core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesTokenAuthenticationService.java index 2efe2081b..b3c1a1a75 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/BioSamplesTokenAuthenticationService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesTokenAuthenticationService.java @@ -8,12 +8,13 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.security; +package uk.ac.ebi.biosamples.security.service; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Primary; import org.springframework.security.core.Authentication; import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.security.model.UserAuthentication; @Service @Primary diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/BioSamplesTokenHandler.java b/core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesTokenHandler.java similarity index 90% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/BioSamplesTokenHandler.java rename to core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesTokenHandler.java index c694b4d74..bb7f67185 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/BioSamplesTokenHandler.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesTokenHandler.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.security; +package uk.ac.ebi.biosamples.security.service; import java.util.ArrayList; import java.util.Optional; @@ -16,8 +16,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.core.userdetails.User; import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.AuthToken; -import uk.ac.ebi.biosamples.service.security.AccessControlService; +import uk.ac.ebi.biosamples.security.model.AuthToken; @Component @Slf4j diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/BioSamplesWebSecurityConfig.java b/core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesWebSecurityConfig.java similarity index 98% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/BioSamplesWebSecurityConfig.java rename to core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesWebSecurityConfig.java index a9f5aa15b..8c7a51fc4 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/security/BioSamplesWebSecurityConfig.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/security/service/BioSamplesWebSecurityConfig.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.security; +package uk.ac.ebi.biosamples.security.service; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; diff --git a/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java b/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java new file mode 100644 index 000000000..da9afd3c1 --- /dev/null +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java @@ -0,0 +1,57 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import java.util.Collection; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.model.SolrSample; +import uk.ac.ebi.biosamples.solr.service.SolrSampleService; +import uk.ac.ebi.biosamples.utils.LinkUtils; + +@Service +public class AccessionsService { + private final SolrSampleService solrSampleService; + private final FilterService filterService; + + public AccessionsService( + final SolrSampleService solrSampleService, final FilterService filterService) { + this.solrSampleService = solrSampleService; + this.filterService = filterService; + } + + public Page getAccessions( + final String text, + final String[] requestfilters, + final String webinSubmissionAccountId, + final Integer page, + final Integer size) { + final PageRequest pageable = PageRequest.of(page, size); + final String decodedText = LinkUtils.decodeText(text); + final String[] decodedFilter = LinkUtils.decodeTexts(requestfilters); + final Collection filtersAfterDecode = filterService.getFiltersCollection(decodedFilter); + + return fetchAccessions(pageable, decodedText, filtersAfterDecode, webinSubmissionAccountId); + } + + private Page fetchAccessions( + final PageRequest pageable, + final String decodedText, + final Collection filtersAfterDecode, + final String webinSubmissionAccountId) { + final Page pageSolrSample = + solrSampleService.fetchSolrSampleByText( + decodedText, filtersAfterDecode, webinSubmissionAccountId, pageable); + return pageSolrSample.map(SolrSample::getAccession); + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/ExceptionAsTextHttpMessageConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/service/ExceptionAsTextHttpMessageConverter.java similarity index 95% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/ExceptionAsTextHttpMessageConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/service/ExceptionAsTextHttpMessageConverter.java index 8f112d777..ee8179e2a 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/ExceptionAsTextHttpMessageConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/ExceptionAsTextHttpMessageConverter.java @@ -1,57 +1,57 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.io.IOException; -import java.io.PrintWriter; -import java.util.Arrays; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpInputMessage; -import org.springframework.http.HttpOutputMessage; -import org.springframework.http.MediaType; -import org.springframework.http.converter.AbstractHttpMessageConverter; -import org.springframework.http.converter.HttpMessageNotReadableException; -import org.springframework.http.converter.HttpMessageNotWritableException; -import uk.ac.ebi.biosamples.model.Sample; - -public class ExceptionAsTextHttpMessageConverter extends AbstractHttpMessageConverter { - - private final List DEFAULT_SUPPORTED_MEDIA_TYPES = Arrays.asList(MediaType.TEXT_PLAIN); - - private final Logger log = LoggerFactory.getLogger(getClass()); - - public ExceptionAsTextHttpMessageConverter() { - setSupportedMediaTypes(DEFAULT_SUPPORTED_MEDIA_TYPES); - } - - @Override - protected boolean supports(final Class clazz) { - return Sample.class.isAssignableFrom(clazz); - } - - @Override - protected Exception readInternal( - final Class clazz, final HttpInputMessage inputMessage) - throws IOException, HttpMessageNotReadableException { - throw new HttpMessageNotReadableException("Cannot read exceptions"); - } - - @Override - protected void writeInternal(final Exception exception, final HttpOutputMessage outputMessage) - throws IOException, HttpMessageNotWritableException { - log.trace("Writing message"); - final PrintWriter printWritter = new PrintWriter(outputMessage.getBody()); - exception.printStackTrace(printWritter); - // don't close the writer, underlying outputstream will be closed elsewhere - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Arrays; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpInputMessage; +import org.springframework.http.HttpOutputMessage; +import org.springframework.http.MediaType; +import org.springframework.http.converter.AbstractHttpMessageConverter; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.http.converter.HttpMessageNotWritableException; +import uk.ac.ebi.biosamples.core.model.Sample; + +public class ExceptionAsTextHttpMessageConverter extends AbstractHttpMessageConverter { + + private final List DEFAULT_SUPPORTED_MEDIA_TYPES = Arrays.asList(MediaType.TEXT_PLAIN); + + private final Logger log = LoggerFactory.getLogger(getClass()); + + public ExceptionAsTextHttpMessageConverter() { + setSupportedMediaTypes(DEFAULT_SUPPORTED_MEDIA_TYPES); + } + + @Override + protected boolean supports(final Class clazz) { + return Sample.class.isAssignableFrom(clazz); + } + + @Override + protected Exception readInternal( + final Class clazz, final HttpInputMessage inputMessage) + throws IOException, HttpMessageNotReadableException { + throw new HttpMessageNotReadableException("Cannot read exceptions"); + } + + @Override + protected void writeInternal(final Exception exception, final HttpOutputMessage outputMessage) + throws IOException, HttpMessageNotWritableException { + log.trace("Writing message"); + final PrintWriter printWritter = new PrintWriter(outputMessage.getBody()); + exception.printStackTrace(printWritter); + // don't close the writer, underlying outputstream will be closed elsewhere + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/FilterService.java b/core/src/main/java/uk/ac/ebi/biosamples/service/FilterService.java similarity index 93% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/FilterService.java rename to core/src/main/java/uk/ac/ebi/biosamples/service/FilterService.java index dd26eb3e1..d22b3d9d4 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/FilterService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/FilterService.java @@ -1,43 +1,44 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.util.*; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.filter.Filter; - -@Service -public class FilterService { - /** - * Converts an array of serialized filters to the corresponding collection of object - * - * @param filterStrings an array of serialized filters - * @return - */ - public Collection getFiltersCollection(final String[] filterStrings) { - final List outputFilters = new ArrayList<>(); - if (filterStrings == null || filterStrings.length == 0) { - return outputFilters; - } - - Arrays.sort(filterStrings); - - final SortedSet filterStringSet = new TreeSet<>(Arrays.asList(filterStrings)); - - for (final String filterString : filterStringSet) { - if (!filterString.isEmpty()) { - outputFilters.add(FilterBuilder.create().buildFromString(filterString)); - } - } - - return outputFilters; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import java.util.*; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.service.FilterBuilder; + +@Service +public class FilterService { + /** + * Converts an array of serialized filters to the corresponding collection of object + * + * @param filterStrings an array of serialized filters + * @return + */ + public Collection getFiltersCollection(final String[] filterStrings) { + final List outputFilters = new ArrayList<>(); + if (filterStrings == null || filterStrings.length == 0) { + return outputFilters; + } + + Arrays.sort(filterStrings); + + final SortedSet filterStringSet = new TreeSet<>(Arrays.asList(filterStrings)); + + for (final String filterString : filterStringSet) { + if (!filterString.isEmpty()) { + outputFilters.add(FilterBuilder.create().buildFromString(filterString)); + } + } + + return outputFilters; + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java b/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java similarity index 90% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java rename to core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java index 419092a28..958a4c6ed 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java @@ -1,136 +1,136 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.amqp.core.AmqpTemplate; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.MessageContent; -import uk.ac.ebi.biosamples.Messaging; -import uk.ac.ebi.biosamples.model.CurationLink; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.mongo.service.SampleReadService; - -@Service -public class MessagingService { - private final Logger log = LoggerFactory.getLogger(getClass()); - private final SampleReadService sampleReadService; - private final AmqpTemplate amqpTemplate; - - public MessagingService( - final SampleReadService sampleReadService, final AmqpTemplate amqpTemplate) { - this.sampleReadService = sampleReadService; - this.amqpTemplate = amqpTemplate; - } - - public void sendFileUploadedMessage(final String fileId) { - log.info("Uploaded file queued " + fileId); - - amqpTemplate.convertAndSend(Messaging.UPLOAD_EXCHANGE, "", fileId); - } - - void fetchThenSendMessage(final String accession) { - fetchThenSendMessage(accession, Collections.emptyList()); - } - - void fetchThenSendMessage( - final String accession, final List existingRelationshipTargets) { - if (accession == null) { - throw new IllegalArgumentException("accession cannot be null"); - } - - if (accession.trim().isEmpty()) { - throw new IllegalArgumentException("accession cannot be empty"); - } - - final Optional sample = sampleReadService.fetch(accession, true); - - if (sample.isPresent()) { - // for each sample we have a relationship to, update it to index this sample as an - // inverse - // relationship - // TODO do this async - final List related = - updateInverseRelationships(sample.get(), existingRelationshipTargets); - - // send the original sample with the extras as related samples - amqpTemplate.convertAndSend( - Messaging.INDEXING_EXCHANGE, - Messaging.INDEXING_QUEUE, - MessageContent.build(sample.get(), null, related, false)); - } - } - - private List updateInverseRelationships( - final Sample sample, final List existingRelationshipTargets) { - final List>> futures = new ArrayList<>(); - - // remove deleted relationships - for (final String accession : existingRelationshipTargets) { - futures.add(sampleReadService.fetchAsync(accession, true)); - } - - for (final Relationship relationship : sample.getRelationships()) { - if (relationship.getSource() != null - && relationship.getSource().equals(sample.getAccession()) - && !existingRelationshipTargets.contains(sample.getAccession())) { - futures.add(sampleReadService.fetchAsync(relationship.getTarget(), true)); - } - } - - final List related = new ArrayList<>(); - for (final Future> sampleFuture : futures) { - try { - sampleFuture.get().ifPresent(related::add); - } catch (final InterruptedException e) { - log.warn("Interrupted fetching future relationships", e); - } catch (final ExecutionException e) { - log.error("Problem fetching future relationships", e); - } - } - return related; - } - - @Deprecated - public void sendMessages(final CurationLink curationLink) { - fetchThenSendMessage(curationLink.getSample()); - } - - @Deprecated - public void sendMessages(final Sample sample) { - fetchThenSendMessage(sample.getAccession()); - } - - private List getDerivedFromSamples(final Sample sample, final List related) { - for (final Relationship relationship : sample.getRelationships()) { - if (relationship.getSource().equals(sample.getAccession())) { - if (relationship.getType().toLowerCase().equals("derived from")) { - final Optional target = sampleReadService.fetch(relationship.getTarget(), true); - if (target.isPresent()) { - if (!related.contains(target.get())) { - related.add(target.get()); - getDerivedFromSamples(target.get(), related); - } - } - } - } - } - return related; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.core.AmqpTemplate; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.CurationLink; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.messaging.MessagingConstants; +import uk.ac.ebi.biosamples.messaging.model.MessageContent; +import uk.ac.ebi.biosamples.mongo.service.SampleReadService; + +@Service +public class MessagingService { + private final Logger log = LoggerFactory.getLogger(getClass()); + private final SampleReadService sampleReadService; + private final AmqpTemplate amqpTemplate; + + public MessagingService( + final SampleReadService sampleReadService, final AmqpTemplate amqpTemplate) { + this.sampleReadService = sampleReadService; + this.amqpTemplate = amqpTemplate; + } + + public void sendFileUploadedMessage(final String fileId) { + log.info("Uploaded file queued " + fileId); + + amqpTemplate.convertAndSend(MessagingConstants.UPLOAD_EXCHANGE, "", fileId); + } + + void fetchThenSendMessage(final String accession) { + fetchThenSendMessage(accession, Collections.emptyList()); + } + + void fetchThenSendMessage( + final String accession, final List existingRelationshipTargets) { + if (accession == null) { + throw new IllegalArgumentException("accession cannot be null"); + } + + if (accession.trim().isEmpty()) { + throw new IllegalArgumentException("accession cannot be empty"); + } + + final Optional sample = sampleReadService.fetch(accession, true); + + if (sample.isPresent()) { + // for each sample we have a relationship to, update it to index this sample as an + // inverse + // relationship + // TODO do this async + final List related = + updateInverseRelationships(sample.get(), existingRelationshipTargets); + + // send the original sample with the extras as related samples + amqpTemplate.convertAndSend( + MessagingConstants.INDEXING_EXCHANGE, + MessagingConstants.INDEXING_QUEUE, + MessageContent.build(sample.get(), null, related, false)); + } + } + + private List updateInverseRelationships( + final Sample sample, final List existingRelationshipTargets) { + final List>> futures = new ArrayList<>(); + + // remove deleted relationships + for (final String accession : existingRelationshipTargets) { + futures.add(sampleReadService.fetchAsync(accession, true)); + } + + for (final Relationship relationship : sample.getRelationships()) { + if (relationship.getSource() != null + && relationship.getSource().equals(sample.getAccession()) + && !existingRelationshipTargets.contains(sample.getAccession())) { + futures.add(sampleReadService.fetchAsync(relationship.getTarget(), true)); + } + } + + final List related = new ArrayList<>(); + for (final Future> sampleFuture : futures) { + try { + sampleFuture.get().ifPresent(related::add); + } catch (final InterruptedException e) { + log.warn("Interrupted fetching future relationships", e); + } catch (final ExecutionException e) { + log.error("Problem fetching future relationships", e); + } + } + return related; + } + + @Deprecated + public void sendMessages(final CurationLink curationLink) { + fetchThenSendMessage(curationLink.getSample()); + } + + @Deprecated + public void sendMessages(final Sample sample) { + fetchThenSendMessage(sample.getAccession()); + } + + private List getDerivedFromSamples(final Sample sample, final List related) { + for (final Relationship relationship : sample.getRelationships()) { + if (relationship.getSource().equals(sample.getAccession())) { + if (relationship.getType().toLowerCase().equals("derived from")) { + final Optional target = sampleReadService.fetch(relationship.getTarget(), true); + if (target.isPresent()) { + if (!related.contains(target.get())) { + related.add(target.get()); + getDerivedFromSamples(target.get(), related); + } + } + } + } + } + return related; + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/StructuredDataService.java b/core/src/main/java/uk/ac/ebi/biosamples/service/StructuredDataService.java similarity index 95% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/StructuredDataService.java rename to core/src/main/java/uk/ac/ebi/biosamples/service/StructuredDataService.java index 09ff11d41..521bf2276 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/StructuredDataService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/StructuredDataService.java @@ -1,122 +1,122 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.time.Instant; -import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.structured.StructuredData; -import uk.ac.ebi.biosamples.mongo.model.MongoStructuredData; -import uk.ac.ebi.biosamples.mongo.repository.MongoSampleRepository; -import uk.ac.ebi.biosamples.mongo.repository.MongoStructuredDataRepository; -import uk.ac.ebi.biosamples.mongo.service.MongoStructuredDataToStructuredDataConverter; -import uk.ac.ebi.biosamples.mongo.service.StructuredDataToMongoStructuredDataConverter; - -@Service -public class StructuredDataService { - private static final Logger log = LoggerFactory.getLogger(StructuredDataService.class); - - private final MongoSampleRepository mongoSampleRepository; - private final MongoStructuredDataRepository mongoStructuredDataRepository; - private final MongoStructuredDataToStructuredDataConverter - mongoStructuredDataToStructuredDataConverter; - private final StructuredDataToMongoStructuredDataConverter - structuredDataToMongoStructuredDataConverter; - private final MessagingService messagingService; - - public StructuredDataService( - final MongoSampleRepository mongoSampleRepository, - final MongoStructuredDataRepository mongoStructuredDataRepository, - final MongoStructuredDataToStructuredDataConverter - mongoStructuredDataToStructuredDataConverter, - final StructuredDataToMongoStructuredDataConverter - structuredDataToMongoStructuredDataConverter, - final MessagingService messagingService) { - this.mongoSampleRepository = mongoSampleRepository; - this.mongoStructuredDataRepository = mongoStructuredDataRepository; - this.mongoStructuredDataToStructuredDataConverter = - mongoStructuredDataToStructuredDataConverter; - this.structuredDataToMongoStructuredDataConverter = - structuredDataToMongoStructuredDataConverter; - this.messagingService = messagingService; - } - - public Optional getStructuredData(final String accession) { - final Optional byId = mongoStructuredDataRepository.findById(accession); - final MongoStructuredData mongoData = byId.orElse(null); - - if (mongoData == null) { - return Optional.empty(); - } - - return Optional.of(mongoStructuredDataToStructuredDataConverter.convert(mongoData)); - } - - public StructuredData saveStructuredData(StructuredData structuredData) { - validate(structuredData); - Instant create = Instant.now(); - final Optional byId = - mongoStructuredDataRepository.findById(structuredData.getAccession()); - final MongoStructuredData oldMongoData = byId.orElse(null); - - if (oldMongoData != null) { - final StructuredData oldData = - mongoStructuredDataToStructuredDataConverter.convert(oldMongoData); - // we consider if domain and types are equal StructuredDataTable - // holds the same data and changed equals method accordingly - structuredData.getData().addAll(oldData.getData()); - create = oldData.getCreate(); - } - - structuredData = - StructuredData.build(structuredData.getAccession(), create, structuredData.getData()); - MongoStructuredData mongoStructuredData = - structuredDataToMongoStructuredDataConverter.convert(structuredData); - mongoStructuredData = mongoStructuredDataRepository.save(mongoStructuredData); - - messagingService.fetchThenSendMessage(structuredData.getAccession()); - return mongoStructuredDataToStructuredDataConverter.convert(mongoStructuredData); - } - - private void validate(final StructuredData structuredData) { - if (structuredData.getAccession() == null - || !isExistingAccession(structuredData.getAccession())) { - log.info( - "Structured data validation failed: Misisng accession {}", structuredData.getAccession()); - throw new GlobalExceptions.SampleValidationException( - "Missing accession. Structured data should have an accession"); - } - - if (structuredData.getData() == null || structuredData.getData().isEmpty()) { - log.info("Structured data validation failed: Misisng data {}", structuredData.getData()); - throw new GlobalExceptions.SampleValidationException( - "Missing data. Empty data is not accepted"); - } - - structuredData - .getData() - .forEach( - d -> { - if (d.getType() == null || d.getType().isEmpty()) { - log.info("Structured data validation failed: Misisng data type {}", d.getType()); - throw new GlobalExceptions.SampleValidationException( - "Empty structured data type. Type must be specified."); - } - }); - } - - private boolean isExistingAccession(final String accession) { - return mongoSampleRepository.findById(accession).isPresent(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import java.time.Instant; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; +import uk.ac.ebi.biosamples.mongo.model.MongoStructuredData; +import uk.ac.ebi.biosamples.mongo.repository.MongoSampleRepository; +import uk.ac.ebi.biosamples.mongo.repository.MongoStructuredDataRepository; +import uk.ac.ebi.biosamples.mongo.service.MongoStructuredDataToStructuredDataConverter; +import uk.ac.ebi.biosamples.mongo.service.StructuredDataToMongoStructuredDataConverter; + +@Service +public class StructuredDataService { + private static final Logger log = LoggerFactory.getLogger(StructuredDataService.class); + + private final MongoSampleRepository mongoSampleRepository; + private final MongoStructuredDataRepository mongoStructuredDataRepository; + private final MongoStructuredDataToStructuredDataConverter + mongoStructuredDataToStructuredDataConverter; + private final StructuredDataToMongoStructuredDataConverter + structuredDataToMongoStructuredDataConverter; + private final MessagingService messagingService; + + public StructuredDataService( + final MongoSampleRepository mongoSampleRepository, + final MongoStructuredDataRepository mongoStructuredDataRepository, + final MongoStructuredDataToStructuredDataConverter + mongoStructuredDataToStructuredDataConverter, + final StructuredDataToMongoStructuredDataConverter + structuredDataToMongoStructuredDataConverter, + final MessagingService messagingService) { + this.mongoSampleRepository = mongoSampleRepository; + this.mongoStructuredDataRepository = mongoStructuredDataRepository; + this.mongoStructuredDataToStructuredDataConverter = + mongoStructuredDataToStructuredDataConverter; + this.structuredDataToMongoStructuredDataConverter = + structuredDataToMongoStructuredDataConverter; + this.messagingService = messagingService; + } + + public Optional getStructuredData(final String accession) { + final Optional byId = mongoStructuredDataRepository.findById(accession); + final MongoStructuredData mongoData = byId.orElse(null); + + if (mongoData == null) { + return Optional.empty(); + } + + return Optional.of(mongoStructuredDataToStructuredDataConverter.convert(mongoData)); + } + + public StructuredData saveStructuredData(StructuredData structuredData) { + validate(structuredData); + Instant create = Instant.now(); + final Optional byId = + mongoStructuredDataRepository.findById(structuredData.getAccession()); + final MongoStructuredData oldMongoData = byId.orElse(null); + + if (oldMongoData != null) { + final StructuredData oldData = + mongoStructuredDataToStructuredDataConverter.convert(oldMongoData); + // we consider if domain and types are equal StructuredDataTable + // holds the same data and changed equals method accordingly + structuredData.getData().addAll(oldData.getData()); + create = oldData.getCreate(); + } + + structuredData = + StructuredData.build(structuredData.getAccession(), create, structuredData.getData()); + MongoStructuredData mongoStructuredData = + structuredDataToMongoStructuredDataConverter.convert(structuredData); + mongoStructuredData = mongoStructuredDataRepository.save(mongoStructuredData); + + messagingService.fetchThenSendMessage(structuredData.getAccession()); + return mongoStructuredDataToStructuredDataConverter.convert(mongoStructuredData); + } + + private void validate(final StructuredData structuredData) { + if (structuredData.getAccession() == null + || !isExistingAccession(structuredData.getAccession())) { + log.info( + "Structured data validation failed: Misisng accession {}", structuredData.getAccession()); + throw new GlobalExceptions.SampleValidationException( + "Missing accession. Structured data should have an accession"); + } + + if (structuredData.getData() == null || structuredData.getData().isEmpty()) { + log.info("Structured data validation failed: Misisng data {}", structuredData.getData()); + throw new GlobalExceptions.SampleValidationException( + "Missing data. Empty data is not accepted"); + } + + structuredData + .getData() + .forEach( + d -> { + if (d.getType() == null || d.getType().isEmpty()) { + log.info("Structured data validation failed: Misisng data type {}", d.getType()); + throw new GlobalExceptions.SampleValidationException( + "Empty structured data type. Type must be specified."); + } + }); + } + + private boolean isExistingAccession(final String accession) { + return mongoSampleRepository.findById(accession).isPresent(); + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/TaxonomyClientService.java b/core/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/TaxonomyClientService.java similarity index 93% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/TaxonomyClientService.java rename to core/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/TaxonomyClientService.java index 9330fb24b..0f945527a 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/TaxonomyClientService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/TaxonomyClientService.java @@ -1,60 +1,60 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service.taxonomy; - -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.ena.taxonomy.client.TaxonomyClientImpl; -import uk.ac.ebi.ena.taxonomy.taxon.SubmittableTaxon; - -@Service -public class TaxonomyClientService extends TaxonomyClientImpl { - public Sample performTaxonomyValidationAndUpdateTaxIdInSample(Sample sample) { - SubmittableTaxon submittableTaxon; - - try { - if (sample.getTaxId() != null && sample.getTaxId() != 0) { - submittableTaxon = getSubmittableTaxonByTaxId(sample.getTaxId()); - - if (submittableTaxon.getTaxon() == null) { - submittableTaxon = getSubmittableTaxonByScientificName(sample); - } - } else { - submittableTaxon = getSubmittableTaxonByScientificName(sample); - } - - if (submittableTaxon.getTaxon() != null) { - sample = - Sample.Builder.fromSample(sample) - .withTaxId(submittableTaxon.getTaxon().getTaxId()) - .build(); - return sample; - } else { - throw new GlobalExceptions.ENATaxonUnresolvedException(); - } - } catch (final Exception e) { - throw new GlobalExceptions.ENATaxonUnresolvedException(); - } - } - - private SubmittableTaxon getSubmittableTaxonByScientificName(final Sample sample) { - return getSubmittableTaxonByScientificName( - sample.getAttributes().stream() - .filter( - attribute -> - (attribute.getType().equalsIgnoreCase("Organism") - || attribute.getType().equalsIgnoreCase("Species"))) - .findFirst() - .get() - .getValue()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.taxonomy; + +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; +import uk.ac.ebi.ena.taxonomy.client.TaxonomyClientImpl; +import uk.ac.ebi.ena.taxonomy.taxon.SubmittableTaxon; + +@Service +public class TaxonomyClientService extends TaxonomyClientImpl { + public Sample performTaxonomyValidationAndUpdateTaxIdInSample(Sample sample) { + SubmittableTaxon submittableTaxon; + + try { + if (sample.getTaxId() != null && sample.getTaxId() != 0) { + submittableTaxon = getSubmittableTaxonByTaxId(sample.getTaxId()); + + if (submittableTaxon.getTaxon() == null) { + submittableTaxon = getSubmittableTaxonByScientificName(sample); + } + } else { + submittableTaxon = getSubmittableTaxonByScientificName(sample); + } + + if (submittableTaxon.getTaxon() != null) { + sample = + Sample.Builder.fromSample(sample) + .withTaxId(submittableTaxon.getTaxon().getTaxId()) + .build(); + return sample; + } else { + throw new GlobalExceptions.ENATaxonUnresolvedException(); + } + } catch (final Exception e) { + throw new GlobalExceptions.ENATaxonUnresolvedException(); + } + } + + private SubmittableTaxon getSubmittableTaxonByScientificName(final Sample sample) { + return getSubmittableTaxonByScientificName( + sample.getAttributes().stream() + .filter( + attribute -> + (attribute.getType().equalsIgnoreCase("Organism") + || attribute.getType().equalsIgnoreCase("Species"))) + .findFirst() + .get() + .getValue()); + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/ZoomaService.java b/core/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/ZoomaService.java similarity index 97% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/ZoomaService.java rename to core/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/ZoomaService.java index 222976842..ef70bceed 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/ZoomaService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/taxonomy/ZoomaService.java @@ -1,100 +1,100 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service.taxonomy; - -import com.fasterxml.jackson.databind.JsonNode; -import java.net.URI; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicReference; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.hateoas.MediaTypes; -import org.springframework.http.RequestEntity; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestOperations; -import org.springframework.web.util.UriComponents; -import org.springframework.web.util.UriComponentsBuilder; - -@Service -public class ZoomaService { - private static final String SEMANTIC_TAGS = "semanticTags"; - private static final String ZOOMA_BASE_URL = - "https://www.ebi.ac.uk/spot/zooma/v2/api/services/annotate"; - private static final String NCBI_TAXON = "NCBITaxon"; - - private final RestOperations restOperations; - private final UriComponents uriBuilder; - - private static final Logger log = LoggerFactory.getLogger(ZoomaService.class); - - public ZoomaService() { - restOperations = - new RestTemplateBuilder( - restTemplate -> { - // no cusomizations are required for us here - simple RestTemplate should work - }) - .build(); - - // Add filter in the URL to fetch only ncbitaxon - uriBuilder = - UriComponentsBuilder.fromUriString( - ZOOMA_BASE_URL - + "?propertyValue={value}&propertyType={type}]&filter=ontologies:[ncbitaxon]") - .build(); - } - - public Optional queryZooma(final String type, final String value) { - log.trace("Zooma getting : " + type + " : " + value); - - final URI uri = uriBuilder.expand(value, type).encode().toUri(); - - final RequestEntity requestEntity = - RequestEntity.get(uri).accept(MediaTypes.HAL_JSON).build(); - final long startTime = System.nanoTime(); - - final ResponseEntity> responseEntity = - restOperations.exchange(requestEntity, new ParameterizedTypeReference>() {}); - - final long endTime = System.nanoTime(); - - log.trace("Got zooma response in " + ((endTime - startTime) / 1000000) + "ms"); - - final List jsonNodes = responseEntity.getBody(); - - return jsonNodes.stream() - .map( - jsonNode -> { - final AtomicReference ncbiSemanticTag = new AtomicReference<>(null); - - jsonNode - .get(SEMANTIC_TAGS) - .forEach( - semanticTag -> { - final String semanticTagText = semanticTag.asText(); - - // fetching only ncbitaxon, but still cross check if there is anything else, - // accept only ncbitaxon - if (semanticTagText.contains(NCBI_TAXON)) { - ncbiSemanticTag.set(semanticTagText); - } - }); - - return ncbiSemanticTag.get(); - }) - .filter(Objects::nonNull) - .findFirst(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.taxonomy; + +import com.fasterxml.jackson.databind.JsonNode; +import java.net.URI; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.hateoas.MediaTypes; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestOperations; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; + +@Service +public class ZoomaService { + private static final String SEMANTIC_TAGS = "semanticTags"; + private static final String ZOOMA_BASE_URL = + "https://www.ebi.ac.uk/spot/zooma/v2/api/services/annotate"; + private static final String NCBI_TAXON = "NCBITaxon"; + + private final RestOperations restOperations; + private final UriComponents uriBuilder; + + private static final Logger log = LoggerFactory.getLogger(ZoomaService.class); + + public ZoomaService() { + restOperations = + new RestTemplateBuilder( + restTemplate -> { + // no cusomizations are required for us here - simple RestTemplate should work + }) + .build(); + + // Add filter in the URL to fetch only ncbitaxon + uriBuilder = + UriComponentsBuilder.fromUriString( + ZOOMA_BASE_URL + + "?propertyValue={value}&propertyType={type}]&filter=ontologies:[ncbitaxon]") + .build(); + } + + public Optional queryZooma(final String type, final String value) { + log.trace("Zooma getting : " + type + " : " + value); + + final URI uri = uriBuilder.expand(value, type).encode().toUri(); + + final RequestEntity requestEntity = + RequestEntity.get(uri).accept(MediaTypes.HAL_JSON).build(); + final long startTime = System.nanoTime(); + + final ResponseEntity> responseEntity = + restOperations.exchange(requestEntity, new ParameterizedTypeReference>() {}); + + final long endTime = System.nanoTime(); + + log.trace("Got zooma response in " + ((endTime - startTime) / 1000000) + "ms"); + + final List jsonNodes = responseEntity.getBody(); + + return jsonNodes.stream() + .map( + jsonNode -> { + final AtomicReference ncbiSemanticTag = new AtomicReference<>(null); + + jsonNode + .get(SEMANTIC_TAGS) + .forEach( + semanticTag -> { + final String semanticTagText = semanticTag.asText(); + + // fetching only ncbitaxon, but still cross check if there is anything else, + // accept only ncbitaxon + if (semanticTagText.contains(NCBI_TAXON)) { + ncbiSemanticTag.set(semanticTagText); + } + }); + + return ncbiSemanticTag.get(); + }) + .filter(Objects::nonNull) + .findFirst(); + } +} diff --git a/utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/ElixirSchemaValidator.java b/core/src/main/java/uk/ac/ebi/biosamples/service/validation/ElixirSchemaValidator.java similarity index 96% rename from utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/ElixirSchemaValidator.java rename to core/src/main/java/uk/ac/ebi/biosamples/service/validation/ElixirSchemaValidator.java index 01cfe6dc0..628f06485 100644 --- a/utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/ElixirSchemaValidator.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/validation/ElixirSchemaValidator.java @@ -1,140 +1,140 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.validation; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.net.URI; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.RequestEntity; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.UriComponentsBuilder; -import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; - -@Service -@Qualifier("elixirValidator") -public class ElixirSchemaValidator implements ValidatorI { - private final Logger log = LoggerFactory.getLogger(getClass()); - - private final RestTemplate restTemplate; - private final BioSamplesProperties bioSamplesProperties; - private final ObjectMapper objectMapper; - - public ElixirSchemaValidator( - final RestTemplate restTemplate, - final BioSamplesProperties bioSamplesProperties, - final ObjectMapper objectMapper) { - this.restTemplate = restTemplate; - this.bioSamplesProperties = bioSamplesProperties; - this.objectMapper = objectMapper; - } - - @Override - public void validate(final String schemaId, final String sample) throws IOException { - final JsonNode sampleJson = objectMapper.readTree(sample); - final JsonSchema schema = getSchema(schemaId); - - final ValidationRequest validationRequest = - new ValidationRequest(schema.getSchema(), sampleJson); - final URI validatorUri = URI.create(bioSamplesProperties.getSchemaValidator()); - final RequestEntity requestEntity = - RequestEntity.post(validatorUri) - .contentType(MediaType.APPLICATION_JSON) - .accept(MediaType.APPLICATION_JSON) - .body(validationRequest); - final JsonNode validationResponse = - restTemplate.exchange(requestEntity, JsonNode.class).getBody(); - - if (validationResponse.get("validationState").asText().equalsIgnoreCase("INVALID")) { - throw new GlobalExceptions.SampleValidationException( - "Sample validation failed: " + validationResponse.get("validationErrors").toString()); - } - } - - @Override - public String validateById(final String schemaAccession, final String sample) - throws IOException, GlobalExceptions.SchemaValidationException { - final JsonNode sampleJson = objectMapper.readTree(sample); - final JsonNode schema = getSchemaByAccession(schemaAccession); - - final ValidationRequest validationRequest = new ValidationRequest(schema, sampleJson); - final URI validatorUri = URI.create(bioSamplesProperties.getSchemaValidator()); - final RequestEntity requestEntity = - RequestEntity.post(validatorUri) - .contentType(MediaType.APPLICATION_JSON) - .accept(MediaType.APPLICATION_JSON) - .body(validationRequest); - final JsonNode validationResponse = - restTemplate.exchange(requestEntity, JsonNode.class).getBody(); - - if (!validationResponse.isEmpty()) { - throw new GlobalExceptions.SampleValidationException(validationResponse.toString()); - } - - return schemaAccession; - } - - private JsonSchema getSchema(final String schemaId) { - final URI schemaStoreUri = - UriComponentsBuilder.fromUriString( - bioSamplesProperties.getSchemaStore() + "/api/v2/schemas?id={schemaId}") - .build() - .expand(schemaId) - .encode() - .toUri(); - - final RequestEntity requestEntity = - RequestEntity.get(schemaStoreUri).accept(MediaType.APPLICATION_JSON).build(); - final ResponseEntity schemaResponse = - restTemplate.exchange(requestEntity, JsonSchema.class); - if (schemaResponse.getStatusCode() != HttpStatus.OK) { - log.error( - "Failed to retrieve schema from JSON Schema Store: {} {}", schemaId, schemaResponse); - throw new GlobalExceptions.SampleValidationException( - "Failed to retrieve schema from JSON Schema Store"); - } - - return schemaResponse.getBody(); - } - - private JsonNode getSchemaByAccession(final String schemaAccession) { - final URI schemaStoreUri = - UriComponentsBuilder.fromUriString( - bioSamplesProperties.getSchemaStore() + "/registry/schemas/{accession}") - .build() - .expand(schemaAccession) - .encode() - .toUri(); - - final RequestEntity requestEntity = - RequestEntity.get(schemaStoreUri).accept(MediaType.APPLICATION_JSON).build(); - final ResponseEntity schemaResponse = - restTemplate.exchange(requestEntity, JsonNode.class); - if (schemaResponse.getStatusCode() != HttpStatus.OK) { - log.error( - "Failed to retrieve schema from JSON Schema Store: {} {}", - schemaAccession, - schemaResponse); - throw new GlobalExceptions.SampleValidationException( - "Failed to retrieve schema from JSON Schema Store: " + schemaAccession); - } - - return schemaResponse.getBody(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.validation; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.net.URI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; +import uk.ac.ebi.biosamples.BioSamplesProperties; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; + +@Service +@Qualifier("elixirValidator") +public class ElixirSchemaValidator implements ValidatorI { + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final RestTemplate restTemplate; + private final BioSamplesProperties bioSamplesProperties; + private final ObjectMapper objectMapper; + + public ElixirSchemaValidator( + final RestTemplate restTemplate, + final BioSamplesProperties bioSamplesProperties, + final ObjectMapper objectMapper) { + this.restTemplate = restTemplate; + this.bioSamplesProperties = bioSamplesProperties; + this.objectMapper = objectMapper; + } + + @Override + public void validate(final String schemaId, final String sample) throws IOException { + final JsonNode sampleJson = objectMapper.readTree(sample); + final JsonSchema schema = getSchema(schemaId); + + final ValidationRequest validationRequest = + new ValidationRequest(schema.getSchema(), sampleJson); + final URI validatorUri = URI.create(bioSamplesProperties.getSchemaValidator()); + final RequestEntity requestEntity = + RequestEntity.post(validatorUri) + .contentType(MediaType.APPLICATION_JSON) + .accept(MediaType.APPLICATION_JSON) + .body(validationRequest); + final JsonNode validationResponse = + restTemplate.exchange(requestEntity, JsonNode.class).getBody(); + + if (validationResponse.get("validationState").asText().equalsIgnoreCase("INVALID")) { + throw new GlobalExceptions.SampleValidationException( + "Sample validation failed: " + validationResponse.get("validationErrors").toString()); + } + } + + @Override + public String validateById(final String schemaAccession, final String sample) + throws IOException, GlobalExceptions.SchemaValidationException { + final JsonNode sampleJson = objectMapper.readTree(sample); + final JsonNode schema = getSchemaByAccession(schemaAccession); + + final ValidationRequest validationRequest = new ValidationRequest(schema, sampleJson); + final URI validatorUri = URI.create(bioSamplesProperties.getSchemaValidator()); + final RequestEntity requestEntity = + RequestEntity.post(validatorUri) + .contentType(MediaType.APPLICATION_JSON) + .accept(MediaType.APPLICATION_JSON) + .body(validationRequest); + final JsonNode validationResponse = + restTemplate.exchange(requestEntity, JsonNode.class).getBody(); + + if (!validationResponse.isEmpty()) { + throw new GlobalExceptions.SampleValidationException(validationResponse.toString()); + } + + return schemaAccession; + } + + private JsonSchema getSchema(final String schemaId) { + final URI schemaStoreUri = + UriComponentsBuilder.fromUriString( + bioSamplesProperties.getSchemaStore() + "/api/v2/schemas?id={schemaId}") + .build() + .expand(schemaId) + .encode() + .toUri(); + + final RequestEntity requestEntity = + RequestEntity.get(schemaStoreUri).accept(MediaType.APPLICATION_JSON).build(); + final ResponseEntity schemaResponse = + restTemplate.exchange(requestEntity, JsonSchema.class); + if (schemaResponse.getStatusCode() != HttpStatus.OK) { + log.error( + "Failed to retrieve schema from JSON Schema Store: {} {}", schemaId, schemaResponse); + throw new GlobalExceptions.SampleValidationException( + "Failed to retrieve schema from JSON Schema Store"); + } + + return schemaResponse.getBody(); + } + + private JsonNode getSchemaByAccession(final String schemaAccession) { + final URI schemaStoreUri = + UriComponentsBuilder.fromUriString( + bioSamplesProperties.getSchemaStore() + "/registry/schemas/{accession}") + .build() + .expand(schemaAccession) + .encode() + .toUri(); + + final RequestEntity requestEntity = + RequestEntity.get(schemaStoreUri).accept(MediaType.APPLICATION_JSON).build(); + final ResponseEntity schemaResponse = + restTemplate.exchange(requestEntity, JsonNode.class); + if (schemaResponse.getStatusCode() != HttpStatus.OK) { + log.error( + "Failed to retrieve schema from JSON Schema Store: {} {}", + schemaAccession, + schemaResponse); + throw new GlobalExceptions.SampleValidationException( + "Failed to retrieve schema from JSON Schema Store: " + schemaAccession); + } + + return schemaResponse.getBody(); + } +} diff --git a/utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/JsonSchema.java b/core/src/main/java/uk/ac/ebi/biosamples/service/validation/JsonSchema.java similarity index 93% rename from utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/JsonSchema.java rename to core/src/main/java/uk/ac/ebi/biosamples/service/validation/JsonSchema.java index 1a6874d7b..86e51bf7a 100644 --- a/utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/JsonSchema.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/validation/JsonSchema.java @@ -1,88 +1,88 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.validation; - -import com.fasterxml.jackson.databind.JsonNode; - -public class JsonSchema { - private String id; - private String name; - private String version; - private String title; - private String description; - private String domain; - private String metaSchema; - private JsonNode schema; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getDomain() { - return domain; - } - - public void setDomain(String domain) { - this.domain = domain; - } - - public String getMetaSchema() { - return metaSchema; - } - - public void setMetaSchema(String metaSchema) { - this.metaSchema = metaSchema; - } - - public JsonNode getSchema() { - return schema; - } - - public void setSchema(JsonNode schema) { - this.schema = schema; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.validation; + +import com.fasterxml.jackson.databind.JsonNode; + +public class JsonSchema { + private String id; + private String name; + private String version; + private String title; + private String description; + private String domain; + private String metaSchema; + private JsonNode schema; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getDomain() { + return domain; + } + + public void setDomain(String domain) { + this.domain = domain; + } + + public String getMetaSchema() { + return metaSchema; + } + + public void setMetaSchema(String metaSchema) { + this.metaSchema = metaSchema; + } + + public JsonNode getSchema() { + return schema; + } + + public void setSchema(JsonNode schema) { + this.schema = schema; + } +} diff --git a/utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/SchemaValidationService.java b/core/src/main/java/uk/ac/ebi/biosamples/service/validation/SchemaValidationService.java similarity index 92% rename from utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/SchemaValidationService.java rename to core/src/main/java/uk/ac/ebi/biosamples/service/validation/SchemaValidationService.java index a7076f46e..803bb2ebf 100644 --- a/utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/SchemaValidationService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/validation/SchemaValidationService.java @@ -1,83 +1,83 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.validation; - -import com.fasterxml.jackson.databind.ObjectMapper; -import javax.validation.ValidationException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.http.HttpStatus; -import org.springframework.stereotype.Service; -import org.springframework.web.server.ResponseStatusException; -import uk.ac.ebi.biosamples.BioSamplesConstants; -import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; - -@Service -public class SchemaValidationService { - private final Logger log = LoggerFactory.getLogger(getClass()); - - private final BioSamplesProperties bioSamplesProperties; - private final ObjectMapper objectMapper; - private final ValidatorI validator; - - public SchemaValidationService( - final ObjectMapper mapper, - final BioSamplesProperties bioSamplesProperties, - @Qualifier("elixirValidator") final ValidatorI validator) { - objectMapper = mapper; - this.bioSamplesProperties = bioSamplesProperties; - this.validator = validator; - } - - public Sample validate(final Sample sample, final String webinId) { - final String schemaId = - sample.getCharacteristics().stream() - .filter( - s -> BioSamplesConstants.CHECKLIST_ATTRIBUTES.contains(s.getType().toLowerCase())) - .findFirst() - .map(Attribute::getValue) - .orElse(bioSamplesProperties.getBiosamplesDefaultSchema()); - - // check if the schema referred in the sample is accessible or protected - if (schemaId.equalsIgnoreCase(bioSamplesProperties.getBiosamplesRestrictedSchema()) - && !bioSamplesProperties.getBiosamplesClientWebinUsername().equals(webinId)) { - throw new ResponseStatusException( - HttpStatus.FORBIDDEN, - "Validation of sample metadata against this schema is not permitted"); - } - - try { - final String sampleString = objectMapper.writeValueAsString(sample); - - validator.validateById(schemaId, sampleString); - - /*if (sampleAttributes.stream() - .noneMatch(attribute -> attribute.getType().equalsIgnoreCase("checklist"))) { - sampleAttributes.add(Attribute.build("checklist", schemaId)); - - return Sample.Builder.fromSample(sample).withAttributes(sampleAttributes).build(); - }*/ - - return sample; - } catch (final ValidationException | GlobalExceptions.SampleValidationException e) { - throw new GlobalExceptions.SchemaValidationException(e.getMessage(), e); - } catch (final Exception e) { - log.error("Schema validation error: " + e.getMessage(), e); - - throw new GlobalExceptions.SchemaValidationException( - "Sample validation error: " + e.getMessage(), e); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.validation; + +import com.fasterxml.jackson.databind.ObjectMapper; +import javax.validation.ValidationException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; +import uk.ac.ebi.biosamples.BioSamplesConstants; +import uk.ac.ebi.biosamples.BioSamplesProperties; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; + +@Service +public class SchemaValidationService { + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final BioSamplesProperties bioSamplesProperties; + private final ObjectMapper objectMapper; + private final ValidatorI validator; + + public SchemaValidationService( + final ObjectMapper mapper, + final BioSamplesProperties bioSamplesProperties, + @Qualifier("elixirValidator") final ValidatorI validator) { + objectMapper = mapper; + this.bioSamplesProperties = bioSamplesProperties; + this.validator = validator; + } + + public Sample validate(final Sample sample, final String webinId) { + final String schemaId = + sample.getCharacteristics().stream() + .filter( + s -> BioSamplesConstants.CHECKLIST_ATTRIBUTES.contains(s.getType().toLowerCase())) + .findFirst() + .map(Attribute::getValue) + .orElse(bioSamplesProperties.getBiosamplesDefaultSchema()); + + // check if the schema referred in the sample is accessible or protected + if (schemaId.equalsIgnoreCase(bioSamplesProperties.getBiosamplesRestrictedSchema()) + && !bioSamplesProperties.getBiosamplesClientWebinUsername().equals(webinId)) { + throw new ResponseStatusException( + HttpStatus.FORBIDDEN, + "Validation of sample metadata against this schema is not permitted"); + } + + try { + final String sampleString = objectMapper.writeValueAsString(sample); + + validator.validateById(schemaId, sampleString); + + /*if (sampleAttributes.stream() + .noneMatch(attribute -> attribute.getType().equalsIgnoreCase("checklist"))) { + sampleAttributes.add(Attribute.build("checklist", schemaId)); + + return Sample.Builder.fromSample(sample).withAttributes(sampleAttributes).build(); + }*/ + + return sample; + } catch (final ValidationException | GlobalExceptions.SampleValidationException e) { + throw new GlobalExceptions.SchemaValidationException(e.getMessage(), e); + } catch (final Exception e) { + log.error("Schema validation error: " + e.getMessage(), e); + + throw new GlobalExceptions.SchemaValidationException( + "Sample validation error: " + e.getMessage(), e); + } + } +} diff --git a/utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/ValidationRequest.java b/core/src/main/java/uk/ac/ebi/biosamples/service/validation/ValidationRequest.java similarity index 93% rename from utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/ValidationRequest.java rename to core/src/main/java/uk/ac/ebi/biosamples/service/validation/ValidationRequest.java index 56fc2c195..a6f374d4b 100644 --- a/utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/ValidationRequest.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/validation/ValidationRequest.java @@ -1,39 +1,39 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.validation; - -import com.fasterxml.jackson.databind.JsonNode; - -public class ValidationRequest { - private JsonNode schema; - private JsonNode data; - - public ValidationRequest(JsonNode schema, JsonNode data) { - this.schema = schema; - this.data = data; - } - - public JsonNode getSchema() { - return schema; - } - - public void setSchema(JsonNode schema) { - this.schema = schema; - } - - public JsonNode getData() { - return data; - } - - public void setData(JsonNode data) { - this.data = data; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.validation; + +import com.fasterxml.jackson.databind.JsonNode; + +public class ValidationRequest { + private JsonNode schema; + private JsonNode data; + + public ValidationRequest(JsonNode schema, JsonNode data) { + this.schema = schema; + this.data = data; + } + + public JsonNode getSchema() { + return schema; + } + + public void setSchema(JsonNode schema) { + this.schema = schema; + } + + public JsonNode getData() { + return data; + } + + public void setData(JsonNode data) { + this.data = data; + } +} diff --git a/utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/ValidatorI.java b/core/src/main/java/uk/ac/ebi/biosamples/service/validation/ValidatorI.java similarity index 87% rename from utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/ValidatorI.java rename to core/src/main/java/uk/ac/ebi/biosamples/service/validation/ValidatorI.java index f32ff2573..5554817ac 100644 --- a/utils/validation/src/main/java/uk/ac/ebi/biosamples/validation/ValidatorI.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/validation/ValidatorI.java @@ -1,21 +1,21 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.validation; - -import java.io.IOException; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; - -public interface ValidatorI { - void validate(String schemaId, String sample) throws IOException; - - String validateById(String schemaId, String document) - throws IOException, GlobalExceptions.SchemaValidationException; -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service.validation; + +import java.io.IOException; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; + +public interface ValidatorI { + void validate(String schemaId, String sample) throws IOException; + + String validateById(String schemaId, String document) + throws IOException, GlobalExceptions.SchemaValidationException; +} diff --git a/models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlSitemap.java b/core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlSitemap.java similarity index 84% rename from models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlSitemap.java rename to core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlSitemap.java index f624e6097..e7e8a7a8f 100644 --- a/models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlSitemap.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlSitemap.java @@ -1,45 +1,44 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.redfin.sitemapgenerator.W3CDateFormat; -import java.util.Date; -import java.util.TimeZone; -import javax.xml.bind.annotation.XmlElement; - -/** Class representing the sitemap entry */ -public class XmlSitemap { - - private final W3CDateFormat dateFormat; - - public XmlSitemap() { - this(""); - } - - public XmlSitemap(final String loc) { - dateFormat = new W3CDateFormat(W3CDateFormat.Pattern.DAY); - dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); - lastmod = dateFormat.format(new Date()); - this.loc = loc; - } - - @XmlElement private final String loc; - - @XmlElement private final String lastmod; - - public String getLoc() { - return loc; - } - - public String getLastmod() { - return lastmod; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.sitemap.model; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.redfin.sitemapgenerator.W3CDateFormat; +import java.util.Date; +import java.util.TimeZone; + +/** Class representing the sitemap entry */ +public class XmlSitemap { + private final W3CDateFormat dateFormat; + + public XmlSitemap() { + this(""); + } + + public XmlSitemap(final String loc) { + dateFormat = new W3CDateFormat(W3CDateFormat.Pattern.DAY); + dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); + lastmod = dateFormat.format(new Date()); + this.loc = loc; + } + + @JacksonXmlProperty private final String loc; + + @JacksonXmlProperty private final String lastmod; + + public String getLoc() { + return loc; + } + + public String getLastmod() { + return lastmod; + } +} diff --git a/models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlSitemapIndex.java b/core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlSitemapIndex.java similarity index 70% rename from models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlSitemapIndex.java rename to core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlSitemapIndex.java index 695e0fd16..e4c4db6fc 100644 --- a/models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlSitemapIndex.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlSitemapIndex.java @@ -1,32 +1,34 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import java.util.ArrayList; -import java.util.Collection; -import javax.xml.bind.annotation.*; - -/** @author mrelac */ -@XmlAccessorType(value = XmlAccessType.NONE) -@XmlRootElement(name = "sitemapindex") -public class XmlSitemapIndex { - - @XmlElements({@XmlElement(name = "sitemap", type = uk.ac.ebi.biosamples.model.XmlSitemap.class)}) - private final Collection xmlSitemaps = new ArrayList(); - - public void addSitemap(final XmlSitemap xmlSitemap) { - xmlSitemaps.add(xmlSitemap); - } - - public Collection getXmlSitemaps() { - return xmlSitemaps; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.sitemap.model; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; +import java.util.ArrayList; +import java.util.Collection; +import javax.xml.bind.annotation.*; + +/** @author mrelac */ +@JacksonXmlRootElement(localName = "sitemapindex") +public class XmlSitemapIndex { + @JacksonXmlElementWrapper(useWrapping = false) + @JacksonXmlProperty(localName = "sitemap") + private final Collection xmlSitemaps = new ArrayList(); + + public void addSitemap(final XmlSitemap xmlSitemap) { + xmlSitemaps.add(xmlSitemap); + } + + public Collection getXmlSitemaps() { + return xmlSitemaps; + } +} diff --git a/models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlSitemapSet.java b/core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlSitemapSet.java similarity index 67% rename from models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlSitemapSet.java rename to core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlSitemapSet.java index 8e9f0898e..3409defbf 100644 --- a/models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlSitemapSet.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlSitemapSet.java @@ -1,31 +1,33 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import java.util.ArrayList; -import java.util.Collection; -import javax.xml.bind.annotation.*; - -/** @author mrelac */ -@XmlAccessorType(value = XmlAccessType.NONE) -@XmlRootElement(name = "sitemapset") -public class XmlSitemapSet { - @XmlElements({@XmlElement(name = "model", type = XmlSitemap.class)}) - private final Collection xmlSitemaps = new ArrayList(); - - public void addSitemap(final XmlSitemap xmlSitemap) { - xmlSitemaps.add(xmlSitemap); - } - - public Collection getXmlSitemaps() { - return xmlSitemaps; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.sitemap.model; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; +import java.util.ArrayList; +import java.util.Collection; + +@JacksonXmlRootElement(localName = "sitemapset") +public class XmlSitemapSet { + + @JacksonXmlElementWrapper(localName = "model", useWrapping = false) + @JacksonXmlProperty(localName = "sitemap") + private final Collection xmlSitemaps = new ArrayList<>(); + + public void addSitemap(final XmlSitemap xmlSitemap) { + xmlSitemaps.add(xmlSitemap); + } + + public Collection getXmlSitemaps() { + return xmlSitemaps; + } +} diff --git a/models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlUrl.java b/core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlUrl.java similarity index 77% rename from models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlUrl.java rename to core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlUrl.java index 96c245396..28e26c7c1 100644 --- a/models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlUrl.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlUrl.java @@ -1,148 +1,140 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import com.redfin.sitemapgenerator.W3CDateFormat; -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.Date; -import java.util.TimeZone; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; - -/** @author mrelac */ -@XmlAccessorType(value = XmlAccessType.NONE) -@XmlRootElement(name = "url") -public class XmlUrl { - - private final W3CDateFormat dateFormat = getDateFormat(); - - private XmlUrl() { - this(null, LocalDate.now(), ChangeFrequency.WEEKLY, Priority.MEDIUM); - } - - // - // private XmlUrl(String loc, Priority priority) { - // lastmod = dateFormat.format(new Date()); - // this.loc = loc; - // this.priority = priority.getValue(); - // this.changefreq = ChangeFrequency.WEEKLY.getValue(); - // } - - private XmlUrl( - final String loc, - final LocalDate lastModifiedDate, - final ChangeFrequency freq, - final Priority priority) { - lastmod = - dateFormat.format(Date.from(lastModifiedDate.atStartOfDay(ZoneId.of("GMT")).toInstant())); - this.loc = loc; - this.priority = priority.getValue(); - changefreq = freq.getValue(); - } - - private W3CDateFormat getDateFormat() { - final W3CDateFormat dateFormat = new W3CDateFormat(W3CDateFormat.Pattern.DAY); - dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); - return dateFormat; - } - - @XmlElement private final String loc; - - @XmlElement private final String lastmod; - - @XmlElement private final String changefreq; - - @XmlElement private final String priority; - - public String getLoc() { - return loc; - } - - public String getPriority() { - return priority; - } - - public String getChangefreq() { - return changefreq; - } - - public String getLastmod() { - return lastmod; - } - - public enum Priority { - HIGH("1.0"), - MEDIUM("0.5"); - - private final String value; - - Priority(final String value) { - this.value = value; - } - - public String getValue() { - return value; - } - } - - public static class XmlUrlBuilder { - private final String loc; - private LocalDate lastModification; - private ChangeFrequency frequency; - private Priority priority; - - public XmlUrlBuilder(final String loc) { - this.loc = loc; - lastModification = LocalDate.now(); - frequency = ChangeFrequency.WEEKLY; - priority = Priority.MEDIUM; - } - - public XmlUrlBuilder lastModified(final LocalDate date) { - lastModification = date; - return this; - } - - public XmlUrlBuilder changeWithFrequency(final ChangeFrequency freq) { - frequency = freq; - return this; - } - - public XmlUrlBuilder hasPriority(final Priority p) { - priority = p; - return this; - } - - public XmlUrl build() { - return new XmlUrl(loc, lastModification, frequency, priority); - } - } - - public enum ChangeFrequency { - DAILY("daily"), - WEEKLY("weekly"), - MONTHLY("monthly"), - YEARLY("yearly"); - - private final String freq; - - ChangeFrequency(final String freq) { - this.freq = freq; - } - - public String getValue() { - return freq; - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.sitemap.model; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; +import com.redfin.sitemapgenerator.W3CDateFormat; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.Date; +import java.util.TimeZone; + +@JacksonXmlRootElement(localName = "url") +public class XmlUrl { + + private final W3CDateFormat dateFormat = getDateFormat(); + + private XmlUrl() { + this(null, LocalDate.now(), ChangeFrequency.WEEKLY, Priority.MEDIUM); + } + + private XmlUrl( + final String loc, + final LocalDate lastModifiedDate, + final ChangeFrequency freq, + final Priority priority) { + lastmod = + dateFormat.format(Date.from(lastModifiedDate.atStartOfDay(ZoneId.of("GMT")).toInstant())); + this.loc = loc; + this.priority = priority.getValue(); + changefreq = freq.getValue(); + } + + private W3CDateFormat getDateFormat() { + final W3CDateFormat dateFormat = new W3CDateFormat(W3CDateFormat.Pattern.DAY); + dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); + return dateFormat; + } + + @JacksonXmlProperty(localName = "loc") + private final String loc; + + @JacksonXmlProperty(localName = "lastmod") + private final String lastmod; + + @JacksonXmlProperty(localName = "changefreq") + private final String changefreq; + + @JacksonXmlProperty(localName = "priority") + private final String priority; + + public String getLoc() { + return loc; + } + + public String getPriority() { + return priority; + } + + public String getChangefreq() { + return changefreq; + } + + public String getLastmod() { + return lastmod; + } + + public enum Priority { + HIGH("1.0"), + MEDIUM("0.5"); + + private final String value; + + Priority(final String value) { + this.value = value; + } + + public String getValue() { + return value; + } + } + + public static class XmlUrlBuilder { + private final String loc; + private LocalDate lastModification; + private ChangeFrequency frequency; + private Priority priority; + + public XmlUrlBuilder(final String loc) { + this.loc = loc; + lastModification = LocalDate.now(); + frequency = ChangeFrequency.WEEKLY; + priority = Priority.MEDIUM; + } + + public XmlUrlBuilder lastModified(final LocalDate date) { + lastModification = date; + return this; + } + + public XmlUrlBuilder changeWithFrequency(final ChangeFrequency freq) { + frequency = freq; + return this; + } + + public XmlUrlBuilder hasPriority(final Priority p) { + priority = p; + return this; + } + + public XmlUrl build() { + return new XmlUrl(loc, lastModification, frequency, priority); + } + } + + public enum ChangeFrequency { + DAILY("daily"), + WEEKLY("weekly"), + MONTHLY("monthly"), + YEARLY("yearly"); + + private final String freq; + + ChangeFrequency(final String freq) { + this.freq = freq; + } + + public String getValue() { + return freq; + } + } +} diff --git a/models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlUrlSet.java b/core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlUrlSet.java similarity index 62% rename from models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlUrlSet.java rename to core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlUrlSet.java index 9c1bf5b0d..512e6fc3f 100644 --- a/models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/XmlUrlSet.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/XmlUrlSet.java @@ -1,48 +1,46 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import javax.xml.bind.annotation.*; - -/** @author mrelac */ -@XmlAccessorType(value = XmlAccessType.NONE) -@XmlRootElement(name = "urlset") -public class XmlUrlSet { - - private static final int MAX_SITEMAP_ENTRIES = 50000; - - @XmlElements({@XmlElement(name = "url", type = XmlUrl.class)}) - private final List xmlUrls = new ArrayList(); - - public void addUrl(final XmlUrl xmlUrl) { - - final int size = xmlUrls.size(); - - // Reservoir sampling to get random entries in the model (this will get them all - // eventually, but not exceed the model entry size restriction) - if (size >= MAX_SITEMAP_ENTRIES) { - // Randomly replace elements in the reservoir with a decreasing probability. - final int idx = new Double(Math.floor(Math.random() * size)).intValue(); - if (idx < MAX_SITEMAP_ENTRIES) { - xmlUrls.set(idx, xmlUrl); - } - } else { - xmlUrls.add(xmlUrl); - } - } - - public Collection getXmlUrls() { - return xmlUrls; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.sitemap.model; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@JacksonXmlRootElement(localName = "urlset") +public class XmlUrlSet { + + private static final int MAX_SITEMAP_ENTRIES = 50000; + + @JacksonXmlProperty(localName = "url") + @JacksonXmlElementWrapper(useWrapping = false) + private final List xmlUrls = new ArrayList<>(); + + public void addUrl(final XmlUrl xmlUrl) { + final int size = xmlUrls.size(); + + // Reservoir sampling + if (size >= MAX_SITEMAP_ENTRIES) { + final int idx = (int) Math.floor(Math.random() * size); + if (idx < MAX_SITEMAP_ENTRIES) { + xmlUrls.set(idx, xmlUrl); + } + } else { + xmlUrls.add(xmlUrl); + } + } + + public Collection getXmlUrls() { + return xmlUrls; + } +} diff --git a/models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/package-info.java b/core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/package-info.java similarity index 82% rename from models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/package-info.java rename to core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/package-info.java index aefc7eabd..16b153d56 100644 --- a/models/sitemap/src/main/java/uk/ac/ebi/biosamples/model/package-info.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/sitemap/model/package-info.java @@ -1,7 +1,7 @@ -@XmlSchema( - namespace = "http://www.sitemaps.org/schemas/sitemap/0.9", - elementFormDefault = XmlNsForm.QUALIFIED) -package uk.ac.ebi.biosamples.model; - -import javax.xml.bind.annotation.XmlNsForm; -import javax.xml.bind.annotation.XmlSchema; +@XmlSchema( + namespace = "http://www.sitemaps.org/schemas/sitemap/0.9", + elementFormDefault = XmlNsForm.QUALIFIED) +package uk.ac.ebi.biosamples.sitemap.model; + +import javax.xml.bind.annotation.XmlNsForm; +import javax.xml.bind.annotation.XmlSchema; diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrDeserializer.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrDeserializer.java similarity index 97% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrDeserializer.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrDeserializer.java index 95bafe47a..8d32200c1 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrDeserializer.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrDeserializer.java @@ -1,35 +1,35 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import java.io.IOException; -import java.time.Instant; - -public class CustomInstantSolrDeserializer extends StdDeserializer { - - public CustomInstantSolrDeserializer() { - this(null); - } - - private CustomInstantSolrDeserializer(final Class t) { - super(t); - } - - @Override - public Instant deserialize(final JsonParser p, final DeserializationContext ctxt) - throws IOException, JsonProcessingException { - return Instant.parse(p.readValueAs(String.class)); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import java.io.IOException; +import java.time.Instant; + +public class CustomInstantSolrDeserializer extends StdDeserializer { + + public CustomInstantSolrDeserializer() { + this(null); + } + + private CustomInstantSolrDeserializer(final Class t) { + super(t); + } + + @Override + public Instant deserialize(final JsonParser p, final DeserializationContext ctxt) + throws IOException, JsonProcessingException { + return Instant.parse(p.readValueAs(String.class)); + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrSerializer.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrSerializer.java similarity index 97% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrSerializer.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrSerializer.java index 2afd45c00..b8e7305c1 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrSerializer.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/CustomInstantSolrSerializer.java @@ -1,36 +1,36 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import java.io.IOException; -import java.time.Instant; -import java.time.format.DateTimeFormatter; - -public class CustomInstantSolrSerializer extends StdSerializer { - - public CustomInstantSolrSerializer() { - this(null); - } - - private CustomInstantSolrSerializer(final Class t) { - super(t); - } - - @Override - public void serialize(final Instant value, final JsonGenerator gen, final SerializerProvider arg2) - throws IOException, JsonProcessingException { - gen.writeString(DateTimeFormatter.ISO_INSTANT.format(value)); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.time.Instant; +import java.time.format.DateTimeFormatter; + +public class CustomInstantSolrSerializer extends StdSerializer { + + public CustomInstantSolrSerializer() { + this(null); + } + + private CustomInstantSolrSerializer(final Class t) { + super(t); + } + + @Override + public void serialize(final Instant value, final JsonGenerator gen, final SerializerProvider arg2) + throws IOException, JsonProcessingException { + gen.writeString(DateTimeFormatter.ISO_INSTANT.format(value)); + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/SolrConfig.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/SolrConfig.java similarity index 97% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/SolrConfig.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/SolrConfig.java index 43e0497f6..987eeddff 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/SolrConfig.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/SolrConfig.java @@ -1,20 +1,20 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr; - -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.solr.repository.config.EnableSolrRepositories; - -@Configuration -@EnableSolrRepositories(basePackages = "uk.ac.ebi.biosamples.solr.repo") -@ComponentScan -public class SolrConfig {} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr; + +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.solr.repository.config.EnableSolrRepositories; + +@Configuration +@EnableSolrRepositories(basePackages = "uk.ac.ebi.biosamples.solr.repo") +@ComponentScan +public class SolrConfig {} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/SolrSample.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/SolrSample.java similarity index 96% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/SolrSample.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/model/SolrSample.java index e77d3ea9c..b5f82c214 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/SolrSample.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/SolrSample.java @@ -1,300 +1,300 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.model; - -import java.util.*; -import java.util.Map.Entry; -import java.util.stream.Collectors; -import org.springframework.data.annotation.Id; -import org.springframework.data.solr.core.mapping.Dynamic; -import org.springframework.data.solr.core.mapping.Indexed; -import org.springframework.data.solr.core.mapping.SolrDocument; -import uk.ac.ebi.biosamples.solr.service.SolrFieldService; - -@SolrDocument(solrCoreName = "samples") -public class SolrSample { - - /** Use the accession as the primary document identifier */ - @Id - @Indexed(name = "id", required = true) - protected String accession; - - @Indexed(name = "name_s", required = true, copyTo = "keywords_ss") - protected String name; - - @Indexed(name = "domain_s", required = true) - protected String domain; - - @Indexed(name = "webinId_s", required = true) - protected String webinSubmissionAcccountId; - - @Indexed(name = "status_s", required = true) - protected String status; - - // TODO - /** - * Store the release date as a string so that it can be used easily by solr Use a TrieDate type - * for better range query performance - */ - @Indexed(name = "release_dt", required = true, type = "date") - protected String release; - /** - * Store the update date as a string so that it can be used easily by solr Use a TrieDate type for - * better range query performance - */ - @Indexed(name = "update_dt", required = true, type = "date") // TODO why type=date ? - protected String update; - - @Indexed(name = "modified_dt", required = true, type = "date") // TODO why type=date ? - protected String modified; - - @Indexed(name = "indexed_dt", required = true, type = "date") // TODO why type=date ? - protected String indexed; - - @Indexed(name = "*_av_ss") - @Dynamic - protected Map> attributeValues; - - @Indexed( - name = "*_ai_ss", - copyTo = {"ontologyiri_ss"}) - @Dynamic - protected Map> attributeIris; - - @Indexed(name = "*_au_ss") - @Dynamic - protected Map> attributeUnits; - - /** Relationships for which this sample is the source */ - @Indexed(name = "*_or_ss", copyTo = "keywords_ss") - @Dynamic - protected Map> outgoingRelationships; - - /** Relationships for which this sample is the target */ - @Indexed(name = "*_ir_ss", copyTo = "keywords_ss") - @Dynamic - protected Map> incomingRelationships; - - /** - * This field shouldn't be populated directly, instead Solr will copy all the ontology terms from - * the attributes into it. - */ - @Indexed(name = "ontologyiri_ss") - protected List ontologyIris; - - /** This field is used to store external references only */ - @Indexed(name = "*_erd_ss", copyTo = "facetfields_ss") - @Dynamic - protected Map> externalReferencesData; - - /** - * This field is required to get a list of attribute to use for faceting. * It includes attributes - * and relationships of the sample Since faceting does not require it to be stored, it wont be to - * save space. - */ - @Indexed(name = "facetfields_ss") - protected List facetFields; - - /** This field is required to store the ontology expansion and attributes from related samples */ - @Indexed(name = "keywords_ss") - protected List keywords; - - public SolrSample() {} - - public String getAccession() { - return accession; - } - - public String getName() { - return name; - } - - public String getDomain() { - return domain; - } - - public String getWebinSubmissionAcccountId() { - return webinSubmissionAcccountId; - } - - public String getStatus() { - return status; - } - - public String getRelease() { - return release; - } - - public String getUpdate() { - return update; - } - - public String getModified() { - return modified; - } - - public String getIndexed() { - return indexed; - } - - public Map> getAttributeValues() { - return attributeValues; - } - - public Map> getAttributeIris() { - return attributeIris; - } - - public Map> getAttributeUnits() { - return attributeUnits; - } - - public List getOntologyIris() { - return ontologyIris; - } - - public Map> getExternalReferencesData() { - return externalReferencesData; - } - - public Map> getIncomingRelationships() { - return incomingRelationships; - } - - public Map> getOutgoingRelationships() { - return outgoingRelationships; - } - - public List getKeywords() { - return keywords; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - sb.append("Sample("); - sb.append(name); - sb.append(","); - sb.append(accession); - sb.append(","); - sb.append(release); - sb.append(","); - sb.append(update); - sb.append(","); - sb.append(attributeValues); - sb.append(","); - sb.append(attributeIris); - sb.append(","); - sb.append(attributeUnits); - sb.append(","); - sb.append(outgoingRelationships); - sb.append(","); - sb.append(incomingRelationships); - sb.append(","); - sb.append(externalReferencesData); - sb.append(")"); - return sb.toString(); - } - - /** - * Avoid using this directly, use the SolrSampleToSampleConverter or SampleToSolrSampleConverter - * instead - */ - public static SolrSample build( - final String name, - final String accession, - final String domain, - final String webinSubmissionAcccountId, - final String status, - final String release, - final String update, - final String modified, - final String indexed, - Map> attributeValues, - Map> attributeIris, - Map> attributeUnits, - final Map> outgoingRelationships, - final Map> incomingRelationships, - final Map> externalReferencesData, - final List keywords) { - - // TODO validate maps - if (attributeValues == null) { - attributeValues = new HashMap<>(); - } - if (attributeIris == null) { - attributeIris = new HashMap<>(); - } - if (attributeUnits == null) { - attributeUnits = new HashMap<>(); - } - - final SolrSample sample = new SolrSample(); - sample.accession = accession; - sample.name = name; - sample.release = release; - sample.update = update; - sample.domain = domain; - sample.webinSubmissionAcccountId = webinSubmissionAcccountId; - sample.status = status; - sample.modified = modified; - sample.indexed = indexed; - sample.attributeValues = attributeValues; - sample.attributeIris = attributeIris; - sample.attributeUnits = attributeUnits; - sample.incomingRelationships = incomingRelationships; - sample.outgoingRelationships = outgoingRelationships; - sample.externalReferencesData = externalReferencesData; - - final SortedSet facetFieldSet = new TreeSet<>(); - if (!attributeValues.keySet().isEmpty()) { - for (final String attributeValueKey : attributeValues.keySet()) { - facetFieldSet.add(attributeValueKey + "_av_ss"); - } - } - - if (outgoingRelationships != null && !outgoingRelationships.keySet().isEmpty()) { - for (final String outgoingRelationshipsKey : outgoingRelationships.keySet()) { - facetFieldSet.add(outgoingRelationshipsKey + "_or_ss"); - } - } - - if (incomingRelationships != null && !incomingRelationships.keySet().isEmpty()) { - for (final String incomingRelationshipsKey : incomingRelationships.keySet()) { - facetFieldSet.add(incomingRelationshipsKey + "_ir_ss"); - } - } - - if (externalReferencesData != null && !externalReferencesData.keySet().isEmpty()) { - for (final String externalReferencesDataKey : externalReferencesData.keySet()) { - facetFieldSet.add(externalReferencesDataKey + "_erd_ss"); - } - } - - sample.facetFields = new ArrayList<>(facetFieldSet); - - // copy into the other fields - // this should be done in a copyfield but that doesn't work for some reason? - final Set searchTerms = new HashSet<>(); - - searchTerms.add(sample.name.toLowerCase()); - searchTerms.addAll(keywords); - - for (final Entry> entry : attributeValues.entrySet()) { - searchTerms.add(SolrFieldService.decodeFieldName(entry.getKey()).toLowerCase()); - searchTerms.addAll( - entry.getValue().stream().map(String::toLowerCase).collect(Collectors.toSet())); - } - sample.keywords = new ArrayList<>(searchTerms); - - return sample; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.model; + +import java.util.*; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import org.springframework.data.annotation.Id; +import org.springframework.data.solr.core.mapping.Dynamic; +import org.springframework.data.solr.core.mapping.Indexed; +import org.springframework.data.solr.core.mapping.SolrDocument; +import uk.ac.ebi.biosamples.solr.service.SolrFieldService; + +@SolrDocument(solrCoreName = "samples") +public class SolrSample { + + /** Use the accession as the primary document identifier */ + @Id + @Indexed(name = "id", required = true) + protected String accession; + + @Indexed(name = "name_s", required = true, copyTo = "keywords_ss") + protected String name; + + @Indexed(name = "domain_s", required = true) + protected String domain; + + @Indexed(name = "webinId_s", required = true) + protected String webinSubmissionAcccountId; + + @Indexed(name = "status_s", required = true) + protected String status; + + // TODO + /** + * Store the release date as a string so that it can be used easily by solr Use a TrieDate type + * for better range query performance + */ + @Indexed(name = "release_dt", required = true, type = "date") + protected String release; + /** + * Store the update date as a string so that it can be used easily by solr Use a TrieDate type for + * better range query performance + */ + @Indexed(name = "update_dt", required = true, type = "date") // TODO why type=date ? + protected String update; + + @Indexed(name = "modified_dt", required = true, type = "date") // TODO why type=date ? + protected String modified; + + @Indexed(name = "indexed_dt", required = true, type = "date") // TODO why type=date ? + protected String indexed; + + @Indexed(name = "*_av_ss") + @Dynamic + protected Map> attributeValues; + + @Indexed( + name = "*_ai_ss", + copyTo = {"ontologyiri_ss"}) + @Dynamic + protected Map> attributeIris; + + @Indexed(name = "*_au_ss") + @Dynamic + protected Map> attributeUnits; + + /** Relationships for which this sample is the source */ + @Indexed(name = "*_or_ss", copyTo = "keywords_ss") + @Dynamic + protected Map> outgoingRelationships; + + /** Relationships for which this sample is the target */ + @Indexed(name = "*_ir_ss", copyTo = "keywords_ss") + @Dynamic + protected Map> incomingRelationships; + + /** + * This field shouldn't be populated directly, instead Solr will copy all the ontology terms from + * the attributes into it. + */ + @Indexed(name = "ontologyiri_ss") + protected List ontologyIris; + + /** This field is used to store external references only */ + @Indexed(name = "*_erd_ss", copyTo = "facetfields_ss") + @Dynamic + protected Map> externalReferencesData; + + /** + * This field is required to get a list of attribute to use for faceting. * It includes attributes + * and relationships of the sample Since faceting does not require it to be stored, it wont be to + * save space. + */ + @Indexed(name = "facetfields_ss") + protected List facetFields; + + /** This field is required to store the ontology expansion and attributes from related samples */ + @Indexed(name = "keywords_ss") + protected List keywords; + + public SolrSample() {} + + public String getAccession() { + return accession; + } + + public String getName() { + return name; + } + + public String getDomain() { + return domain; + } + + public String getWebinSubmissionAcccountId() { + return webinSubmissionAcccountId; + } + + public String getStatus() { + return status; + } + + public String getRelease() { + return release; + } + + public String getUpdate() { + return update; + } + + public String getModified() { + return modified; + } + + public String getIndexed() { + return indexed; + } + + public Map> getAttributeValues() { + return attributeValues; + } + + public Map> getAttributeIris() { + return attributeIris; + } + + public Map> getAttributeUnits() { + return attributeUnits; + } + + public List getOntologyIris() { + return ontologyIris; + } + + public Map> getExternalReferencesData() { + return externalReferencesData; + } + + public Map> getIncomingRelationships() { + return incomingRelationships; + } + + public Map> getOutgoingRelationships() { + return outgoingRelationships; + } + + public List getKeywords() { + return keywords; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("Sample("); + sb.append(name); + sb.append(","); + sb.append(accession); + sb.append(","); + sb.append(release); + sb.append(","); + sb.append(update); + sb.append(","); + sb.append(attributeValues); + sb.append(","); + sb.append(attributeIris); + sb.append(","); + sb.append(attributeUnits); + sb.append(","); + sb.append(outgoingRelationships); + sb.append(","); + sb.append(incomingRelationships); + sb.append(","); + sb.append(externalReferencesData); + sb.append(")"); + return sb.toString(); + } + + /** + * Avoid using this directly, use the SolrSampleToSampleConverter or SampleToSolrSampleConverter + * instead + */ + public static SolrSample build( + final String name, + final String accession, + final String domain, + final String webinSubmissionAcccountId, + final String status, + final String release, + final String update, + final String modified, + final String indexed, + Map> attributeValues, + Map> attributeIris, + Map> attributeUnits, + final Map> outgoingRelationships, + final Map> incomingRelationships, + final Map> externalReferencesData, + final List keywords) { + + // TODO validate maps + if (attributeValues == null) { + attributeValues = new HashMap<>(); + } + if (attributeIris == null) { + attributeIris = new HashMap<>(); + } + if (attributeUnits == null) { + attributeUnits = new HashMap<>(); + } + + final SolrSample sample = new SolrSample(); + sample.accession = accession; + sample.name = name; + sample.release = release; + sample.update = update; + sample.domain = domain; + sample.webinSubmissionAcccountId = webinSubmissionAcccountId; + sample.status = status; + sample.modified = modified; + sample.indexed = indexed; + sample.attributeValues = attributeValues; + sample.attributeIris = attributeIris; + sample.attributeUnits = attributeUnits; + sample.incomingRelationships = incomingRelationships; + sample.outgoingRelationships = outgoingRelationships; + sample.externalReferencesData = externalReferencesData; + + final SortedSet facetFieldSet = new TreeSet<>(); + if (!attributeValues.keySet().isEmpty()) { + for (final String attributeValueKey : attributeValues.keySet()) { + facetFieldSet.add(attributeValueKey + "_av_ss"); + } + } + + if (outgoingRelationships != null && !outgoingRelationships.keySet().isEmpty()) { + for (final String outgoingRelationshipsKey : outgoingRelationships.keySet()) { + facetFieldSet.add(outgoingRelationshipsKey + "_or_ss"); + } + } + + if (incomingRelationships != null && !incomingRelationships.keySet().isEmpty()) { + for (final String incomingRelationshipsKey : incomingRelationships.keySet()) { + facetFieldSet.add(incomingRelationshipsKey + "_ir_ss"); + } + } + + if (externalReferencesData != null && !externalReferencesData.keySet().isEmpty()) { + for (final String externalReferencesDataKey : externalReferencesData.keySet()) { + facetFieldSet.add(externalReferencesDataKey + "_erd_ss"); + } + } + + sample.facetFields = new ArrayList<>(facetFieldSet); + + // copy into the other fields + // this should be done in a copyfield but that doesn't work for some reason? + final Set searchTerms = new HashSet<>(); + + searchTerms.add(sample.name.toLowerCase()); + searchTerms.addAll(keywords); + + for (final Entry> entry : attributeValues.entrySet()) { + searchTerms.add(SolrFieldService.decodeFieldName(entry.getKey()).toLowerCase()); + searchTerms.addAll( + entry.getValue().stream().map(String::toLowerCase).collect(Collectors.toSet())); + } + sample.keywords = new ArrayList<>(searchTerms); + + return sample; + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/FilterCriteriaBuilder.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/FilterCriteriaBuilder.java similarity index 92% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/FilterCriteriaBuilder.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/FilterCriteriaBuilder.java index e8a6b72cb..8dd9bc9f6 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/FilterCriteriaBuilder.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/FilterCriteriaBuilder.java @@ -1,19 +1,19 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.model.field; - -import org.springframework.data.solr.core.query.Criteria; -import uk.ac.ebi.biosamples.model.filter.Filter; - -public interface FilterCriteriaBuilder { - - Criteria getFilterCriteria(Filter filter); -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.model.field; + +import org.springframework.data.solr.core.query.Criteria; +import uk.ac.ebi.biosamples.core.model.filter.Filter; + +public interface FilterCriteriaBuilder { + + Criteria getFilterCriteria(Filter filter); +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAccessionField.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAccessionField.java similarity index 92% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAccessionField.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAccessionField.java index 1a6e1f5a6..52aafd7f6 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAccessionField.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAccessionField.java @@ -1,86 +1,86 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.model.field; - -import java.util.regex.Pattern; -import org.springframework.data.solr.core.query.Criteria; -import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.filter.AccessionFilter; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; - -@Component -public class SolrSampleAccessionField extends SolrSampleField { - - public SolrSampleAccessionField() { - super(); - } - - public SolrSampleAccessionField(final String readableLabel) { - super(readableLabel); - } - - /** - * All subclasses should implement this constructor - * - * @param readableLabel - * @param solrDocumentLabel - */ - public SolrSampleAccessionField(final String readableLabel, final String solrDocumentLabel) { - super(readableLabel, solrDocumentLabel); - } - - @Override - public Pattern getSolrFieldPattern() { - return Pattern.compile("^(?id)$"); - } - - @Override - public boolean isEncodedField() { - return false; - } - - @Override - public String getSolrFieldSuffix() { - return ""; - } - - @Override - public boolean isCompatibleWith(final Filter filter) { - return filter instanceof AccessionFilter; - } - - @Override - public FacetFetchStrategy getFacetCollectionStrategy() { - return null; - } - - @Override - public Criteria getFilterCriteria(final Filter filter) { - - Criteria filterCriteria = null; - - if (filter instanceof AccessionFilter) { - - filterCriteria = new Criteria(getSolrLabel()); - - final AccessionFilter accessionFilter = (AccessionFilter) filter; - if (accessionFilter.getContent().isPresent()) { - filterCriteria = - filterCriteria.expression(String.format("/%s/", accessionFilter.getContent().get())); - } else { - filterCriteria = filterCriteria.isNotNull(); - } - } - - return filterCriteria; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.model.field; + +import java.util.regex.Pattern; +import org.springframework.data.solr.core.query.Criteria; +import org.springframework.stereotype.Component; +import uk.ac.ebi.biosamples.core.model.filter.AccessionFilter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; + +@Component +public class SolrSampleAccessionField extends SolrSampleField { + + public SolrSampleAccessionField() { + super(); + } + + public SolrSampleAccessionField(final String readableLabel) { + super(readableLabel); + } + + /** + * All subclasses should implement this constructor + * + * @param readableLabel + * @param solrDocumentLabel + */ + public SolrSampleAccessionField(final String readableLabel, final String solrDocumentLabel) { + super(readableLabel, solrDocumentLabel); + } + + @Override + public Pattern getSolrFieldPattern() { + return Pattern.compile("^(?id)$"); + } + + @Override + public boolean isEncodedField() { + return false; + } + + @Override + public String getSolrFieldSuffix() { + return ""; + } + + @Override + public boolean isCompatibleWith(final Filter filter) { + return filter instanceof AccessionFilter; + } + + @Override + public FacetFetchStrategy getFacetCollectionStrategy() { + return null; + } + + @Override + public Criteria getFilterCriteria(final Filter filter) { + + Criteria filterCriteria = null; + + if (filter instanceof AccessionFilter) { + + filterCriteria = new Criteria(getSolrLabel()); + + final AccessionFilter accessionFilter = (AccessionFilter) filter; + if (accessionFilter.getContent().isPresent()) { + filterCriteria = + filterCriteria.expression(String.format("/%s/", accessionFilter.getContent().get())); + } else { + filterCriteria = filterCriteria.isNotNull(); + } + } + + return filterCriteria; + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAttributeValueField.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAttributeValueField.java similarity index 90% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAttributeValueField.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAttributeValueField.java index 0f7ed8301..3c62ead2d 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAttributeValueField.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAttributeValueField.java @@ -1,96 +1,96 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.model.field; - -import java.util.regex.Pattern; -import org.springframework.data.solr.core.query.Criteria; -import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.facet.AttributeFacet; -import uk.ac.ebi.biosamples.model.facet.Facet; -import uk.ac.ebi.biosamples.model.filter.AttributeFilter; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; -import uk.ac.ebi.biosamples.solr.model.strategy.RegularFacetFetchStrategy; - -@Component -public class SolrSampleAttributeValueField extends SolrSampleField { - - public SolrSampleAttributeValueField() { - super(); - } - - public SolrSampleAttributeValueField(final String readableLabel) { - super(readableLabel); - } - - public SolrSampleAttributeValueField(final String label, final String documentField) { - super(label, documentField); - } - - @Override - public boolean isEncodedField() { - return true; - } - - @Override - public Pattern getSolrFieldPattern() { - return Pattern.compile( - "^(?[A-Z0-9_]+)(?" + getSolrFieldSuffix() + ")$"); - } - - @Override - public String getSolrFieldSuffix() { - return "_av_ss"; - } - - @Override - public boolean canGenerateFacets() { - return true; - } - - @Override - public Facet.Builder getFacetBuilder(final String facetLabel, final Long facetCount) { - return new AttributeFacet.Builder(facetLabel, facetCount); - } - - @Override - public FacetFetchStrategy getFacetCollectionStrategy() { - return new RegularFacetFetchStrategy(); - } - - @Override - public boolean isCompatibleWith(final Filter filter) { - return filter instanceof AttributeFilter; - } - - @Override - public Criteria getFilterCriteria(final Filter filter) { - - Criteria filterCriteria = null; - if (filter instanceof AttributeFilter) { - - filterCriteria = new Criteria(getSolrLabel()); - - final AttributeFilter attributeFilter = (AttributeFilter) filter; - if (attributeFilter.getContent().isPresent()) - // - // filterCriteria.expression(String.format("/%s/",attributeFilter.getContent().get())); - { - filterCriteria = - filterCriteria.expression(String.format("\"%s\"", attributeFilter.getContent().get())); - } else { - filterCriteria = filterCriteria.isNotNull(); - } - } - - return filterCriteria; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.model.field; + +import java.util.regex.Pattern; +import org.springframework.data.solr.core.query.Criteria; +import org.springframework.stereotype.Component; +import uk.ac.ebi.biosamples.core.model.facet.AttributeFacet; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.filter.AttributeFilter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; +import uk.ac.ebi.biosamples.solr.model.strategy.RegularFacetFetchStrategy; + +@Component +public class SolrSampleAttributeValueField extends SolrSampleField { + + public SolrSampleAttributeValueField() { + super(); + } + + public SolrSampleAttributeValueField(final String readableLabel) { + super(readableLabel); + } + + public SolrSampleAttributeValueField(final String label, final String documentField) { + super(label, documentField); + } + + @Override + public boolean isEncodedField() { + return true; + } + + @Override + public Pattern getSolrFieldPattern() { + return Pattern.compile( + "^(?[A-Z0-9_]+)(?" + getSolrFieldSuffix() + ")$"); + } + + @Override + public String getSolrFieldSuffix() { + return "_av_ss"; + } + + @Override + public boolean canGenerateFacets() { + return true; + } + + @Override + public Facet.Builder getFacetBuilder(final String facetLabel, final Long facetCount) { + return new AttributeFacet.Builder(facetLabel, facetCount); + } + + @Override + public FacetFetchStrategy getFacetCollectionStrategy() { + return new RegularFacetFetchStrategy(); + } + + @Override + public boolean isCompatibleWith(final Filter filter) { + return filter instanceof AttributeFilter; + } + + @Override + public Criteria getFilterCriteria(final Filter filter) { + + Criteria filterCriteria = null; + if (filter instanceof AttributeFilter) { + + filterCriteria = new Criteria(getSolrLabel()); + + final AttributeFilter attributeFilter = (AttributeFilter) filter; + if (attributeFilter.getContent().isPresent()) + // + // filterCriteria.expression(String.format("/%s/",attributeFilter.getContent().get())); + { + filterCriteria = + filterCriteria.expression(String.format("\"%s\"", attributeFilter.getContent().get())); + } else { + filterCriteria = filterCriteria.isNotNull(); + } + } + + return filterCriteria; + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAuthenticationField.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAuthenticationField.java similarity index 95% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAuthenticationField.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAuthenticationField.java index bea2e7815..ab3d31cdd 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAuthenticationField.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleAuthenticationField.java @@ -13,8 +13,8 @@ import java.util.regex.Pattern; import org.springframework.data.solr.core.query.Criteria; import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.filter.AuthenticationFilter; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.filter.AuthenticationFilter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; @Component diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleDateField.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleDateField.java similarity index 91% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleDateField.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleDateField.java index 17938491d..d0d978ddc 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleDateField.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleDateField.java @@ -1,120 +1,120 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.model.field; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.time.temporal.TemporalAccessor; -import java.util.regex.Pattern; -import org.springframework.data.solr.core.query.Criteria; -import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.facet.DateRangeFacet; -import uk.ac.ebi.biosamples.model.facet.Facet; -import uk.ac.ebi.biosamples.model.filter.DateRangeFilter; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; -import uk.ac.ebi.biosamples.solr.model.strategy.RegularFacetFetchStrategy; - -@Component -public class SolrSampleDateField extends SolrSampleField { - - public SolrSampleDateField() { - super(); - } - - public SolrSampleDateField(final String readableLabel) { - super(readableLabel); - } - - /** - * All subclasses should implement this constructor - * - * @param readableLabel - * @param solrDocumentLabel - */ - public SolrSampleDateField(final String readableLabel, final String solrDocumentLabel) { - super(readableLabel, solrDocumentLabel); - } - - @Override - public Pattern getSolrFieldPattern() { - return Pattern.compile( - "^(?release|update)(?" + getSolrFieldSuffix() + ")$"); - } - - @Override - public String getSolrFieldSuffix() { - return "_dt"; - } - - @Override - public boolean canGenerateFacets() { - return true; - } - - @Override - public Facet.Builder getFacetBuilder(final String facetLabel, final Long facetCount) { - return new DateRangeFacet.Builder(facetLabel, facetCount); - } - - @Override - public boolean isEncodedField() { - return false; - } - - @Override - public boolean isCompatibleWith(final Filter filter) { - return filter instanceof DateRangeFilter; - } - - @Override - public FacetFetchStrategy getFacetCollectionStrategy() { - return new RegularFacetFetchStrategy(); - } - - @Override - public Criteria getFilterCriteria(final Filter filter) { - Criteria filterCriteria = null; - - if (filter instanceof DateRangeFilter) { - - final DateRangeFilter dateRangeFilter = (DateRangeFilter) filter; - filterCriteria = new Criteria(getSolrLabel()); - - if (dateRangeFilter.getContent().isPresent()) { - final DateRangeFilter.DateRange dateRange = dateRangeFilter.getContent().get(); - if (dateRange.isFromMinDate() && dateRange.isUntilMaxDate()) { - filterCriteria = filterCriteria.isNotNull(); - } else if (dateRange.isFromMinDate()) { - filterCriteria = filterCriteria.lessThanEqual(toSolrDateString(dateRange.getUntil())); - } else if (dateRange.isUntilMaxDate()) { - filterCriteria = filterCriteria.greaterThanEqual(toSolrDateString(dateRange.getFrom())); - } else { - filterCriteria = - filterCriteria.between( - toSolrDateString(dateRange.getFrom()), - toSolrDateString(dateRange.getUntil()), - true, - false); - } - - } else { - filterCriteria = filterCriteria.isNotNull(); - } - } - - return filterCriteria; - } - - private String toSolrDateString(final TemporalAccessor temporal) { - return DateTimeFormatter.ISO_INSTANT.withZone(ZoneId.of("UTC")).format(temporal); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.model.field; + +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.temporal.TemporalAccessor; +import java.util.regex.Pattern; +import org.springframework.data.solr.core.query.Criteria; +import org.springframework.stereotype.Component; +import uk.ac.ebi.biosamples.core.model.facet.DateRangeFacet; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; +import uk.ac.ebi.biosamples.solr.model.strategy.RegularFacetFetchStrategy; + +@Component +public class SolrSampleDateField extends SolrSampleField { + + public SolrSampleDateField() { + super(); + } + + public SolrSampleDateField(final String readableLabel) { + super(readableLabel); + } + + /** + * All subclasses should implement this constructor + * + * @param readableLabel + * @param solrDocumentLabel + */ + public SolrSampleDateField(final String readableLabel, final String solrDocumentLabel) { + super(readableLabel, solrDocumentLabel); + } + + @Override + public Pattern getSolrFieldPattern() { + return Pattern.compile( + "^(?release|update)(?" + getSolrFieldSuffix() + ")$"); + } + + @Override + public String getSolrFieldSuffix() { + return "_dt"; + } + + @Override + public boolean canGenerateFacets() { + return true; + } + + @Override + public Facet.Builder getFacetBuilder(final String facetLabel, final Long facetCount) { + return new DateRangeFacet.Builder(facetLabel, facetCount); + } + + @Override + public boolean isEncodedField() { + return false; + } + + @Override + public boolean isCompatibleWith(final Filter filter) { + return filter instanceof DateRangeFilter; + } + + @Override + public FacetFetchStrategy getFacetCollectionStrategy() { + return new RegularFacetFetchStrategy(); + } + + @Override + public Criteria getFilterCriteria(final Filter filter) { + Criteria filterCriteria = null; + + if (filter instanceof DateRangeFilter) { + + final DateRangeFilter dateRangeFilter = (DateRangeFilter) filter; + filterCriteria = new Criteria(getSolrLabel()); + + if (dateRangeFilter.getContent().isPresent()) { + final DateRangeFilter.DateRange dateRange = dateRangeFilter.getContent().get(); + if (dateRange.isFromMinDate() && dateRange.isUntilMaxDate()) { + filterCriteria = filterCriteria.isNotNull(); + } else if (dateRange.isFromMinDate()) { + filterCriteria = filterCriteria.lessThanEqual(toSolrDateString(dateRange.getUntil())); + } else if (dateRange.isUntilMaxDate()) { + filterCriteria = filterCriteria.greaterThanEqual(toSolrDateString(dateRange.getFrom())); + } else { + filterCriteria = + filterCriteria.between( + toSolrDateString(dateRange.getFrom()), + toSolrDateString(dateRange.getUntil()), + true, + false); + } + + } else { + filterCriteria = filterCriteria.isNotNull(); + } + } + + return filterCriteria; + } + + private String toSolrDateString(final TemporalAccessor temporal) { + return DateTimeFormatter.ISO_INSTANT.withZone(ZoneId.of("UTC")).format(temporal); + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleExternalReferenceDataField.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleExternalReferenceDataField.java similarity index 90% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleExternalReferenceDataField.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleExternalReferenceDataField.java index 123db1aca..7d6247717 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleExternalReferenceDataField.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleExternalReferenceDataField.java @@ -1,102 +1,102 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.model.field; - -import java.util.regex.Pattern; -import org.springframework.data.solr.core.query.Criteria; -import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.facet.ExternalReferenceDataFacet; -import uk.ac.ebi.biosamples.model.facet.Facet; -import uk.ac.ebi.biosamples.model.filter.ExternalReferenceDataFilter; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; -import uk.ac.ebi.biosamples.solr.model.strategy.RegularFacetFetchStrategy; - -@Component -public class SolrSampleExternalReferenceDataField extends SolrSampleField { - - public SolrSampleExternalReferenceDataField() { - super(); - } - - public SolrSampleExternalReferenceDataField(final String readableLabel) { - super(readableLabel); - } - - /** - * All subclasses should implement this constructor. - * - * @param readableLabel - * @param solrDocumentLabel - */ - public SolrSampleExternalReferenceDataField( - final String readableLabel, final String solrDocumentLabel) { - super(readableLabel, solrDocumentLabel); - } - - @Override - public Pattern getSolrFieldPattern() { - return Pattern.compile( - "^(?[A-Z0-9_]+)(?" + getSolrFieldSuffix() + ")$"); - } - - @Override - public String getSolrFieldSuffix() { - return "_erd_ss"; - } - - @Override - public boolean isEncodedField() { - return true; - } - - @Override - public boolean isCompatibleWith(final Filter filter) { - return filter instanceof ExternalReferenceDataFilter; - } - - @Override - public boolean canGenerateFacets() { - return true; - } - - @Override - public Facet.Builder getFacetBuilder(final String facetLabel, final Long facetCount) { - return new ExternalReferenceDataFacet.Builder(facetLabel, facetCount); - } - - @Override - public FacetFetchStrategy getFacetCollectionStrategy() { - return new RegularFacetFetchStrategy(); - } - - @Override - public Criteria getFilterCriteria(final Filter filter) { - Criteria filterCriteria = null; - - if (filter instanceof ExternalReferenceDataFilter) { - - filterCriteria = new Criteria(getSolrLabel()); - - final ExternalReferenceDataFilter extRefFilter = (ExternalReferenceDataFilter) filter; - if (extRefFilter.getContent().isPresent()) { - // filterCriteria = filterCriteria.expression("/" + - // extRefFilter.getContent().get() + "/"); - filterCriteria = - filterCriteria.expression(String.format("\"%s\"", extRefFilter.getContent().get())); - } else { - filterCriteria = filterCriteria.isNotNull(); - } - } - - return filterCriteria; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.model.field; + +import java.util.regex.Pattern; +import org.springframework.data.solr.core.query.Criteria; +import org.springframework.stereotype.Component; +import uk.ac.ebi.biosamples.core.model.facet.ExternalReferenceDataFacet; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.filter.ExternalReferenceDataFilter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; +import uk.ac.ebi.biosamples.solr.model.strategy.RegularFacetFetchStrategy; + +@Component +public class SolrSampleExternalReferenceDataField extends SolrSampleField { + + public SolrSampleExternalReferenceDataField() { + super(); + } + + public SolrSampleExternalReferenceDataField(final String readableLabel) { + super(readableLabel); + } + + /** + * All subclasses should implement this constructor. + * + * @param readableLabel + * @param solrDocumentLabel + */ + public SolrSampleExternalReferenceDataField( + final String readableLabel, final String solrDocumentLabel) { + super(readableLabel, solrDocumentLabel); + } + + @Override + public Pattern getSolrFieldPattern() { + return Pattern.compile( + "^(?[A-Z0-9_]+)(?" + getSolrFieldSuffix() + ")$"); + } + + @Override + public String getSolrFieldSuffix() { + return "_erd_ss"; + } + + @Override + public boolean isEncodedField() { + return true; + } + + @Override + public boolean isCompatibleWith(final Filter filter) { + return filter instanceof ExternalReferenceDataFilter; + } + + @Override + public boolean canGenerateFacets() { + return true; + } + + @Override + public Facet.Builder getFacetBuilder(final String facetLabel, final Long facetCount) { + return new ExternalReferenceDataFacet.Builder(facetLabel, facetCount); + } + + @Override + public FacetFetchStrategy getFacetCollectionStrategy() { + return new RegularFacetFetchStrategy(); + } + + @Override + public Criteria getFilterCriteria(final Filter filter) { + Criteria filterCriteria = null; + + if (filter instanceof ExternalReferenceDataFilter) { + + filterCriteria = new Criteria(getSolrLabel()); + + final ExternalReferenceDataFilter extRefFilter = (ExternalReferenceDataFilter) filter; + if (extRefFilter.getContent().isPresent()) { + // filterCriteria = filterCriteria.expression("/" + + // extRefFilter.getContent().get() + "/"); + filterCriteria = + filterCriteria.expression(String.format("\"%s\"", extRefFilter.getContent().get())); + } else { + filterCriteria = filterCriteria.isNotNull(); + } + } + + return filterCriteria; + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleField.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleField.java similarity index 94% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleField.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleField.java index 590060a4f..162ac5d6f 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleField.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleField.java @@ -1,142 +1,142 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.model.field; - -import java.util.regex.Pattern; -import uk.ac.ebi.biosamples.model.facet.Facet; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; -import uk.ac.ebi.biosamples.solr.service.SolrFieldService; - -public abstract class SolrSampleField implements FilterCriteriaBuilder { - - private String readableLabel; - private String solrDocumentLabel; - - /** Constructor meant to be used only for reflection purposes */ - protected SolrSampleField() { - readableLabel = null; - solrDocumentLabel = null; - } - - protected SolrSampleField(final String readableLabel) { - this.readableLabel = readableLabel; - if (isEncodedField()) { - solrDocumentLabel = SolrFieldService.encodeFieldName(readableLabel) + getSolrFieldSuffix(); - } else { - solrDocumentLabel = readableLabel + getSolrFieldSuffix(); - } - } - - /** - * All subclasses should implement this constructor. - * - * @param readableLabel - * @param solrDocumentLabel - */ - protected SolrSampleField(final String readableLabel, final String solrDocumentLabel) { - this.readableLabel = readableLabel; - this.solrDocumentLabel = solrDocumentLabel; - } - - /** - * Check if the provided string matches the field regularExpression - * - * @param fieldName string to check against the field pattern - * @return - */ - public boolean matches(final String fieldName) { - return getSolrFieldPattern().matcher(fieldName).find(); - } - - /** - * Return the regular expression pattern that matches the field in a Solr document. The pattern - * contains usually two groups: fieldname and fieldsuffix Usually is structured this way - * - * 

/^(?pattern>(?suffix)$
- * - * @return the Pattern to match the field in a solr document - */ - public abstract Pattern getSolrFieldPattern(); - - /** - * Return the solr field suffix - * - * @return - */ - public abstract String getSolrFieldSuffix(); - - /** - * Return if the field in the solr document is encoded or not - * - * @return - */ - public abstract boolean isEncodedField(); - - /** - * Return if the solr field is compatible with a specific Filter class TODO: if in future we want - * to compose filters (NOT, REGEX, etc.) This should look inside the filter itself - * - * @param filter the filter to test for compatibility - * @return if the field is compatible with the filter class - */ - public abstract boolean isCompatibleWith(Filter filter); - - /** - * Return if the field can be used to generate facets - * - * @return - */ - public boolean canGenerateFacets() { - return false; - } - - /** - * Return an optional builder for the facet corresponding to the field - * - * @param facetLabel - * @param facetCount - * @return - */ - public Facet.Builder getFacetBuilder(final String facetLabel, final Long facetCount) { - return null; - } - - /** - * For each field a specific strategy to get the facet content need to be implemented Facet - * content retrieve will be delegated to the facet fetch strategy - * - * @return a facet fetch strategy - */ - public abstract FacetFetchStrategy getFacetCollectionStrategy(); - - /** @return the readable label of the field */ - public String getReadableLabel() { - return readableLabel; - } - - /** - * @return the document field, which could be encoded or not encoded based on the SolrFieldType - */ - public String getSolrLabel() { - return solrDocumentLabel; - } - - public SolrSampleField setReadableLabel(final String readableLabel) { - this.readableLabel = readableLabel; - return this; - } - - public SolrSampleField setSolrLabel(final String solrDocumentLabel) { - this.solrDocumentLabel = solrDocumentLabel; - return this; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.model.field; + +import java.util.regex.Pattern; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; +import uk.ac.ebi.biosamples.solr.service.SolrFieldService; + +public abstract class SolrSampleField implements FilterCriteriaBuilder { + + private String readableLabel; + private String solrDocumentLabel; + + /** Constructor meant to be used only for reflection purposes */ + protected SolrSampleField() { + readableLabel = null; + solrDocumentLabel = null; + } + + protected SolrSampleField(final String readableLabel) { + this.readableLabel = readableLabel; + if (isEncodedField()) { + solrDocumentLabel = SolrFieldService.encodeFieldName(readableLabel) + getSolrFieldSuffix(); + } else { + solrDocumentLabel = readableLabel + getSolrFieldSuffix(); + } + } + + /** + * All subclasses should implement this constructor. + * + * @param readableLabel + * @param solrDocumentLabel + */ + protected SolrSampleField(final String readableLabel, final String solrDocumentLabel) { + this.readableLabel = readableLabel; + this.solrDocumentLabel = solrDocumentLabel; + } + + /** + * Check if the provided string matches the field regularExpression + * + * @param fieldName string to check against the field pattern + * @return + */ + public boolean matches(final String fieldName) { + return getSolrFieldPattern().matcher(fieldName).find(); + } + + /** + * Return the regular expression pattern that matches the field in a Solr document. The pattern + * contains usually two groups: fieldname and fieldsuffix Usually is structured this way + * + * 
/^(?pattern>(?suffix)$
+ * + * @return the Pattern to match the field in a solr document + */ + public abstract Pattern getSolrFieldPattern(); + + /** + * Return the solr field suffix + * + * @return + */ + public abstract String getSolrFieldSuffix(); + + /** + * Return if the field in the solr document is encoded or not + * + * @return + */ + public abstract boolean isEncodedField(); + + /** + * Return if the solr field is compatible with a specific Filter class TODO: if in future we want + * to compose filters (NOT, REGEX, etc.) This should look inside the filter itself + * + * @param filter the filter to test for compatibility + * @return if the field is compatible with the filter class + */ + public abstract boolean isCompatibleWith(Filter filter); + + /** + * Return if the field can be used to generate facets + * + * @return + */ + public boolean canGenerateFacets() { + return false; + } + + /** + * Return an optional builder for the facet corresponding to the field + * + * @param facetLabel + * @param facetCount + * @return + */ + public Facet.Builder getFacetBuilder(final String facetLabel, final Long facetCount) { + return null; + } + + /** + * For each field a specific strategy to get the facet content need to be implemented Facet + * content retrieve will be delegated to the facet fetch strategy + * + * @return a facet fetch strategy + */ + public abstract FacetFetchStrategy getFacetCollectionStrategy(); + + /** @return the readable label of the field */ + public String getReadableLabel() { + return readableLabel; + } + + /** + * @return the document field, which could be encoded or not encoded based on the SolrFieldType + */ + public String getSolrLabel() { + return solrDocumentLabel; + } + + public SolrSampleField setReadableLabel(final String readableLabel) { + this.readableLabel = readableLabel; + return this; + } + + public SolrSampleField setSolrLabel(final String solrDocumentLabel) { + this.solrDocumentLabel = solrDocumentLabel; + return this; + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleInverseRelationField.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleInverseRelationField.java similarity index 90% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleInverseRelationField.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleInverseRelationField.java index 837c699b8..f19f0e063 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleInverseRelationField.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleInverseRelationField.java @@ -1,103 +1,103 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.model.field; - -import java.util.regex.Pattern; -import org.springframework.data.solr.core.query.Criteria; -import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.facet.Facet; -import uk.ac.ebi.biosamples.model.facet.InverseRelationFacet; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.model.filter.InverseRelationFilter; -import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; -import uk.ac.ebi.biosamples.solr.model.strategy.RegularFacetFetchStrategy; - -@Component -public class SolrSampleInverseRelationField extends SolrSampleField { - - public SolrSampleInverseRelationField() { - super(); - } - - public SolrSampleInverseRelationField(final String readableLabel) { - super(readableLabel); - } - - /** - * All subclasses should implement this constructor - * - * @param readableLabel - * @param solrDocumentLabel - */ - public SolrSampleInverseRelationField( - final String readableLabel, final String solrDocumentLabel) { - super(readableLabel, solrDocumentLabel); - } - - @Override - public Pattern getSolrFieldPattern() { - return Pattern.compile( - "^(?[A-Z0-9_]+)(?" + getSolrFieldSuffix() + ")$"); - } - - @Override - public String getSolrFieldSuffix() { - return "_ir_ss"; - } - - @Override - public boolean isEncodedField() { - return true; - } - - @Override - public boolean isCompatibleWith(final Filter filter) { - return filter instanceof InverseRelationFilter; - } - - @Override - public boolean canGenerateFacets() { - return true; - } - - @Override - public Facet.Builder getFacetBuilder(final String facetLabel, final Long facetCount) { - return new InverseRelationFacet.Builder(facetLabel, facetCount); - } - - @Override - public FacetFetchStrategy getFacetCollectionStrategy() { - return new RegularFacetFetchStrategy(); - } - - @Override - public Criteria getFilterCriteria(final Filter filter) { - Criteria filterCriteria = null; - - if (filter instanceof InverseRelationFilter) { - - filterCriteria = new Criteria(getSolrLabel()); - - final InverseRelationFilter inverseRelationFilter = (InverseRelationFilter) filter; - if (inverseRelationFilter.getContent().isPresent()) { - // filterCriteria = filterCriteria.expression("/" + - // inverseRelationFilter.getContent().get() + "/"); - filterCriteria = - filterCriteria.expression( - String.format("\"%s\"", inverseRelationFilter.getContent().get())); - } else { - filterCriteria = filterCriteria.isNotNull(); - } - } - - return filterCriteria; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.model.field; + +import java.util.regex.Pattern; +import org.springframework.data.solr.core.query.Criteria; +import org.springframework.stereotype.Component; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.facet.InverseRelationFacet; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.filter.InverseRelationFilter; +import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; +import uk.ac.ebi.biosamples.solr.model.strategy.RegularFacetFetchStrategy; + +@Component +public class SolrSampleInverseRelationField extends SolrSampleField { + + public SolrSampleInverseRelationField() { + super(); + } + + public SolrSampleInverseRelationField(final String readableLabel) { + super(readableLabel); + } + + /** + * All subclasses should implement this constructor + * + * @param readableLabel + * @param solrDocumentLabel + */ + public SolrSampleInverseRelationField( + final String readableLabel, final String solrDocumentLabel) { + super(readableLabel, solrDocumentLabel); + } + + @Override + public Pattern getSolrFieldPattern() { + return Pattern.compile( + "^(?[A-Z0-9_]+)(?" + getSolrFieldSuffix() + ")$"); + } + + @Override + public String getSolrFieldSuffix() { + return "_ir_ss"; + } + + @Override + public boolean isEncodedField() { + return true; + } + + @Override + public boolean isCompatibleWith(final Filter filter) { + return filter instanceof InverseRelationFilter; + } + + @Override + public boolean canGenerateFacets() { + return true; + } + + @Override + public Facet.Builder getFacetBuilder(final String facetLabel, final Long facetCount) { + return new InverseRelationFacet.Builder(facetLabel, facetCount); + } + + @Override + public FacetFetchStrategy getFacetCollectionStrategy() { + return new RegularFacetFetchStrategy(); + } + + @Override + public Criteria getFilterCriteria(final Filter filter) { + Criteria filterCriteria = null; + + if (filter instanceof InverseRelationFilter) { + + filterCriteria = new Criteria(getSolrLabel()); + + final InverseRelationFilter inverseRelationFilter = (InverseRelationFilter) filter; + if (inverseRelationFilter.getContent().isPresent()) { + // filterCriteria = filterCriteria.expression("/" + + // inverseRelationFilter.getContent().get() + "/"); + filterCriteria = + filterCriteria.expression( + String.format("\"%s\"", inverseRelationFilter.getContent().get())); + } else { + filterCriteria = filterCriteria.isNotNull(); + } + } + + return filterCriteria; + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleNameField.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleNameField.java similarity index 92% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleNameField.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleNameField.java index e2ccf07da..92693efa4 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleNameField.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleNameField.java @@ -1,87 +1,87 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.model.field; - -import java.util.regex.Pattern; -import org.springframework.data.solr.core.query.Criteria; -import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.model.filter.NameFilter; -import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; - -@Component -public class SolrSampleNameField extends SolrSampleField { - - public SolrSampleNameField() { - super(); - } - - public SolrSampleNameField(final String readableLabel) { - super(readableLabel); - } - - /** - * All subclasses should implement this constructor - * - * @param readableLabel - * @param solrDocumentLabel - */ - public SolrSampleNameField(final String readableLabel, final String solrDocumentLabel) { - super(readableLabel, solrDocumentLabel); - } - - @Override - public Pattern getSolrFieldPattern() { - return Pattern.compile("^(?name)(?" + getSolrFieldSuffix() + ")$"); - } - - @Override - public String getSolrFieldSuffix() { - return "_s"; - } - - @Override - public boolean isEncodedField() { - return false; - } - - @Override - public boolean isCompatibleWith(final Filter filter) { - return filter instanceof NameFilter; - } - - @Override - public FacetFetchStrategy getFacetCollectionStrategy() { - return null; - } - - @Override - public Criteria getFilterCriteria(final Filter filter) { - Criteria filterCriteria = null; - - if (filter instanceof NameFilter) { - - filterCriteria = new Criteria(getSolrLabel()); - - final NameFilter nameFilter = (NameFilter) filter; - if (nameFilter.getContent().isPresent()) { - // filterCriteria = filterCriteria.expression("/" + - // nameFilter.getContent().get() + "/"); - filterCriteria = - filterCriteria.expression(String.format("\"%s\"", nameFilter.getContent().get())); - } else { - filterCriteria = filterCriteria.isNotNull(); - } - } - - return filterCriteria; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.model.field; + +import java.util.regex.Pattern; +import org.springframework.data.solr.core.query.Criteria; +import org.springframework.stereotype.Component; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.filter.NameFilter; +import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; + +@Component +public class SolrSampleNameField extends SolrSampleField { + + public SolrSampleNameField() { + super(); + } + + public SolrSampleNameField(final String readableLabel) { + super(readableLabel); + } + + /** + * All subclasses should implement this constructor + * + * @param readableLabel + * @param solrDocumentLabel + */ + public SolrSampleNameField(final String readableLabel, final String solrDocumentLabel) { + super(readableLabel, solrDocumentLabel); + } + + @Override + public Pattern getSolrFieldPattern() { + return Pattern.compile("^(?name)(?" + getSolrFieldSuffix() + ")$"); + } + + @Override + public String getSolrFieldSuffix() { + return "_s"; + } + + @Override + public boolean isEncodedField() { + return false; + } + + @Override + public boolean isCompatibleWith(final Filter filter) { + return filter instanceof NameFilter; + } + + @Override + public FacetFetchStrategy getFacetCollectionStrategy() { + return null; + } + + @Override + public Criteria getFilterCriteria(final Filter filter) { + Criteria filterCriteria = null; + + if (filter instanceof NameFilter) { + + filterCriteria = new Criteria(getSolrLabel()); + + final NameFilter nameFilter = (NameFilter) filter; + if (nameFilter.getContent().isPresent()) { + // filterCriteria = filterCriteria.expression("/" + + // nameFilter.getContent().get() + "/"); + filterCriteria = + filterCriteria.expression(String.format("\"%s\"", nameFilter.getContent().get())); + } else { + filterCriteria = filterCriteria.isNotNull(); + } + } + + return filterCriteria; + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleRelationField.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleRelationField.java similarity index 90% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleRelationField.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleRelationField.java index 4d2169010..89a660c5f 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleRelationField.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleRelationField.java @@ -1,101 +1,101 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.model.field; - -import java.util.regex.Pattern; -import org.springframework.data.solr.core.query.Criteria; -import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.facet.Facet; -import uk.ac.ebi.biosamples.model.facet.RelationFacet; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.model.filter.RelationFilter; -import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; -import uk.ac.ebi.biosamples.solr.model.strategy.RegularFacetFetchStrategy; - -@Component -public class SolrSampleRelationField extends SolrSampleField { - - public SolrSampleRelationField() { - super(); - } - - public SolrSampleRelationField(final String readableLabel) { - super(readableLabel); - } - - /** - * All subclasses should implement this constructor - * - * @param readableLabel - * @param solrDocumentLabel - */ - public SolrSampleRelationField(final String readableLabel, final String solrDocumentLabel) { - super(readableLabel, solrDocumentLabel); - } - - @Override - public Pattern getSolrFieldPattern() { - return Pattern.compile( - "^(?[A-Z0-9_]+)(?" + getSolrFieldSuffix() + ")$"); - } - - @Override - public String getSolrFieldSuffix() { - return "_or_ss"; - } - - @Override - public boolean isEncodedField() { - return true; - } - - @Override - public boolean isCompatibleWith(final Filter filter) { - return filter instanceof RelationFilter; - } - - @Override - public boolean canGenerateFacets() { - return true; - } - - @Override - public Facet.Builder getFacetBuilder(final String facetLabel, final Long facetCount) { - return new RelationFacet.Builder(facetLabel, facetCount); - } - - @Override - public FacetFetchStrategy getFacetCollectionStrategy() { - return new RegularFacetFetchStrategy(); - } - - @Override - public Criteria getFilterCriteria(final Filter filter) { - Criteria filterCriteria = null; - - if (filter instanceof RelationFilter) { - - filterCriteria = new Criteria(getSolrLabel()); - - final RelationFilter relationFilter = (RelationFilter) filter; - if (relationFilter.getContent().isPresent()) { - // filterCriteria = filterCriteria.expression("/" + - // relationFilter.getContent().get() + "/"); - filterCriteria = - filterCriteria.expression(String.format("\"%s\"", relationFilter.getContent().get())); - } else { - filterCriteria = filterCriteria.isNotNull(); - } - } - - return filterCriteria; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.model.field; + +import java.util.regex.Pattern; +import org.springframework.data.solr.core.query.Criteria; +import org.springframework.stereotype.Component; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.facet.RelationFacet; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.filter.RelationFilter; +import uk.ac.ebi.biosamples.solr.model.strategy.FacetFetchStrategy; +import uk.ac.ebi.biosamples.solr.model.strategy.RegularFacetFetchStrategy; + +@Component +public class SolrSampleRelationField extends SolrSampleField { + + public SolrSampleRelationField() { + super(); + } + + public SolrSampleRelationField(final String readableLabel) { + super(readableLabel); + } + + /** + * All subclasses should implement this constructor + * + * @param readableLabel + * @param solrDocumentLabel + */ + public SolrSampleRelationField(final String readableLabel, final String solrDocumentLabel) { + super(readableLabel, solrDocumentLabel); + } + + @Override + public Pattern getSolrFieldPattern() { + return Pattern.compile( + "^(?[A-Z0-9_]+)(?" + getSolrFieldSuffix() + ")$"); + } + + @Override + public String getSolrFieldSuffix() { + return "_or_ss"; + } + + @Override + public boolean isEncodedField() { + return true; + } + + @Override + public boolean isCompatibleWith(final Filter filter) { + return filter instanceof RelationFilter; + } + + @Override + public boolean canGenerateFacets() { + return true; + } + + @Override + public Facet.Builder getFacetBuilder(final String facetLabel, final Long facetCount) { + return new RelationFacet.Builder(facetLabel, facetCount); + } + + @Override + public FacetFetchStrategy getFacetCollectionStrategy() { + return new RegularFacetFetchStrategy(); + } + + @Override + public Criteria getFilterCriteria(final Filter filter) { + Criteria filterCriteria = null; + + if (filter instanceof RelationFilter) { + + filterCriteria = new Criteria(getSolrLabel()); + + final RelationFilter relationFilter = (RelationFilter) filter; + if (relationFilter.getContent().isPresent()) { + // filterCriteria = filterCriteria.expression("/" + + // relationFilter.getContent().get() + "/"); + filterCriteria = + filterCriteria.expression(String.format("\"%s\"", relationFilter.getContent().get())); + } else { + filterCriteria = filterCriteria.isNotNull(); + } + } + + return filterCriteria; + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/FacetFetchStrategy.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/FacetFetchStrategy.java similarity index 95% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/FacetFetchStrategy.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/FacetFetchStrategy.java index 37c140434..dee12e64b 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/FacetFetchStrategy.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/FacetFetchStrategy.java @@ -1,45 +1,45 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.model.strategy; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import org.springframework.data.domain.Pageable; -import org.springframework.data.solr.core.query.FacetQuery; -import uk.ac.ebi.biosamples.model.facet.Facet; -import uk.ac.ebi.biosamples.solr.model.field.SolrSampleField; -import uk.ac.ebi.biosamples.solr.repo.SolrSampleRepository; - -public interface FacetFetchStrategy { - - /** - * The strategy uses the results from the facet query to return a list of optional facet - * - * @param sampleRepository the repository that will be used to retrieve the facets - * @param query the FacetQuery to retrieve the facet - * @param facetFieldCountEntries the facet fields/count on which the facet will be calculated - * @param pageable a page information - * @return a list of optional facets - */ - public List> fetchFacetsUsing( - SolrSampleRepository sampleRepository, - FacetQuery query, - List> facetFieldCountEntries, - Pageable pageable); - - public List> fetchFacetsUsing( - SolrSampleRepository sampleRepository, - FacetQuery query, - List> facetFieldCountEntries, - List> rangeFieldCountEntries, - Pageable pageable); -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.model.strategy; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.springframework.data.domain.Pageable; +import org.springframework.data.solr.core.query.FacetQuery; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.solr.model.field.SolrSampleField; +import uk.ac.ebi.biosamples.solr.repo.SolrSampleRepository; + +public interface FacetFetchStrategy { + + /** + * The strategy uses the results from the facet query to return a list of optional facet + * + * @param sampleRepository the repository that will be used to retrieve the facets + * @param query the FacetQuery to retrieve the facet + * @param facetFieldCountEntries the facet fields/count on which the facet will be calculated + * @param pageable a page information + * @return a list of optional facets + */ + public List> fetchFacetsUsing( + SolrSampleRepository sampleRepository, + FacetQuery query, + List> facetFieldCountEntries, + Pageable pageable); + + public List> fetchFacetsUsing( + SolrSampleRepository sampleRepository, + FacetQuery query, + List> facetFieldCountEntries, + List> rangeFieldCountEntries, + Pageable pageable); +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/RegularFacetFetchStrategy.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/RegularFacetFetchStrategy.java similarity index 95% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/RegularFacetFetchStrategy.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/RegularFacetFetchStrategy.java index 758a818e3..7ce01f07b 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/RegularFacetFetchStrategy.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/strategy/RegularFacetFetchStrategy.java @@ -1,177 +1,177 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.model.strategy; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Optional; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.domain.Pageable; -import org.springframework.data.solr.core.query.FacetQuery; -import org.springframework.data.solr.core.query.Field; -import org.springframework.data.solr.core.query.result.FacetFieldEntry; -import org.springframework.data.solr.core.query.result.FacetPage; -import org.springframework.data.solr.core.query.result.FacetQueryEntry; -import uk.ac.ebi.biosamples.model.facet.Facet; -import uk.ac.ebi.biosamples.model.facet.content.LabelCountEntry; -import uk.ac.ebi.biosamples.model.facet.content.LabelCountListContent; -import uk.ac.ebi.biosamples.solr.model.field.SolrSampleField; -import uk.ac.ebi.biosamples.solr.repo.SolrSampleRepository; - -public class RegularFacetFetchStrategy implements FacetFetchStrategy { - - private final Logger log = LoggerFactory.getLogger(getClass()); - - /* - - */ - // @Override - @Override - public List> fetchFacetsUsing( - final SolrSampleRepository solrSampleRepository, - final FacetQuery query, - final List> facetFieldCountEntries, - final Pageable facetPageable) { - - final List facetFieldNames = - facetFieldCountEntries.stream() - .map(Entry::getKey) - .map(SolrSampleField::getSolrLabel) - .collect(Collectors.toList()); - - final FacetPage facetPage = - solrSampleRepository.getFacets(query, facetFieldNames, facetPageable); - - final List> facetResults = new ArrayList<>(); - for (final Field field : facetPage.getFacetFields()) { - - // Get the field info associated with returned field from solr - final Optional> optionalFieldInfo = - facetFieldCountEntries.stream() - .filter(entry -> entry.getKey().getSolrLabel().equals(field.getName())) - .findFirst(); - - if (!optionalFieldInfo.isPresent()) { - throw new RuntimeException( - "Unexpected field returned when getting facets for " + facetFieldCountEntries); - } - - final Entry fieldCountEntry = optionalFieldInfo.get(); - - // Create the list of facet value-count for the returned field - final List listFacetContent = new ArrayList<>(); - for (final FacetFieldEntry ffe : facetPage.getFacetResultPage(field)) { - log.trace( - "Adding " - + fieldCountEntry.getKey().getReadableLabel() - + " : " - + ffe.getValue() - + " with count " - + ffe.getValueCount()); - listFacetContent.add(LabelCountEntry.build(ffe.getValue(), ffe.getValueCount())); - } - - // Build the facet - final SolrSampleField solrSampleField = fieldCountEntry.getKey(); - if (solrSampleField.canGenerateFacets()) { - final Facet facet = - solrSampleField - .getFacetBuilder(solrSampleField.getReadableLabel(), fieldCountEntry.getValue()) - .withContent(new LabelCountListContent(listFacetContent)) - .build(); - facetResults.add(Optional.of(facet)); - } - // Optional associatedFacetType = - // solrSampleField.getSolrFieldType().getFacetFilterFieldType().getFacetType(); - // if(associatedFacetType.isPresent()) { - // FacetType facetType = associatedFacetType.get(); - // String facetLabel = solrSampleField.getReadableLabel(); - // Long facetCount = fieldCountEntry.getValue(); - // Facet facet = facetType - // .getBuilderForLabelAndCount(facetLabel, facetCount) - // .withContent(new LabelCountListContent(listFacetContent)) - // .build(); - // facetResults.add(Optional.of(facet)); - // } - } - - return facetResults; - } - - // @Override - @Override - public List> fetchFacetsUsing( - final SolrSampleRepository solrSampleRepository, - final FacetQuery query, - final List> facetFieldCountEntries, - final List> rangeFieldCountEntries, - final Pageable facetPageable) { - - final List facetFieldNames = - facetFieldCountEntries.stream() - .map(Entry::getKey) - .map(SolrSampleField::getSolrLabel) - .collect(Collectors.toList()); - final List rangeFacetFieldNames = - rangeFieldCountEntries.stream() - .map(Entry::getKey) - .map(SolrSampleField::getSolrLabel) - .collect(Collectors.toList()); - - final Map fieldMap = - facetFieldCountEntries.stream() - .map(Entry::getKey) - .collect(Collectors.toMap(SolrSampleField::getSolrLabel, s -> s)); - rangeFieldCountEntries.stream() - .map(Entry::getKey) - .forEach(s -> fieldMap.put(s.getSolrLabel(), s)); - - final FacetPage facetPage = - solrSampleRepository.getFacets(query, facetFieldNames, rangeFacetFieldNames, facetPageable); - - final List> facetResults = new ArrayList<>(); - for (final FacetQueryEntry q : facetPage.getFacetQueryResult().getContent()) { - final long fieldCount = q.getValueCount(); - if (fieldCount > 0) { - final String fieldName = q.getValue().split(":")[0]; - // String readableFieldName = SolrFieldService.decodeFieldName(fieldName); - final SolrSampleField solrSampleField = fieldMap.get(fieldName); - - final List listFacetContent = new ArrayList<>(); - for (final FacetFieldEntry ffe : facetPage.getFacetResultPage(fieldName)) { - listFacetContent.add(LabelCountEntry.build(ffe.getValue(), ffe.getValueCount())); - } - - // todo only add if not empty or remove - // if (facetPage.getRangeFacetResultPage(fieldName).getTotalElements() < 1) { - // break; - // } - for (final FacetFieldEntry ffe : facetPage.getRangeFacetResultPage(fieldName)) { - listFacetContent.add(LabelCountEntry.build(ffe.getValue(), ffe.getValueCount())); - } - - if (!listFacetContent.isEmpty()) { - final Facet facet = - solrSampleField - .getFacetBuilder(solrSampleField.getReadableLabel(), fieldCount) - .withContent(new LabelCountListContent(listFacetContent)) - .build(); - facetResults.add(Optional.of(facet)); - } - } - } - return facetResults; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.model.strategy; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.domain.Pageable; +import org.springframework.data.solr.core.query.FacetQuery; +import org.springframework.data.solr.core.query.Field; +import org.springframework.data.solr.core.query.result.FacetFieldEntry; +import org.springframework.data.solr.core.query.result.FacetPage; +import org.springframework.data.solr.core.query.result.FacetQueryEntry; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountEntry; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; +import uk.ac.ebi.biosamples.solr.model.field.SolrSampleField; +import uk.ac.ebi.biosamples.solr.repo.SolrSampleRepository; + +public class RegularFacetFetchStrategy implements FacetFetchStrategy { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + /* + + */ + // @Override + @Override + public List> fetchFacetsUsing( + final SolrSampleRepository solrSampleRepository, + final FacetQuery query, + final List> facetFieldCountEntries, + final Pageable facetPageable) { + + final List facetFieldNames = + facetFieldCountEntries.stream() + .map(Entry::getKey) + .map(SolrSampleField::getSolrLabel) + .collect(Collectors.toList()); + + final FacetPage facetPage = + solrSampleRepository.getFacets(query, facetFieldNames, facetPageable); + + final List> facetResults = new ArrayList<>(); + for (final Field field : facetPage.getFacetFields()) { + + // Get the field info associated with returned field from solr + final Optional> optionalFieldInfo = + facetFieldCountEntries.stream() + .filter(entry -> entry.getKey().getSolrLabel().equals(field.getName())) + .findFirst(); + + if (!optionalFieldInfo.isPresent()) { + throw new RuntimeException( + "Unexpected field returned when getting facets for " + facetFieldCountEntries); + } + + final Entry fieldCountEntry = optionalFieldInfo.get(); + + // Create the list of facet value-count for the returned field + final List listFacetContent = new ArrayList<>(); + for (final FacetFieldEntry ffe : facetPage.getFacetResultPage(field)) { + log.trace( + "Adding " + + fieldCountEntry.getKey().getReadableLabel() + + " : " + + ffe.getValue() + + " with count " + + ffe.getValueCount()); + listFacetContent.add(LabelCountEntry.build(ffe.getValue(), ffe.getValueCount())); + } + + // Build the facet + final SolrSampleField solrSampleField = fieldCountEntry.getKey(); + if (solrSampleField.canGenerateFacets()) { + final Facet facet = + solrSampleField + .getFacetBuilder(solrSampleField.getReadableLabel(), fieldCountEntry.getValue()) + .withContent(new LabelCountListContent(listFacetContent)) + .build(); + facetResults.add(Optional.of(facet)); + } + // Optional associatedFacetType = + // solrSampleField.getSolrFieldType().getFacetFilterFieldType().getFacetType(); + // if(associatedFacetType.isPresent()) { + // FacetType facetType = associatedFacetType.get(); + // String facetLabel = solrSampleField.getReadableLabel(); + // Long facetCount = fieldCountEntry.getValue(); + // Facet facet = facetType + // .getBuilderForLabelAndCount(facetLabel, facetCount) + // .withContent(new LabelCountListContent(listFacetContent)) + // .build(); + // facetResults.add(Optional.of(facet)); + // } + } + + return facetResults; + } + + // @Override + @Override + public List> fetchFacetsUsing( + final SolrSampleRepository solrSampleRepository, + final FacetQuery query, + final List> facetFieldCountEntries, + final List> rangeFieldCountEntries, + final Pageable facetPageable) { + + final List facetFieldNames = + facetFieldCountEntries.stream() + .map(Entry::getKey) + .map(SolrSampleField::getSolrLabel) + .collect(Collectors.toList()); + final List rangeFacetFieldNames = + rangeFieldCountEntries.stream() + .map(Entry::getKey) + .map(SolrSampleField::getSolrLabel) + .collect(Collectors.toList()); + + final Map fieldMap = + facetFieldCountEntries.stream() + .map(Entry::getKey) + .collect(Collectors.toMap(SolrSampleField::getSolrLabel, s -> s)); + rangeFieldCountEntries.stream() + .map(Entry::getKey) + .forEach(s -> fieldMap.put(s.getSolrLabel(), s)); + + final FacetPage facetPage = + solrSampleRepository.getFacets(query, facetFieldNames, rangeFacetFieldNames, facetPageable); + + final List> facetResults = new ArrayList<>(); + for (final FacetQueryEntry q : facetPage.getFacetQueryResult().getContent()) { + final long fieldCount = q.getValueCount(); + if (fieldCount > 0) { + final String fieldName = q.getValue().split(":")[0]; + // String readableFieldName = SolrFieldService.decodeFieldName(fieldName); + final SolrSampleField solrSampleField = fieldMap.get(fieldName); + + final List listFacetContent = new ArrayList<>(); + for (final FacetFieldEntry ffe : facetPage.getFacetResultPage(fieldName)) { + listFacetContent.add(LabelCountEntry.build(ffe.getValue(), ffe.getValueCount())); + } + + // todo only add if not empty or remove + // if (facetPage.getRangeFacetResultPage(fieldName).getTotalElements() < 1) { + // break; + // } + for (final FacetFieldEntry ffe : facetPage.getRangeFacetResultPage(fieldName)) { + listFacetContent.add(LabelCountEntry.build(ffe.getValue(), ffe.getValueCount())); + } + + if (!listFacetContent.isEmpty()) { + final Facet facet = + solrSampleField + .getFacetBuilder(solrSampleField.getReadableLabel(), fieldCount) + .withContent(new LabelCountListContent(listFacetContent)) + .build(); + facetResults.add(Optional.of(facet)); + } + } + } + return facetResults; + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/repo/CursorArrayList.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/repo/CursorArrayList.java similarity index 97% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/repo/CursorArrayList.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/repo/CursorArrayList.java index e33d08cdd..da73d9ecd 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/repo/CursorArrayList.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/repo/CursorArrayList.java @@ -1,38 +1,38 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.repo; - -import java.util.ArrayList; -import java.util.Collection; - -public class CursorArrayList extends ArrayList { - private static final long serialVersionUID = -1828085550269612339L; - private final String nextCursorMark; - - public CursorArrayList(String nextCursorMark) { - super(); - this.nextCursorMark = nextCursorMark; - } - - public CursorArrayList(Collection c, String nextCursorMark) { - super(c); - this.nextCursorMark = nextCursorMark; - } - - public CursorArrayList(int initialCapacity, String nextCursorMark) { - super(initialCapacity); - this.nextCursorMark = nextCursorMark; - } - - public String getNextCursorMark() { - return nextCursorMark; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.repo; + +import java.util.ArrayList; +import java.util.Collection; + +public class CursorArrayList extends ArrayList { + private static final long serialVersionUID = -1828085550269612339L; + private final String nextCursorMark; + + public CursorArrayList(String nextCursorMark) { + super(); + this.nextCursorMark = nextCursorMark; + } + + public CursorArrayList(Collection c, String nextCursorMark) { + super(c); + this.nextCursorMark = nextCursorMark; + } + + public CursorArrayList(int initialCapacity, String nextCursorMark) { + super(initialCapacity); + this.nextCursorMark = nextCursorMark; + } + + public String getNextCursorMark() { + return nextCursorMark; + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepository.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepository.java similarity index 98% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepository.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepository.java index 5c27b8629..d3a60c5da 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepository.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepository.java @@ -1,17 +1,17 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.repo; - -import org.springframework.data.solr.repository.SolrCrudRepository; -import uk.ac.ebi.biosamples.solr.model.SolrSample; - -public interface SolrSampleRepository - extends SolrCrudRepository, SolrSampleRepositoryCustom {} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.repo; + +import org.springframework.data.solr.repository.SolrCrudRepository; +import uk.ac.ebi.biosamples.solr.model.SolrSample; + +public interface SolrSampleRepository + extends SolrCrudRepository, SolrSampleRepositoryCustom {} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryCustom.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryCustom.java similarity index 97% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryCustom.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryCustom.java index 01f234592..a2ad86f16 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryCustom.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryCustom.java @@ -1,92 +1,92 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.repo; - -import java.util.List; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.solr.core.query.FacetQuery; -import org.springframework.data.solr.core.query.Query; -import org.springframework.data.solr.core.query.result.FacetFieldEntry; -import org.springframework.data.solr.core.query.result.FacetPage; -import uk.ac.ebi.biosamples.solr.model.SolrSample; - -public interface SolrSampleRepositoryCustom { - /** - * Get the attribute types (or other facet fields) for a particular query and using the pageable - * to determine the number of offset of *the facets* returned as a page of facet fields - * - * @param query - * @param facetPageable - * @return - */ - Page getFacetFields(FacetQuery query, Pageable facetPageable); - - /** - * Return a result of facets over the provided fields with the provided facet paging information - * (offset and count). - * - * @param query - * @param facetFields - * @param facetPageable - * @return - */ - FacetPage getFacets(FacetQuery query, List facetFields, Pageable facetPageable); - - /** - * Return a result of facets over the provided fields with the provided facet paging information - * (offset and count). - * - * @param query - * @param facetFields - * @param rangeFacetFields - * @param facetPageable - * @return - */ - FacetPage getFacets( - FacetQuery query, - List facetFields, - List rangeFacetFields, - Pageable facetPageable); - - /** - * Return a results of range facets over the provided fields with the provided facet paging - * information (offset and count). - * - * @param query - * @param facetFields - * @param facetPageable - * @return - */ - FacetPage getRangeFacets(FacetQuery query, List facetFields, Pageable facetPageable); - - /** - * Use a query object to get a page of results. This allows for more complicated query - * construction compared to a simple string e.g. filtering - * - * @param query - * @return - */ - Page findByQuery(Query query); - - /** - * Use a query object to get a page of results. This allows for more complicated query - * construction compared to a simple string e.g. filtering - * - * @param query - * @return - */ - FacetPage findByFacetQuery(FacetQuery query); - - SolrSample saveWithoutCommit(SolrSample entity); - - CursorArrayList findByQueryCursorMark(Query query, String cursorMark, int size); -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.repo; + +import java.util.List; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.solr.core.query.FacetQuery; +import org.springframework.data.solr.core.query.Query; +import org.springframework.data.solr.core.query.result.FacetFieldEntry; +import org.springframework.data.solr.core.query.result.FacetPage; +import uk.ac.ebi.biosamples.solr.model.SolrSample; + +public interface SolrSampleRepositoryCustom { + /** + * Get the attribute types (or other facet fields) for a particular query and using the pageable + * to determine the number of offset of *the facets* returned as a page of facet fields + * + * @param query + * @param facetPageable + * @return + */ + Page getFacetFields(FacetQuery query, Pageable facetPageable); + + /** + * Return a result of facets over the provided fields with the provided facet paging information + * (offset and count). + * + * @param query + * @param facetFields + * @param facetPageable + * @return + */ + FacetPage getFacets(FacetQuery query, List facetFields, Pageable facetPageable); + + /** + * Return a result of facets over the provided fields with the provided facet paging information + * (offset and count). + * + * @param query + * @param facetFields + * @param rangeFacetFields + * @param facetPageable + * @return + */ + FacetPage getFacets( + FacetQuery query, + List facetFields, + List rangeFacetFields, + Pageable facetPageable); + + /** + * Return a results of range facets over the provided fields with the provided facet paging + * information (offset and count). + * + * @param query + * @param facetFields + * @param facetPageable + * @return + */ + FacetPage getRangeFacets(FacetQuery query, List facetFields, Pageable facetPageable); + + /** + * Use a query object to get a page of results. This allows for more complicated query + * construction compared to a simple string e.g. filtering + * + * @param query + * @return + */ + Page findByQuery(Query query); + + /** + * Use a query object to get a page of results. This allows for more complicated query + * construction compared to a simple string e.g. filtering + * + * @param query + * @return + */ + FacetPage findByFacetQuery(FacetQuery query); + + SolrSample saveWithoutCommit(SolrSample entity); + + CursorArrayList findByQueryCursorMark(Query query, String cursorMark, int size); +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryImpl.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryImpl.java similarity index 97% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryImpl.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryImpl.java index 1bbcf25a6..1aad22b79 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryImpl.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/repo/SolrSampleRepositoryImpl.java @@ -1,191 +1,191 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.repo; - -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.Date; -import java.util.List; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.CursorMarkParams; -import org.apache.solr.common.params.FacetParams; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.solr.core.QueryParsers; -import org.springframework.data.solr.core.SolrTemplate; -import org.springframework.data.solr.core.mapping.SimpleSolrMappingContext; -import org.springframework.data.solr.core.query.*; -import org.springframework.data.solr.core.query.result.FacetFieldEntry; -import org.springframework.data.solr.core.query.result.FacetPage; -import org.springframework.stereotype.Component; -import org.springframework.util.Assert; -import uk.ac.ebi.biosamples.solr.model.SolrSample; - -@Component -public class SolrSampleRepositoryImpl implements SolrSampleRepositoryCustom { - // this must be SolrTemplate not SolrOperations because we use some of the details - private final SolrTemplate solrTemplate; - private final QueryParsers queryParsers = new QueryParsers(new SimpleSolrMappingContext()); - - /** - * Constructor with required fields to build its own SolrOperations object because one is not - * normally exposed as a bean. - * - * @param solrClient - */ - public SolrSampleRepositoryImpl(final SolrClient solrClient) { - solrTemplate = createTemplate(solrClient); - } - - /** - * Private method to create a SolrTemplate object. Copied from SolrRepositoryFactory - * - * @param solrClient - * @return - */ - private SolrTemplate createTemplate(final SolrClient solrClient) { - final SolrTemplate template = new SolrTemplate(solrClient); - template.afterPropertiesSet(); - return template; - } - - @Override - // TODO cacheing - public Page getFacetFields( - final FacetQuery query, final Pageable facetPageable) { - // configure the facet options to use the attribute types fields - // and to have the appropriate paging - final FacetOptions facetOptions = new FacetOptions(); - facetOptions.addFacetOnField("facetfields_ss"); - facetOptions.setPageable(facetPageable); - - query.setFacetOptions(facetOptions); - // execute the query against the solr server - final FacetPage page = - solrTemplate.queryForFacetPage("samples", query, SolrSample.class); - return page.getFacetResultPage("facetfields_ss"); - } - - @Override - public FacetPage getFacets( - final FacetQuery query, final List facetFields, final Pageable facetPageable) { - - if (facetFields == null || facetFields.size() == 0) { - throw new IllegalArgumentException("Must provide fields to facet on"); - } - - // configure the facet options to use the provided fields - // and to have the appropriate paging - final FacetOptions facetOptions = new FacetOptions(); - for (final String field : facetFields) { - facetOptions.addFacetOnField(field); - } - facetOptions.setPageable(facetPageable); - - query.setFacetOptions(facetOptions); - // execute the query against the solr server - return solrTemplate.queryForFacetPage("samples", query, SolrSample.class); - } - - @Override - public FacetPage getFacets( - final FacetQuery query, - final List facetFields, - final List rangeFacetFields, - final Pageable facetPageable) { - - if (facetFields == null || facetFields.isEmpty()) { - throw new IllegalArgumentException("Must provide fields to facet on"); - } - - // //if we are retrieving large number of facets limit for front page, remove this for new - // implementation - // if (query.getFilterQueries().size() <= 1 && - // query.getCriteria().getPredicates().stream().filter(f -> - // f.getValue().equals("*:*")).count() >= 1) { - // facetFields = facetFields.subList(0, Math.min(facetPageable.getPageSize(), - // facetFields.size())); - // } - - // configure the facet options to use the provided fields - // and to have the appropriate paging - final FacetOptions facetOptions = new FacetOptions(); - for (final String field : facetFields) { - facetOptions.addFacetOnField(field); - facetOptions.addFacetQuery(new SimpleFacetQuery(new Criteria(field))); - } - facetOptions.setPageable(facetPageable); - - // todo generalise range facets apart from dates and remove hardcoded date boundaries - for (final String field : rangeFacetFields) { - final LocalDateTime dateTime = LocalDateTime.now(); - final Date end = Date.from(dateTime.atZone(ZoneId.systemDefault()).toInstant()); - final Date start = - Date.from(dateTime.minusYears(5).atZone(ZoneId.systemDefault()).toInstant()); - facetOptions.addFacetByRange( - new FacetOptions.FieldWithDateRangeParameters(field, start, end, "+1YEAR") - .setInclude(FacetParams.FacetRangeInclude.ALL) - .setOther(FacetParams.FacetRangeOther.BEFORE)); - facetOptions.addFacetQuery(new SimpleFacetQuery(new Criteria(field))); - } - - query.setFacetOptions(facetOptions); - // execute the query against the solr server - return solrTemplate.queryForFacetPage("samples", query, SolrSample.class); - } - - @Override - public FacetPage getRangeFacets( - final FacetQuery query, final List facetFields, final Pageable facetPageable) { - // TODO Implement the method - return null; - } - - @Override - public Page findByQuery(final Query query) { - return solrTemplate.query("samples", query, SolrSample.class); - } - - @Override - public CursorArrayList findByQueryCursorMark( - final Query query, final String cursorMark, final int size) { - - // TODO this is a different set of query parsers than the solrOperation has itself - final SolrQuery solrQuery = - queryParsers.getForClass(query.getClass()).constructSolrQuery(query, SolrSample.class); - - solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); - solrQuery.set(CommonParams.ROWS, size); - - final QueryResponse response = - solrTemplate.execute(solrClient -> solrClient.query("samples", solrQuery)); - response.getNextCursorMark(); - final List solrSampleList = - solrTemplate.convertQueryResponseToBeans(response, SolrSample.class); - - return new CursorArrayList<>(solrSampleList, response.getNextCursorMark()); - } - - @Override - public FacetPage findByFacetQuery(final FacetQuery query) { - return solrTemplate.queryForFacetPage("samples", query, SolrSample.class); - } - - @Override - public SolrSample saveWithoutCommit(final SolrSample entity) { - Assert.notNull(entity, "Cannot save 'null' entity."); - solrTemplate.saveBean("samples", entity); - return entity; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.repo; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Date; +import java.util.List; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.CursorMarkParams; +import org.apache.solr.common.params.FacetParams; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.solr.core.QueryParsers; +import org.springframework.data.solr.core.SolrTemplate; +import org.springframework.data.solr.core.mapping.SimpleSolrMappingContext; +import org.springframework.data.solr.core.query.*; +import org.springframework.data.solr.core.query.result.FacetFieldEntry; +import org.springframework.data.solr.core.query.result.FacetPage; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; +import uk.ac.ebi.biosamples.solr.model.SolrSample; + +@Component +public class SolrSampleRepositoryImpl implements SolrSampleRepositoryCustom { + // this must be SolrTemplate not SolrOperations because we use some of the details + private final SolrTemplate solrTemplate; + private final QueryParsers queryParsers = new QueryParsers(new SimpleSolrMappingContext()); + + /** + * Constructor with required fields to build its own SolrOperations object because one is not + * normally exposed as a bean. + * + * @param solrClient + */ + public SolrSampleRepositoryImpl(final SolrClient solrClient) { + solrTemplate = createTemplate(solrClient); + } + + /** + * Private method to create a SolrTemplate object. Copied from SolrRepositoryFactory + * + * @param solrClient + * @return + */ + private SolrTemplate createTemplate(final SolrClient solrClient) { + final SolrTemplate template = new SolrTemplate(solrClient); + template.afterPropertiesSet(); + return template; + } + + @Override + // TODO cacheing + public Page getFacetFields( + final FacetQuery query, final Pageable facetPageable) { + // configure the facet options to use the attribute types fields + // and to have the appropriate paging + final FacetOptions facetOptions = new FacetOptions(); + facetOptions.addFacetOnField("facetfields_ss"); + facetOptions.setPageable(facetPageable); + + query.setFacetOptions(facetOptions); + // execute the query against the solr server + final FacetPage page = + solrTemplate.queryForFacetPage("samples", query, SolrSample.class); + return page.getFacetResultPage("facetfields_ss"); + } + + @Override + public FacetPage getFacets( + final FacetQuery query, final List facetFields, final Pageable facetPageable) { + + if (facetFields == null || facetFields.size() == 0) { + throw new IllegalArgumentException("Must provide fields to facet on"); + } + + // configure the facet options to use the provided fields + // and to have the appropriate paging + final FacetOptions facetOptions = new FacetOptions(); + for (final String field : facetFields) { + facetOptions.addFacetOnField(field); + } + facetOptions.setPageable(facetPageable); + + query.setFacetOptions(facetOptions); + // execute the query against the solr server + return solrTemplate.queryForFacetPage("samples", query, SolrSample.class); + } + + @Override + public FacetPage getFacets( + final FacetQuery query, + final List facetFields, + final List rangeFacetFields, + final Pageable facetPageable) { + + if (facetFields == null || facetFields.isEmpty()) { + throw new IllegalArgumentException("Must provide fields to facet on"); + } + + // //if we are retrieving large number of facets limit for front page, remove this for new + // implementation + // if (query.getFilterQueries().size() <= 1 && + // query.getCriteria().getPredicates().stream().filter(f -> + // f.getValue().equals("*:*")).count() >= 1) { + // facetFields = facetFields.subList(0, Math.min(facetPageable.getPageSize(), + // facetFields.size())); + // } + + // configure the facet options to use the provided fields + // and to have the appropriate paging + final FacetOptions facetOptions = new FacetOptions(); + for (final String field : facetFields) { + facetOptions.addFacetOnField(field); + facetOptions.addFacetQuery(new SimpleFacetQuery(new Criteria(field))); + } + facetOptions.setPageable(facetPageable); + + // todo generalise range facets apart from dates and remove hardcoded date boundaries + for (final String field : rangeFacetFields) { + final LocalDateTime dateTime = LocalDateTime.now(); + final Date end = Date.from(dateTime.atZone(ZoneId.systemDefault()).toInstant()); + final Date start = + Date.from(dateTime.minusYears(5).atZone(ZoneId.systemDefault()).toInstant()); + facetOptions.addFacetByRange( + new FacetOptions.FieldWithDateRangeParameters(field, start, end, "+1YEAR") + .setInclude(FacetParams.FacetRangeInclude.ALL) + .setOther(FacetParams.FacetRangeOther.BEFORE)); + facetOptions.addFacetQuery(new SimpleFacetQuery(new Criteria(field))); + } + + query.setFacetOptions(facetOptions); + // execute the query against the solr server + return solrTemplate.queryForFacetPage("samples", query, SolrSample.class); + } + + @Override + public FacetPage getRangeFacets( + final FacetQuery query, final List facetFields, final Pageable facetPageable) { + // TODO Implement the method + return null; + } + + @Override + public Page findByQuery(final Query query) { + return solrTemplate.query("samples", query, SolrSample.class); + } + + @Override + public CursorArrayList findByQueryCursorMark( + final Query query, final String cursorMark, final int size) { + + // TODO this is a different set of query parsers than the solrOperation has itself + final SolrQuery solrQuery = + queryParsers.getForClass(query.getClass()).constructSolrQuery(query, SolrSample.class); + + solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); + solrQuery.set(CommonParams.ROWS, size); + + final QueryResponse response = + solrTemplate.execute(solrClient -> solrClient.query("samples", solrQuery)); + response.getNextCursorMark(); + final List solrSampleList = + solrTemplate.convertQueryResponseToBeans(response, SolrSample.class); + + return new CursorArrayList<>(solrSampleList, response.getNextCursorMark()); + } + + @Override + public FacetPage findByFacetQuery(final FacetQuery query) { + return solrTemplate.queryForFacetPage("samples", query, SolrSample.class); + } + + @Override + public SolrSample saveWithoutCommit(final SolrSample entity) { + Assert.notNull(entity, "Cannot save 'null' entity."); + solrTemplate.saveBean("samples", entity); + return entity; + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SampleToSolrSampleConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SampleToSolrSampleConverter.java similarity index 89% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SampleToSolrSampleConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/service/SampleToSolrSampleConverter.java index 05bde8534..9f3390c2e 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SampleToSolrSampleConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SampleToSolrSampleConverter.java @@ -1,254 +1,273 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.service; - -import java.time.format.DateTimeFormatter; -import java.util.*; -import java.util.stream.Collectors; -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.ExternalReference; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.structured.StructuredDataEntry; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; -import uk.ac.ebi.biosamples.service.ExternalReferenceService; -import uk.ac.ebi.biosamples.service.SampleRelationshipUtils; -import uk.ac.ebi.biosamples.solr.model.SolrSample; - -@Service -public class SampleToSolrSampleConverter implements Converter { - private final ExternalReferenceService externalReferenceService; - - public SampleToSolrSampleConverter(final ExternalReferenceService externalReferenceService) { - this.externalReferenceService = externalReferenceService; - } - - @Override - public SolrSample convert(final Sample sample) { - final Map> attributeValues = new HashMap<>(); - final Map> attributeIris = new HashMap<>(); - final Map> attributeUnits = new HashMap<>(); - Map> outgoingRelationships = new HashMap<>(); - Map> incomingRelationships = new HashMap<>(); - final Map> externalReferencesData = new HashMap<>(); - final List keywords = new ArrayList<>(); - - if (sample.getCharacteristics() != null && sample.getCharacteristics().size() > 0) { - for (final Attribute attr : sample.getCharacteristics()) { - final String key = SolrFieldService.encodeFieldName(attr.getType()); - - String value = attr.getValue(); - // if its longer than 255 characters, don't add it to solr - // solr cant index long things well, and its probably not useful for search - if (value.length() > 255) { - continue; - } - - if (!attributeValues.containsKey(key)) { - attributeValues.put(key, new ArrayList<>()); - } - - // if there is a unit, add it to the value for search & facet purposes - if (attr.getUnit() != null) { - value = value + " (" + attr.getUnit() + ")"; - } - attributeValues.get(key).add(value); - - // TODO this can't differentiate which iris go with which attribute if there - // are multiple attributes with the same type - if (!attributeIris.containsKey(key)) { - attributeIris.put(key, new ArrayList<>()); - } - if (attr.getIri().isEmpty()) { - attributeIris.get(key).add(""); - } else { - final List iris = - attr.getIri().stream().map(this::getOntologyFromIri).collect(Collectors.toList()); - attributeIris.get(key).addAll(iris); - keywords.addAll(iris.stream().map(String::toLowerCase).collect(Collectors.toList())); - } - - if (!attributeUnits.containsKey(key)) { - attributeUnits.put(key, new ArrayList<>()); - } - if (attr.getUnit() == null) { - attributeUnits.get(key).add(""); - } else { - attributeUnits.get(key).add(attr.getUnit()); - } - } - } - - // turn external reference into additional attributes for facet & filter - for (final ExternalReference externalReference : sample.getExternalReferences()) { - final String externalReferenceNickname = - externalReferenceService.getNickname(externalReference); - final String externalReferenceNicknameKey = - SolrFieldService.encodeFieldName(externalReferenceNickname); - final String key = SolrFieldService.encodeFieldName("external reference"); - final String keyDuo = SolrFieldService.encodeFieldName("data use conditions"); - - if (!attributeValues.containsKey(key)) { - attributeValues.put(key, new ArrayList<>()); - } - attributeValues.get(key).add(externalReferenceNickname); - - if (externalReference.getDuo() != null && !externalReference.getDuo().isEmpty()) { - if (!attributeValues.containsKey(keyDuo)) { - attributeValues.put(keyDuo, new ArrayList<>()); - } - attributeValues.get(keyDuo).addAll(externalReference.getDuo()); - } - - // Add the external reference data id - final Optional externalReferenceDataId = - externalReferenceService.getDataId(externalReference); - if (externalReferenceDataId.isPresent()) { - if (!externalReferencesData.containsKey(externalReferenceNicknameKey)) { - externalReferencesData.put(externalReferenceNicknameKey, new ArrayList<>()); - } - externalReferencesData.get(externalReferenceNicknameKey).add(externalReferenceDataId.get()); - } - } - - // Add relationships owned by sample - final SortedSet sampleOutgoingRelationships = - SampleRelationshipUtils.getOutgoingRelationships(sample); - if (!sampleOutgoingRelationships.isEmpty()) { - final String attributeValueKey = SolrFieldService.encodeFieldName("outgoing relationships"); - if (!attributeValues.containsKey(attributeValueKey)) { - attributeValues.put(attributeValueKey, new ArrayList<>()); - } - outgoingRelationships = new HashMap<>(); - for (final Relationship rel : sampleOutgoingRelationships) { - final String key = SolrFieldService.encodeFieldName(rel.getType()); - if (!outgoingRelationships.containsKey(key)) { - outgoingRelationships.put(key, new ArrayList<>()); - } - outgoingRelationships.get(key).add(rel.getTarget()); - attributeValues.get(attributeValueKey).add(rel.getTarget()); - } - } - - // Add relationships for which sample is the target - final SortedSet sampleIngoingRelationships = - SampleRelationshipUtils.getIncomingRelationships(sample); - if (sampleIngoingRelationships != null && !sampleIngoingRelationships.isEmpty()) { - final String attributeValueKey = SolrFieldService.encodeFieldName("incoming relationships"); - if (!attributeValues.containsKey(attributeValueKey)) { - attributeValues.put(attributeValueKey, new ArrayList<>()); - } - incomingRelationships = new HashMap<>(); - for (final Relationship rel : sampleIngoingRelationships) { - final String key = SolrFieldService.encodeFieldName(rel.getType()); - if (!incomingRelationships.containsKey(key)) { - incomingRelationships.put(key, new ArrayList<>()); - } - incomingRelationships.get(key).add(rel.getSource()); - attributeValues.get(attributeValueKey).add(rel.getSource()); - } - } - - // Add structured data - final Set structuredDataSet = sample.getStructuredData(); - - if (!CollectionUtils.isEmpty(structuredDataSet)) { - final String key = SolrFieldService.encodeFieldName("structured data"); - - for (final StructuredDataTable sd : structuredDataSet) { - keywords.add(sd.getType().toLowerCase()); - - if (!attributeValues.containsKey(key)) { - attributeValues.put(key, new ArrayList<>()); - } - - attributeValues.get(key).add(sd.getType()); - - for (final Map sdMap : sd.getContent()) { - for (final Map.Entry sdMapEntry : sdMap.entrySet()) { - if (sdMapEntry != null) { - final StructuredDataEntry sdMapEntryValue = sdMapEntry.getValue(); - - if (sdMapEntry.getKey() != null && sdMapEntryValue != null) { - final String sdMapEntryValueValue = sdMapEntryValue.getValue(); - - if (sdMapEntryValueValue != null) { - keywords.addAll( - Arrays.asList(sdMapEntry.getKey(), sdMapEntryValueValue.toLowerCase())); - } - } - } - } - } - } - } - - final String releaseSolr = DateTimeFormatter.ISO_INSTANT.format(sample.getRelease()); - final String updateSolr = DateTimeFormatter.ISO_INSTANT.format(sample.getUpdate()); - - if (sample.getStatus() != null) { - final String attributeValueKey = SolrFieldService.encodeFieldName("status"); - if (!attributeValues.containsKey(attributeValueKey)) { - attributeValues.put(attributeValueKey, new ArrayList<>()); - } - attributeValues.get(attributeValueKey).add(sample.getStatus().name()); - } - - sample - .getOrganizations() - .forEach( - org -> { - keywords.addAll( - Arrays.asList(org.getName(), org.getEmail(), org.getRole(), org.getUrl())); - }); - - sample - .getContacts() - .forEach( - contact -> { - keywords.addAll( - Arrays.asList(contact.getName(), contact.getAffiliation(), contact.getUrl())); - }); - - sample - .getPublications() - .forEach( - pub -> { - keywords.addAll(Arrays.asList(pub.getDoi(), pub.getPubMedId())); - }); - - return SolrSample.build( - sample.getName(), - sample.getAccession(), - sample.getDomain(), - sample.getWebinSubmissionAccountId(), - sample.getStatus().name(), - releaseSolr, - updateSolr, - null, - null, - attributeValues, - attributeIris, - attributeUnits, - outgoingRelationships, - incomingRelationships, - externalReferencesData, - keywords); - } - - private String getOntologyFromIri(final String iri) { - final String[] iriParts = iri.split("/"); - return iriParts[iriParts.length - 1]; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.service; + +import java.time.format.DateTimeFormatter; +import java.util.*; +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.ExternalReference; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataEntry; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; +import uk.ac.ebi.biosamples.core.service.ExternalReferenceService; +import uk.ac.ebi.biosamples.core.service.SampleRelationshipUtils; +import uk.ac.ebi.biosamples.solr.model.SolrSample; + +@Service +public class SampleToSolrSampleConverter implements Converter { + private final ExternalReferenceService externalReferenceService; + + public SampleToSolrSampleConverter(final ExternalReferenceService externalReferenceService) { + this.externalReferenceService = externalReferenceService; + } + + @Override + public SolrSample convert(final Sample sample) { + final Map> attributeValues = new HashMap<>(); + final Map> attributeIris = new HashMap<>(); + final Map> attributeUnits = new HashMap<>(); + final Map> externalReferencesData = new HashMap<>(); + final List keywords = new ArrayList<>(); + + Map> outgoingRelationships = new HashMap<>(); + Map> incomingRelationships = new HashMap<>(); + + if (sample.getCharacteristics() != null && !sample.getCharacteristics().isEmpty()) { + for (final Attribute attr : sample.getCharacteristics()) { + final String key = SolrFieldService.encodeFieldName(attr.getType()); + + String value = attr.getValue(); + // if it's longer than 255 characters, don't add it to solr + // cant index long things well, and It's probably not useful for search + if (value.length() > 255) { + continue; + } + + if (!attributeValues.containsKey(key)) { + attributeValues.put(key, new ArrayList<>()); + } + + // if there is a unit, add it to the value for search & facet purposes + if (attr.getUnit() != null) { + value = value + " (" + attr.getUnit() + ")"; + } + + attributeValues.get(key).add(value); + + // TODO this can't differentiate which iris go with which attribute if there + // are multiple attributes with the same type + if (!attributeIris.containsKey(key)) { + attributeIris.put(key, new ArrayList<>()); + } + + if (attr.getIri().isEmpty()) { + attributeIris.get(key).add(""); + } else { + final List iris = attr.getIri().stream().map(this::getOntologyFromIri).toList(); + + attributeIris.get(key).addAll(iris); + keywords.addAll(iris.stream().map(String::toLowerCase).toList()); + } + + if (!attributeUnits.containsKey(key)) { + attributeUnits.put(key, new ArrayList<>()); + } + + if (attr.getUnit() == null) { + attributeUnits.get(key).add(""); + } else { + attributeUnits.get(key).add(attr.getUnit()); + } + } + } + + // turn external reference into additional attributes for facet & filter + for (final ExternalReference externalReference : sample.getExternalReferences()) { + final String externalReferenceNickname = + externalReferenceService.getNickname(externalReference); + final String externalReferenceNicknameKey = + SolrFieldService.encodeFieldName(externalReferenceNickname); + final String key = SolrFieldService.encodeFieldName("external reference"); + final String keyDuo = SolrFieldService.encodeFieldName("data use conditions"); + + if (!attributeValues.containsKey(key)) { + attributeValues.put(key, new ArrayList<>()); + } + + attributeValues.get(key).add(externalReferenceNickname); + + if (externalReference.getDuo() != null && !externalReference.getDuo().isEmpty()) { + if (!attributeValues.containsKey(keyDuo)) { + attributeValues.put(keyDuo, new ArrayList<>()); + } + + attributeValues.get(keyDuo).addAll(externalReference.getDuo()); + } + + // Add the external reference data id + final Optional externalReferenceDataId = + externalReferenceService.getDataId(externalReference); + if (externalReferenceDataId.isPresent()) { + if (!externalReferencesData.containsKey(externalReferenceNicknameKey)) { + externalReferencesData.put(externalReferenceNicknameKey, new ArrayList<>()); + } + externalReferencesData.get(externalReferenceNicknameKey).add(externalReferenceDataId.get()); + } + } + + // Add relationships owned by sample + final SortedSet sampleOutgoingRelationships = + SampleRelationshipUtils.getOutgoingRelationships(sample); + if (!sampleOutgoingRelationships.isEmpty()) { + final String attributeValueKey = SolrFieldService.encodeFieldName("outgoing relationships"); + + if (!attributeValues.containsKey(attributeValueKey)) { + attributeValues.put(attributeValueKey, new ArrayList<>()); + } + + outgoingRelationships = new HashMap<>(); + + for (final Relationship rel : sampleOutgoingRelationships) { + final String key = SolrFieldService.encodeFieldName(rel.getType()); + + if (!outgoingRelationships.containsKey(key)) { + outgoingRelationships.put(key, new ArrayList<>()); + } + + outgoingRelationships.get(key).add(rel.getTarget()); + attributeValues.get(attributeValueKey).add(rel.getTarget()); + } + } + + // Add relationships for which sample is the target + final SortedSet sampleIngoingRelationships = + SampleRelationshipUtils.getIncomingRelationships(sample); + + if (sampleIngoingRelationships != null && !sampleIngoingRelationships.isEmpty()) { + final String attributeValueKey = SolrFieldService.encodeFieldName("incoming relationships"); + + if (!attributeValues.containsKey(attributeValueKey)) { + attributeValues.put(attributeValueKey, new ArrayList<>()); + } + + incomingRelationships = new HashMap<>(); + + for (final Relationship rel : sampleIngoingRelationships) { + final String key = SolrFieldService.encodeFieldName(rel.getType()); + + if (!incomingRelationships.containsKey(key)) { + incomingRelationships.put(key, new ArrayList<>()); + } + + incomingRelationships.get(key).add(rel.getSource()); + attributeValues.get(attributeValueKey).add(rel.getSource()); + } + } + + // Add structured data + final Set structuredDataSet = sample.getStructuredData(); + + if (!CollectionUtils.isEmpty(structuredDataSet)) { + final String key = SolrFieldService.encodeFieldName("structured data"); + + for (final StructuredDataTable sd : structuredDataSet) { + keywords.add(sd.getType().toLowerCase()); + + if (!attributeValues.containsKey(key)) { + attributeValues.put(key, new ArrayList<>()); + } + + attributeValues.get(key).add(sd.getType()); + + for (final Map sdMap : sd.getContent()) { + for (final Map.Entry sdMapEntry : sdMap.entrySet()) { + if (sdMapEntry != null) { + final StructuredDataEntry sdMapEntryValue = sdMapEntry.getValue(); + + if (sdMapEntry.getKey() != null && sdMapEntryValue != null) { + final String sdMapEntryValueValue = sdMapEntryValue.getValue(); + + if (sdMapEntryValueValue != null) { + keywords.addAll( + Arrays.asList(sdMapEntry.getKey(), sdMapEntryValueValue.toLowerCase())); + } + } + } + } + } + } + } + + final String releaseSolr = DateTimeFormatter.ISO_INSTANT.format(sample.getRelease()); + final String updateSolr = DateTimeFormatter.ISO_INSTANT.format(sample.getUpdate()); + + if (sample.getStatus() != null) { + final String attributeValueKey = SolrFieldService.encodeFieldName("status"); + + if (!attributeValues.containsKey(attributeValueKey)) { + attributeValues.put(attributeValueKey, new ArrayList<>()); + } + + attributeValues.get(attributeValueKey).add(sample.getStatus().name()); + } + + sample + .getOrganizations() + .forEach( + org -> { + keywords.addAll( + Arrays.asList(org.getName(), org.getEmail(), org.getRole(), org.getUrl())); + }); + + sample + .getContacts() + .forEach( + contact -> { + keywords.addAll( + Arrays.asList(contact.getName(), contact.getAffiliation(), contact.getUrl())); + }); + + sample + .getPublications() + .forEach( + pub -> { + keywords.addAll(Arrays.asList(pub.getDoi(), pub.getPubMedId())); + }); + + return SolrSample.build( + sample.getName(), + sample.getAccession(), + sample.getDomain(), + sample.getWebinSubmissionAccountId(), + sample.getStatus().name(), + releaseSolr, + updateSolr, + null, + null, + attributeValues, + attributeIris, + attributeUnits, + outgoingRelationships, + incomingRelationships, + externalReferencesData, + keywords); + } + + private String getOntologyFromIri(final String iri) { + final String[] iriParts = iri.split("/"); + + return iriParts[iriParts.length - 1]; + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFacetService.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFacetService.java similarity index 95% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFacetService.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFacetService.java index 542055474..7a5a53b4d 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFacetService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFacetService.java @@ -1,210 +1,210 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.service; - -import java.util.*; -import java.util.AbstractMap.SimpleEntry; -import java.util.Map.Entry; -import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.solr.core.query.Criteria; -import org.springframework.data.solr.core.query.FacetQuery; -import org.springframework.data.solr.core.query.FilterQuery; -import org.springframework.data.solr.core.query.SimpleFacetQuery; -import org.springframework.data.solr.core.query.result.FacetFieldEntry; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.facet.Facet; -import uk.ac.ebi.biosamples.model.facet.FacetHelper; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.solr.model.field.SolrSampleField; -import uk.ac.ebi.biosamples.solr.repo.SolrSampleRepository; - -@Service -public class SolrFacetService { - private static final int TIME_ALLOWED = 55; - private final SolrSampleRepository solrSampleRepository; - private final SolrFieldService solrFieldService; - private final SolrFilterService solrFilterService; - - public SolrFacetService( - final SolrSampleRepository solrSampleRepository, - final SolrFieldService solrFieldService, - final SolrFilterService solrFilterService) { - this.solrSampleRepository = solrSampleRepository; - this.solrFieldService = solrFieldService; - this.solrFilterService = solrFilterService; - } - - public List getFacets( - final String searchTerm, - final Collection filters, - final Pageable facetFieldPageInfo, - final Pageable facetValuesPageInfo, - final String facetField, - final List facetFields) { - boolean isLandingPage = false; - - final List facets = new ArrayList<>(); - final FacetQuery query; - if (StringUtils.isBlank(searchTerm) || "*:*".equals(searchTerm.trim())) { - query = new SimpleFacetQuery(new Criteria().expression("*:*")); // default to search all - isLandingPage = filters.isEmpty(); - } else { - final String lowerCasedSearchTerm = searchTerm.toLowerCase().replace("\\", ""); - - // search for copied fields keywords_ss - query = new SimpleFacetQuery(); - final Criteria searchCriteria = new Criteria("keywords_ss").fuzzy(lowerCasedSearchTerm); - searchCriteria.setPartIsOr(true); - query.addCriteria(searchCriteria); - - // boosting accession to bring accession matches to the top - final Criteria boostId = new Criteria("id").is(searchTerm).boost(5); - boostId.setPartIsOr(true); - query.addCriteria(boostId); - - // boosting name to bring name matches to the top - final Criteria boostName = new Criteria("name_s").is(searchTerm).boost(3); - boostName.setPartIsOr(true); - query.addCriteria(boostName); - } - query.setTimeAllowed(TIME_ALLOWED * 1000); // some facet queries could take longer to return - - // Add domains and release date filters - final Optional domainAndPublicFilterQuery = - solrFilterService.getPublicFilterQuery(null); - domainAndPublicFilterQuery.ifPresent(query::addFilterQuery); - - // Add all the provided filters - solrFilterService.getFilterQuery(filters).forEach(query::addFilterQuery); - - final List> allFacetFields = - getFacetFields(facetFieldPageInfo, query, isLandingPage, facetField, facetFields); - - List> rangeFacetFields = Collections.emptyList(); - if (facetField == null || facetFields == null) { - rangeFacetFields = - FacetHelper.RANGE_FACETING_FIELDS.stream() - .map( - s -> - new SimpleEntry<>( - solrFieldService.decodeField(s + FacetHelper.getEncodingSuffix(s)), 0L)) - .collect(Collectors.toList()); - } - - if (!allFacetFields.isEmpty()) { - allFacetFields - .get(0) - .getKey() - .getFacetCollectionStrategy() - .fetchFacetsUsing( - solrSampleRepository, query, allFacetFields, rangeFacetFields, facetValuesPageInfo) - .forEach(opt -> opt.ifPresent(facets::add)); - } - - // Return the list of facets - Collections.sort(facets); - Collections.reverse(facets); - - return facets; - } - - public List getFacets( - final String searchTerm, - final Collection filters, - final Pageable facetFieldPageInfo, - final Pageable facetValuesPageInfo) { - - return getFacets(searchTerm, filters, facetFieldPageInfo, facetValuesPageInfo, null, null); - } - - private List> getFacetFields( - final Pageable facetFieldPageInfo, - final FacetQuery query, - final boolean isLandingPage, - final String facetField, - final List facetFields) { - final int facetLimit = 10; - final List> allFacetFields; - - // short-circuit for landing search page - if (facetField != null) { - allFacetFields = - Collections.singletonList( - new SimpleEntry<>( - solrFieldService.decodeField( - SolrFieldService.encodeFieldName(facetField) - + FacetHelper.getEncodingSuffix(facetField)), - 0L)); - } else if (facetFields != null && !facetFields.isEmpty()) { - allFacetFields = - facetFields.stream() - .map( - s -> - new SimpleEntry<>( - solrFieldService.decodeField( - SolrFieldService.encodeFieldName(s) - + FacetHelper.getEncodingSuffixForFacetingFields()), - 0L)) - .collect(Collectors.toList()); - } else if (isLandingPage) { - allFacetFields = - FacetHelper.FACETING_FIELDS.stream() - .limit(facetLimit) - .map( - s -> - new SimpleEntry<>( - solrFieldService.decodeField( - SolrFieldService.encodeFieldName(s) - + FacetHelper.getEncodingSuffix(s)), - 0L)) - .collect(Collectors.toList()); - } else { - allFacetFields = getDynamicFacetFields(facetFieldPageInfo, query, facetLimit); - } - - return allFacetFields; - } - - private List> getDynamicFacetFields( - final Pageable facetFieldPageInfo, final FacetQuery query, final int facetLimit) { - final List> allFacetFields = new ArrayList<>(); - final Page facetFields = - solrSampleRepository.getFacetFields(query, facetFieldPageInfo); - int facetCount = 0; - for (final FacetFieldEntry ffe : facetFields) { - final SolrSampleField solrSampleField = solrFieldService.decodeField(ffe.getValue()); - if (FacetHelper.FACETING_FIELDS.contains(solrSampleField.getReadableLabel())) { - final Long facetFieldCount = ffe.getValueCount(); - allFacetFields.add(new SimpleEntry<>(solrSampleField, facetFieldCount)); - if (++facetCount >= facetLimit) { - break; - } - } - } - - for (final FacetFieldEntry ffe : facetFields) { - if (facetCount++ >= facetLimit) { - break; - } - final SolrSampleField solrSampleField = solrFieldService.decodeField(ffe.getValue()); - if (!FacetHelper.FACETING_FIELDS.contains(solrSampleField.getReadableLabel()) - && !FacetHelper.IGNORE_FACETING_FIELDS.contains(solrSampleField.getReadableLabel())) { - final Long facetFieldCount = ffe.getValueCount(); - allFacetFields.add(new SimpleEntry<>(solrSampleField, facetFieldCount)); - } - } - - return allFacetFields; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.service; + +import java.util.*; +import java.util.AbstractMap.SimpleEntry; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.solr.core.query.Criteria; +import org.springframework.data.solr.core.query.FacetQuery; +import org.springframework.data.solr.core.query.FilterQuery; +import org.springframework.data.solr.core.query.SimpleFacetQuery; +import org.springframework.data.solr.core.query.result.FacetFieldEntry; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.facet.FacetHelper; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.model.field.SolrSampleField; +import uk.ac.ebi.biosamples.solr.repo.SolrSampleRepository; + +@Service +public class SolrFacetService { + private static final int TIME_ALLOWED = 55; + private final SolrSampleRepository solrSampleRepository; + private final SolrFieldService solrFieldService; + private final SolrFilterService solrFilterService; + + public SolrFacetService( + final SolrSampleRepository solrSampleRepository, + final SolrFieldService solrFieldService, + final SolrFilterService solrFilterService) { + this.solrSampleRepository = solrSampleRepository; + this.solrFieldService = solrFieldService; + this.solrFilterService = solrFilterService; + } + + public List getFacets( + final String searchTerm, + final Collection filters, + final Pageable facetFieldPageInfo, + final Pageable facetValuesPageInfo, + final String facetField, + final List facetFields) { + boolean isLandingPage = false; + + final List facets = new ArrayList<>(); + final FacetQuery query; + if (StringUtils.isBlank(searchTerm) || "*:*".equals(searchTerm.trim())) { + query = new SimpleFacetQuery(new Criteria().expression("*:*")); // default to search all + isLandingPage = filters.isEmpty(); + } else { + final String lowerCasedSearchTerm = searchTerm.toLowerCase().replace("\\", ""); + + // search for copied fields keywords_ss + query = new SimpleFacetQuery(); + final Criteria searchCriteria = new Criteria("keywords_ss").fuzzy(lowerCasedSearchTerm); + searchCriteria.setPartIsOr(true); + query.addCriteria(searchCriteria); + + // boosting accession to bring accession matches to the top + final Criteria boostId = new Criteria("id").is(searchTerm).boost(5); + boostId.setPartIsOr(true); + query.addCriteria(boostId); + + // boosting name to bring name matches to the top + final Criteria boostName = new Criteria("name_s").is(searchTerm).boost(3); + boostName.setPartIsOr(true); + query.addCriteria(boostName); + } + query.setTimeAllowed(TIME_ALLOWED * 1000); // some facet queries could take longer to return + + // Add domains and release date filters + final Optional domainAndPublicFilterQuery = + solrFilterService.getPublicFilterQuery(null); + domainAndPublicFilterQuery.ifPresent(query::addFilterQuery); + + // Add all the provided filters + solrFilterService.getFilterQuery(filters).forEach(query::addFilterQuery); + + final List> allFacetFields = + getFacetFields(facetFieldPageInfo, query, isLandingPage, facetField, facetFields); + + List> rangeFacetFields = Collections.emptyList(); + if (facetField == null || facetFields == null) { + rangeFacetFields = + FacetHelper.RANGE_FACETING_FIELDS.stream() + .map( + s -> + new SimpleEntry<>( + solrFieldService.decodeField(s + FacetHelper.getEncodingSuffix(s)), 0L)) + .collect(Collectors.toList()); + } + + if (!allFacetFields.isEmpty()) { + allFacetFields + .get(0) + .getKey() + .getFacetCollectionStrategy() + .fetchFacetsUsing( + solrSampleRepository, query, allFacetFields, rangeFacetFields, facetValuesPageInfo) + .forEach(opt -> opt.ifPresent(facets::add)); + } + + // Return the list of facets + Collections.sort(facets); + Collections.reverse(facets); + + return facets; + } + + public List getFacets( + final String searchTerm, + final Collection filters, + final Pageable facetFieldPageInfo, + final Pageable facetValuesPageInfo) { + + return getFacets(searchTerm, filters, facetFieldPageInfo, facetValuesPageInfo, null, null); + } + + private List> getFacetFields( + final Pageable facetFieldPageInfo, + final FacetQuery query, + final boolean isLandingPage, + final String facetField, + final List facetFields) { + final int facetLimit = 10; + final List> allFacetFields; + + // short-circuit for landing search page + if (facetField != null) { + allFacetFields = + Collections.singletonList( + new SimpleEntry<>( + solrFieldService.decodeField( + SolrFieldService.encodeFieldName(facetField) + + FacetHelper.getEncodingSuffix(facetField)), + 0L)); + } else if (facetFields != null && !facetFields.isEmpty()) { + allFacetFields = + facetFields.stream() + .map( + s -> + new SimpleEntry<>( + solrFieldService.decodeField( + SolrFieldService.encodeFieldName(s) + + FacetHelper.getEncodingSuffixForFacetingFields()), + 0L)) + .collect(Collectors.toList()); + } else if (isLandingPage) { + allFacetFields = + FacetHelper.FACETING_FIELDS.stream() + .limit(facetLimit) + .map( + s -> + new SimpleEntry<>( + solrFieldService.decodeField( + SolrFieldService.encodeFieldName(s) + + FacetHelper.getEncodingSuffix(s)), + 0L)) + .collect(Collectors.toList()); + } else { + allFacetFields = getDynamicFacetFields(facetFieldPageInfo, query, facetLimit); + } + + return allFacetFields; + } + + private List> getDynamicFacetFields( + final Pageable facetFieldPageInfo, final FacetQuery query, final int facetLimit) { + final List> allFacetFields = new ArrayList<>(); + final Page facetFields = + solrSampleRepository.getFacetFields(query, facetFieldPageInfo); + int facetCount = 0; + for (final FacetFieldEntry ffe : facetFields) { + final SolrSampleField solrSampleField = solrFieldService.decodeField(ffe.getValue()); + if (FacetHelper.FACETING_FIELDS.contains(solrSampleField.getReadableLabel())) { + final Long facetFieldCount = ffe.getValueCount(); + allFacetFields.add(new SimpleEntry<>(solrSampleField, facetFieldCount)); + if (++facetCount >= facetLimit) { + break; + } + } + } + + for (final FacetFieldEntry ffe : facetFields) { + if (facetCount++ >= facetLimit) { + break; + } + final SolrSampleField solrSampleField = solrFieldService.decodeField(ffe.getValue()); + if (!FacetHelper.FACETING_FIELDS.contains(solrSampleField.getReadableLabel()) + && !FacetHelper.IGNORE_FACETING_FIELDS.contains(solrSampleField.getReadableLabel())) { + final Long facetFieldCount = ffe.getValueCount(); + allFacetFields.add(new SimpleEntry<>(solrSampleField, facetFieldCount)); + } + } + + return allFacetFields; + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFieldService.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFieldService.java similarity index 96% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFieldService.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFieldService.java index 49e15bfc4..cd4e70ce0 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFieldService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFieldService.java @@ -1,145 +1,145 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.service; - -import com.google.common.io.BaseEncoding; -import java.io.UnsupportedEncodingException; -import java.lang.reflect.InvocationTargetException; -import java.util.List; -import java.util.Optional; -import java.util.regex.Matcher; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.solr.model.field.SolrSampleField; - -/** - * SolrFieldService is the service that should be able to deal with all field matters - Encode and - * decode of a field is the main reason behind it - */ -@Service -public class SolrFieldService { - - private final List solrFieldList; - - @Autowired - public SolrFieldService(final List solrSampleFields) { - solrFieldList = solrSampleFields; - } - - public List getSolrFieldList() { - return solrFieldList; - } - - public static String encodeFieldName(String field) { - // solr only allows alphanumeric field types - try { - field = BaseEncoding.base32().encode(field.getBytes("UTF-8")); - } catch (final UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - // although its base32 encoded, that include = which solr doesn't allow - field = field.replaceAll("=", "_"); - - return field; - } - - public static String decodeFieldName(final String encodedField) { - // although its base32 encoded, that include = which solr doesn't allow - String decodedField = encodedField.replace("_", "="); - try { - decodedField = new String(BaseEncoding.base32().decode(decodedField), "UTF-8"); - } catch (final UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - return decodedField; - } - - /** - * Try to decode a field guessing the facet type - * - * @param encodedField encoded version of the field with the type suffix - * @return the field name decoded - */ - public SolrSampleField decodeField(final String encodedField) { - - final Optional optionalType = - solrFieldList.stream().filter(solrField -> solrField.matches(encodedField)).findFirst(); - if (optionalType.isPresent()) { - final SolrSampleField fieldCandidate = optionalType.get(); - final Matcher m = fieldCandidate.getSolrFieldPattern().matcher(encodedField); - if (m.find()) { - String baseLabel = m.group("fieldname"); - - if (fieldCandidate.isEncodedField()) { - baseLabel = decodeFieldName(baseLabel); - } - try { - - return getNewFieldInstance(fieldCandidate.getClass(), baseLabel, encodedField); - - } catch (final NoSuchMethodException - | IllegalAccessException - | InvocationTargetException - | InstantiationException e) { - throw new RuntimeException( - "An error occurred while instantiating creating a new instance of class " - + fieldCandidate.getClass()); - } - } - } - - throw new RuntimeException("Provide field " + encodedField + " is unknown"); - } - - public SolrSampleField getCompatibleField(final Filter filter) { - - final Optional optionalType = - solrFieldList.stream().filter(solrField -> solrField.isCompatibleWith(filter)).findFirst(); - if (optionalType.isPresent()) { - final SolrSampleField fieldCandidate = optionalType.get(); - // TODO implement methods to extract suffix and generate also the encoded label - - try { - - return getNewFieldInstance(fieldCandidate.getClass(), filter.getLabel()); - - } catch (final NoSuchMethodException - | IllegalAccessException - | InvocationTargetException - | InstantiationException e) { - throw new RuntimeException( - "An error occurred while instantiating creating a new instance of class " - + fieldCandidate.getClass()); - } - } - - throw new RuntimeException("Provide filter " + filter + " is unknown"); - } - - public SolrSampleField getNewFieldInstance( - final Class prototype, - final String baseLabel, - final String encodedLabel) - throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, - InstantiationException { - return prototype - .getConstructor(String.class, String.class) - .newInstance(baseLabel, encodedLabel); - } - - public SolrSampleField getNewFieldInstance( - final Class prototype, final String baseLabel) - throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, - InstantiationException { - return prototype.getConstructor(String.class).newInstance(baseLabel); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.service; + +import com.google.common.io.BaseEncoding; +import java.io.UnsupportedEncodingException; +import java.lang.reflect.InvocationTargetException; +import java.util.List; +import java.util.Optional; +import java.util.regex.Matcher; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.model.field.SolrSampleField; + +/** + * SolrFieldService is the service that should be able to deal with all field matters - Encode and + * decode of a field is the main reason behind it + */ +@Service +public class SolrFieldService { + + private final List solrFieldList; + + @Autowired + public SolrFieldService(final List solrSampleFields) { + solrFieldList = solrSampleFields; + } + + public List getSolrFieldList() { + return solrFieldList; + } + + public static String encodeFieldName(String field) { + // solr only allows alphanumeric field types + try { + field = BaseEncoding.base32().encode(field.getBytes("UTF-8")); + } catch (final UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + // although its base32 encoded, that include = which solr doesn't allow + field = field.replaceAll("=", "_"); + + return field; + } + + public static String decodeFieldName(final String encodedField) { + // although its base32 encoded, that include = which solr doesn't allow + String decodedField = encodedField.replace("_", "="); + try { + decodedField = new String(BaseEncoding.base32().decode(decodedField), "UTF-8"); + } catch (final UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + return decodedField; + } + + /** + * Try to decode a field guessing the facet type + * + * @param encodedField encoded version of the field with the type suffix + * @return the field name decoded + */ + public SolrSampleField decodeField(final String encodedField) { + + final Optional optionalType = + solrFieldList.stream().filter(solrField -> solrField.matches(encodedField)).findFirst(); + if (optionalType.isPresent()) { + final SolrSampleField fieldCandidate = optionalType.get(); + final Matcher m = fieldCandidate.getSolrFieldPattern().matcher(encodedField); + if (m.find()) { + String baseLabel = m.group("fieldname"); + + if (fieldCandidate.isEncodedField()) { + baseLabel = decodeFieldName(baseLabel); + } + try { + + return getNewFieldInstance(fieldCandidate.getClass(), baseLabel, encodedField); + + } catch (final NoSuchMethodException + | IllegalAccessException + | InvocationTargetException + | InstantiationException e) { + throw new RuntimeException( + "An error occurred while instantiating creating a new instance of class " + + fieldCandidate.getClass()); + } + } + } + + throw new RuntimeException("Provide field " + encodedField + " is unknown"); + } + + public SolrSampleField getCompatibleField(final Filter filter) { + + final Optional optionalType = + solrFieldList.stream().filter(solrField -> solrField.isCompatibleWith(filter)).findFirst(); + if (optionalType.isPresent()) { + final SolrSampleField fieldCandidate = optionalType.get(); + // TODO implement methods to extract suffix and generate also the encoded label + + try { + + return getNewFieldInstance(fieldCandidate.getClass(), filter.getLabel()); + + } catch (final NoSuchMethodException + | IllegalAccessException + | InvocationTargetException + | InstantiationException e) { + throw new RuntimeException( + "An error occurred while instantiating creating a new instance of class " + + fieldCandidate.getClass()); + } + } + + throw new RuntimeException("Provide filter " + filter + " is unknown"); + } + + public SolrSampleField getNewFieldInstance( + final Class prototype, + final String baseLabel, + final String encodedLabel) + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, + InstantiationException { + return prototype + .getConstructor(String.class, String.class) + .newInstance(baseLabel, encodedLabel); + } + + public SolrSampleField getNewFieldInstance( + final Class prototype, final String baseLabel) + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, + InstantiationException { + return prototype.getConstructor(String.class).newInstance(baseLabel); + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java similarity index 95% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java index 2c18b8df5..7ceecb839 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java @@ -1,162 +1,161 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.service; - -import java.util.*; -import java.util.AbstractMap.SimpleEntry; -import java.util.stream.Collectors; -import org.springframework.data.solr.core.query.Criteria; -import org.springframework.data.solr.core.query.FilterQuery; -import org.springframework.data.solr.core.query.SimpleFilterQuery; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.model.filter.AccessionFilter; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.solr.model.field.SolrSampleField; - -@Service -public class SolrFilterService { - private final SolrFieldService solrFieldService; - private final BioSamplesProperties bioSamplesProperties; - - public SolrFilterService( - final SolrFieldService solrFieldService, final BioSamplesProperties bioSamplesProperties) { - this.solrFieldService = solrFieldService; - this.bioSamplesProperties = bioSamplesProperties; - } - - /** - * Build a filter criteria based on the filter type and filter content - * - * @param filter - * @return an optional solr criteria for filtering purpose - */ - public Optional getFilterCriteria(final Filter filter) { - - // TODO rename to getFilterTargetField - final SolrSampleField solrField = solrFieldService.getCompatibleField(filter); - final Criteria filterCriteria = solrField.getFilterCriteria(filter); - return Optional.ofNullable(filterCriteria); - } - - /** - * Return an optional list of criterias based on filters with same type and label of a reference - * filter - * - * @param availableFilters the list of filters to scan - * @param referenceFilter - * @return Optional List of criteria - */ - public Optional> getCompatibleFilters( - final List availableFilters, final Filter referenceFilter) { - final List compatibleFilterList = new ArrayList<>(); - for (final Filter nextFilter : availableFilters) { - if (nextFilter.getLabel().equals(referenceFilter.getLabel()) - && nextFilter.getType().equals(referenceFilter.getType())) { - compatibleFilterList.add(nextFilter); - } - } - if (compatibleFilterList.isEmpty()) { - return Optional.empty(); - } - return Optional.of(compatibleFilterList); - } - - /** - * Produce a filter query based on the provided filters - * - * @param filters a collection of filters - * @return the corresponding filter query - */ - List getFilterQuery(final Collection filters) { - if (filters == null || filters.size() == 0) { - return Collections.emptyList(); - } - - boolean filterActive = false; - final FilterQuery filterQuery = new SimpleFilterQuery(); - final List filterQueries = new ArrayList<>(); - final Collection> filterGroups = - filters.stream() - .collect( - Collectors.groupingBy( - filter -> new SimpleEntry(filter.getLabel(), filter.getType()))) - .values(); - for (final List group : filterGroups) { - // Compose all the or criteria available for the filters, if any available - // Reduce will go through all criteria - final boolean isAccessionFilter = group.stream().findFirst().get() instanceof AccessionFilter; - final Optional filterCriteria = - group.stream() - .map(this::getFilterCriteria) - .reduce( - Optional.empty(), - (composedCriteria, currentCriteria) -> { - if (currentCriteria.isPresent()) { - if (composedCriteria.isPresent()) { - // Compose with an OR - if (isAccessionFilter) { - return Optional.of(composedCriteria.get().and(currentCriteria.get())); - } else { - return Optional.of(composedCriteria.get().or(currentCriteria.get())); - } - } else { - // Create a new criteria - return currentCriteria; - } - } - return Optional.empty(); - }); - - filterCriteria.ifPresent(criteria -> filterQueries.add(new SimpleFilterQuery(criteria))); - } - return filterQueries; - } - - /** - * Return a filter query for public samples (released in the past) or samples part of the - * provided, and their own samples using auth domains - * - * @return a filter query for public and domain relevant samples - */ - Optional getPublicFilterQuery(final String webinSubmissionAccountId) { - // check if this is a read superuser - if (webinSubmissionAccountId != null - && webinSubmissionAccountId.equalsIgnoreCase( - bioSamplesProperties.getBiosamplesClientWebinUsername())) { - return Optional.empty(); - } - - // filter out non-public - final FilterQuery filterQuery = new SimpleFilterQuery(); - Criteria publicSampleCriteria = new Criteria("release_dt").lessThan("NOW"); - - // publicSampleCriteria = - // publicSampleCriteria.and( - // new Criteria("status_s").not().in(SampleStatus.getSearchHiddenStatuses())); - publicSampleCriteria = - publicSampleCriteria.or( - new Criteria(SolrFieldService.encodeFieldName("INSDC status") + "_av_ss") - .not() - .in(Collections.singletonList("suppressed"))); - - if (webinSubmissionAccountId != null && !webinSubmissionAccountId.isEmpty()) { - // user can see public and private samples submitted by them using their webin auth tokens - - publicSampleCriteria = - publicSampleCriteria.or(new Criteria("webinId_s").is(webinSubmissionAccountId)); - } - - filterQuery.addCriteria(publicSampleCriteria); - - return Optional.of(filterQuery); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.service; + +import java.util.*; +import java.util.AbstractMap.SimpleEntry; +import java.util.stream.Collectors; +import org.springframework.data.solr.core.query.Criteria; +import org.springframework.data.solr.core.query.FilterQuery; +import org.springframework.data.solr.core.query.SimpleFilterQuery; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.BioSamplesProperties; +import uk.ac.ebi.biosamples.core.model.filter.AccessionFilter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.model.field.SolrSampleField; + +@Service +public class SolrFilterService { + private final SolrFieldService solrFieldService; + private final BioSamplesProperties bioSamplesProperties; + + public SolrFilterService( + final SolrFieldService solrFieldService, final BioSamplesProperties bioSamplesProperties) { + this.solrFieldService = solrFieldService; + this.bioSamplesProperties = bioSamplesProperties; + } + + /** + * Build a filter criteria based on the filter type and filter content + * + * @param filter + * @return an optional solr criteria for filtering purpose + */ + public Optional getFilterCriteria(final Filter filter) { + // TODO rename to getFilterTargetField + final SolrSampleField solrField = solrFieldService.getCompatibleField(filter); + final Criteria filterCriteria = solrField.getFilterCriteria(filter); + return Optional.ofNullable(filterCriteria); + } + + /** + * Return an optional list of criterias based on filters with same type and label of a reference + * filter + * + * @param availableFilters the list of filters to scan + * @param referenceFilter + * @return Optional List of criteria + */ + public Optional> getCompatibleFilters( + final List availableFilters, final Filter referenceFilter) { + final List compatibleFilterList = new ArrayList<>(); + for (final Filter nextFilter : availableFilters) { + if (nextFilter.getLabel().equals(referenceFilter.getLabel()) + && nextFilter.getType().equals(referenceFilter.getType())) { + compatibleFilterList.add(nextFilter); + } + } + if (compatibleFilterList.isEmpty()) { + return Optional.empty(); + } + return Optional.of(compatibleFilterList); + } + + /** + * Produce a filter query based on the provided filters + * + * @param filters a collection of filters + * @return the corresponding filter query + */ + List getFilterQuery(final Collection filters) { + if (filters == null || filters.size() == 0) { + return Collections.emptyList(); + } + + boolean filterActive = false; + final FilterQuery filterQuery = new SimpleFilterQuery(); + final List filterQueries = new ArrayList<>(); + final Collection> filterGroups = + filters.stream() + .collect( + Collectors.groupingBy( + filter -> new SimpleEntry(filter.getLabel(), filter.getType()))) + .values(); + for (final List group : filterGroups) { + // Compose all the or criteria available for the filters, if any available + // Reduce will go through all criteria + final boolean isAccessionFilter = group.stream().findFirst().get() instanceof AccessionFilter; + final Optional filterCriteria = + group.stream() + .map(this::getFilterCriteria) + .reduce( + Optional.empty(), + (composedCriteria, currentCriteria) -> { + if (currentCriteria.isPresent()) { + if (composedCriteria.isPresent()) { + // Compose with an OR + if (isAccessionFilter) { + return Optional.of(composedCriteria.get().and(currentCriteria.get())); + } else { + return Optional.of(composedCriteria.get().or(currentCriteria.get())); + } + } else { + // Create a new criteria + return currentCriteria; + } + } + return Optional.empty(); + }); + + filterCriteria.ifPresent(criteria -> filterQueries.add(new SimpleFilterQuery(criteria))); + } + return filterQueries; + } + + /** + * Return a filter query for public samples (released in the past) or samples part of the + * provided, and their own samples using auth domains + * + * @return a filter query for public and domain relevant samples + */ + Optional getPublicFilterQuery(final String webinSubmissionAccountId) { + // check if this is a read superuser + if (webinSubmissionAccountId != null + && webinSubmissionAccountId.equalsIgnoreCase( + bioSamplesProperties.getBiosamplesClientWebinUsername())) { + return Optional.empty(); + } + + // filter out non-public + final FilterQuery filterQuery = new SimpleFilterQuery(); + Criteria publicSampleCriteria = new Criteria("release_dt").lessThan("NOW"); + + // publicSampleCriteria = + // publicSampleCriteria.and( + // new Criteria("status_s").not().in(SampleStatus.getSearchHiddenStatuses())); + publicSampleCriteria = + publicSampleCriteria.or( + new Criteria(SolrFieldService.encodeFieldName("INSDC status") + "_av_ss") + .not() + .in(Collections.singletonList("suppressed"))); + + if (webinSubmissionAccountId != null && !webinSubmissionAccountId.isEmpty()) { + // user can see public and private samples submitted by them using their webin auth tokens + + publicSampleCriteria = + publicSampleCriteria.or(new Criteria("webinId_s").is(webinSubmissionAccountId)); + } + + filterQuery.addCriteria(publicSampleCriteria); + + return Optional.of(filterQuery); + } +} diff --git a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java similarity index 96% rename from models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java rename to core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java index b043a4143..40312c3f5 100644 --- a/models/solr/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java @@ -1,132 +1,132 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.service; - -import java.util.Collection; -import java.util.Optional; -import org.apache.commons.lang3.StringUtils; -import org.apache.solr.client.solrj.util.ClientUtils; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Sort; -import org.springframework.data.solr.core.query.*; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.solr.model.SolrSample; -import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; -import uk.ac.ebi.biosamples.solr.repo.SolrSampleRepository; - -@Service -public class SolrSampleService { - private final SolrSampleRepository solrSampleRepository; - private final SolrFilterService solrFilterService; - - // maximum time allowed for a solr search in s - // TODO application.properties this - private static final int TIMEALLOWED = 55; - - public SolrSampleService( - final SolrSampleRepository solrSampleRepository, final SolrFilterService solrFilterService) { - this.solrSampleRepository = solrSampleRepository; - this.solrFilterService = solrFilterService; - } - - /** - * Fetch the solr samples based on query specification - * - * @param searchTerm the term to search for in solr - * @param filters a Collection of filters used in the solr query - * @param pageable pagination information - * @return a page of Samples full-filling the query - */ - public Page fetchSolrSampleByText( - final String searchTerm, - final Collection filters, - final String webinSubmissionAccountId, - final Pageable pageable) { - Page result; - try { - final Query query = buildQuery(searchTerm, filters, webinSubmissionAccountId); - query.setPageRequest(pageable); - query.setTimeAllowed(TIMEALLOWED * 1000); - // return the samples from solr that match the query - result = solrSampleRepository.findByQuery(query); - } catch (final Exception e) { - // If it is not possible to use the search as a filter treat search string as text - final String escapedSearchTerm = - searchTerm == null ? null : ClientUtils.escapeQueryChars(searchTerm); - final Query query = buildQuery(escapedSearchTerm, filters, webinSubmissionAccountId); - query.setPageRequest(pageable); - query.setTimeAllowed(TIMEALLOWED * 1000); - // return the samples from solr that match the query - result = solrSampleRepository.findByQuery(query); - } - return result; - } - - /** - * Fetch the solr samples based on query specification - * - * @param searchTerm the term to search for in solr - * @param filters a Collection of filters used in the solr query - * @param cursorMark cursor serialization - * @return a page of Samples full-filling the query - */ - public CursorArrayList fetchSolrSampleByText( - final String searchTerm, - final Collection filters, - final String webinSubmissionAccountId, - final String cursorMark, - final int size) { - final Query query = buildQuery(searchTerm, filters, webinSubmissionAccountId); - query.addSort(Sort.by("id")); // this must match the field in solr - - return solrSampleRepository.findByQueryCursorMark(query, cursorMark, size); - } - - private Query buildQuery( - final String searchTerm, - final Collection filters, - final String webinSubmissionAccountId) { - final Query query; - if (StringUtils.isBlank(searchTerm) || "*:*".equals(searchTerm.trim())) { - query = new SimpleQuery("*:*"); // default to search all - } else { - final String lowerCasedSearchTerm = searchTerm.toLowerCase(); - - // search for copied fields keywords_ss. - query = new SimpleQuery(); - final Criteria searchCriteria = new Criteria("keywords_ss").fuzzy(lowerCasedSearchTerm); - searchCriteria.setPartIsOr(true); - query.addCriteria(searchCriteria); - - // boosting accession to bring accession matches to the top - final Criteria boostId = new Criteria("id").is(searchTerm).boost(5); - boostId.setPartIsOr(true); - query.addCriteria(boostId); - - // boosting name to bring accession matches to the top - final Criteria boostName = new Criteria("name_s").is(searchTerm).boost(3); - boostName.setPartIsOr(true); - query.addCriteria(boostName); - } - - query.addProjectionOnField(new SimpleField("id")); - - final Optional publicFilterQuery = - solrFilterService.getPublicFilterQuery(webinSubmissionAccountId); - publicFilterQuery.ifPresent(query::addFilterQuery); - - solrFilterService.getFilterQuery(filters).forEach(query::addFilterQuery); - - return query; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr.service; + +import java.util.Collection; +import java.util.Optional; +import org.apache.commons.lang3.StringUtils; +import org.apache.solr.client.solrj.util.ClientUtils; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.data.solr.core.query.*; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.solr.model.SolrSample; +import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; +import uk.ac.ebi.biosamples.solr.repo.SolrSampleRepository; + +@Service +public class SolrSampleService { + private final SolrSampleRepository solrSampleRepository; + private final SolrFilterService solrFilterService; + + // maximum time allowed for a solr search in s + // TODO application.properties this + private static final int TIMEALLOWED = 55; + + public SolrSampleService( + final SolrSampleRepository solrSampleRepository, final SolrFilterService solrFilterService) { + this.solrSampleRepository = solrSampleRepository; + this.solrFilterService = solrFilterService; + } + + /** + * Fetch the solr samples based on query specification + * + * @param searchTerm the term to search for in solr + * @param filters a Collection of filters used in the solr query + * @param pageable pagination information + * @return a page of Samples full-filling the query + */ + public Page fetchSolrSampleByText( + final String searchTerm, + final Collection filters, + final String webinSubmissionAccountId, + final Pageable pageable) { + Page result; + try { + final Query query = buildQuery(searchTerm, filters, webinSubmissionAccountId); + query.setPageRequest(pageable); + query.setTimeAllowed(TIMEALLOWED * 1000); + // return the samples from solr that match the query + result = solrSampleRepository.findByQuery(query); + } catch (final Exception e) { + // If it is not possible to use the search as a filter treat search string as text + final String escapedSearchTerm = + searchTerm == null ? null : ClientUtils.escapeQueryChars(searchTerm); + final Query query = buildQuery(escapedSearchTerm, filters, webinSubmissionAccountId); + query.setPageRequest(pageable); + query.setTimeAllowed(TIMEALLOWED * 1000); + // return the samples from solr that match the query + result = solrSampleRepository.findByQuery(query); + } + return result; + } + + /** + * Fetch the solr samples based on query specification + * + * @param searchTerm the term to search for in solr + * @param filters a Collection of filters used in the solr query + * @param cursorMark cursor serialization + * @return a page of Samples full-filling the query + */ + public CursorArrayList fetchSolrSampleByText( + final String searchTerm, + final Collection filters, + final String webinSubmissionAccountId, + final String cursorMark, + final int size) { + final Query query = buildQuery(searchTerm, filters, webinSubmissionAccountId); + query.addSort(Sort.by("id")); // this must match the field in solr + + return solrSampleRepository.findByQueryCursorMark(query, cursorMark, size); + } + + private Query buildQuery( + final String searchTerm, + final Collection filters, + final String webinSubmissionAccountId) { + final Query query; + if (StringUtils.isBlank(searchTerm) || "*:*".equals(searchTerm.trim())) { + query = new SimpleQuery("*:*"); // default to search all + } else { + final String lowerCasedSearchTerm = searchTerm.toLowerCase(); + + // search for copied fields keywords_ss. + query = new SimpleQuery(); + final Criteria searchCriteria = new Criteria("keywords_ss").fuzzy(lowerCasedSearchTerm); + searchCriteria.setPartIsOr(true); + query.addCriteria(searchCriteria); + + // boosting accession to bring accession matches to the top + final Criteria boostId = new Criteria("id").is(searchTerm).boost(5); + boostId.setPartIsOr(true); + query.addCriteria(boostId); + + // boosting name to bring accession matches to the top + final Criteria boostName = new Criteria("name_s").is(searchTerm).boost(3); + boostName.setPartIsOr(true); + query.addCriteria(boostName); + } + + query.addProjectionOnField(new SimpleField("id")); + + final Optional publicFilterQuery = + solrFilterService.getPublicFilterQuery(webinSubmissionAccountId); + publicFilterQuery.ifPresent(query::addFilterQuery); + + solrFilterService.getFilterQuery(filters).forEach(query::addFilterQuery); + + return query; + } +} diff --git a/utils/client/src/main/java/uk/ac/ebi/biosamples/utils/ClientUtils.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/ClientUtils.java similarity index 97% rename from utils/client/src/main/java/uk/ac/ebi/biosamples/utils/ClientUtils.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/ClientUtils.java index f3a76e9d7..a090600a7 100644 --- a/utils/client/src/main/java/uk/ac/ebi/biosamples/utils/ClientUtils.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/ClientUtils.java @@ -1,85 +1,85 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils; - -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.RequestEntity; -import org.springframework.http.ResponseEntity; -import org.springframework.web.client.HttpStatusCodeException; -import org.springframework.web.client.RestOperations; - -public class ClientUtils { - public static ResponseEntity doRetryQuery( - final RequestEntity requestEntity, - final RestOperations restOperations, - final int maxRetries, - final ParameterizedTypeReference parameterizedTypeReference) { - ResponseEntity responseEntity = null; - int retries = 0; - while (responseEntity == null && retries < maxRetries) { - try { - responseEntity = restOperations.exchange(requestEntity, parameterizedTypeReference); - } catch (final HttpStatusCodeException e) { - if (e.getStatusCode() == HttpStatus.TOO_MANY_REQUESTS - || e.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) { - // need to wait and retry - String retryString = ""; - // may include a header for how long to wait till retry - if (responseEntity.getHeaders().keySet().contains(HttpHeaders.RETRY_AFTER)) { - retryString = responseEntity.getHeaders().get(HttpHeaders.RETRY_AFTER).get(0); - } - // retry header after could be an integer number of seconds, or a date - ZonedDateTime retryTime = null; - try { - retryTime = ZonedDateTime.parse(retryString, DateTimeFormatter.RFC_1123_DATE_TIME); - } catch (final DateTimeParseException e2) { - // do nothing - } - if (retryTime == null) { - int delaySeconds = 10 * retries; // default to waiting 10 seconds per retry before - // trying again - try { - delaySeconds = Integer.parseInt(retryString); - } catch (final NumberFormatException e2) { - // do nothing - } - // TODO use the date/time from the response, instead of now? - retryTime = ZonedDateTime.now(ZoneOffset.UTC).plusSeconds(delaySeconds); - } - - // at this point, we will have a time after which we can retry - // sleep until after that time - while (ZonedDateTime.now(ZoneOffset.UTC).isBefore(retryTime)) { - try { - Thread.sleep(1000); - } catch (final InterruptedException e2) { - throw new RuntimeException(e2); - } - } - - // now we can retry and hope it works - retries += 1; - responseEntity = null; - - } else { - throw e; - } - } - } - return responseEntity; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils; + +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpStatusCodeException; +import org.springframework.web.client.RestOperations; + +public class ClientUtils { + public static ResponseEntity doRetryQuery( + final RequestEntity requestEntity, + final RestOperations restOperations, + final int maxRetries, + final ParameterizedTypeReference parameterizedTypeReference) { + ResponseEntity responseEntity = null; + int retries = 0; + while (responseEntity == null && retries < maxRetries) { + try { + responseEntity = restOperations.exchange(requestEntity, parameterizedTypeReference); + } catch (final HttpStatusCodeException e) { + if (e.getStatusCode() == HttpStatus.TOO_MANY_REQUESTS + || e.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) { + // need to wait and retry + String retryString = ""; + // may include a header for how long to wait till retry + if (responseEntity.getHeaders().keySet().contains(HttpHeaders.RETRY_AFTER)) { + retryString = responseEntity.getHeaders().get(HttpHeaders.RETRY_AFTER).get(0); + } + // retry header after could be an integer number of seconds, or a date + ZonedDateTime retryTime = null; + try { + retryTime = ZonedDateTime.parse(retryString, DateTimeFormatter.RFC_1123_DATE_TIME); + } catch (final DateTimeParseException e2) { + // do nothing + } + if (retryTime == null) { + int delaySeconds = 10 * retries; // default to waiting 10 seconds per retry before + // trying again + try { + delaySeconds = Integer.parseInt(retryString); + } catch (final NumberFormatException e2) { + // do nothing + } + // TODO use the date/time from the response, instead of now? + retryTime = ZonedDateTime.now(ZoneOffset.UTC).plusSeconds(delaySeconds); + } + + // at this point, we will have a time after which we can retry + // sleep until after that time + while (ZonedDateTime.now(ZoneOffset.UTC).isBefore(retryTime)) { + try { + Thread.sleep(1000); + } catch (final InterruptedException e2) { + throw new RuntimeException(e2); + } + } + + // now we can retry and hope it works + retries += 1; + responseEntity = null; + + } else { + throw e; + } + } + } + return responseEntity; + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/utils/LinkUtils.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/LinkUtils.java similarity index 97% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/utils/LinkUtils.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/LinkUtils.java index c22c249d0..1ae3c97ad 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/utils/LinkUtils.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/LinkUtils.java @@ -1,64 +1,64 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import org.springframework.hateoas.Link; -import org.springframework.web.util.UriUtils; - -public class LinkUtils { - - public static String decodeText(String text) { - if (text != null) { - // URLDecoder doesn't work right... - // text = URLDecoder.decode(text, "UTF-8"); - text = UriUtils.decode(text, "UTF-8"); - } - return text; - } - - public static String[] decodeTexts(final String[] texts) { - if (texts != null) { - for (int i = 0; i < texts.length; i++) { - // URLDecoder doesn't work right... - // filter[i] = URLDecoder.decode(filter[i], "UTF-8"); - texts[i] = UriUtils.decode(texts[i], "UTF-8"); - } - } - return texts; - } - - public static Optional> decodeTextsToArray(final String[] texts) { - if (texts == null) { - return Optional.empty(); - } else { - final List decoded = new ArrayList<>(texts.length); - for (final String text : texts) { - // URLDecoder doesn't work right... - // filter[i] = URLDecoder.decode(filter[i], "UTF-8"); - decoded.add(UriUtils.decode(text, "UTF-8")); - } - return Optional.of(decoded); - } - } - - public static Link cleanLink(Link link) { - // expand template to nothing - link = link.expand(Collections.emptyMap()); - // this won't handle encodings correctly, so need to manually fix that - link = Link.of(decodeText(decodeText(link.getHref())), link.getRel()); - - return link; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import org.springframework.hateoas.Link; +import org.springframework.web.util.UriUtils; + +public class LinkUtils { + + public static String decodeText(String text) { + if (text != null) { + // URLDecoder doesn't work right... + // text = URLDecoder.decode(text, "UTF-8"); + text = UriUtils.decode(text, "UTF-8"); + } + return text; + } + + public static String[] decodeTexts(final String[] texts) { + if (texts != null) { + for (int i = 0; i < texts.length; i++) { + // URLDecoder doesn't work right... + // filter[i] = URLDecoder.decode(filter[i], "UTF-8"); + texts[i] = UriUtils.decode(texts[i], "UTF-8"); + } + } + return texts; + } + + public static Optional> decodeTextsToArray(final String[] texts) { + if (texts == null) { + return Optional.empty(); + } else { + final List decoded = new ArrayList<>(texts.length); + for (final String text : texts) { + // URLDecoder doesn't work right... + // filter[i] = URLDecoder.decode(filter[i], "UTF-8"); + decoded.add(UriUtils.decode(text, "UTF-8")); + } + return Optional.of(decoded); + } + } + + public static Link cleanLink(Link link) { + // expand template to nothing + link = link.expand(Collections.emptyMap()); + // this won't handle encodings correctly, so need to manually fix that + link = Link.of(decodeText(decodeText(link.getHref())), link.getRel()); + + return link; + } +} diff --git a/commons/src/main/java/uk/ac/ebi/biosamples/utils/json/JsonFragmenter.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/json/JsonFragmenter.java similarity index 97% rename from commons/src/main/java/uk/ac/ebi/biosamples/utils/json/JsonFragmenter.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/json/JsonFragmenter.java index d6dd3e36e..4ef08c9b4 100644 --- a/commons/src/main/java/uk/ac/ebi/biosamples/utils/json/JsonFragmenter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/json/JsonFragmenter.java @@ -1,55 +1,55 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils.json; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.InputStream; -import org.springframework.stereotype.Service; - -/** - * Utility class that reads an input stream of JSON and calls a provided handler for each element of - * interest. The handler is given a DOM populated element to do something with - */ -@Service -public class JsonFragmenter { - private JsonFragmenter() {} - - public void handleStream( - final InputStream inputStream, final String encoding, final JsonCallback callback) - throws Exception { - - final ObjectMapper mapper = new ObjectMapper(); - final JsonParser parser = mapper.getFactory().createParser(inputStream); - if (parser.nextToken() != JsonToken.START_ARRAY) { - throw new IllegalStateException("A JSON array was expected"); - } - while (parser.nextToken() == JsonToken.START_OBJECT) { - final JsonNode sampleNode = mapper.readTree(parser); - if (sampleNode.has("accession")) { - final String biosampleSerialization = mapper.writeValueAsString(sampleNode); - callback.handleJson(biosampleSerialization); - } - } - } - - public interface JsonCallback { - /** - * This function is passed a DOM element of interest for further processing. - * - * @param json - * @throws Exception - */ - public void handleJson(String json) throws Exception; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.json; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.InputStream; +import org.springframework.stereotype.Service; + +/** + * Utility class that reads an input stream of JSON and calls a provided handler for each element of + * interest. The handler is given a DOM populated element to do something with + */ +@Service +public class JsonFragmenter { + private JsonFragmenter() {} + + public void handleStream( + final InputStream inputStream, final String encoding, final JsonCallback callback) + throws Exception { + + final ObjectMapper mapper = new ObjectMapper(); + final JsonParser parser = mapper.getFactory().createParser(inputStream); + if (parser.nextToken() != JsonToken.START_ARRAY) { + throw new IllegalStateException("A JSON array was expected"); + } + while (parser.nextToken() == JsonToken.START_OBJECT) { + final JsonNode sampleNode = mapper.readTree(parser); + if (sampleNode.has("accession")) { + final String biosampleSerialization = mapper.writeValueAsString(sampleNode); + callback.handleJson(biosampleSerialization); + } + } + } + + public interface JsonCallback { + /** + * This function is passed a DOM element of interest for further processing. + * + * @param json + * @throws Exception + */ + public void handleJson(String json) throws Exception; + } +} diff --git a/utils/ols/src/main/java/uk/ac/ebi/biosamples/ols/OlsProcessor.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/ols/OlsProcessor.java similarity index 97% rename from utils/ols/src/main/java/uk/ac/ebi/biosamples/ols/OlsProcessor.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/ols/OlsProcessor.java index c7d7f2eb3..e8c0897db 100644 --- a/utils/ols/src/main/java/uk/ac/ebi/biosamples/ols/OlsProcessor.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/ols/OlsProcessor.java @@ -1,240 +1,240 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.ols; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.net.URI; -import java.util.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.cache.annotation.Cacheable; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.RequestEntity; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Service; -import org.springframework.web.client.HttpStatusCodeException; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.UriComponents; -import org.springframework.web.util.UriComponentsBuilder; -import org.springframework.web.util.UriUtils; -import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.utils.ClientUtils; - -@Service -public class OlsProcessor { - private final Logger log = LoggerFactory.getLogger(getClass()); - private final RestTemplate restTemplate; - private final BioSamplesProperties bioSamplesProperties; - - public OlsProcessor( - final RestTemplate restTemplate, final BioSamplesProperties bioSamplesProperties) { - this.restTemplate = restTemplate; - this.bioSamplesProperties = bioSamplesProperties; - } - - /** - * @param ontology must be unencoded - * @param iri must be unencoded - * @return - */ - @Cacheable("ols_ancestors_synonyms") - public Collection ancestorsAndSynonyms(String ontology, String iri) { - final Set synonyms = new HashSet<>(); - if (ontology == null || ontology.trim().length() == 0) { - return synonyms; - } - if (iri == null || iri.trim().length() == 0) { - return synonyms; - } - - // check if the iri is a full iri with all the necessary parts - // build has to flag this iri as having already been encoded - final UriComponents iriComponents = UriComponentsBuilder.fromUriString(iri).build(); - if (iriComponents.getScheme() == null - || iriComponents.getHost() == null - || iriComponents.getPath() == null) { - // incomplete iri (e.g. 9606, EFO_12345) don't bother to check - return synonyms; - } - - // TODO do more by hal links, needs OLS to support - // build has to flag this iri as having already been encoded - final UriComponents uriComponents = - UriComponentsBuilder.fromUriString( - bioSamplesProperties.getOls() - + "/api/ontologies/{ontology}/terms/{term}/hierarchicalAncestors?size=1000") - .build(); - - log.trace("Base uriComponents = " + uriComponents); - - // have to *double* encode the things we are going to put in the URI - // uriComponents will encode it once, so we only need to encode it one more time manually - ontology = UriUtils.encodePathSegment(ontology, "UTF-8"); - iri = UriUtils.encodePathSegment(iri, "UTF-8"); - // expand the template using the variables - final URI uri = uriComponents.expand(ontology, iri).toUri(); - - log.debug("Contacting " + uri); - - // Note: OLS won't accept hal+json on that endpoint - final RequestEntity requestEntity = - RequestEntity.get(uri).accept(MediaType.APPLICATION_JSON).build(); - ResponseEntity responseEntity = null; - try { - responseEntity = - ClientUtils.doRetryQuery( - requestEntity, restTemplate, 5, new ParameterizedTypeReference() {}); - } catch (final HttpStatusCodeException e) { - // if we get a 404, return an empty list - if (e.getStatusCode() == HttpStatus.NOT_FOUND) { - return Collections.emptyList(); - } - } - - final JsonNode n = responseEntity.getBody(); - if (n.has("_embedded")) { - if (n.get("_embedded").has("terms")) { - for (final JsonNode o : n.get("_embedded").get("terms")) { - if (o.has("label")) { - final String synonym = o.get("label").asText(); - if (synonym != null && synonym.trim().length() > 0) { - log.trace("adding synonym " + synonym); - synonyms.add(synonym); - } - } - if (o.has("synonyms")) { - for (final JsonNode p : o.get("synonyms")) { - final String synonym = p.asText(); - if (synonym != null && synonym.trim().length() > 0) { - log.trace("adding synonym " + synonym); - synonyms.add(synonym); - } - } - } - } - } - } - - return synonyms; - } - - // @Cacheable("ols_short") - public Optional queryOlsForShortcode(final String shortcode) { - log.trace("OLS getting : " + shortcode); - - // TODO do more by hal links, needs OLS to support - final UriComponents uriComponents = - UriComponentsBuilder.fromUriString( - bioSamplesProperties.getOls() + "/api/terms?id={shortcode}&size=500") - .build(); - final URI uri = uriComponents.expand(shortcode).encode().toUri(); - - log.trace("OLS query for shortcode " + shortcode + " against " + uri); - - final RequestEntity requestEntity = - RequestEntity.get(uri).accept(MediaType.APPLICATION_JSON).build(); - final ResponseEntity responseEntity = - restTemplate.exchange(requestEntity, new ParameterizedTypeReference() {}); - - // non-200 status code - if (!responseEntity.getStatusCode().is2xxSuccessful()) { - log.trace( - "Got status " - + responseEntity.getStatusCodeValue() - + " for shortcode " - + shortcode - + " against " - + uri); - return Optional.empty(); - } - - // if zero result found, abort - if (responseEntity.getBody() == null) { - log.trace("Found empty body for shortcode " + shortcode + " against " + uri); - return Optional.empty(); - } - final ObjectNode n = responseEntity.getBody(); - - String iri = null; - if (n.has("_embedded")) { - if (n.get("_embedded").has("terms")) { - for (final JsonNode term : n.get("_embedded").get("terms")) { - if (term.has("iri") && term.has("is_defining_ontology")) { - log.trace( - "iri: " - + term.get("iri") - + ", is_defining_ontology: " - + term.get("is_defining_ontology")); - // if we don't have an iri use this but if there is a defining ontoloy use - // this in - // preference - if (iri == null || term.get("is_defining_ontology").booleanValue()) { - iri = term.get("iri").asText(); - log.trace("set iri: " + iri); - } - } - } - } - } - return Optional.ofNullable(iri); - } - - public Optional queryForOlsObject(final String shortcode) { - final UriComponents uriComponents = - UriComponentsBuilder.fromUriString( - bioSamplesProperties.getOls() + "/api/terms?id={shortcode}&size=500") - .build(); - final URI uri = uriComponents.expand(shortcode).encode().toUri(); - - log.trace("OLS query for shortcode {} against {}", shortcode, uri); - - final RequestEntity requestEntity = - RequestEntity.get(uri).accept(MediaType.APPLICATION_JSON).build(); - final ResponseEntity responseEntity = - restTemplate.exchange(requestEntity, new ParameterizedTypeReference() {}); - - // non-200 status code or empty body - if (!responseEntity.getStatusCode().is2xxSuccessful() || responseEntity.getBody() == null) { - log.trace( - "Http status: {}, Body: {}", - responseEntity.getStatusCodeValue(), - responseEntity.getBody()); - return Optional.empty(); - } - - final ObjectNode node = responseEntity.getBody(); - - OlsResult olsResult = null; - if (node.has("_embedded") && node.get("_embedded").has("terms")) { - for (final JsonNode term : node.get("_embedded").get("terms")) { - if (term.has("iri") && term.has("is_defining_ontology")) { - // if we don't have an iri use this but if there is a defining ontoloy use this - // in - // preference - if (olsResult == null || term.get("is_defining_ontology").booleanValue()) { - // if this term is obsolete get the most upto-date one - if (term.get("is_obsolete").booleanValue()) { - final String[] replaceTermArray = term.get("term_replaced_by").asText().split("/"); - final String replaceTerm = replaceTermArray[replaceTermArray.length - 1]; - return queryForOlsObject(replaceTerm); - } - - olsResult = new OlsResult(term.get("label").asText(), term.get("iri").asText()); - } - } - } - } - return Optional.ofNullable(olsResult); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.ols; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.net.URI; +import java.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpStatusCodeException; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; +import org.springframework.web.util.UriUtils; +import uk.ac.ebi.biosamples.BioSamplesProperties; +import uk.ac.ebi.biosamples.utils.ClientUtils; + +@Service +public class OlsProcessor { + private final Logger log = LoggerFactory.getLogger(getClass()); + private final RestTemplate restTemplate; + private final BioSamplesProperties bioSamplesProperties; + + public OlsProcessor( + final RestTemplate restTemplate, final BioSamplesProperties bioSamplesProperties) { + this.restTemplate = restTemplate; + this.bioSamplesProperties = bioSamplesProperties; + } + + /** + * @param ontology must be unencoded + * @param iri must be unencoded + * @return + */ + @Cacheable("ols_ancestors_synonyms") + public Collection ancestorsAndSynonyms(String ontology, String iri) { + final Set synonyms = new HashSet<>(); + if (ontology == null || ontology.trim().length() == 0) { + return synonyms; + } + if (iri == null || iri.trim().length() == 0) { + return synonyms; + } + + // check if the iri is a full iri with all the necessary parts + // build has to flag this iri as having already been encoded + final UriComponents iriComponents = UriComponentsBuilder.fromUriString(iri).build(); + if (iriComponents.getScheme() == null + || iriComponents.getHost() == null + || iriComponents.getPath() == null) { + // incomplete iri (e.g. 9606, EFO_12345) don't bother to check + return synonyms; + } + + // TODO do more by hal links, needs OLS to support + // build has to flag this iri as having already been encoded + final UriComponents uriComponents = + UriComponentsBuilder.fromUriString( + bioSamplesProperties.getOls() + + "/api/ontologies/{ontology}/terms/{term}/hierarchicalAncestors?size=1000") + .build(); + + log.trace("Base uriComponents = " + uriComponents); + + // have to *double* encode the things we are going to put in the URI + // uriComponents will encode it once, so we only need to encode it one more time manually + ontology = UriUtils.encodePathSegment(ontology, "UTF-8"); + iri = UriUtils.encodePathSegment(iri, "UTF-8"); + // expand the template using the variables + final URI uri = uriComponents.expand(ontology, iri).toUri(); + + log.debug("Contacting " + uri); + + // Note: OLS won't accept hal+json on that endpoint + final RequestEntity requestEntity = + RequestEntity.get(uri).accept(MediaType.APPLICATION_JSON).build(); + ResponseEntity responseEntity = null; + try { + responseEntity = + ClientUtils.doRetryQuery( + requestEntity, restTemplate, 5, new ParameterizedTypeReference() {}); + } catch (final HttpStatusCodeException e) { + // if we get a 404, return an empty list + if (e.getStatusCode() == HttpStatus.NOT_FOUND) { + return Collections.emptyList(); + } + } + + final JsonNode n = responseEntity.getBody(); + if (n.has("_embedded")) { + if (n.get("_embedded").has("terms")) { + for (final JsonNode o : n.get("_embedded").get("terms")) { + if (o.has("label")) { + final String synonym = o.get("label").asText(); + if (synonym != null && synonym.trim().length() > 0) { + log.trace("adding synonym " + synonym); + synonyms.add(synonym); + } + } + if (o.has("synonyms")) { + for (final JsonNode p : o.get("synonyms")) { + final String synonym = p.asText(); + if (synonym != null && synonym.trim().length() > 0) { + log.trace("adding synonym " + synonym); + synonyms.add(synonym); + } + } + } + } + } + } + + return synonyms; + } + + // @Cacheable("ols_short") + public Optional queryOlsForShortcode(final String shortcode) { + log.trace("OLS getting : " + shortcode); + + // TODO do more by hal links, needs OLS to support + final UriComponents uriComponents = + UriComponentsBuilder.fromUriString( + bioSamplesProperties.getOls() + "/api/terms?id={shortcode}&size=500") + .build(); + final URI uri = uriComponents.expand(shortcode).encode().toUri(); + + log.trace("OLS query for shortcode " + shortcode + " against " + uri); + + final RequestEntity requestEntity = + RequestEntity.get(uri).accept(MediaType.APPLICATION_JSON).build(); + final ResponseEntity responseEntity = + restTemplate.exchange(requestEntity, new ParameterizedTypeReference() {}); + + // non-200 status code + if (!responseEntity.getStatusCode().is2xxSuccessful()) { + log.trace( + "Got status " + + responseEntity.getStatusCodeValue() + + " for shortcode " + + shortcode + + " against " + + uri); + return Optional.empty(); + } + + // if zero result found, abort + if (responseEntity.getBody() == null) { + log.trace("Found empty body for shortcode " + shortcode + " against " + uri); + return Optional.empty(); + } + final ObjectNode n = responseEntity.getBody(); + + String iri = null; + if (n.has("_embedded")) { + if (n.get("_embedded").has("terms")) { + for (final JsonNode term : n.get("_embedded").get("terms")) { + if (term.has("iri") && term.has("is_defining_ontology")) { + log.trace( + "iri: " + + term.get("iri") + + ", is_defining_ontology: " + + term.get("is_defining_ontology")); + // if we don't have an iri use this but if there is a defining ontoloy use + // this in + // preference + if (iri == null || term.get("is_defining_ontology").booleanValue()) { + iri = term.get("iri").asText(); + log.trace("set iri: " + iri); + } + } + } + } + } + return Optional.ofNullable(iri); + } + + public Optional queryForOlsObject(final String shortcode) { + final UriComponents uriComponents = + UriComponentsBuilder.fromUriString( + bioSamplesProperties.getOls() + "/api/terms?id={shortcode}&size=500") + .build(); + final URI uri = uriComponents.expand(shortcode).encode().toUri(); + + log.trace("OLS query for shortcode {} against {}", shortcode, uri); + + final RequestEntity requestEntity = + RequestEntity.get(uri).accept(MediaType.APPLICATION_JSON).build(); + final ResponseEntity responseEntity = + restTemplate.exchange(requestEntity, new ParameterizedTypeReference() {}); + + // non-200 status code or empty body + if (!responseEntity.getStatusCode().is2xxSuccessful() || responseEntity.getBody() == null) { + log.trace( + "Http status: {}, Body: {}", + responseEntity.getStatusCodeValue(), + responseEntity.getBody()); + return Optional.empty(); + } + + final ObjectNode node = responseEntity.getBody(); + + OlsResult olsResult = null; + if (node.has("_embedded") && node.get("_embedded").has("terms")) { + for (final JsonNode term : node.get("_embedded").get("terms")) { + if (term.has("iri") && term.has("is_defining_ontology")) { + // if we don't have an iri use this but if there is a defining ontoloy use this + // in + // preference + if (olsResult == null || term.get("is_defining_ontology").booleanValue()) { + // if this term is obsolete get the most upto-date one + if (term.get("is_obsolete").booleanValue()) { + final String[] replaceTermArray = term.get("term_replaced_by").asText().split("/"); + final String replaceTerm = replaceTermArray[replaceTermArray.length - 1]; + return queryForOlsObject(replaceTerm); + } + + olsResult = new OlsResult(term.get("label").asText(), term.get("iri").asText()); + } + } + } + } + return Optional.ofNullable(olsResult); + } +} diff --git a/utils/ols/src/main/java/uk/ac/ebi/biosamples/ols/OlsResult.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/ols/OlsResult.java similarity index 93% rename from utils/ols/src/main/java/uk/ac/ebi/biosamples/ols/OlsResult.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/ols/OlsResult.java index 3b9b64263..201cd02a6 100644 --- a/utils/ols/src/main/java/uk/ac/ebi/biosamples/ols/OlsResult.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/ols/OlsResult.java @@ -1,29 +1,29 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.ols; - -public class OlsResult { - private String label; - private String iri; - - public OlsResult(String label, String iri) { - this.label = label; - this.iri = iri; - } - - public String getLabel() { - return label; - } - - public String getIri() { - return iri; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.ols; + +public class OlsResult { + private String label; + private String iri; + + public OlsResult(String label, String iri) { + this.label = label; + this.iri = iri; + } + + public String getLabel() { + return label; + } + + public String getIri() { + return iri; + } +} diff --git a/commons/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetriever.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetriever.java similarity index 96% rename from commons/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetriever.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetriever.java index 754066d05..fd6a48f5a 100644 --- a/commons/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetriever.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetriever.java @@ -1,135 +1,135 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils.phenopacket; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLEncoder; -import java.util.HashMap; -import java.util.Map; -import org.springframework.stereotype.Service; - -/** - * OLSDAtaRetriever is api class for working with EBI OLS. It fetchs specific (see methods of class) - * metadata for phenopackets. - * - * @author Dilshat Salikhov - */ -@Service -public class OLSDataRetriever { - private JsonNode node; - private final Map ontologyPrefixMapping = new HashMap<>(); - - public OLSDataRetriever() { - ontologyPrefixMapping.put("orphanet", "ordo"); - } - /** - * Read json from OLS by iri provided in GA4GH sample - * - * @param iri - */ - void readOntologyJsonFromUrl(final String iri) { - final String linkToTerm; - try { - // TODO move to application properties - linkToTerm = "https://www.ebi.ac.uk/ols/api/terms?iri=" + URLEncoder.encode(iri, "UTF-8"); - } catch (final UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - readJson(linkToTerm); - } - - /** - * Read json from OLS by ontology id, for example 'efo' - * - * @param id - */ - void readResourceInfoFromUrl(String id) { - - // Necessary for ontologies like orphanet, where the prefix is not orphanet but ordo - if (ontologyPrefixMapping.containsKey(id.toLowerCase())) { - id = ontologyPrefixMapping.get(id.toLowerCase()); - } - - String linkToResourceInfo = null; - // TODO move to application properties - try { - linkToResourceInfo = - "https://www.ebi.ac.uk/ols/api/ontologies/" - + URLEncoder.encode(id.toLowerCase(), "UTF-8"); - } catch (final UnsupportedEncodingException e) { - e.printStackTrace(); - } - readJson(linkToResourceInfo); - } - - /** - * Fetchs json from given url and store it as Json tree in node attribute - * - * @param link link to OLS element - */ - private void readJson(final String link) { - final URL urlToTerm; - try { - urlToTerm = new URL(link); - } catch (final MalformedURLException e) { - throw new RuntimeException(e); - } - final ObjectMapper mapper = new ObjectMapper(); - try { - node = mapper.readTree(urlToTerm); - } catch (final IOException e) { - throw new RuntimeException(e); - } - } - - String getOntologyTermId() { - JsonNode terms = node.get("_embedded").get("terms"); - terms = terms.get(0); - return terms.get("obo_id").asText(); - } - - String getOntologyTermLabel() { - JsonNode terms = node.get("_embedded").get("terms"); - terms = terms.get(0); - return terms.get("label").asText(); - } - - String getResourceId() { - return node.get("ontologyId").asText(); - } - - String getResourceName() { - final JsonNode config = node.get("config"); - return config.get("title").isNull() - ? config.get("localizedTitles").get("en").asText() - : config.get("title").asText(); - } - - String getResourcePrefix() { - final JsonNode config = node.get("config"); - return config.get("preferredPrefix").asText(); - } - - String getResourceUrl() { - final JsonNode config = node.get("config"); - return config.get("fileLocation").asText(); - } - - String getResourceVersion() { - final JsonNode config = node.get("config"); - return config.get("version").asText(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.phenopacket; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLEncoder; +import java.util.HashMap; +import java.util.Map; +import org.springframework.stereotype.Service; + +/** + * OLSDAtaRetriever is api class for working with EBI OLS. It fetchs specific (see methods of class) + * metadata for phenopackets. + * + * @author Dilshat Salikhov + */ +@Service +public class OLSDataRetriever { + private JsonNode node; + private final Map ontologyPrefixMapping = new HashMap<>(); + + public OLSDataRetriever() { + ontologyPrefixMapping.put("orphanet", "ordo"); + } + /** + * Read json from OLS by iri provided in GA4GH sample + * + * @param iri + */ + void readOntologyJsonFromUrl(final String iri) { + final String linkToTerm; + try { + // TODO move to application properties + linkToTerm = "https://www.ebi.ac.uk/ols/api/terms?iri=" + URLEncoder.encode(iri, "UTF-8"); + } catch (final UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + readJson(linkToTerm); + } + + /** + * Read json from OLS by ontology id, for example 'efo' + * + * @param id + */ + void readResourceInfoFromUrl(String id) { + + // Necessary for ontologies like orphanet, where the prefix is not orphanet but ordo + if (ontologyPrefixMapping.containsKey(id.toLowerCase())) { + id = ontologyPrefixMapping.get(id.toLowerCase()); + } + + String linkToResourceInfo = null; + // TODO move to application properties + try { + linkToResourceInfo = + "https://www.ebi.ac.uk/ols/api/ontologies/" + + URLEncoder.encode(id.toLowerCase(), "UTF-8"); + } catch (final UnsupportedEncodingException e) { + e.printStackTrace(); + } + readJson(linkToResourceInfo); + } + + /** + * Fetchs json from given url and store it as Json tree in node attribute + * + * @param link link to OLS element + */ + private void readJson(final String link) { + final URL urlToTerm; + try { + urlToTerm = new URL(link); + } catch (final MalformedURLException e) { + throw new RuntimeException(e); + } + final ObjectMapper mapper = new ObjectMapper(); + try { + node = mapper.readTree(urlToTerm); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + + String getOntologyTermId() { + JsonNode terms = node.get("_embedded").get("terms"); + terms = terms.get(0); + return terms.get("obo_id").asText(); + } + + String getOntologyTermLabel() { + JsonNode terms = node.get("_embedded").get("terms"); + terms = terms.get(0); + return terms.get("label").asText(); + } + + String getResourceId() { + return node.get("ontologyId").asText(); + } + + String getResourceName() { + final JsonNode config = node.get("config"); + return config.get("title").isNull() + ? config.get("localizedTitles").get("en").asText() + : config.get("title").asText(); + } + + String getResourcePrefix() { + final JsonNode config = node.get("config"); + return config.get("preferredPrefix").asText(); + } + + String getResourceUrl() { + final JsonNode config = node.get("config"); + return config.get("fileLocation").asText(); + } + + String getResourceVersion() { + final JsonNode config = node.get("config"); + return config.get("version").asText(); + } +} diff --git a/commons/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketAttribute.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketAttribute.java similarity index 96% rename from commons/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketAttribute.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketAttribute.java index c8ed2d83b..fbf4fa92e 100644 --- a/commons/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketAttribute.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketAttribute.java @@ -1,64 +1,64 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils.phenopacket; - -public class PhenopacketAttribute { - private String type; - private String value; - private String ontologyId; - private String ontologyLabel; - private boolean negate; - - private PhenopacketAttribute() { - // hide constructor - } - - public static PhenopacketAttribute build( - final String type, - final String value, - final String ontologyId, - final String ontologyLabel, - final boolean negate) { - final PhenopacketAttribute phenopacketAttribute = new PhenopacketAttribute(); - phenopacketAttribute.type = type; - phenopacketAttribute.value = value; - phenopacketAttribute.ontologyId = ontologyId; - phenopacketAttribute.ontologyLabel = ontologyLabel; - phenopacketAttribute.negate = negate; - - return phenopacketAttribute; - } - - public static PhenopacketAttribute build( - final String type, final String value, final String ontologyId, final String ontologyLabel) { - return build(type, value, ontologyId, ontologyLabel, false); - } - - public String getType() { - return type; - } - - public String getValue() { - return value; - } - - String getOntologyId() { - return ontologyId; - } - - String getOntologyLabel() { - return ontologyLabel; - } - - boolean isNegate() { - return negate; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.phenopacket; + +public class PhenopacketAttribute { + private String type; + private String value; + private String ontologyId; + private String ontologyLabel; + private boolean negate; + + private PhenopacketAttribute() { + // hide constructor + } + + public static PhenopacketAttribute build( + final String type, + final String value, + final String ontologyId, + final String ontologyLabel, + final boolean negate) { + final PhenopacketAttribute phenopacketAttribute = new PhenopacketAttribute(); + phenopacketAttribute.type = type; + phenopacketAttribute.value = value; + phenopacketAttribute.ontologyId = ontologyId; + phenopacketAttribute.ontologyLabel = ontologyLabel; + phenopacketAttribute.negate = negate; + + return phenopacketAttribute; + } + + public static PhenopacketAttribute build( + final String type, final String value, final String ontologyId, final String ontologyLabel) { + return build(type, value, ontologyId, ontologyLabel, false); + } + + public String getType() { + return type; + } + + public String getValue() { + return value; + } + + String getOntologyId() { + return ontologyId; + } + + String getOntologyLabel() { + return ontologyLabel; + } + + boolean isNegate() { + return negate; + } +} diff --git a/commons/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConversionHelper.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConversionHelper.java similarity index 96% rename from commons/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConversionHelper.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConversionHelper.java index f93fb2eff..7fccf1454 100644 --- a/commons/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConversionHelper.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConversionHelper.java @@ -1,99 +1,99 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils.phenopacket; - -import java.util.Optional; -import org.phenopackets.schema.v1.core.OntologyClass; -import org.phenopackets.schema.v1.core.PhenotypicFeature; -import org.phenopackets.schema.v1.core.Resource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.Attribute; - -@Component -class PhenopacketConversionHelper { - private static final Logger LOG = LoggerFactory.getLogger(PhenopacketConversionHelper.class); - - Optional convertAttribute( - final String type, final String value, final String ontologyId, final String ontologyLabel) { - return Optional.of(PhenopacketAttribute.build(type, value, ontologyId, ontologyLabel, false)); - } - - Optional convertAttributeWithNegation( - final String type, final String value, final String ontologyId, final String ontologyLabel) { - return Optional.of(PhenopacketAttribute.build(type, value, ontologyId, ontologyLabel, true)); - } - - Optional convertAttribute(final String type, final Attribute attribute) { - Optional optionalPhenopacketAttribute = Optional.empty(); - if (!attribute.getIri().isEmpty()) { - try { - final OLSDataRetriever retriever = new OLSDataRetriever(); - final String firstIri = attribute.getIri().first(); - retriever.readOntologyJsonFromUrl(firstIri); - optionalPhenopacketAttribute = - Optional.of( - PhenopacketAttribute.build( - type, - attribute.getValue(), - retriever.getOntologyTermId(), - retriever.getOntologyTermLabel())); - } catch (final Exception e) { - LOG.warn( - "Failed to get IRI from OLS, possibly a wrong IRI format: {}", - attribute.getIri().first(), - e); - } - } else { - optionalPhenopacketAttribute = - Optional.of(PhenopacketAttribute.build(type, attribute.getValue(), null, null)); - } - - return optionalPhenopacketAttribute; - } - - OntologyClass getOntology(final PhenopacketAttribute attribute) { - return OntologyClass.newBuilder() - .setId(attribute.getOntologyId()) - .setLabel(attribute.getOntologyLabel()) - .build(); - } - - PhenotypicFeature getPhenotype(final PhenopacketAttribute attribute) { - return PhenotypicFeature.newBuilder() - .setType(getOntology(attribute)) - .setNegated(attribute.isNegate()) - .setDescription(attribute.getValue()) - .build(); - } - - Optional getResource(final PhenopacketAttribute attribute) { - Optional optionalResource = Optional.empty(); - if (attribute.getOntologyId() != null) { - final String ontology = attribute.getOntologyId().split(":")[0]; - final OLSDataRetriever retriever = new OLSDataRetriever(); - retriever.readResourceInfoFromUrl(ontology); - - optionalResource = - Optional.of( - Resource.newBuilder() - .setId(retriever.getResourceId()) - .setName(retriever.getResourceName()) - .setUrl(retriever.getResourceUrl()) - .setNamespacePrefix(retriever.getResourcePrefix()) - .setVersion(retriever.getResourceVersion()) - .build()); - } - - return optionalResource; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.phenopacket; + +import java.util.Optional; +import org.phenopackets.schema.v1.core.OntologyClass; +import org.phenopackets.schema.v1.core.PhenotypicFeature; +import org.phenopackets.schema.v1.core.Resource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import uk.ac.ebi.biosamples.core.model.Attribute; + +@Component +class PhenopacketConversionHelper { + private static final Logger LOG = LoggerFactory.getLogger(PhenopacketConversionHelper.class); + + Optional convertAttribute( + final String type, final String value, final String ontologyId, final String ontologyLabel) { + return Optional.of(PhenopacketAttribute.build(type, value, ontologyId, ontologyLabel, false)); + } + + Optional convertAttributeWithNegation( + final String type, final String value, final String ontologyId, final String ontologyLabel) { + return Optional.of(PhenopacketAttribute.build(type, value, ontologyId, ontologyLabel, true)); + } + + Optional convertAttribute(final String type, final Attribute attribute) { + Optional optionalPhenopacketAttribute = Optional.empty(); + if (!attribute.getIri().isEmpty()) { + try { + final OLSDataRetriever retriever = new OLSDataRetriever(); + final String firstIri = attribute.getIri().first(); + retriever.readOntologyJsonFromUrl(firstIri); + optionalPhenopacketAttribute = + Optional.of( + PhenopacketAttribute.build( + type, + attribute.getValue(), + retriever.getOntologyTermId(), + retriever.getOntologyTermLabel())); + } catch (final Exception e) { + LOG.warn( + "Failed to get IRI from OLS, possibly a wrong IRI format: {}", + attribute.getIri().first(), + e); + } + } else { + optionalPhenopacketAttribute = + Optional.of(PhenopacketAttribute.build(type, attribute.getValue(), null, null)); + } + + return optionalPhenopacketAttribute; + } + + OntologyClass getOntology(final PhenopacketAttribute attribute) { + return OntologyClass.newBuilder() + .setId(attribute.getOntologyId()) + .setLabel(attribute.getOntologyLabel()) + .build(); + } + + PhenotypicFeature getPhenotype(final PhenopacketAttribute attribute) { + return PhenotypicFeature.newBuilder() + .setType(getOntology(attribute)) + .setNegated(attribute.isNegate()) + .setDescription(attribute.getValue()) + .build(); + } + + Optional getResource(final PhenopacketAttribute attribute) { + Optional optionalResource = Optional.empty(); + if (attribute.getOntologyId() != null) { + final String ontology = attribute.getOntologyId().split(":")[0]; + final OLSDataRetriever retriever = new OLSDataRetriever(); + retriever.readResourceInfoFromUrl(ontology); + + optionalResource = + Optional.of( + Resource.newBuilder() + .setId(retriever.getResourceId()) + .setName(retriever.getResourceName()) + .setUrl(retriever.getResourceUrl()) + .setNamespacePrefix(retriever.getResourcePrefix()) + .setVersion(retriever.getResourceVersion()) + .build()); + } + + return optionalResource; + } +} diff --git a/commons/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverter.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverter.java similarity index 97% rename from commons/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverter.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverter.java index fd9ce1f78..4cfb827fb 100644 --- a/commons/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverter.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverter.java @@ -1,578 +1,578 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils.phenopacket; - -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.JsonFormat; -import java.util.*; -import org.phenopackets.schema.v1.Phenopacket; -import org.phenopackets.schema.v1.core.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; - -@Service -public class PhenopacketConverter { - private static final Logger LOG = LoggerFactory.getLogger(PhenopacketConverter.class); - private final PhenopacketConversionHelper phenopacketConversionHelper; - - public PhenopacketConverter(final PhenopacketConversionHelper phenopacketConversionHelper) { - this.phenopacketConversionHelper = phenopacketConversionHelper; - } - - public String convertToJsonPhenopacket(final Sample sample) { - if (sample.getTaxId() == null || sample.getTaxId() != 9606) { - LOG.warn( - "Trying to export invalid sample in phenopacket format: accession = {}, tax_id = {}", - sample.getAccession(), - sample.getTaxId()); - throw new GlobalExceptions.SampleConversionException( - "Non human sample can not be exported into phenopacket."); - } - - final Phenopacket phenopacket = convert(sample); - try { - return JsonFormat.printer().print(phenopacket); - } catch (final InvalidProtocolBufferException e) { - LOG.error("Failed to serialise phenopacket into JSON", e); - throw new GlobalExceptions.SampleConversionException( - "Failed to serialise phenopacket into JSON."); - } - } - - public Phenopacket convert(final Sample sample) { - final List diseases = new ArrayList<>(); - final Map attributes = new HashMap<>(); - normalizeAttributes(sample, attributes, diseases); - - return Phenopacket.newBuilder() - .setId(sample.getAccession()) - .setMetaData(populateMetadata(attributes, diseases)) - .setSubject(populateSubject(sample, attributes)) - .addBiosamples(populateBiosample(sample, attributes)) - .addAllDiseases(populateDiseases(diseases)) - .build(); - } - - private MetaData populateMetadata( - final Map attributes, - final List diseases) { - final Set resources = new HashSet<>(); - for (final PhenopacketAttribute a : attributes.values()) { - phenopacketConversionHelper.getResource(a).ifPresent(resources::add); - } - for (final PhenopacketAttribute a : diseases) { - phenopacketConversionHelper.getResource(a).ifPresent(resources::add); - } - - return MetaData.newBuilder() - .setCreated(Timestamp.newBuilder()) - .setCreatedBy("Biosamples phenopacket exporter") - .addAllResources(resources) - .build(); - } - - private Individual populateSubject( - final Sample sample, final Map attributes) { - final Individual.Builder builder = Individual.newBuilder(); - builder.setId(sample.getAccession() + "-individual"); - if (attributes.containsKey("sex")) { - final Sex sex; - if ("male".equalsIgnoreCase(attributes.get("sex").getValue())) { - sex = Sex.MALE; - } else if ("female".equalsIgnoreCase(attributes.get("sex").getValue())) { - sex = Sex.FEMALE; - } else { - sex = Sex.UNKNOWN_SEX; - } - builder.setSex(sex); - } - if (attributes.containsKey("organism")) { - builder.setTaxonomy(phenopacketConversionHelper.getOntology(attributes.get("organism"))); - } - - return builder.build(); - } - - private Biosample populateBiosample( - final Sample sample, final Map attributes) { - final Biosample.Builder builder = Biosample.newBuilder(); - builder.setId(sample.getAccession()); - builder.setIndividualId(getIndividualId(sample)); - if (attributes.containsKey("organism")) { - builder.setTaxonomy(phenopacketConversionHelper.getOntology(attributes.get("organism"))); - } - if (attributes.containsKey("description")) { - builder.setDescription(attributes.get("description").getValue()); - } - if (attributes.containsKey("age")) { - builder.setAgeOfIndividualAtCollection( - Age.newBuilder().setAge(attributes.get("age").getValue()).build()); - } - // phenotypic feature - if (attributes.containsKey("phenotype")) { - builder.addPhenotypicFeatures( - phenopacketConversionHelper.getPhenotype(attributes.get("phenotype"))); - } - if (attributes.containsKey("developmental stage")) { - builder.addPhenotypicFeatures( - phenopacketConversionHelper.getPhenotype(attributes.get("developmental stage"))); - } - - if (attributes.containsKey("tissue")) { - builder.setSampledTissue(phenopacketConversionHelper.getOntology(attributes.get("tissue"))); - } - if (attributes.containsKey("diagnosis")) { - builder.setSampledTissue( - phenopacketConversionHelper.getOntology(attributes.get("diagnosis"))); - } - if (attributes.containsKey("tumor grade")) { - builder.setSampledTissue( - phenopacketConversionHelper.getOntology(attributes.get("tumor grade"))); - } - if (attributes.containsKey("tumor progression")) { - builder.setSampledTissue( - phenopacketConversionHelper.getOntology(attributes.get("tumor progression"))); - } - if (attributes.containsKey("biomarker")) { - builder.setSampledTissue( - phenopacketConversionHelper.getOntology(attributes.get("biomarker"))); - } - if (attributes.containsKey("procedure")) { - builder.setProcedure( - Procedure.newBuilder() - .setCode(phenopacketConversionHelper.getOntology(attributes.get("procedure"))) - .build()); - } - if (attributes.containsKey("variants")) { - builder.setSampledTissue(phenopacketConversionHelper.getOntology(attributes.get("variants"))); - } - return builder.build(); - } - - private List populateDiseases(final List diseases) { - final List diseaseList = new ArrayList<>(); - for (final PhenopacketAttribute disease : diseases) { - if (disease.getOntologyId() != null) { - final Disease.Builder builder = Disease.newBuilder(); - builder.setTerm(phenopacketConversionHelper.getOntology(disease)); - diseaseList.add(builder.build()); - } - } - return diseaseList; - } - - private void normalizeAttributes( - final Sample sample, - final Map attributeMap, - final List diseases) { - for (final Attribute attribute : sample.getAttributes()) { - getAge(attribute).ifPresent(a -> addToMap(attributeMap, a)); - getDescription(attribute).ifPresent(a -> addToMap(attributeMap, a)); - getSex(attribute).ifPresent(a -> addToMap(attributeMap, a)); - - if (!attribute.getIri().isEmpty()) { - getOrganism(attribute).ifPresent(a -> addToMap(attributeMap, a)); - getDisease(attribute) - .ifPresent( - a -> { - addToMap(attributeMap, a); - diseases.add(a); - }); - - getPhenotypicFeatures(attribute).ifPresent(a -> addToMap(attributeMap, a)); - getSampleTissue(attribute).ifPresent(a -> addToMap(attributeMap, a)); - getDiagnosis(attribute).ifPresent(a -> addToMap(attributeMap, a)); - getTumorGrade(attribute).ifPresent(a -> addToMap(attributeMap, a)); - getTumorProgression(attribute).ifPresent(a -> addToMap(attributeMap, a)); - getBiomarker(attribute).ifPresent(a -> addToMap(attributeMap, a)); - getProcedure(attribute).ifPresent(a -> addToMap(attributeMap, a)); - getVariants(attribute).ifPresent(a -> addToMap(attributeMap, a)); - } - - extractDisease(attribute).ifPresent(diseases::add); - extractMentalDisease(attribute).ifPresent(diseases::add); - } - } - - private String getIndividualId(final Sample sample) { - return sample.getAccession() + "-individual"; - } - - private Optional getSex(final Attribute attribute) { - final Optional normalisedAttribute; - if ("sex".equalsIgnoreCase(attribute.getType()) - || "gender".equalsIgnoreCase(attribute.getType()) - || "vioscreen gender".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("sex", attribute); - } else { - normalisedAttribute = Optional.empty(); - } - return normalisedAttribute; - } - - private Optional getOrganism(final Attribute attribute) { - final Optional normalisedAttribute; - if ("organism".equalsIgnoreCase(attribute.getType()) - || "species".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("organism", attribute); - } else { - normalisedAttribute = Optional.empty(); - } - return normalisedAttribute; - } - - private Optional getDisease(final Attribute attribute) { - final Optional normalisedAttribute; - if ("disease".equalsIgnoreCase(attribute.getType()) - || "disease state".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("disease", attribute); - } else { - normalisedAttribute = Optional.empty(); - } - return normalisedAttribute; - } - - private Optional getAge(final Attribute attribute) { - final Optional normalisedAttribute; - if ("age".equalsIgnoreCase(attribute.getType()) - || "age_years".equalsIgnoreCase(attribute.getType()) - || "age(years)".equalsIgnoreCase(attribute.getType()) - || "age at collection months".equalsIgnoreCase(attribute.getType()) - || "age at collection".equalsIgnoreCase(attribute.getType()) - || "age at collection mo".equalsIgnoreCase(attribute.getType()) - || "age at sample months".equalsIgnoreCase(attribute.getType()) - || "age at collection (months)".equalsIgnoreCase(attribute.getType()) - || "age at sampling".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("age", attribute); - } else { - normalisedAttribute = Optional.empty(); - } - return normalisedAttribute; - } - - private Optional getDescription(final Attribute attribute) { - final Optional normalisedAttribute; - if ("description".equalsIgnoreCase(attribute.getType()) - || "description title".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("description", attribute); - } else { - normalisedAttribute = Optional.empty(); - } - return normalisedAttribute; - } - - private Optional getPhenotypicFeatures(final Attribute attribute) { - final Optional normalisedAttribute; - if ("phenotype".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("phenotype", attribute); - } else if ("development stage".equalsIgnoreCase(attribute.getType()) - || "developmental stage".equalsIgnoreCase(attribute.getType()) - || "dev stage".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute("developmental stage", attribute); - } else { - normalisedAttribute = Optional.empty(); - } - return normalisedAttribute; - } - - private Optional getSampleTissue(final Attribute attribute) { - final Optional normalisedAttribute; - if ("tissue".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("tissue", attribute); - } else { - normalisedAttribute = Optional.empty(); - } - return normalisedAttribute; - } - - private Optional getDiagnosis(final Attribute attribute) { - final Optional normalisedAttribute; - if ("histological diagnosis".equalsIgnoreCase(attribute.getType()) - || "diagnosis".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("diagnosis", attribute); - } else { - normalisedAttribute = Optional.empty(); - } - return normalisedAttribute; - } - - private Optional getTumorGrade(final Attribute attribute) { - final Optional normalisedAttribute; - if ("tumor grade".equalsIgnoreCase(attribute.getType()) - || "tumor stage".equalsIgnoreCase(attribute.getType()) - || "tumor grading".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("tumor grade", attribute); - } else { - normalisedAttribute = Optional.empty(); - } - return normalisedAttribute; - } - - private Optional getTumorProgression(final Attribute attribute) { - final Optional normalisedAttribute; - if ("tumor progression".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute("tumor progression", attribute); - } else { - normalisedAttribute = Optional.empty(); - } - return normalisedAttribute; - } - - private Optional getBiomarker(final Attribute attribute) { - final Optional normalisedAttribute; - if ("biomarker".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("biomarker", attribute); - } else { - normalisedAttribute = Optional.empty(); - } - return normalisedAttribute; - } - - private Optional getProcedure(final Attribute attribute) { - final Optional normalisedAttribute; - if ("procedure".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("procedure", attribute); - } else { - normalisedAttribute = Optional.empty(); - } - return normalisedAttribute; - } - - private Optional getVariants(final Attribute attribute) { - final Optional normalisedAttribute; - if ("variants".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("variants", attribute); - } else { - normalisedAttribute = Optional.empty(); - } - return normalisedAttribute; - } - - private Optional getOtherFields(final Attribute attribute) { - Optional normalisedAttribute = Optional.empty(); - if ("tissue".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("tissue", attribute); - } - - return normalisedAttribute; - } - - private Optional extractDisease(final Attribute attribute) { - Optional normalisedAttribute = Optional.empty(); - if ("diabetes".equalsIgnoreCase(attribute.getType()) - || "diabetes type".equalsIgnoreCase(attribute.getType())) { - if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", "diabetes mellitus", "MONDO:0005015", "diabetes mellitus"); - } else { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "diabetes mellitus", "MONDO:0005015", "diabetes mellitus"); - } - } else if ("lung disease".equalsIgnoreCase(attribute.getType())) { - if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", "lung disease", "MONDO:0005275", "lung disease"); - } else { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "lung disease", "MONDO:0005275", "lung disease"); - } - } else if ("COVID-19".equalsIgnoreCase(attribute.getType())) { - if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue()) - || "negative".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", "COVID-19", "MONDO:0100096", "COVID-19"); - } else { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "COVID-19", "MONDO:0100096", "COVID-19"); - } - } else if ("liver disease".equalsIgnoreCase(attribute.getType())) { - if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", "liver disease", "MONDO:0005154", "liver disease"); - } else { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "liver disease", "MONDO:0005154", "liver disease"); - } - } else if ("kidney disease".equalsIgnoreCase(attribute.getType())) { - if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", "kidney disease", "MONDO:0005240", "kidney disease"); - } else { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "kidney disease", "MONDO:0005240", "kidney disease"); - } - } else if ("cardiovascular disease".equalsIgnoreCase(attribute.getType())) { - if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", "cardiovascular disease", "MONDO:0004995", "cardiovascular disease"); - } else { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "cardiovascular disease", "MONDO:0004995", "cardiovascular disease"); - } - } else if ("cancer".equalsIgnoreCase(attribute.getType())) { - if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", "cancer", "MONDO:0004992", "cancer"); - } else { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "cancer", "MONDO:0004992", "cancer"); - } - } else if ("ibd".equalsIgnoreCase(attribute.getType()) - || ("inflammatory bowel disease".equalsIgnoreCase(attribute.getType()))) { - if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", - "inflammatory bowel disease", - "MONDO:0005265", - "inflammatory bowel disease"); - } else { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", - "inflammatory bowel disease", - "MONDO:0005265", - "inflammatory bowel disease"); - } - } else if ("Biosamples inferred disease".equalsIgnoreCase(attribute.getType())) { - normalisedAttribute = phenopacketConversionHelper.convertAttribute("disease", attribute); - } - - return normalisedAttribute; - } - - private Optional extractMentalDisease(final Attribute attribute) { - Optional normalisedAttribute = Optional.empty(); - if ("mental illness".equalsIgnoreCase(attribute.getType())) { - if ("yes".equalsIgnoreCase(attribute.getValue()) - || "true".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "mental illness", "MONDO_0005084", "mental illness"); - } - } else if (attribute.getType().toLowerCase().startsWith("mental illness type")) { - if (attribute.getType().toLowerCase().contains("substance abuse")) { - if ("yes".equalsIgnoreCase(attribute.getValue()) - || "true".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "substance abuse", "MONDO:0002491", "substance abuse"); - } else if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", "substance abuse", "MONDO:0002491", "substance abuse"); - } - } else if (attribute.getType().toLowerCase().contains("anorexia nervosa")) { - if ("yes".equalsIgnoreCase(attribute.getValue()) - || "true".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "anorexia nervosa", "MONDO:0005351", "anorexia nervosa"); - } else if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", "anorexia nervosa", "MONDO:0005351", "anorexia nervosa"); - } - } else if (attribute.getType().toLowerCase().contains("schizophrenia")) { - if ("yes".equalsIgnoreCase(attribute.getValue()) - || "true".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "schizophrenia", "MONDO:0005090", "schizophrenia"); - } else if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", "schizophrenia", "MONDO:0005090", "schizophrenia"); - } - } else if (attribute.getType().toLowerCase().contains("depression")) { - if ("yes".equalsIgnoreCase(attribute.getValue()) - || "true".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "depression", "MONDO:0002050", "depression"); - } else if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", "depression", "MONDO:0002050", "depression"); - } - } else if (attribute.getType().toLowerCase().contains("bulimia nervosa")) { - if ("yes".equalsIgnoreCase(attribute.getValue()) - || "true".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "bulimia nervosa", "MONDO:0005452", "bulimia nervosa"); - } else if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", "bulimia nervosa", "MONDO:0005452", "bulimia nervosa"); - } - } else if (attribute.getType().toLowerCase().contains("bipolar disorder")) { - if ("yes".equalsIgnoreCase(attribute.getValue()) - || "true".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttribute( - "disease", "bipolar disorder", "MONDO:0004985", "bipolar disorder"); - } else if ("no".equalsIgnoreCase(attribute.getValue()) - || "false".equalsIgnoreCase(attribute.getValue())) { - normalisedAttribute = - phenopacketConversionHelper.convertAttributeWithNegation( - "disease", "bipolar disorder", "MONDO:0004985", "bipolar disorder"); - } - } - } else if ("study disease".equalsIgnoreCase(attribute.getType())) { - // todo check attribute 'subject is affected' - normalisedAttribute = phenopacketConversionHelper.convertAttribute("disease", attribute); - } else if ("study name".equalsIgnoreCase(attribute.getType())) { - // todo check attribute 'subject is affected' - normalisedAttribute = phenopacketConversionHelper.convertAttribute("disease", attribute); - } - - return normalisedAttribute; - } - - private void addToMap(final Map m, final PhenopacketAttribute a) { - m.put(a.getType(), a); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.phenopacket; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Timestamp; +import com.google.protobuf.util.JsonFormat; +import java.util.*; +import org.phenopackets.schema.v1.Phenopacket; +import org.phenopackets.schema.v1.core.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; + +@Service +public class PhenopacketConverter { + private static final Logger LOG = LoggerFactory.getLogger(PhenopacketConverter.class); + private final PhenopacketConversionHelper phenopacketConversionHelper; + + public PhenopacketConverter(final PhenopacketConversionHelper phenopacketConversionHelper) { + this.phenopacketConversionHelper = phenopacketConversionHelper; + } + + public String convertToJsonPhenopacket(final Sample sample) { + if (sample.getTaxId() == null || sample.getTaxId() != 9606) { + LOG.warn( + "Trying to export invalid sample in phenopacket format: accession = {}, tax_id = {}", + sample.getAccession(), + sample.getTaxId()); + throw new GlobalExceptions.SampleConversionException( + "Non human sample can not be exported into phenopacket."); + } + + final Phenopacket phenopacket = convert(sample); + try { + return JsonFormat.printer().print(phenopacket); + } catch (final InvalidProtocolBufferException e) { + LOG.error("Failed to serialise phenopacket into JSON", e); + throw new GlobalExceptions.SampleConversionException( + "Failed to serialise phenopacket into JSON."); + } + } + + public Phenopacket convert(final Sample sample) { + final List diseases = new ArrayList<>(); + final Map attributes = new HashMap<>(); + normalizeAttributes(sample, attributes, diseases); + + return Phenopacket.newBuilder() + .setId(sample.getAccession()) + .setMetaData(populateMetadata(attributes, diseases)) + .setSubject(populateSubject(sample, attributes)) + .addBiosamples(populateBiosample(sample, attributes)) + .addAllDiseases(populateDiseases(diseases)) + .build(); + } + + private MetaData populateMetadata( + final Map attributes, + final List diseases) { + final Set resources = new HashSet<>(); + for (final PhenopacketAttribute a : attributes.values()) { + phenopacketConversionHelper.getResource(a).ifPresent(resources::add); + } + for (final PhenopacketAttribute a : diseases) { + phenopacketConversionHelper.getResource(a).ifPresent(resources::add); + } + + return MetaData.newBuilder() + .setCreated(Timestamp.newBuilder()) + .setCreatedBy("Biosamples phenopacket exporter") + .addAllResources(resources) + .build(); + } + + private Individual populateSubject( + final Sample sample, final Map attributes) { + final Individual.Builder builder = Individual.newBuilder(); + builder.setId(sample.getAccession() + "-individual"); + if (attributes.containsKey("sex")) { + final Sex sex; + if ("male".equalsIgnoreCase(attributes.get("sex").getValue())) { + sex = Sex.MALE; + } else if ("female".equalsIgnoreCase(attributes.get("sex").getValue())) { + sex = Sex.FEMALE; + } else { + sex = Sex.UNKNOWN_SEX; + } + builder.setSex(sex); + } + if (attributes.containsKey("organism")) { + builder.setTaxonomy(phenopacketConversionHelper.getOntology(attributes.get("organism"))); + } + + return builder.build(); + } + + private Biosample populateBiosample( + final Sample sample, final Map attributes) { + final Biosample.Builder builder = Biosample.newBuilder(); + builder.setId(sample.getAccession()); + builder.setIndividualId(getIndividualId(sample)); + if (attributes.containsKey("organism")) { + builder.setTaxonomy(phenopacketConversionHelper.getOntology(attributes.get("organism"))); + } + if (attributes.containsKey("description")) { + builder.setDescription(attributes.get("description").getValue()); + } + if (attributes.containsKey("age")) { + builder.setAgeOfIndividualAtCollection( + Age.newBuilder().setAge(attributes.get("age").getValue()).build()); + } + // phenotypic feature + if (attributes.containsKey("phenotype")) { + builder.addPhenotypicFeatures( + phenopacketConversionHelper.getPhenotype(attributes.get("phenotype"))); + } + if (attributes.containsKey("developmental stage")) { + builder.addPhenotypicFeatures( + phenopacketConversionHelper.getPhenotype(attributes.get("developmental stage"))); + } + + if (attributes.containsKey("tissue")) { + builder.setSampledTissue(phenopacketConversionHelper.getOntology(attributes.get("tissue"))); + } + if (attributes.containsKey("diagnosis")) { + builder.setSampledTissue( + phenopacketConversionHelper.getOntology(attributes.get("diagnosis"))); + } + if (attributes.containsKey("tumor grade")) { + builder.setSampledTissue( + phenopacketConversionHelper.getOntology(attributes.get("tumor grade"))); + } + if (attributes.containsKey("tumor progression")) { + builder.setSampledTissue( + phenopacketConversionHelper.getOntology(attributes.get("tumor progression"))); + } + if (attributes.containsKey("biomarker")) { + builder.setSampledTissue( + phenopacketConversionHelper.getOntology(attributes.get("biomarker"))); + } + if (attributes.containsKey("procedure")) { + builder.setProcedure( + Procedure.newBuilder() + .setCode(phenopacketConversionHelper.getOntology(attributes.get("procedure"))) + .build()); + } + if (attributes.containsKey("variants")) { + builder.setSampledTissue(phenopacketConversionHelper.getOntology(attributes.get("variants"))); + } + return builder.build(); + } + + private List populateDiseases(final List diseases) { + final List diseaseList = new ArrayList<>(); + for (final PhenopacketAttribute disease : diseases) { + if (disease.getOntologyId() != null) { + final Disease.Builder builder = Disease.newBuilder(); + builder.setTerm(phenopacketConversionHelper.getOntology(disease)); + diseaseList.add(builder.build()); + } + } + return diseaseList; + } + + private void normalizeAttributes( + final Sample sample, + final Map attributeMap, + final List diseases) { + for (final Attribute attribute : sample.getAttributes()) { + getAge(attribute).ifPresent(a -> addToMap(attributeMap, a)); + getDescription(attribute).ifPresent(a -> addToMap(attributeMap, a)); + getSex(attribute).ifPresent(a -> addToMap(attributeMap, a)); + + if (!attribute.getIri().isEmpty()) { + getOrganism(attribute).ifPresent(a -> addToMap(attributeMap, a)); + getDisease(attribute) + .ifPresent( + a -> { + addToMap(attributeMap, a); + diseases.add(a); + }); + + getPhenotypicFeatures(attribute).ifPresent(a -> addToMap(attributeMap, a)); + getSampleTissue(attribute).ifPresent(a -> addToMap(attributeMap, a)); + getDiagnosis(attribute).ifPresent(a -> addToMap(attributeMap, a)); + getTumorGrade(attribute).ifPresent(a -> addToMap(attributeMap, a)); + getTumorProgression(attribute).ifPresent(a -> addToMap(attributeMap, a)); + getBiomarker(attribute).ifPresent(a -> addToMap(attributeMap, a)); + getProcedure(attribute).ifPresent(a -> addToMap(attributeMap, a)); + getVariants(attribute).ifPresent(a -> addToMap(attributeMap, a)); + } + + extractDisease(attribute).ifPresent(diseases::add); + extractMentalDisease(attribute).ifPresent(diseases::add); + } + } + + private String getIndividualId(final Sample sample) { + return sample.getAccession() + "-individual"; + } + + private Optional getSex(final Attribute attribute) { + final Optional normalisedAttribute; + if ("sex".equalsIgnoreCase(attribute.getType()) + || "gender".equalsIgnoreCase(attribute.getType()) + || "vioscreen gender".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("sex", attribute); + } else { + normalisedAttribute = Optional.empty(); + } + return normalisedAttribute; + } + + private Optional getOrganism(final Attribute attribute) { + final Optional normalisedAttribute; + if ("organism".equalsIgnoreCase(attribute.getType()) + || "species".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("organism", attribute); + } else { + normalisedAttribute = Optional.empty(); + } + return normalisedAttribute; + } + + private Optional getDisease(final Attribute attribute) { + final Optional normalisedAttribute; + if ("disease".equalsIgnoreCase(attribute.getType()) + || "disease state".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("disease", attribute); + } else { + normalisedAttribute = Optional.empty(); + } + return normalisedAttribute; + } + + private Optional getAge(final Attribute attribute) { + final Optional normalisedAttribute; + if ("age".equalsIgnoreCase(attribute.getType()) + || "age_years".equalsIgnoreCase(attribute.getType()) + || "age(years)".equalsIgnoreCase(attribute.getType()) + || "age at collection months".equalsIgnoreCase(attribute.getType()) + || "age at collection".equalsIgnoreCase(attribute.getType()) + || "age at collection mo".equalsIgnoreCase(attribute.getType()) + || "age at sample months".equalsIgnoreCase(attribute.getType()) + || "age at collection (months)".equalsIgnoreCase(attribute.getType()) + || "age at sampling".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("age", attribute); + } else { + normalisedAttribute = Optional.empty(); + } + return normalisedAttribute; + } + + private Optional getDescription(final Attribute attribute) { + final Optional normalisedAttribute; + if ("description".equalsIgnoreCase(attribute.getType()) + || "description title".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("description", attribute); + } else { + normalisedAttribute = Optional.empty(); + } + return normalisedAttribute; + } + + private Optional getPhenotypicFeatures(final Attribute attribute) { + final Optional normalisedAttribute; + if ("phenotype".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("phenotype", attribute); + } else if ("development stage".equalsIgnoreCase(attribute.getType()) + || "developmental stage".equalsIgnoreCase(attribute.getType()) + || "dev stage".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute("developmental stage", attribute); + } else { + normalisedAttribute = Optional.empty(); + } + return normalisedAttribute; + } + + private Optional getSampleTissue(final Attribute attribute) { + final Optional normalisedAttribute; + if ("tissue".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("tissue", attribute); + } else { + normalisedAttribute = Optional.empty(); + } + return normalisedAttribute; + } + + private Optional getDiagnosis(final Attribute attribute) { + final Optional normalisedAttribute; + if ("histological diagnosis".equalsIgnoreCase(attribute.getType()) + || "diagnosis".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("diagnosis", attribute); + } else { + normalisedAttribute = Optional.empty(); + } + return normalisedAttribute; + } + + private Optional getTumorGrade(final Attribute attribute) { + final Optional normalisedAttribute; + if ("tumor grade".equalsIgnoreCase(attribute.getType()) + || "tumor stage".equalsIgnoreCase(attribute.getType()) + || "tumor grading".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("tumor grade", attribute); + } else { + normalisedAttribute = Optional.empty(); + } + return normalisedAttribute; + } + + private Optional getTumorProgression(final Attribute attribute) { + final Optional normalisedAttribute; + if ("tumor progression".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute("tumor progression", attribute); + } else { + normalisedAttribute = Optional.empty(); + } + return normalisedAttribute; + } + + private Optional getBiomarker(final Attribute attribute) { + final Optional normalisedAttribute; + if ("biomarker".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("biomarker", attribute); + } else { + normalisedAttribute = Optional.empty(); + } + return normalisedAttribute; + } + + private Optional getProcedure(final Attribute attribute) { + final Optional normalisedAttribute; + if ("procedure".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("procedure", attribute); + } else { + normalisedAttribute = Optional.empty(); + } + return normalisedAttribute; + } + + private Optional getVariants(final Attribute attribute) { + final Optional normalisedAttribute; + if ("variants".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("variants", attribute); + } else { + normalisedAttribute = Optional.empty(); + } + return normalisedAttribute; + } + + private Optional getOtherFields(final Attribute attribute) { + Optional normalisedAttribute = Optional.empty(); + if ("tissue".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("tissue", attribute); + } + + return normalisedAttribute; + } + + private Optional extractDisease(final Attribute attribute) { + Optional normalisedAttribute = Optional.empty(); + if ("diabetes".equalsIgnoreCase(attribute.getType()) + || "diabetes type".equalsIgnoreCase(attribute.getType())) { + if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", "diabetes mellitus", "MONDO:0005015", "diabetes mellitus"); + } else { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "diabetes mellitus", "MONDO:0005015", "diabetes mellitus"); + } + } else if ("lung disease".equalsIgnoreCase(attribute.getType())) { + if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", "lung disease", "MONDO:0005275", "lung disease"); + } else { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "lung disease", "MONDO:0005275", "lung disease"); + } + } else if ("COVID-19".equalsIgnoreCase(attribute.getType())) { + if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue()) + || "negative".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", "COVID-19", "MONDO:0100096", "COVID-19"); + } else { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "COVID-19", "MONDO:0100096", "COVID-19"); + } + } else if ("liver disease".equalsIgnoreCase(attribute.getType())) { + if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", "liver disease", "MONDO:0005154", "liver disease"); + } else { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "liver disease", "MONDO:0005154", "liver disease"); + } + } else if ("kidney disease".equalsIgnoreCase(attribute.getType())) { + if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", "kidney disease", "MONDO:0005240", "kidney disease"); + } else { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "kidney disease", "MONDO:0005240", "kidney disease"); + } + } else if ("cardiovascular disease".equalsIgnoreCase(attribute.getType())) { + if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", "cardiovascular disease", "MONDO:0004995", "cardiovascular disease"); + } else { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "cardiovascular disease", "MONDO:0004995", "cardiovascular disease"); + } + } else if ("cancer".equalsIgnoreCase(attribute.getType())) { + if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", "cancer", "MONDO:0004992", "cancer"); + } else { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "cancer", "MONDO:0004992", "cancer"); + } + } else if ("ibd".equalsIgnoreCase(attribute.getType()) + || ("inflammatory bowel disease".equalsIgnoreCase(attribute.getType()))) { + if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", + "inflammatory bowel disease", + "MONDO:0005265", + "inflammatory bowel disease"); + } else { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", + "inflammatory bowel disease", + "MONDO:0005265", + "inflammatory bowel disease"); + } + } else if ("Biosamples inferred disease".equalsIgnoreCase(attribute.getType())) { + normalisedAttribute = phenopacketConversionHelper.convertAttribute("disease", attribute); + } + + return normalisedAttribute; + } + + private Optional extractMentalDisease(final Attribute attribute) { + Optional normalisedAttribute = Optional.empty(); + if ("mental illness".equalsIgnoreCase(attribute.getType())) { + if ("yes".equalsIgnoreCase(attribute.getValue()) + || "true".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "mental illness", "MONDO_0005084", "mental illness"); + } + } else if (attribute.getType().toLowerCase().startsWith("mental illness type")) { + if (attribute.getType().toLowerCase().contains("substance abuse")) { + if ("yes".equalsIgnoreCase(attribute.getValue()) + || "true".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "substance abuse", "MONDO:0002491", "substance abuse"); + } else if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", "substance abuse", "MONDO:0002491", "substance abuse"); + } + } else if (attribute.getType().toLowerCase().contains("anorexia nervosa")) { + if ("yes".equalsIgnoreCase(attribute.getValue()) + || "true".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "anorexia nervosa", "MONDO:0005351", "anorexia nervosa"); + } else if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", "anorexia nervosa", "MONDO:0005351", "anorexia nervosa"); + } + } else if (attribute.getType().toLowerCase().contains("schizophrenia")) { + if ("yes".equalsIgnoreCase(attribute.getValue()) + || "true".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "schizophrenia", "MONDO:0005090", "schizophrenia"); + } else if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", "schizophrenia", "MONDO:0005090", "schizophrenia"); + } + } else if (attribute.getType().toLowerCase().contains("depression")) { + if ("yes".equalsIgnoreCase(attribute.getValue()) + || "true".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "depression", "MONDO:0002050", "depression"); + } else if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", "depression", "MONDO:0002050", "depression"); + } + } else if (attribute.getType().toLowerCase().contains("bulimia nervosa")) { + if ("yes".equalsIgnoreCase(attribute.getValue()) + || "true".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "bulimia nervosa", "MONDO:0005452", "bulimia nervosa"); + } else if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", "bulimia nervosa", "MONDO:0005452", "bulimia nervosa"); + } + } else if (attribute.getType().toLowerCase().contains("bipolar disorder")) { + if ("yes".equalsIgnoreCase(attribute.getValue()) + || "true".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttribute( + "disease", "bipolar disorder", "MONDO:0004985", "bipolar disorder"); + } else if ("no".equalsIgnoreCase(attribute.getValue()) + || "false".equalsIgnoreCase(attribute.getValue())) { + normalisedAttribute = + phenopacketConversionHelper.convertAttributeWithNegation( + "disease", "bipolar disorder", "MONDO:0004985", "bipolar disorder"); + } + } + } else if ("study disease".equalsIgnoreCase(attribute.getType())) { + // todo check attribute 'subject is affected' + normalisedAttribute = phenopacketConversionHelper.convertAttribute("disease", attribute); + } else if ("study name".equalsIgnoreCase(attribute.getType())) { + // todo check attribute 'subject is affected' + normalisedAttribute = phenopacketConversionHelper.convertAttribute("disease", attribute); + } + + return normalisedAttribute; + } + + private void addToMap(final Map m, final PhenopacketAttribute a) { + m.put(a.getType(), a); + } +} diff --git a/utils/thread/src/main/java/uk/ac/ebi/biosamples/utils/AdaptiveThreadPoolExecutor.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/thread/AdaptiveThreadPoolExecutor.java similarity index 96% rename from utils/thread/src/main/java/uk/ac/ebi/biosamples/utils/AdaptiveThreadPoolExecutor.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/thread/AdaptiveThreadPoolExecutor.java index af7c3f828..d12505339 100644 --- a/utils/thread/src/main/java/uk/ac/ebi/biosamples/utils/AdaptiveThreadPoolExecutor.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/thread/AdaptiveThreadPoolExecutor.java @@ -1,252 +1,252 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class AdaptiveThreadPoolExecutor extends ThreadPoolExecutor implements AutoCloseable { - private final AtomicInteger completedJobs = new AtomicInteger(0); - - private AdaptiveThreadPoolExecutor( - final int corePoolSize, - final int maximumPoolSize, - final long keepAliveTime, - final TimeUnit unit, - final BlockingQueue workQueue, - final RejectedExecutionHandler rejectedExecutionHandler) { - - super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, rejectedExecutionHandler); - } - - @Override - protected void afterExecute(final Runnable r, final Throwable t) { - if (t != null) { - return; - } - - completedJobs.incrementAndGet(); - } - - /** - * This is required to implement the AutoClosable interface. It will stop accepting new jobs and - * wait up to 24h before termination; - */ - @Override - public void close() { - shutdown(); - try { - awaitTermination(1, TimeUnit.MINUTES); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - /** - * By default creates a pool with a queue size of 1000 that will test to increase/decrease threads - * every 60 seconds and does not guarantee to distribute jobs fairly among threads - * - * @return - */ - public static AdaptiveThreadPoolExecutor create() { - return create(1000, 60000, false); - } - - public static AdaptiveThreadPoolExecutor create( - final int maxQueueSize, final int pollInterval, final boolean fairness) { - return create( - maxQueueSize, - pollInterval, - fairness, - Runtime.getRuntime().availableProcessors(), - Runtime.getRuntime().availableProcessors() * 8); - } - - public static AdaptiveThreadPoolExecutor create( - final int maxQueueSize, - final int pollInterval, - final boolean fairness, - final int initialPoolSize, - final int maxThreads) { - // default to the number of processors - final int corePoolSize = initialPoolSize; - final int maximumPoolSize = corePoolSize; - // keep alive is not relevant, since core == maximum - final long keepAliveTime = 1; - final TimeUnit unit = TimeUnit.DAYS; - // a queue constructed with fairness set to true grants threads access - // in FIFO order. - // Fairness generally decreases throughput but reduces variability and - // avoids starvation. - final BlockingQueue workQueue = new ArrayBlockingQueue<>(maxQueueSize, fairness); - // A handler for rejected tasks that runs the rejected task directly in - // the calling thread of the execute method, - // unless the executor has been shut down, in which case the task is - // discarded. - final RejectedExecutionHandler rejectedExecutionHandler = - new ThreadPoolExecutor.CallerRunsPolicy(); - final AdaptiveThreadPoolExecutor threadPool = - new AdaptiveThreadPoolExecutor( - corePoolSize, - maximumPoolSize, - keepAliveTime, - unit, - workQueue, - rejectedExecutionHandler); - final Thread monitorThread = new Thread(new PoolMonitor(threadPool, pollInterval, maxThreads)); - - monitorThread.setDaemon(true); - monitorThread.start(); - - return threadPool; - } - - /** - * This is a separate thread that monitors a thread pool and increases or decreases the number of - * threads within the pool in order to try to maximize the throughput. - * - * @author faulcon - */ - private static class PoolMonitor implements Runnable { - private final Logger log = LoggerFactory.getLogger(getClass()); - private final AdaptiveThreadPoolExecutor pool; - private final int pollInterval; - private final Map threadsScores = new HashMap<>(); - private final Map threadsTime = new HashMap<>(); - private final double margin = 1.0; - private final int maxThreads; - - PoolMonitor( - final AdaptiveThreadPoolExecutor pool, final int pollInterval, final int maxThreads) { - this.pool = pool; - this.pollInterval = pollInterval; - this.maxThreads = maxThreads; - } - - @Override - public void run() { - long lastStep = System.nanoTime(); - - while (!pool.isTerminated()) { - // wait for it to do stuff - try { - Thread.sleep(pollInterval); - } catch (final InterruptedException e) { - if (Thread.interrupted()) { // Clears interrupted status! - throw new RuntimeException(e); - } - } - - // test the number of jobs done - // get number of threads they were done with - - final long now = System.nanoTime(); - final long interval = now - lastStep; - lastStep = now; - - final int currentThreads = pool.getMaximumPoolSize(); - final int doneJobs = pool.completedJobs.getAndSet(0); - // number of jobs per sec - final double score = (((double) doneJobs) * 1000000000.0d) / (interval); - - log.trace( - "Completed " - + doneJobs - + " in " - + interval - + "ns using " - + currentThreads - + " threads : score = " - + score); - // store the result of this score - threadsScores.put(currentThreads, score); - threadsTime.put(currentThreads, now); - // remove any scores that are too old - final Iterator iterator = threadsTime.keySet().iterator(); - - while (iterator.hasNext()) { - final int testThreads = iterator.next(); - final long testTime = threadsTime.get(testThreads); - // more than 25 pollings ago? - if (testTime + (pollInterval * 1000000l * 25) < now) { - // too old score, remove it - log.trace( - "Remove out-of-date score for " - + testThreads - + " of " - + threadsScores.get(testThreads)); - iterator.remove(); - threadsScores.remove(testThreads); - } - } - - // work out what the best number of threads is - double bestScore = score; - int bestThreads = currentThreads; - - for (final int testThreads : threadsScores.keySet()) { - final double testScore = threadsScores.get(testThreads); - - if (testScore > bestScore) { - bestScore = testScore; - bestThreads = testThreads; - } - } - log.trace("Best scoring number of threads is " + bestThreads + " with " + bestScore); - - // if we are more than margin below the best, change to the best - if (bestThreads != currentThreads && margin * score < bestScore) { - log.trace("Adjusting to use " + (bestThreads) + " threads"); - setPoolSizesCoreFirst(bestThreads); - } else { - // experiment if we might do better increase or decreasing the threads - if ((!threadsScores.containsKey(currentThreads + 1) - || threadsScores.get(currentThreads + 1) > margin * score) - && currentThreads < maxThreads) { - // increase the number of threads - log.trace("Adjusting to try " + (currentThreads + 1) + " threads"); - setPoolSizesMaxFirst(currentThreads + 1); - } else if (currentThreads > 1 - && (!threadsScores.containsKey(currentThreads - 1) - || threadsScores.get(currentThreads - 1) > margin * score)) { - // decrease the number of threads - // only decrease threads if there are at least 2 (so we don't drop to zero!) - log.trace("Adjusting to try " + (currentThreads - 1) + " threads"); - setPoolSizesCoreFirst(currentThreads - 1); - } - } - } - } - - private void setPoolSizesCoreFirst(final int bestThreads) { - try { - pool.setCorePoolSize(bestThreads); - pool.setMaximumPoolSize(bestThreads); - } catch (final IllegalArgumentException illPoolSizes) { - pool.setMaximumPoolSize(pool.getCorePoolSize() + 1); - } - } - - private void setPoolSizesMaxFirst(final int bestThreads) { - try { - pool.setMaximumPoolSize(bestThreads); - pool.setCorePoolSize(bestThreads); - } catch (final IllegalArgumentException illPoolSizes) { - pool.setMaximumPoolSize(pool.getCorePoolSize() + 1); - } - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.thread; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AdaptiveThreadPoolExecutor extends ThreadPoolExecutor implements AutoCloseable { + private final AtomicInteger completedJobs = new AtomicInteger(0); + + private AdaptiveThreadPoolExecutor( + final int corePoolSize, + final int maximumPoolSize, + final long keepAliveTime, + final TimeUnit unit, + final BlockingQueue workQueue, + final RejectedExecutionHandler rejectedExecutionHandler) { + + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, rejectedExecutionHandler); + } + + @Override + protected void afterExecute(final Runnable r, final Throwable t) { + if (t != null) { + return; + } + + completedJobs.incrementAndGet(); + } + + /** + * This is required to implement the AutoClosable interface. It will stop accepting new jobs and + * wait up to 24h before termination; + */ + @Override + public void close() { + shutdown(); + try { + awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + /** + * By default creates a pool with a queue size of 1000 that will test to increase/decrease threads + * every 60 seconds and does not guarantee to distribute jobs fairly among threads + * + * @return + */ + public static AdaptiveThreadPoolExecutor create() { + return create(1000, 60000, false); + } + + public static AdaptiveThreadPoolExecutor create( + final int maxQueueSize, final int pollInterval, final boolean fairness) { + return create( + maxQueueSize, + pollInterval, + fairness, + Runtime.getRuntime().availableProcessors(), + Runtime.getRuntime().availableProcessors() * 8); + } + + public static AdaptiveThreadPoolExecutor create( + final int maxQueueSize, + final int pollInterval, + final boolean fairness, + final int initialPoolSize, + final int maxThreads) { + // default to the number of processors + final int corePoolSize = initialPoolSize; + final int maximumPoolSize = corePoolSize; + // keep alive is not relevant, since core == maximum + final long keepAliveTime = 1; + final TimeUnit unit = TimeUnit.DAYS; + // a queue constructed with fairness set to true grants threads access + // in FIFO order. + // Fairness generally decreases throughput but reduces variability and + // avoids starvation. + final BlockingQueue workQueue = new ArrayBlockingQueue<>(maxQueueSize, fairness); + // A handler for rejected tasks that runs the rejected task directly in + // the calling thread of the execute method, + // unless the executor has been shut down, in which case the task is + // discarded. + final RejectedExecutionHandler rejectedExecutionHandler = + new ThreadPoolExecutor.CallerRunsPolicy(); + final AdaptiveThreadPoolExecutor threadPool = + new AdaptiveThreadPoolExecutor( + corePoolSize, + maximumPoolSize, + keepAliveTime, + unit, + workQueue, + rejectedExecutionHandler); + final Thread monitorThread = new Thread(new PoolMonitor(threadPool, pollInterval, maxThreads)); + + monitorThread.setDaemon(true); + monitorThread.start(); + + return threadPool; + } + + /** + * This is a separate thread that monitors a thread pool and increases or decreases the number of + * threads within the pool in order to try to maximize the throughput. + * + * @author faulcon + */ + private static class PoolMonitor implements Runnable { + private final Logger log = LoggerFactory.getLogger(getClass()); + private final AdaptiveThreadPoolExecutor pool; + private final int pollInterval; + private final Map threadsScores = new HashMap<>(); + private final Map threadsTime = new HashMap<>(); + private final double margin = 1.0; + private final int maxThreads; + + PoolMonitor( + final AdaptiveThreadPoolExecutor pool, final int pollInterval, final int maxThreads) { + this.pool = pool; + this.pollInterval = pollInterval; + this.maxThreads = maxThreads; + } + + @Override + public void run() { + long lastStep = System.nanoTime(); + + while (!pool.isTerminated()) { + // wait for it to do stuff + try { + Thread.sleep(pollInterval); + } catch (final InterruptedException e) { + if (Thread.interrupted()) { // Clears interrupted status! + throw new RuntimeException(e); + } + } + + // test the number of jobs done + // get number of threads they were done with + + final long now = System.nanoTime(); + final long interval = now - lastStep; + lastStep = now; + + final int currentThreads = pool.getMaximumPoolSize(); + final int doneJobs = pool.completedJobs.getAndSet(0); + // number of jobs per sec + final double score = (((double) doneJobs) * 1000000000.0d) / (interval); + + log.trace( + "Completed " + + doneJobs + + " in " + + interval + + "ns using " + + currentThreads + + " threads : score = " + + score); + // store the result of this score + threadsScores.put(currentThreads, score); + threadsTime.put(currentThreads, now); + // remove any scores that are too old + final Iterator iterator = threadsTime.keySet().iterator(); + + while (iterator.hasNext()) { + final int testThreads = iterator.next(); + final long testTime = threadsTime.get(testThreads); + // more than 25 pollings ago? + if (testTime + (pollInterval * 1000000l * 25) < now) { + // too old score, remove it + log.trace( + "Remove out-of-date score for " + + testThreads + + " of " + + threadsScores.get(testThreads)); + iterator.remove(); + threadsScores.remove(testThreads); + } + } + + // work out what the best number of threads is + double bestScore = score; + int bestThreads = currentThreads; + + for (final int testThreads : threadsScores.keySet()) { + final double testScore = threadsScores.get(testThreads); + + if (testScore > bestScore) { + bestScore = testScore; + bestThreads = testThreads; + } + } + log.trace("Best scoring number of threads is " + bestThreads + " with " + bestScore); + + // if we are more than margin below the best, change to the best + if (bestThreads != currentThreads && margin * score < bestScore) { + log.trace("Adjusting to use " + (bestThreads) + " threads"); + setPoolSizesCoreFirst(bestThreads); + } else { + // experiment if we might do better increase or decreasing the threads + if ((!threadsScores.containsKey(currentThreads + 1) + || threadsScores.get(currentThreads + 1) > margin * score) + && currentThreads < maxThreads) { + // increase the number of threads + log.trace("Adjusting to try " + (currentThreads + 1) + " threads"); + setPoolSizesMaxFirst(currentThreads + 1); + } else if (currentThreads > 1 + && (!threadsScores.containsKey(currentThreads - 1) + || threadsScores.get(currentThreads - 1) > margin * score)) { + // decrease the number of threads + // only decrease threads if there are at least 2 (so we don't drop to zero!) + log.trace("Adjusting to try " + (currentThreads - 1) + " threads"); + setPoolSizesCoreFirst(currentThreads - 1); + } + } + } + } + + private void setPoolSizesCoreFirst(final int bestThreads) { + try { + pool.setCorePoolSize(bestThreads); + pool.setMaximumPoolSize(bestThreads); + } catch (final IllegalArgumentException illPoolSizes) { + pool.setMaximumPoolSize(pool.getCorePoolSize() + 1); + } + } + + private void setPoolSizesMaxFirst(final int bestThreads) { + try { + pool.setMaximumPoolSize(bestThreads); + pool.setCorePoolSize(bestThreads); + } catch (final IllegalArgumentException illPoolSizes) { + pool.setMaximumPoolSize(pool.getCorePoolSize() + 1); + } + } + } +} diff --git a/utils/thread/src/main/java/uk/ac/ebi/biosamples/utils/ThreadUtils.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/thread/ThreadUtils.java similarity index 95% rename from utils/thread/src/main/java/uk/ac/ebi/biosamples/utils/ThreadUtils.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/thread/ThreadUtils.java index 6647f1619..57f8fab34 100644 --- a/utils/thread/src/main/java/uk/ac/ebi/biosamples/utils/ThreadUtils.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/thread/ThreadUtils.java @@ -1,84 +1,84 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ThreadUtils { - private static final Logger log = LoggerFactory.getLogger(ThreadUtils.class); - - public static void checkFutures( - final Map> futures, final int maxSize) - throws InterruptedException, ExecutionException { - - while (futures.size() > maxSize) { - for (final Iterator i = futures.keySet().iterator(); i.hasNext(); ) { - final Object key = i.next(); - - futures.get(key).get(); - i.remove(); - } - } - } - - public static void checkAndCallbackFutures( - final Map> futures, final int maxSize, final Callback callback) - throws InterruptedException, ExecutionException { - while (futures.size() > maxSize) { - for (final Iterator i = futures.keySet().iterator(); i.hasNext(); ) { - final Object key = i.next(); - - callback.call(futures.get(key).get()); - i.remove(); - } - } - } - - public interface Callback { - void call(T t); - } - - public static void checkAndRetryFutures( - final Map> futures, - final Map> callables, - final int maxSize, - final ExecutorService executionService) - throws InterruptedException { - while (futures.size() > maxSize) { - final List toReRun = new ArrayList<>(); - - for (final Iterator i = futures.keySet().iterator(); i.hasNext(); ) { - final T key = i.next(); - - try { - futures.get(key).get(); - } catch (final ExecutionException e) { - toReRun.add(key); - } - i.remove(); - } - - for (final T key : toReRun) { - log.info("Re-executing " + key); - futures.put(key, executionService.submit(callables.get(key))); - } - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.thread; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ThreadUtils { + private static final Logger log = LoggerFactory.getLogger(ThreadUtils.class); + + public static void checkFutures( + final Map> futures, final int maxSize) + throws InterruptedException, ExecutionException { + + while (futures.size() > maxSize) { + for (final Iterator i = futures.keySet().iterator(); i.hasNext(); ) { + final Object key = i.next(); + + futures.get(key).get(); + i.remove(); + } + } + } + + public static void checkAndCallbackFutures( + final Map> futures, final int maxSize, final Callback callback) + throws InterruptedException, ExecutionException { + while (futures.size() > maxSize) { + for (final Iterator i = futures.keySet().iterator(); i.hasNext(); ) { + final Object key = i.next(); + + callback.call(futures.get(key).get()); + i.remove(); + } + } + } + + public interface Callback { + void call(T t); + } + + public static void checkAndRetryFutures( + final Map> futures, + final Map> callables, + final int maxSize, + final ExecutorService executionService) + throws InterruptedException { + while (futures.size() > maxSize) { + final List toReRun = new ArrayList<>(); + + for (final Iterator i = futures.keySet().iterator(); i.hasNext(); ) { + final T key = i.next(); + + try { + futures.get(key).get(); + } catch (final ExecutionException e) { + toReRun.add(key); + } + i.remove(); + } + + for (final T key : toReRun) { + log.info("Re-executing " + key); + futures.put(key, executionService.submit(callables.get(key))); + } + } + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/utils/upload/Characteristics.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/upload/Characteristics.java similarity index 97% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/utils/upload/Characteristics.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/upload/Characteristics.java index 3af2472fc..14522f6a6 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/utils/upload/Characteristics.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/upload/Characteristics.java @@ -1,28 +1,28 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils.upload; - -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -@Getter -@Setter -@EqualsAndHashCode -@ToString -public class Characteristics { - private static final long serialVersionUID = 1L; - String name; - String value; - String iri; - String unit; -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.upload; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Getter +@Setter +@EqualsAndHashCode +@ToString +public class Characteristics { + private static final long serialVersionUID = 1L; + String name; + String value; + String iri; + String unit; +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/utils/upload/FileUploadUtils.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/upload/FileUploadUtils.java similarity index 96% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/utils/upload/FileUploadUtils.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/upload/FileUploadUtils.java index e682c523c..a11891278 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/utils/upload/FileUploadUtils.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/upload/FileUploadUtils.java @@ -1,707 +1,707 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils.upload; - -import static com.google.common.collect.Multimaps.toMultimap; - -import com.google.common.collect.LinkedListMultimap; -import com.google.common.collect.Multimap; -import java.io.*; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.CSVRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.*; - -@Service -public class FileUploadUtils { - private final Logger log = LoggerFactory.getLogger(getClass()); - private static final DateTimeFormatter DATE_TIME_FORMATTER = - DateTimeFormatter.ofPattern("dd LLL yyyy HH:mm"); - - public List> getISATABDataInMap(final CSVParser csvParser) - throws IOException { - final List> csvDataMap = new ArrayList<>(); - final List headers = csvParser.getHeaderNames(); - - csvParser - .getRecords() - .forEach( - csvRecord -> { - final AtomicInteger i = new AtomicInteger(0); - final Multimap listMultiMap = LinkedListMultimap.create(); - - headers.forEach( - header -> { - final String record = csvRecord.get(i.get()); - listMultiMap.put( - header != null - && !(header.startsWith("characteristics") - || header.startsWith("Characteristics")) - ? header.toLowerCase() - : header, - record); - i.getAndIncrement(); - }); - - csvDataMap.add(listMultiMap); - }); - - return csvDataMap; - } - - public Sample buildSample( - final Multimap multiMap, final ValidationResult validationResult) { - String sampleName = getSampleName(multiMap); - String sampleReleaseDate = getReleaseDate(multiMap); - String accession = getSampleAccession(multiMap); - List characteristicsList = handleCharacteristics(multiMap); - List externalReferenceList = handleExternalReferences(multiMap); - List contactsList = handleContacts(multiMap); - List publicationsList = handlePublications(multiMap); - List organizationList = handleOrganizations(multiMap); - - if (isValidSample(sampleName, sampleReleaseDate, validationResult)) { - return buildSample( - sampleName, - accession, - sampleReleaseDate, - characteristicsList, - externalReferenceList, - contactsList, - publicationsList, - organizationList); - } - - return null; - } - - private List handleOrganizations(final Multimap multiMap) { - final List organizationsList = new ArrayList<>(); - final List organizationNames = new ArrayList<>(); - final List organizationRoles = new ArrayList<>(); - final List organizationAddresses = new ArrayList<>(); - final List organizationEmails = new ArrayList<>(); - final List organizationUrls = new ArrayList<>(); - - multiMap - .entries() - .forEach( - entry -> { - final String entryKey = entry.getKey(); - final String entryValue = entry.getValue(); - - if (entryKey.startsWith("comment[submission_organization:")) { - if (entryKey.contains("email")) { - organizationEmails.add(entryValue); - } - - if (entryKey.contains("name")) { - organizationNames.add(entryValue); - } - - if (entryKey.contains("address")) { - organizationAddresses.add(entryValue); - } - - if (entryKey.contains("role")) { - organizationRoles.add(entryValue); - } - - if (entryKey.contains("url")) { - organizationUrls.add(entryValue); - } - } - }); - - /*Contact email is mandatory for the contact information to be built*/ - for (int iter = 0; iter < organizationNames.size(); iter++) { - final Organization.Builder organizationBuilder = new Organization.Builder(); - - organizationBuilder.name(organizationNames.get(iter)); - - organizationBuilder.email( - iter >= organizationEmails.size() ? null : organizationEmails.get(iter)); - organizationBuilder.role( - iter >= organizationRoles.size() ? null : organizationRoles.get(iter)); - organizationBuilder.address( - iter >= organizationAddresses.size() ? null : organizationAddresses.get(iter)); - organizationBuilder.url(iter >= organizationUrls.size() ? null : organizationUrls.get(iter)); - - if (organizationBuilder.isNotEmpty()) { - organizationsList.add(organizationBuilder.build()); - } - } - - return organizationsList; - } - - private Sample buildSample( - final String sampleName, - final String accession, - final String sampleReleaseDate, - final List characteristicsList, - final List externalReferencesList, - final List contactsList, - final List publicationsList, - final List organizationsList) { - return new Sample.Builder(sampleName.trim()) - .withAccession(accession) - .withAttributes( - characteristicsList.stream() - .map( - characteristics -> { - final String name = characteristics.getName(); - final String trimmedCharacteristicsName = - name.substring(name.indexOf('[') + 1, name.indexOf(']')); - final String characteristicsValue = characteristics.getValue(); - - if (isValidCharacteristics( - name, trimmedCharacteristicsName, characteristicsValue)) { - - return new Attribute.Builder( - trimmedCharacteristicsName, characteristicsValue) - .withUnit(characteristics.getUnit()) - .withIri(characteristics.getIri()) - .build(); - } else { - return null; - } - }) - .filter(Objects::nonNull) - .collect(Collectors.toList())) - .withRelease(sampleReleaseDate) - .withExternalReferences(externalReferencesList) - .withContacts(contactsList) - .withPublications(publicationsList) - .withOrganizations(organizationsList) - .withSubmittedVia(SubmittedViaType.FILE_UPLOADER) - .build(); - } - - private boolean isValidCharacteristics( - final String name, - final String trimmedCharacteristicsName, - final String characteristicsValue) { - return (name != null && !trimmedCharacteristicsName.isEmpty()) - && (characteristicsValue != null && !characteristicsValue.isEmpty()); - } - - private List handleContacts(final Multimap multiMap) { - final List contactList = new ArrayList<>(); - final List contactEmails = new ArrayList<>(); - final List contactNames = new ArrayList<>(); - final List contactAffiliations = new ArrayList<>(); - final List contactRoles = new ArrayList<>(); - final List contactUrls = new ArrayList<>(); - - multiMap - .entries() - .forEach( - entry -> { - final String entryKey = entry.getKey(); - final String entryValue = entry.getValue(); - - if (entryKey.startsWith("comment[submission_contact:")) { - if (entryKey.contains("email")) { - contactEmails.add(entryValue); - } - - if (entryKey.contains("name")) { - contactNames.add(entryValue); - } - - if (entryKey.contains("affiliation")) { - contactAffiliations.add(entryValue); - } - - if (entryKey.contains("role")) { - contactRoles.add(entryValue); - } - - if (entryKey.contains("url")) { - contactUrls.add(entryValue); - } - } - }); - - /*Contact email is mandatory for the contact information to be built*/ - for (int iter = 0; iter < contactEmails.size(); iter++) { - final Contact.Builder contactBuilder = new Contact.Builder(); - - contactBuilder.email(contactEmails.get(iter)); - - contactBuilder.name(iter >= contactNames.size() ? null : contactNames.get(iter)); - contactBuilder.affiliation( - iter >= contactAffiliations.size() ? null : contactAffiliations.get(iter)); - contactBuilder.role(iter >= contactRoles.size() ? null : contactRoles.get(iter)); - contactBuilder.url(iter >= contactUrls.size() ? null : contactUrls.get(iter)); - - if (contactBuilder.isNotEmpty()) { - contactList.add(contactBuilder.build()); - } - } - - return contactList; - } - - private List handlePublications(final Multimap multiMap) { - final List publicationDois = handlePublicationDois(multiMap); - final List publicationPubMedIds = handlePublicationPubMedIds(multiMap); - final List publicationList = new ArrayList<>(); - - final int pubMedSize = publicationPubMedIds.size(); - final int doiSize = publicationDois.size(); - - if (pubMedSize >= doiSize) { - for (int iter = 0; iter < pubMedSize; iter++) { - publicationList.add( - buildPublication(publicationDois, publicationPubMedIds, iter, pubMedSize, doiSize)); - } - } else { - for (int iter = 0; iter < doiSize; iter++) { - publicationList.add( - buildPublication(publicationDois, publicationPubMedIds, iter, pubMedSize, doiSize)); - } - } - - return publicationList; - } - - private Publication buildPublication( - final List publicationDois, - final List publicationPubMedIds, - final int iter, - final int pubMedSize, - final int doiSize) { - final Publication.Builder publicationBuilder = new Publication.Builder(); - - final String pubMedId = iter >= pubMedSize ? null : publicationPubMedIds.get(iter); - final String doi = iter >= doiSize ? null : publicationDois.get(iter); - - publicationBuilder.pubmed_id(pubMedId); - publicationBuilder.doi(doi); - - return publicationBuilder.build(); - } - - private List handlePublicationPubMedIds(final Multimap multiMap) { - final List pubMedIds = new ArrayList<>(); - - multiMap - .entries() - .forEach( - entry -> { - final String entryKey = entry.getKey(); - final String entryValue = entry.getValue(); - - if (entryKey.startsWith("comment[publication:")) { - if (entryKey.contains("pubmed_id")) { - pubMedIds.add((entryValue != null && !entryValue.isEmpty()) ? entryValue : ""); - } - } - }); - - return pubMedIds.stream().filter(Objects::nonNull).collect(Collectors.toList()); - } - - private List handlePublicationDois(final Multimap multiMap) { - final List dois = new ArrayList<>(); - - multiMap - .entries() - .forEach( - entry -> { - final String entryKey = entry.getKey(); - final String entryValue = entry.getValue(); - - if (entryKey.startsWith("comment[publication:")) { - if (entryKey.contains("doi")) { - dois.add((entryValue != null && !entryValue.isEmpty()) ? entryValue : ""); - } - } - }); - - return dois.stream().filter(Objects::nonNull).collect(Collectors.toList()); - } - - public String getSampleAccession(final Multimap multiMap) { - return multiMap.get("sample identifier").stream().findFirst().orElse(null); - } - - private List handleExternalReferences( - final Multimap multiMap) { - final List externalReferenceList = new ArrayList<>(); - - multiMap - .entries() - .forEach( - entry -> { - final String entryKey = entry.getKey(); - final String entryValue = entry.getValue(); - - log.trace(entryKey + " " + entryValue); - - if (entryKey.startsWith("comment") && entryKey.contains("external db ref")) { - if (entryValue != null && entryValue.length() > 1) { - externalReferenceList.add(ExternalReference.build(entryValue)); - } - } - }); - - return externalReferenceList; - } - - public List createRelationships( - final Sample sample, - final Map sampleNameToAccessionMap, - final Multimap relationshipMap, - final ValidationResult validationResult) { - return relationshipMap.entries().stream() - .map(entry -> getRelationship(sample, sampleNameToAccessionMap, entry, validationResult)) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - - private Relationship getRelationship( - final Sample sample, - final Map sampleNameToAccessionMap, - final Map.Entry entry, - final ValidationResult validationResult) { - if (sample != null && sample.getAccession() != null) { - try { - final String relationshipTarget = getRelationshipTarget(sampleNameToAccessionMap, entry); - - if (relationshipTarget != null) { - return Relationship.build( - sample.getAccession(), entry.getKey().trim(), relationshipTarget); - } else { - validationResult.addValidationMessage( - new ValidationResult.ValidationMessage( - sample.getAccession(), - "Failed to add all relationships for " + sample.getAccession(), - true)); - - return null; - } - } catch (final Exception e) { - log.info("Failed to add relationship"); - validationResult.addValidationMessage( - new ValidationResult.ValidationMessage( - sample.getAccession(), - "Failed to add all relationships for " + sample.getAccession(), - true)); - - return null; - } - } - - return null; - } - - private String getRelationshipTarget( - final Map sampleNameToAccessionMap, final Map.Entry entry) { - final String relationshipTarget = entry.getValue().trim(); - - if (relationshipTarget.startsWith("SAM")) { - return relationshipTarget; - } else { - return sampleNameToAccessionMap.get(relationshipTarget); - } - } - - public Multimap parseRelationships(final Multimap multiMap) { - return multiMap.entries().stream() - .filter( - e -> { - final String entryKey = e.getKey(); - - return entryKey.startsWith("comment") && entryKey.contains("bsd_relationship"); - }) - .collect( - toMultimap( - e -> { - final String key = e.getKey().trim(); - return key.substring(key.indexOf(":") + 1, key.length() - 1); - }, - e -> { - final String value = e.getValue(); - - return value != null ? value.trim() : null; - }, - LinkedListMultimap::create)); - } - - private List handleCharacteristics(final Multimap multiMap) { - final List characteristicsList = new ArrayList<>(); - - multiMap - .entries() - .forEach( - entry -> { - final Characteristics characteristics = new Characteristics(); - - if (entry.getKey().startsWith("Characteristics")) { - characteristics.setName(entry.getKey().trim()); - characteristics.setValue(entry.getValue()); - - characteristicsList.add(characteristics); - } - }); - - final List termRefList = - multiMap.entries().stream() - .map( - entry -> { - if (entry.getKey().startsWith("term accession number")) { - final String value = entry.getValue(); - - return value != null ? value.trim() : null; - } else { - return null; - } - }) - .filter(Objects::nonNull) - .toList(); - - final List unitList = - multiMap.entries().stream() - .map( - entry -> { - if (entry.getKey().startsWith("unit")) { - final String value = entry.getValue(); - - return value != null ? value.trim() : null; - } else { - return null; - } - }) - .filter(Objects::nonNull) - .toList(); // handle units - - final AtomicInteger i = new AtomicInteger(0); - - characteristicsList.forEach( - characteristics -> { - final int val = i.getAndIncrement(); - - if (val < termRefList.size() && termRefList.get(val) != null) { - characteristics.setIri(termRefList.get(val)); - } - - if (val < unitList.size() && unitList.get(val) != null) { - characteristics.setUnit(unitList.get(val)); - } - }); - - return characteristicsList; - } - - public static String getSampleName(final Multimap multiMap) { - return multiMap.get("sample name").stream().findFirst().orElse(null); - } - - private static String getReleaseDate(final Multimap multiMap) { - return multiMap.get("release date").stream().findFirst().orElse(null); - } - - public Sample addChecklistAttributeAndBuildSample(final String checklist, Sample sample) { - final Set attributeSet = sample.getAttributes(); - final Attribute attribute = new Attribute.Builder("checklist", checklist).build(); - - attributeSet.add(attribute); - sample = Sample.Builder.fromSample(sample).withAttributes(attributeSet).build(); - - return sample; - } - - private boolean isValidSample( - final String sampleName, - final String sampleReleaseDate, - final ValidationResult validationResult) { - boolean isValidSample = true; - - if (sampleName == null || sampleName.isEmpty()) { - validationResult.addValidationMessage( - new ValidationResult.ValidationMessage( - "MESSAGE#1", - "All samples in the file must have a sample name, some samples are missing sample name and hence are not created", - true)); - isValidSample = false; - } - - if (sampleReleaseDate == null || sampleReleaseDate.isEmpty()) { - validationResult.addValidationMessage( - new ValidationResult.ValidationMessage( - "MESSAGE#2", - "All samples in the file must have a release date " - + sampleName - + " doesn't have a release date and is not created", - true)); - isValidSample = false; - } - - return isValidSample; - } - - private List getPrintableListFromCsvRow(final Iterator iterator) { - final Iterable iterable = () -> iterator; - - return StreamSupport.stream(iterable.spliterator(), false).collect(Collectors.toList()); - } - - public File writeToFile( - final File fileToBeUploaded, - final List headers, - final List samples, - final ValidationResult validationResult) { - try { - log.info("Writing to file"); - final Path temp = Files.createTempFile("upload_result", ".tsv"); - final boolean headerHasIdentifier = - headers.stream().anyMatch(header -> header.equalsIgnoreCase("Sample Identifier")); - - final List outputFileHeaders = new ArrayList<>(headers); - - if (!headerHasIdentifier) { - outputFileHeaders.add("Sample Identifier"); - } - - final Reader in = new FileReader(fileToBeUploaded); - final String[] headerParsed = outputFileHeaders.toArray(new String[outputFileHeaders.size()]); - - final Iterable records = - CSVFormat.TDF - .withHeader(headerParsed) - .withFirstRecordAsHeader() - .withAllowDuplicateHeaderNames() - .withFirstRecordAsHeader() - .withIgnoreEmptyLines() - .withIgnoreHeaderCase() - .withAllowMissingColumnNames() - .withIgnoreSurroundingSpaces() - .withTrim() - .parse(in); - - try (final BufferedWriter writer = Files.newBufferedWriter(temp, StandardCharsets.UTF_8); - final CSVPrinter csvPrinter = - new CSVPrinter(writer, CSVFormat.TDF.withHeader(headerParsed)); - final PrintWriter out = new PrintWriter(writer)) { - for (final CSVRecord row : records) { - if (headerHasIdentifier) { - csvPrinter.printRecord(row); - } else { - csvPrinter.printRecord( - addAccessionToSamplesForPrint(getPrintableListFromCsvRow(row.iterator()), samples)); - } - } - - out.println("\n\n"); - out.println("********RECEIPT START********"); - out.println( - validationResult.getValidationMessagesList().stream() - .map( - validationMessage -> - validationMessage.getMessageKey() - + " : " - + validationMessage.getMessageValue()) - .collect(Collectors.joining("\n"))); - out.println("********RECEIPT END********"); - out.println("\n\n"); - } - - return temp.toFile(); - } catch (final Exception e) { - log.info("Writing to file has failed " + e.getMessage(), e); - - e.printStackTrace(); - return null; - } - } - - private Iterable addAccessionToSamplesForPrint( - final List listFromIterator, final List samples) { - final String sampleName = listFromIterator.get(1); - final Optional sampleOptional = - samples.stream().filter(sample -> sample.getName().equals(sampleName)).findFirst(); - - sampleOptional.ifPresent(sample -> listFromIterator.add(sample.getAccession())); - - return listFromIterator; - } - - public CSVParser buildParser(final BufferedReader reader) throws IOException { - return new CSVParser( - reader, - CSVFormat.TDF - .withAllowDuplicateHeaderNames(true) - .withFirstRecordAsHeader() - .withIgnoreEmptyLines(true) - .withIgnoreHeaderCase(true) - .withAllowMissingColumnNames(true) - .withIgnoreSurroundingSpaces(true) - .withTrim(true)); - } - - public File writeQueueMessageToFile(final String submissionId) throws IOException { - final Path temp = Files.createTempFile("queue_result", ".txt"); - - try (final BufferedWriter writer = Files.newBufferedWriter(temp, StandardCharsets.UTF_8)) { - writer.write( - "Your submission has been queued and your submission id is " - + submissionId - + ". Please use the View Submissions tab and use your submission ID to get the submission result."); - } - - return temp.toFile(); - } - - public void validateHeaderPositions( - final List headers, final ValidationResult validationResult) { - if (!headers.isEmpty()) { - if ((!headers.get(0).equalsIgnoreCase("Source Name") - && (!headers.get(1).equalsIgnoreCase("Sample Name")) - && (!headers.get(2).equalsIgnoreCase("Release Date")))) { - validationResult.addValidationMessage( - new ValidationResult.ValidationMessage( - "GENERAL_VALIDATION_MESSAGE", - "ISA tab file must have Source Name as first column, followed by Sample Name and Release Date.", - true)); - - throw new GlobalExceptions.UploadInvalidException( - validationResult.getValidationMessagesList().stream() - .map( - validationMessage -> - validationMessage.getMessageKey() - + ":" - + validationMessage.getMessageValue()) - .collect(Collectors.joining("\n"))); - } - } - } - - public static String formatDateString(LocalDateTime dateTime) { - return dateTime.format(DATE_TIME_FORMATTER); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.upload; + +import static com.google.common.collect.Multimaps.toMultimap; + +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.Multimap; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.CSVRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.*; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; + +@Service +public class FileUploadUtils { + private final Logger log = LoggerFactory.getLogger(getClass()); + private static final DateTimeFormatter DATE_TIME_FORMATTER = + DateTimeFormatter.ofPattern("dd LLL yyyy HH:mm"); + + public List> getISATABDataInMap(final CSVParser csvParser) + throws IOException { + final List> csvDataMap = new ArrayList<>(); + final List headers = csvParser.getHeaderNames(); + + csvParser + .getRecords() + .forEach( + csvRecord -> { + final AtomicInteger i = new AtomicInteger(0); + final Multimap listMultiMap = LinkedListMultimap.create(); + + headers.forEach( + header -> { + final String record = csvRecord.get(i.get()); + listMultiMap.put( + header != null + && !(header.startsWith("characteristics") + || header.startsWith("Characteristics")) + ? header.toLowerCase() + : header, + record); + i.getAndIncrement(); + }); + + csvDataMap.add(listMultiMap); + }); + + return csvDataMap; + } + + public Sample buildSample( + final Multimap multiMap, final ValidationResult validationResult) { + String sampleName = getSampleName(multiMap); + String sampleReleaseDate = getReleaseDate(multiMap); + String accession = getSampleAccession(multiMap); + List characteristicsList = handleCharacteristics(multiMap); + List externalReferenceList = handleExternalReferences(multiMap); + List contactsList = handleContacts(multiMap); + List publicationsList = handlePublications(multiMap); + List organizationList = handleOrganizations(multiMap); + + if (isValidSample(sampleName, sampleReleaseDate, validationResult)) { + return buildSample( + sampleName, + accession, + sampleReleaseDate, + characteristicsList, + externalReferenceList, + contactsList, + publicationsList, + organizationList); + } + + return null; + } + + private List handleOrganizations(final Multimap multiMap) { + final List organizationsList = new ArrayList<>(); + final List organizationNames = new ArrayList<>(); + final List organizationRoles = new ArrayList<>(); + final List organizationAddresses = new ArrayList<>(); + final List organizationEmails = new ArrayList<>(); + final List organizationUrls = new ArrayList<>(); + + multiMap + .entries() + .forEach( + entry -> { + final String entryKey = entry.getKey(); + final String entryValue = entry.getValue(); + + if (entryKey.startsWith("comment[submission_organization:")) { + if (entryKey.contains("email")) { + organizationEmails.add(entryValue); + } + + if (entryKey.contains("name")) { + organizationNames.add(entryValue); + } + + if (entryKey.contains("address")) { + organizationAddresses.add(entryValue); + } + + if (entryKey.contains("role")) { + organizationRoles.add(entryValue); + } + + if (entryKey.contains("url")) { + organizationUrls.add(entryValue); + } + } + }); + + /*Contact email is mandatory for the contact information to be built*/ + for (int iter = 0; iter < organizationNames.size(); iter++) { + final Organization.Builder organizationBuilder = new Organization.Builder(); + + organizationBuilder.name(organizationNames.get(iter)); + + organizationBuilder.email( + iter >= organizationEmails.size() ? null : organizationEmails.get(iter)); + organizationBuilder.role( + iter >= organizationRoles.size() ? null : organizationRoles.get(iter)); + organizationBuilder.address( + iter >= organizationAddresses.size() ? null : organizationAddresses.get(iter)); + organizationBuilder.url(iter >= organizationUrls.size() ? null : organizationUrls.get(iter)); + + if (organizationBuilder.isNotEmpty()) { + organizationsList.add(organizationBuilder.build()); + } + } + + return organizationsList; + } + + private Sample buildSample( + final String sampleName, + final String accession, + final String sampleReleaseDate, + final List characteristicsList, + final List externalReferencesList, + final List contactsList, + final List publicationsList, + final List organizationsList) { + return new Sample.Builder(sampleName.trim()) + .withAccession(accession) + .withAttributes( + characteristicsList.stream() + .map( + characteristics -> { + final String name = characteristics.getName(); + final String trimmedCharacteristicsName = + name.substring(name.indexOf('[') + 1, name.indexOf(']')); + final String characteristicsValue = characteristics.getValue(); + + if (isValidCharacteristics( + name, trimmedCharacteristicsName, characteristicsValue)) { + + return new Attribute.Builder( + trimmedCharacteristicsName, characteristicsValue) + .withUnit(characteristics.getUnit()) + .withIri(characteristics.getIri()) + .build(); + } else { + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList())) + .withRelease(sampleReleaseDate) + .withExternalReferences(externalReferencesList) + .withContacts(contactsList) + .withPublications(publicationsList) + .withOrganizations(organizationsList) + .withSubmittedVia(SubmittedViaType.FILE_UPLOADER) + .build(); + } + + private boolean isValidCharacteristics( + final String name, + final String trimmedCharacteristicsName, + final String characteristicsValue) { + return (name != null && !trimmedCharacteristicsName.isEmpty()) + && (characteristicsValue != null && !characteristicsValue.isEmpty()); + } + + private List handleContacts(final Multimap multiMap) { + final List contactList = new ArrayList<>(); + final List contactEmails = new ArrayList<>(); + final List contactNames = new ArrayList<>(); + final List contactAffiliations = new ArrayList<>(); + final List contactRoles = new ArrayList<>(); + final List contactUrls = new ArrayList<>(); + + multiMap + .entries() + .forEach( + entry -> { + final String entryKey = entry.getKey(); + final String entryValue = entry.getValue(); + + if (entryKey.startsWith("comment[submission_contact:")) { + if (entryKey.contains("email")) { + contactEmails.add(entryValue); + } + + if (entryKey.contains("name")) { + contactNames.add(entryValue); + } + + if (entryKey.contains("affiliation")) { + contactAffiliations.add(entryValue); + } + + if (entryKey.contains("role")) { + contactRoles.add(entryValue); + } + + if (entryKey.contains("url")) { + contactUrls.add(entryValue); + } + } + }); + + /*Contact email is mandatory for the contact information to be built*/ + for (int iter = 0; iter < contactEmails.size(); iter++) { + final Contact.Builder contactBuilder = new Contact.Builder(); + + contactBuilder.email(contactEmails.get(iter)); + + contactBuilder.name(iter >= contactNames.size() ? null : contactNames.get(iter)); + contactBuilder.affiliation( + iter >= contactAffiliations.size() ? null : contactAffiliations.get(iter)); + contactBuilder.role(iter >= contactRoles.size() ? null : contactRoles.get(iter)); + contactBuilder.url(iter >= contactUrls.size() ? null : contactUrls.get(iter)); + + if (contactBuilder.isNotEmpty()) { + contactList.add(contactBuilder.build()); + } + } + + return contactList; + } + + private List handlePublications(final Multimap multiMap) { + final List publicationDois = handlePublicationDois(multiMap); + final List publicationPubMedIds = handlePublicationPubMedIds(multiMap); + final List publicationList = new ArrayList<>(); + + final int pubMedSize = publicationPubMedIds.size(); + final int doiSize = publicationDois.size(); + + if (pubMedSize >= doiSize) { + for (int iter = 0; iter < pubMedSize; iter++) { + publicationList.add( + buildPublication(publicationDois, publicationPubMedIds, iter, pubMedSize, doiSize)); + } + } else { + for (int iter = 0; iter < doiSize; iter++) { + publicationList.add( + buildPublication(publicationDois, publicationPubMedIds, iter, pubMedSize, doiSize)); + } + } + + return publicationList; + } + + private Publication buildPublication( + final List publicationDois, + final List publicationPubMedIds, + final int iter, + final int pubMedSize, + final int doiSize) { + final Publication.Builder publicationBuilder = new Publication.Builder(); + + final String pubMedId = iter >= pubMedSize ? null : publicationPubMedIds.get(iter); + final String doi = iter >= doiSize ? null : publicationDois.get(iter); + + publicationBuilder.pubmed_id(pubMedId); + publicationBuilder.doi(doi); + + return publicationBuilder.build(); + } + + private List handlePublicationPubMedIds(final Multimap multiMap) { + final List pubMedIds = new ArrayList<>(); + + multiMap + .entries() + .forEach( + entry -> { + final String entryKey = entry.getKey(); + final String entryValue = entry.getValue(); + + if (entryKey.startsWith("comment[publication:")) { + if (entryKey.contains("pubmed_id")) { + pubMedIds.add((entryValue != null && !entryValue.isEmpty()) ? entryValue : ""); + } + } + }); + + return pubMedIds.stream().filter(Objects::nonNull).collect(Collectors.toList()); + } + + private List handlePublicationDois(final Multimap multiMap) { + final List dois = new ArrayList<>(); + + multiMap + .entries() + .forEach( + entry -> { + final String entryKey = entry.getKey(); + final String entryValue = entry.getValue(); + + if (entryKey.startsWith("comment[publication:")) { + if (entryKey.contains("doi")) { + dois.add((entryValue != null && !entryValue.isEmpty()) ? entryValue : ""); + } + } + }); + + return dois.stream().filter(Objects::nonNull).collect(Collectors.toList()); + } + + public String getSampleAccession(final Multimap multiMap) { + return multiMap.get("sample identifier").stream().findFirst().orElse(null); + } + + private List handleExternalReferences( + final Multimap multiMap) { + final List externalReferenceList = new ArrayList<>(); + + multiMap + .entries() + .forEach( + entry -> { + final String entryKey = entry.getKey(); + final String entryValue = entry.getValue(); + + log.trace(entryKey + " " + entryValue); + + if (entryKey.startsWith("comment") && entryKey.contains("external db ref")) { + if (entryValue != null && entryValue.length() > 1) { + externalReferenceList.add(ExternalReference.build(entryValue)); + } + } + }); + + return externalReferenceList; + } + + public List createRelationships( + final Sample sample, + final Map sampleNameToAccessionMap, + final Multimap relationshipMap, + final ValidationResult validationResult) { + return relationshipMap.entries().stream() + .map(entry -> getRelationship(sample, sampleNameToAccessionMap, entry, validationResult)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private Relationship getRelationship( + final Sample sample, + final Map sampleNameToAccessionMap, + final Map.Entry entry, + final ValidationResult validationResult) { + if (sample != null && sample.getAccession() != null) { + try { + final String relationshipTarget = getRelationshipTarget(sampleNameToAccessionMap, entry); + + if (relationshipTarget != null) { + return Relationship.build( + sample.getAccession(), entry.getKey().trim(), relationshipTarget); + } else { + validationResult.addValidationMessage( + new ValidationResult.ValidationMessage( + sample.getAccession(), + "Failed to add all relationships for " + sample.getAccession(), + true)); + + return null; + } + } catch (final Exception e) { + log.info("Failed to add relationship"); + validationResult.addValidationMessage( + new ValidationResult.ValidationMessage( + sample.getAccession(), + "Failed to add all relationships for " + sample.getAccession(), + true)); + + return null; + } + } + + return null; + } + + private String getRelationshipTarget( + final Map sampleNameToAccessionMap, final Map.Entry entry) { + final String relationshipTarget = entry.getValue().trim(); + + if (relationshipTarget.startsWith("SAM")) { + return relationshipTarget; + } else { + return sampleNameToAccessionMap.get(relationshipTarget); + } + } + + public Multimap parseRelationships(final Multimap multiMap) { + return multiMap.entries().stream() + .filter( + e -> { + final String entryKey = e.getKey(); + + return entryKey.startsWith("comment") && entryKey.contains("bsd_relationship"); + }) + .collect( + toMultimap( + e -> { + final String key = e.getKey().trim(); + return key.substring(key.indexOf(":") + 1, key.length() - 1); + }, + e -> { + final String value = e.getValue(); + + return value != null ? value.trim() : null; + }, + LinkedListMultimap::create)); + } + + private List handleCharacteristics(final Multimap multiMap) { + final List characteristicsList = new ArrayList<>(); + + multiMap + .entries() + .forEach( + entry -> { + final Characteristics characteristics = new Characteristics(); + + if (entry.getKey().startsWith("Characteristics")) { + characteristics.setName(entry.getKey().trim()); + characteristics.setValue(entry.getValue()); + + characteristicsList.add(characteristics); + } + }); + + final List termRefList = + multiMap.entries().stream() + .map( + entry -> { + if (entry.getKey().startsWith("term accession number")) { + final String value = entry.getValue(); + + return value != null ? value.trim() : null; + } else { + return null; + } + }) + .filter(Objects::nonNull) + .toList(); + + final List unitList = + multiMap.entries().stream() + .map( + entry -> { + if (entry.getKey().startsWith("unit")) { + final String value = entry.getValue(); + + return value != null ? value.trim() : null; + } else { + return null; + } + }) + .filter(Objects::nonNull) + .toList(); // handle units + + final AtomicInteger i = new AtomicInteger(0); + + characteristicsList.forEach( + characteristics -> { + final int val = i.getAndIncrement(); + + if (val < termRefList.size() && termRefList.get(val) != null) { + characteristics.setIri(termRefList.get(val)); + } + + if (val < unitList.size() && unitList.get(val) != null) { + characteristics.setUnit(unitList.get(val)); + } + }); + + return characteristicsList; + } + + public static String getSampleName(final Multimap multiMap) { + return multiMap.get("sample name").stream().findFirst().orElse(null); + } + + private static String getReleaseDate(final Multimap multiMap) { + return multiMap.get("release date").stream().findFirst().orElse(null); + } + + public Sample addChecklistAttributeAndBuildSample(final String checklist, Sample sample) { + final Set attributeSet = sample.getAttributes(); + final Attribute attribute = new Attribute.Builder("checklist", checklist).build(); + + attributeSet.add(attribute); + sample = Sample.Builder.fromSample(sample).withAttributes(attributeSet).build(); + + return sample; + } + + private boolean isValidSample( + final String sampleName, + final String sampleReleaseDate, + final ValidationResult validationResult) { + boolean isValidSample = true; + + if (sampleName == null || sampleName.isEmpty()) { + validationResult.addValidationMessage( + new ValidationResult.ValidationMessage( + "MESSAGE#1", + "All samples in the file must have a sample name, some samples are missing sample name and hence are not created", + true)); + isValidSample = false; + } + + if (sampleReleaseDate == null || sampleReleaseDate.isEmpty()) { + validationResult.addValidationMessage( + new ValidationResult.ValidationMessage( + "MESSAGE#2", + "All samples in the file must have a release date " + + sampleName + + " doesn't have a release date and is not created", + true)); + isValidSample = false; + } + + return isValidSample; + } + + private List getPrintableListFromCsvRow(final Iterator iterator) { + final Iterable iterable = () -> iterator; + + return StreamSupport.stream(iterable.spliterator(), false).collect(Collectors.toList()); + } + + public File writeToFile( + final File fileToBeUploaded, + final List headers, + final List samples, + final ValidationResult validationResult) { + try { + log.info("Writing to file"); + final Path temp = Files.createTempFile("upload_result", ".tsv"); + final boolean headerHasIdentifier = + headers.stream().anyMatch(header -> header.equalsIgnoreCase("Sample Identifier")); + + final List outputFileHeaders = new ArrayList<>(headers); + + if (!headerHasIdentifier) { + outputFileHeaders.add("Sample Identifier"); + } + + final Reader in = new FileReader(fileToBeUploaded); + final String[] headerParsed = outputFileHeaders.toArray(new String[outputFileHeaders.size()]); + + final Iterable records = + CSVFormat.TDF + .withHeader(headerParsed) + .withFirstRecordAsHeader() + .withAllowDuplicateHeaderNames() + .withFirstRecordAsHeader() + .withIgnoreEmptyLines() + .withIgnoreHeaderCase() + .withAllowMissingColumnNames() + .withIgnoreSurroundingSpaces() + .withTrim() + .parse(in); + + try (final BufferedWriter writer = Files.newBufferedWriter(temp, StandardCharsets.UTF_8); + final CSVPrinter csvPrinter = + new CSVPrinter(writer, CSVFormat.TDF.withHeader(headerParsed)); + final PrintWriter out = new PrintWriter(writer)) { + for (final CSVRecord row : records) { + if (headerHasIdentifier) { + csvPrinter.printRecord(row); + } else { + csvPrinter.printRecord( + addAccessionToSamplesForPrint(getPrintableListFromCsvRow(row.iterator()), samples)); + } + } + + out.println("\n\n"); + out.println("********RECEIPT START********"); + out.println( + validationResult.getValidationMessagesList().stream() + .map( + validationMessage -> + validationMessage.getMessageKey() + + " : " + + validationMessage.getMessageValue()) + .collect(Collectors.joining("\n"))); + out.println("********RECEIPT END********"); + out.println("\n\n"); + } + + return temp.toFile(); + } catch (final Exception e) { + log.info("Writing to file has failed " + e.getMessage(), e); + + e.printStackTrace(); + return null; + } + } + + private Iterable addAccessionToSamplesForPrint( + final List listFromIterator, final List samples) { + final String sampleName = listFromIterator.get(1); + final Optional sampleOptional = + samples.stream().filter(sample -> sample.getName().equals(sampleName)).findFirst(); + + sampleOptional.ifPresent(sample -> listFromIterator.add(sample.getAccession())); + + return listFromIterator; + } + + public CSVParser buildParser(final BufferedReader reader) throws IOException { + return new CSVParser( + reader, + CSVFormat.TDF + .withAllowDuplicateHeaderNames(true) + .withFirstRecordAsHeader() + .withIgnoreEmptyLines(true) + .withIgnoreHeaderCase(true) + .withAllowMissingColumnNames(true) + .withIgnoreSurroundingSpaces(true) + .withTrim(true)); + } + + public File writeQueueMessageToFile(final String submissionId) throws IOException { + final Path temp = Files.createTempFile("queue_result", ".txt"); + + try (final BufferedWriter writer = Files.newBufferedWriter(temp, StandardCharsets.UTF_8)) { + writer.write( + "Your submission has been queued and your submission id is " + + submissionId + + ". Please use the View Submissions tab and use your submission ID to get the submission result."); + } + + return temp.toFile(); + } + + public void validateHeaderPositions( + final List headers, final ValidationResult validationResult) { + if (!headers.isEmpty()) { + if ((!headers.get(0).equalsIgnoreCase("Source Name") + && (!headers.get(1).equalsIgnoreCase("Sample Name")) + && (!headers.get(2).equalsIgnoreCase("Release Date")))) { + validationResult.addValidationMessage( + new ValidationResult.ValidationMessage( + "GENERAL_VALIDATION_MESSAGE", + "ISA tab file must have Source Name as first column, followed by Sample Name and Release Date.", + true)); + + throw new GlobalExceptions.UploadInvalidException( + validationResult.getValidationMessagesList().stream() + .map( + validationMessage -> + validationMessage.getMessageKey() + + ":" + + validationMessage.getMessageValue()) + .collect(Collectors.joining("\n"))); + } + } + } + + public static String formatDateString(LocalDateTime dateTime) { + return dateTime.format(DATE_TIME_FORMATTER); + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/utils/upload/ValidationResult.java b/core/src/main/java/uk/ac/ebi/biosamples/utils/upload/ValidationResult.java similarity index 96% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/utils/upload/ValidationResult.java rename to core/src/main/java/uk/ac/ebi/biosamples/utils/upload/ValidationResult.java index cd27a06cb..ed8f4c661 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/utils/upload/ValidationResult.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/utils/upload/ValidationResult.java @@ -1,56 +1,56 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils.upload; - -import java.util.ArrayList; -import java.util.List; - -public class ValidationResult { - private final List validationMessagesList = new ArrayList<>(); - - public void addValidationMessage(final ValidationMessage message) { - validationMessagesList.add(message); - } - - public List getValidationMessagesList() { - return validationMessagesList; - } - - public void clear() { - validationMessagesList.clear(); - } - - public static class ValidationMessage { - private final String messageKey; - private final String messageValue; - - private final boolean error; - - public ValidationMessage( - final String messageKey, final String messageValue, final boolean error) { - this.messageKey = messageKey; - this.messageValue = messageValue; - this.error = error; - } - - public String getMessageKey() { - return messageKey; - } - - public String getMessageValue() { - return messageValue; - } - - public boolean isError() { - return error; - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.upload; + +import java.util.ArrayList; +import java.util.List; + +public class ValidationResult { + private final List validationMessagesList = new ArrayList<>(); + + public void addValidationMessage(final ValidationMessage message) { + validationMessagesList.add(message); + } + + public List getValidationMessagesList() { + return validationMessagesList; + } + + public void clear() { + validationMessagesList.clear(); + } + + public static class ValidationMessage { + private final String messageKey; + private final String messageValue; + + private final boolean error; + + public ValidationMessage( + final String messageKey, final String messageValue, final boolean error) { + this.messageKey = messageKey; + this.messageValue = messageValue; + this.error = error; + } + + public String getMessageKey() { + return messageKey; + } + + public String getMessageValue() { + return messageValue; + } + + public boolean isError() { + return error; + } + } +} diff --git a/models/curami/src/main/resources/abbreviations.csv b/core/src/main/resources/abbreviations.csv similarity index 86% rename from models/curami/src/main/resources/abbreviations.csv rename to core/src/main/resources/abbreviations.csv index afdf002d1..181c22464 100644 --- a/models/curami/src/main/resources/abbreviations.csv +++ b/core/src/main/resources/abbreviations.csv @@ -1,273 +1,273 @@ -ATTRIBUTE -NCBI -ENA -EGA -SRA -INSDC -DNA -RNA -BioSD -dbGAP -1000genomes -ADMET -ALMF -AntiFam -API -ArchSchema -ArrayExpress -ASTD -ATC -ATC -AUP -AV -AWERB -BAC -BAGMSC -BioJS -BioJS -BioMart -BioMedBridges -BioModels -BioProject -BioSamples -BioSample -BioServices -BioStudies Database -BioSource -BLAST [nucleotide] -BLAST [protein] -BoM -BSC -BSL-(1-4) -CAFM -CamSIS -CCO -CCPE -CellComm -CellNOpt -CENSOR -CGaP Comm -CGHSSg -CGP -ChEBI -ChEMBL -CHIPseq -CL3OG -CMS -CoGS -Confluence -Confluence -Converis -CORBEL -CPP -CSA -CSSB -CTTV -CVS -DAC -DAS -DB -DBA -DDBJ -DDMoRe -DESY -DGVa -DiNA -DLS -DNAseq -DoS -Dry-Lab -DSEasy -e-PTW -EBI -EMBL-EBI -EBiSC -EC -EC-PDB -ECA -EFO -EGA -EICAT -EICAT -EIPOD -EIPP -ELIXIR -ELLS -EMBL -EMBL -EMBL ATC CPP -EMBL GR -EMBL MR -EMBL-EBI -EMBL-EBI -EMBL-HD -EMBL-HH -EMBLEM -EMBO -EMBOSS -EMBRIC -EMC -EMC -EMDataBank -EMIF -EMMA -EMMA -EMTRAIN -ENA -ENA -ENA/SVA -Ensembl -EoC -EoP -EP -ERC -ERIC -eSCAMPS -ESFRI -ESPOD -ESRF -ESS -ESTA -EuroCarbDB -Europe PMC -EVA -EXCELERATE -Expression Atlas -FAANG -Faculty -FAIR Data -FGED -FP7 -FTP -GCSSC -Gen2Phen -Geuvadis -GFF -GO -GOBLET -GPCR -GPCR SARfari -GRADSAF -GRIs -GRL -GTF -GTL -GTL -GWAS -GWAS Catalog -H2020 -HAVANA -Helix -HGNC -HMDMC -HMMER -HoU -HR -HTML -HTS -HTTP -HUGO -IAA -ICRF -IM -IMeX -IMGT -IMGT/HLA -IMI -IMPC -INSDC -INSDC -IntAct -IPI -ISE -JANET -JDispatcher -JS -KEGG -KOMP2 -LRG -MAFFT -ManIC -MASSIF-1 -MEIGOR -MGP -MIG -MMPU -ModComm -MRM -MSA -MX -NCBI -NHGRI -NIH -OA Infrastructure -OIPA -OLS -OoP -OpenAIRE -ORCID -PA -PacBio Sequencing -PDB -PDBe -PDBeFold -PEPCore -PICR -PMC -Pop-up banner -PostDoc -PRIDE -ps_scan -PSB -PSI -PSI-BLAST -Pull-up banner -RA -RADAR -Reactome -Rhea -RIs -RNA-seq -RSF -SAB -SAP -SAPS -SAS -SAXS -SBO -SCM -SeqCksum -SHSC -SIFTS -SMC -SMEs -SOB -SPOT -SRS -SSH -SSMAC -SVA -SVN -SWAT -SWO -TAC -Taxonomy -TSC -UKERNA -UniChem -UniProt -URI -URL -UVHCI -VCS -VWP -WCC -WEEE -Wet-Lab -WGC -WTGC -WTSI -wwPDB -XML -ZMP +ATTRIBUTE +NCBI +ENA +EGA +SRA +INSDC +DNA +RNA +BioSD +dbGAP +1000genomes +ADMET +ALMF +AntiFam +API +ArchSchema +ArrayExpress +ASTD +ATC +ATC +AUP +AV +AWERB +BAC +BAGMSC +BioJS +BioJS +BioMart +BioMedBridges +BioModels +BioProject +BioSamples +BioSample +BioServices +BioStudies Database +BioSource +BLAST [nucleotide] +BLAST [protein] +BoM +BSC +BSL-(1-4) +CAFM +CamSIS +CCO +CCPE +CellComm +CellNOpt +CENSOR +CGaP Comm +CGHSSg +CGP +ChEBI +ChEMBL +CHIPseq +CL3OG +CMS +CoGS +Confluence +Confluence +Converis +CORBEL +CPP +CSA +CSSB +CTTV +CVS +DAC +DAS +DB +DBA +DDBJ +DDMoRe +DESY +DGVa +DiNA +DLS +DNAseq +DoS +Dry-Lab +DSEasy +e-PTW +EBI +EMBL-EBI +EBiSC +EC +EC-PDB +ECA +EFO +EGA +EICAT +EICAT +EIPOD +EIPP +ELIXIR +ELLS +EMBL +EMBL +EMBL ATC CPP +EMBL GR +EMBL MR +EMBL-EBI +EMBL-EBI +EMBL-HD +EMBL-HH +EMBLEM +EMBO +EMBOSS +EMBRIC +EMC +EMC +EMDataBank +EMIF +EMMA +EMMA +EMTRAIN +ENA +ENA +ENA/SVA +Ensembl +EoC +EoP +EP +ERC +ERIC +eSCAMPS +ESFRI +ESPOD +ESRF +ESS +ESTA +EuroCarbDB +Europe PMC +EVA +EXCELERATE +Expression Atlas +FAANG +Faculty +FAIR Data +FGED +FP7 +FTP +GCSSC +Gen2Phen +Geuvadis +GFF +GO +GOBLET +GPCR +GPCR SARfari +GRADSAF +GRIs +GRL +GTF +GTL +GTL +GWAS +GWAS Catalog +H2020 +HAVANA +Helix +HGNC +HMDMC +HMMER +HoU +HR +HTML +HTS +HTTP +HUGO +IAA +ICRF +IM +IMeX +IMGT +IMGT/HLA +IMI +IMPC +INSDC +INSDC +IntAct +IPI +ISE +JANET +JDispatcher +JS +KEGG +KOMP2 +LRG +MAFFT +ManIC +MASSIF-1 +MEIGOR +MGP +MIG +MMPU +ModComm +MRM +MSA +MX +NCBI +NHGRI +NIH +OA Infrastructure +OIPA +OLS +OoP +OpenAIRE +ORCID +PA +PacBio Sequencing +PDB +PDBe +PDBeFold +PEPCore +PICR +PMC +Pop-up banner +PostDoc +PRIDE +ps_scan +PSB +PSI +PSI-BLAST +Pull-up banner +RA +RADAR +Reactome +Rhea +RIs +RNA-seq +RSF +SAB +SAP +SAPS +SAS +SAXS +SBO +SCM +SeqCksum +SHSC +SIFTS +SMC +SMEs +SOB +SPOT +SRS +SSH +SSMAC +SVA +SVN +SWAT +SWO +TAC +Taxonomy +TSC +UKERNA +UniChem +UniProt +URI +URL +UVHCI +VCS +VWP +WCC +WEEE +Wet-Lab +WGC +WTGC +WTSI +wwPDB +XML +ZMP PubMed \ No newline at end of file diff --git a/models/curami/src/main/resources/attributes.csv b/core/src/main/resources/attributes.csv similarity index 95% rename from models/curami/src/main/resources/attributes.csv rename to core/src/main/resources/attributes.csv index 07c785f86..b2baec427 100644 --- a/models/curami/src/main/resources/attributes.csv +++ b/core/src/main/resources/attributes.csv @@ -1,1019 +1,1019 @@ -ATTRIBUTE,COUNT -organism,10670271 -INSDC status,8527784 -INSDC center name,8359205 -INSDC first public,8245159 -INSDC last update,7435715 -description title,6741070 -NCBI submission model,6599497 -NCBI submission package,5919641 -sex,4072824 -INSDC secondary accession,3863891 -tissue,3250905 -gap subject id,3132347 -study name,3119844 -study design,3119622 -submitted sample id,3119583 -submitter handle,3119575 -gap sample id,3119575 -gap accession,3119575 -biospecimen repository sample id,3119575 -biospecimen repository,3119575 -submitted subject id,3119547 -analyte type,2832324 -description,2652959 -gap consent code,2577581 -gap consent short name,2563836 -collection date,1993101 -synonym,1971398 -title,1773985 -strain,1733133 -is tumor,1658905 -geographic location,1653701 -molecular data type,1643893 -alias,1610918 -SRA accession,1606737 -ENA checklist,1504866 -subject is affected,1436901 -sample source name,1322866 -isolation source,1278916 -INSDC center alias,1255447 -histological type,1066169 -source name,1004350 -age,965126 -host,921501 -sample title,867518 -organism part,860789 -study disease,805049 -cell type,802787 -gap parent phs,686777 -isolate,648647 -latitude and longitude,567030 -sample type,546434 -genotype,543726 -collected by,541632 -broad scale environmental context,448040 -environmental medium,418170 -geographic location (country and/or sea),413996 -local scale environmental context,387588 -project name,377310 -secondary description,350108 -biomaterial provider,345933 -development stage,340959 -investigation type,314234 -geographic location (longitude),299297 -geographic location (latitude),299267 -environment (material),290203 -environment (biome),290041 -environment (feature),288971 -host subject id,285009 -sequencing method,284732 -disease state,278361 -cell line,276654 -treatment,269393 -depth,233115 -host sex,231049 -cultivar,230875 -elevation,229687 -host disease,222831 -replicate,194303 -subject id,193857 -serovar,190893 -individual,187418 -broker name,179206 -anonymized name,178294 -longitude,168775 -latitude,168628 -developmental stage,167885 -sample characteristics,161609 -env feature,160196 -env biome,159985 -disease,143254 -host age,139737 -sample name,137773 -physical specimen location,137380 -host taxid,136749 -DNA extracted,135947 -submission identifier,135859 -submission title,135859 -submission description,135857 -collection timestamp,135300 -physical specimen remaining,134995 -host common name,134195 -env package,129920 -phenotype,128218 -isolate name alias,127108 -ecotype,125912 -sample id,125545 -public,122013 -sub species,119860 -env material,118455 -geo loc name,117766 -country,112616 -host scientific name,108831 -common name,103122 -source material identifiers,96869 -ethnicity,90190 -sample collection device or method,89896 -sample derived from,88232 -assembly quality,86201 -contamination score,86047 -completeness score,86047 -metagenomic source,86030 -sample description,83785 -race,83532 -assembly software,82436 -binning software,82231 -completeness software,82230 -binning parameters,82203 -taxonomic identity marker,82147 -altitude,81861 -body site,80508 -population,78497 -body product,78478 -body habitat,77979 -breed,77826 -time point,73839 -biosourceprovider,69041 -batch,65040 -sequence type,64837 -chip antibody,60041 -env matter,59582 -assigned from geo,58930 -specific host,58895 -time,57903 -temperature,54994 -host tissue sampled,54903 -biosourcetype,54573 -bio source type,54485 -environmental sample,54332 -sample size,53820 -timepoint,51104 -scientific name,51029 -material,50936 -submission reference layer,50605 -ph,50567 -host body product,50060 -target gene,48276 -group,47815 -sample material processing,46835 -geographic location (region and locality),46315 -tissue type,45837 -diet,44872 -bmi,44281 -environmental package,44272 -human gut environmental package,43620 -geographic location (elevation),43320 -host health state,43080 -label,42962 -patient id,42822 -stage,42510 -attribute package,40728 -host status,40570 -specimen voucher,39979 -subject,39579 -well,39545 -required sample info status,39067 -age unit,39054 -has physical specimen,38775 -supplier name,38674 -antibody,38429 -geographic location (depth),38247 -growth condition,37230 -host associated environmental package,35985 -target subfragment,35981 -sample storage temperature,35268 -experiment,35260 -pcr primers,34049 -potential contaminant,34043 -soil environmental package,33842 -hhs region,33352 -specimen with known storage state,32926 -barcode,32377 -metagenomic,32074 -metagenome source,31905 -host body habitat,31797 -clinical information,31249 -condition,30927 -isolation and growth condition,30885 -species,30740 -collection time,30585 -samp size,30525 -culture collection,30389 -diagnosis,30051 -vehicle,29989 -ploidy,29351 -library name,29263 -extracted DNA avail now,29165 -weight units,29061 -site,28864 -plate,28856 -state,28749 -sampleid,28502 -age years,28474 -health state,28364 -note,28140 -empo 2,27885 -empo 1,27885 -empo 3,27885 -run,27538 -disease stage,27308 -host Taxonomy id,27227 -donor id,27030 -day,26994 -total number of reads,26602 -number of mapped reads,26560 -weight kg,26507 -reference,26426 -taxon id,26320 -diabetes,26100 -height cm,25852 -reference for biomaterial,25823 -lane,25571 -ibd,25461 -is technical control,25392 -histology,25312 -pregnant,25275 -height units,25027 -host diet,24875 -hybridization date,24832 -csection,24587 -chickenpox,24586 -gender,24262 -rna extraction date,24254 -id,24211 -genetic background,23984 -dev stage,23533 -route,23316 -location,23280 -single cell well quality,23165 -relationship to oxygen,23160 -sample,23056 -dominant hand,22951 -sampling site,22767 -barcode sequence,22433 -family id,22367 -serotype,22302 -barcodesequence,22221 -lims id,22122 -strain/background,22053 -vector,21895 -cat,21643 -dog,21633 -sleep duration,21441 -EGA sample id,21432 -EGA dataset id,21432 -biological replicate,21430 -age cat,21191 -teethbrushing frequency,21128 -pool frequency,21115 -alcohol frequency,21115 -multivitamin,21114 -cosmetics frequency,21113 -exercise frequency,21113 -appendix removed,21111 -flossing frequency,21110 -deodorant use,21108 -migraine,21108 -gluten,21108 -skin condition,21107 -lactose,21107 -tonsils removed,21103 -seasonal allergies,21102 -pku,21101 -contraceptive,21099 -weight change,21094 -country of birth,21092 -exercise location,21087 -softener,21087 -alcohol consumption,21052 -birth year,21023 -flu vaccine date,21014 -lung disease,20943 -diet type,20937 -collection month,20904 -clinical condition,20881 -acne medication,20860 -patient,20843 -subset diabetes,20817 -subset antibiotic history,20817 -subset age,20817 -subset bmi,20817 -last travel,20813 -other supplement frequency,20792 -subset ibd,20792 -subset healthy,20792 -last move,20786 -prepared meals frequency,20785 -sugar sweetened drink frequency,20784 -salted snacks frequency,20783 -liver disease,20783 -alcohol types red wine,20780 -allergic to shellfish,20780 -alcohol types white wine,20780 -alcohol types unspecified,20780 -allergic to tree nuts,20780 -non food allergies pet dander,20780 -allergic to peanuts,20780 -non food allergies beestings,20780 -non food allergies unspecified,20780 -allergic to i have no food allergies that i know of,20780 -allergic to unspecified,20780 -non food allergies sun,20780 -survey id,20780 -allergic to other,20780 -alcohol types sour beers,20780 -drinking water source,20777 -one liter of water a day frequency,20777 -smoking frequency,20776 -livingwith,20770 -acne medication otc,20769 -vitamin d supplement frequency,20769 -consume animal products abx,20765 -alzheimers,20761 -kidney disease,20761 -cardiovascular disease,20760 -homecooked meals frequency,20760 -milk cheese frequency,20760 -whole eggs,20760 -meat eggs frequency,20759 -poultry frequency,20759 -probiotic frequency,20759 -seafood frequency,20759 -epilepsy or seizure disorder,20759 -add adhd,20758 -milk substitute frequency,20758 -olive oil,20758 -ready to eat meals frequency,20758 -whole grain frequency,20758 -vegetable frequency,20757 -asd,20757 -depression bipolar schizophrenia,20757 -thyroid,20756 -sugary sweets frequency,20756 -fruit frequency,20755 -ibs,20755 -types of plants,20755 -fed as infant,20754 -autoimmune,20754 -bowel movement frequency,20754 -level of education,20754 -vitamin b supplement frequency,20754 -lowgrain diet type,20752 -red meat frequency,20752 -cdiff,20752 -high fat red meat frequency,20751 -fungal overgrowth,20750 -bowel movement quality,20748 -fermented plant frequency,20747 -breastmilk formula ensure,20745 -sibo,20745 -roommates in study,20743 -cancer,20610 -cell subtype,20602 -protocol,20589 -host genotype,20546 -diabetes type,20468 -bmi cat,20451 -ibd diagnosis,20444 -project,20441 -sample progress,20430 -collection season,20402 -cancer treatment,20373 -subgroup,20358 -country residence,20354 -ibd diagnosis refined,20353 -non food allergies drug eg penicillin,20353 -drinks per session,20353 -pets other,20353 -alcohol types beercider,20353 -non food allergies poison ivyoak,20353 -alcohol types spiritshard alcohol,20353 -mental illness,20353 -nail biter,20353 -vivid dreams,20353 -economic region,20353 -antibiotic history,20353 -age corrected,20353 -frozen dessert frequency,20353 -acid reflux,20353 -census region,20353 -artificial sweeteners,20353 -bmi corrected,20353 -specialized diet westenprice or other lowgrain low processed fo,20311 -specialized diet unspecified,20311 -mental illness type substance abuse,20286 -specialized diet paleodiet or primal diet,20286 -specialized diet fodmap,20286 -specialized diet other restrictions not described here,20286 -specialized diet i do not eat a specialized diet,20286 -mental illness type anorexia nervosa,20286 -mental illness type unspecified,20286 -specialized diet exclude refined sugars,20286 -specialized diet kosher,20286 -specialized diet raw food diet,20286 -mental illness type schizophrenia,20286 -specialized diet exclude dairy,20286 -mental illness type depression,20286 -mental illness type bulimia nervosa,20286 -specialized diet halaal,20286 -specialized diet modified paleo diet,20286 -mental illness type ptsd posttraumatic stress disorder,20286 -mental illness type bipolar disorder,20286 -specialized diet exclude nightshades,20286 -estimated size,20063 -compound,19753 -host body site,19746 -growth protocol,19667 -specialized diet,19638 -mental illness type,19638 -allergic to,19638 -non food allergies,19638 -alcohol types,19638 -age units,19627 -genus,19538 -cohort,19361 -life stage,19326 -seq methods,19304 -subtype,19280 -number of replicons,19222 -time unit,18952 -type,18931 -salinity,18757 -infection,18492 -well id,18074 -plant associated environmental package,18033 -host life stage,17966 -linkerprimersequence,17845 -donor,17615 -subjectid,17601 -biosamplemodel,17530 -body mass index,17351 -distinquish,17337 -genetic modification,17259 -clinical history,17209 -collection,17186 -public accession,17081 -source,16954 -vioscreen methion,16898 -vioscreen vitb12,16898 -vioscreen daidzein,16898 -vioscreen hei oils,16898 -vioscreen biochana,16898 -vioscreen v potato,16898 -vioscreen vita rae,16898 -vioscreen protveg,16898 -vioscreen d milk,16898 -vioscreen hei veg,16898 -vioscreen aspartam,16898 -vioscreen serine,16898 -vioscreen phosphor,16898 -vioscreen isoleuc,16898 -vioscreen rgrain,16898 -vioscreen totfolat,16898 -vioscreen adsugtot,16898 -vioscreen formontn,16898 -vioscreen v other,16898 -vioscreen totsugar,16898 -vioscreen glycine,16898 -vioscreen hei2010 fatty acids,16898 -vioscreen vitc,16898 -vioscreen galactos,16898 -vioscreen hei sat fat,16898 -vioscreen hei fruit,16898 -vioscreen sodium,16898 -vioscreen mfa181,16898 -vioscreen g total,16898 -vioscreen a bev,16898 -vioscreen f nj other,16898 -vioscreen calcium servings,16898 -vioscreen discfat oil,16898 -vioscreen v orange,16898 -vioscreen gltc,16898 -vioscreen tfa182t,16898 -vioscreen hei2010 protien foods,16898 -vioscreen v tomato,16898 -vioscreen tyrosine,16898 -vioscreen vitd iu,16898 -vioscreen hei2010 sodium,16898 -vioscreen add sug,16898 -vioscreen fish servings,16898 -vioscreen inositol,16898 -vioscreen legumes,16898 -vioscreen erythr,16898 -vioscreen hei drk g org veg leg,16898 -vioscreen glutamic,16898 -vioscreen caffeine,16898 -vioscreen histidin,16898 -vioscreen mfatot,16898 -vioscreen aspartic,16898 -vioscreen line gi,16898 -vioscreen cystine,16898 -vioscreen methhis3,16898 -vioscreen totcla,16898 -vioscreen vitd2,16898 -vioscreen salad vegetable servings,16898 -vioscreen vite iu,16898 -vioscreen water,16898 -vioscreen wgrain,16898 -vioscreen betatoco,16898 -vioscreen hei2010 dairy,16898 -vioscreen sfa160,16898 -vioscreen clac9t11,16898 -vioscreen retinol,16898 -vioscreen hei2010 sea foods plant protiens,16898 -vioscreen fried food servings,16898 -vioscreen copper,16898 -vioscreen pfa183,16898 -vioscreen hei sodium,16898 -vioscreen f other,16898 -vioscreen arginine,16898 -vioscreen protein,16898 -vioscreen m frank,16898 -vioscreen sfa120,16898 -vioscreen pfa205,16898 -vioscreen d yogurt,16898 -vioscreen vegsumm,16898 -vioscreen fibinso,16898 -vioscreen alcohol,16898 -vioscreen questionnaire,16898 -vioscreen fruit servings,16898 -vioscreen v drkgr,16898 -vioscreen carbo,16898 -vioscreen g nwhl,16898 -vioscreen mannitol,16898 -vioscreen alphtoce,16898 -vioscreen fried fish servings,16898 -vioscreen frtsumm,16898 -vioscreen sfa80,16898 -vioscreen f citmlb,16898 -vioscreen alcohol servings,16898 -vioscreen vita re,16898 -vioscreen isomalt,16898 -vioscreen alanine,16898 -vioscreen database,16898 -vioscreen pfa184,16898 -vioscreen m meat,16898 -vioscreen hei whl grains,16898 -vioscreen glac,16898 -vioscreen satoco,16898 -vioscreen tagatose,16898 -vioscreen maltose,16898 -vioscreen joules,16898 -vioscreen pfa226,16898 -vioscreen hei meat beans,16898 -vioscreen pfa204,16898 -vioscreen m fish lo,16898 -vioscreen gammtoco,16898 -vioscreen calcium from dairy servings,16898 -vioscreen nccglgr,16898 -vioscreen betacryp,16898 -vioscreen tfa161t,16898 -vioscreen vitd,16898 -vioscreen fructose,16898 -vioscreen pinitol,16898 -vioscreen m mpf,16898 -vioscreen tfa181t,16898 -vioscreen f nj total,16898 -vioscreen proline,16898 -vioscreen hei2010 whole grains,16898 -vioscreen glycitn,16898 -vioscreen whole grain servings,16898 -vioscreen v total,16898 -vioscreen sfa100,16898 -vioscreen delttoco,16898 -vioscreen pantothe,16898 -vioscreen low fat dairy serving,16898 -vioscreen zinc,16898 -vioscreen sucrlose,16898 -vioscreen selenium,16898 -vioscreen vitk,16898 -vioscreen d tot soym,16898 -vioscreen fibh2o,16898 -vioscreen oxalic,16898 -vioscreen m egg,16898 -vioscreen natoco,16898 -vioscreen ash,16898 -vioscreen sweet servings,16898 -vioscreen coumest,16898 -vioscreen sfatot,16898 -vioscreen iron,16898 -vioscreen m nutsd,16898 -vioscreen d cheese,16898 -vioscreen f total,16898 -vioscreen d total,16898 -vioscreen hei2010 fruit,16898 -vioscreen magnes,16898 -vioscreen hei score,16898 -vioscreen v starcy,16898 -vioscreen acesupot,16898 -vioscreen hei non juice frt,16898 -vioscreen hei2010 whole fruit,16898 -vioscreen pfa182,16898 -vioscreen sorbitol,16898 -vioscreen fol syn,16898 -vioscreen cholest,16898 -vioscreen nccglbr,16898 -vioscreen juice servings,16898 -vioscreen choline,16898 -vioscreen g whl,16898 -vioscreen pectins,16898 -vioscreen addsugar,16898 -vioscreen sucrose,16898 -vioscreen sfa140,16898 -vioscreen betaine,16898 -vioscreen niacin,16898 -vioscreen sacchar,16898 -vioscreen calories,16898 -vioscreen m poult,16898 -vioscreen avcarb,16898 -vioscreen clat10c12,16898 -vioscreen m organ,16898 -vioscreen vitb6,16898 -vioscreen sucpoly,16898 -vioscreen lutzeax,16898 -vioscreen vitd3,16898 -vioscreen omega3,16898 -vioscreen valine,16898 -vioscreen oxalicm,16898 -vioscreen hei sol fat alc add sug,16898 -vioscreen xylitol,16898 -vioscreen m soy,16898 -vioscreen potass,16898 -vioscreen sfa220,16898 -vioscreen lactitol,16898 -vioscreen phytic,16898 -vioscreen a cal,16898 -vioscreen lysine,16898 -vioscreen leucine,16898 -vioscreen tgrain,16898 -vioscreen mfa141,16898 -vioscreen protanim,16898 -vioscreen niacineq,16898 -vioscreen hei2010 empty calories,16898 -vioscreen lycopene,16898 -vioscreen grams,16898 -vioscreen starch,16898 -vioscreen hei grains,16898 -vioscreen veg5 day,16898 -vioscreen sfa60,16898 -vioscreen maltitol,16898 -vioscreen alphtoco,16898 -vioscreen mfa201,16898 -vioscreen lactose,16898 -vioscreen sfa180,16898 -vioscreen threonin,16898 -vioscreen m fish hi,16898 -vioscreen thiamin,16898 -vioscreen ribofla,16898 -vioscreen betacar,16898 -vioscreen fiber,16898 -vioscreen vita iu,16898 -vioscreen tryptoph,16898 -vioscreen sfa200,16898 -vioscreen fol nat,16898 -vioscreen pfatot,16898 -vioscreen pfa225,16898 -vioscreen sfa170,16898 -vioscreen sfa40,16898 -vioscreen hei milk,16898 -vioscreen nitrogen,16898 -vioscreen hei2010 veg,16898 -vioscreen alphacar,16898 -vioscreen f nj citmlb,16898 -vioscreen vegetable servings,16898 -vioscreen totaltfa,16898 -vioscreen mfa161,16898 -vioscreen glucose,16898 -vioscreen mangan,16898 -vioscreen mfa221,16898 -vioscreen fol deqv,16898 -vioscreen hei2010 score,16898 -vioscreen non fried fish servings,16898 -vioscreen hei2010 greens beans,16898 -vioscreen frt5 day,16898 -vioscreen phenylal,16898 -vioscreen fat,16898 -vioscreen hei2010 refined grains,16898 -vioscreen discfat sol,16898 -vioscreen genistn,16898 -vioscreen multivitamin,16896 -vioscreen multivitamin freq,16896 -vioscreen multi calcium avg,16896 -vioscreen multi calcium dose,16896 -vioscreen calcium avg,16892 -vioscreen calcium freq,16892 -vioscreen calcium dose,16892 -vioscreen calcium,16892 -variety,16725 -linker primer sequence,16678 -lab host,16631 -strain background,16562 -variation,16509 -stimulus,16480 -passages,16433 -fda lab id,16408 -tag,16277 -individual id,16203 -vioscreen user id,16176 -vioscreen gender,16176 -vioscreen srvid,16176 -vioscreen finished,16176 -vioscreen recno,16176 -vioscreen age,16176 -vioscreen time,16176 -vioscreen protocol,16176 -vioscreen dob,16176 -vioscreen started,16176 -vioscreen procdate,16176 -vioscreen nutrient recommendation,16176 -vioscreen visit,16176 -vioscreen weight,16167 -vioscreen email,16167 -vioscreen subject id,16167 -vioscreen height,16167 -vioscreen bmi,16167 -vioscreen bcodeid,16167 -vioscreen eer,16167 -vioscreen activity level,16167 -v type,16107 -sample volume or weight for DNA extraction,16098 -disease staging,15994 -weight,15892 -stimulation,15804 -sample number,15767 -sample comment,15740 -"geographic location (country and/or sea,region)",15630 -passage,15374 -vioscreen scfv,15368 -vioscreen scf,15368 -miscellaneous parameter,15253 -inferred cell type,15131 -host body mass index,15009 -dose,14930 -post analysis well quality,14670 -disease status,14661 -season,14659 -sub strain,14581 -status,14578 -age at diagnosis,14541 -region,14498 -initial time point,14385 -re 1,14246 -birth date,14167 -passage history,14155 -family role,14058 -propagation,13985 -study id,13871 -re 2,13465 -mating type,13185 -numbervirusreads,13126 -diseaseseverity,13126 -numberhumanuniquelymappedreads,13126 -BioSD sample,12986 -birth location,12979 -original body site annotation,12950 -multiplex identifiers,12805 -sample group,12801 -dbGAP sample,12772 -chemical administration,12762 -grade,12628 -library id,12391 -tumor stage,12363 -temp,12335 -tmp,12254 -family,12233 -has extracted data,12193 -treatment duration,12050 -visit,11841 -water environmental package,11829 -pfge primary enzyme pattern,11789 -soil type,11769 -lat lon,11756 -cage,11657 -plot,11602 -brain region,11598 -animal id,11592 -colection date,11556 -assembly method,11465 -mouse id,11361 -fraction,11180 -tumor,11074 -clinicalinformation,11042 -mapping method,10992 -quality assessment method,10992 -subsource note,10987 -age yrs,10899 -perturbation,10832 -identified by,10784 -unique id,10701 -origin,10638 -ph method,10495 -treated with,10494 -human associated environmental package,10492 -plate id,10475 -name,10428 -treatment group,10363 -illumina technology,10357 -library,10325 -year,10270 -tumor-of-origin,10252 -collector name,10237 -targeted cell type,10205 -control,10111 -line,10091 -X.SampleID,10084 -physical location,10073 -collecting institution,9999 -pool,9983 -height or length,9977 -row,9789 -physical samp avail now,9784 -specimen collection protocol,9554 -specimen collection date,9554 -animal age at collection,9536 -cd34,9497 -cd38,9494 -cd45ra,9476 -cd90,9476 -background strain,9442 -flowcell,9436 -storage conditions,9392 -tissue supergroup,9361 -host height,9340 -adapters,9309 -emp status,9287 -er status,9252 -host visit number,9203 -nitrate,9189 -week,9134 -repository,9115 -roommates,9073 -health status at collection,9027 -date,8971 -phosphate,8941 -cell cluster,8915 -sample storage location,8795 -host weight,8733 -antibiotics,8569 -rep,8548 -lab,8515 -smoking status,8450 -geneticmodification,8421 -cluster,8382 -recurated from,8272 -rin,8255 -projectid,8245 -run name,8166 -experimental factor 1,8072 -block,8071 -dataset,8040 -village,7950 -platform,7945 -observed biotic relationship,7914 -ethnic group,7818 -postnatal day,7757 -smoker,7673 -her2 status,7669 -sequencing batch,7663 -collection method,7604 -reverse primer sequence,7583 -nucleic acid extraction,7566 -tot mass,7564 -well information,7551 -oxygen,7528 -mutation,7512 -tumor type,7496 -tumor size,7477 -forward filename,7412 -reverse filename,7412 -karyotype,7411 -cre line,7378 -medium,7301 -studyid,7247 -is the sequenced pathogen host associated?,7228 -sampling time,7178 -single cell id,7078 -cd123,7063 -instrument model,7036 -molecule,6943 -affected by,6929 -generation,6884 -molecule subtype,6854 -media,6785 -total organic carbon,6779 -host description,6770 -experimental factor,6751 -number,6745 -tissue source,6734 -method,6685 -sequencing run,6671 -days in culture,6634 -agent,6607 -total nitrogen,6577 -family relationship,6542 -exposure,6534 -treatment time,6510 -project accession,6493 -tumor site,6490 -miscellaneous environmental package,6469 -phase,6444 -growth phase,6441 -experiment date,6422 -facs gating,6388 -well position,6378 -sampletype,6370 -tumor grading,6358 -host disease status,6349 -administration route,6304 -frequency of dosing,6302 -solvent,6298 -plant product,6244 -extraction method,6219 -type status,6196 -height,6193 -host phenotype,6188 -diseasestaging,6161 -total reads,6149 -cell source,6127 -sample location,6115 -primer plate,6094 -host associated,6068 -calcium,6068 -magnesium,6060 -biological specimen,6042 -risk group,6007 -protocol label,6006 -er,5981 -cell class,5958 -chip antibody vendor,5958 -current land use,5951 -index,5916 -building setting,5914 -enzyme used,5901 -dob,5880 -sampling time point,5875 -study arm,5873 -env,5872 -seqc sample,5872 -building occupancy type,5866 -host disease outcome,5800 -study center,5777 -host age units,5768 -sample site,5754 -rna amplification set,5732 -empo 0,5720 -type sample,5716 -cats,5710 -treatment compound,5709 -dissected layer,5702 -chip id,5696 -bio material,5689 -dogs,5684 -experiment center,5682 -racegendersite,5664 -nkcell,5664 -bcell,5664 -neutro,5664 -lib const meth,5634 -project id,5609 -seq meth,5593 -experimental factor 2,5573 -stress,5554 -tumor grade,5551 -shipment,5547 -replicate id,5543 -conductivity,5528 -cell gating,5510 -biosample,5508 -cell population,5467 -pcrplate,5417 -amount or size of sample collected,5412 -infect,5399 -transfection,5397 -operator variation,5372 -369 groups,5372 -Blood/NonBlood meta-groups,5372 -4 meta-groups,5372 -data source,5372 -15 meta-groups,5372 -PFGE_SecondaryEnzyme_pattern,5324 -isolation source host associated,5318 -sample source,5305 -culture condition,5293 -current vegetation,5286 -sample storage duration,5278 -biosource type,5272 -CRISPR-Cas9_nuclease,5261 -CRISPR-Cas9_gene_target,5261 -sodium,5212 -DNA id,5210 -month,5150 -chlorophyll,5144 -pam50 subtype,5117 -surface,5094 -scaffold,5090 -technical replicate,5071 -notes,5029 -donor genotype,5022 +ATTRIBUTE,COUNT +organism,10670271 +INSDC status,8527784 +INSDC center name,8359205 +INSDC first public,8245159 +INSDC last update,7435715 +description title,6741070 +NCBI submission model,6599497 +NCBI submission package,5919641 +sex,4072824 +INSDC secondary accession,3863891 +tissue,3250905 +gap subject id,3132347 +study name,3119844 +study design,3119622 +submitted sample id,3119583 +submitter handle,3119575 +gap sample id,3119575 +gap accession,3119575 +biospecimen repository sample id,3119575 +biospecimen repository,3119575 +submitted subject id,3119547 +analyte type,2832324 +description,2652959 +gap consent code,2577581 +gap consent short name,2563836 +collection date,1993101 +synonym,1971398 +title,1773985 +strain,1733133 +is tumor,1658905 +geographic location,1653701 +molecular data type,1643893 +alias,1610918 +SRA accession,1606737 +ENA checklist,1504866 +subject is affected,1436901 +sample source name,1322866 +isolation source,1278916 +INSDC center alias,1255447 +histological type,1066169 +source name,1004350 +age,965126 +host,921501 +sample title,867518 +organism part,860789 +study disease,805049 +cell type,802787 +gap parent phs,686777 +isolate,648647 +latitude and longitude,567030 +sample type,546434 +genotype,543726 +collected by,541632 +broad scale environmental context,448040 +environmental medium,418170 +geographic location (country and/or sea),413996 +local scale environmental context,387588 +project name,377310 +secondary description,350108 +biomaterial provider,345933 +development stage,340959 +investigation type,314234 +geographic location (longitude),299297 +geographic location (latitude),299267 +environment (material),290203 +environment (biome),290041 +environment (feature),288971 +host subject id,285009 +sequencing method,284732 +disease state,278361 +cell line,276654 +treatment,269393 +depth,233115 +host sex,231049 +cultivar,230875 +elevation,229687 +host disease,222831 +replicate,194303 +subject id,193857 +serovar,190893 +individual,187418 +broker name,179206 +anonymized name,178294 +longitude,168775 +latitude,168628 +developmental stage,167885 +sample characteristics,161609 +env feature,160196 +env biome,159985 +disease,143254 +host age,139737 +sample name,137773 +physical specimen location,137380 +host taxid,136749 +DNA extracted,135947 +submission identifier,135859 +submission title,135859 +submission description,135857 +collection timestamp,135300 +physical specimen remaining,134995 +host common name,134195 +env package,129920 +phenotype,128218 +isolate name alias,127108 +ecotype,125912 +sample id,125545 +public,122013 +sub species,119860 +env material,118455 +geo loc name,117766 +country,112616 +host scientific name,108831 +common name,103122 +source material identifiers,96869 +ethnicity,90190 +sample collection device or method,89896 +sample derived from,88232 +assembly quality,86201 +contamination score,86047 +completeness score,86047 +metagenomic source,86030 +sample description,83785 +race,83532 +assembly software,82436 +binning software,82231 +completeness software,82230 +binning parameters,82203 +taxonomic identity marker,82147 +altitude,81861 +body site,80508 +population,78497 +body product,78478 +body habitat,77979 +breed,77826 +time point,73839 +biosourceprovider,69041 +batch,65040 +sequence type,64837 +chip antibody,60041 +env matter,59582 +assigned from geo,58930 +specific host,58895 +time,57903 +temperature,54994 +host tissue sampled,54903 +biosourcetype,54573 +bio source type,54485 +environmental sample,54332 +sample size,53820 +timepoint,51104 +scientific name,51029 +material,50936 +submission reference layer,50605 +ph,50567 +host body product,50060 +target gene,48276 +group,47815 +sample material processing,46835 +geographic location (region and locality),46315 +tissue type,45837 +diet,44872 +bmi,44281 +environmental package,44272 +human gut environmental package,43620 +geographic location (elevation),43320 +host health state,43080 +label,42962 +patient id,42822 +stage,42510 +attribute package,40728 +host status,40570 +specimen voucher,39979 +subject,39579 +well,39545 +required sample info status,39067 +age unit,39054 +has physical specimen,38775 +supplier name,38674 +antibody,38429 +geographic location (depth),38247 +growth condition,37230 +host associated environmental package,35985 +target subfragment,35981 +sample storage temperature,35268 +experiment,35260 +pcr primers,34049 +potential contaminant,34043 +soil environmental package,33842 +hhs region,33352 +specimen with known storage state,32926 +barcode,32377 +metagenomic,32074 +metagenome source,31905 +host body habitat,31797 +clinical information,31249 +condition,30927 +isolation and growth condition,30885 +species,30740 +collection time,30585 +samp size,30525 +culture collection,30389 +diagnosis,30051 +vehicle,29989 +ploidy,29351 +library name,29263 +extracted DNA avail now,29165 +weight units,29061 +site,28864 +plate,28856 +state,28749 +sampleid,28502 +age years,28474 +health state,28364 +note,28140 +empo 2,27885 +empo 1,27885 +empo 3,27885 +run,27538 +disease stage,27308 +host Taxonomy id,27227 +donor id,27030 +day,26994 +total number of reads,26602 +number of mapped reads,26560 +weight kg,26507 +reference,26426 +taxon id,26320 +diabetes,26100 +height cm,25852 +reference for biomaterial,25823 +lane,25571 +ibd,25461 +is technical control,25392 +histology,25312 +pregnant,25275 +height units,25027 +host diet,24875 +hybridization date,24832 +csection,24587 +chickenpox,24586 +gender,24262 +rna extraction date,24254 +id,24211 +genetic background,23984 +dev stage,23533 +route,23316 +location,23280 +single cell well quality,23165 +relationship to oxygen,23160 +sample,23056 +dominant hand,22951 +sampling site,22767 +barcode sequence,22433 +family id,22367 +serotype,22302 +barcodesequence,22221 +lims id,22122 +strain/background,22053 +vector,21895 +cat,21643 +dog,21633 +sleep duration,21441 +EGA sample id,21432 +EGA dataset id,21432 +biological replicate,21430 +age cat,21191 +teethbrushing frequency,21128 +pool frequency,21115 +alcohol frequency,21115 +multivitamin,21114 +cosmetics frequency,21113 +exercise frequency,21113 +appendix removed,21111 +flossing frequency,21110 +deodorant use,21108 +migraine,21108 +gluten,21108 +skin condition,21107 +lactose,21107 +tonsils removed,21103 +seasonal allergies,21102 +pku,21101 +contraceptive,21099 +weight change,21094 +country of birth,21092 +exercise location,21087 +softener,21087 +alcohol consumption,21052 +birth year,21023 +flu vaccine date,21014 +lung disease,20943 +diet type,20937 +collection month,20904 +clinical condition,20881 +acne medication,20860 +patient,20843 +subset diabetes,20817 +subset antibiotic history,20817 +subset age,20817 +subset bmi,20817 +last travel,20813 +other supplement frequency,20792 +subset ibd,20792 +subset healthy,20792 +last move,20786 +prepared meals frequency,20785 +sugar sweetened drink frequency,20784 +salted snacks frequency,20783 +liver disease,20783 +alcohol types red wine,20780 +allergic to shellfish,20780 +alcohol types white wine,20780 +alcohol types unspecified,20780 +allergic to tree nuts,20780 +non food allergies pet dander,20780 +allergic to peanuts,20780 +non food allergies beestings,20780 +non food allergies unspecified,20780 +allergic to i have no food allergies that i know of,20780 +allergic to unspecified,20780 +non food allergies sun,20780 +survey id,20780 +allergic to other,20780 +alcohol types sour beers,20780 +drinking water source,20777 +one liter of water a day frequency,20777 +smoking frequency,20776 +livingwith,20770 +acne medication otc,20769 +vitamin d supplement frequency,20769 +consume animal products abx,20765 +alzheimers,20761 +kidney disease,20761 +cardiovascular disease,20760 +homecooked meals frequency,20760 +milk cheese frequency,20760 +whole eggs,20760 +meat eggs frequency,20759 +poultry frequency,20759 +probiotic frequency,20759 +seafood frequency,20759 +epilepsy or seizure disorder,20759 +add adhd,20758 +milk substitute frequency,20758 +olive oil,20758 +ready to eat meals frequency,20758 +whole grain frequency,20758 +vegetable frequency,20757 +asd,20757 +depression bipolar schizophrenia,20757 +thyroid,20756 +sugary sweets frequency,20756 +fruit frequency,20755 +ibs,20755 +types of plants,20755 +fed as infant,20754 +autoimmune,20754 +bowel movement frequency,20754 +level of education,20754 +vitamin b supplement frequency,20754 +lowgrain diet type,20752 +red meat frequency,20752 +cdiff,20752 +high fat red meat frequency,20751 +fungal overgrowth,20750 +bowel movement quality,20748 +fermented plant frequency,20747 +breastmilk formula ensure,20745 +sibo,20745 +roommates in study,20743 +cancer,20610 +cell subtype,20602 +protocol,20589 +host genotype,20546 +diabetes type,20468 +bmi cat,20451 +ibd diagnosis,20444 +project,20441 +sample progress,20430 +collection season,20402 +cancer treatment,20373 +subgroup,20358 +country residence,20354 +ibd diagnosis refined,20353 +non food allergies drug eg penicillin,20353 +drinks per session,20353 +pets other,20353 +alcohol types beercider,20353 +non food allergies poison ivyoak,20353 +alcohol types spiritshard alcohol,20353 +mental illness,20353 +nail biter,20353 +vivid dreams,20353 +economic region,20353 +antibiotic history,20353 +age corrected,20353 +frozen dessert frequency,20353 +acid reflux,20353 +census region,20353 +artificial sweeteners,20353 +bmi corrected,20353 +specialized diet westenprice or other lowgrain low processed fo,20311 +specialized diet unspecified,20311 +mental illness type substance abuse,20286 +specialized diet paleodiet or primal diet,20286 +specialized diet fodmap,20286 +specialized diet other restrictions not described here,20286 +specialized diet i do not eat a specialized diet,20286 +mental illness type anorexia nervosa,20286 +mental illness type unspecified,20286 +specialized diet exclude refined sugars,20286 +specialized diet kosher,20286 +specialized diet raw food diet,20286 +mental illness type schizophrenia,20286 +specialized diet exclude dairy,20286 +mental illness type depression,20286 +mental illness type bulimia nervosa,20286 +specialized diet halaal,20286 +specialized diet modified paleo diet,20286 +mental illness type ptsd posttraumatic stress disorder,20286 +mental illness type bipolar disorder,20286 +specialized diet exclude nightshades,20286 +estimated size,20063 +compound,19753 +host body site,19746 +growth protocol,19667 +specialized diet,19638 +mental illness type,19638 +allergic to,19638 +non food allergies,19638 +alcohol types,19638 +age units,19627 +genus,19538 +cohort,19361 +life stage,19326 +seq methods,19304 +subtype,19280 +number of replicons,19222 +time unit,18952 +type,18931 +salinity,18757 +infection,18492 +well id,18074 +plant associated environmental package,18033 +host life stage,17966 +linkerprimersequence,17845 +donor,17615 +subjectid,17601 +biosamplemodel,17530 +body mass index,17351 +distinquish,17337 +genetic modification,17259 +clinical history,17209 +collection,17186 +public accession,17081 +source,16954 +vioscreen methion,16898 +vioscreen vitb12,16898 +vioscreen daidzein,16898 +vioscreen hei oils,16898 +vioscreen biochana,16898 +vioscreen v potato,16898 +vioscreen vita rae,16898 +vioscreen protveg,16898 +vioscreen d milk,16898 +vioscreen hei veg,16898 +vioscreen aspartam,16898 +vioscreen serine,16898 +vioscreen phosphor,16898 +vioscreen isoleuc,16898 +vioscreen rgrain,16898 +vioscreen totfolat,16898 +vioscreen adsugtot,16898 +vioscreen formontn,16898 +vioscreen v other,16898 +vioscreen totsugar,16898 +vioscreen glycine,16898 +vioscreen hei2010 fatty acids,16898 +vioscreen vitc,16898 +vioscreen galactos,16898 +vioscreen hei sat fat,16898 +vioscreen hei fruit,16898 +vioscreen sodium,16898 +vioscreen mfa181,16898 +vioscreen g total,16898 +vioscreen a bev,16898 +vioscreen f nj other,16898 +vioscreen calcium servings,16898 +vioscreen discfat oil,16898 +vioscreen v orange,16898 +vioscreen gltc,16898 +vioscreen tfa182t,16898 +vioscreen hei2010 protien foods,16898 +vioscreen v tomato,16898 +vioscreen tyrosine,16898 +vioscreen vitd iu,16898 +vioscreen hei2010 sodium,16898 +vioscreen add sug,16898 +vioscreen fish servings,16898 +vioscreen inositol,16898 +vioscreen legumes,16898 +vioscreen erythr,16898 +vioscreen hei drk g org veg leg,16898 +vioscreen glutamic,16898 +vioscreen caffeine,16898 +vioscreen histidin,16898 +vioscreen mfatot,16898 +vioscreen aspartic,16898 +vioscreen line gi,16898 +vioscreen cystine,16898 +vioscreen methhis3,16898 +vioscreen totcla,16898 +vioscreen vitd2,16898 +vioscreen salad vegetable servings,16898 +vioscreen vite iu,16898 +vioscreen water,16898 +vioscreen wgrain,16898 +vioscreen betatoco,16898 +vioscreen hei2010 dairy,16898 +vioscreen sfa160,16898 +vioscreen clac9t11,16898 +vioscreen retinol,16898 +vioscreen hei2010 sea foods plant protiens,16898 +vioscreen fried food servings,16898 +vioscreen copper,16898 +vioscreen pfa183,16898 +vioscreen hei sodium,16898 +vioscreen f other,16898 +vioscreen arginine,16898 +vioscreen protein,16898 +vioscreen m frank,16898 +vioscreen sfa120,16898 +vioscreen pfa205,16898 +vioscreen d yogurt,16898 +vioscreen vegsumm,16898 +vioscreen fibinso,16898 +vioscreen alcohol,16898 +vioscreen questionnaire,16898 +vioscreen fruit servings,16898 +vioscreen v drkgr,16898 +vioscreen carbo,16898 +vioscreen g nwhl,16898 +vioscreen mannitol,16898 +vioscreen alphtoce,16898 +vioscreen fried fish servings,16898 +vioscreen frtsumm,16898 +vioscreen sfa80,16898 +vioscreen f citmlb,16898 +vioscreen alcohol servings,16898 +vioscreen vita re,16898 +vioscreen isomalt,16898 +vioscreen alanine,16898 +vioscreen database,16898 +vioscreen pfa184,16898 +vioscreen m meat,16898 +vioscreen hei whl grains,16898 +vioscreen glac,16898 +vioscreen satoco,16898 +vioscreen tagatose,16898 +vioscreen maltose,16898 +vioscreen joules,16898 +vioscreen pfa226,16898 +vioscreen hei meat beans,16898 +vioscreen pfa204,16898 +vioscreen m fish lo,16898 +vioscreen gammtoco,16898 +vioscreen calcium from dairy servings,16898 +vioscreen nccglgr,16898 +vioscreen betacryp,16898 +vioscreen tfa161t,16898 +vioscreen vitd,16898 +vioscreen fructose,16898 +vioscreen pinitol,16898 +vioscreen m mpf,16898 +vioscreen tfa181t,16898 +vioscreen f nj total,16898 +vioscreen proline,16898 +vioscreen hei2010 whole grains,16898 +vioscreen glycitn,16898 +vioscreen whole grain servings,16898 +vioscreen v total,16898 +vioscreen sfa100,16898 +vioscreen delttoco,16898 +vioscreen pantothe,16898 +vioscreen low fat dairy serving,16898 +vioscreen zinc,16898 +vioscreen sucrlose,16898 +vioscreen selenium,16898 +vioscreen vitk,16898 +vioscreen d tot soym,16898 +vioscreen fibh2o,16898 +vioscreen oxalic,16898 +vioscreen m egg,16898 +vioscreen natoco,16898 +vioscreen ash,16898 +vioscreen sweet servings,16898 +vioscreen coumest,16898 +vioscreen sfatot,16898 +vioscreen iron,16898 +vioscreen m nutsd,16898 +vioscreen d cheese,16898 +vioscreen f total,16898 +vioscreen d total,16898 +vioscreen hei2010 fruit,16898 +vioscreen magnes,16898 +vioscreen hei score,16898 +vioscreen v starcy,16898 +vioscreen acesupot,16898 +vioscreen hei non juice frt,16898 +vioscreen hei2010 whole fruit,16898 +vioscreen pfa182,16898 +vioscreen sorbitol,16898 +vioscreen fol syn,16898 +vioscreen cholest,16898 +vioscreen nccglbr,16898 +vioscreen juice servings,16898 +vioscreen choline,16898 +vioscreen g whl,16898 +vioscreen pectins,16898 +vioscreen addsugar,16898 +vioscreen sucrose,16898 +vioscreen sfa140,16898 +vioscreen betaine,16898 +vioscreen niacin,16898 +vioscreen sacchar,16898 +vioscreen calories,16898 +vioscreen m poult,16898 +vioscreen avcarb,16898 +vioscreen clat10c12,16898 +vioscreen m organ,16898 +vioscreen vitb6,16898 +vioscreen sucpoly,16898 +vioscreen lutzeax,16898 +vioscreen vitd3,16898 +vioscreen omega3,16898 +vioscreen valine,16898 +vioscreen oxalicm,16898 +vioscreen hei sol fat alc add sug,16898 +vioscreen xylitol,16898 +vioscreen m soy,16898 +vioscreen potass,16898 +vioscreen sfa220,16898 +vioscreen lactitol,16898 +vioscreen phytic,16898 +vioscreen a cal,16898 +vioscreen lysine,16898 +vioscreen leucine,16898 +vioscreen tgrain,16898 +vioscreen mfa141,16898 +vioscreen protanim,16898 +vioscreen niacineq,16898 +vioscreen hei2010 empty calories,16898 +vioscreen lycopene,16898 +vioscreen grams,16898 +vioscreen starch,16898 +vioscreen hei grains,16898 +vioscreen veg5 day,16898 +vioscreen sfa60,16898 +vioscreen maltitol,16898 +vioscreen alphtoco,16898 +vioscreen mfa201,16898 +vioscreen lactose,16898 +vioscreen sfa180,16898 +vioscreen threonin,16898 +vioscreen m fish hi,16898 +vioscreen thiamin,16898 +vioscreen ribofla,16898 +vioscreen betacar,16898 +vioscreen fiber,16898 +vioscreen vita iu,16898 +vioscreen tryptoph,16898 +vioscreen sfa200,16898 +vioscreen fol nat,16898 +vioscreen pfatot,16898 +vioscreen pfa225,16898 +vioscreen sfa170,16898 +vioscreen sfa40,16898 +vioscreen hei milk,16898 +vioscreen nitrogen,16898 +vioscreen hei2010 veg,16898 +vioscreen alphacar,16898 +vioscreen f nj citmlb,16898 +vioscreen vegetable servings,16898 +vioscreen totaltfa,16898 +vioscreen mfa161,16898 +vioscreen glucose,16898 +vioscreen mangan,16898 +vioscreen mfa221,16898 +vioscreen fol deqv,16898 +vioscreen hei2010 score,16898 +vioscreen non fried fish servings,16898 +vioscreen hei2010 greens beans,16898 +vioscreen frt5 day,16898 +vioscreen phenylal,16898 +vioscreen fat,16898 +vioscreen hei2010 refined grains,16898 +vioscreen discfat sol,16898 +vioscreen genistn,16898 +vioscreen multivitamin,16896 +vioscreen multivitamin freq,16896 +vioscreen multi calcium avg,16896 +vioscreen multi calcium dose,16896 +vioscreen calcium avg,16892 +vioscreen calcium freq,16892 +vioscreen calcium dose,16892 +vioscreen calcium,16892 +variety,16725 +linker primer sequence,16678 +lab host,16631 +strain background,16562 +variation,16509 +stimulus,16480 +passages,16433 +fda lab id,16408 +tag,16277 +individual id,16203 +vioscreen user id,16176 +vioscreen gender,16176 +vioscreen srvid,16176 +vioscreen finished,16176 +vioscreen recno,16176 +vioscreen age,16176 +vioscreen time,16176 +vioscreen protocol,16176 +vioscreen dob,16176 +vioscreen started,16176 +vioscreen procdate,16176 +vioscreen nutrient recommendation,16176 +vioscreen visit,16176 +vioscreen weight,16167 +vioscreen email,16167 +vioscreen subject id,16167 +vioscreen height,16167 +vioscreen bmi,16167 +vioscreen bcodeid,16167 +vioscreen eer,16167 +vioscreen activity level,16167 +v type,16107 +sample volume or weight for DNA extraction,16098 +disease staging,15994 +weight,15892 +stimulation,15804 +sample number,15767 +sample comment,15740 +"geographic location (country and/or sea,region)",15630 +passage,15374 +vioscreen scfv,15368 +vioscreen scf,15368 +miscellaneous parameter,15253 +inferred cell type,15131 +host body mass index,15009 +dose,14930 +post analysis well quality,14670 +disease status,14661 +season,14659 +sub strain,14581 +status,14578 +age at diagnosis,14541 +region,14498 +initial time point,14385 +re 1,14246 +birth date,14167 +passage history,14155 +family role,14058 +propagation,13985 +study id,13871 +re 2,13465 +mating type,13185 +numbervirusreads,13126 +diseaseseverity,13126 +numberhumanuniquelymappedreads,13126 +BioSD sample,12986 +birth location,12979 +original body site annotation,12950 +multiplex identifiers,12805 +sample group,12801 +dbGAP sample,12772 +chemical administration,12762 +grade,12628 +library id,12391 +tumor stage,12363 +temp,12335 +tmp,12254 +family,12233 +has extracted data,12193 +treatment duration,12050 +visit,11841 +water environmental package,11829 +pfge primary enzyme pattern,11789 +soil type,11769 +lat lon,11756 +cage,11657 +plot,11602 +brain region,11598 +animal id,11592 +colection date,11556 +assembly method,11465 +mouse id,11361 +fraction,11180 +tumor,11074 +clinicalinformation,11042 +mapping method,10992 +quality assessment method,10992 +subsource note,10987 +age yrs,10899 +perturbation,10832 +identified by,10784 +unique id,10701 +origin,10638 +ph method,10495 +treated with,10494 +human associated environmental package,10492 +plate id,10475 +name,10428 +treatment group,10363 +illumina technology,10357 +library,10325 +year,10270 +tumor-of-origin,10252 +collector name,10237 +targeted cell type,10205 +control,10111 +line,10091 +X.SampleID,10084 +physical location,10073 +collecting institution,9999 +pool,9983 +height or length,9977 +row,9789 +physical samp avail now,9784 +specimen collection protocol,9554 +specimen collection date,9554 +animal age at collection,9536 +cd34,9497 +cd38,9494 +cd45ra,9476 +cd90,9476 +background strain,9442 +flowcell,9436 +storage conditions,9392 +tissue supergroup,9361 +host height,9340 +adapters,9309 +emp status,9287 +er status,9252 +host visit number,9203 +nitrate,9189 +week,9134 +repository,9115 +roommates,9073 +health status at collection,9027 +date,8971 +phosphate,8941 +cell cluster,8915 +sample storage location,8795 +host weight,8733 +antibiotics,8569 +rep,8548 +lab,8515 +smoking status,8450 +geneticmodification,8421 +cluster,8382 +recurated from,8272 +rin,8255 +projectid,8245 +run name,8166 +experimental factor 1,8072 +block,8071 +dataset,8040 +village,7950 +platform,7945 +observed biotic relationship,7914 +ethnic group,7818 +postnatal day,7757 +smoker,7673 +her2 status,7669 +sequencing batch,7663 +collection method,7604 +reverse primer sequence,7583 +nucleic acid extraction,7566 +tot mass,7564 +well information,7551 +oxygen,7528 +mutation,7512 +tumor type,7496 +tumor size,7477 +forward filename,7412 +reverse filename,7412 +karyotype,7411 +cre line,7378 +medium,7301 +studyid,7247 +is the sequenced pathogen host associated?,7228 +sampling time,7178 +single cell id,7078 +cd123,7063 +instrument model,7036 +molecule,6943 +affected by,6929 +generation,6884 +molecule subtype,6854 +media,6785 +total organic carbon,6779 +host description,6770 +experimental factor,6751 +number,6745 +tissue source,6734 +method,6685 +sequencing run,6671 +days in culture,6634 +agent,6607 +total nitrogen,6577 +family relationship,6542 +exposure,6534 +treatment time,6510 +project accession,6493 +tumor site,6490 +miscellaneous environmental package,6469 +phase,6444 +growth phase,6441 +experiment date,6422 +facs gating,6388 +well position,6378 +sampletype,6370 +tumor grading,6358 +host disease status,6349 +administration route,6304 +frequency of dosing,6302 +solvent,6298 +plant product,6244 +extraction method,6219 +type status,6196 +height,6193 +host phenotype,6188 +diseasestaging,6161 +total reads,6149 +cell source,6127 +sample location,6115 +primer plate,6094 +host associated,6068 +calcium,6068 +magnesium,6060 +biological specimen,6042 +risk group,6007 +protocol label,6006 +er,5981 +cell class,5958 +chip antibody vendor,5958 +current land use,5951 +index,5916 +building setting,5914 +enzyme used,5901 +dob,5880 +sampling time point,5875 +study arm,5873 +env,5872 +seqc sample,5872 +building occupancy type,5866 +host disease outcome,5800 +study center,5777 +host age units,5768 +sample site,5754 +rna amplification set,5732 +empo 0,5720 +type sample,5716 +cats,5710 +treatment compound,5709 +dissected layer,5702 +chip id,5696 +bio material,5689 +dogs,5684 +experiment center,5682 +racegendersite,5664 +nkcell,5664 +bcell,5664 +neutro,5664 +lib const meth,5634 +project id,5609 +seq meth,5593 +experimental factor 2,5573 +stress,5554 +tumor grade,5551 +shipment,5547 +replicate id,5543 +conductivity,5528 +cell gating,5510 +biosample,5508 +cell population,5467 +pcrplate,5417 +amount or size of sample collected,5412 +infect,5399 +transfection,5397 +operator variation,5372 +369 groups,5372 +Blood/NonBlood meta-groups,5372 +4 meta-groups,5372 +data source,5372 +15 meta-groups,5372 +PFGE_SecondaryEnzyme_pattern,5324 +isolation source host associated,5318 +sample source,5305 +culture condition,5293 +current vegetation,5286 +sample storage duration,5278 +biosource type,5272 +CRISPR-Cas9_nuclease,5261 +CRISPR-Cas9_gene_target,5261 +sodium,5212 +DNA id,5210 +month,5150 +chlorophyll,5144 +pam50 subtype,5117 +surface,5094 +scaffold,5090 +technical replicate,5071 +notes,5029 +donor genotype,5022 type material,5017 \ No newline at end of file diff --git a/models/curami/src/main/resources/curations.csv b/core/src/main/resources/curations.csv similarity index 96% rename from models/curami/src/main/resources/curations.csv rename to core/src/main/resources/curations.csv index 4a73359f6..165672b40 100644 --- a/models/curami/src/main/resources/curations.csv +++ b/core/src/main/resources/curations.csv @@ -1,4 +1,4 @@ -ATTRIBUTE,CURATION -organism_part,organism part -gender,sex +ATTRIBUTE,CURATION +organism_part,organism part +gender,sex species,organism \ No newline at end of file diff --git a/utils/webapp/src/main/resources/public.explore.der b/core/src/main/resources/public.explore.der similarity index 100% rename from utils/webapp/src/main/resources/public.explore.der rename to core/src/main/resources/public.explore.der diff --git a/utils/webapp/src/main/resources/public.old.explore.der b/core/src/main/resources/public.old.explore.der similarity index 100% rename from utils/webapp/src/main/resources/public.old.explore.der rename to core/src/main/resources/public.old.explore.der diff --git a/utils/webapp/src/main/resources/public.prod.der b/core/src/main/resources/public.prod.der similarity index 100% rename from utils/webapp/src/main/resources/public.prod.der rename to core/src/main/resources/public.prod.der diff --git a/models/core/src/test/java/uk/ac/ebi/biosamples/model/AttributeTest.java b/core/src/test/java/uk/ac/ebi/biosamples/core/model/AttributeTest.java similarity index 93% rename from models/core/src/test/java/uk/ac/ebi/biosamples/model/AttributeTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/core/model/AttributeTest.java index b176b18c1..df94dfed8 100644 --- a/models/core/src/test/java/uk/ac/ebi/biosamples/model/AttributeTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/core/model/AttributeTest.java @@ -1,62 +1,62 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import static org.hamcrest.Matchers.*; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; - -import java.io.UnsupportedEncodingException; -import org.junit.Test; -import uk.ac.ebi.biosamples.service.HttpOlsUrlResolutionService; - -// @RunWith(SpringRunner.class) -// @JsonTest -public class AttributeTest { - - @Test - public void test_getIriOls_method_returns_null_if_invalid_iri_is_provided() { - final Attribute testAttribute = - Attribute.build("WrongIRIAttributeKey", "WrongIRIAttributeValue", "Something else", null); - - final HttpOlsUrlResolutionService httpOlsUrlResolutionService = - new HttpOlsUrlResolutionService(); - - assertThat(httpOlsUrlResolutionService.getIriOls(testAttribute.getIri()), nullValue()); - } - - @Test - public void test_getIriOls_method_returns_iri_if_valid_iri_is_provided() - throws UnsupportedEncodingException { - final String iri = "http://purl.obolibrary.org/obo/NCBITaxon_291302"; - final Attribute testAttribute = - Attribute.build("Organism", "Miniopterus natalensis", iri, null); - - final HttpOlsUrlResolutionService httpOlsUrlResolutionService = - new HttpOlsUrlResolutionService(); - - System.out.println(httpOlsUrlResolutionService.getIriOls(testAttribute.getIri())); - assertThat( - httpOlsUrlResolutionService.getIriOls(testAttribute.getIri()), - allOf(endsWith("NCBITaxon:291302"), startsWith("https://www.ebi.ac.uk/ols4?termId="))); - } - - @Test - public void test_getIriOls_method_returns_correctly_formatted_curie() { - final String curie = "CL:0000451"; - final Attribute testAttribute = Attribute.build("Cell type", "dendritic cell", curie, null); - - final HttpOlsUrlResolutionService httpOlsUrlResolutionService = - new HttpOlsUrlResolutionService(); - - assertEquals("CL:0000451", httpOlsUrlResolutionService.getIriOls(testAttribute.getIri())); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import java.io.UnsupportedEncodingException; +import org.junit.Test; +import uk.ac.ebi.biosamples.core.service.HttpOlsUrlResolutionService; + +// @RunWith(SpringRunner.class) +// @JsonTest +public class AttributeTest { + + @Test + public void test_getIriOls_method_returns_null_if_invalid_iri_is_provided() { + final Attribute testAttribute = + Attribute.build("WrongIRIAttributeKey", "WrongIRIAttributeValue", "Something else", null); + + final HttpOlsUrlResolutionService httpOlsUrlResolutionService = + new HttpOlsUrlResolutionService(); + + assertThat(httpOlsUrlResolutionService.getIriOls(testAttribute.getIri()), nullValue()); + } + + @Test + public void test_getIriOls_method_returns_iri_if_valid_iri_is_provided() + throws UnsupportedEncodingException { + final String iri = "http://purl.obolibrary.org/obo/NCBITaxon_291302"; + final Attribute testAttribute = + Attribute.build("Organism", "Miniopterus natalensis", iri, null); + + final HttpOlsUrlResolutionService httpOlsUrlResolutionService = + new HttpOlsUrlResolutionService(); + + System.out.println(httpOlsUrlResolutionService.getIriOls(testAttribute.getIri())); + assertThat( + httpOlsUrlResolutionService.getIriOls(testAttribute.getIri()), + allOf(endsWith("NCBITaxon:291302"), startsWith("https://www.ebi.ac.uk/ols4?termId="))); + } + + @Test + public void test_getIriOls_method_returns_correctly_formatted_curie() { + final String curie = "CL:0000451"; + final Attribute testAttribute = Attribute.build("Cell type", "dendritic cell", curie, null); + + final HttpOlsUrlResolutionService httpOlsUrlResolutionService = + new HttpOlsUrlResolutionService(); + + assertEquals("CL:0000451", httpOlsUrlResolutionService.getIriOls(testAttribute.getIri())); + } +} diff --git a/models/core/src/test/java/uk/ac/ebi/biosamples/model/ContactComparisonTest.java b/core/src/test/java/uk/ac/ebi/biosamples/core/model/ContactComparisonTest.java similarity index 96% rename from models/core/src/test/java/uk/ac/ebi/biosamples/model/ContactComparisonTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/core/model/ContactComparisonTest.java index cc4ddc167..058ccfcd4 100644 --- a/models/core/src/test/java/uk/ac/ebi/biosamples/model/ContactComparisonTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/core/model/ContactComparisonTest.java @@ -1,156 +1,156 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; - -@RunWith(SpringRunner.class) -@SpringBootTest -public class ContactComparisonTest { - - @Test - public void testComparisonOfNames() { - Contact firstContact = new Contact.Builder().firstName("Arnold").lastName("Pigeon").build(); - - Contact secondContact = new Contact.Builder().firstName("Arnold").lastName("Bigeon").build(); - - Assert.assertEquals( - firstContact.compareTo(secondContact), oldComparison(firstContact, secondContact)); - } - - @Test - public void testGeneral() { - Contact firstContact = - new Contact.Builder() - .firstName("Timmy") - .midInitials("TT") - .lastName("Tommy") - .email("tt@ttcorp.com") - .role("eater") - .affiliation("TTCorp") - .url("ttcorp.com") - .build(); - - Contact secondContact = - new Contact.Builder() - .firstName("Timmy") - .midInitials("TT") - .lastName("Tommy") - .email("tu@ttcorp.com") - .role("eater") - .affiliation("TTCorp") - .url("ttcorp.com") - .build(); - - Contact thirdContact = new Contact.Builder().email("tt@ttcorp.com").build(); - - Contact fourthContact = new Contact.Builder().email("tu@ttcorp.com").build(); - - Assert.assertEquals( - firstContact.compareTo(secondContact), oldComparison(firstContact, secondContact)); - Assert.assertEquals(firstContact.compareTo(secondContact), -1); - - Assert.assertEquals( - thirdContact.compareTo(fourthContact), oldComparison(thirdContact, fourthContact)); - } - - private int oldComparison(Contact first, Contact second) { - if (first.getFirstName() == null && second.getFirstName() != null) { - return -1; - } - if (first.getFirstName() != null && second.getFirstName() == null) { - return 1; - } - if (first.getFirstName() != null - && second.getFirstName() != null - && !first.getFirstName().equals(second.getFirstName())) { - return first.getFirstName().compareTo(second.getFirstName()); - } - - if (first.getLastName() == null && second.getLastName() != null) { - return -1; - } - if (first.getLastName() != null && second.getLastName() == null) { - return 1; - } - if (first.getLastName() != null - && second.getLastName() != null - && !first.getLastName().equals(second.getLastName())) { - return first.getLastName().compareTo(second.getLastName()); - } - - if (first.getMidInitials() == null && second.getMidInitials() != null) { - return -1; - } - if (first.getMidInitials() != null && second.getMidInitials() == null) { - return 1; - } - if (first.getMidInitials() != null - && second.getMidInitials() != null - && !first.getMidInitials().equals(second.getMidInitials())) { - return first.getMidInitials().compareTo(second.getMidInitials()); - } - - if (first.getRole() == null && second.getRole() != null) { - return -1; - } - if (first.getRole() != null && second.getRole() == null) { - return 1; - } - if (first.getRole() != null - && second.getRole() != null - && !first.getRole().equals(second.getRole())) { - return first.getRole().compareTo(second.getRole()); - } - - if (first.getEmail() == null && second.getEmail() != null) { - return -1; - } - if (first.getEmail() != null && second.getEmail() == null) { - return 1; - } - if (first.getEmail() != null - && second.getEmail() != null - && !first.getEmail().equals(second.getEmail())) { - return first.getEmail().compareTo(second.getEmail()); - } - - if (first.getAffiliation() == null && second.getAffiliation() != null) { - return -1; - } - if (first.getAffiliation() != null && second.getAffiliation() == null) { - return 1; - } - if (first.getAffiliation() != null - && second.getAffiliation() != null - && !first.getAffiliation().equals(second.getAffiliation())) { - return first.getAffiliation().compareTo(second.getAffiliation()); - } - - if (first.getUrl() == null && second.getUrl() != null) { - return -1; - } - if (first.getUrl() != null && second.getUrl() == null) { - return 1; - } - if (first.getUrl() != null - && second.getUrl() != null - && !first.getUrl().equals(second.getUrl())) { - return first.getUrl().compareTo(second.getUrl()); - } - // no differences, must be the same - return 0; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +public class ContactComparisonTest { + + @Test + public void testComparisonOfNames() { + Contact firstContact = new Contact.Builder().firstName("Arnold").lastName("Pigeon").build(); + + Contact secondContact = new Contact.Builder().firstName("Arnold").lastName("Bigeon").build(); + + Assert.assertEquals( + firstContact.compareTo(secondContact), oldComparison(firstContact, secondContact)); + } + + @Test + public void testGeneral() { + Contact firstContact = + new Contact.Builder() + .firstName("Timmy") + .midInitials("TT") + .lastName("Tommy") + .email("tt@ttcorp.com") + .role("eater") + .affiliation("TTCorp") + .url("ttcorp.com") + .build(); + + Contact secondContact = + new Contact.Builder() + .firstName("Timmy") + .midInitials("TT") + .lastName("Tommy") + .email("tu@ttcorp.com") + .role("eater") + .affiliation("TTCorp") + .url("ttcorp.com") + .build(); + + Contact thirdContact = new Contact.Builder().email("tt@ttcorp.com").build(); + + Contact fourthContact = new Contact.Builder().email("tu@ttcorp.com").build(); + + Assert.assertEquals( + firstContact.compareTo(secondContact), oldComparison(firstContact, secondContact)); + Assert.assertEquals(firstContact.compareTo(secondContact), -1); + + Assert.assertEquals( + thirdContact.compareTo(fourthContact), oldComparison(thirdContact, fourthContact)); + } + + private int oldComparison(Contact first, Contact second) { + if (first.getFirstName() == null && second.getFirstName() != null) { + return -1; + } + if (first.getFirstName() != null && second.getFirstName() == null) { + return 1; + } + if (first.getFirstName() != null + && second.getFirstName() != null + && !first.getFirstName().equals(second.getFirstName())) { + return first.getFirstName().compareTo(second.getFirstName()); + } + + if (first.getLastName() == null && second.getLastName() != null) { + return -1; + } + if (first.getLastName() != null && second.getLastName() == null) { + return 1; + } + if (first.getLastName() != null + && second.getLastName() != null + && !first.getLastName().equals(second.getLastName())) { + return first.getLastName().compareTo(second.getLastName()); + } + + if (first.getMidInitials() == null && second.getMidInitials() != null) { + return -1; + } + if (first.getMidInitials() != null && second.getMidInitials() == null) { + return 1; + } + if (first.getMidInitials() != null + && second.getMidInitials() != null + && !first.getMidInitials().equals(second.getMidInitials())) { + return first.getMidInitials().compareTo(second.getMidInitials()); + } + + if (first.getRole() == null && second.getRole() != null) { + return -1; + } + if (first.getRole() != null && second.getRole() == null) { + return 1; + } + if (first.getRole() != null + && second.getRole() != null + && !first.getRole().equals(second.getRole())) { + return first.getRole().compareTo(second.getRole()); + } + + if (first.getEmail() == null && second.getEmail() != null) { + return -1; + } + if (first.getEmail() != null && second.getEmail() == null) { + return 1; + } + if (first.getEmail() != null + && second.getEmail() != null + && !first.getEmail().equals(second.getEmail())) { + return first.getEmail().compareTo(second.getEmail()); + } + + if (first.getAffiliation() == null && second.getAffiliation() != null) { + return -1; + } + if (first.getAffiliation() != null && second.getAffiliation() == null) { + return 1; + } + if (first.getAffiliation() != null + && second.getAffiliation() != null + && !first.getAffiliation().equals(second.getAffiliation())) { + return first.getAffiliation().compareTo(second.getAffiliation()); + } + + if (first.getUrl() == null && second.getUrl() != null) { + return -1; + } + if (first.getUrl() != null && second.getUrl() == null) { + return 1; + } + if (first.getUrl() != null + && second.getUrl() != null + && !first.getUrl().equals(second.getUrl())) { + return first.getUrl().compareTo(second.getUrl()); + } + // no differences, must be the same + return 0; + } +} diff --git a/models/core/src/test/java/uk/ac/ebi/biosamples/model/CurationTest.java b/core/src/test/java/uk/ac/ebi/biosamples/core/model/CurationTest.java similarity index 95% rename from models/core/src/test/java/uk/ac/ebi/biosamples/model/CurationTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/core/model/CurationTest.java index b4a428594..ec55b01b0 100644 --- a/models/core/src/test/java/uk/ac/ebi/biosamples/model/CurationTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/core/model/CurationTest.java @@ -1,51 +1,51 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import java.util.ArrayList; -import java.util.Collection; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.test.context.junit4.SpringRunner; - -@RunWith(SpringRunner.class) -public class CurationTest { - - private final Logger log = LoggerFactory.getLogger(getClass()); - - private Curation getCuration() { - final Collection attributePre = new ArrayList<>(); - attributePre.add(Attribute.build("organism", "human")); - attributePre.add(Attribute.build("taxid", "9606")); - final Collection attributePost = new ArrayList<>(); - attributePost.add( - Attribute.build( - "organism", "Homo sapiens", "http://purl.obolibrary.org/obo/NCBITaxon_9606", null)); - return Curation.build(attributePre, attributePost); - } - - @Test - public void testEquality() { - final Curation curation1 = getCuration(); - final Curation curation2 = getCuration(); - - log.info("curation1 = " + curation1); - log.info("curation2 = " + curation2); - log.info("curation1 == curation2 " + curation1.equals(curation2)); - log.info("curation2 == curation1 " + curation2.equals(curation1)); - - Assert.assertEquals("curation objects should be equaly", curation1, curation2); - Assert.assertEquals("curation objects should be equaly", curation2, curation1); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import java.util.ArrayList; +import java.util.Collection; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +public class CurationTest { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private Curation getCuration() { + final Collection attributePre = new ArrayList<>(); + attributePre.add(Attribute.build("organism", "human")); + attributePre.add(Attribute.build("taxid", "9606")); + final Collection attributePost = new ArrayList<>(); + attributePost.add( + Attribute.build( + "organism", "Homo sapiens", "http://purl.obolibrary.org/obo/NCBITaxon_9606", null)); + return Curation.build(attributePre, attributePost); + } + + @Test + public void testEquality() { + final Curation curation1 = getCuration(); + final Curation curation2 = getCuration(); + + log.info("curation1 = " + curation1); + log.info("curation2 = " + curation2); + log.info("curation1 == curation2 " + curation1.equals(curation2)); + log.info("curation2 == curation1 " + curation2.equals(curation1)); + + Assert.assertEquals("curation objects should be equaly", curation1, curation2); + Assert.assertEquals("curation objects should be equaly", curation2, curation1); + } +} diff --git a/models/core/src/test/java/uk/ac/ebi/biosamples/model/FilterSerializationTest.java b/core/src/test/java/uk/ac/ebi/biosamples/core/model/FilterSerializationTest.java similarity index 95% rename from models/core/src/test/java/uk/ac/ebi/biosamples/model/FilterSerializationTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/core/model/FilterSerializationTest.java index 650ccf416..a7436f36f 100644 --- a/models/core/src/test/java/uk/ac/ebi/biosamples/model/FilterSerializationTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/core/model/FilterSerializationTest.java @@ -1,162 +1,162 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; - -import java.time.Instant; -import java.time.LocalDate; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.service.FilterBuilder; - -@RunWith(SpringJUnit4ClassRunner.class) -@SpringBootTest(classes = {FilterBuilder.class}) -public class FilterSerializationTest { - private final Logger log = LoggerFactory.getLogger(getClass()); - - @Autowired public FilterBuilder filterBuilder; - - @Test - public void testAttributeFilterDeserialization() { - final String stringToTest = "attr:organism:Homo sapiens"; - final Filter expectedFilter = - FilterBuilder.create().onAttribute("organism").withValue("Homo sapiens").build(); - - final Filter attributeFilter = FilterBuilder.create().buildFromString(stringToTest); - assertEquals(attributeFilter, expectedFilter); - } - - @Test - public void testFromLocalDateFilterDeserialization() { - final String stringToTest = "dt:update:from=2017-01-10"; - final Instant from = ZonedDateTime.of(2017, 1, 10, 0, 0, 0, 0, ZoneOffset.UTC).toInstant(); - - final Filter expectedFilter = FilterBuilder.create().onUpdateDate().from(from).build(); - final Filter dateRangeFilter = FilterBuilder.create().buildFromString(stringToTest); - assertEquals(dateRangeFilter, expectedFilter); - } - - @Test - public void testDateRangeFromLocalDateTimeToLocalDateFilterDeserialization() { - final String stringToTest = "dt:release:from=2014-01-01T20:30:00until=2015-01-01"; - final Instant from = ZonedDateTime.of(2014, 1, 1, 20, 30, 0, 0, ZoneOffset.UTC).toInstant(); - final Instant until = - LocalDate.of(2015, 1, 1).atStartOfDay().plusDays(1).toInstant(ZoneOffset.UTC); - final Filter expectedFilter = - FilterBuilder.create().onReleaseDate().from(from).until(until).build(); - final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); - log.info( - "testDateRangeFromLocalDateTimeToLocalDateFilterDeserialization expected = " - + expectedFilter.getSerialization()); - log.info( - "testDateRangeFromLocalDateTimeToLocalDateFilterDeserialization actual = " - + actualFilter.getSerialization()); - assertEquals(actualFilter, expectedFilter); - } - - @Test - public void testInvertedDateRangeFilterDeserialization() { - final String stringToTest = "dt:update:until=2018-01-01from=2016-01-01"; - final Instant from = LocalDate.of(2016, 1, 1).atStartOfDay().toInstant(ZoneOffset.UTC); - final Instant until = - LocalDate.of(2018, 1, 1).atStartOfDay().plusDays(1).toInstant(ZoneOffset.UTC); - final Filter expectedFilter = - FilterBuilder.create().onUpdateDate().from(from).until(until).build(); - final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); - log.info( - "testInvertedDateRangeFilterDeserialization expected = " - + expectedFilter.getSerialization()); - log.info( - "testInvertedDateRangeFilterDeserialization actual = " + actualFilter.getSerialization()); - assertEquals(actualFilter, expectedFilter); - } - - @Test - public void testAccessionFilterSerialization() { - final String stringToTest = "acc:SAMEA123123"; - final Filter expectedFilter = FilterBuilder.create().onAccession("SAMEA123123").build(); - final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); - assertEquals(expectedFilter, actualFilter); - } - - @Test - public void testExternalReferenceFilterSerialization() { - final String stringToTest = "extd:ENA:E-MTAB-123123"; - final Filter expectedFilter = - FilterBuilder.create() - .onDataFromExternalReference("ENA") - .withValue("E-MTAB-123123") - .build(); - final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); - assertEquals(expectedFilter, actualFilter); - } - - @Test - public void testRelationFilterSerialization() { - final String stringToTest = "rel:derive From:SAMEA123123"; - final Filter expectedFilter = - FilterBuilder.create().onRelation("derive From").withValue("SAMEA123123").build(); - final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); - assertEquals(expectedFilter, actualFilter); - } - - @Test - public void testInverseRelationFilterSerialization() { - final String stringToTest = "rrel:derive From:SAMEA123123"; - final Filter expectedFilter = - FilterBuilder.create().onInverseRelation("derive From").withValue("SAMEA123123").build(); - final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); - assertEquals(expectedFilter, actualFilter); - } - - @Test - public void testNameFilterSerialization() { - final String stringToTest = "name:Test filter 2"; - final Filter expectedFilter = FilterBuilder.create().onName("Test filter 2").build(); - final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); - assertEquals(expectedFilter, actualFilter); - } - - @Test - public void testWildcardFilterSerialization() { - final String stringToTest = "name:Test filter *"; - final Filter expectedFilter = FilterBuilder.create().onName("Test filter *").build(); - final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); - assertEquals(expectedFilter, actualFilter); - } - - @Test - public void testEscapeSerialization() { - final String stringToTest = "attr:(?\\:O)organism"; - final Filter expectedFilter = FilterBuilder.create().onAttribute("(?:O)organism").build(); - final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); - assertEquals(expectedFilter, actualFilter); - } - - @Test - public void testForDifferentFiltersWithEscapedCharacters() { - final String stringToTest = "attr:(?\\:O)rganism"; - final Filter wrongDeserializedFilter = - FilterBuilder.create().onAttribute("(?").withValue("O)rganism").build(); - final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); - assertNotEquals(wrongDeserializedFilter, actualFilter); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.service.FilterBuilder; + +@RunWith(SpringJUnit4ClassRunner.class) +@SpringBootTest(classes = {FilterBuilder.class}) +public class FilterSerializationTest { + private final Logger log = LoggerFactory.getLogger(getClass()); + + @Autowired public FilterBuilder filterBuilder; + + @Test + public void testAttributeFilterDeserialization() { + final String stringToTest = "attr:organism:Homo sapiens"; + final Filter expectedFilter = + FilterBuilder.create().onAttribute("organism").withValue("Homo sapiens").build(); + + final Filter attributeFilter = FilterBuilder.create().buildFromString(stringToTest); + assertEquals(attributeFilter, expectedFilter); + } + + @Test + public void testFromLocalDateFilterDeserialization() { + final String stringToTest = "dt:update:from=2017-01-10"; + final Instant from = ZonedDateTime.of(2017, 1, 10, 0, 0, 0, 0, ZoneOffset.UTC).toInstant(); + + final Filter expectedFilter = FilterBuilder.create().onUpdateDate().from(from).build(); + final Filter dateRangeFilter = FilterBuilder.create().buildFromString(stringToTest); + assertEquals(dateRangeFilter, expectedFilter); + } + + @Test + public void testDateRangeFromLocalDateTimeToLocalDateFilterDeserialization() { + final String stringToTest = "dt:release:from=2014-01-01T20:30:00until=2015-01-01"; + final Instant from = ZonedDateTime.of(2014, 1, 1, 20, 30, 0, 0, ZoneOffset.UTC).toInstant(); + final Instant until = + LocalDate.of(2015, 1, 1).atStartOfDay().plusDays(1).toInstant(ZoneOffset.UTC); + final Filter expectedFilter = + FilterBuilder.create().onReleaseDate().from(from).until(until).build(); + final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); + log.info( + "testDateRangeFromLocalDateTimeToLocalDateFilterDeserialization expected = " + + expectedFilter.getSerialization()); + log.info( + "testDateRangeFromLocalDateTimeToLocalDateFilterDeserialization actual = " + + actualFilter.getSerialization()); + assertEquals(actualFilter, expectedFilter); + } + + @Test + public void testInvertedDateRangeFilterDeserialization() { + final String stringToTest = "dt:update:until=2018-01-01from=2016-01-01"; + final Instant from = LocalDate.of(2016, 1, 1).atStartOfDay().toInstant(ZoneOffset.UTC); + final Instant until = + LocalDate.of(2018, 1, 1).atStartOfDay().plusDays(1).toInstant(ZoneOffset.UTC); + final Filter expectedFilter = + FilterBuilder.create().onUpdateDate().from(from).until(until).build(); + final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); + log.info( + "testInvertedDateRangeFilterDeserialization expected = " + + expectedFilter.getSerialization()); + log.info( + "testInvertedDateRangeFilterDeserialization actual = " + actualFilter.getSerialization()); + assertEquals(actualFilter, expectedFilter); + } + + @Test + public void testAccessionFilterSerialization() { + final String stringToTest = "acc:SAMEA123123"; + final Filter expectedFilter = FilterBuilder.create().onAccession("SAMEA123123").build(); + final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); + assertEquals(expectedFilter, actualFilter); + } + + @Test + public void testExternalReferenceFilterSerialization() { + final String stringToTest = "extd:ENA:E-MTAB-123123"; + final Filter expectedFilter = + FilterBuilder.create() + .onDataFromExternalReference("ENA") + .withValue("E-MTAB-123123") + .build(); + final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); + assertEquals(expectedFilter, actualFilter); + } + + @Test + public void testRelationFilterSerialization() { + final String stringToTest = "rel:derive From:SAMEA123123"; + final Filter expectedFilter = + FilterBuilder.create().onRelation("derive From").withValue("SAMEA123123").build(); + final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); + assertEquals(expectedFilter, actualFilter); + } + + @Test + public void testInverseRelationFilterSerialization() { + final String stringToTest = "rrel:derive From:SAMEA123123"; + final Filter expectedFilter = + FilterBuilder.create().onInverseRelation("derive From").withValue("SAMEA123123").build(); + final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); + assertEquals(expectedFilter, actualFilter); + } + + @Test + public void testNameFilterSerialization() { + final String stringToTest = "name:Test filter 2"; + final Filter expectedFilter = FilterBuilder.create().onName("Test filter 2").build(); + final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); + assertEquals(expectedFilter, actualFilter); + } + + @Test + public void testWildcardFilterSerialization() { + final String stringToTest = "name:Test filter *"; + final Filter expectedFilter = FilterBuilder.create().onName("Test filter *").build(); + final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); + assertEquals(expectedFilter, actualFilter); + } + + @Test + public void testEscapeSerialization() { + final String stringToTest = "attr:(?\\:O)organism"; + final Filter expectedFilter = FilterBuilder.create().onAttribute("(?:O)organism").build(); + final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); + assertEquals(expectedFilter, actualFilter); + } + + @Test + public void testForDifferentFiltersWithEscapedCharacters() { + final String stringToTest = "attr:(?\\:O)rganism"; + final Filter wrongDeserializedFilter = + FilterBuilder.create().onAttribute("(?").withValue("O)rganism").build(); + final Filter actualFilter = FilterBuilder.create().buildFromString(stringToTest); + assertNotEquals(wrongDeserializedFilter, actualFilter); + } +} diff --git a/models/core/src/test/java/uk/ac/ebi/biosamples/model/RelationshipValidationTests.java b/core/src/test/java/uk/ac/ebi/biosamples/core/model/RelationshipValidationTests.java similarity index 91% rename from models/core/src/test/java/uk/ac/ebi/biosamples/model/RelationshipValidationTests.java rename to core/src/test/java/uk/ac/ebi/biosamples/core/model/RelationshipValidationTests.java index 5865ef02e..c650f7b4a 100644 --- a/models/core/src/test/java/uk/ac/ebi/biosamples/model/RelationshipValidationTests.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/core/model/RelationshipValidationTests.java @@ -1,38 +1,38 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.Collection; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.test.context.junit4.SpringRunner; -import uk.ac.ebi.biosamples.service.RelationshipValidator; - -@RunWith(SpringRunner.class) -public class RelationshipValidationTests { - - private final RelationshipValidator relationshipValidator; - - public RelationshipValidationTests() { - relationshipValidator = new RelationshipValidator(); - } - - @Test - public void throws_exception_if_relationship_source_or_target_are_not_accessions() { - final Relationship rel = Relationship.build("Animal", "derivedFrom", "SAMEG1234123"); - final Collection errors = relationshipValidator.validate(rel); - assertThat(errors) - .containsExactly( - "Source of a relationship must be an accession but was \"" + rel.getSource() + "\""); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Collection; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.junit4.SpringRunner; +import uk.ac.ebi.biosamples.core.service.RelationshipValidator; + +@RunWith(SpringRunner.class) +public class RelationshipValidationTests { + + private final RelationshipValidator relationshipValidator; + + public RelationshipValidationTests() { + relationshipValidator = new RelationshipValidator(); + } + + @Test + public void throws_exception_if_relationship_source_or_target_are_not_accessions() { + final Relationship rel = Relationship.build("Animal", "derivedFrom", "SAMEG1234123"); + final Collection errors = relationshipValidator.validate(rel); + assertThat(errors) + .containsExactly( + "Source of a relationship must be an accession but was \"" + rel.getSource() + "\""); + } +} diff --git a/models/core/src/test/java/uk/ac/ebi/biosamples/model/SampleBuilderTest.java b/core/src/test/java/uk/ac/ebi/biosamples/core/model/SampleBuilderTest.java similarity index 93% rename from models/core/src/test/java/uk/ac/ebi/biosamples/model/SampleBuilderTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/core/model/SampleBuilderTest.java index 6b72cad1b..49a8f72e3 100644 --- a/models/core/src/test/java/uk/ac/ebi/biosamples/model/SampleBuilderTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/core/model/SampleBuilderTest.java @@ -1,25 +1,25 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import static org.assertj.core.api.Java6Assertions.assertThat; - -import org.junit.Test; - -public class SampleBuilderTest { - - @Test - public void sample_build_even_if_null_data_is_provided() { - final Sample sample = new Sample.Builder("TestSample").withData(null).build(); - - assertThat(sample.getName()).isEqualTo("TestSample"); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import static org.assertj.core.api.Java6Assertions.assertThat; + +import org.junit.Test; + +public class SampleBuilderTest { + + @Test + public void sample_build_even_if_null_data_is_provided() { + final Sample sample = new Sample.Builder("TestSample").withData(null).build(); + + assertThat(sample.getName()).isEqualTo("TestSample"); + } +} diff --git a/models/core/src/test/java/uk/ac/ebi/biosamples/model/SampleTaxIdTest.java b/core/src/test/java/uk/ac/ebi/biosamples/core/model/SampleTaxIdTest.java similarity index 96% rename from models/core/src/test/java/uk/ac/ebi/biosamples/model/SampleTaxIdTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/core/model/SampleTaxIdTest.java index 042b19a80..df49249c5 100644 --- a/models/core/src/test/java/uk/ac/ebi/biosamples/model/SampleTaxIdTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/core/model/SampleTaxIdTest.java @@ -1,106 +1,106 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import static junit.framework.TestCase.assertTrue; - -import java.time.Instant; -import java.util.*; -import org.junit.Test; - -public class SampleTaxIdTest { - - @Test - public void given_single_ontologyTerm_return_taxId() { - final String olsValue = "http://purl.obolibrary.org/obo/NCBITaxon_10116"; - final Attribute attribute = - Attribute.build("Organism", "", null, Collections.singletonList(olsValue), null); - final Sample sample = generateTestSample(Collections.singletonList(attribute)); - assertTrue(10116 == sample.getTaxId()); - } - - @Test - public void given_single_ontologyTerm_return_taxId_with_lowercase_organism() { - final String olsValue = "http://purl.obolibrary.org/obo/NCBITaxon_9685"; - final Attribute attribute = - Attribute.build("organism", "Felis catu", null, Collections.singletonList(olsValue), null); - final Sample sample = generateTestSample(Collections.singletonList(attribute)); - assertTrue(9685 == sample.getTaxId()); - } - - @Test - public void given_an_Organism_with_multiple_entries() { - final List olsValues = - Arrays.asList( - "http://purl.obolibrary.org/obo/NCBITaxon_10116", - "http://purl.obolibrary.org/obo/NCBITaxon_9685"); - final Attribute attribute = Attribute.build("Organism", "Felis catu", null, olsValues, null); - final Sample sample = generateTestSample(Collections.singletonList(attribute)); - assertTrue(10116 == sample.getTaxId()); - } - - @Test - public void given_multiple_Organisms() { - final String olsValue1 = "http://purl.obolibrary.org/obo/NCBITaxon_10116"; - final String olsValue2 = "http://purl.obolibrary.org/obo/NCBITaxon_9685"; - final Attribute attribute1 = Attribute.build("Organism", "", olsValue1, null); - final Attribute attribute2 = Attribute.build("Organism", "", olsValue2, null); - final Sample sample = generateTestSample(Arrays.asList(attribute1, attribute2)); - assertTrue(10116 == sample.getTaxId()); - } - - @Test - public void given_single_ontologyTerm_return_taxId_with_empty_iri() { - final String olsValue = ""; - final Attribute attribute = - Attribute.build("Organism", "", null, Collections.singletonList(olsValue), null); - final Sample sample = generateTestSample(Collections.singletonList(attribute)); - assertTrue(null == sample.getTaxId()); - } - - @Test - public void given_9606_ontologyTerm_return_taxId() { - final String value = "9606"; - final Attribute attribute = - Attribute.build("Organism", "", null, Collections.singletonList(value), null); - final Sample sample = generateTestSample(Collections.singletonList(attribute)); - assertTrue(9606 == sample.getTaxId()); - } - - @Test - public void given_no_ontologyTerm_return_unknown_taxId() { - final Attribute attribute = - Attribute.build("Organism", "s", null, Collections.EMPTY_LIST, null); - final Sample sample = generateTestSample(Collections.singletonList(attribute)); - assertTrue(null == sample.getTaxId()); - } - - private Sample generateTestSample(final List attributes) { - final Set attributeSet = new HashSet<>(attributes); - - return Sample.build( - "", - "", - "", - "", - "", - null, - null, - Instant.now(), - Instant.now(), - Instant.now(), - Instant.now(), - Instant.now(), - attributeSet, - Collections.emptySet(), - Collections.emptySet()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import static junit.framework.TestCase.assertTrue; + +import java.time.Instant; +import java.util.*; +import org.junit.Test; + +public class SampleTaxIdTest { + + @Test + public void given_single_ontologyTerm_return_taxId() { + final String olsValue = "http://purl.obolibrary.org/obo/NCBITaxon_10116"; + final Attribute attribute = + Attribute.build("Organism", "", null, Collections.singletonList(olsValue), null); + final Sample sample = generateTestSample(Collections.singletonList(attribute)); + assertTrue(10116 == sample.getTaxId()); + } + + @Test + public void given_single_ontologyTerm_return_taxId_with_lowercase_organism() { + final String olsValue = "http://purl.obolibrary.org/obo/NCBITaxon_9685"; + final Attribute attribute = + Attribute.build("organism", "Felis catu", null, Collections.singletonList(olsValue), null); + final Sample sample = generateTestSample(Collections.singletonList(attribute)); + assertTrue(9685 == sample.getTaxId()); + } + + @Test + public void given_an_Organism_with_multiple_entries() { + final List olsValues = + Arrays.asList( + "http://purl.obolibrary.org/obo/NCBITaxon_10116", + "http://purl.obolibrary.org/obo/NCBITaxon_9685"); + final Attribute attribute = Attribute.build("Organism", "Felis catu", null, olsValues, null); + final Sample sample = generateTestSample(Collections.singletonList(attribute)); + assertTrue(10116 == sample.getTaxId()); + } + + @Test + public void given_multiple_Organisms() { + final String olsValue1 = "http://purl.obolibrary.org/obo/NCBITaxon_10116"; + final String olsValue2 = "http://purl.obolibrary.org/obo/NCBITaxon_9685"; + final Attribute attribute1 = Attribute.build("Organism", "", olsValue1, null); + final Attribute attribute2 = Attribute.build("Organism", "", olsValue2, null); + final Sample sample = generateTestSample(Arrays.asList(attribute1, attribute2)); + assertTrue(10116 == sample.getTaxId()); + } + + @Test + public void given_single_ontologyTerm_return_taxId_with_empty_iri() { + final String olsValue = ""; + final Attribute attribute = + Attribute.build("Organism", "", null, Collections.singletonList(olsValue), null); + final Sample sample = generateTestSample(Collections.singletonList(attribute)); + assertTrue(null == sample.getTaxId()); + } + + @Test + public void given_9606_ontologyTerm_return_taxId() { + final String value = "9606"; + final Attribute attribute = + Attribute.build("Organism", "", null, Collections.singletonList(value), null); + final Sample sample = generateTestSample(Collections.singletonList(attribute)); + assertTrue(9606 == sample.getTaxId()); + } + + @Test + public void given_no_ontologyTerm_return_unknown_taxId() { + final Attribute attribute = + Attribute.build("Organism", "s", null, Collections.EMPTY_LIST, null); + final Sample sample = generateTestSample(Collections.singletonList(attribute)); + assertTrue(null == sample.getTaxId()); + } + + private Sample generateTestSample(final List attributes) { + final Set attributeSet = new HashSet<>(attributes); + + return Sample.build( + "", + "", + "", + "", + "", + null, + null, + Instant.now(), + Instant.now(), + Instant.now(), + Instant.now(), + Instant.now(), + attributeSet, + Collections.emptySet(), + Collections.emptySet()); + } +} diff --git a/models/core/src/test/java/uk/ac/ebi/biosamples/model/SerializationTest.java b/core/src/test/java/uk/ac/ebi/biosamples/core/model/SerializationTest.java similarity index 95% rename from models/core/src/test/java/uk/ac/ebi/biosamples/model/SerializationTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/core/model/SerializationTest.java index d4d605bfe..9880e7659 100644 --- a/models/core/src/test/java/uk/ac/ebi/biosamples/model/SerializationTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/core/model/SerializationTest.java @@ -1,198 +1,198 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Sets; -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.SortedSet; -import java.util.TreeSet; -import org.assertj.core.util.Lists; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.test.autoconfigure.json.JsonTest; -import org.springframework.boot.test.json.JacksonTester; -import org.springframework.core.io.ClassPathResource; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - -@RunWith(SpringRunner.class) -@JsonTest -// @TestPropertySource(properties = -// {"spring.jackson.serialization.WRITE_DATES_AS_TIMESTAMPS=false","spring.jackson.serialization.WRITE_NULL_MAP_VALUES=false"}) -@TestPropertySource(properties = {"spring.jackson.serialization.INDENT_OUTPUT=true"}) -public class SerializationTest { - - private final Logger log = LoggerFactory.getLogger(getClass()); - - private JacksonTester json; - - @Before - public void setup() { - final ObjectMapper objectMapper = new ObjectMapper(); - JacksonTester.initFields(this, objectMapper); - } - - private Sample getSimpleSample() { - final String name = "Test Sample"; - final String accession = "SAMEA1234"; - final String domain = "abcde12345"; - final Instant update = Instant.parse("2016-05-05T11:36:57.00Z"); - final Instant release = Instant.parse("2016-04-01T11:36:57.00Z"); - final Instant submitted = Instant.parse("2016-04-01T11:36:57.00Z"); - - final SortedSet attributes = new TreeSet<>(); - attributes.add( - Attribute.build( - "organism", "Homo sapiens", "http://purl.obolibrary.org/obo/NCBITaxon_9606", null)); - attributes.add(Attribute.build("age", "3", null, Lists.emptyList(), "year")); - attributes.add(Attribute.build("organism part", "lung")); - attributes.add(Attribute.build("organism part", "heart")); - attributes.add( - Attribute.build( - "sex", - "female", - null, - Sets.newHashSet( - "http://purl.obolibrary.org/obo/PATO_0000383", - "http://www.ebi.ac.uk/efo/EFO_0001265"), - null)); - - final SortedSet relationships = new TreeSet<>(); - relationships.add(Relationship.build("SAMEA1234", "derived from", "SAMD4321")); - - final SortedSet externalReferences = new TreeSet<>(); - externalReferences.add(ExternalReference.build("http://www.google.com")); - - final SortedSet organizations = new TreeSet<>(); - // organizations.add(Organization.build("Jo Bloggs Inc", "user", "help@jobloggs.com", - // "http://www.jobloggs.com")); - organizations.add( - new Organization.Builder() - .name("Jo Bloggs Inc") - .role("user") - .email("help@jobloggs.com") - .url("http://www.jobloggs.com") - .build()); - - final SortedSet contacts = new TreeSet<>(); - // contacts.add(Contact.build("Joe Bloggs","Jo Bloggs Inc", - // "http://www.jobloggs.com/joe")); - contacts.add( - new Contact.Builder() - .firstName("Joe") - .lastName("Bloggs") - // .affiliation("Jo Bloggs Inc") - // .url("http://www.jobloggs.com/joe") - .name("Joe Bloggs") - .role("Submitter") - .email("jobloggs@joblogs.com") - .build()); - - final SortedSet publications = new TreeSet<>(); - // publications.add(Publication.build("10.1093/nar/gkt1081", "24265224")); - publications.add( - new Publication.Builder().doi("10.1093/nar/gkt1081").pubmed_id("24265224").build()); - - // return Sample.build(name, accession, domain, release, update, attributes, relationships, - // externalReferences, organizations, contacts, publications); - return new Sample.Builder(name, accession) - .withDomain(domain) - .withRelease(release) - .withUpdate(update) - .withSubmitted(submitted) - .withAttributes(attributes) - .withRelationships(relationships) - .withExternalReferences(externalReferences) - .withOrganizations(organizations) - .withContacts(contacts) - .withPublications(publications) - .build(); - } - - @Test - public void testSerialize() throws Exception { - final Sample details = getSimpleSample(); - - log.info(json.write(details).getJson()); - - // Use JSON path based assertions - assertThat(json.write(details)).hasJsonPathStringValue("@.accession"); - assertThat(json.write(details)) - .extractingJsonPathStringValue("@.accession") - .isEqualTo("SAMEA1234"); - - // Assert against a `.json` file in the same package as the test - assertThat(json.write(details)).isEqualToJson("/TEST1.json"); - } - - @Test - public void testDeserialize() throws Exception { - final Sample fileSample = json.readObject("/TEST1.json"); - final Sample simpleSample = getSimpleSample(); - log.info("fileSample = " + fileSample); - log.info("simpleSample = " + simpleSample); - // Use JSON path based assertions - assertThat(fileSample.getName()).isEqualTo("Test Sample"); - assertThat(fileSample.getAccession()).isEqualTo("SAMEA1234"); - // Assert against a `.json` file - assertThat(fileSample).isEqualTo(simpleSample); - - // check that a specific attribute exists - assertThat(fileSample.getCharacteristics().contains(Attribute.build("organism part", "heart"))); - } - - @Test - public void testRoundTrip() throws Exception { - final Sample sample = getSimpleSample(); - log.info("roundTrip sample = " + sample); - - final String json = this.json.write(sample).getJson(); - log.info("roundTrip json = " + json); - - final InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8)); - final Sample sampleRedux = this.json.readObject(inputStream); - log.info("roundTrip sampleRedux = " + sampleRedux); - - final String jsonRedux = this.json.write(sampleRedux).getJson(); - log.info("roundTrip jsonRedux = " + jsonRedux); - - final BufferedReader br = - new BufferedReader( - new InputStreamReader(new ClassPathResource("/TEST1.json").getInputStream()), 1024); - final StringBuilder stringBuilder = new StringBuilder(); - String line; - while ((line = br.readLine()) != null) { - stringBuilder.append(line).append('\n'); - } - br.close(); - final String jsonFile = stringBuilder.toString(); - - assertThat(sample.equals(sampleRedux)); - assertThat(sample.equals(jsonFile)); - assertThat(json.equals(jsonRedux)); - assertThat(json.equals(jsonFile)); - } - - @SpringBootConfiguration - public static class TestConfig {} -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.core.model; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.SortedSet; +import java.util.TreeSet; +import org.assertj.core.util.Lists; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.test.autoconfigure.json.JsonTest; +import org.springframework.boot.test.json.JacksonTester; +import org.springframework.core.io.ClassPathResource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@JsonTest +// @TestPropertySource(properties = +// {"spring.jackson.serialization.WRITE_DATES_AS_TIMESTAMPS=false","spring.jackson.serialization.WRITE_NULL_MAP_VALUES=false"}) +@TestPropertySource(properties = {"spring.jackson.serialization.INDENT_OUTPUT=true"}) +public class SerializationTest { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private JacksonTester json; + + @Before + public void setup() { + final ObjectMapper objectMapper = new ObjectMapper(); + JacksonTester.initFields(this, objectMapper); + } + + private Sample getSimpleSample() { + final String name = "Test Sample"; + final String accession = "SAMEA1234"; + final String domain = "abcde12345"; + final Instant update = Instant.parse("2016-05-05T11:36:57.00Z"); + final Instant release = Instant.parse("2016-04-01T11:36:57.00Z"); + final Instant submitted = Instant.parse("2016-04-01T11:36:57.00Z"); + + final SortedSet attributes = new TreeSet<>(); + attributes.add( + Attribute.build( + "organism", "Homo sapiens", "http://purl.obolibrary.org/obo/NCBITaxon_9606", null)); + attributes.add(Attribute.build("age", "3", null, Lists.emptyList(), "year")); + attributes.add(Attribute.build("organism part", "lung")); + attributes.add(Attribute.build("organism part", "heart")); + attributes.add( + Attribute.build( + "sex", + "female", + null, + Sets.newHashSet( + "http://purl.obolibrary.org/obo/PATO_0000383", + "http://www.ebi.ac.uk/efo/EFO_0001265"), + null)); + + final SortedSet relationships = new TreeSet<>(); + relationships.add(Relationship.build("SAMEA1234", "derived from", "SAMD4321")); + + final SortedSet externalReferences = new TreeSet<>(); + externalReferences.add(ExternalReference.build("http://www.google.com")); + + final SortedSet organizations = new TreeSet<>(); + // organizations.add(Organization.build("Jo Bloggs Inc", "user", "help@jobloggs.com", + // "http://www.jobloggs.com")); + organizations.add( + new Organization.Builder() + .name("Jo Bloggs Inc") + .role("user") + .email("help@jobloggs.com") + .url("http://www.jobloggs.com") + .build()); + + final SortedSet contacts = new TreeSet<>(); + // contacts.add(Contact.build("Joe Bloggs","Jo Bloggs Inc", + // "http://www.jobloggs.com/joe")); + contacts.add( + new Contact.Builder() + .firstName("Joe") + .lastName("Bloggs") + // .affiliation("Jo Bloggs Inc") + // .url("http://www.jobloggs.com/joe") + .name("Joe Bloggs") + .role("Submitter") + .email("jobloggs@joblogs.com") + .build()); + + final SortedSet publications = new TreeSet<>(); + // publications.add(Publication.build("10.1093/nar/gkt1081", "24265224")); + publications.add( + new Publication.Builder().doi("10.1093/nar/gkt1081").pubmed_id("24265224").build()); + + // return Sample.build(name, accession, domain, release, update, attributes, relationships, + // externalReferences, organizations, contacts, publications); + return new Sample.Builder(name, accession) + .withDomain(domain) + .withRelease(release) + .withUpdate(update) + .withSubmitted(submitted) + .withAttributes(attributes) + .withRelationships(relationships) + .withExternalReferences(externalReferences) + .withOrganizations(organizations) + .withContacts(contacts) + .withPublications(publications) + .build(); + } + + @Test + public void testSerialize() throws Exception { + final Sample details = getSimpleSample(); + + log.info(json.write(details).getJson()); + + // Use JSON path based assertions + assertThat(json.write(details)).hasJsonPathStringValue("@.accession"); + assertThat(json.write(details)) + .extractingJsonPathStringValue("@.accession") + .isEqualTo("SAMEA1234"); + + // Assert against a `.json` file in the same package as the test + assertThat(json.write(details)).isEqualToJson("/core/TEST1.json"); + } + + @Test + public void testDeserialize() throws Exception { + final Sample fileSample = json.readObject("/core/TEST1.json"); + final Sample simpleSample = getSimpleSample(); + log.info("fileSample = " + fileSample); + log.info("simpleSample = " + simpleSample); + // Use JSON path based assertions + assertThat(fileSample.getName()).isEqualTo("Test Sample"); + assertThat(fileSample.getAccession()).isEqualTo("SAMEA1234"); + // Assert against a `.json` file + assertThat(fileSample).isEqualTo(simpleSample); + + // check that a specific attribute exists + assertThat(fileSample.getCharacteristics().contains(Attribute.build("organism part", "heart"))); + } + + @Test + public void testRoundTrip() throws Exception { + final Sample sample = getSimpleSample(); + log.info("roundTrip sample = " + sample); + + final String json = this.json.write(sample).getJson(); + log.info("roundTrip json = " + json); + + final InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8)); + final Sample sampleRedux = this.json.readObject(inputStream); + log.info("roundTrip sampleRedux = " + sampleRedux); + + final String jsonRedux = this.json.write(sampleRedux).getJson(); + log.info("roundTrip jsonRedux = " + jsonRedux); + + final BufferedReader br = + new BufferedReader( + new InputStreamReader(new ClassPathResource("/TEST1.json").getInputStream()), 1024); + final StringBuilder stringBuilder = new StringBuilder(); + String line; + while ((line = br.readLine()) != null) { + stringBuilder.append(line).append('\n'); + } + br.close(); + final String jsonFile = stringBuilder.toString(); + + assertThat(sample.equals(sampleRedux)); + assertThat(sample.equals(jsonFile)); + assertThat(json.equals(jsonRedux)); + assertThat(json.equals(jsonFile)); + } + + @SpringBootConfiguration + public static class TestConfig {} +} diff --git a/models/core/src/test/java/uk/ac/ebi/biosamples/service/HttpOlsUrlResolutionServiceTest.java b/core/src/test/java/uk/ac/ebi/biosamples/core/service/HttpOlsUrlResolutionServiceTest.java similarity index 84% rename from models/core/src/test/java/uk/ac/ebi/biosamples/service/HttpOlsUrlResolutionServiceTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/core/service/HttpOlsUrlResolutionServiceTest.java index d00597845..52ec9d00c 100644 --- a/models/core/src/test/java/uk/ac/ebi/biosamples/service/HttpOlsUrlResolutionServiceTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/core/service/HttpOlsUrlResolutionServiceTest.java @@ -8,15 +8,14 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.service; +package uk.ac.ebi.biosamples.core.service; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; -import org.junit.jupiter.api.Test; +import org.junit.Test; class HttpOlsUrlResolutionServiceTest { - @Test public void testIsCurie() { assertTrue(HttpOlsUrlResolutionService.isCurie("NCBI_1234")); diff --git a/models/jsonld/src/test/java/uk/ac/ebi/biosamples/model/BioschemasContextTest.java b/core/src/test/java/uk/ac/ebi/biosamples/jsonld/model/BioschemasContextTest.java similarity index 96% rename from models/jsonld/src/test/java/uk/ac/ebi/biosamples/model/BioschemasContextTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/jsonld/model/BioschemasContextTest.java index 8044388e3..4bab273df 100644 --- a/models/jsonld/src/test/java/uk/ac/ebi/biosamples/model/BioschemasContextTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/jsonld/model/BioschemasContextTest.java @@ -1,80 +1,80 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.net.URI; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.skyscreamer.jsonassert.Customization; -import org.skyscreamer.jsonassert.JSONAssert; -import org.skyscreamer.jsonassert.JSONCompareMode; -import org.skyscreamer.jsonassert.comparator.CustomComparator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.test.autoconfigure.json.JsonTest; -import org.springframework.boot.test.json.JacksonTester; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - -@RunWith(SpringRunner.class) -@JsonTest -// @TestPropertySource(properties = -// {"spring.jackson.serialization.WRITE_DATES_AS_TIMESTAMPS=false","spring.jackson.serialization.WRITE_NULL_MAP_VALUES=false"}) -@TestPropertySource(properties = {"spring.jackson.serialization.INDENT_OUTPUT=true"}) -public class BioschemasContextTest { - - private final Logger log = LoggerFactory.getLogger(getClass()); - - private JacksonTester json; - - @Test - public void testDeserialize() throws Exception { - final String contextJson = - "[" - + "\"http://schema.org\"," - + "{\"OBI\":\"http://purl.obolibrary.org/obo/OBI_\",\"biosample\":\"http://identifiers.org/biosample\"}" - + "]"; - - final BioSchemasContext context = - new ObjectMapper().readerFor(BioSchemasContext.class).readValue(contextJson); - - // Use JSON path based assertions - assertThat(context.getSchemaOrgContext()).isEqualTo(URI.create("http://schema.org")); - } - - @Test - public void testSerialize() throws Exception { - final ObjectMapper mapper = new ObjectMapper(); - final BioSchemasContext context = new BioSchemasContext(); - context.addOtherContexts("OBI", URI.create("http://purl.obolibrary.org/obo/OBI_")); - context.addOtherContexts("biosample", URI.create("http://identifiers.org/biosample")); - final String actualJson = mapper.writeValueAsString(context); - - final String expectedJson = - "[" - + "\"http://schema.org\"," - + "{\"OBI\":\"http://purl.obolibrary.org/obo/OBI_\",\"biosample\":\"http://identifiers.org/biosample\"}" - + "]"; - - JSONAssert.assertEquals( - expectedJson, - actualJson, - new CustomComparator( - JSONCompareMode.LENIENT, new Customization("metaData.created", (o1, o2) -> true))); - } - - @SpringBootConfiguration - public static class TestConfig {} -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.model; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.net.URI; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.skyscreamer.jsonassert.Customization; +import org.skyscreamer.jsonassert.JSONAssert; +import org.skyscreamer.jsonassert.JSONCompareMode; +import org.skyscreamer.jsonassert.comparator.CustomComparator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.test.autoconfigure.json.JsonTest; +import org.springframework.boot.test.json.JacksonTester; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@JsonTest +// @TestPropertySource(properties = +// {"spring.jackson.serialization.WRITE_DATES_AS_TIMESTAMPS=false","spring.jackson.serialization.WRITE_NULL_MAP_VALUES=false"}) +@TestPropertySource(properties = {"spring.jackson.serialization.INDENT_OUTPUT=true"}) +public class BioschemasContextTest { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private JacksonTester json; + + @Test + public void testDeserialize() throws Exception { + final String contextJson = + "[" + + "\"http://schema.org\"," + + "{\"OBI\":\"http://purl.obolibrary.org/obo/OBI_\",\"biosample\":\"http://identifiers.org/biosample\"}" + + "]"; + + final BioSchemasContext context = + new ObjectMapper().readerFor(BioSchemasContext.class).readValue(contextJson); + + // Use JSON path based assertions + assertThat(context.getSchemaOrgContext()).isEqualTo(URI.create("http://schema.org")); + } + + @Test + public void testSerialize() throws Exception { + final ObjectMapper mapper = new ObjectMapper(); + final BioSchemasContext context = new BioSchemasContext(); + context.addOtherContexts("OBI", URI.create("http://purl.obolibrary.org/obo/OBI_")); + context.addOtherContexts("biosample", URI.create("http://identifiers.org/biosample")); + final String actualJson = mapper.writeValueAsString(context); + + final String expectedJson = + "[" + + "\"http://schema.org\"," + + "{\"OBI\":\"http://purl.obolibrary.org/obo/OBI_\",\"biosample\":\"http://identifiers.org/biosample\"}" + + "]"; + + JSONAssert.assertEquals( + expectedJson, + actualJson, + new CustomComparator( + JSONCompareMode.LENIENT, new Customization("metaData.created", (o1, o2) -> true))); + } + + @SpringBootConfiguration + public static class TestConfig {} +} diff --git a/models/jsonld/src/test/java/uk/ac/ebi/biosamples/service/ContextDeserializerTest.java b/core/src/test/java/uk/ac/ebi/biosamples/jsonld/service/ContextDeserializerTest.java similarity index 94% rename from models/jsonld/src/test/java/uk/ac/ebi/biosamples/service/ContextDeserializerTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/jsonld/service/ContextDeserializerTest.java index aa0624f48..d166a00aa 100644 --- a/models/jsonld/src/test/java/uk/ac/ebi/biosamples/service/ContextDeserializerTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/jsonld/service/ContextDeserializerTest.java @@ -1,62 +1,62 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.net.URI; -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import uk.ac.ebi.biosamples.model.BioSchemasContext; - -public class ContextDeserializerTest { - private static final Logger log = LoggerFactory.getLogger(ContextDeserializerTest.class); - - @Test - public void testSerialize_schemaOrgContext() { - final String contextString = - "[\"http://schema.org\",{\"OBI\":\"http://purl.obolibrary.org/obo/OBI_\"," - + "\"biosample\":\"http://identifiers.org/biosample\"}]"; - final BioSchemasContext expectedContext = new BioSchemasContext(); - final ObjectMapper mapper = new ObjectMapper(); - BioSchemasContext context = null; - try { - context = mapper.readValue(contextString, BioSchemasContext.class); - } catch (final IOException e) { - log.error("Failed to deserialize context"); - Assert.fail(); - } - - Assert.assertEquals(expectedContext.getSchemaOrgContext(), context.getSchemaOrgContext()); - } - - @Test - public void testSerialize_otherContext() { - final String contextString = - "[\"http://schema.org\",{\"OBI\":\"http://purl.obolibrary.org/obo/OBI_\"," - + "\"biosample\":\"http://identifiers.org/biosample\",\"ebi\":\"https://www.ebi.ac.uk/biosamples/\"}]"; - final BioSchemasContext expectedContext = new BioSchemasContext(); - expectedContext.addOtherContexts("ebi", URI.create("https://www.ebi.ac.uk/biosamples/")); - final ObjectMapper mapper = new ObjectMapper(); - BioSchemasContext context = null; - try { - context = mapper.readValue(contextString, BioSchemasContext.class); - } catch (final IOException e) { - log.error("Failed to deserialize context"); - Assert.fail(); - } - - Assert.assertEquals( - expectedContext.getOtherContexts().size(), context.getOtherContexts().size()); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.net.URI; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import uk.ac.ebi.biosamples.jsonld.model.BioSchemasContext; + +public class ContextDeserializerTest { + private static final Logger log = LoggerFactory.getLogger(ContextDeserializerTest.class); + + @Test + public void testSerialize_schemaOrgContext() { + final String contextString = + "[\"http://schema.org\",{\"OBI\":\"http://purl.obolibrary.org/obo/OBI_\"," + + "\"biosample\":\"http://identifiers.org/biosample\"}]"; + final BioSchemasContext expectedContext = new BioSchemasContext(); + final ObjectMapper mapper = new ObjectMapper(); + BioSchemasContext context = null; + try { + context = mapper.readValue(contextString, BioSchemasContext.class); + } catch (final IOException e) { + log.error("Failed to deserialize context"); + Assert.fail(); + } + + Assert.assertEquals(expectedContext.getSchemaOrgContext(), context.getSchemaOrgContext()); + } + + @Test + public void testSerialize_otherContext() { + final String contextString = + "[\"http://schema.org\",{\"OBI\":\"http://purl.obolibrary.org/obo/OBI_\"," + + "\"biosample\":\"http://identifiers.org/biosample\",\"ebi\":\"https://www.ebi.ac.uk/biosamples/\"}]"; + final BioSchemasContext expectedContext = new BioSchemasContext(); + expectedContext.addOtherContexts("ebi", URI.create("https://www.ebi.ac.uk/biosamples/")); + final ObjectMapper mapper = new ObjectMapper(); + BioSchemasContext context = null; + try { + context = mapper.readValue(contextString, BioSchemasContext.class); + } catch (final IOException e) { + log.error("Failed to deserialize context"); + Assert.fail(); + } + + Assert.assertEquals( + expectedContext.getOtherContexts().size(), context.getOtherContexts().size()); + } +} diff --git a/models/jsonld/src/test/java/uk/ac/ebi/biosamples/service/ContextSerializerTest.java b/core/src/test/java/uk/ac/ebi/biosamples/jsonld/service/ContextSerializerTest.java similarity index 92% rename from models/jsonld/src/test/java/uk/ac/ebi/biosamples/service/ContextSerializerTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/jsonld/service/ContextSerializerTest.java index ad6e3c7db..280b72ac5 100644 --- a/models/jsonld/src/test/java/uk/ac/ebi/biosamples/service/ContextSerializerTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/jsonld/service/ContextSerializerTest.java @@ -1,40 +1,40 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import uk.ac.ebi.biosamples.model.BioSchemasContext; - -public class ContextSerializerTest { - private static final Logger log = LoggerFactory.getLogger(ContextSerializerTest.class); - - @Test - public void testSerialize() { - final String expectedSerializedContext = - "[\"http://schema.org\",{\"OBI\":\"http://purl.obolibrary.org/obo/OBI_\"," - + "\"biosample\":\"http://identifiers.org/biosample/\"}]"; - final BioSchemasContext context = new BioSchemasContext(); - final ObjectMapper mapper = new ObjectMapper(); - String serializedContext = null; - try { - serializedContext = mapper.writeValueAsString(context); - } catch (final IOException e) { - log.error("Failed to serialize context"); - } - - Assert.assertEquals(expectedSerializedContext, serializedContext); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import uk.ac.ebi.biosamples.jsonld.model.BioSchemasContext; + +public class ContextSerializerTest { + private static final Logger log = LoggerFactory.getLogger(ContextSerializerTest.class); + + @Test + public void testSerialize() { + final String expectedSerializedContext = + "[\"http://schema.org\",{\"OBI\":\"http://purl.obolibrary.org/obo/OBI_\"," + + "\"biosample\":\"http://identifiers.org/biosample/\"}]"; + final BioSchemasContext context = new BioSchemasContext(); + final ObjectMapper mapper = new ObjectMapper(); + String serializedContext = null; + try { + serializedContext = mapper.writeValueAsString(context); + } catch (final IOException e) { + log.error("Failed to serialize context"); + } + + Assert.assertEquals(expectedSerializedContext, serializedContext); + } +} diff --git a/models/jsonld/src/test/java/uk/ac/ebi/biosamples/service/SampleToJsonLDSampleRecordConverterTest.java b/core/src/test/java/uk/ac/ebi/biosamples/jsonld/service/SampleToJsonLDSampleRecordConverterTest.java similarity index 91% rename from models/jsonld/src/test/java/uk/ac/ebi/biosamples/service/SampleToJsonLDSampleRecordConverterTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/jsonld/service/SampleToJsonLDSampleRecordConverterTest.java index aa9afd22d..7f0813d96 100644 --- a/models/jsonld/src/test/java/uk/ac/ebi/biosamples/service/SampleToJsonLDSampleRecordConverterTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/jsonld/service/SampleToJsonLDSampleRecordConverterTest.java @@ -1,70 +1,70 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.JsonLDDataRecord; -import uk.ac.ebi.biosamples.model.Sample; - -public class SampleToJsonLDSampleRecordConverterTest { - private static final Logger log = - LoggerFactory.getLogger(SampleToJsonLDSampleRecordConverterTest.class); - - @Test - public void testConvert() { - final Sample sample = getSample(); - - final SampleToJsonLDSampleRecordConverter converter = new SampleToJsonLDSampleRecordConverter(); - final JsonLDDataRecord record = converter.convert(sample); - - Assert.assertEquals( - sample.getAttributes().first().getIri().first(), - record.getMainEntity().getAdditionalProperties().get(0).getValueReference().get(0).getId()); - } - - @Test - public void testSerializeDeserialize() { - final Sample sample = getSample(); - final SampleToJsonLDSampleRecordConverter converter = new SampleToJsonLDSampleRecordConverter(); - final JsonLDDataRecord record = converter.convert(sample); - - JsonLDDataRecord deserializedRecord = null; - final ObjectMapper mapper = new ObjectMapper(); - try { - final String serializedRecord = - mapper.writerWithDefaultPrettyPrinter().writeValueAsString(record); - deserializedRecord = mapper.readValue(serializedRecord, JsonLDDataRecord.class); - } catch (final IOException e) { - log.error("Failed to serialize JsonLD record"); - e.printStackTrace(); - Assert.fail(); - } - - Assert.assertEquals( - record.getContext().getOtherContexts().size(), - deserializedRecord.getContext().getOtherContexts().size()); - } - - private Sample getSample() { - return new Sample.Builder("FakeName", "FakeAccession") - .withDomain("test.fake.domain") - .addAttribute( - Attribute.build( - "Organism", "Homo Sapiens", "http://purl.obolibrary.org/obo/NCBITaxon_9606", null)) - .build(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.jsonld.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.jsonld.model.JsonLDDataRecord; + +public class SampleToJsonLDSampleRecordConverterTest { + private static final Logger log = + LoggerFactory.getLogger(SampleToJsonLDSampleRecordConverterTest.class); + + @Test + public void testConvert() { + final Sample sample = getSample(); + + final SampleToJsonLDSampleRecordConverter converter = new SampleToJsonLDSampleRecordConverter(); + final JsonLDDataRecord record = converter.convert(sample); + + Assert.assertEquals( + sample.getAttributes().first().getIri().first(), + record.getMainEntity().getAdditionalProperties().get(0).getValueReference().get(0).getId()); + } + + @Test + public void testSerializeDeserialize() { + final Sample sample = getSample(); + final SampleToJsonLDSampleRecordConverter converter = new SampleToJsonLDSampleRecordConverter(); + final JsonLDDataRecord record = converter.convert(sample); + + JsonLDDataRecord deserializedRecord = null; + final ObjectMapper mapper = new ObjectMapper(); + try { + final String serializedRecord = + mapper.writerWithDefaultPrettyPrinter().writeValueAsString(record); + deserializedRecord = mapper.readValue(serializedRecord, JsonLDDataRecord.class); + } catch (final IOException e) { + log.error("Failed to serialize JsonLD record"); + e.printStackTrace(); + Assert.fail(); + } + + Assert.assertEquals( + record.getContext().getOtherContexts().size(), + deserializedRecord.getContext().getOtherContexts().size()); + } + + private Sample getSample() { + return new Sample.Builder("FakeName", "FakeAccession") + .withDomain("test.fake.domain") + .addAttribute( + Attribute.build( + "Organism", "Homo Sapiens", "http://purl.obolibrary.org/obo/NCBITaxon_9606", null)) + .build(); + } +} diff --git a/models/mongo/src/test/java/uk/ac/ebi/biosamples/model/MongoSerializationTest.java b/core/src/test/java/uk/ac/ebi/biosamples/mongo/model/MongoSerializationTest.java similarity index 94% rename from models/mongo/src/test/java/uk/ac/ebi/biosamples/model/MongoSerializationTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/mongo/model/MongoSerializationTest.java index 2c5d01497..71557bb34 100644 --- a/models/mongo/src/test/java/uk/ac/ebi/biosamples/model/MongoSerializationTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/mongo/model/MongoSerializationTest.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.model; +package uk.ac.ebi.biosamples.mongo.model; import static org.assertj.core.api.Assertions.assertThat; @@ -25,13 +25,10 @@ import org.springframework.boot.test.json.JacksonTester; import org.springframework.context.annotation.Configuration; import org.springframework.test.context.junit4.SpringRunner; -import uk.ac.ebi.biosamples.model.structured.AbstractData; -import uk.ac.ebi.biosamples.model.structured.StructuredDataEntry; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; -import uk.ac.ebi.biosamples.mongo.model.MongoExternalReference; -import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; -import uk.ac.ebi.biosamples.mongo.model.MongoSample; -import uk.ac.ebi.biosamples.mongo.model.MongoStructuredData; +import uk.ac.ebi.biosamples.core.model.*; +import uk.ac.ebi.biosamples.core.model.structured.AbstractData; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataEntry; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; @RunWith(SpringRunner.class) @JsonTest diff --git a/models/mongo/src/test/java/uk/ac/ebi/biosamples/service/CurationReadServiceTest.java b/core/src/test/java/uk/ac/ebi/biosamples/mongo/service/CurationReadServiceTest.java similarity index 97% rename from models/mongo/src/test/java/uk/ac/ebi/biosamples/service/CurationReadServiceTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/mongo/service/CurationReadServiceTest.java index 9cb5170ca..ca62d6781 100644 --- a/models/mongo/src/test/java/uk/ac/ebi/biosamples/service/CurationReadServiceTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/mongo/service/CurationReadServiceTest.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.service; +package uk.ac.ebi.biosamples.mongo.service; import java.time.Instant; import java.util.*; @@ -25,14 +25,10 @@ import org.springframework.data.domain.PageRequest; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; -import uk.ac.ebi.biosamples.model.*; +import uk.ac.ebi.biosamples.core.model.*; import uk.ac.ebi.biosamples.mongo.model.MongoCurationLink; import uk.ac.ebi.biosamples.mongo.repository.MongoCurationLinkRepository; import uk.ac.ebi.biosamples.mongo.repository.MongoCurationRepository; -import uk.ac.ebi.biosamples.mongo.service.CurationLinkToMongoCurationLinkConverter; -import uk.ac.ebi.biosamples.mongo.service.CurationReadService; -import uk.ac.ebi.biosamples.mongo.service.MongoCurationLinkToCurationLinkConverter; -import uk.ac.ebi.biosamples.mongo.service.MongoCurationToCurationConverter; @RunWith(SpringRunner.class) @SpringBootTest( diff --git a/models/neo4j/src/test/java/uk/ac/ebi/biosamples/AppTest.java b/core/src/test/java/uk/ac/ebi/biosamples/solr/SolrSampleTestConfiguration.java similarity index 68% rename from models/neo4j/src/test/java/uk/ac/ebi/biosamples/AppTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/solr/SolrSampleTestConfiguration.java index 86a3cf38d..13a885235 100644 --- a/models/neo4j/src/test/java/uk/ac/ebi/biosamples/AppTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/solr/SolrSampleTestConfiguration.java @@ -1,24 +1,18 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples; - -import static org.junit.Assert.assertTrue; - -import org.junit.Test; - -/** Unit test for simple App. */ -public class AppTest { - /** Rigorous Test :-) */ - @Test - public void shouldAnswerWithTrue() { - assertTrue(true); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr; + +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.ComponentScan; + +@TestConfiguration +@ComponentScan +public class SolrSampleTestConfiguration {} diff --git a/core/src/test/java/uk/ac/ebi/biosamples/solr/SolrServiceTests.java b/core/src/test/java/uk/ac/ebi/biosamples/solr/SolrServiceTests.java new file mode 100644 index 000000000..499bd47ea --- /dev/null +++ b/core/src/test/java/uk/ac/ebi/biosamples/solr/SolrServiceTests.java @@ -0,0 +1,85 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.solr; + +import static org.junit.Assert.*; + +import java.util.Optional; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.solr.core.query.Criteria; +import org.springframework.test.context.junit4.SpringRunner; +import uk.ac.ebi.biosamples.BioSamplesProperties; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.service.FilterBuilder; +import uk.ac.ebi.biosamples.solr.model.field.*; +import uk.ac.ebi.biosamples.solr.service.SolrFieldService; +import uk.ac.ebi.biosamples.solr.service.SolrFilterService; + +@RunWith(SpringRunner.class) +@SpringBootTest( + classes = { + SolrSampleAccessionField.class, + SolrSampleAttributeValueField.class, + SolrSampleDateField.class, + SolrSampleExternalReferenceDataField.class, + SolrSampleInverseRelationField.class, + SolrSampleNameField.class, + SolrSampleRelationField.class, + SolrFilterService.class, + SolrFieldService.class, + BioSamplesProperties.class + }) +public class SolrServiceTests { + @Autowired SolrFilterService solrFilterService; + + @Autowired private SolrFieldService fieldService; + + // @Before + // public void setup() { + // this.fieldService = new SolrFieldService(solrSampleFieldList); + // } + + @Test + public void given_encoded_sample_decode_it_correctly_and_of_the_right_type() { + String encodedField = "MRSXGY3SNFYHI2LPNY_______av_ss"; + String expectedDecodedField = "description"; + + SolrSampleField sampleField = fieldService.decodeField(encodedField); + assertEquals(sampleField.getReadableLabel(), expectedDecodedField); + assertTrue(sampleField instanceof SolrSampleAttributeValueField); + } + + @Test + public void given_fields_with_similar_suffix_return_the_correct_type() { + + SolrSampleField attributeField = fieldService.decodeField("MRSXGY3SNFYHI2LPNY_______av_ss"); + SolrSampleField nameField = fieldService.decodeField("name_s"); + // SolrSampleField domainField = fieldService.decodeField("domain_s"); + + assertTrue(attributeField instanceof SolrSampleAttributeValueField); + assertTrue(nameField instanceof SolrSampleNameField); + } + + @Test + public void given_filter_object_return_the_corresponding_solr_field() { + Filter organismFilter = + FilterBuilder.create().onAttribute("organism").withValue("Homo sapiens").build(); + SolrSampleAttributeValueField organism = new SolrSampleAttributeValueField("organism"); + Optional criteriaFromField = + Optional.ofNullable(organism.getFilterCriteria(organismFilter)); + Optional criteriaFromService = solrFilterService.getFilterCriteria(organismFilter); + + assertEquals(criteriaFromField.toString(), criteriaFromService.toString()); + } +} diff --git a/utils/webapp/src/test/java/uk/ac/ebi/biosamples/utils/LinkUtilsTest.java b/core/src/test/java/uk/ac/ebi/biosamples/utils/LinkUtilsTest.java similarity index 97% rename from utils/webapp/src/test/java/uk/ac/ebi/biosamples/utils/LinkUtilsTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/utils/LinkUtilsTest.java index 355b4f382..f170cd4a0 100644 --- a/utils/webapp/src/test/java/uk/ac/ebi/biosamples/utils/LinkUtilsTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/utils/LinkUtilsTest.java @@ -1,47 +1,47 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils; - -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.hateoas.IanaLinkRelations; -import org.springframework.hateoas.Link; -import org.springframework.test.context.junit4.SpringRunner; - -@RunWith(SpringRunner.class) -public class LinkUtilsTest { - @Test - public void testCleanLink() { - Link link = Link.of("http://localhost:8080/test/end{?foo,bar}", IanaLinkRelations.SELF); - String uri = "http://localhost:8080/test/end"; - Link cleanLink = LinkUtils.cleanLink(link); - Assert.assertEquals(uri, cleanLink.getHref()); - } - - @Test - @Ignore - public void testCleanLinkEncoded() { - Link link = Link.of("http://localhost:8080/test/end?foo=%21{&bar}", IanaLinkRelations.SELF); - String uri = "http://localhost:8080/test/end?foo=%21"; - Link cleanLink = LinkUtils.cleanLink(link); - Assert.assertEquals(uri, cleanLink.getHref()); - } - - // @Test - // public void testCleanLinkPartial() { - // Link link = new Link("http://localhost:8080/test/end{?foo,bar}", Link.REL_SELF) ; - // String uri = "http://localhost:8080/test/end{?foo}"; - // Link cleanLink = LinkUtils.removeTemplateVariable(link, "bar"); - // Assert.assertEquals(uri, cleanLink.getHref()); - // } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils; + +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.hateoas.IanaLinkRelations; +import org.springframework.hateoas.Link; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +public class LinkUtilsTest { + @Test + public void testCleanLink() { + Link link = Link.of("http://localhost:8080/test/end{?foo,bar}", IanaLinkRelations.SELF); + String uri = "http://localhost:8080/test/end"; + Link cleanLink = LinkUtils.cleanLink(link); + Assert.assertEquals(uri, cleanLink.getHref()); + } + + @Test + @Ignore + public void testCleanLinkEncoded() { + Link link = Link.of("http://localhost:8080/test/end?foo=%21{&bar}", IanaLinkRelations.SELF); + String uri = "http://localhost:8080/test/end?foo=%21"; + Link cleanLink = LinkUtils.cleanLink(link); + Assert.assertEquals(uri, cleanLink.getHref()); + } + + // @Test + // public void testCleanLinkPartial() { + // Link link = new Link("http://localhost:8080/test/end{?foo,bar}", Link.REL_SELF) ; + // String uri = "http://localhost:8080/test/end{?foo}"; + // Link cleanLink = LinkUtils.removeTemplateVariable(link, "bar"); + // Assert.assertEquals(uri, cleanLink.getHref()); + // } +} diff --git a/utils/ols/src/test/java/uk/ac/ebi/biosamples/ols/OlsProcessorTest.java b/core/src/test/java/uk/ac/ebi/biosamples/utils/ols/OlsProcessorTest.java similarity index 95% rename from utils/ols/src/test/java/uk/ac/ebi/biosamples/ols/OlsProcessorTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/utils/ols/OlsProcessorTest.java index 3ba0b87fb..ab331cc7b 100644 --- a/utils/ols/src/test/java/uk/ac/ebi/biosamples/ols/OlsProcessorTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/utils/ols/OlsProcessorTest.java @@ -1,128 +1,128 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.ols; - -import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; -import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; -import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; - -import com.google.common.collect.ImmutableMap; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Map; -import java.util.Optional; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.web.client.AutoConfigureWebClient; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.http.HttpMethod; -import org.springframework.http.MediaType; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.test.web.client.MockRestServiceServer; -import org.springframework.web.client.RestTemplate; - -@RunWith(SpringRunner.class) -@SpringBootTest(classes = TestApplication.class) -@AutoConfigureWebClient -public class OlsProcessorTest { - - private MockRestServiceServer mockServer; - - @Autowired private OlsProcessor olsProcessor; - - @Autowired private RestTemplate restTemplate; - - @Before - public void setUp() { - mockServer = MockRestServiceServer.createServer(restTemplate); - } - - private final Map expectedValues = - ImmutableMap.of( - "NCIT:C2985", "http://purl.obolibrary.org/obo/NCIT_C2985", - "UBERON:0003978", "http://purl.obolibrary.org/obo/UBERON_0003978", - "FOODON:03304708", "http://purl.obolibrary.org/obo/FOODON_03304708", - "NCIT_C14207", "http://purl.obolibrary.org/obo/NCIT_C14207"); - - @Test - public void test_OlsProcessor_returns_correct_value_for_invalid_ols_term() throws IOException { - final String shortcode = "invalid-term"; - final Optional result = performQuery(shortcode); - assertFalse(result.isPresent()); - } - - private String readFile(final String filePath) throws IOException { - final InputStream inputStream = OlsProcessorTest.class.getResourceAsStream(filePath); - final StringBuilder resultStringBuilder = new StringBuilder(); - try (final BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) { - String line; - while ((line = br.readLine()) != null) { - resultStringBuilder.append(line).append("\n"); - } - } - return resultStringBuilder.toString(); - } - - private Optional performQuery(final String shortcode) throws IOException { - final String expectedResponse = readFile("/examples/ols-responses/" + shortcode + ".json"); - mockServer.reset(); - mockServer - .expect(requestTo("https://www.ebi.ac.uk/ols/api/terms?id=" + shortcode + "&size=500")) - .andExpect(method(HttpMethod.GET)) - .andRespond(withSuccess(expectedResponse, MediaType.APPLICATION_JSON)); - final Optional result = olsProcessor.queryOlsForShortcode(shortcode); - mockServer.verify(); - return result; - } - - @Test - public void test_OlsProcessor_returns_correct_value_for_multi_ols_term() throws IOException { - final String shortcode = "PATO_0000384"; - final Optional result = performQuery(shortcode); - assertTrue(result.isPresent()); - assertEquals("http://purl.obolibrary.org/obo/PATO_0000384", result.get()); - } - - @Test - public void - test_OlsProcessor_returns_correct_value_for_standard_ols_term_that_has_defining_ontology() - throws IOException { - final String shortcode = "NCBITaxon_3702"; - final Optional result = performQuery(shortcode); - assertTrue(result.isPresent()); - assertEquals("http://purl.obolibrary.org/obo/NCBITaxon_3702", result.get()); - } - - @Test - public void - test_OlsProcessor_returns_correct_value_for_standard_ols_term_does_not_have_defining_ontology() - throws IOException { - final String shortcode = "FBcv_0003016"; - final Optional result = performQuery(shortcode); - assertTrue(result.isPresent()); - assertEquals("http://purl.obolibrary.org/obo/FBcv_0003016", result.get()); - } - /*@Test - public void test_OlsProcessor_returns_correct_value_for_example_terms() throws IOException { - for (Map.Entry entry : expectedValues.entrySet()) { - Optional result = performQuery(entry.getKey()); - assertTrue(result.isPresent()); - assertEquals(entry.getValue(), result.get()); - } - }*/ -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.ols; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +import com.google.common.collect.ImmutableMap; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Map; +import java.util.Optional; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.client.AutoConfigureWebClient; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestTemplate; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = TestApplication.class) +@AutoConfigureWebClient +public class OlsProcessorTest { + + private MockRestServiceServer mockServer; + + @Autowired private OlsProcessor olsProcessor; + + @Autowired private RestTemplate restTemplate; + + @Before + public void setUp() { + mockServer = MockRestServiceServer.createServer(restTemplate); + } + + private final Map expectedValues = + ImmutableMap.of( + "NCIT:C2985", "http://purl.obolibrary.org/obo/NCIT_C2985", + "UBERON:0003978", "http://purl.obolibrary.org/obo/UBERON_0003978", + "FOODON:03304708", "http://purl.obolibrary.org/obo/FOODON_03304708", + "NCIT_C14207", "http://purl.obolibrary.org/obo/NCIT_C14207"); + + @Test + public void test_OlsProcessor_returns_correct_value_for_invalid_ols_term() throws IOException { + final String shortcode = "invalid-term"; + final Optional result = performQuery(shortcode); + assertFalse(result.isPresent()); + } + + private String readFile(final String filePath) throws IOException { + final InputStream inputStream = OlsProcessorTest.class.getResourceAsStream(filePath); + final StringBuilder resultStringBuilder = new StringBuilder(); + try (final BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) { + String line; + while ((line = br.readLine()) != null) { + resultStringBuilder.append(line).append("\n"); + } + } + return resultStringBuilder.toString(); + } + + private Optional performQuery(final String shortcode) throws IOException { + final String expectedResponse = readFile("/ols-responses/" + shortcode + ".json"); + mockServer.reset(); + mockServer + .expect(requestTo("https://www.ebi.ac.uk/ols/api/terms?id=" + shortcode + "&size=500")) + .andExpect(method(HttpMethod.GET)) + .andRespond(withSuccess(expectedResponse, MediaType.APPLICATION_JSON)); + final Optional result = olsProcessor.queryOlsForShortcode(shortcode); + mockServer.verify(); + return result; + } + + @Test + public void test_OlsProcessor_returns_correct_value_for_multi_ols_term() throws IOException { + final String shortcode = "PATO_0000384"; + final Optional result = performQuery(shortcode); + assertTrue(result.isPresent()); + assertEquals("http://purl.obolibrary.org/obo/PATO_0000384", result.get()); + } + + @Test + public void + test_OlsProcessor_returns_correct_value_for_standard_ols_term_that_has_defining_ontology() + throws IOException { + final String shortcode = "NCBITaxon_3702"; + final Optional result = performQuery(shortcode); + assertTrue(result.isPresent()); + assertEquals("http://purl.obolibrary.org/obo/NCBITaxon_3702", result.get()); + } + + @Test + public void + test_OlsProcessor_returns_correct_value_for_standard_ols_term_does_not_have_defining_ontology() + throws IOException { + final String shortcode = "FBcv_0003016"; + final Optional result = performQuery(shortcode); + assertTrue(result.isPresent()); + assertEquals("http://purl.obolibrary.org/obo/FBcv_0003016", result.get()); + } + /*@Test + public void test_OlsProcessor_returns_correct_value_for_example_terms() throws IOException { + for (Map.Entry entry : expectedValues.entrySet()) { + Optional result = performQuery(entry.getKey()); + assertTrue(result.isPresent()); + assertEquals(entry.getValue(), result.get()); + } + }*/ +} diff --git a/utils/ols/src/test/java/uk/ac/ebi/biosamples/ols/TestApplication.java b/core/src/test/java/uk/ac/ebi/biosamples/utils/ols/TestApplication.java similarity index 94% rename from utils/ols/src/test/java/uk/ac/ebi/biosamples/ols/TestApplication.java rename to core/src/test/java/uk/ac/ebi/biosamples/utils/ols/TestApplication.java index ce35edd6a..c05f69e1a 100644 --- a/utils/ols/src/test/java/uk/ac/ebi/biosamples/ols/TestApplication.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/utils/ols/TestApplication.java @@ -1,32 +1,32 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.ols; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.client.RestTemplate; -import uk.ac.ebi.biosamples.BioSamplesProperties; - -@Configuration -@ComponentScan -public class TestApplication { - - @Bean - public RestTemplate restTemplate() { - return new RestTemplate(); - } - - @Bean - public BioSamplesProperties bioSamplesProperties() { - return new BioSamplesProperties(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.ols; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.biosamples.BioSamplesProperties; + +@Configuration +@ComponentScan +public class TestApplication { + + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } + + @Bean + public BioSamplesProperties bioSamplesProperties() { + return new BioSamplesProperties(); + } +} diff --git a/commons/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetrieverTest.java b/core/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetrieverTest.java similarity index 97% rename from commons/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetrieverTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetrieverTest.java index 6e142921b..629f47939 100644 --- a/commons/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetrieverTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/OLSDataRetrieverTest.java @@ -1,37 +1,37 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils.phenopacket; - -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -public class OLSDataRetrieverTest { - @Test - @Ignore - public void id_retrieving_test() { - OLSDataRetriever retriever = new OLSDataRetriever(); - retriever.readOntologyJsonFromUrl("http://purl.obolibrary.org/obo/NCBITaxon_9606"); - String expected_id = "NCBITaxon:9606"; - String actual_id = retriever.getOntologyTermId(); - Assert.assertEquals(actual_id, expected_id); - } - - @Test - @Ignore - public void label_retreiving_test() { - OLSDataRetriever retriever = new OLSDataRetriever(); - retriever.readOntologyJsonFromUrl("http://purl.obolibrary.org/obo/NCBITaxon_9606"); - String expected_label = "Homo sapiens"; - String actual_label = retriever.getOntologyTermLabel(); - Assert.assertEquals(actual_label, expected_label); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.phenopacket; + +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +public class OLSDataRetrieverTest { + @Test + @Ignore + public void id_retrieving_test() { + OLSDataRetriever retriever = new OLSDataRetriever(); + retriever.readOntologyJsonFromUrl("http://purl.obolibrary.org/obo/NCBITaxon_9606"); + String expected_id = "NCBITaxon:9606"; + String actual_id = retriever.getOntologyTermId(); + Assert.assertEquals(actual_id, expected_id); + } + + @Test + @Ignore + public void label_retreiving_test() { + OLSDataRetriever retriever = new OLSDataRetriever(); + retriever.readOntologyJsonFromUrl("http://purl.obolibrary.org/obo/NCBITaxon_9606"); + String expected_label = "Homo sapiens"; + String actual_label = retriever.getOntologyTermLabel(); + Assert.assertEquals(actual_label, expected_label); + } +} diff --git a/commons/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverterTest.java b/core/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverterTest.java similarity index 96% rename from commons/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverterTest.java rename to core/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverterTest.java index 3b1dbf204..3c250cd7b 100644 --- a/commons/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverterTest.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/utils/phenopacket/PhenopacketConverterTest.java @@ -1,126 +1,126 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils.phenopacket; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.Arrays; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; -import org.phenopackets.schema.v1.Phenopacket; -import org.springframework.core.io.ClassPathResource; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; - -public class PhenopacketConverterTest { - private final PhenopacketConverter phenopacketConverter = - new PhenopacketConverter(new PhenopacketConversionHelper()); - private final ObjectMapper jsonMapper = new ObjectMapper(); - - @Test - @Ignore - public void testPhenopacketConversion() throws Exception { - final Sample sample = getTestSample(); - final Phenopacket phenopacket = phenopacketConverter.convert(sample); - - final JsonNode phenopacketExpected = - jsonMapper.readValue( - new ClassPathResource("phenopacket/phenopacket_1.json").getInputStream(), - JsonNode.class); - Assert.assertEquals( - phenopacketExpected.get("biosamples").get(0).get("id").textValue(), - phenopacket.getBiosamples(0).getId()); - } - - @Test - public void testPhenopacketConversion_withoutOrganismOntology() { - final Sample sample = getTestSampleWithoutOrganismOntology(); - final Phenopacket phenopacket = phenopacketConverter.convert(sample); - Assert.assertSame("", phenopacket.getBiosamples(0).getTaxonomy().getId()); - } - - @Test - @Ignore - public void testPhenopacketConversion_resources() throws Exception { - final Sample sample = getTestSample_2(); - final Phenopacket phenopacket = phenopacketConverter.convert(sample); - Assert.assertTrue(phenopacket.getMetaData().getResourcesList().size() > 0); - } - - private Sample getTestSample() { - return new Sample.Builder("phenopacket_test") - .withAccession("SAMETAG2031") - .withDomain("self.BiosampleIntegrationTest") - .withRelease("2017-01-01T12:00:00") - .withUpdate("2017-01-01T12:00:00") - .addAttribute( - Attribute.build( - "organism", "human", "http://purl.obolibrary.org/obo/NCBITaxon_9606", "")) - .addAttribute( - Attribute.build( - "disease", "colorectal adenocarcinoma", "http://www.ebi.ac.uk/efo/EFO_0000365", "")) - .addAttribute( - Attribute.build("sex", "male", "http://purl.obolibrary.org/obo/PATO_0000384", "")) - .addAttribute(Attribute.build("age", "30")) - .addAttribute(Attribute.build("test", "test", "http://purl.obolibrary.org/obo/test", "")) - .addAttribute(Attribute.build("description", "test description")) - .addAttribute( - Attribute.build("tissue", "liver", "http://purl.obolibrary.org/obo/UBERON_0002107", "")) - .addAttribute( - Attribute.build( - "lung disease", "yes", "http://purl.obolibrary.org/obo/UBERON_0002107", "")) - .addAttribute( - Attribute.build( - "disease state", - "Duchenne muscular dystrophy", - "http://www.orpha.net/ORDO/Orphanet_98896", - null)) - .build(); - } - - private Sample getTestSampleWithoutOrganismOntology() { - return new Sample.Builder("phenopacket_test") - .withAccession("SAMETAG2031") - .withDomain("self.BiosampleIntegrationTest") - .withRelease("2017-01-01T12:00:00") - .withUpdate("2017-01-01T12:00:00") - .addAttribute(Attribute.build("organism", "human")) - .build(); - } - - private Sample getTestSample_2() { - final Sample.Builder sampleBuilder = - new Sample.Builder("Phenopacket_ERS1790018", "Phenopacket_ERS1790018"); - sampleBuilder - .withDomain("self.BiosampleIntegrationTest") - .withRelease("2017-01-01T12:00:00") - .withUpdate("2017-01-01T12:00:00") - .withAttributes( - Arrays.asList( - Attribute.build( - "Organism", - "Homo sapiens", - "http://purl.obolibrary.org/obo/NCBITaxon_9606", - null), - Attribute.build( - "cell type", "myoblast", "http://purl.obolibrary.org/obo/CL_0000056", null), - Attribute.build( - "disease state", - "Duchenne muscular dystrophy", - "http://www.orpha.net/ORDO/Orphanet_98896", - null), - Attribute.build("genotype", "BMI1 overexpression"), - Attribute.build("individual", "SD-8306I"))); - - return sampleBuilder.build(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils.phenopacket; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Arrays; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; +import org.phenopackets.schema.v1.Phenopacket; +import org.springframework.core.io.ClassPathResource; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; + +public class PhenopacketConverterTest { + private final PhenopacketConverter phenopacketConverter = + new PhenopacketConverter(new PhenopacketConversionHelper()); + private final ObjectMapper jsonMapper = new ObjectMapper(); + + @Test + @Ignore + public void testPhenopacketConversion() throws Exception { + final Sample sample = getTestSample(); + final Phenopacket phenopacket = phenopacketConverter.convert(sample); + + final JsonNode phenopacketExpected = + jsonMapper.readValue( + new ClassPathResource("phenopacket/phenopacket_1.json").getInputStream(), + JsonNode.class); + Assert.assertEquals( + phenopacketExpected.get("biosamples").get(0).get("id").textValue(), + phenopacket.getBiosamples(0).getId()); + } + + @Test + public void testPhenopacketConversion_withoutOrganismOntology() { + final Sample sample = getTestSampleWithoutOrganismOntology(); + final Phenopacket phenopacket = phenopacketConverter.convert(sample); + Assert.assertSame("", phenopacket.getBiosamples(0).getTaxonomy().getId()); + } + + @Test + @Ignore + public void testPhenopacketConversion_resources() throws Exception { + final Sample sample = getTestSample_2(); + final Phenopacket phenopacket = phenopacketConverter.convert(sample); + Assert.assertTrue(phenopacket.getMetaData().getResourcesList().size() > 0); + } + + private Sample getTestSample() { + return new Sample.Builder("phenopacket_test") + .withAccession("SAMETAG2031") + .withDomain("self.BiosampleIntegrationTest") + .withRelease("2017-01-01T12:00:00") + .withUpdate("2017-01-01T12:00:00") + .addAttribute( + Attribute.build( + "organism", "human", "http://purl.obolibrary.org/obo/NCBITaxon_9606", "")) + .addAttribute( + Attribute.build( + "disease", "colorectal adenocarcinoma", "http://www.ebi.ac.uk/efo/EFO_0000365", "")) + .addAttribute( + Attribute.build("sex", "male", "http://purl.obolibrary.org/obo/PATO_0000384", "")) + .addAttribute(Attribute.build("age", "30")) + .addAttribute(Attribute.build("test", "test", "http://purl.obolibrary.org/obo/test", "")) + .addAttribute(Attribute.build("description", "test description")) + .addAttribute( + Attribute.build("tissue", "liver", "http://purl.obolibrary.org/obo/UBERON_0002107", "")) + .addAttribute( + Attribute.build( + "lung disease", "yes", "http://purl.obolibrary.org/obo/UBERON_0002107", "")) + .addAttribute( + Attribute.build( + "disease state", + "Duchenne muscular dystrophy", + "http://www.orpha.net/ORDO/Orphanet_98896", + null)) + .build(); + } + + private Sample getTestSampleWithoutOrganismOntology() { + return new Sample.Builder("phenopacket_test") + .withAccession("SAMETAG2031") + .withDomain("self.BiosampleIntegrationTest") + .withRelease("2017-01-01T12:00:00") + .withUpdate("2017-01-01T12:00:00") + .addAttribute(Attribute.build("organism", "human")) + .build(); + } + + private Sample getTestSample_2() { + final Sample.Builder sampleBuilder = + new Sample.Builder("Phenopacket_ERS1790018", "Phenopacket_ERS1790018"); + sampleBuilder + .withDomain("self.BiosampleIntegrationTest") + .withRelease("2017-01-01T12:00:00") + .withUpdate("2017-01-01T12:00:00") + .withAttributes( + Arrays.asList( + Attribute.build( + "Organism", + "Homo sapiens", + "http://purl.obolibrary.org/obo/NCBITaxon_9606", + null), + Attribute.build( + "cell type", "myoblast", "http://purl.obolibrary.org/obo/CL_0000056", null), + Attribute.build( + "disease state", + "Duchenne muscular dystrophy", + "http://www.orpha.net/ORDO/Orphanet_98896", + null), + Attribute.build("genotype", "BMI1 overexpression"), + Attribute.build("individual", "SD-8306I"))); + + return sampleBuilder.build(); + } +} diff --git a/utils/validation/src/test/java/uk/ac/ebi/biosamples/validation/TestApplication.java b/core/src/test/java/uk/ac/ebi/biosamples/validation/TestApplication.java similarity index 97% rename from utils/validation/src/test/java/uk/ac/ebi/biosamples/validation/TestApplication.java rename to core/src/test/java/uk/ac/ebi/biosamples/validation/TestApplication.java index e29325e97..0700dd2b4 100644 --- a/utils/validation/src/test/java/uk/ac/ebi/biosamples/validation/TestApplication.java +++ b/core/src/test/java/uk/ac/ebi/biosamples/validation/TestApplication.java @@ -1,32 +1,32 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.validation; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.client.RestTemplate; -import uk.ac.ebi.biosamples.BioSamplesProperties; - -@Configuration -@ComponentScan -public class TestApplication { - - @Bean - public RestTemplate restTemplate() { - return new RestTemplate(); - } - - @Bean - public BioSamplesProperties bioSamplesProperties() { - return new BioSamplesProperties(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.validation; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.biosamples.BioSamplesProperties; + +@Configuration +@ComponentScan +public class TestApplication { + + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } + + @Bean + public BioSamplesProperties bioSamplesProperties() { + return new BioSamplesProperties(); + } +} diff --git a/utils/ols/src/test/resources/logback-test.xml b/core/src/test/logback-test.xml similarity index 98% rename from utils/ols/src/test/resources/logback-test.xml rename to core/src/test/logback-test.xml index 4449069d9..2f6b67ec9 100644 --- a/utils/ols/src/test/resources/logback-test.xml +++ b/core/src/test/logback-test.xml @@ -1,12 +1,12 @@ - - - - - - - - - - - + + + + + + + + + + + \ No newline at end of file diff --git a/models/mongo/src/test/resources/TEST1.json b/core/src/test/resources/TEST1.json similarity index 94% rename from models/mongo/src/test/resources/TEST1.json rename to core/src/test/resources/TEST1.json index d95e8f64d..925fb0edb 100644 --- a/models/mongo/src/test/resources/TEST1.json +++ b/core/src/test/resources/TEST1.json @@ -1,69 +1,69 @@ -{ - "accession": "TEST1", - "sraAccession": "ERS01", - "name": "Test Sample", - "domain": "foozit", - "webinSubmissionAccountId": "", - "taxId": "9606", - "status": "PUBLIC", - "update": "2016-05-05T11:36:57Z", - "create": "2016-05-05T11:36:57Z", - "submitted": "2016-05-05T11:36:57Z", - "release": "2016-04-01T11:36:57Z", - "attributes": [ - { - "type": "organism", - "value": "Homo sapiens", - "iri": ["http://purl.obolibrary.org/obo/NCBITaxon_9606"] - }, - { - "type": "age", - "value": "3", - "unit": "year" - }, - { - "type": "organism part", - "value": "heart" - }, - { - "type": "organism part", - "value": "lung" - } - ], - "relationships": [ - { - "source": "TEST1", - "type": "derived from", - "target": "TEST2" - } - ], - "externalReferences": [ - { - "url": "http://www.google.com" - } - ], - "organizations": [ - { - "Name": "Jo Bloggs Inc", - "Role": "user", - "E-mail": "help@jobloggs.com", - "URL": "http://www.jobloggs.com" - } - ], - "contacts": [ - { - "FirstName": "Joe", - "LastName": "Bloggs", - "Name": "Joe Bloggs", - "E-mail": "jobloggs@joblogs.com", - "Role": "Submitter" - } - ], - "publications": [ - { - "doi": "10.1093/nar/gkt1081", - "pubmed_id": "24265224" - } - ], - "submittedVia": "JSON_API" -} +{ + "accession": "TEST1", + "sraAccession": "ERS01", + "name": "Test Sample", + "domain": "foozit", + "webinSubmissionAccountId": "", + "taxId": "9606", + "status": "PUBLIC", + "update": "2016-05-05T11:36:57Z", + "create": "2016-05-05T11:36:57Z", + "submitted": "2016-05-05T11:36:57Z", + "release": "2016-04-01T11:36:57Z", + "attributes": [ + { + "type": "organism", + "value": "Homo sapiens", + "iri": ["http://purl.obolibrary.org/obo/NCBITaxon_9606"] + }, + { + "type": "age", + "value": "3", + "unit": "year" + }, + { + "type": "organism part", + "value": "heart" + }, + { + "type": "organism part", + "value": "lung" + } + ], + "relationships": [ + { + "source": "TEST1", + "type": "derived from", + "target": "TEST2" + } + ], + "externalReferences": [ + { + "url": "http://www.google.com" + } + ], + "organizations": [ + { + "Name": "Jo Bloggs Inc", + "Role": "user", + "E-mail": "help@jobloggs.com", + "URL": "http://www.jobloggs.com" + } + ], + "contacts": [ + { + "FirstName": "Joe", + "LastName": "Bloggs", + "Name": "Joe Bloggs", + "E-mail": "jobloggs@joblogs.com", + "Role": "Submitter" + } + ], + "publications": [ + { + "doi": "10.1093/nar/gkt1081", + "pubmed_id": "24265224" + } + ], + "submittedVia": "JSON_API" +} diff --git a/models/mongo/src/test/resources/TEST2.json b/core/src/test/resources/TEST2.json similarity index 95% rename from models/mongo/src/test/resources/TEST2.json rename to core/src/test/resources/TEST2.json index 6cafe652c..564966f12 100644 --- a/models/mongo/src/test/resources/TEST2.json +++ b/core/src/test/resources/TEST2.json @@ -1,14 +1,14 @@ -{ - "accession": "TEST2", - "name": "Test Sample the second", - "domain": "foozit", - "update": "2016-05-05T11:36:57Z", - "release": "2016-04-01T11:36:57Z", - "attributes": [ - { - "type": "organism", - "value": "Homo sapiens", - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606" - } - ] -} +{ + "accession": "TEST2", + "name": "Test Sample the second", + "domain": "foozit", + "update": "2016-05-05T11:36:57Z", + "release": "2016-04-01T11:36:57Z", + "attributes": [ + { + "type": "organism", + "value": "Homo sapiens", + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606" + } + ] +} diff --git a/models/core/src/test/resources/TEST1.json b/core/src/test/resources/core/TEST1.json similarity index 94% rename from models/core/src/test/resources/TEST1.json rename to core/src/test/resources/core/TEST1.json index 4a604cad9..db4972bb9 100644 --- a/models/core/src/test/resources/TEST1.json +++ b/core/src/test/resources/core/TEST1.json @@ -1,77 +1,77 @@ -{ - "accession": "SAMEA1234", - "name": "Test Sample", - "domain": "abcde12345", - "update": "2016-05-05T11:36:57Z", - "release": "2016-04-01T11:36:57Z", - "submitted": "2016-04-01T11:36:57Z", - "characteristics": { - "age": [ - { - "text": "3", - "unit": "year" - } - ], - "organism": [ - { - "text": "Homo sapiens", - "ontologyTerms": [ - "http://purl.obolibrary.org/obo/NCBITaxon_9606" - ] - } - ], - "organism part": [ - { - "text": "heart" - }, - { - "text": "lung" - } - ], - "sex": [ - { - "text": "female", - "ontologyTerms": [ - "http://purl.obolibrary.org/obo/PATO_0000383", - "http://www.ebi.ac.uk/efo/EFO_0001265" - ] - } - ] - }, - "relationships": [ - { - "source": "SAMEA1234", - "type": "derived from", - "target": "SAMD4321" - } - ], - "externalReferences": [ - { - "url": "http://www.google.com" - } - ], - "organization": [ - { - "Name": "Jo Bloggs Inc", - "Role": "user", - "E-mail": "help@jobloggs.com", - "URL": "http://www.jobloggs.com" - } - ], - "contact": [ - { - "Name": "Joe Bloggs", - "FirstName": "Joe", - "LastName": "Bloggs", - "Role": "Submitter", - "E-mail": "jobloggs@joblogs.com" - - } - ], - "publications": [ - { - "doi": "10.1093/nar/gkt1081", - "pubmed_id": "24265224" - } - ] -} +{ + "accession": "SAMEA1234", + "name": "Test Sample", + "domain": "abcde12345", + "update": "2016-05-05T11:36:57Z", + "release": "2016-04-01T11:36:57Z", + "submitted": "2016-04-01T11:36:57Z", + "characteristics": { + "age": [ + { + "text": "3", + "unit": "year" + } + ], + "organism": [ + { + "text": "Homo sapiens", + "ontologyTerms": [ + "http://purl.obolibrary.org/obo/NCBITaxon_9606" + ] + } + ], + "organism part": [ + { + "text": "heart" + }, + { + "text": "lung" + } + ], + "sex": [ + { + "text": "female", + "ontologyTerms": [ + "http://purl.obolibrary.org/obo/PATO_0000383", + "http://www.ebi.ac.uk/efo/EFO_0001265" + ] + } + ] + }, + "relationships": [ + { + "source": "SAMEA1234", + "type": "derived from", + "target": "SAMD4321" + } + ], + "externalReferences": [ + { + "url": "http://www.google.com" + } + ], + "organization": [ + { + "Name": "Jo Bloggs Inc", + "Role": "user", + "E-mail": "help@jobloggs.com", + "URL": "http://www.jobloggs.com" + } + ], + "contact": [ + { + "Name": "Joe Bloggs", + "FirstName": "Joe", + "LastName": "Bloggs", + "Role": "Submitter", + "E-mail": "jobloggs@joblogs.com" + + } + ], + "publications": [ + { + "doi": "10.1093/nar/gkt1081", + "pubmed_id": "24265224" + } + ] +} diff --git a/models/core/src/test/resources/TEST1.xml b/core/src/test/resources/core/TEST1.xml similarity index 97% rename from models/core/src/test/resources/TEST1.xml rename to core/src/test/resources/core/TEST1.xml index c0c5a8903..3e1e5a816 100644 --- a/models/core/src/test/resources/TEST1.xml +++ b/core/src/test/resources/core/TEST1.xml @@ -1,58 +1,58 @@ - - - - - Test Sample - - - - - 3 - year - - - - - Homo sapiens - - - http://purl.obolibrary.org/obo/NCBITaxon_9606 - - - - - - heart - - - lung - - - - - female - - - http://purl.obolibrary.org/obo/PATO_0000383 - - - - - SAMD4321 - - - other - http://www.google.com - - + + + + + Test Sample + + + + + 3 + year + + + + + Homo sapiens + + + http://purl.obolibrary.org/obo/NCBITaxon_9606 + + + + + + heart + + + lung + + + + + female + + + http://purl.obolibrary.org/obo/PATO_0000383 + + + + + SAMD4321 + + + other + http://www.google.com + + diff --git a/models/core/src/test/resources/TEST2.json b/core/src/test/resources/core/TEST2.json similarity index 95% rename from models/core/src/test/resources/TEST2.json rename to core/src/test/resources/core/TEST2.json index 50d14051c..4ec521577 100644 --- a/models/core/src/test/resources/TEST2.json +++ b/core/src/test/resources/core/TEST2.json @@ -1,18 +1,18 @@ -{ - "accession": "TEST2", - "name": "Test Sample the second", - "update": "2016-05-05T11:36:57.00", - "release": "2016-04-01T11:36:57.00", - "updateDate": "2016-05-05", - "releaseDate": "2016-04-01", - "characteristics": { - "organism": [ - { - "text": "Homo sapiens", - "ontologyTerms": [ - "http://purl.obolibrary.org/obo/NCBITaxon_9606" - ] - } - ] - } -} +{ + "accession": "TEST2", + "name": "Test Sample the second", + "update": "2016-05-05T11:36:57.00", + "release": "2016-04-01T11:36:57.00", + "updateDate": "2016-05-05", + "releaseDate": "2016-04-01", + "characteristics": { + "organism": [ + { + "text": "Homo sapiens", + "ontologyTerms": [ + "http://purl.obolibrary.org/obo/NCBITaxon_9606" + ] + } + ] + } +} diff --git a/models/core/src/test/resources/curation.json b/core/src/test/resources/core/curation.json similarity index 93% rename from models/core/src/test/resources/curation.json rename to core/src/test/resources/core/curation.json index 626240c13..4d79be9dd 100644 --- a/models/core/src/test/resources/curation.json +++ b/core/src/test/resources/core/curation.json @@ -1,19 +1,19 @@ -{ - "attributesPre": [ - { - "type" : "organism", - "value" : "human" - },{ - "type" : "taxid", - "value" : "9606" - } - ], - "attributesPost": [ - { - "type" : "organism", - "value" : "Homo sapiens", - "iris" : [ "http://purl.obolibrary.org/obo/NCBITaxon_9606" ] - } - ] - +{ + "attributesPre": [ + { + "type" : "organism", + "value" : "human" + },{ + "type" : "taxid", + "value" : "9606" + } + ], + "attributesPost": [ + { + "type" : "organism", + "value" : "Homo sapiens", + "iris" : [ "http://purl.obolibrary.org/obo/NCBITaxon_9606" ] + } + ] + } \ No newline at end of file diff --git a/utils/validation/src/test/resources/examples/ols-responses/FBcv_0003016.json b/core/src/test/resources/ols-responses/FBcv_0003016.json similarity index 97% rename from utils/validation/src/test/resources/examples/ols-responses/FBcv_0003016.json rename to core/src/test/resources/ols-responses/FBcv_0003016.json index 4073dc4ba..14956d73e 100644 --- a/utils/validation/src/test/resources/examples/ols-responses/FBcv_0003016.json +++ b/core/src/test/resources/ols-responses/FBcv_0003016.json @@ -1,94 +1,94 @@ -{ - "_embedded": { - "terms": [ - { - "iri": "http://purl.obolibrary.org/obo/FBcv_0003016", - "label": "protein complex group", - "description": [ - "Genes whose protein products form a macromolecular complex (GO:0043234)." - ], - "annotation": { - "created_by": [ - "mmc46" - ], - "creation_date": [ - "2014-03-28T15:23:42Z" - ], - "has_obo_namespace": [ - "group_descriptor" - ], - "id": [ - "FBcv:0003016" - ] - }, - "synonyms": null, - "ontology_name": "fbcv", - "ontology_prefix": "FBcv", - "ontology_iri": "http://purl.obolibrary.org/obo/fbcv.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "FBcv_0003016", - "obo_id": "FBcv:0003016", - "in_subset": null, - "obo_definition_citation": [ - { - "definition": "Genes whose protein products form a macromolecular complex (GO:0043234).", - "oboXrefs": [ - { - "database": "FBC", - "id": "SM", - "description": null, - "url": null - }, - { - "database": "GOC", - "id": "go_curators", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/graph" - } - } - } - ] - }, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" - } - }, - "page": { - "size": 500, - "totalElements": 1, - "totalPages": 1, - "number": 0 - } +{ + "_embedded": { + "terms": [ + { + "iri": "http://purl.obolibrary.org/obo/FBcv_0003016", + "label": "protein complex group", + "description": [ + "Genes whose protein products form a macromolecular complex (GO:0043234)." + ], + "annotation": { + "created_by": [ + "mmc46" + ], + "creation_date": [ + "2014-03-28T15:23:42Z" + ], + "has_obo_namespace": [ + "group_descriptor" + ], + "id": [ + "FBcv:0003016" + ] + }, + "synonyms": null, + "ontology_name": "fbcv", + "ontology_prefix": "FBcv", + "ontology_iri": "http://purl.obolibrary.org/obo/fbcv.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "FBcv_0003016", + "obo_id": "FBcv:0003016", + "in_subset": null, + "obo_definition_citation": [ + { + "definition": "Genes whose protein products form a macromolecular complex (GO:0043234).", + "oboXrefs": [ + { + "database": "FBC", + "id": "SM", + "description": null, + "url": null + }, + { + "database": "GOC", + "id": "go_curators", + "description": null, + "url": null + } + ] + } + ], + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/graph" + } + } + } + ] + }, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" + } + }, + "page": { + "size": 500, + "totalElements": 1, + "totalPages": 1, + "number": 0 + } } \ No newline at end of file diff --git a/utils/ols/src/test/resources/examples/ols-responses/NCBITaxon_3702.json b/core/src/test/resources/ols-responses/NCBITaxon_3702.json similarity index 97% rename from utils/ols/src/test/resources/examples/ols-responses/NCBITaxon_3702.json rename to core/src/test/resources/ols-responses/NCBITaxon_3702.json index da9c1343c..5b81ec0b5 100644 --- a/utils/ols/src/test/resources/examples/ols-responses/NCBITaxon_3702.json +++ b/core/src/test/resources/ols-responses/NCBITaxon_3702.json @@ -1,609 +1,609 @@ -{ - "_embedded": { - "terms": [ - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "has_obo_namespace": [ - "ncbi_taxonomy" - ], - "has_related_synonym": [ - "At-" - ], - "id": [ - "NCBITaxon:3702" - ] - }, - "synonyms": null, - "ontology_name": "pr", - "ontology_prefix": "PR", - "ontology_iri": "http://purl.obolibrary.org/obo/pr.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": [ - { - "name": "At-", - "scope": "hasRelatedSynonym", - "type": "Unique short label for PRO terms for display purposes", - "xrefs": [ - { - "database": "PRO", - "id": "DAN", - "description": null, - "url": null - } - ] - } - ], - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": [ - "An organism of the species Arabidopsis thaliana" - ], - "annotation": { - "id": [ - "NCBITaxon:3702" - ] - }, - "synonyms": null, - "ontology_name": "cco", - "ontology_prefix": "CCO", - "ontology_iri": "http://purl.obolibrary.org/obo/cco", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "alternative term": [ - "thale cress", - "mouse-ear cress", - "Arabidopsis thaliana (thale cress)", - "thale-cress", - "Arbisopsis thaliana" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/ncbitaxon.owl" - ] - }, - "synonyms": null, - "ontology_name": "ero", - "ontology_prefix": "ERO", - "ontology_iri": "http://purl.obolibrary.org/obo/ero", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": [ - "A species of genus Arabidopsis." - ], - "annotation": { - "has_related_synonym": [ - "Arabidopsis thaliana (thale cress)", - "Arbisopsis thaliana" - ] - }, - "synonyms": [ - "Mouse-ear cress", - "Thale cress", - "Thale-cress" - ], - "ontology_name": "mirnao", - "ontology_prefix": "MIRNAO", - "ontology_iri": "http://purl.obolibrary.org/obo/mirnao", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "editor preferred label": [ - "Arabidopsis thaliana" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/ncbitaxon.owl" - ] - }, - "synonyms": [ - "thale cress", - "mouse-ear cress", - "Arabidopsis thaliana (thale cress)", - "thale-cress", - "Arabidopsis_thaliana", - "Arbisopsis thaliana" - ], - "ontology_name": "bao", - "ontology_prefix": "BAO", - "ontology_iri": "http://www.bioassayontology.org/bao/bao_complete.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "alternative term": [ - "thale cress", - "mouse-ear cress", - "thale-cress" - ], - "editor preferred label": [ - "Arabidopsis thaliana" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/ncbitaxon.owl" - ] - }, - "synonyms": null, - "ontology_name": "obi", - "ontology_prefix": "OBI", - "ontology_iri": "http://purl.obolibrary.org/obo/obi.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "alternative term": [ - "thale cress", - "mouse-ear cress", - "thale-cress" - ], - "editor preferred label": [ - "Arabidopsis thaliana" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/ncbitaxon.owl" - ] - }, - "synonyms": null, - "ontology_name": "ohmi", - "ontology_prefix": "OHMI", - "ontology_iri": "http://purl.obolibrary.org/obo/ohmi.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "has_obo_namespace": [ - "ncbi_taxonomy" - ], - "has_rank": [ - "http://purl.obolibrary.org/obo/NCBITaxon_species" - ], - "id": [ - "NCBITaxon:3702" - ], - "term editor": [ - "Class imported / merged by efoimporter", - "James Malone" - ] - }, - "synonyms": [ - "thale cress", - "mouse-ear cress", - "Arabidopsis thaliana (thale cress)", - "thale-cress", - "Arbisopsis thaliana" - ], - "ontology_name": "efo", - "ontology_prefix": "EFO", - "ontology_iri": "http://www.ebi.ac.uk/efo/efo.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - }, - "part_of": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FBFO_0000050" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "database_cross_reference": [ - "GC_ID:1" - ], - "has_exact_synonym": [ - "thale cress", - "mouse-ear cress", - "thale-cress" - ], - "has_obo_namespace": [ - "ncbi_taxonomy" - ], - "has_rank": [ - "http://purl.obolibrary.org/obo/NCBITaxon_species" - ], - "has_related_synonym": [ - "Arabidopsis thaliana (thale cress)", - "Arabidopsis_thaliana", - "thale kress", - "Arbisopsis thaliana" - ] - }, - "synonyms": null, - "ontology_name": "ncbitaxon", - "ontology_prefix": "NCBITAXON", - "ontology_iri": "http://purl.obolibrary.org/obo/ncbitaxon.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": true, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": [ - { - "database": "GC_ID", - "id": "1", - "description": null, - "url": null - } - ], - "obo_synonym": [ - { - "name": "thale-cress", - "scope": "hasExactSynonym", - "type": "common name", - "xrefs": [] - }, - { - "name": "thale kress", - "scope": "hasRelatedSynonym", - "type": "misspelling", - "xrefs": [] - }, - { - "name": "Arbisopsis thaliana", - "scope": "hasRelatedSynonym", - "type": "misspelling", - "xrefs": [] - }, - { - "name": "thale cress", - "scope": "hasExactSynonym", - "type": "genbank common name", - "xrefs": [] - }, - { - "name": "Arabidopsis_thaliana", - "scope": "hasRelatedSynonym", - "type": "misspelling", - "xrefs": [] - }, - { - "name": "mouse-ear cress", - "scope": "hasExactSynonym", - "type": "common name", - "xrefs": [] - }, - { - "name": "Arabidopsis thaliana (thale cress)", - "scope": "hasRelatedSynonym", - "type": "misspelling", - "xrefs": [] - } - ], - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - } - ] - }, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" - } - }, - "page": { - "size": 500, - "totalElements": 9, - "totalPages": 1, - "number": 0 - } +{ + "_embedded": { + "terms": [ + { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", + "label": "Arabidopsis thaliana", + "description": null, + "annotation": { + "has_obo_namespace": [ + "ncbi_taxonomy" + ], + "has_related_synonym": [ + "At-" + ], + "id": [ + "NCBITaxon:3702" + ] + }, + "synonyms": null, + "ontology_name": "pr", + "ontology_prefix": "PR", + "ontology_iri": "http://purl.obolibrary.org/obo/pr.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "NCBITaxon_3702", + "obo_id": "NCBITaxon:3702", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": [ + { + "name": "At-", + "scope": "hasRelatedSynonym", + "type": "Unique short label for PRO terms for display purposes", + "xrefs": [ + { + "database": "PRO", + "id": "DAN", + "description": null, + "url": null + } + ] + } + ], + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", + "label": "Arabidopsis thaliana", + "description": [ + "An organism of the species Arabidopsis thaliana" + ], + "annotation": { + "id": [ + "NCBITaxon:3702" + ] + }, + "synonyms": null, + "ontology_name": "cco", + "ontology_prefix": "CCO", + "ontology_iri": "http://purl.obolibrary.org/obo/cco", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "NCBITaxon_3702", + "obo_id": "NCBITaxon:3702", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", + "label": "Arabidopsis thaliana", + "description": null, + "annotation": { + "alternative term": [ + "thale cress", + "mouse-ear cress", + "Arabidopsis thaliana (thale cress)", + "thale-cress", + "Arbisopsis thaliana" + ], + "imported from": [ + "http://purl.obolibrary.org/obo/ncbitaxon.owl" + ] + }, + "synonyms": null, + "ontology_name": "ero", + "ontology_prefix": "ERO", + "ontology_iri": "http://purl.obolibrary.org/obo/ero", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "NCBITaxon_3702", + "obo_id": "NCBITaxon:3702", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", + "label": "Arabidopsis thaliana", + "description": [ + "A species of genus Arabidopsis." + ], + "annotation": { + "has_related_synonym": [ + "Arabidopsis thaliana (thale cress)", + "Arbisopsis thaliana" + ] + }, + "synonyms": [ + "Mouse-ear cress", + "Thale cress", + "Thale-cress" + ], + "ontology_name": "mirnao", + "ontology_prefix": "MIRNAO", + "ontology_iri": "http://purl.obolibrary.org/obo/mirnao", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "NCBITaxon_3702", + "obo_id": "NCBITaxon:3702", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", + "label": "Arabidopsis thaliana", + "description": null, + "annotation": { + "editor preferred label": [ + "Arabidopsis thaliana" + ], + "imported from": [ + "http://purl.obolibrary.org/obo/ncbitaxon.owl" + ] + }, + "synonyms": [ + "thale cress", + "mouse-ear cress", + "Arabidopsis thaliana (thale cress)", + "thale-cress", + "Arabidopsis_thaliana", + "Arbisopsis thaliana" + ], + "ontology_name": "bao", + "ontology_prefix": "BAO", + "ontology_iri": "http://www.bioassayontology.org/bao/bao_complete.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "NCBITaxon_3702", + "obo_id": "NCBITaxon:3702", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", + "label": "Arabidopsis thaliana", + "description": null, + "annotation": { + "alternative term": [ + "thale cress", + "mouse-ear cress", + "thale-cress" + ], + "editor preferred label": [ + "Arabidopsis thaliana" + ], + "imported from": [ + "http://purl.obolibrary.org/obo/ncbitaxon.owl" + ] + }, + "synonyms": null, + "ontology_name": "obi", + "ontology_prefix": "OBI", + "ontology_iri": "http://purl.obolibrary.org/obo/obi.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "NCBITaxon_3702", + "obo_id": "NCBITaxon:3702", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", + "label": "Arabidopsis thaliana", + "description": null, + "annotation": { + "alternative term": [ + "thale cress", + "mouse-ear cress", + "thale-cress" + ], + "editor preferred label": [ + "Arabidopsis thaliana" + ], + "imported from": [ + "http://purl.obolibrary.org/obo/ncbitaxon.owl" + ] + }, + "synonyms": null, + "ontology_name": "ohmi", + "ontology_prefix": "OHMI", + "ontology_iri": "http://purl.obolibrary.org/obo/ohmi.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "NCBITaxon_3702", + "obo_id": "NCBITaxon:3702", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", + "label": "Arabidopsis thaliana", + "description": null, + "annotation": { + "has_obo_namespace": [ + "ncbi_taxonomy" + ], + "has_rank": [ + "http://purl.obolibrary.org/obo/NCBITaxon_species" + ], + "id": [ + "NCBITaxon:3702" + ], + "term editor": [ + "Class imported / merged by efoimporter", + "James Malone" + ] + }, + "synonyms": [ + "thale cress", + "mouse-ear cress", + "Arabidopsis thaliana (thale cress)", + "thale-cress", + "Arbisopsis thaliana" + ], + "ontology_name": "efo", + "ontology_prefix": "EFO", + "ontology_iri": "http://www.ebi.ac.uk/efo/efo.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": true, + "is_root": false, + "short_form": "NCBITaxon_3702", + "obo_id": "NCBITaxon:3702", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" + }, + "children": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/children" + }, + "descendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/descendants" + }, + "hierarchicalChildren": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalChildren" + }, + "hierarchicalDescendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalDescendants" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" + }, + "part_of": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FBFO_0000050" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", + "label": "Arabidopsis thaliana", + "description": null, + "annotation": { + "database_cross_reference": [ + "GC_ID:1" + ], + "has_exact_synonym": [ + "thale cress", + "mouse-ear cress", + "thale-cress" + ], + "has_obo_namespace": [ + "ncbi_taxonomy" + ], + "has_rank": [ + "http://purl.obolibrary.org/obo/NCBITaxon_species" + ], + "has_related_synonym": [ + "Arabidopsis thaliana (thale cress)", + "Arabidopsis_thaliana", + "thale kress", + "Arbisopsis thaliana" + ] + }, + "synonyms": null, + "ontology_name": "ncbitaxon", + "ontology_prefix": "NCBITAXON", + "ontology_iri": "http://purl.obolibrary.org/obo/ncbitaxon.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": true, + "has_children": false, + "is_root": false, + "short_form": "NCBITaxon_3702", + "obo_id": "NCBITaxon:3702", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": [ + { + "database": "GC_ID", + "id": "1", + "description": null, + "url": null + } + ], + "obo_synonym": [ + { + "name": "thale-cress", + "scope": "hasExactSynonym", + "type": "common name", + "xrefs": [] + }, + { + "name": "thale kress", + "scope": "hasRelatedSynonym", + "type": "misspelling", + "xrefs": [] + }, + { + "name": "Arbisopsis thaliana", + "scope": "hasRelatedSynonym", + "type": "misspelling", + "xrefs": [] + }, + { + "name": "thale cress", + "scope": "hasExactSynonym", + "type": "genbank common name", + "xrefs": [] + }, + { + "name": "Arabidopsis_thaliana", + "scope": "hasRelatedSynonym", + "type": "misspelling", + "xrefs": [] + }, + { + "name": "mouse-ear cress", + "scope": "hasExactSynonym", + "type": "common name", + "xrefs": [] + }, + { + "name": "Arabidopsis thaliana (thale cress)", + "scope": "hasRelatedSynonym", + "type": "misspelling", + "xrefs": [] + } + ], + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" + } + } + } + ] + }, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" + } + }, + "page": { + "size": 500, + "totalElements": 9, + "totalPages": 1, + "number": 0 + } } \ No newline at end of file diff --git a/utils/ols/src/test/resources/examples/ols-responses/NCIT_C14207.json b/core/src/test/resources/ols-responses/NCIT_C14207.json similarity index 97% rename from utils/ols/src/test/resources/examples/ols-responses/NCIT_C14207.json rename to core/src/test/resources/ols-responses/NCIT_C14207.json index b91ccceba..e37e82c42 100644 --- a/utils/ols/src/test/resources/examples/ols-responses/NCIT_C14207.json +++ b/core/src/test/resources/ols-responses/NCIT_C14207.json @@ -1,136 +1,136 @@ -{ - "_embedded": { - "terms": [ - { - "iri": "http://purl.obolibrary.org/obo/NCIT_C14207", - "label": "Fish", - "description": [ - "A grouping of jawed and jawless vertebrate animals usually having fins and a covering of scales or plates, breathing by means of gills, and living almost entirely in the water." - ], - "annotation": { - "ALT_DEFINITION": [ - "Any jawed or jawless organisms in the phylum Chordata including the jawless fish, armored fish, cartilaginous fish, ray-finned fish and lobe-finned fish." - ], - "Contributing_Source": [ - "CDISC" - ], - "Legacy_Concept_Name": [ - "Fish" - ], - "Preferred_Name": [ - "Fish" - ], - "Semantic_Type": [ - "Fish" - ], - "UMLS_CUI": [ - "C0016163" - ], - "code": [ - "C14207" - ] - }, - "synonyms": [ - "FISH", - "Fish", - "Fish (NOS)" - ], - "ontology_name": "ncit", - "ontology_prefix": "NCIT", - "ontology_iri": "http://purl.obolibrary.org/obo/ncit.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": true, - "has_children": true, - "is_root": false, - "short_form": "NCIT_C14207", - "obo_id": "NCIT:C14207", - "in_subset": [ - "NCIT_C61410", - "NCIT_C77526", - "NCIT_C77808" - ], - "obo_definition_citation": [ - { - "definition": "A grouping of jawed and jawless vertebrate animals usually having fins and a covering of scales or plates, breathing by means of gills, and living almost entirely in the water.", - "oboXrefs": [ - { - "database": null, - "id": "NCI", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": [ - { - "name": "Fish (NOS)", - "scope": "hasExactSynonym", - "type": null, - "xrefs": [] - }, - { - "name": "FISH", - "scope": "hasExactSynonym", - "type": null, - "xrefs": [] - }, - { - "name": "Fish", - "scope": "hasExactSynonym", - "type": null, - "xrefs": [] - } - ], - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/graph" - } - } - } - ] - }, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" - } - }, - "page": { - "size": 500, - "totalElements": 1, - "totalPages": 1, - "number": 0 - } +{ + "_embedded": { + "terms": [ + { + "iri": "http://purl.obolibrary.org/obo/NCIT_C14207", + "label": "Fish", + "description": [ + "A grouping of jawed and jawless vertebrate animals usually having fins and a covering of scales or plates, breathing by means of gills, and living almost entirely in the water." + ], + "annotation": { + "ALT_DEFINITION": [ + "Any jawed or jawless organisms in the phylum Chordata including the jawless fish, armored fish, cartilaginous fish, ray-finned fish and lobe-finned fish." + ], + "Contributing_Source": [ + "CDISC" + ], + "Legacy_Concept_Name": [ + "Fish" + ], + "Preferred_Name": [ + "Fish" + ], + "Semantic_Type": [ + "Fish" + ], + "UMLS_CUI": [ + "C0016163" + ], + "code": [ + "C14207" + ] + }, + "synonyms": [ + "FISH", + "Fish", + "Fish (NOS)" + ], + "ontology_name": "ncit", + "ontology_prefix": "NCIT", + "ontology_iri": "http://purl.obolibrary.org/obo/ncit.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": true, + "has_children": true, + "is_root": false, + "short_form": "NCIT_C14207", + "obo_id": "NCIT:C14207", + "in_subset": [ + "NCIT_C61410", + "NCIT_C77526", + "NCIT_C77808" + ], + "obo_definition_citation": [ + { + "definition": "A grouping of jawed and jawless vertebrate animals usually having fins and a covering of scales or plates, breathing by means of gills, and living almost entirely in the water.", + "oboXrefs": [ + { + "database": null, + "id": "NCI", + "description": null, + "url": null + } + ] + } + ], + "obo_xref": null, + "obo_synonym": [ + { + "name": "Fish (NOS)", + "scope": "hasExactSynonym", + "type": null, + "xrefs": [] + }, + { + "name": "FISH", + "scope": "hasExactSynonym", + "type": null, + "xrefs": [] + }, + { + "name": "Fish", + "scope": "hasExactSynonym", + "type": null, + "xrefs": [] + } + ], + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/jstree" + }, + "children": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/children" + }, + "descendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/descendants" + }, + "hierarchicalChildren": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/hierarchicalChildren" + }, + "hierarchicalDescendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/hierarchicalDescendants" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/graph" + } + } + } + ] + }, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" + } + }, + "page": { + "size": 500, + "totalElements": 1, + "totalPages": 1, + "number": 0 + } } \ No newline at end of file diff --git a/utils/ols/src/test/resources/examples/ols-responses/PATO_0000384.json b/core/src/test/resources/ols-responses/PATO_0000384.json similarity index 97% rename from utils/ols/src/test/resources/examples/ols-responses/PATO_0000384.json rename to core/src/test/resources/ols-responses/PATO_0000384.json index 23873380a..03271d22c 100644 --- a/utils/ols/src/test/resources/examples/ols-responses/PATO_0000384.json +++ b/core/src/test/resources/ols-responses/PATO_0000384.json @@ -1,1369 +1,1369 @@ -{ - "_embedded": { - "terms": [ - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "bcgo", - "ontology_prefix": "BCGO", - "ontology_iri": "http://purl.obolibrary.org/obo/bcgo.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": null, - "annotation": { - "alternative term": [ - "http://mged.sourceforge.net/ontologies/MGEDOntology.owl#male" - ], - "comment": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "definition source": [ - "CARO:0000027", - "MO_652", - "PATO:0000384" - ], - "term editor": [ - "James Malone", - "Tomasz Adamusiak", - "Jie Zheng" - ] - }, - "synonyms": null, - "ontology_name": "clo", - "ontology_prefix": "CLO", - "ontology_iri": "http://purl.obolibrary.org/obo/clo.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "PATO_0000384", - "description": null, - "annotation": {}, - "synonyms": null, - "ontology_name": "ero", - "ontology_prefix": "ERO", - "ontology_iri": "http://purl.obolibrary.org/obo/ero", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": true, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ] - }, - "synonyms": null, - "ontology_name": "dpo", - "ontology_prefix": "FBcv", - "ontology_iri": "http://purl.obolibrary.org/obo/dpo", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": "MGED", - "id": "MGED", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ] - }, - "synonyms": null, - "ontology_name": "fbcv", - "ontology_prefix": "FBcv", - "ontology_iri": "http://purl.obolibrary.org/obo/fbcv.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": "MGED", - "id": "MGED", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "created_by": [ - "Topito" - ], - "creation_date": [ - "2013-05-28T14:34:52Z" - ], - "database_cross_reference": [ - "PATO:0000384" - ], - "has_obo_namespace": [ - "idomal_namespace" - ], - "id": [ - "PATO:0000384" - ] - }, - "synonyms": null, - "ontology_name": "idomal", - "ontology_prefix": "IDOMAL", - "ontology_iri": "http://purl.obolibrary.org/obo/idomal", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": "PATO", - "id": "0000384", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": [ - { - "database": "PATO", - "id": "0000384", - "description": null, - "url": null - } - ], - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ] - }, - "synonyms": null, - "ontology_name": "pato", - "ontology_prefix": "PATO", - "ontology_iri": "http://purl.obolibrary.org/obo/pato.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": true, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": "MGED", - "id": "MGED", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "imported from": [ - "pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "omiabis", - "ontology_prefix": "OMIABIS", - "ontology_iri": "http://purl.obolibrary.org/obo/omiabis", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": null, - "annotation": {}, - "synonyms": null, - "ontology_name": "oba", - "ontology_prefix": "OBA", - "ontology_iri": "http://purl.obolibrary.org/obo/oba", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ], - "in_subset": [ - "value_slim", - "mpath_slim" - ] - }, - "synonyms": null, - "ontology_name": "flopo", - "ontology_prefix": "FLOPO", - "ontology_iri": "http://purl.obolibrary.org/obo/flopo", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": null, - "id": "MGED:MGED", - "description": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "hasDbXref": [ - "NCBI BioSample: male" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "genepio", - "ontology_prefix": "GENEPIO", - "ontology_iri": "http://purl.obolibrary.org/obo/genepio.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": null, - "obo_xref": [ - { - "database": "NCBI BioSample", - "id": " male", - "description": null, - "url": null - } - ], - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "imported from": [ - "http://purl.obolibrary.org/obo/pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "eupath", - "ontology_prefix": "EUPATH", - "ontology_iri": "http://purl.obolibrary.org/obo/eupath.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": null, - "annotation": {}, - "synonyms": null, - "ontology_name": "hp", - "ontology_prefix": "HP", - "ontology_iri": "http://purl.obolibrary.org/obo/hp.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "bao", - "ontology_prefix": "BAO", - "ontology_iri": "http://www.bioassayontology.org/bao/bao_complete.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": null, - "annotation": {}, - "synonyms": null, - "ontology_name": "geno", - "ontology_prefix": "GENO", - "ontology_iri": "http://purl.obolibrary.org/obo/geno.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.org/obo/owl/PATO#PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "editor preferred term": [ - "male" - ], - "imported from": [ - "http://purl.org/obo/owl/PATO" - ] - }, - "synonyms": null, - "ontology_name": "flu", - "ontology_prefix": "FLU", - "ontology_iri": "http://purl.obolibrary.org/obo/flu/dev/flu.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "CARO_definition_citation": [ - "CARO:0000027" - ], - "MO_definition_citation": [ - "MO:652" - ], - "MSH_definition_citation": [ - "MSH:D008297" - ], - "NCI_Thesaurus_definition_citation": [ - "NCIt:C20197" - ], - "PATO_definition_citation": [ - "PATO:0000384" - ], - "SNOMEDCT_definition_citation": [ - "SNOMEDCT:248153007" - ], - "bioportal_provenance": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.[accessedResource: PATO:0000384][accessDate: 05-04-2011]" - ], - "term editor": [ - "James Malone", - "Tomasz Adamusiak", - "Jie Zheng" - ] - }, - "synonyms": null, - "ontology_name": "efo", - "ontology_prefix": "EFO", - "ontology_iri": "http://www.ebi.ac.uk/efo/efo.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "micro", - "ontology_prefix": "MICRO", - "ontology_iri": "http://purl.obolibrary.org/obo/MicrO.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ] - }, - "synonyms": null, - "ontology_name": "nbo", - "ontology_prefix": "NBO", - "ontology_iri": "http://purl.obolibrary.org/obo/nbo.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": "MGED", - "id": "MGED", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "editor preferred label": [ - "male" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "obi", - "ontology_prefix": "OBI", - "ontology_iri": "http://purl.obolibrary.org/obo/obi.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": null, - "annotation": {}, - "synonyms": null, - "ontology_name": "mp", - "ontology_prefix": "MP", - "ontology_iri": "http://purl.obolibrary.org/obo/mp.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ] - }, - "synonyms": null, - "ontology_name": "ms", - "ontology_prefix": "MS", - "ontology_iri": "http://purl.obolibrary.org/obo/ms.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": "MGED", - "id": "MGED", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - } - ] - }, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" - } - }, - "page": { - "size": 500, - "totalElements": 22, - "totalPages": 1, - "number": 0 - } +{ + "_embedded": { + "terms": [ + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "has_obo_namespace": [ + "quality" + ], + "id": [ + "PATO:0000384" + ], + "imported from": [ + "http://purl.obolibrary.org/obo/pato.owl" + ] + }, + "synonyms": null, + "ontology_name": "bcgo", + "ontology_prefix": "BCGO", + "ontology_iri": "http://purl.obolibrary.org/obo/bcgo.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": [ + "value_slim", + "mpath_slim" + ], + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": null, + "annotation": { + "alternative term": [ + "http://mged.sourceforge.net/ontologies/MGEDOntology.owl#male" + ], + "comment": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "definition source": [ + "CARO:0000027", + "MO_652", + "PATO:0000384" + ], + "term editor": [ + "James Malone", + "Tomasz Adamusiak", + "Jie Zheng" + ] + }, + "synonyms": null, + "ontology_name": "clo", + "ontology_prefix": "CLO", + "ontology_iri": "http://purl.obolibrary.org/obo/clo.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "PATO_0000384", + "description": null, + "annotation": {}, + "synonyms": null, + "ontology_name": "ero", + "ontology_prefix": "ERO", + "ontology_iri": "http://purl.obolibrary.org/obo/ero", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": true, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "has_obo_namespace": [ + "quality" + ], + "id": [ + "PATO:0000384" + ] + }, + "synonyms": null, + "ontology_name": "dpo", + "ontology_prefix": "FBcv", + "ontology_iri": "http://purl.obolibrary.org/obo/dpo", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": [ + { + "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", + "oboXrefs": [ + { + "database": "MGED", + "id": "MGED", + "description": null, + "url": null + } + ] + } + ], + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "has_obo_namespace": [ + "quality" + ], + "id": [ + "PATO:0000384" + ] + }, + "synonyms": null, + "ontology_name": "fbcv", + "ontology_prefix": "FBcv", + "ontology_iri": "http://purl.obolibrary.org/obo/fbcv.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": [ + { + "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", + "oboXrefs": [ + { + "database": "MGED", + "id": "MGED", + "description": null, + "url": null + } + ] + } + ], + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "created_by": [ + "Topito" + ], + "creation_date": [ + "2013-05-28T14:34:52Z" + ], + "database_cross_reference": [ + "PATO:0000384" + ], + "has_obo_namespace": [ + "idomal_namespace" + ], + "id": [ + "PATO:0000384" + ] + }, + "synonyms": null, + "ontology_name": "idomal", + "ontology_prefix": "IDOMAL", + "ontology_iri": "http://purl.obolibrary.org/obo/idomal", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": [ + { + "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", + "oboXrefs": [ + { + "database": "PATO", + "id": "0000384", + "description": null, + "url": null + } + ] + } + ], + "obo_xref": [ + { + "database": "PATO", + "id": "0000384", + "description": null, + "url": null + } + ], + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "has_obo_namespace": [ + "quality" + ], + "id": [ + "PATO:0000384" + ] + }, + "synonyms": null, + "ontology_name": "pato", + "ontology_prefix": "PATO", + "ontology_iri": "http://purl.obolibrary.org/obo/pato.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": true, + "has_children": true, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": [ + "value_slim", + "mpath_slim" + ], + "obo_definition_citation": [ + { + "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", + "oboXrefs": [ + { + "database": "MGED", + "id": "MGED", + "description": null, + "url": null + } + ] + } + ], + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "children": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" + }, + "descendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" + }, + "hierarchicalChildren": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" + }, + "hierarchicalDescendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "imported from": [ + "pato.owl" + ] + }, + "synonyms": null, + "ontology_name": "omiabis", + "ontology_prefix": "OMIABIS", + "ontology_iri": "http://purl.obolibrary.org/obo/omiabis", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": null, + "annotation": {}, + "synonyms": null, + "ontology_name": "oba", + "ontology_prefix": "OBA", + "ontology_iri": "http://purl.obolibrary.org/obo/oba", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": true, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "children": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" + }, + "descendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" + }, + "hierarchicalChildren": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" + }, + "hierarchicalDescendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "has_obo_namespace": [ + "quality" + ], + "id": [ + "PATO:0000384" + ], + "in_subset": [ + "value_slim", + "mpath_slim" + ] + }, + "synonyms": null, + "ontology_name": "flopo", + "ontology_prefix": "FLOPO", + "ontology_iri": "http://purl.obolibrary.org/obo/flopo", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": true, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": [ + { + "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", + "oboXrefs": [ + { + "database": null, + "id": "MGED:MGED", + "description": null + } + ] + } + ], + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "children": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" + }, + "descendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" + }, + "hierarchicalChildren": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" + }, + "hierarchicalDescendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "hasDbXref": [ + "NCBI BioSample: male" + ], + "imported from": [ + "http://purl.obolibrary.org/obo/pato.owl" + ] + }, + "synonyms": null, + "ontology_name": "genepio", + "ontology_prefix": "GENEPIO", + "ontology_iri": "http://purl.obolibrary.org/obo/genepio.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": true, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": [ + "value_slim", + "mpath_slim" + ], + "obo_definition_citation": null, + "obo_xref": [ + { + "database": "NCBI BioSample", + "id": " male", + "description": null, + "url": null + } + ], + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "children": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" + }, + "descendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" + }, + "hierarchicalChildren": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" + }, + "hierarchicalDescendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "imported from": [ + "http://purl.obolibrary.org/obo/pato.owl" + ] + }, + "synonyms": null, + "ontology_name": "eupath", + "ontology_prefix": "EUPATH", + "ontology_iri": "http://purl.obolibrary.org/obo/eupath.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": null, + "annotation": {}, + "synonyms": null, + "ontology_name": "hp", + "ontology_prefix": "HP", + "ontology_iri": "http://purl.obolibrary.org/obo/hp.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "has_obo_namespace": [ + "quality" + ], + "id": [ + "PATO:0000384" + ], + "imported from": [ + "http://purl.obolibrary.org/obo/pato.owl" + ] + }, + "synonyms": null, + "ontology_name": "bao", + "ontology_prefix": "BAO", + "ontology_iri": "http://www.bioassayontology.org/bao/bao_complete.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": [ + "value_slim", + "mpath_slim" + ], + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": null, + "annotation": {}, + "synonyms": null, + "ontology_name": "geno", + "ontology_prefix": "GENO", + "ontology_iri": "http://purl.obolibrary.org/obo/geno.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.org/obo/owl/PATO#PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "editor preferred term": [ + "male" + ], + "imported from": [ + "http://purl.org/obo/owl/PATO" + ] + }, + "synonyms": null, + "ontology_name": "flu", + "ontology_prefix": "FLU", + "ontology_iri": "http://purl.obolibrary.org/obo/flu/dev/flu.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "CARO_definition_citation": [ + "CARO:0000027" + ], + "MO_definition_citation": [ + "MO:652" + ], + "MSH_definition_citation": [ + "MSH:D008297" + ], + "NCI_Thesaurus_definition_citation": [ + "NCIt:C20197" + ], + "PATO_definition_citation": [ + "PATO:0000384" + ], + "SNOMEDCT_definition_citation": [ + "SNOMEDCT:248153007" + ], + "bioportal_provenance": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.[accessedResource: PATO:0000384][accessDate: 05-04-2011]" + ], + "term editor": [ + "James Malone", + "Tomasz Adamusiak", + "Jie Zheng" + ] + }, + "synonyms": null, + "ontology_name": "efo", + "ontology_prefix": "EFO", + "ontology_iri": "http://www.ebi.ac.uk/efo/efo.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "has_obo_namespace": [ + "quality" + ], + "id": [ + "PATO:0000384" + ], + "imported from": [ + "http://purl.obolibrary.org/obo/pato.owl" + ] + }, + "synonyms": null, + "ontology_name": "micro", + "ontology_prefix": "MICRO", + "ontology_iri": "http://purl.obolibrary.org/obo/MicrO.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": true, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": [ + "value_slim", + "mpath_slim" + ], + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "children": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" + }, + "descendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" + }, + "hierarchicalChildren": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" + }, + "hierarchicalDescendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "has_obo_namespace": [ + "quality" + ], + "id": [ + "PATO:0000384" + ] + }, + "synonyms": null, + "ontology_name": "nbo", + "ontology_prefix": "NBO", + "ontology_iri": "http://purl.obolibrary.org/obo/nbo.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": true, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": [ + "value_slim", + "mpath_slim" + ], + "obo_definition_citation": [ + { + "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", + "oboXrefs": [ + { + "database": "MGED", + "id": "MGED", + "description": null, + "url": null + } + ] + } + ], + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "children": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" + }, + "descendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" + }, + "hierarchicalChildren": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" + }, + "hierarchicalDescendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "editor preferred label": [ + "male" + ], + "imported from": [ + "http://purl.obolibrary.org/obo/pato.owl" + ] + }, + "synonyms": null, + "ontology_name": "obi", + "ontology_prefix": "OBI", + "ontology_iri": "http://purl.obolibrary.org/obo/obi.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": null, + "annotation": {}, + "synonyms": null, + "ontology_name": "mp", + "ontology_prefix": "MP", + "ontology_iri": "http://purl.obolibrary.org/obo/mp.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": false, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": null, + "obo_definition_citation": null, + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + }, + { + "iri": "http://purl.obolibrary.org/obo/PATO_0000384", + "label": "male", + "description": [ + "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." + ], + "annotation": { + "has_obo_namespace": [ + "quality" + ], + "id": [ + "PATO:0000384" + ] + }, + "synonyms": null, + "ontology_name": "ms", + "ontology_prefix": "MS", + "ontology_iri": "http://purl.obolibrary.org/obo/ms.owl", + "is_obsolete": false, + "term_replaced_by": null, + "is_defining_ontology": false, + "has_children": true, + "is_root": false, + "short_form": "PATO_0000384", + "obo_id": "PATO:0000384", + "in_subset": [ + "value_slim", + "mpath_slim" + ], + "obo_definition_citation": [ + { + "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", + "oboXrefs": [ + { + "database": "MGED", + "id": "MGED", + "description": null, + "url": null + } + ] + } + ], + "obo_xref": null, + "obo_synonym": null, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" + }, + "parents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" + }, + "ancestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" + }, + "hierarchicalParents": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" + }, + "hierarchicalAncestors": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" + }, + "jstree": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" + }, + "children": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" + }, + "descendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" + }, + "hierarchicalChildren": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" + }, + "hierarchicalDescendants": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" + }, + "graph": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" + } + } + } + ] + }, + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" + } + }, + "page": { + "size": 500, + "totalElements": 22, + "totalPages": 1, + "number": 0 + } } \ No newline at end of file diff --git a/utils/ols/src/test/resources/examples/ols-responses/invalid-term.json b/core/src/test/resources/ols-responses/invalid-term.json similarity index 94% rename from utils/ols/src/test/resources/examples/ols-responses/invalid-term.json rename to core/src/test/resources/ols-responses/invalid-term.json index d98e82679..dfc8c75d6 100644 --- a/utils/ols/src/test/resources/examples/ols-responses/invalid-term.json +++ b/core/src/test/resources/ols-responses/invalid-term.json @@ -1,13 +1,13 @@ -{ - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" - } - }, - "page": { - "size": 500, - "totalElements": 0, - "totalPages": 0, - "number": 0 - } +{ + "_links": { + "self": { + "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" + } + }, + "page": { + "size": 500, + "totalElements": 0, + "totalPages": 0, + "number": 0 + } } \ No newline at end of file diff --git a/commons/src/test/resources/phenopacket/basicPhenopacketJson.json b/core/src/test/resources/phenopacket/basicPhenopacketJson.json similarity index 95% rename from commons/src/test/resources/phenopacket/basicPhenopacketJson.json rename to core/src/test/resources/phenopacket/basicPhenopacketJson.json index 41d9e9f7a..fbd460991 100644 --- a/commons/src/test/resources/phenopacket/basicPhenopacketJson.json +++ b/core/src/test/resources/phenopacket/basicPhenopacketJson.json @@ -1,32 +1,32 @@ -{ - "subject": - { - "id": "id-individual" - }, - "biosamples": [ - { - "id": "id", - "description": "description", - "phenotypicFeatures": [{ - "type": { - "id": "term id", - "label": "term label" - } - }], - "individualId": "id-individual", - "ageOfIndividualAtCollection": { - "age": "0" - } - }], - "metaData": { - "created": "2018-07-04T14:00:23.294Z", - "createdBy": "Biosamples phenopacket exporter", - "resources": [{ - "id": "resource id", - "name": "resource name", - "namespacePrefix": "resource prefix", - "url": "url", - "version": "version" - }] - } -} +{ + "subject": + { + "id": "id-individual" + }, + "biosamples": [ + { + "id": "id", + "description": "description", + "phenotypicFeatures": [{ + "type": { + "id": "term id", + "label": "term label" + } + }], + "individualId": "id-individual", + "ageOfIndividualAtCollection": { + "age": "0" + } + }], + "metaData": { + "created": "2018-07-04T14:00:23.294Z", + "createdBy": "Biosamples phenopacket exporter", + "resources": [{ + "id": "resource id", + "name": "resource name", + "namespacePrefix": "resource prefix", + "url": "url", + "version": "version" + }] + } +} diff --git a/commons/src/test/resources/phenopacket/biosample1.json b/core/src/test/resources/phenopacket/biosample1.json similarity index 95% rename from commons/src/test/resources/phenopacket/biosample1.json rename to core/src/test/resources/phenopacket/biosample1.json index 180acbb80..989954e1a 100644 --- a/commons/src/test/resources/phenopacket/biosample1.json +++ b/core/src/test/resources/phenopacket/biosample1.json @@ -1,75 +1,75 @@ -{ - "name":"source GSM729765 1", - "accession":"SAMEA190234", - "domain":"self.BioSamplesMigration", - "release":"2011-05-23T23:00:00Z", - "update":"2018-01-20T13:55:29.870Z", - "characteristics":{ - "Organism":[ - { - "text":"Homo sapiens", - "ontologyTerms":[ - "http://purl.obolibrary.org/obo/NCBITaxon_9606" - ] - } - ], - "Sample_source_name":[ - { - "text":"A549/vec cells" - } - ], - "description":[ - { - "text":"Biological replicate 1" - } - ], - "disease":[ - { - "text":"Lung adenocarcinoma", - "ontologyTerms":[ - "http://www.ebi.ac.uk/efo/EFO_0000571" - ] - } - ], - "strain":[ - { - "text":"A549 lung cancer cells" - } - ], - "transfection":[ - { - "text":"Empty pLemiR plasmid" - } - ] - }, - "relationships":[ - { - "source":"SAMEG12970", - "type":"has member", - "target":"SAMEA190234" - } - ], - "organization":[ - { - "Name":"Roswell Park Cancer Institute", - "Role":"submitter " - } - ], - "contact":[ - { - "Name":"Eric Kannisto" - }, - { - "Name":"Reema Mallick" - }, - { - "Name":"Sai Yendamuri" - }, - { - "Name":"Santosh Patnaik" - } - ], - "releaseDate":"2011-05-23", - "updateDate":"2018-01-20" - +{ + "name":"source GSM729765 1", + "accession":"SAMEA190234", + "domain":"self.BioSamplesMigration", + "release":"2011-05-23T23:00:00Z", + "update":"2018-01-20T13:55:29.870Z", + "characteristics":{ + "Organism":[ + { + "text":"Homo sapiens", + "ontologyTerms":[ + "http://purl.obolibrary.org/obo/NCBITaxon_9606" + ] + } + ], + "Sample_source_name":[ + { + "text":"A549/vec cells" + } + ], + "description":[ + { + "text":"Biological replicate 1" + } + ], + "disease":[ + { + "text":"Lung adenocarcinoma", + "ontologyTerms":[ + "http://www.ebi.ac.uk/efo/EFO_0000571" + ] + } + ], + "strain":[ + { + "text":"A549 lung cancer cells" + } + ], + "transfection":[ + { + "text":"Empty pLemiR plasmid" + } + ] + }, + "relationships":[ + { + "source":"SAMEG12970", + "type":"has member", + "target":"SAMEA190234" + } + ], + "organization":[ + { + "Name":"Roswell Park Cancer Institute", + "Role":"submitter " + } + ], + "contact":[ + { + "Name":"Eric Kannisto" + }, + { + "Name":"Reema Mallick" + }, + { + "Name":"Sai Yendamuri" + }, + { + "Name":"Santosh Patnaik" + } + ], + "releaseDate":"2011-05-23", + "updateDate":"2018-01-20" + } \ No newline at end of file diff --git a/commons/src/test/resources/phenopacket/biosample3.json b/core/src/test/resources/phenopacket/biosample3.json similarity index 95% rename from commons/src/test/resources/phenopacket/biosample3.json rename to core/src/test/resources/phenopacket/biosample3.json index b6e2c7a27..544d6c249 100644 --- a/commons/src/test/resources/phenopacket/biosample3.json +++ b/core/src/test/resources/phenopacket/biosample3.json @@ -1,87 +1,87 @@ -{ - "name":"source GSM996317 1", - "accession":"SAMEA1534312", - "domain":"self.BioSamplesMigration", - "release":"2012-09-30T23:00:00Z", - "update":"2018-04-04T15:37:26.093Z", - "characteristics":{ - "Organism":[ - { - "text":"Homo sapiens", - "ontologyTerms":[ - "http://purl.obolibrary.org/obo/NCBITaxon_9606" - ] - } - ], - "Sample_source_name":[ - { - "text":"Peripheral blood", - "ontologyTerms":[ - "http://purl.obolibrary.org/obo/UBERON_0000178" - ] - } - ], - "Sample_title":[ - { - "text":"active TB 2 months post treatment [416/02]" - } - ], - "description":[ - { - "text":"South Africa patient blood" - } - ], - "disease":[ - { - "text":"PTB" - } - ], - "geographical location":[ - { - "text":"South Africa patient blood" - } - ], - "organism part":[ - { - "text":"whole blood", - "ontologyTerms":[ - "http://www.ebi.ac.uk/efo/EFO_0000296" - ] - } - ], - "status":[ - { - "text":"active TB 2 months post treatment" - } - ], - "time point months":[ - { - "text":"2" - } - ] - }, - "relationships":[ - { - "source":"SAMEG114472", - "type":"has member", - "target":"SAMEA1534312" - } - ], - "organization":[ - { - "Name":"MRC National Institute for Medical Research", - "Role":"submitter " - } - ], - "contact":[ - { - "Name":"Anne O'Garra" - }, - { - "Name":"Chloe Bloom" - } - ], - "releaseDate":"2012-09-30", - "updateDate":"2018-04-04" - +{ + "name":"source GSM996317 1", + "accession":"SAMEA1534312", + "domain":"self.BioSamplesMigration", + "release":"2012-09-30T23:00:00Z", + "update":"2018-04-04T15:37:26.093Z", + "characteristics":{ + "Organism":[ + { + "text":"Homo sapiens", + "ontologyTerms":[ + "http://purl.obolibrary.org/obo/NCBITaxon_9606" + ] + } + ], + "Sample_source_name":[ + { + "text":"Peripheral blood", + "ontologyTerms":[ + "http://purl.obolibrary.org/obo/UBERON_0000178" + ] + } + ], + "Sample_title":[ + { + "text":"active TB 2 months post treatment [416/02]" + } + ], + "description":[ + { + "text":"South Africa patient blood" + } + ], + "disease":[ + { + "text":"PTB" + } + ], + "geographical location":[ + { + "text":"South Africa patient blood" + } + ], + "organism part":[ + { + "text":"whole blood", + "ontologyTerms":[ + "http://www.ebi.ac.uk/efo/EFO_0000296" + ] + } + ], + "status":[ + { + "text":"active TB 2 months post treatment" + } + ], + "time point months":[ + { + "text":"2" + } + ] + }, + "relationships":[ + { + "source":"SAMEG114472", + "type":"has member", + "target":"SAMEA1534312" + } + ], + "organization":[ + { + "Name":"MRC National Institute for Medical Research", + "Role":"submitter " + } + ], + "contact":[ + { + "Name":"Anne O'Garra" + }, + { + "Name":"Chloe Bloom" + } + ], + "releaseDate":"2012-09-30", + "updateDate":"2018-04-04" + } \ No newline at end of file diff --git a/commons/src/test/resources/phenopacket/biosamples2.json b/core/src/test/resources/phenopacket/biosamples2.json similarity index 95% rename from commons/src/test/resources/phenopacket/biosamples2.json rename to core/src/test/resources/phenopacket/biosamples2.json index 92ac05e36..59504bd00 100644 --- a/commons/src/test/resources/phenopacket/biosamples2.json +++ b/core/src/test/resources/phenopacket/biosamples2.json @@ -1,85 +1,85 @@ -{ - "name":"source GSM498707 1", - "accession":"SAMEA182635", - "domain":"self.BioSamplesMigration", - "release":"2010-12-01T00:00:00Z", - "update":"2018-04-04T15:56:20.902Z", - "characteristics":{ - "Organism":[ - { - "text":"Homo sapiens", - "ontologyTerms":[ - "http://purl.obolibrary.org/obo/NCBITaxon_9606" - ] - } - ], - "Sample_source_name":[ - { - "text":"Normal colonic mucosa located in proximal colon from a familial HPPS patient" - } - ], - "colon location":[ - { - "text":"proximal", - "ontologyTerms":[ - "http://www.ebi.ac.uk/efo/EFO_0001661" - ] - } - ], - "description":[ - { - "text":"A643MNP1" - } - ], - "disease":[ - { - "text":"hyperplastic polyposis syndrome (HPPS)" - } - ], - "hereditary nature of hpps":[ - { - "text":"familial" - } - ], - "histology":[ - { - "text":"normal colonic mucosa" - } - ], - "organism part":[ - { - "text":"colon", - "ontologyTerms":[ - "http://www.ebi.ac.uk/efo/EFO_0000361" - ] - } - ] - }, - "relationships":[ - { - "source":"SAMEG12675", - "type":"has member", - "target":"SAMEA182635" - } - ], - "organization":[ - { - "Name":"Instituto Português de Oncologia de Lisboa Francisco Gentil, EPE", - "Role":"submitter " - } - ], - "contact":[ - { - "Name":"Cristina Albuquerque" - }, - { - "Name":"Joana Dinis" - }, - { - "Name":"Patrícia Silva" - } - ], - "releaseDate":"2010-12-01", - "updateDate":"2018-04-04" - +{ + "name":"source GSM498707 1", + "accession":"SAMEA182635", + "domain":"self.BioSamplesMigration", + "release":"2010-12-01T00:00:00Z", + "update":"2018-04-04T15:56:20.902Z", + "characteristics":{ + "Organism":[ + { + "text":"Homo sapiens", + "ontologyTerms":[ + "http://purl.obolibrary.org/obo/NCBITaxon_9606" + ] + } + ], + "Sample_source_name":[ + { + "text":"Normal colonic mucosa located in proximal colon from a familial HPPS patient" + } + ], + "colon location":[ + { + "text":"proximal", + "ontologyTerms":[ + "http://www.ebi.ac.uk/efo/EFO_0001661" + ] + } + ], + "description":[ + { + "text":"A643MNP1" + } + ], + "disease":[ + { + "text":"hyperplastic polyposis syndrome (HPPS)" + } + ], + "hereditary nature of hpps":[ + { + "text":"familial" + } + ], + "histology":[ + { + "text":"normal colonic mucosa" + } + ], + "organism part":[ + { + "text":"colon", + "ontologyTerms":[ + "http://www.ebi.ac.uk/efo/EFO_0000361" + ] + } + ] + }, + "relationships":[ + { + "source":"SAMEG12675", + "type":"has member", + "target":"SAMEA182635" + } + ], + "organization":[ + { + "Name":"Instituto Português de Oncologia de Lisboa Francisco Gentil, EPE", + "Role":"submitter " + } + ], + "contact":[ + { + "Name":"Cristina Albuquerque" + }, + { + "Name":"Joana Dinis" + }, + { + "Name":"Patrícia Silva" + } + ], + "releaseDate":"2010-12-01", + "updateDate":"2018-04-04" + } \ No newline at end of file diff --git a/commons/src/test/resources/phenopacket/diseaseTestPhenopacket.json b/core/src/test/resources/phenopacket/diseaseTestPhenopacket.json similarity index 95% rename from commons/src/test/resources/phenopacket/diseaseTestPhenopacket.json rename to core/src/test/resources/phenopacket/diseaseTestPhenopacket.json index abfd9864a..51b36649a 100644 --- a/commons/src/test/resources/phenopacket/diseaseTestPhenopacket.json +++ b/core/src/test/resources/phenopacket/diseaseTestPhenopacket.json @@ -1,36 +1,36 @@ -{ - "subject": { - "id": "id-individual" - }, - "biosamples": [ - { - "id": "id", - "description": "description", - "individualId": "id-individual", - "ageOfIndividualAtCollection": { - "age": "0" - } - } - ], - "diseases": [ - { - "term": { - "id": "term id", - "label": "term label" - } - } - ], - "metaData": { - "created": "2018-07-04T17:02:13.103Z", - "createdBy": "Biosamples phenopacket exporter", - "resources": [ - { - "id": "resource id", - "name": "resource name", - "namespacePrefix": "resource prefix", - "url": "url", - "version": "version" - } - ] - } +{ + "subject": { + "id": "id-individual" + }, + "biosamples": [ + { + "id": "id", + "description": "description", + "individualId": "id-individual", + "ageOfIndividualAtCollection": { + "age": "0" + } + } + ], + "diseases": [ + { + "term": { + "id": "term id", + "label": "term label" + } + } + ], + "metaData": { + "created": "2018-07-04T17:02:13.103Z", + "createdBy": "Biosamples phenopacket exporter", + "resources": [ + { + "id": "resource id", + "name": "resource name", + "namespacePrefix": "resource prefix", + "url": "url", + "version": "version" + } + ] + } } \ No newline at end of file diff --git a/commons/src/test/resources/phenopacket/individualMappingTestPhenopacket.json b/core/src/test/resources/phenopacket/individualMappingTestPhenopacket.json similarity index 95% rename from commons/src/test/resources/phenopacket/individualMappingTestPhenopacket.json rename to core/src/test/resources/phenopacket/individualMappingTestPhenopacket.json index e1876eb2f..9e4ede08a 100644 --- a/commons/src/test/resources/phenopacket/individualMappingTestPhenopacket.json +++ b/core/src/test/resources/phenopacket/individualMappingTestPhenopacket.json @@ -1,38 +1,38 @@ -{ - "subject": - { - "id": "id-individual", - "taxonomy": { - "id": "term id", - "label": "term label" - } - } - , - "biosamples": [ - { - "id": "id", - "description": "description", - "individualId": "id-individual", - "taxonomy": { - "id": "term id", - "label": "term label" - }, - "ageOfIndividualAtCollection": { - "age": "0" - } - } - ], - "metaData": { - "created": "2018-07-30T14:33:18.284Z", - "createdBy": "Biosamples phenopacket exporter", - "resources": [ - { - "id": "resource id", - "name": "resource name", - "namespacePrefix": "resource prefix", - "url": "url", - "version": "version" - } - ] - } +{ + "subject": + { + "id": "id-individual", + "taxonomy": { + "id": "term id", + "label": "term label" + } + } + , + "biosamples": [ + { + "id": "id", + "description": "description", + "individualId": "id-individual", + "taxonomy": { + "id": "term id", + "label": "term label" + }, + "ageOfIndividualAtCollection": { + "age": "0" + } + } + ], + "metaData": { + "created": "2018-07-30T14:33:18.284Z", + "createdBy": "Biosamples phenopacket exporter", + "resources": [ + { + "id": "resource id", + "name": "resource name", + "namespacePrefix": "resource prefix", + "url": "url", + "version": "version" + } + ] + } } \ No newline at end of file diff --git a/commons/src/test/resources/phenopacket/phenopacket1.json b/core/src/test/resources/phenopacket/phenopacket1.json similarity index 95% rename from commons/src/test/resources/phenopacket/phenopacket1.json rename to core/src/test/resources/phenopacket/phenopacket1.json index b45e7ab54..fc2fb5002 100644 --- a/commons/src/test/resources/phenopacket/phenopacket1.json +++ b/core/src/test/resources/phenopacket/phenopacket1.json @@ -1,50 +1,50 @@ -{ - "subject": { - "id": "SAMEA190234-individual", - "taxonomy": { - "id": "NCBITaxon:9606", - "label": "Homo sapiens" - } - }, - "biosamples": [ - { - "id": "SAMEA190234", - "description": "Biological replicate 1", - "individualId": "SAMEA190234-individual", - "taxonomy": { - "id": "NCBITaxon:9606", - "label": "Homo sapiens" - }, - "ageOfIndividualAtCollection": { - } - } - ], - "diseases": [ - { - "term": { - "id": "EFO:0000571", - "label": "lung adenocarcinoma" - } - } - ], - "metaData": { - "created": "2018-06-28T15:47:17.684Z", - "createdBy": "Biosamples phenopacket exporter", - "resources": [ - { - "id": "ncbitaxon", - "name": "NCBI organismal classification", - "namespacePrefix": "NCBITaxon", - "url": "http://purl.obolibrary.org/obo/ncbitaxon.owl", - "version": "2018-07-27" - }, - { - "id": "efo", - "name": "Experimental Factor Ontology", - "namespacePrefix": "EFO", - "url": "http://www.ebi.ac.uk/efo/efo.owl", - "version": "3.7.0" - } - ] - } -} +{ + "subject": { + "id": "SAMEA190234-individual", + "taxonomy": { + "id": "NCBITaxon:9606", + "label": "Homo sapiens" + } + }, + "biosamples": [ + { + "id": "SAMEA190234", + "description": "Biological replicate 1", + "individualId": "SAMEA190234-individual", + "taxonomy": { + "id": "NCBITaxon:9606", + "label": "Homo sapiens" + }, + "ageOfIndividualAtCollection": { + } + } + ], + "diseases": [ + { + "term": { + "id": "EFO:0000571", + "label": "lung adenocarcinoma" + } + } + ], + "metaData": { + "created": "2018-06-28T15:47:17.684Z", + "createdBy": "Biosamples phenopacket exporter", + "resources": [ + { + "id": "ncbitaxon", + "name": "NCBI organismal classification", + "namespacePrefix": "NCBITaxon", + "url": "http://purl.obolibrary.org/obo/ncbitaxon.owl", + "version": "2018-07-27" + }, + { + "id": "efo", + "name": "Experimental Factor Ontology", + "namespacePrefix": "EFO", + "url": "http://www.ebi.ac.uk/efo/efo.owl", + "version": "3.7.0" + } + ] + } +} diff --git a/commons/src/test/resources/phenopacket/phenopacket2.json b/core/src/test/resources/phenopacket/phenopacket2.json similarity index 95% rename from commons/src/test/resources/phenopacket/phenopacket2.json rename to core/src/test/resources/phenopacket/phenopacket2.json index d429e23cf..3e261158f 100644 --- a/commons/src/test/resources/phenopacket/phenopacket2.json +++ b/core/src/test/resources/phenopacket/phenopacket2.json @@ -1,57 +1,57 @@ -{ - "subject": { - "id": "SAMEA182635-individual", - "taxonomy": { - "id": "NCBITaxon:9606", - "label": "Homo sapiens" - } - }, - "biosamples": [ - { - "id": "SAMEA182635", - "description": "A643MNP1", - "phenotypicFeatures": [ - { - "type": { - "id": "EFO:0000361", - "label": "obsolete_colon" - } - } - ], - "individualId": "SAMEA182635-individual", - "taxonomy": { - "id": "NCBITaxon:9606", - "label": "Homo sapiens" - }, - "ageOfIndividualAtCollection": { - } - } - ], - "diseases": [ - { - "term": { - "label": "hyperplastic polyposis syndrome (HPPS)" - } - } - ], - "metaData": { - "created": "2018-06-28T15:50:05.007Z", - "createdBy": "Biosamples phenopacket exporter", - "resources": [ - { - "id": "ncbitaxon", - "name": "NCBI organismal classification", - "namespacePrefix": "NCBITaxon", - "url": "http://purl.obolibrary.org/obo/ncbitaxon.owl", - "version": "2018-07-27" - }, - { - "id": "efo", - "name": "Experimental Factor Ontology", - "namespacePrefix": "EFO", - "url": "http://www.ebi.ac.uk/efo/efo.owl", - "version": "2.104" - } - ] - } -} +{ + "subject": { + "id": "SAMEA182635-individual", + "taxonomy": { + "id": "NCBITaxon:9606", + "label": "Homo sapiens" + } + }, + "biosamples": [ + { + "id": "SAMEA182635", + "description": "A643MNP1", + "phenotypicFeatures": [ + { + "type": { + "id": "EFO:0000361", + "label": "obsolete_colon" + } + } + ], + "individualId": "SAMEA182635-individual", + "taxonomy": { + "id": "NCBITaxon:9606", + "label": "Homo sapiens" + }, + "ageOfIndividualAtCollection": { + } + } + ], + "diseases": [ + { + "term": { + "label": "hyperplastic polyposis syndrome (HPPS)" + } + } + ], + "metaData": { + "created": "2018-06-28T15:50:05.007Z", + "createdBy": "Biosamples phenopacket exporter", + "resources": [ + { + "id": "ncbitaxon", + "name": "NCBI organismal classification", + "namespacePrefix": "NCBITaxon", + "url": "http://purl.obolibrary.org/obo/ncbitaxon.owl", + "version": "2018-07-27" + }, + { + "id": "efo", + "name": "Experimental Factor Ontology", + "namespacePrefix": "EFO", + "url": "http://www.ebi.ac.uk/efo/efo.owl", + "version": "2.104" + } + ] + } +} diff --git a/commons/src/test/resources/phenopacket/phenopacket3.json b/core/src/test/resources/phenopacket/phenopacket3.json similarity index 95% rename from commons/src/test/resources/phenopacket/phenopacket3.json rename to core/src/test/resources/phenopacket/phenopacket3.json index eff130e11..6e98150d4 100644 --- a/commons/src/test/resources/phenopacket/phenopacket3.json +++ b/core/src/test/resources/phenopacket/phenopacket3.json @@ -1,70 +1,70 @@ -{ - "subject": { - "id": "SAMEA1534312-individual", - "taxonomy": { - "id": "NCBITaxon:9606", - "label": "Homo sapiens" - } - }, - "biosamples": [ - { - "id": "SAMEA1534312", - "description": "South Africa patient blood", - "phenotypicFeatures": [ - { - "type": { - "id": "UBERON:0000178", - "label": "blood" - } - }, - { - "type": { - "id": "EFO:0000296", - "label": "blood" - } - } - ], - "individualId": "SAMEA1534312-individual", - "taxonomy": { - "id": "NCBITaxon:9606", - "label": "Homo sapiens" - }, - "ageOfIndividualAtCollection": { - } - } - ], - "diseases": [ - { - "term": { - "label": "PTB" - } - } - ], - "metaData": { - "created": "2018-06-28T15:52:50.638Z", - "createdBy": "Biosamples phenopacket exporter", - "resources": [ - { - "id": "ncbitaxon", - "name": "NCBI organismal classification", - "namespacePrefix": "NCBITaxon", - "url": "http://purl.obolibrary.org/obo/ncbitaxon.owl", - "version": "2018-07-27" - }, - { - "id": "uberon", - "name": "Uber-anatomy ontology", - "namespacePrefix": "UBERON", - "url": "http://purl.obolibrary.org/obo/uberon.owl", - "version": "2018-11-25" - }, - { - "id": "efo", - "name": "Experimental Factor Ontology", - "namespacePrefix": "EFO", - "url": "http://www.ebi.ac.uk/efo/efo.owl", - "version": "2.104" - } - ] - } -} +{ + "subject": { + "id": "SAMEA1534312-individual", + "taxonomy": { + "id": "NCBITaxon:9606", + "label": "Homo sapiens" + } + }, + "biosamples": [ + { + "id": "SAMEA1534312", + "description": "South Africa patient blood", + "phenotypicFeatures": [ + { + "type": { + "id": "UBERON:0000178", + "label": "blood" + } + }, + { + "type": { + "id": "EFO:0000296", + "label": "blood" + } + } + ], + "individualId": "SAMEA1534312-individual", + "taxonomy": { + "id": "NCBITaxon:9606", + "label": "Homo sapiens" + }, + "ageOfIndividualAtCollection": { + } + } + ], + "diseases": [ + { + "term": { + "label": "PTB" + } + } + ], + "metaData": { + "created": "2018-06-28T15:52:50.638Z", + "createdBy": "Biosamples phenopacket exporter", + "resources": [ + { + "id": "ncbitaxon", + "name": "NCBI organismal classification", + "namespacePrefix": "NCBITaxon", + "url": "http://purl.obolibrary.org/obo/ncbitaxon.owl", + "version": "2018-07-27" + }, + { + "id": "uberon", + "name": "Uber-anatomy ontology", + "namespacePrefix": "UBERON", + "url": "http://purl.obolibrary.org/obo/uberon.owl", + "version": "2018-11-25" + }, + { + "id": "efo", + "name": "Experimental Factor Ontology", + "namespacePrefix": "EFO", + "url": "http://www.ebi.ac.uk/efo/efo.owl", + "version": "2.104" + } + ] + } +} diff --git a/commons/src/test/resources/phenopacket/phenopacket_1.json b/core/src/test/resources/phenopacket/phenopacket_1.json similarity index 96% rename from commons/src/test/resources/phenopacket/phenopacket_1.json rename to core/src/test/resources/phenopacket/phenopacket_1.json index 4a5166031..07db2484f 100644 --- a/commons/src/test/resources/phenopacket/phenopacket_1.json +++ b/core/src/test/resources/phenopacket/phenopacket_1.json @@ -1,68 +1,68 @@ -{ - "id": "SAMETAG2031", - "subject": { - "id": "SAMETAG2031-individual", - "sex": "MALE", - "taxonomy": { - "id": "NCBITaxon:9606", - "label": "Homo sapiens" - } - }, - "biosamples": [{ - "id": "SAMETAG2031", - "individualId": "SAMETAG2031-individual", - "description": "test description", - "sampledTissue": { - "id": "UBERON:0002107", - "label": "liver" - }, - "taxonomy": { - "id": "NCBITaxon:9606", - "label": "Homo sapiens" - }, - "ageOfIndividualAtCollection": { - "age": "30" - } - }], - "diseases": [{ - "term": { - "id": "EFO:0000365", - "label": "colorectal adenocarcinoma" - } - }, { - "term": { - "id": "MONDO:0002491", - "label": "lung disease" - } - }], - "metaData": { - "created": "1970-01-01T00:00:00Z", - "createdBy": "Biosamples phenopacket exporter", - "resources": [{ - "id": "ncbitaxon", - "name": "NCBI organismal classification", - "url": "http://purl.obolibrary.org/obo/ncbitaxon.owl", - "version": "2020-04-18" - }, { - "id": "pato", - "name": "PATO - the Phenotype And Trait Ontology", - "url": "http://purl.obolibrary.org/obo/pato.owl", - "version": "2020-08-02" - }, { - "id": "mondo", - "name": "Mondo Disease Ontology", - "url": "http://purl.obolibrary.org/obo/mondo.owl", - "version": "2020-10-21" - }, { - "id": "efo", - "name": "Experimental Factor Ontology", - "url": "http://www.ebi.ac.uk/efo/efo.owl", - "version": "3.23.0" - }, { - "id": "uberon", - "name": "Uber-anatomy ontology", - "url": "http://purl.obolibrary.org/obo/uberon.owl", - "version": "2020-09-16" - }] - } -} +{ + "id": "SAMETAG2031", + "subject": { + "id": "SAMETAG2031-individual", + "sex": "MALE", + "taxonomy": { + "id": "NCBITaxon:9606", + "label": "Homo sapiens" + } + }, + "biosamples": [{ + "id": "SAMETAG2031", + "individualId": "SAMETAG2031-individual", + "description": "test description", + "sampledTissue": { + "id": "UBERON:0002107", + "label": "liver" + }, + "taxonomy": { + "id": "NCBITaxon:9606", + "label": "Homo sapiens" + }, + "ageOfIndividualAtCollection": { + "age": "30" + } + }], + "diseases": [{ + "term": { + "id": "EFO:0000365", + "label": "colorectal adenocarcinoma" + } + }, { + "term": { + "id": "MONDO:0002491", + "label": "lung disease" + } + }], + "metaData": { + "created": "1970-01-01T00:00:00Z", + "createdBy": "Biosamples phenopacket exporter", + "resources": [{ + "id": "ncbitaxon", + "name": "NCBI organismal classification", + "url": "http://purl.obolibrary.org/obo/ncbitaxon.owl", + "version": "2020-04-18" + }, { + "id": "pato", + "name": "PATO - the Phenotype And Trait Ontology", + "url": "http://purl.obolibrary.org/obo/pato.owl", + "version": "2020-08-02" + }, { + "id": "mondo", + "name": "Mondo Disease Ontology", + "url": "http://purl.obolibrary.org/obo/mondo.owl", + "version": "2020-10-21" + }, { + "id": "efo", + "name": "Experimental Factor Ontology", + "url": "http://www.ebi.ac.uk/efo/efo.owl", + "version": "3.23.0" + }, { + "id": "uberon", + "name": "Uber-anatomy ontology", + "url": "http://purl.obolibrary.org/obo/uberon.owl", + "version": "2020-09-16" + }] + } +} diff --git a/commons/src/test/resources/phenopacket/phenotypeTestPhenopacket.json b/core/src/test/resources/phenopacket/phenotypeTestPhenopacket.json similarity index 95% rename from commons/src/test/resources/phenopacket/phenotypeTestPhenopacket.json rename to core/src/test/resources/phenopacket/phenotypeTestPhenopacket.json index 8a6dbcfad..0310bfb98 100644 --- a/commons/src/test/resources/phenopacket/phenotypeTestPhenopacket.json +++ b/core/src/test/resources/phenopacket/phenotypeTestPhenopacket.json @@ -1,42 +1,42 @@ -{ - "subject": { - "id": "id-individual" - }, - "biosamples": [ - { - "id": "id", - "description": "description", - "phenotypicFeatures": [ - { - "type": { - "id": "term id", - "label": "term label" - } - }, - { - "type": { - "id": "term id", - "label": "term label" - } - } - ], - "individualId": "id-individual", - "ageOfIndividualAtCollection": { - "age": "0" - } - } - ], - "metaData": { - "created": "2018-07-04T17:12:46.284Z", - "createdBy": "Biosamples phenopacket exporter", - "resources": [ - { - "id": "resource id", - "name": "resource name", - "namespacePrefix": "resource prefix", - "url": "url", - "version": "version" - } - ] - } -} +{ + "subject": { + "id": "id-individual" + }, + "biosamples": [ + { + "id": "id", + "description": "description", + "phenotypicFeatures": [ + { + "type": { + "id": "term id", + "label": "term label" + } + }, + { + "type": { + "id": "term id", + "label": "term label" + } + } + ], + "individualId": "id-individual", + "ageOfIndividualAtCollection": { + "age": "0" + } + } + ], + "metaData": { + "created": "2018-07-04T17:12:46.284Z", + "createdBy": "Biosamples phenopacket exporter", + "resources": [ + { + "id": "resource id", + "name": "resource name", + "namespacePrefix": "resource prefix", + "url": "url", + "version": "version" + } + ] + } +} diff --git a/docker-agents.sh b/docker-agents.sh index 25510b0ed..5336455a7 100755 --- a/docker-agents.sh +++ b/docker-agents.sh @@ -1,5 +1,5 @@ -#!/bin/bash -set -e - -docker-compose run --rm --service-ports biosamples-agents-solr java -jar agents-solr-5.3.12-SNAPSHOT.jar -echo "Successfully runned agents" +#!/bin/bash +set -e + +docker-compose run --rm --service-ports biosamples-agents-solr java -jar agents-solr-5.3.13-SNAPSHOT.jar +echo "Successfully runned agents" diff --git a/docker-compose.yml b/docker-compose.yml index 967b1eac8..cc4c01570 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: command: - java - -jar - - webapps-core-5.3.12-SNAPSHOT.war + - webapps-core-5.3.13-SNAPSHOT.war environment: - env=local #application properties @@ -106,7 +106,7 @@ services: command: - java - -jar - - webapps-core-v2-5.3.12-SNAPSHOT.jar + - webapps-core-v2-5.3.13-SNAPSHOT.jar environment: #application properties - biosamples.bulksubmisison.webin.superuser.validation=true @@ -183,7 +183,7 @@ services: command: - java - -jar - - agents-solr-5.3.12-SNAPSHOT.jar + - agents-solr-5.3.13-SNAPSHOT.jar environment: - SPRING_RABBITMQ_HOST=rabbitmq - SPRING_RABBITMQ_PUBLISHER-CONFIRMS=true @@ -208,7 +208,7 @@ services: command: - java - -jar - - agents-uploadworkers-5.3.12-SNAPSHOT.jar + - agents-uploadworkers-5.3.13-SNAPSHOT.jar environment: - SPRING_RABBITMQ_HOST=rabbitmq - SPRING_RABBITMQ_PUBLISHER-CONFIRMS=true @@ -232,7 +232,7 @@ services: command: - java - -jar - - pipelines-ena-5.3.12-SNAPSHOT.jar + - pipelines-ena-5.3.13-SNAPSHOT.jar environment: - BIOSAMPLES_CLIENT_URI=http://biosamples-webapps-core:8080/biosamples - LOGGING_FILE=/logs/pipelines-ena.log @@ -247,7 +247,7 @@ services: command: - java - -jar - - pipelines-ncbi-5.3.12-SNAPSHOT.jar + - pipelines-ncbi-5.3.13-SNAPSHOT.jar environment: - BIOSAMPLES_CLIENT_URI=http://biosamples-webapps-core:8080/biosamples - LOGGING_FILE=/logs/pipelines-ncbi.log @@ -262,7 +262,7 @@ services: command: - java - -jar - - pipelines-accession-5.3.12-SNAPSHOT.jar + - pipelines-accession-5.3.13-SNAPSHOT.jar environment: - BIOSAMPLES_CLIENT_URI=http://biosamples-webapps-core:8080/biosamples - LOGGING_FILE=/logs/pipelines-accession.log @@ -277,7 +277,7 @@ services: command: - java - -jar - - pipelines-copydown-5.3.12-SNAPSHOT.jar + - pipelines-copydown-5.3.13-SNAPSHOT.jar environment: - BIOSAMPLES_CLIENT_URI=http://biosamples-webapps-core:8080/biosamples - LOGGING_FILE=/logs/pipelines-copydown.log @@ -292,7 +292,7 @@ services: command: - java - -jar - - pipelines-curation-5.3.12-SNAPSHOT.jar + - pipelines-curation-5.3.13-SNAPSHOT.jar environment: - BIOSAMPLES_CLIENT_URI=http://biosamples-webapps-core:8080/biosamples - LOGGING_FILE=/logs/pipelines-curation.log @@ -309,7 +309,7 @@ services: command: - java - -jar - - pipelines-zooma-5.3.12-SNAPSHOT.jar + - pipelines-zooma-5.3.13-SNAPSHOT.jar environment: - BIOSAMPLES_CLIENT_URI=http://biosamples-webapps-core:8080/biosamples - LOGGING_FILE=/logs/pipelines-zooma.log @@ -326,7 +326,7 @@ services: command: - java - -jar - - pipelines-reindex-5.3.12-SNAPSHOT.jar + - pipelines-reindex-5.3.13-SNAPSHOT.jar environment: - BIOSAMPLES_CLIENT_URI=http://biosamples-webapps-core:8080/biosamples - LOGGING_FILE=/logs/pipelines-reindex.log @@ -344,7 +344,7 @@ services: command: - java - -jar - - pipelines-export-5.3.12-SNAPSHOT.jar + - pipelines-export-5.3.13-SNAPSHOT.jar - --gzip - /export/export.json.gzip environment: @@ -362,7 +362,7 @@ services: command: - java - -jar - - integration-5.3.12-SNAPSHOT.jar + - integration-5.3.13-SNAPSHOT.jar environment: - BIOSAMPLES_CLIENT_URI=http://biosamples-webapps-core:8080/biosamples - BIOSAMPLES_CLIENT_URI_V2=http://biosamples-webapps-core-v2:8080/biosamples/v2 diff --git a/docker-integration.big.sh b/docker-integration.big.sh index 917fafa4a..e81107947 100755 --- a/docker-integration.big.sh +++ b/docker-integration.big.sh @@ -1,23 +1,23 @@ -#!/bin/bash -set -e - -./docker-webapp.sh --clean - - -docker-compose up -d biosamples-agents-solr -docker-compose up -d biosamples-agents-upload-workers - -ARGS=--spring.profiles.active=big -for X in 1 2 3 4 5 -do - #java -jar integration/target/integration-4.0.0-SNAPSHOT.jar --phase=$X $ARGS $@ - docker-compose run --rm --service-ports biosamples-integration java -jar integration-5.3.12-SNAPSHOT.jar --phase=$X $ARGS $@ - sleep 10 #solr is configured to commit every 5 seconds - -done - -#leave the agents up at the end -docker-compose up -d biosamples-agents-solr -docker-compose up -d biosamples-agents-upload-workers - -echo "Successfully completed" +#!/bin/bash +set -e + +./docker-webapp.sh --clean + + +docker-compose up -d biosamples-agents-solr +docker-compose up -d biosamples-agents-upload-workers + +ARGS=--spring.profiles.active=big +for X in 1 2 3 4 5 +do + #java -jar integration/target/integration-4.0.0-SNAPSHOT.jar --phase=$X $ARGS $@ + docker-compose run --rm --service-ports biosamples-integration java -jar integration-5.3.13-SNAPSHOT.jar --phase=$X $ARGS $@ + sleep 10 #solr is configured to commit every 5 seconds + +done + +#leave the agents up at the end +docker-compose up -d biosamples-agents-solr +docker-compose up -d biosamples-agents-upload-workers + +echo "Successfully completed" diff --git a/docker-integration.sh b/docker-integration.sh index c00d9db70..97a1cbb8f 100755 --- a/docker-integration.sh +++ b/docker-integration.sh @@ -14,7 +14,7 @@ do echo "=================================== STARTING INTEGRATION TESTS PHASE-"$X "=====================================" echo "============================================================================================================" #java -jar integration/target/integration-4.0.0-SNAPSHOT.jar --phase=$X $ARGS $@ - docker-compose run --rm --service-ports biosamples-integration java -jar integration-5.3.12-SNAPSHOT.jar --phase=$X $ARGS $@ + docker-compose run --rm --service-ports biosamples-integration java -jar integration-5.3.13-SNAPSHOT.jar --phase=$X $ARGS $@ sleep 10 #solr is configured to commit every 5 seconds done diff --git a/docker-test.sh b/docker-test.sh index df43f1b4f..5a2d2389a 100755 --- a/docker-test.sh +++ b/docker-test.sh @@ -1,16 +1,16 @@ -#!/bin/bash -set -e - -docker-compose up -d biosamples-agents-solr - -for X in "$@" -do - docker-compose run --rm --service-ports biosamples-integration java -jar integration-5.3.12-SNAPSHOT.jar --phase=$X $ARGS $@ - sleep 30 #solr is configured to commit every 5 seconds - -done - -docker-compose up -d biosamples-agents-solr - -echo "Successfully completed" - +#!/bin/bash +set -e + +docker-compose up -d biosamples-agents-solr + +for X in "$@" +do + docker-compose run --rm --service-ports biosamples-integration java -jar integration-5.3.13-SNAPSHOT.jar --phase=$X $ARGS $@ + sleep 30 #solr is configured to commit every 5 seconds + +done + +docker-compose up -d biosamples-agents-solr + +echo "Successfully completed" + diff --git a/integration/pom.xml b/integration/pom.xml index 68533e1d7..0d6d4c1a7 100644 --- a/integration/pom.xml +++ b/integration/pom.xml @@ -1,48 +1,48 @@ - + 4.0.0 integration jar + uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../ + + uk.ac.ebi.biosamples biosamples-spring-boot-starter - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - models-sitemap - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - models-jsonld - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples properties - 5.3.12-SNAPSHOT - compile - - - uk.ac.ebi.biosamples - models-neo4j - 5.3.12-SNAPSHOT - compile + 5.3.13-SNAPSHOT + + org.springframework.hateoas spring-hateoas 1.3.4 + + org.springframework + spring-webmvc + + + org.springframework.boot + spring-boot-starter-tomcat + + + org.xmlunit xmlunit-core @@ -54,24 +54,38 @@ 2.4.0 - org.hamcrest - hamcrest-library + dom4j + dom4j + 1.6.1 + + - io.rest-assured - rest-assured - 3.0.7 + junit + junit + test org.hamcrest hamcrest-all 1.3 + + org.hamcrest + hamcrest-library + + + io.rest-assured + rest-assured + 3.0.7 + io.rest-assured json-path 3.0.7 + + commons-io commons-io @@ -85,20 +99,8 @@ junit junit - 4.13.2 - test - - - junit - junit - 4.13.2 - - - dom4j - dom4j - 1.6.1 - compile + @@ -124,4 +126,5 @@ + diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/AbstractIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/AbstractIntegration.java index 06dfa01bd..cfc7e4246 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/AbstractIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/AbstractIntegration.java @@ -23,10 +23,10 @@ import org.springframework.boot.ExitCodeGenerator; import org.springframework.hateoas.EntityModel; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.service.FilterBuilder; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.service.FilterBuilder; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; public abstract class AbstractIntegration implements ApplicationRunner, ExitCodeGenerator { diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/AmrDataIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/AmrDataIntegration.java index 42eebc586..4f8533883 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/AmrDataIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/AmrDataIntegration.java @@ -24,11 +24,11 @@ import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.*; -import uk.ac.ebi.biosamples.model.structured.StructuredData; -import uk.ac.ebi.biosamples.model.structured.StructuredDataEntry; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; -import uk.ac.ebi.biosamples.model.structured.StructuredDataType; +import uk.ac.ebi.biosamples.core.model.*; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataEntry; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataType; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; import uk.ac.ebi.biosamples.utils.TestUtilities; diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/Application.java b/integration/src/main/java/uk/ac/ebi/biosamples/Application.java index dc74f1073..04aa0d587 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -20,6 +20,7 @@ import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; +import org.springframework.web.client.RestTemplate; @SpringBootApplication public class Application { @@ -30,6 +31,11 @@ public static PropertySourcesPlaceholderConfigurer getPropertySourcesPlaceholder return new PropertySourcesPlaceholderConfigurer(); } + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } + public static void exitApplication(final ConfigurableApplicationContext ctx) { int exitCode = SpringApplication.exit(ctx, () -> 0); log.info("Exit Spring Boot"); diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/BigIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/BigIntegration.java index 059cba150..dafb5451b 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/BigIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/BigIntegration.java @@ -35,9 +35,9 @@ import org.springframework.web.util.UriComponentsBuilder; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Sample; @Component @Profile({"big"}) diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/ETagIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/ETagIntegration.java index dc173b985..f1408368d 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/ETagIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/ETagIntegration.java @@ -29,9 +29,9 @@ import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.Sample; @Component public class ETagIntegration extends AbstractIntegration { diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/JsonLdIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/JsonLdIntegration.java index 84c90879f..74d49b724 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/JsonLdIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/JsonLdIntegration.java @@ -26,7 +26,11 @@ import org.springframework.web.util.UriComponentsBuilder; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.*; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.ExternalReference; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.jsonld.model.JsonLDDataRecord; +import uk.ac.ebi.biosamples.jsonld.model.JsonLDSample; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/JsonSchemaValidationIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/JsonSchemaValidationIntegration.java index f49283db2..13c68e8df 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/JsonSchemaValidationIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/JsonSchemaValidationIntegration.java @@ -22,7 +22,8 @@ import org.springframework.stereotype.Component; import org.springframework.web.client.HttpClientErrorException; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.*; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/PhenopacketIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/PhenopacketIntegration.java index 22a3a8ee6..9a44237f5 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/PhenopacketIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/PhenopacketIntegration.java @@ -29,8 +29,8 @@ import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; @Component // @Order(1) diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestCurationIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestCurationIntegration.java index e14e82ffc..65a24ff9b 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestCurationIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestCurationIntegration.java @@ -10,10 +10,11 @@ */ package uk.ac.ebi.biosamples; +import static org.junit.Assert.assertTrue; + import java.net.URI; import java.time.Instant; import java.util.*; -import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.web.client.RestTemplateBuilder; @@ -32,10 +33,7 @@ import org.springframework.web.util.UriComponentsBuilder; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.*; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component @@ -255,7 +253,7 @@ protected void phaseFive() { "Sample does not exist, sample name: " + sample3.getName(), Phase.TWO); } - Assert.assertTrue(sample3.getRelationships().isEmpty()); + assertTrue(sample3.getRelationships().isEmpty()); } @Override diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestExternalReferenceIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestExternalReferenceIntegration.java index 00dfba8bd..eec3eb9ea 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestExternalReferenceIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestExternalReferenceIntegration.java @@ -16,7 +16,7 @@ import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.*; +import uk.ac.ebi.biosamples.core.model.*; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java index 164268742..3e0a26511 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java @@ -27,9 +27,7 @@ import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.ExternalReference; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.*; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestFilterIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestFilterIntegration.java index 6e3a06733..8ca4de78f 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestFilterIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestFilterIntegration.java @@ -23,11 +23,9 @@ import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.service.FilterBuilder; +import uk.ac.ebi.biosamples.core.model.*; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.service.FilterBuilder; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestIntegration.java index e783fdb32..e4e6debee 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestIntegration.java @@ -32,7 +32,7 @@ import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.*; +import uk.ac.ebi.biosamples.core.model.*; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestIntegrationSRAAccessioningV2.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestIntegrationSRAAccessioningV2.java index c88d9ce29..0c99047bf 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestIntegrationSRAAccessioningV2.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestIntegrationSRAAccessioningV2.java @@ -21,7 +21,7 @@ import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.*; +import uk.ac.ebi.biosamples.core.model.*; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestIntegrationV2.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestIntegrationV2.java index 494bf74db..d6acdd40a 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestIntegrationV2.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestIntegrationV2.java @@ -23,7 +23,7 @@ import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.*; +import uk.ac.ebi.biosamples.core.model.*; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java index 1a3d338cb..fff390782 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java @@ -17,8 +17,7 @@ import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.*; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestSampleStatusIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestSampleStatusIntegration.java index 57373e3da..622b37f30 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestSampleStatusIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestSampleStatusIntegration.java @@ -21,9 +21,7 @@ import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SampleStatus; +import uk.ac.ebi.biosamples.core.model.*; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestSearchIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestSearchIntegration.java index 466847f83..b8de76eb9 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestSearchIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestSearchIntegration.java @@ -17,9 +17,7 @@ import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.*; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/SamplesGraphIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/SamplesGraphIntegration.java index f0e89d1f1..b57f23a95 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/SamplesGraphIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/SamplesGraphIntegration.java @@ -16,7 +16,7 @@ import org.springframework.hateoas.EntityModel; import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.*; import uk.ac.ebi.biosamples.neo4j.model.GraphNode; import uk.ac.ebi.biosamples.neo4j.model.GraphSearchQuery; import uk.ac.ebi.biosamples.neo4j.model.NeoSample; diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/SitemapIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/SitemapIntegration.java index 03f6ea729..7e9d8db3c 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/SitemapIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/SitemapIntegration.java @@ -12,6 +12,8 @@ import java.net.URI; import java.util.*; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.core.annotation.Order; @@ -23,13 +25,14 @@ import org.springframework.web.util.UriComponentsBuilder; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.XmlSitemap; -import uk.ac.ebi.biosamples.model.XmlSitemapIndex; -import uk.ac.ebi.biosamples.model.XmlUrlSet; +import uk.ac.ebi.biosamples.core.model.*; +import uk.ac.ebi.biosamples.sitemap.model.XmlSitemap; +import uk.ac.ebi.biosamples.sitemap.model.XmlSitemapIndex; +import uk.ac.ebi.biosamples.sitemap.model.XmlUrlSet; @Order @Component +@Slf4j public class SitemapIntegration extends AbstractIntegration { private final URI biosamplesSubmissionUri; private final RestOperations restTemplate; @@ -42,6 +45,7 @@ public SitemapIntegration( final RestTemplateBuilder restTemplateBuilder, final ClientProperties clientProperties) { super(client); + biosamplesSubmissionUri = clientProperties.getBiosamplesClientUri(); restTemplate = restTemplateBuilder.build(); } @@ -53,6 +57,7 @@ protected void phaseOne() {} protected void phaseTwo() { final List> samples = new ArrayList<>(); final Map lookupTable = new HashMap<>(); + for (final EntityModel sample : noAuthClient.fetchSampleResourceAll()) { samples.add(sample); lookupTable.put(Objects.requireNonNull(sample.getContent()).getAccession(), Boolean.FALSE); @@ -63,12 +68,13 @@ protected void phaseTwo() { } final int expectedSitemapIndexSize = Math.floorDiv(samples.size(), sitemapPageSize) + 1; - final XmlSitemapIndex index = getSitemapIndex(); - if (index.getXmlSitemaps().size() != expectedSitemapIndexSize) { + final int modelIndexSize = index.getXmlSitemaps().size(); + + if (modelIndexSize != expectedSitemapIndexSize) { throw new RuntimeException( "The model index size (" - + index.getXmlSitemaps().size() + + modelIndexSize + ") doesn't match the expected size (" + expectedSitemapIndexSize + ")"); @@ -115,20 +121,41 @@ private String getAccessionFromUri(final UriComponents uri) { private XmlSitemapIndex getSitemapIndex() { final UriComponentsBuilder builder = UriComponentsBuilder.fromUri(biosamplesSubmissionUri); final UriComponents sitemapUri = builder.pathSegment("sitemap").build(); + + log.info("Calling sitemap URI: {}", sitemapUri.toUri()); + + ResponseEntity rawResponse = + restTemplate.getForEntity(sitemapUri.toUri(), String.class); + log.info("Raw XML: " + rawResponse.getBody()); + final ResponseEntity responseEntity = restTemplate.getForEntity(sitemapUri.toUri(), XmlSitemapIndex.class); + if (!responseEntity.getStatusCode().is2xxSuccessful()) { throw new RuntimeException("Sitemap not available"); } + + XmlSitemapIndex xmlSitemapIndex = responseEntity.getBody(); + + log.info("Number of sitemaps {}", xmlSitemapIndex.getXmlSitemaps().size()); + + log.info( + "Sitemap response {}", + xmlSitemapIndex.getXmlSitemaps().stream() + .map(xmlSitemap -> xmlSitemap.getLoc() + xmlSitemap.getLastmod()) + .collect(Collectors.joining(","))); + return responseEntity.getBody(); } private XmlUrlSet getUrlSet(final XmlSitemap sitemap) { final ResponseEntity urlSetReponseEntity = restTemplate.getForEntity(sitemap.getLoc(), XmlUrlSet.class); + if (!urlSetReponseEntity.getStatusCode().is2xxSuccessful()) { throw new RuntimeException("Unable to reach a model urlset"); } + return urlSetReponseEntity.getBody(); } } diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/StructuredDataGenericIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/StructuredDataGenericIntegration.java index 58c9f62e8..8ff1addd0 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/StructuredDataGenericIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/StructuredDataGenericIntegration.java @@ -18,9 +18,7 @@ import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Publication; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.*; @Component public class StructuredDataGenericIntegration extends AbstractIntegration { diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/XmlSearchIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/XmlSearchIntegration.java index b0e5c36f3..96b84429c 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/XmlSearchIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/XmlSearchIntegration.java @@ -17,7 +17,7 @@ import org.springframework.hateoas.EntityModel; import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.*; +import uk.ac.ebi.biosamples.core.model.*; import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component diff --git a/messaging/pom.xml b/messaging/pom.xml deleted file mode 100644 index 4c211c3c9..000000000 --- a/messaging/pom.xml +++ /dev/null @@ -1,24 +0,0 @@ - - 4.0.0 - messaging - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - - - - - uk.ac.ebi.biosamples - models-core - 5.3.12-SNAPSHOT - - - org.springframework.boot - spring-boot-starter-amqp - - - diff --git a/models/README.md b/models/README.md deleted file mode 100644 index 903dc6bb2..000000000 --- a/models/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# README -These are all the models that will be used within the system. - -Note that currently they are all one module, so that module has the -dependencies of all of them, and therefore anything that uses any of -the models has to pull all of the transitive dependencies. - -## JsonLD Module -Contains all the classes modeling the ld+json world for the BioSchema project - -## Sitemap Module -All classes used to create the sitemap are in this module and reused in the -**webpps-core** module diff --git a/models/core/pom.xml b/models/core/pom.xml deleted file mode 100644 index 8f3c53b9b..000000000 --- a/models/core/pom.xml +++ /dev/null @@ -1,68 +0,0 @@ - - 4.0.0 - models-core - jar - - - 3.5.1 - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 8 - 8 - - - - - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - - - - - org.springframework - spring-web - - - org.springframework.hateoas - spring-hateoas - 1.3.4 - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-databind - - - org.apache.commons - commons-collections4 - 4.1 - - - - - - - - xml-apis - xml-apis - 1.4.01 - - - - diff --git a/models/core/src/main/xsd/BioSD/BioSDSchema.xsd b/models/core/src/main/xsd/BioSD/BioSDSchema.xsd deleted file mode 100644 index 543c3fd5e..000000000 --- a/models/core/src/main/xsd/BioSD/BioSDSchema.xsd +++ /dev/null @@ -1,184 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/models/core/src/main/xsd/ResultQuerySample/ResultQuerySampleSchema.xsd b/models/core/src/main/xsd/ResultQuerySample/ResultQuerySampleSchema.xsd deleted file mode 100644 index cdc257d5f..000000000 --- a/models/core/src/main/xsd/ResultQuerySample/ResultQuerySampleSchema.xsd +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/models/core/src/main/xsd/ResultQuerySampleGroup/ResultQuerySampleGroupSchema.xsd b/models/core/src/main/xsd/ResultQuerySampleGroup/ResultQuerySampleGroupSchema.xsd deleted file mode 100644 index b838bfdf7..000000000 --- a/models/core/src/main/xsd/ResultQuerySampleGroup/ResultQuerySampleGroupSchema.xsd +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/models/curami/pom.xml b/models/curami/pom.xml deleted file mode 100644 index d73a444a4..000000000 --- a/models/curami/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - 4.0.0 - - models-curami - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - - - - uk.ac.ebi.biosamples - models-core - 5.3.12-SNAPSHOT - - - org.apache.commons - commons-csv - 1.8 - - - org.apache.commons - commons-text - 1.8 - - - diff --git a/models/jsonld/pom.xml b/models/jsonld/pom.xml deleted file mode 100644 index 15cfb30fc..000000000 --- a/models/jsonld/pom.xml +++ /dev/null @@ -1,38 +0,0 @@ - - 4.0.0 - - models-jsonld - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - - - - com.fasterxml.jackson.core - jackson-annotations - 2.12.5 - - - uk.ac.ebi.biosamples - models-core - 5.3.12-SNAPSHOT - - - - - - - - xml-apis - xml-apis - 1.4.01 - - - - diff --git a/models/mongo/pom.xml b/models/mongo/pom.xml deleted file mode 100644 index 4047cfe4c..000000000 --- a/models/mongo/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - 4.0.0 - - models-mongo - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - - - - uk.ac.ebi.biosamples - models-core - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - properties - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - - - org.springframework.boot - spring-boot-starter-data-mongodb - - - javax.annotation - javax.annotation-api - 1.3.1 - - - diff --git a/models/neo4j/pom.xml b/models/neo4j/pom.xml deleted file mode 100644 index 35a1f96d4..000000000 --- a/models/neo4j/pom.xml +++ /dev/null @@ -1,27 +0,0 @@ - - 4.0.0 - - models-neo4j - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - - - - uk.ac.ebi.biosamples - models-core - 5.3.12-SNAPSHOT - - - org.neo4j.driver - neo4j-java-driver - 4.0.0 - - - diff --git a/models/pom.xml b/models/pom.xml deleted file mode 100644 index 23f88d2f0..000000000 --- a/models/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - 4.0.0 - - models - - pom - - - com.fasterxml.jackson.core - jackson-annotations - 2.12.5 - - - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - - - - core - mongo - solr - sitemap - jsonld - neo4j - curami - security - - diff --git a/models/security/pom.xml b/models/security/pom.xml deleted file mode 100644 index 4a03c27db..000000000 --- a/models/security/pom.xml +++ /dev/null @@ -1,22 +0,0 @@ - - 4.0.0 - - models-security - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - - - - uk.ac.ebi.biosamples - models-core - 5.3.12-SNAPSHOT - - - diff --git a/models/security/src/main/resources/abbreviations.csv b/models/security/src/main/resources/abbreviations.csv deleted file mode 100644 index afdf002d1..000000000 --- a/models/security/src/main/resources/abbreviations.csv +++ /dev/null @@ -1,273 +0,0 @@ -ATTRIBUTE -NCBI -ENA -EGA -SRA -INSDC -DNA -RNA -BioSD -dbGAP -1000genomes -ADMET -ALMF -AntiFam -API -ArchSchema -ArrayExpress -ASTD -ATC -ATC -AUP -AV -AWERB -BAC -BAGMSC -BioJS -BioJS -BioMart -BioMedBridges -BioModels -BioProject -BioSamples -BioSample -BioServices -BioStudies Database -BioSource -BLAST [nucleotide] -BLAST [protein] -BoM -BSC -BSL-(1-4) -CAFM -CamSIS -CCO -CCPE -CellComm -CellNOpt -CENSOR -CGaP Comm -CGHSSg -CGP -ChEBI -ChEMBL -CHIPseq -CL3OG -CMS -CoGS -Confluence -Confluence -Converis -CORBEL -CPP -CSA -CSSB -CTTV -CVS -DAC -DAS -DB -DBA -DDBJ -DDMoRe -DESY -DGVa -DiNA -DLS -DNAseq -DoS -Dry-Lab -DSEasy -e-PTW -EBI -EMBL-EBI -EBiSC -EC -EC-PDB -ECA -EFO -EGA -EICAT -EICAT -EIPOD -EIPP -ELIXIR -ELLS -EMBL -EMBL -EMBL ATC CPP -EMBL GR -EMBL MR -EMBL-EBI -EMBL-EBI -EMBL-HD -EMBL-HH -EMBLEM -EMBO -EMBOSS -EMBRIC -EMC -EMC -EMDataBank -EMIF -EMMA -EMMA -EMTRAIN -ENA -ENA -ENA/SVA -Ensembl -EoC -EoP -EP -ERC -ERIC -eSCAMPS -ESFRI -ESPOD -ESRF -ESS -ESTA -EuroCarbDB -Europe PMC -EVA -EXCELERATE -Expression Atlas -FAANG -Faculty -FAIR Data -FGED -FP7 -FTP -GCSSC -Gen2Phen -Geuvadis -GFF -GO -GOBLET -GPCR -GPCR SARfari -GRADSAF -GRIs -GRL -GTF -GTL -GTL -GWAS -GWAS Catalog -H2020 -HAVANA -Helix -HGNC -HMDMC -HMMER -HoU -HR -HTML -HTS -HTTP -HUGO -IAA -ICRF -IM -IMeX -IMGT -IMGT/HLA -IMI -IMPC -INSDC -INSDC -IntAct -IPI -ISE -JANET -JDispatcher -JS -KEGG -KOMP2 -LRG -MAFFT -ManIC -MASSIF-1 -MEIGOR -MGP -MIG -MMPU -ModComm -MRM -MSA -MX -NCBI -NHGRI -NIH -OA Infrastructure -OIPA -OLS -OoP -OpenAIRE -ORCID -PA -PacBio Sequencing -PDB -PDBe -PDBeFold -PEPCore -PICR -PMC -Pop-up banner -PostDoc -PRIDE -ps_scan -PSB -PSI -PSI-BLAST -Pull-up banner -RA -RADAR -Reactome -Rhea -RIs -RNA-seq -RSF -SAB -SAP -SAPS -SAS -SAXS -SBO -SCM -SeqCksum -SHSC -SIFTS -SMC -SMEs -SOB -SPOT -SRS -SSH -SSMAC -SVA -SVN -SWAT -SWO -TAC -Taxonomy -TSC -UKERNA -UniChem -UniProt -URI -URL -UVHCI -VCS -VWP -WCC -WEEE -Wet-Lab -WGC -WTGC -WTSI -wwPDB -XML -ZMP -PubMed \ No newline at end of file diff --git a/models/security/src/main/resources/attributes.csv b/models/security/src/main/resources/attributes.csv deleted file mode 100644 index 07c785f86..000000000 --- a/models/security/src/main/resources/attributes.csv +++ /dev/null @@ -1,1019 +0,0 @@ -ATTRIBUTE,COUNT -organism,10670271 -INSDC status,8527784 -INSDC center name,8359205 -INSDC first public,8245159 -INSDC last update,7435715 -description title,6741070 -NCBI submission model,6599497 -NCBI submission package,5919641 -sex,4072824 -INSDC secondary accession,3863891 -tissue,3250905 -gap subject id,3132347 -study name,3119844 -study design,3119622 -submitted sample id,3119583 -submitter handle,3119575 -gap sample id,3119575 -gap accession,3119575 -biospecimen repository sample id,3119575 -biospecimen repository,3119575 -submitted subject id,3119547 -analyte type,2832324 -description,2652959 -gap consent code,2577581 -gap consent short name,2563836 -collection date,1993101 -synonym,1971398 -title,1773985 -strain,1733133 -is tumor,1658905 -geographic location,1653701 -molecular data type,1643893 -alias,1610918 -SRA accession,1606737 -ENA checklist,1504866 -subject is affected,1436901 -sample source name,1322866 -isolation source,1278916 -INSDC center alias,1255447 -histological type,1066169 -source name,1004350 -age,965126 -host,921501 -sample title,867518 -organism part,860789 -study disease,805049 -cell type,802787 -gap parent phs,686777 -isolate,648647 -latitude and longitude,567030 -sample type,546434 -genotype,543726 -collected by,541632 -broad scale environmental context,448040 -environmental medium,418170 -geographic location (country and/or sea),413996 -local scale environmental context,387588 -project name,377310 -secondary description,350108 -biomaterial provider,345933 -development stage,340959 -investigation type,314234 -geographic location (longitude),299297 -geographic location (latitude),299267 -environment (material),290203 -environment (biome),290041 -environment (feature),288971 -host subject id,285009 -sequencing method,284732 -disease state,278361 -cell line,276654 -treatment,269393 -depth,233115 -host sex,231049 -cultivar,230875 -elevation,229687 -host disease,222831 -replicate,194303 -subject id,193857 -serovar,190893 -individual,187418 -broker name,179206 -anonymized name,178294 -longitude,168775 -latitude,168628 -developmental stage,167885 -sample characteristics,161609 -env feature,160196 -env biome,159985 -disease,143254 -host age,139737 -sample name,137773 -physical specimen location,137380 -host taxid,136749 -DNA extracted,135947 -submission identifier,135859 -submission title,135859 -submission description,135857 -collection timestamp,135300 -physical specimen remaining,134995 -host common name,134195 -env package,129920 -phenotype,128218 -isolate name alias,127108 -ecotype,125912 -sample id,125545 -public,122013 -sub species,119860 -env material,118455 -geo loc name,117766 -country,112616 -host scientific name,108831 -common name,103122 -source material identifiers,96869 -ethnicity,90190 -sample collection device or method,89896 -sample derived from,88232 -assembly quality,86201 -contamination score,86047 -completeness score,86047 -metagenomic source,86030 -sample description,83785 -race,83532 -assembly software,82436 -binning software,82231 -completeness software,82230 -binning parameters,82203 -taxonomic identity marker,82147 -altitude,81861 -body site,80508 -population,78497 -body product,78478 -body habitat,77979 -breed,77826 -time point,73839 -biosourceprovider,69041 -batch,65040 -sequence type,64837 -chip antibody,60041 -env matter,59582 -assigned from geo,58930 -specific host,58895 -time,57903 -temperature,54994 -host tissue sampled,54903 -biosourcetype,54573 -bio source type,54485 -environmental sample,54332 -sample size,53820 -timepoint,51104 -scientific name,51029 -material,50936 -submission reference layer,50605 -ph,50567 -host body product,50060 -target gene,48276 -group,47815 -sample material processing,46835 -geographic location (region and locality),46315 -tissue type,45837 -diet,44872 -bmi,44281 -environmental package,44272 -human gut environmental package,43620 -geographic location (elevation),43320 -host health state,43080 -label,42962 -patient id,42822 -stage,42510 -attribute package,40728 -host status,40570 -specimen voucher,39979 -subject,39579 -well,39545 -required sample info status,39067 -age unit,39054 -has physical specimen,38775 -supplier name,38674 -antibody,38429 -geographic location (depth),38247 -growth condition,37230 -host associated environmental package,35985 -target subfragment,35981 -sample storage temperature,35268 -experiment,35260 -pcr primers,34049 -potential contaminant,34043 -soil environmental package,33842 -hhs region,33352 -specimen with known storage state,32926 -barcode,32377 -metagenomic,32074 -metagenome source,31905 -host body habitat,31797 -clinical information,31249 -condition,30927 -isolation and growth condition,30885 -species,30740 -collection time,30585 -samp size,30525 -culture collection,30389 -diagnosis,30051 -vehicle,29989 -ploidy,29351 -library name,29263 -extracted DNA avail now,29165 -weight units,29061 -site,28864 -plate,28856 -state,28749 -sampleid,28502 -age years,28474 -health state,28364 -note,28140 -empo 2,27885 -empo 1,27885 -empo 3,27885 -run,27538 -disease stage,27308 -host Taxonomy id,27227 -donor id,27030 -day,26994 -total number of reads,26602 -number of mapped reads,26560 -weight kg,26507 -reference,26426 -taxon id,26320 -diabetes,26100 -height cm,25852 -reference for biomaterial,25823 -lane,25571 -ibd,25461 -is technical control,25392 -histology,25312 -pregnant,25275 -height units,25027 -host diet,24875 -hybridization date,24832 -csection,24587 -chickenpox,24586 -gender,24262 -rna extraction date,24254 -id,24211 -genetic background,23984 -dev stage,23533 -route,23316 -location,23280 -single cell well quality,23165 -relationship to oxygen,23160 -sample,23056 -dominant hand,22951 -sampling site,22767 -barcode sequence,22433 -family id,22367 -serotype,22302 -barcodesequence,22221 -lims id,22122 -strain/background,22053 -vector,21895 -cat,21643 -dog,21633 -sleep duration,21441 -EGA sample id,21432 -EGA dataset id,21432 -biological replicate,21430 -age cat,21191 -teethbrushing frequency,21128 -pool frequency,21115 -alcohol frequency,21115 -multivitamin,21114 -cosmetics frequency,21113 -exercise frequency,21113 -appendix removed,21111 -flossing frequency,21110 -deodorant use,21108 -migraine,21108 -gluten,21108 -skin condition,21107 -lactose,21107 -tonsils removed,21103 -seasonal allergies,21102 -pku,21101 -contraceptive,21099 -weight change,21094 -country of birth,21092 -exercise location,21087 -softener,21087 -alcohol consumption,21052 -birth year,21023 -flu vaccine date,21014 -lung disease,20943 -diet type,20937 -collection month,20904 -clinical condition,20881 -acne medication,20860 -patient,20843 -subset diabetes,20817 -subset antibiotic history,20817 -subset age,20817 -subset bmi,20817 -last travel,20813 -other supplement frequency,20792 -subset ibd,20792 -subset healthy,20792 -last move,20786 -prepared meals frequency,20785 -sugar sweetened drink frequency,20784 -salted snacks frequency,20783 -liver disease,20783 -alcohol types red wine,20780 -allergic to shellfish,20780 -alcohol types white wine,20780 -alcohol types unspecified,20780 -allergic to tree nuts,20780 -non food allergies pet dander,20780 -allergic to peanuts,20780 -non food allergies beestings,20780 -non food allergies unspecified,20780 -allergic to i have no food allergies that i know of,20780 -allergic to unspecified,20780 -non food allergies sun,20780 -survey id,20780 -allergic to other,20780 -alcohol types sour beers,20780 -drinking water source,20777 -one liter of water a day frequency,20777 -smoking frequency,20776 -livingwith,20770 -acne medication otc,20769 -vitamin d supplement frequency,20769 -consume animal products abx,20765 -alzheimers,20761 -kidney disease,20761 -cardiovascular disease,20760 -homecooked meals frequency,20760 -milk cheese frequency,20760 -whole eggs,20760 -meat eggs frequency,20759 -poultry frequency,20759 -probiotic frequency,20759 -seafood frequency,20759 -epilepsy or seizure disorder,20759 -add adhd,20758 -milk substitute frequency,20758 -olive oil,20758 -ready to eat meals frequency,20758 -whole grain frequency,20758 -vegetable frequency,20757 -asd,20757 -depression bipolar schizophrenia,20757 -thyroid,20756 -sugary sweets frequency,20756 -fruit frequency,20755 -ibs,20755 -types of plants,20755 -fed as infant,20754 -autoimmune,20754 -bowel movement frequency,20754 -level of education,20754 -vitamin b supplement frequency,20754 -lowgrain diet type,20752 -red meat frequency,20752 -cdiff,20752 -high fat red meat frequency,20751 -fungal overgrowth,20750 -bowel movement quality,20748 -fermented plant frequency,20747 -breastmilk formula ensure,20745 -sibo,20745 -roommates in study,20743 -cancer,20610 -cell subtype,20602 -protocol,20589 -host genotype,20546 -diabetes type,20468 -bmi cat,20451 -ibd diagnosis,20444 -project,20441 -sample progress,20430 -collection season,20402 -cancer treatment,20373 -subgroup,20358 -country residence,20354 -ibd diagnosis refined,20353 -non food allergies drug eg penicillin,20353 -drinks per session,20353 -pets other,20353 -alcohol types beercider,20353 -non food allergies poison ivyoak,20353 -alcohol types spiritshard alcohol,20353 -mental illness,20353 -nail biter,20353 -vivid dreams,20353 -economic region,20353 -antibiotic history,20353 -age corrected,20353 -frozen dessert frequency,20353 -acid reflux,20353 -census region,20353 -artificial sweeteners,20353 -bmi corrected,20353 -specialized diet westenprice or other lowgrain low processed fo,20311 -specialized diet unspecified,20311 -mental illness type substance abuse,20286 -specialized diet paleodiet or primal diet,20286 -specialized diet fodmap,20286 -specialized diet other restrictions not described here,20286 -specialized diet i do not eat a specialized diet,20286 -mental illness type anorexia nervosa,20286 -mental illness type unspecified,20286 -specialized diet exclude refined sugars,20286 -specialized diet kosher,20286 -specialized diet raw food diet,20286 -mental illness type schizophrenia,20286 -specialized diet exclude dairy,20286 -mental illness type depression,20286 -mental illness type bulimia nervosa,20286 -specialized diet halaal,20286 -specialized diet modified paleo diet,20286 -mental illness type ptsd posttraumatic stress disorder,20286 -mental illness type bipolar disorder,20286 -specialized diet exclude nightshades,20286 -estimated size,20063 -compound,19753 -host body site,19746 -growth protocol,19667 -specialized diet,19638 -mental illness type,19638 -allergic to,19638 -non food allergies,19638 -alcohol types,19638 -age units,19627 -genus,19538 -cohort,19361 -life stage,19326 -seq methods,19304 -subtype,19280 -number of replicons,19222 -time unit,18952 -type,18931 -salinity,18757 -infection,18492 -well id,18074 -plant associated environmental package,18033 -host life stage,17966 -linkerprimersequence,17845 -donor,17615 -subjectid,17601 -biosamplemodel,17530 -body mass index,17351 -distinquish,17337 -genetic modification,17259 -clinical history,17209 -collection,17186 -public accession,17081 -source,16954 -vioscreen methion,16898 -vioscreen vitb12,16898 -vioscreen daidzein,16898 -vioscreen hei oils,16898 -vioscreen biochana,16898 -vioscreen v potato,16898 -vioscreen vita rae,16898 -vioscreen protveg,16898 -vioscreen d milk,16898 -vioscreen hei veg,16898 -vioscreen aspartam,16898 -vioscreen serine,16898 -vioscreen phosphor,16898 -vioscreen isoleuc,16898 -vioscreen rgrain,16898 -vioscreen totfolat,16898 -vioscreen adsugtot,16898 -vioscreen formontn,16898 -vioscreen v other,16898 -vioscreen totsugar,16898 -vioscreen glycine,16898 -vioscreen hei2010 fatty acids,16898 -vioscreen vitc,16898 -vioscreen galactos,16898 -vioscreen hei sat fat,16898 -vioscreen hei fruit,16898 -vioscreen sodium,16898 -vioscreen mfa181,16898 -vioscreen g total,16898 -vioscreen a bev,16898 -vioscreen f nj other,16898 -vioscreen calcium servings,16898 -vioscreen discfat oil,16898 -vioscreen v orange,16898 -vioscreen gltc,16898 -vioscreen tfa182t,16898 -vioscreen hei2010 protien foods,16898 -vioscreen v tomato,16898 -vioscreen tyrosine,16898 -vioscreen vitd iu,16898 -vioscreen hei2010 sodium,16898 -vioscreen add sug,16898 -vioscreen fish servings,16898 -vioscreen inositol,16898 -vioscreen legumes,16898 -vioscreen erythr,16898 -vioscreen hei drk g org veg leg,16898 -vioscreen glutamic,16898 -vioscreen caffeine,16898 -vioscreen histidin,16898 -vioscreen mfatot,16898 -vioscreen aspartic,16898 -vioscreen line gi,16898 -vioscreen cystine,16898 -vioscreen methhis3,16898 -vioscreen totcla,16898 -vioscreen vitd2,16898 -vioscreen salad vegetable servings,16898 -vioscreen vite iu,16898 -vioscreen water,16898 -vioscreen wgrain,16898 -vioscreen betatoco,16898 -vioscreen hei2010 dairy,16898 -vioscreen sfa160,16898 -vioscreen clac9t11,16898 -vioscreen retinol,16898 -vioscreen hei2010 sea foods plant protiens,16898 -vioscreen fried food servings,16898 -vioscreen copper,16898 -vioscreen pfa183,16898 -vioscreen hei sodium,16898 -vioscreen f other,16898 -vioscreen arginine,16898 -vioscreen protein,16898 -vioscreen m frank,16898 -vioscreen sfa120,16898 -vioscreen pfa205,16898 -vioscreen d yogurt,16898 -vioscreen vegsumm,16898 -vioscreen fibinso,16898 -vioscreen alcohol,16898 -vioscreen questionnaire,16898 -vioscreen fruit servings,16898 -vioscreen v drkgr,16898 -vioscreen carbo,16898 -vioscreen g nwhl,16898 -vioscreen mannitol,16898 -vioscreen alphtoce,16898 -vioscreen fried fish servings,16898 -vioscreen frtsumm,16898 -vioscreen sfa80,16898 -vioscreen f citmlb,16898 -vioscreen alcohol servings,16898 -vioscreen vita re,16898 -vioscreen isomalt,16898 -vioscreen alanine,16898 -vioscreen database,16898 -vioscreen pfa184,16898 -vioscreen m meat,16898 -vioscreen hei whl grains,16898 -vioscreen glac,16898 -vioscreen satoco,16898 -vioscreen tagatose,16898 -vioscreen maltose,16898 -vioscreen joules,16898 -vioscreen pfa226,16898 -vioscreen hei meat beans,16898 -vioscreen pfa204,16898 -vioscreen m fish lo,16898 -vioscreen gammtoco,16898 -vioscreen calcium from dairy servings,16898 -vioscreen nccglgr,16898 -vioscreen betacryp,16898 -vioscreen tfa161t,16898 -vioscreen vitd,16898 -vioscreen fructose,16898 -vioscreen pinitol,16898 -vioscreen m mpf,16898 -vioscreen tfa181t,16898 -vioscreen f nj total,16898 -vioscreen proline,16898 -vioscreen hei2010 whole grains,16898 -vioscreen glycitn,16898 -vioscreen whole grain servings,16898 -vioscreen v total,16898 -vioscreen sfa100,16898 -vioscreen delttoco,16898 -vioscreen pantothe,16898 -vioscreen low fat dairy serving,16898 -vioscreen zinc,16898 -vioscreen sucrlose,16898 -vioscreen selenium,16898 -vioscreen vitk,16898 -vioscreen d tot soym,16898 -vioscreen fibh2o,16898 -vioscreen oxalic,16898 -vioscreen m egg,16898 -vioscreen natoco,16898 -vioscreen ash,16898 -vioscreen sweet servings,16898 -vioscreen coumest,16898 -vioscreen sfatot,16898 -vioscreen iron,16898 -vioscreen m nutsd,16898 -vioscreen d cheese,16898 -vioscreen f total,16898 -vioscreen d total,16898 -vioscreen hei2010 fruit,16898 -vioscreen magnes,16898 -vioscreen hei score,16898 -vioscreen v starcy,16898 -vioscreen acesupot,16898 -vioscreen hei non juice frt,16898 -vioscreen hei2010 whole fruit,16898 -vioscreen pfa182,16898 -vioscreen sorbitol,16898 -vioscreen fol syn,16898 -vioscreen cholest,16898 -vioscreen nccglbr,16898 -vioscreen juice servings,16898 -vioscreen choline,16898 -vioscreen g whl,16898 -vioscreen pectins,16898 -vioscreen addsugar,16898 -vioscreen sucrose,16898 -vioscreen sfa140,16898 -vioscreen betaine,16898 -vioscreen niacin,16898 -vioscreen sacchar,16898 -vioscreen calories,16898 -vioscreen m poult,16898 -vioscreen avcarb,16898 -vioscreen clat10c12,16898 -vioscreen m organ,16898 -vioscreen vitb6,16898 -vioscreen sucpoly,16898 -vioscreen lutzeax,16898 -vioscreen vitd3,16898 -vioscreen omega3,16898 -vioscreen valine,16898 -vioscreen oxalicm,16898 -vioscreen hei sol fat alc add sug,16898 -vioscreen xylitol,16898 -vioscreen m soy,16898 -vioscreen potass,16898 -vioscreen sfa220,16898 -vioscreen lactitol,16898 -vioscreen phytic,16898 -vioscreen a cal,16898 -vioscreen lysine,16898 -vioscreen leucine,16898 -vioscreen tgrain,16898 -vioscreen mfa141,16898 -vioscreen protanim,16898 -vioscreen niacineq,16898 -vioscreen hei2010 empty calories,16898 -vioscreen lycopene,16898 -vioscreen grams,16898 -vioscreen starch,16898 -vioscreen hei grains,16898 -vioscreen veg5 day,16898 -vioscreen sfa60,16898 -vioscreen maltitol,16898 -vioscreen alphtoco,16898 -vioscreen mfa201,16898 -vioscreen lactose,16898 -vioscreen sfa180,16898 -vioscreen threonin,16898 -vioscreen m fish hi,16898 -vioscreen thiamin,16898 -vioscreen ribofla,16898 -vioscreen betacar,16898 -vioscreen fiber,16898 -vioscreen vita iu,16898 -vioscreen tryptoph,16898 -vioscreen sfa200,16898 -vioscreen fol nat,16898 -vioscreen pfatot,16898 -vioscreen pfa225,16898 -vioscreen sfa170,16898 -vioscreen sfa40,16898 -vioscreen hei milk,16898 -vioscreen nitrogen,16898 -vioscreen hei2010 veg,16898 -vioscreen alphacar,16898 -vioscreen f nj citmlb,16898 -vioscreen vegetable servings,16898 -vioscreen totaltfa,16898 -vioscreen mfa161,16898 -vioscreen glucose,16898 -vioscreen mangan,16898 -vioscreen mfa221,16898 -vioscreen fol deqv,16898 -vioscreen hei2010 score,16898 -vioscreen non fried fish servings,16898 -vioscreen hei2010 greens beans,16898 -vioscreen frt5 day,16898 -vioscreen phenylal,16898 -vioscreen fat,16898 -vioscreen hei2010 refined grains,16898 -vioscreen discfat sol,16898 -vioscreen genistn,16898 -vioscreen multivitamin,16896 -vioscreen multivitamin freq,16896 -vioscreen multi calcium avg,16896 -vioscreen multi calcium dose,16896 -vioscreen calcium avg,16892 -vioscreen calcium freq,16892 -vioscreen calcium dose,16892 -vioscreen calcium,16892 -variety,16725 -linker primer sequence,16678 -lab host,16631 -strain background,16562 -variation,16509 -stimulus,16480 -passages,16433 -fda lab id,16408 -tag,16277 -individual id,16203 -vioscreen user id,16176 -vioscreen gender,16176 -vioscreen srvid,16176 -vioscreen finished,16176 -vioscreen recno,16176 -vioscreen age,16176 -vioscreen time,16176 -vioscreen protocol,16176 -vioscreen dob,16176 -vioscreen started,16176 -vioscreen procdate,16176 -vioscreen nutrient recommendation,16176 -vioscreen visit,16176 -vioscreen weight,16167 -vioscreen email,16167 -vioscreen subject id,16167 -vioscreen height,16167 -vioscreen bmi,16167 -vioscreen bcodeid,16167 -vioscreen eer,16167 -vioscreen activity level,16167 -v type,16107 -sample volume or weight for DNA extraction,16098 -disease staging,15994 -weight,15892 -stimulation,15804 -sample number,15767 -sample comment,15740 -"geographic location (country and/or sea,region)",15630 -passage,15374 -vioscreen scfv,15368 -vioscreen scf,15368 -miscellaneous parameter,15253 -inferred cell type,15131 -host body mass index,15009 -dose,14930 -post analysis well quality,14670 -disease status,14661 -season,14659 -sub strain,14581 -status,14578 -age at diagnosis,14541 -region,14498 -initial time point,14385 -re 1,14246 -birth date,14167 -passage history,14155 -family role,14058 -propagation,13985 -study id,13871 -re 2,13465 -mating type,13185 -numbervirusreads,13126 -diseaseseverity,13126 -numberhumanuniquelymappedreads,13126 -BioSD sample,12986 -birth location,12979 -original body site annotation,12950 -multiplex identifiers,12805 -sample group,12801 -dbGAP sample,12772 -chemical administration,12762 -grade,12628 -library id,12391 -tumor stage,12363 -temp,12335 -tmp,12254 -family,12233 -has extracted data,12193 -treatment duration,12050 -visit,11841 -water environmental package,11829 -pfge primary enzyme pattern,11789 -soil type,11769 -lat lon,11756 -cage,11657 -plot,11602 -brain region,11598 -animal id,11592 -colection date,11556 -assembly method,11465 -mouse id,11361 -fraction,11180 -tumor,11074 -clinicalinformation,11042 -mapping method,10992 -quality assessment method,10992 -subsource note,10987 -age yrs,10899 -perturbation,10832 -identified by,10784 -unique id,10701 -origin,10638 -ph method,10495 -treated with,10494 -human associated environmental package,10492 -plate id,10475 -name,10428 -treatment group,10363 -illumina technology,10357 -library,10325 -year,10270 -tumor-of-origin,10252 -collector name,10237 -targeted cell type,10205 -control,10111 -line,10091 -X.SampleID,10084 -physical location,10073 -collecting institution,9999 -pool,9983 -height or length,9977 -row,9789 -physical samp avail now,9784 -specimen collection protocol,9554 -specimen collection date,9554 -animal age at collection,9536 -cd34,9497 -cd38,9494 -cd45ra,9476 -cd90,9476 -background strain,9442 -flowcell,9436 -storage conditions,9392 -tissue supergroup,9361 -host height,9340 -adapters,9309 -emp status,9287 -er status,9252 -host visit number,9203 -nitrate,9189 -week,9134 -repository,9115 -roommates,9073 -health status at collection,9027 -date,8971 -phosphate,8941 -cell cluster,8915 -sample storage location,8795 -host weight,8733 -antibiotics,8569 -rep,8548 -lab,8515 -smoking status,8450 -geneticmodification,8421 -cluster,8382 -recurated from,8272 -rin,8255 -projectid,8245 -run name,8166 -experimental factor 1,8072 -block,8071 -dataset,8040 -village,7950 -platform,7945 -observed biotic relationship,7914 -ethnic group,7818 -postnatal day,7757 -smoker,7673 -her2 status,7669 -sequencing batch,7663 -collection method,7604 -reverse primer sequence,7583 -nucleic acid extraction,7566 -tot mass,7564 -well information,7551 -oxygen,7528 -mutation,7512 -tumor type,7496 -tumor size,7477 -forward filename,7412 -reverse filename,7412 -karyotype,7411 -cre line,7378 -medium,7301 -studyid,7247 -is the sequenced pathogen host associated?,7228 -sampling time,7178 -single cell id,7078 -cd123,7063 -instrument model,7036 -molecule,6943 -affected by,6929 -generation,6884 -molecule subtype,6854 -media,6785 -total organic carbon,6779 -host description,6770 -experimental factor,6751 -number,6745 -tissue source,6734 -method,6685 -sequencing run,6671 -days in culture,6634 -agent,6607 -total nitrogen,6577 -family relationship,6542 -exposure,6534 -treatment time,6510 -project accession,6493 -tumor site,6490 -miscellaneous environmental package,6469 -phase,6444 -growth phase,6441 -experiment date,6422 -facs gating,6388 -well position,6378 -sampletype,6370 -tumor grading,6358 -host disease status,6349 -administration route,6304 -frequency of dosing,6302 -solvent,6298 -plant product,6244 -extraction method,6219 -type status,6196 -height,6193 -host phenotype,6188 -diseasestaging,6161 -total reads,6149 -cell source,6127 -sample location,6115 -primer plate,6094 -host associated,6068 -calcium,6068 -magnesium,6060 -biological specimen,6042 -risk group,6007 -protocol label,6006 -er,5981 -cell class,5958 -chip antibody vendor,5958 -current land use,5951 -index,5916 -building setting,5914 -enzyme used,5901 -dob,5880 -sampling time point,5875 -study arm,5873 -env,5872 -seqc sample,5872 -building occupancy type,5866 -host disease outcome,5800 -study center,5777 -host age units,5768 -sample site,5754 -rna amplification set,5732 -empo 0,5720 -type sample,5716 -cats,5710 -treatment compound,5709 -dissected layer,5702 -chip id,5696 -bio material,5689 -dogs,5684 -experiment center,5682 -racegendersite,5664 -nkcell,5664 -bcell,5664 -neutro,5664 -lib const meth,5634 -project id,5609 -seq meth,5593 -experimental factor 2,5573 -stress,5554 -tumor grade,5551 -shipment,5547 -replicate id,5543 -conductivity,5528 -cell gating,5510 -biosample,5508 -cell population,5467 -pcrplate,5417 -amount or size of sample collected,5412 -infect,5399 -transfection,5397 -operator variation,5372 -369 groups,5372 -Blood/NonBlood meta-groups,5372 -4 meta-groups,5372 -data source,5372 -15 meta-groups,5372 -PFGE_SecondaryEnzyme_pattern,5324 -isolation source host associated,5318 -sample source,5305 -culture condition,5293 -current vegetation,5286 -sample storage duration,5278 -biosource type,5272 -CRISPR-Cas9_nuclease,5261 -CRISPR-Cas9_gene_target,5261 -sodium,5212 -DNA id,5210 -month,5150 -chlorophyll,5144 -pam50 subtype,5117 -surface,5094 -scaffold,5090 -technical replicate,5071 -notes,5029 -donor genotype,5022 -type material,5017 \ No newline at end of file diff --git a/models/security/src/main/resources/curations.csv b/models/security/src/main/resources/curations.csv deleted file mode 100644 index 4a73359f6..000000000 --- a/models/security/src/main/resources/curations.csv +++ /dev/null @@ -1,4 +0,0 @@ -ATTRIBUTE,CURATION -organism_part,organism part -gender,sex -species,organism \ No newline at end of file diff --git a/models/sitemap/pom.xml b/models/sitemap/pom.xml deleted file mode 100644 index f2c3e23d8..000000000 --- a/models/sitemap/pom.xml +++ /dev/null @@ -1,61 +0,0 @@ - - 4.0.0 - - models-sitemap - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - - - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-databind - - - com.google.code - sitemapgen4j - 1.0.1 - - - javax.xml.bind - jaxb-api - 2.3.0 - - - com.sun.xml.bind - jaxb-core - 2.3.0 - - - com.sun.xml.bind - jaxb-impl - 2.3.0 - - - - - - - - xml-apis - xml-apis - 1.4.01 - - - - diff --git a/models/solr/pom.xml b/models/solr/pom.xml deleted file mode 100644 index 275f584f8..000000000 --- a/models/solr/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - 4.0.0 - - models-solr - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - - - - uk.ac.ebi.biosamples - models-core - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - properties - 5.3.12-SNAPSHOT - - - org.springframework.data - spring-data-solr - 4.1.0.RELEASE - - - org.springframework.boot - spring-boot-starter-cache - - - - com.github.ben-manes.caffeine - caffeine - - - - - org.springframework.boot - spring-boot-starter-test - 1.5.15.RELEASE - test - - - diff --git a/models/solr/src/test/uk/ac/ebi/biosamples/solr/SolrSampleTestConfiguration.java b/models/solr/src/test/uk/ac/ebi/biosamples/solr/SolrSampleTestConfiguration.java deleted file mode 100644 index e1d49097b..000000000 --- a/models/solr/src/test/uk/ac/ebi/biosamples/solr/SolrSampleTestConfiguration.java +++ /dev/null @@ -1,11 +0,0 @@ -package uk.ac.ebi.biosamples.solr; - -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.context.annotation.ComponentScan; - -@TestConfiguration -@ComponentScan -public class SolrSampleTestConfiguration { - - -} diff --git a/models/solr/src/test/uk/ac/ebi/biosamples/solr/SolrServiceTests.java b/models/solr/src/test/uk/ac/ebi/biosamples/solr/SolrServiceTests.java deleted file mode 100644 index 9a7571d15..000000000 --- a/models/solr/src/test/uk/ac/ebi/biosamples/solr/SolrServiceTests.java +++ /dev/null @@ -1,94 +0,0 @@ -package uk.ac.ebi.biosamples.solr; - -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.test.autoconfigure.json.AutoConfigureJson; -import org.springframework.boot.test.autoconfigure.orm.jpa.AutoConfigureTestEntityManager; -import org.springframework.boot.test.autoconfigure.web.client.AutoConfigureWebClient; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureWebMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Bean; -import org.springframework.data.solr.core.query.Criteria; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; -import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.service.FilterBuilder; -import uk.ac.ebi.biosamples.solr.model.field.*; -import uk.ac.ebi.biosamples.solr.service.SolrFieldService; -import uk.ac.ebi.biosamples.solr.service.SolrFilterService; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import static org.junit.Assert.*; - -@RunWith(SpringRunner.class) -@SpringBootTest(classes = {SolrSampleAccessionField.class, SolrSampleAttributeValueField.class, -SolrSampleDateField.class, SolrSampleDomainField.class, SolrSampleExternalReferenceDataField.class, -SolrSampleInverseRelationField.class, SolrSampleNameField.class, SolrSampleRelationField.class, -SolrFilterService.class, SolrFieldService.class, BioSamplesProperties.class}) -public class SolrServiceTests { - - - @Autowired - SolrFilterService solrFilterService; - - @Autowired - private SolrFieldService fieldService; - - -// @Before -// public void setup() { -// this.fieldService = new SolrFieldService(solrSampleFieldList); -// } - - @Test - public void given_encoded_sample_decode_it_correctly_and_of_the_right_type() { - String encodedField = "MRSXGY3SNFYHI2LPNY_______av_ss"; - String expectedDecodedField = "description"; - - SolrSampleField sampleField = fieldService.decodeField(encodedField); - assertEquals(sampleField.getReadableLabel(), expectedDecodedField); - assertTrue(sampleField instanceof SolrSampleAttributeValueField); - - } - - @Test - public void given_fields_with_similar_suffix_return_the_correct_type() { - - SolrSampleField attributeField = fieldService.decodeField("MRSXGY3SNFYHI2LPNY_______av_ss"); - SolrSampleField nameField = fieldService.decodeField("name_s"); - SolrSampleField domainField = fieldService.decodeField("domain_s"); - - assertTrue(attributeField instanceof SolrSampleAttributeValueField); - assertTrue(nameField instanceof SolrSampleNameField); - assertTrue(domainField instanceof SolrSampleDomainField); - - } - - - @Test - public void given_filter_object_return_the_corresponding_solr_field() { - Filter organismFilter = FilterBuilder.create() - .onAttribute("organism") - .withValue("Homo sapiens") - .build(); - SolrSampleAttributeValueField organism = new SolrSampleAttributeValueField("organism"); - Optional criteriaFromField = Optional.ofNullable(organism.getFilterCriteria(organismFilter)); - Optional criteriaFromService = solrFilterService.getFilterCriteria(organismFilter); - - assertEquals(criteriaFromField.toString(), criteriaFromService.toString()); - } - - -} diff --git a/pipelines/analytics/pom.xml b/pipelines/analytics/pom.xml index a657b562f..3a8184c52 100644 --- a/pipelines/analytics/pom.xml +++ b/pipelines/analytics/pom.xml @@ -13,7 +13,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ @@ -24,33 +24,18 @@ uk.ac.ebi.biosamples - models-mongo - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-ols - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples webapps-core - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT classes @@ -64,7 +49,6 @@ - diff --git a/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java b/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java index 943b55efa..888bc3fe5 100644 --- a/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java +++ b/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java @@ -20,11 +20,11 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SampleAnalytics; -import uk.ac.ebi.biosamples.model.facet.Facet; -import uk.ac.ebi.biosamples.model.facet.content.LabelCountEntry; -import uk.ac.ebi.biosamples.model.facet.content.LabelCountListContent; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SampleAnalytics; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountEntry; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; import uk.ac.ebi.biosamples.service.FacetService; import uk.ac.ebi.biosamples.service.SamplePageService; diff --git a/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/Application.java index 14fd81aa1..c3551f905 100644 --- a/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -34,14 +34,9 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; -import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.configuration.ExclusionConfiguration; -import uk.ac.ebi.biosamples.mongo.repository.MongoSampleRepository; -import uk.ac.ebi.biosamples.mongo.service.MongoAccessionService; -import uk.ac.ebi.biosamples.mongo.service.MongoSampleToSampleConverter; -import uk.ac.ebi.biosamples.mongo.service.SampleToMongoSampleConverter; import uk.ac.ebi.biosamples.service.EnaConfig; import uk.ac.ebi.biosamples.service.EnaSampleToBioSampleConversionService; import uk.ac.ebi.biosamples.service.EraProDao; @@ -153,17 +148,4 @@ public long getKeepAliveDuration( } }; } - - @Bean(name = "SampleAccessionService") - public MongoAccessionService mongoSampleAccessionService( - final MongoSampleRepository mongoSampleRepository, - final SampleToMongoSampleConverter sampleToMongoSampleConverter, - final MongoSampleToSampleConverter mongoSampleToSampleConverter, - final MongoOperations mongoOperations) { - return new MongoAccessionService( - mongoSampleRepository, - sampleToMongoSampleConverter, - mongoSampleToSampleConverter, - mongoOperations); - } } diff --git a/pipelines/chain/pom.xml b/pipelines/chain/pom.xml index 1a337d1d4..1a2b710b5 100644 --- a/pipelines/chain/pom.xml +++ b/pipelines/chain/pom.xml @@ -8,7 +8,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ @@ -18,19 +18,18 @@ spring-boot-starter-cache - uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT + org.springframework + spring-webmvc uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT org.apache.commons @@ -38,12 +37,6 @@ 3.7 test - - uk.ac.ebi.biosamples - models-mongo - 5.3.12-SNAPSHOT - compile - org.apache.commons commons-csv diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/auth/services/AuthChangeHandler.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/auth/services/AuthChangeHandler.java index 3badd1c5f..ddbc25173 100644 --- a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/auth/services/AuthChangeHandler.java +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/auth/services/AuthChangeHandler.java @@ -23,8 +23,8 @@ import org.springframework.hateoas.EntityModel; import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SubmittedViaType; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SubmittedViaType; import uk.ac.ebi.biosamples.mongo.model.MongoAuthChangeRecord; import uk.ac.ebi.biosamples.mongo.repository.MongoAuthChangeRepository; diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/auth/services/SamplesCrawlerAuthChangeHandler.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/auth/services/SamplesCrawlerAuthChangeHandler.java new file mode 100644 index 000000000..8d34e5c94 --- /dev/null +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/auth/services/SamplesCrawlerAuthChangeHandler.java @@ -0,0 +1,92 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.auth.services; + +import java.util.List; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; +import org.springframework.hateoas.EntityModel; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.client.BioSamplesClient; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SubmittedViaType; +import uk.ac.ebi.biosamples.core.model.filter.AuthenticationFilter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.mongo.model.MongoAuthChangeRecord; +import uk.ac.ebi.biosamples.mongo.repository.MongoAuthChangeRepository; + +@Service +@Slf4j +public class SamplesCrawlerAuthChangeHandler { + private static final String WEBIN_ID_TO_CHANGE_TO = "Webin-69232"; + private final BioSamplesClient bioSamplesClient; + private final MongoAuthChangeRepository mongoAuthChangeRepository; + + public SamplesCrawlerAuthChangeHandler( + final BioSamplesClient bioSamplesClient, + final MongoAuthChangeRepository mongoAuthChangeRepository) { + this.bioSamplesClient = bioSamplesClient; + this.mongoAuthChangeRepository = mongoAuthChangeRepository; + } + + public Iterable> getSamples(final String domain) { + final Filter authenticationFilter = new AuthenticationFilter.Builder(domain).build(); + return bioSamplesClient.fetchSampleResourceAllWithoutCuration( + "", List.of(authenticationFilter)); + } + + public void handleAuth(final EntityModel sampleEntityModel, final String domain) { + final Sample sample = sampleEntityModel.getContent(); + + log.info("Handling Sample {}", sample.getAccession()); + + final String accession = sample.getAccession(); + final String sampleDomain = sample.getDomain(); + final String webinId = sample.getWebinSubmissionAccountId(); + + if (!accession.startsWith("SAME") + || (sample.getSubmittedVia() == SubmittedViaType.PIPELINE_IMPORT + || sample.getSubmittedVia() == SubmittedViaType.WEBIN_SERVICES)) { + log.info("Sample is an imported sample: {} please update at source", accession); + + return; + } + + if (sampleDomain != null && sampleDomain.equals(domain) && webinId == null) { + log.info( + "Sample authority needs to change for: " + + accession + + " setting to: " + + WEBIN_ID_TO_CHANGE_TO); + + final Sample updatedSample = + Sample.Builder.fromSample(sample) + .withWebinSubmissionAccountId(WEBIN_ID_TO_CHANGE_TO) + .withNoDomain() + .build(); + final EntityModel savedSampleEntityModel = + bioSamplesClient.persistSampleResource(updatedSample); + + if (Objects.requireNonNull(savedSampleEntityModel.getContent()) + .getWebinSubmissionAccountId() + .equals(WEBIN_ID_TO_CHANGE_TO)) { + log.info("Sample " + accession + " updated"); + + mongoAuthChangeRepository.save( + new MongoAuthChangeRecord(accession, domain, WEBIN_ID_TO_CHANGE_TO)); + } else { + log.info("Sample " + accession + " failed to be updated"); + } + } else { + log.info("Sample from some other domain, no change required"); + } + } +} diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java index ce3098807..c062e6a9b 100644 --- a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java @@ -23,8 +23,8 @@ import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.PipelinesProperties; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; @Component public class SampleChecklistComplianceHandlerEVA { diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleExternalReferenceHandler.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleExternalReferenceHandler.java index dac44aa8b..7a9f461b4 100644 --- a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleExternalReferenceHandler.java +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleExternalReferenceHandler.java @@ -18,9 +18,9 @@ import org.springframework.stereotype.Service; import uk.ac.ebi.biosamples.PipelinesProperties; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.ExternalReference; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.ExternalReference; +import uk.ac.ebi.biosamples.core.model.Sample; @Service public class SampleExternalReferenceHandler { diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleRelationshipHandler.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleRelationshipHandler.java index a7cd39402..de369823f 100644 --- a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleRelationshipHandler.java +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleRelationshipHandler.java @@ -24,8 +24,8 @@ import org.springframework.hateoas.EntityModel; import org.springframework.stereotype.Service; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Sample; @Service @Slf4j diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleRestoreIPK.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleRestoreIPK.java index e93fb0d95..eb9766958 100644 --- a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleRestoreIPK.java +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleRestoreIPK.java @@ -22,8 +22,8 @@ import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.BioSamplesConstants; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; @Service @Slf4j diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleStatusUpdater.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleStatusUpdater.java index 16b1c4db9..5d67dddfc 100644 --- a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleStatusUpdater.java +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleStatusUpdater.java @@ -24,10 +24,10 @@ import org.springframework.hateoas.EntityModel; import org.springframework.stereotype.Service; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.AttributeFilter; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.AttributeFilter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; @Service @Slf4j diff --git a/utils/pipeline/pom.xml b/pipelines/common/pom.xml similarity index 71% rename from utils/pipeline/pom.xml rename to pipelines/common/pom.xml index 0b6d51cb0..6246dc256 100644 --- a/utils/pipeline/pom.xml +++ b/pipelines/common/pom.xml @@ -1,77 +1,82 @@ - - 4.0.0 - utils-pipeline - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - - - - uk.ac.ebi.biosamples - biosamples-spring-boot-starter - 5.3.12-SNAPSHOT - - - - org.springframework.hateoas - spring-hateoas - 1.3.4 - - - - - org.apache.httpcomponents - httpclient-cache - ${httpclient.version} - - - org.springframework.boot - spring-boot-starter-jdbc - - - org.apache.tomcat - tomcat-jdbc - - - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-csv - 2.8.8 - - - uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - compile - - - uk.ac.ebi.biosamples - properties - 5.3.12-SNAPSHOT - compile - - - - xmlunit - xmlunit - 1.4 - - - org.springframework - spring-aop - - - dom4j - dom4j - 1.6.1 - - - + + + 4.0.0 + pipelines-common + jar + + uk.ac.ebi.biosamples + biosamples + 5.3.13-SNAPSHOT + ../../pom.xml + + + + 17 + 17 + UTF-8 + + + + + uk.ac.ebi.biosamples + biosamples-spring-boot-starter + 5.3.13-SNAPSHOT + + + org.springframework.hateoas + spring-hateoas + 1.3.4 + + + + org.apache.httpcomponents + httpclient-cache + ${httpclient.version} + + + org.springframework.boot + spring-boot-starter-jdbc + + + org.apache.tomcat + tomcat-jdbc + + + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-csv + 2.8.8 + + + uk.ac.ebi.biosamples + core + 5.3.13-SNAPSHOT + compile + + + uk.ac.ebi.biosamples + properties + 5.3.13-SNAPSHOT + compile + + + + xmlunit + xmlunit + 1.4 + + + org.springframework + spring-aop + + + dom4j + dom4j + 1.6.1 + + + diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelineApplicationRunner.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelineApplicationRunner.java similarity index 92% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelineApplicationRunner.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelineApplicationRunner.java index 80e421adf..357b9ce4b 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelineApplicationRunner.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelineApplicationRunner.java @@ -1,114 +1,114 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples; - -import java.time.Duration; -import java.time.Instant; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.Callable; -import java.util.concurrent.Future; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.ApplicationArguments; -import org.springframework.boot.ApplicationRunner; -import org.springframework.hateoas.EntityModel; -import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.PipelineAnalytics; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.utils.AdaptiveThreadPoolExecutor; -import uk.ac.ebi.biosamples.utils.PipelineUtils; -import uk.ac.ebi.biosamples.utils.ThreadUtils; - -public abstract class PipelineApplicationRunner implements ApplicationRunner { - protected final Logger LOG = LoggerFactory.getLogger(getClass()); - private static final String PIPELINE_NAME = "TEMPLATE"; - - protected final BioSamplesClient bioSamplesClient; - private final PipelinesProperties pipelinesProperties; - private final PipelineFutureCallback pipelineFutureCallback; - - public PipelineApplicationRunner( - final BioSamplesClient bioSamplesClient, final PipelinesProperties pipelinesProperties) { - this.bioSamplesClient = bioSamplesClient; - this.pipelinesProperties = pipelinesProperties; - // this.analyticsService = analyticsService; - pipelineFutureCallback = new PipelineFutureCallback(); - } - - @Override - public void run(final ApplicationArguments args) throws Exception { - final Instant startTime = Instant.now(); - LOG.info("Pipeline started at {}", startTime); - final Collection filters = PipelineUtils.getDateFilters(args, "update"); - long sampleCount = 0; - - loadPreConfiguration(); - - try (final AdaptiveThreadPoolExecutor executorService = - AdaptiveThreadPoolExecutor.create( - 100, - 10000, - true, - pipelinesProperties.getThreadCount(), - pipelinesProperties.getThreadCountMax())) { - - final Map> futures = new HashMap<>(); - for (final EntityModel sampleResource : - bioSamplesClient.fetchSampleResourceAll("", filters)) { - final Sample sample = Objects.requireNonNull(sampleResource.getContent()); - LOG.trace("Handling {}", sample); - - final Callable task = getNewCallableClassInstance().withSample(sample); - sampleCount++; - if (sampleCount % 10000 == 0) { - LOG.info("{} samples scheduled for processing", sampleCount); - } - futures.put(sample.getAccession(), executorService.submit(task)); - } - - LOG.info("waiting for futures to finish"); - ThreadUtils.checkAndCallbackFutures(futures, 0, pipelineFutureCallback); - } catch (final Exception e) { - LOG.error("Pipeline failed to finish successfully", e); - throw e; - } finally { - final Instant endTime = Instant.now(); - LOG.info("Total samples processed {}", sampleCount); - LOG.info("Total curation objects added {}", pipelineFutureCallback.getTotalCount()); - LOG.info("Pipeline finished at {}", endTime); - LOG.info( - "Pipeline total running time {} seconds", - Duration.between(startTime, endTime).getSeconds()); - - final PipelineAnalytics pipelineAnalytics = - new PipelineAnalytics( - getPipelineName(), - startTime, - endTime, - sampleCount, - pipelineFutureCallback.getTotalCount()); - pipelineAnalytics.setDateRange(filters); - // analyticsService.persistPipelineAnalytics(pipelineAnalytics); - } - } - - private String getPipelineName() { - return PIPELINE_NAME; - } - - protected abstract void loadPreConfiguration(); - - protected abstract PipelineSampleCallable getNewCallableClassInstance(); -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples; + +import java.time.Duration; +import java.time.Instant; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.Future; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.hateoas.EntityModel; +import uk.ac.ebi.biosamples.client.BioSamplesClient; +import uk.ac.ebi.biosamples.core.model.PipelineAnalytics; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.utils.PipelineUtils; +import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; + +public abstract class PipelineApplicationRunner implements ApplicationRunner { + protected final Logger LOG = LoggerFactory.getLogger(getClass()); + private static final String PIPELINE_NAME = "TEMPLATE"; + + protected final BioSamplesClient bioSamplesClient; + private final PipelinesProperties pipelinesProperties; + private final PipelineFutureCallback pipelineFutureCallback; + + public PipelineApplicationRunner( + final BioSamplesClient bioSamplesClient, final PipelinesProperties pipelinesProperties) { + this.bioSamplesClient = bioSamplesClient; + this.pipelinesProperties = pipelinesProperties; + // this.analyticsService = analyticsService; + pipelineFutureCallback = new PipelineFutureCallback(); + } + + @Override + public void run(final ApplicationArguments args) throws Exception { + final Instant startTime = Instant.now(); + LOG.info("Pipeline started at {}", startTime); + final Collection filters = PipelineUtils.getDateFilters(args, "update"); + long sampleCount = 0; + + loadPreConfiguration(); + + try (final AdaptiveThreadPoolExecutor executorService = + AdaptiveThreadPoolExecutor.create( + 100, + 10000, + true, + pipelinesProperties.getThreadCount(), + pipelinesProperties.getThreadCountMax())) { + + final Map> futures = new HashMap<>(); + for (final EntityModel sampleResource : + bioSamplesClient.fetchSampleResourceAll("", filters)) { + final Sample sample = Objects.requireNonNull(sampleResource.getContent()); + LOG.trace("Handling {}", sample); + + final Callable task = getNewCallableClassInstance().withSample(sample); + sampleCount++; + if (sampleCount % 10000 == 0) { + LOG.info("{} samples scheduled for processing", sampleCount); + } + futures.put(sample.getAccession(), executorService.submit(task)); + } + + LOG.info("waiting for futures to finish"); + ThreadUtils.checkAndCallbackFutures(futures, 0, pipelineFutureCallback); + } catch (final Exception e) { + LOG.error("Pipeline failed to finish successfully", e); + throw e; + } finally { + final Instant endTime = Instant.now(); + LOG.info("Total samples processed {}", sampleCount); + LOG.info("Total curation objects added {}", pipelineFutureCallback.getTotalCount()); + LOG.info("Pipeline finished at {}", endTime); + LOG.info( + "Pipeline total running time {} seconds", + Duration.between(startTime, endTime).getSeconds()); + + final PipelineAnalytics pipelineAnalytics = + new PipelineAnalytics( + getPipelineName(), + startTime, + endTime, + sampleCount, + pipelineFutureCallback.getTotalCount()); + pipelineAnalytics.setDateRange(filters); + // analyticsService.persistPipelineAnalytics(pipelineAnalytics); + } + } + + private String getPipelineName() { + return PIPELINE_NAME; + } + + protected abstract void loadPreConfiguration(); + + protected abstract PipelineSampleCallable getNewCallableClassInstance(); +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelineFutureCallback.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelineFutureCallback.java similarity index 93% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelineFutureCallback.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelineFutureCallback.java index 6de3eff0f..c04a83a38 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelineFutureCallback.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelineFutureCallback.java @@ -1,37 +1,37 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples; - -import java.util.ArrayList; -import java.util.List; -import uk.ac.ebi.biosamples.utils.ThreadUtils; - -public class PipelineFutureCallback implements ThreadUtils.Callback { - private long totalCount = 0; - private final List failedSamples = new ArrayList<>(); - - @Override - public void call(final PipelineResult pipelineResult) { - totalCount = totalCount + pipelineResult.getModifiedRecords(); - - if (!pipelineResult.isSuccess()) { - failedSamples.add(pipelineResult.getAccession()); - } - } - - public long getTotalCount() { - return totalCount; - } - - public List getFailedSamples() { - return failedSamples; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples; + +import java.util.ArrayList; +import java.util.List; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; + +public class PipelineFutureCallback implements ThreadUtils.Callback { + private long totalCount = 0; + private final List failedSamples = new ArrayList<>(); + + @Override + public void call(final PipelineResult pipelineResult) { + totalCount = totalCount + pipelineResult.getModifiedRecords(); + + if (!pipelineResult.isSuccess()) { + failedSamples.add(pipelineResult.getAccession()); + } + } + + public long getTotalCount() { + return totalCount; + } + + public List getFailedSamples() { + return failedSamples; + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelineResult.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelineResult.java similarity index 97% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelineResult.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelineResult.java index 0c5a2331b..938e4f0f5 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelineResult.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelineResult.java @@ -1,26 +1,26 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples; - -import lombok.Getter; - -@Getter -public class PipelineResult { - private final long modifiedRecords; - private final boolean success; - private final String accession; - - public PipelineResult(String accession, long modifiedRecords, boolean success) { - this.accession = accession; - this.modifiedRecords = modifiedRecords; - this.success = success; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples; + +import lombok.Getter; + +@Getter +public class PipelineResult { + private final long modifiedRecords; + private final boolean success; + private final String accession; + + public PipelineResult(String accession, long modifiedRecords, boolean success) { + this.accession = accession; + this.modifiedRecords = modifiedRecords; + this.success = success; + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelineSampleCallable.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelineSampleCallable.java similarity index 94% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelineSampleCallable.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelineSampleCallable.java index 986d5dfee..b1995e9a6 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelineSampleCallable.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelineSampleCallable.java @@ -1,48 +1,48 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples; - -import java.util.concurrent.Callable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Sample; - -public abstract class PipelineSampleCallable implements Callable { - protected final Logger LOG = LoggerFactory.getLogger(getClass()); - - protected Sample sample; - private final BioSamplesClient bioSamplesClient; - - public PipelineSampleCallable(final BioSamplesClient bioSamplesClient) { - this.bioSamplesClient = bioSamplesClient; - } - - @Override - public PipelineResult call() { - boolean success = true; - int appliedCurations = 0; - try { - appliedCurations = processSample(sample); - } catch (final Exception e) { - success = false; - LOG.error("Failed to add curation on sample: " + sample.getAccession(), e); - } - return new PipelineResult(sample.getAccession(), appliedCurations, success); - } - - PipelineSampleCallable withSample(final Sample sample) { - this.sample = sample; - return this; - } - - protected abstract int processSample(Sample sample) throws Exception; -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples; + +import java.util.concurrent.Callable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import uk.ac.ebi.biosamples.client.BioSamplesClient; +import uk.ac.ebi.biosamples.core.model.Sample; + +public abstract class PipelineSampleCallable implements Callable { + protected final Logger LOG = LoggerFactory.getLogger(getClass()); + + protected Sample sample; + private final BioSamplesClient bioSamplesClient; + + public PipelineSampleCallable(final BioSamplesClient bioSamplesClient) { + this.bioSamplesClient = bioSamplesClient; + } + + @Override + public PipelineResult call() { + boolean success = true; + int appliedCurations = 0; + try { + appliedCurations = processSample(sample); + } catch (final Exception e) { + success = false; + LOG.error("Failed to add curation on sample: " + sample.getAccession(), e); + } + return new PipelineResult(sample.getAccession(), appliedCurations, success); + } + + PipelineSampleCallable withSample(final Sample sample) { + this.sample = sample; + return this; + } + + protected abstract int processSample(Sample sample) throws Exception; +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelinesHelper.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesHelper.java similarity index 97% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelinesHelper.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesHelper.java index 9f3b266e8..62033f13c 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelinesHelper.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesHelper.java @@ -1,109 +1,109 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples; - -import org.apache.http.HeaderElement; -import org.apache.http.HeaderElementIterator; -import org.apache.http.HttpHost; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.conn.ConnectionKeepAliveStrategy; -import org.apache.http.conn.routing.HttpRoute; -import org.apache.http.impl.client.cache.CacheConfig; -import org.apache.http.impl.client.cache.CachingHttpClientBuilder; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.http.message.BasicHeaderElementIterator; -import org.apache.http.protocol.HTTP; -import org.springframework.boot.web.client.RestTemplateCustomizer; -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; - -public class PipelinesHelper { - public RestTemplateCustomizer getRestTemplateCustomizer( - BioSamplesProperties bioSamplesProperties, PipelinesProperties pipelinesProperties) { - return restTemplate -> { - HttpClient httpClient = configureHttpClient(pipelinesProperties, bioSamplesProperties); - restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory(httpClient)); - }; - } - - private HttpClient configureHttpClient( - PipelinesProperties pipelinesProperties, BioSamplesProperties bioSamplesProperties) { - ConnectionKeepAliveStrategy keepAliveStrategy = configureConnectionKeepAliveStrategy(); - PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = - configurePoolingHttpClientConnectionManager(pipelinesProperties, bioSamplesProperties); - CacheConfig cacheConfig = configureCache(); - RequestConfig config = configureTimeout(pipelinesProperties.getConnectionTimeout()); - - return CachingHttpClientBuilder.create() - .setCacheConfig(cacheConfig) - .useSystemProperties() - .setConnectionManager(poolingHttpClientConnectionManager) - .setKeepAliveStrategy(keepAliveStrategy) - .setDefaultRequestConfig(config) - .build(); - } - - private RequestConfig configureTimeout(int timeout) { - return RequestConfig.custom() - .setConnectTimeout(timeout * 1000) // time to establish the connection with the remote host - .setConnectionRequestTimeout( - timeout * 1000) // maximum time of inactivity between two data packets - .setSocketTimeout(timeout * 1000) - .build(); - } - - private ConnectionKeepAliveStrategy configureConnectionKeepAliveStrategy() { - return (response, context) -> { - long keepAliveDuration = 60 * 1000L; - - // check if there is a non-standard keep alive header present - HeaderElementIterator it = - new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); - while (it.hasNext()) { - HeaderElement he = it.nextElement(); - String param = he.getName(); - String value = he.getValue(); - if (value != null && param.equalsIgnoreCase("timeout")) { - keepAliveDuration = Long.parseLong(value) * 1000; - } - } - - return keepAliveDuration; - }; - } - - private PoolingHttpClientConnectionManager configurePoolingHttpClientConnectionManager( - PipelinesProperties pipelinesProperties, BioSamplesProperties bioSamplesProperties) { - - PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = - new PoolingHttpClientConnectionManager(); - poolingHttpClientConnectionManager.setMaxTotal(pipelinesProperties.getConnectionCountMax()); - poolingHttpClientConnectionManager.setDefaultMaxPerRoute( - pipelinesProperties.getConnectionCountDefault()); - poolingHttpClientConnectionManager.setMaxPerRoute( - new HttpRoute(HttpHost.create(pipelinesProperties.getZooma())), - pipelinesProperties.getConnectionCountZooma()); - poolingHttpClientConnectionManager.setMaxPerRoute( - new HttpRoute(HttpHost.create(bioSamplesProperties.getOls())), - pipelinesProperties.getConnectionCountOls()); - - return poolingHttpClientConnectionManager; - } - - private CacheConfig configureCache() { - return CacheConfig.custom() - .setMaxCacheEntries(1024) - .setMaxObjectSize(1024 * 1024L) // max size of 1Mb - // number of entries x size of entries = 1Gb total cache size - .setSharedCache(false) // act like a browser cache not a middle-hop cache - .build(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples; + +import org.apache.http.HeaderElement; +import org.apache.http.HeaderElementIterator; +import org.apache.http.HttpHost; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.conn.ConnectionKeepAliveStrategy; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.impl.client.cache.CacheConfig; +import org.apache.http.impl.client.cache.CachingHttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicHeaderElementIterator; +import org.apache.http.protocol.HTTP; +import org.springframework.boot.web.client.RestTemplateCustomizer; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; + +public class PipelinesHelper { + public RestTemplateCustomizer getRestTemplateCustomizer( + BioSamplesProperties bioSamplesProperties, PipelinesProperties pipelinesProperties) { + return restTemplate -> { + HttpClient httpClient = configureHttpClient(pipelinesProperties, bioSamplesProperties); + restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory(httpClient)); + }; + } + + private HttpClient configureHttpClient( + PipelinesProperties pipelinesProperties, BioSamplesProperties bioSamplesProperties) { + ConnectionKeepAliveStrategy keepAliveStrategy = configureConnectionKeepAliveStrategy(); + PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = + configurePoolingHttpClientConnectionManager(pipelinesProperties, bioSamplesProperties); + CacheConfig cacheConfig = configureCache(); + RequestConfig config = configureTimeout(pipelinesProperties.getConnectionTimeout()); + + return CachingHttpClientBuilder.create() + .setCacheConfig(cacheConfig) + .useSystemProperties() + .setConnectionManager(poolingHttpClientConnectionManager) + .setKeepAliveStrategy(keepAliveStrategy) + .setDefaultRequestConfig(config) + .build(); + } + + private RequestConfig configureTimeout(int timeout) { + return RequestConfig.custom() + .setConnectTimeout(timeout * 1000) // time to establish the connection with the remote host + .setConnectionRequestTimeout( + timeout * 1000) // maximum time of inactivity between two data packets + .setSocketTimeout(timeout * 1000) + .build(); + } + + private ConnectionKeepAliveStrategy configureConnectionKeepAliveStrategy() { + return (response, context) -> { + long keepAliveDuration = 60 * 1000L; + + // check if there is a non-standard keep alive header present + HeaderElementIterator it = + new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); + while (it.hasNext()) { + HeaderElement he = it.nextElement(); + String param = he.getName(); + String value = he.getValue(); + if (value != null && param.equalsIgnoreCase("timeout")) { + keepAliveDuration = Long.parseLong(value) * 1000; + } + } + + return keepAliveDuration; + }; + } + + private PoolingHttpClientConnectionManager configurePoolingHttpClientConnectionManager( + PipelinesProperties pipelinesProperties, BioSamplesProperties bioSamplesProperties) { + + PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = + new PoolingHttpClientConnectionManager(); + poolingHttpClientConnectionManager.setMaxTotal(pipelinesProperties.getConnectionCountMax()); + poolingHttpClientConnectionManager.setDefaultMaxPerRoute( + pipelinesProperties.getConnectionCountDefault()); + poolingHttpClientConnectionManager.setMaxPerRoute( + new HttpRoute(HttpHost.create(pipelinesProperties.getZooma())), + pipelinesProperties.getConnectionCountZooma()); + poolingHttpClientConnectionManager.setMaxPerRoute( + new HttpRoute(HttpHost.create(bioSamplesProperties.getOls())), + pipelinesProperties.getConnectionCountOls()); + + return poolingHttpClientConnectionManager; + } + + private CacheConfig configureCache() { + return CacheConfig.custom() + .setMaxCacheEntries(1024) + .setMaxObjectSize(1024 * 1024L) // max size of 1Mb + // number of entries x size of entries = 1Gb total cache size + .setSharedCache(false) // act like a browser cache not a middle-hop cache + .build(); + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java similarity index 96% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java index d770d5d9b..baf612762 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java @@ -1,166 +1,166 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Component -public class PipelinesProperties { - @Value("${biosamples.pipelines.ncbi.file:/ncbi/biosample_set.xml.gz}") - private String ncbiFile; - - @Value("${biosamples.pipelines.threadcount:1}") - private int threadCount; - - @Value("${biosamples.pipelines.threadcount.max:60}") - private int threadCountMax; - - @Value("${biosamples.pipelines.connectioncount.max:8}") - private int connectionCountMax; - - @Value("${biosamples.pipelines.connectioncount.default:8}") - private int connectionCountDefault; - - @Value("${biosamples.pipelines.connectioncount.ols:8}") - private int connectionCountOls; - - @Value("${biosamples.pipelines.connectioncount.zooma:1}") - private int connectionCountZooma; - - @Value("${biosamples.pipelines.connectiontimeout:60}") - private int connectionTimeout; - - @Value("${biosamples.pipelines.zooma:https://www.ebi.ac.uk/spot/zooma}") - private String zooma; - - @Value("${biosamples.pipelines.ncbi.domain:self.BiosampleImportNCBI}") - private String ncbiDomain; - - @Value("${biosamples.pipelines.ncbi.controlledaccess:true}") - private Boolean ncbiControlledAccess; - - @Value("${biosamples.pipelines.ena.domain:self.BiosampleImportENA}") - private String enaDomain; - - @Value("${biosamples.pipelines.ena.webin_id:Webin-40894}") - private String proxyWebinId; - - @Value("${biosamples.pipelines.accession.domain:self.BiosampleImportAcccession}") - private String accessionDomain; - - @Value("${biosamples.pipelines.curation.domain:self.BiosampleCuration}") - private String curationDomain; - - @Value("${biosamples.pipelines.zooma.domain:self.BiosampleZooma}") - private String zoomaDomain; - - @Value("${biosamples.pipelines.copydown.domain:self.BiosampleCopydown}") - private String copydownDomain; - - @Value("${biosamples.schemaValidator:http://localhost:3020/validate}") - private String schemaValidator; - - @Value("${biosamples.schemaStore:http://localhost:8085/api/v2/schemas}") - private String schemaStore; - - @Value( - "${biosample.pipelines.samplerelease.get:http://wwwint.ebi.ac.uk/webin/era/service/test/biosample/release/}") - private String webinEraServiceSampleReleaseGet; - - @Value( - "${biosample.pipelines.samplerelease.delete:http://wwwint.ebi.ac.uk/webin/era/service/test/biosample/release/{biosampleAccession}}") - private String webinEraServiceSampleReleaseDelete; - - public String getWebinEraServiceSampleReleaseGet() { - return webinEraServiceSampleReleaseGet; - } - - public String getWebinEraServiceSampleReleaseDelete() { - return webinEraServiceSampleReleaseDelete; - } - - public String getProxyWebinId() { - return proxyWebinId; - } - - public String getNcbiFile() { - return ncbiFile; - } - - public String getNcbiDomain() { - return ncbiDomain; - } - - public Boolean getNcbiControlledAccess() { - return ncbiControlledAccess; - } - - public String getEnaDomain() { - return enaDomain; - } - - public String getAccessionDomain() { - return accessionDomain; - } - - public String getCurationDomain() { - return curationDomain; - } - - public String getZoomaDomain() { - return zoomaDomain; - } - - public String getCopydownDomain() { - return copydownDomain; - } - - public int getThreadCount() { - return threadCount; - } - - public int getThreadCountMax() { - return threadCountMax; - } - - public int getConnectionCountMax() { - return connectionCountMax; - } - - public int getConnectionCountDefault() { - return connectionCountDefault; - } - - public int getConnectionCountOls() { - return connectionCountOls; - } - - public int getConnectionCountZooma() { - return connectionCountZooma; - } - - public int getConnectionTimeout() { - return connectionTimeout; - } - - public String getZooma() { - return zooma; - } - - public String getSchemaValidator() { - return schemaValidator; - } - - public String getSchemaStore() { - return schemaStore; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class PipelinesProperties { + @Value("${biosamples.pipelines.ncbi.file:/ncbi/biosample_set.xml.gz}") + private String ncbiFile; + + @Value("${biosamples.pipelines.threadcount:1}") + private int threadCount; + + @Value("${biosamples.pipelines.threadcount.max:60}") + private int threadCountMax; + + @Value("${biosamples.pipelines.connectioncount.max:8}") + private int connectionCountMax; + + @Value("${biosamples.pipelines.connectioncount.default:8}") + private int connectionCountDefault; + + @Value("${biosamples.pipelines.connectioncount.ols:8}") + private int connectionCountOls; + + @Value("${biosamples.pipelines.connectioncount.zooma:1}") + private int connectionCountZooma; + + @Value("${biosamples.pipelines.connectiontimeout:60}") + private int connectionTimeout; + + @Value("${biosamples.pipelines.zooma:https://www.ebi.ac.uk/spot/zooma}") + private String zooma; + + @Value("${biosamples.pipelines.ncbi.domain:self.BiosampleImportNCBI}") + private String ncbiDomain; + + @Value("${biosamples.pipelines.ncbi.controlledaccess:true}") + private Boolean ncbiControlledAccess; + + @Value("${biosamples.pipelines.ena.domain:self.BiosampleImportENA}") + private String enaDomain; + + @Value("${biosamples.pipelines.ena.webin_id:Webin-40894}") + private String proxyWebinId; + + @Value("${biosamples.pipelines.accession.domain:self.BiosampleImportAcccession}") + private String accessionDomain; + + @Value("${biosamples.pipelines.curation.domain:self.BiosampleCuration}") + private String curationDomain; + + @Value("${biosamples.pipelines.zooma.domain:self.BiosampleZooma}") + private String zoomaDomain; + + @Value("${biosamples.pipelines.copydown.domain:self.BiosampleCopydown}") + private String copydownDomain; + + @Value("${biosamples.schemaValidator:http://localhost:3020/validate}") + private String schemaValidator; + + @Value("${biosamples.schemaStore:http://localhost:8085/api/v2/schemas}") + private String schemaStore; + + @Value( + "${biosample.pipelines.samplerelease.get:http://wwwint.ebi.ac.uk/webin/era/service/test/biosample/release/}") + private String webinEraServiceSampleReleaseGet; + + @Value( + "${biosample.pipelines.samplerelease.delete:http://wwwint.ebi.ac.uk/webin/era/service/test/biosample/release/{biosampleAccession}}") + private String webinEraServiceSampleReleaseDelete; + + public String getWebinEraServiceSampleReleaseGet() { + return webinEraServiceSampleReleaseGet; + } + + public String getWebinEraServiceSampleReleaseDelete() { + return webinEraServiceSampleReleaseDelete; + } + + public String getProxyWebinId() { + return proxyWebinId; + } + + public String getNcbiFile() { + return ncbiFile; + } + + public String getNcbiDomain() { + return ncbiDomain; + } + + public Boolean getNcbiControlledAccess() { + return ncbiControlledAccess; + } + + public String getEnaDomain() { + return enaDomain; + } + + public String getAccessionDomain() { + return accessionDomain; + } + + public String getCurationDomain() { + return curationDomain; + } + + public String getZoomaDomain() { + return zoomaDomain; + } + + public String getCopydownDomain() { + return copydownDomain; + } + + public int getThreadCount() { + return threadCount; + } + + public int getThreadCountMax() { + return threadCountMax; + } + + public int getConnectionCountMax() { + return connectionCountMax; + } + + public int getConnectionCountDefault() { + return connectionCountDefault; + } + + public int getConnectionCountOls() { + return connectionCountOls; + } + + public int getConnectionCountZooma() { + return connectionCountZooma; + } + + public int getConnectionTimeout() { + return connectionTimeout; + } + + public String getZooma() { + return zooma; + } + + public String getSchemaValidator() { + return schemaValidator; + } + + public String getSchemaStore() { + return schemaStore; + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/configuration/ExclusionConfiguration.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/configuration/ExclusionConfiguration.java similarity index 97% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/configuration/ExclusionConfiguration.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/configuration/ExclusionConfiguration.java index 5c22cbe33..b47b428e3 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/configuration/ExclusionConfiguration.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/configuration/ExclusionConfiguration.java @@ -1,21 +1,21 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.configuration; - -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; - -@Configuration -@ComponentScan({ - "uk.ac.ebi.biosamples.model", - "uk.ac.ebi.biosamples.utils", -}) -public class ExclusionConfiguration {} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.configuration; + +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ComponentScan({ + "uk.ac.ebi.biosamples.model", + "uk.ac.ebi.biosamples.utils", +}) +public class ExclusionConfiguration {} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/model/AccessionFtpUrlPair.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/AccessionFtpUrlPair.java similarity index 96% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/model/AccessionFtpUrlPair.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/AccessionFtpUrlPair.java index 946b42461..69c4245a4 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/model/AccessionFtpUrlPair.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/AccessionFtpUrlPair.java @@ -1,39 +1,39 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -public class AccessionFtpUrlPair { - String accession; - String ftpUrl; - - public AccessionFtpUrlPair(final String accession, final String ftpUrl) { - this.accession = accession; - this.ftpUrl = ftpUrl; - } - - public AccessionFtpUrlPair() {} - - public void setAccession(final String accession) { - this.accession = accession; - } - - public void setFtpUrl(final String ftpUrl) { - this.ftpUrl = ftpUrl; - } - - public String getAccession() { - return accession; - } - - public String getFtpUrl() { - return ftpUrl; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.model; + +public class AccessionFtpUrlPair { + String accession; + String ftpUrl; + + public AccessionFtpUrlPair(final String accession, final String ftpUrl) { + this.accession = accession; + this.ftpUrl = ftpUrl; + } + + public AccessionFtpUrlPair() {} + + public void setAccession(final String accession) { + this.accession = accession; + } + + public void setFtpUrl(final String ftpUrl) { + this.ftpUrl = ftpUrl; + } + + public String getAccession() { + return accession; + } + + public String getFtpUrl() { + return ftpUrl; + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/model/PipelineName.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineName.java similarity index 96% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/model/PipelineName.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineName.java index 29bf67ded..b43aaf6af 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/model/PipelineName.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineName.java @@ -1,26 +1,26 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.model; - -public enum PipelineName { - CURATION, - CURAMI, - COPYDOWN, - ENA, - NCBI, - ENAAMR, - EBEYESEARCH, - ZOOMA, - DTOL, - NEOEXPORT, - SAMPLERELEASE, - NCBIENALINK -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.model; + +public enum PipelineName { + CURATION, + CURAMI, + COPYDOWN, + ENA, + NCBI, + ENAAMR, + EBEYESEARCH, + ZOOMA, + DTOL, + NEOEXPORT, + SAMPLERELEASE, + NCBIENALINK +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/AmrDataLoaderService.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/AmrDataLoaderService.java similarity index 95% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/AmrDataLoaderService.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/AmrDataLoaderService.java index f9b4733aa..415b30f6e 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/AmrDataLoaderService.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/AmrDataLoaderService.java @@ -1,233 +1,233 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import com.fasterxml.jackson.databind.ObjectReader; -import com.fasterxml.jackson.dataformat.csv.CsvMapper; -import com.fasterxml.jackson.dataformat.csv.CsvSchema; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.*; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.AccessionFtpUrlPair; -import uk.ac.ebi.biosamples.model.structured.StructuredDataEntry; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; - -@Service -public class AmrDataLoaderService { - private static final Logger log = LoggerFactory.getLogger(AmrDataLoaderService.class); - private static final String BSD_SAMPLE_PREFIX = "SA"; - private static final String FTP = "ftp"; - private static final String MD_5 = "md5"; - private static final String HTTP = "http://"; - private static final String antibiogram = "\"AMR_ANTIBIOGRAM\""; - private static final String URL = - "https://www.ebi.ac.uk/ena/portal/api/search?result=analysis&query=analysis_type=" - + antibiogram - + "&dataPortal=pathogen&dccDataOnly=false&fields=analysis_accession,country,region,scientific_name,location,sample_accession,tax_id,submitted_ftp,first_public,last_updated&sortFields=scientific_name,country&limit=0"; - private static final String TAB = "\t"; - - private Map> loadAmrData() { - log.info("Loading ENA-AMR data"); - - final Map> sampleToAmrMap = new HashMap<>(); - final List pairList; - - try { - pairList = requestHttpAndGetAccessionFtpUrlPairs(); - - if (pairList.isEmpty()) { - log.info( - "Unable to fetch ENA-AMR Antibiogram data from ENA API, Timed out waiting for connection"); - } else { - downloadFtpContent(pairList, sampleToAmrMap); - } - } catch (final Exception e) { - log.info("An exception occured while processing AMR data " + e.getMessage()); - } - - return sampleToAmrMap; - } - - private static List requestHttpAndGetAccessionFtpUrlPairs() - throws Exception { - final URL enaApiUrl = new URL(AmrDataLoaderService.URL); - final HttpURLConnection conn = (HttpURLConnection) enaApiUrl.openConnection(); - List pairList = new ArrayList<>(); - - try { - if (getResponseFromEnaApi(conn) == 200) { - pairList = doGetAccessionFtpUrlPairs(enaApiUrl); - } - } catch (final Exception e) { - throw new RuntimeException(e); - } finally { - conn.disconnect(); - } - - return pairList; - } - - private static int getResponseFromEnaApi(final HttpURLConnection conn) throws IOException { - final int response; - - conn.setRequestMethod("GET"); - conn.connect(); - response = conn.getResponseCode(); - - return response; - } - - private static List doGetAccessionFtpUrlPairs(final URL url) { - final List accessionFtpUrlPairs = new ArrayList<>(); - - try { - final BufferedReader bufferedReader = getReader(url); - - bufferedReader - .lines() - .skip(1) - .forEach(line -> accessionFtpUrlPairs.add(getAccessionFtpUrlPair(line))); - } catch (final IOException e) { - log.info("Failed to get and parse accession and FTP pairs for URL " + url.toString()); - } - - return accessionFtpUrlPairs; - } - - private static BufferedReader getReader(final URL url) throws IOException { - return new BufferedReader(new InputStreamReader(url.openConnection().getInputStream())); - } - - private static AccessionFtpUrlPair getAccessionFtpUrlPair(final String line) { - final StringTokenizer tokenizer = new StringTokenizer(line, TAB); - final AccessionFtpUrlPair accessionFtpUrlPair = new AccessionFtpUrlPair(); - - while (tokenizer.hasMoreTokens()) { - final String value = tokenizer.nextToken(); - - if (value.startsWith(BSD_SAMPLE_PREFIX)) { - accessionFtpUrlPair.setAccession(value); - } - - if (value.startsWith(FTP)) { - dealWithSemicolon(value, accessionFtpUrlPair); - } - } - - return accessionFtpUrlPair; - } - - private static void dealWithSemicolon( - final String value, final AccessionFtpUrlPair accessionFtpUrlPair) { - accessionFtpUrlPair.setFtpUrl(HTTP + value); - } - - private Map> downloadFtpContent( - final List pairList, - final Map> sampleToAmrMap) { - pairList.forEach( - pair -> { - try { - final String accession = pair.getAccession(); - - if (accession != null) { - sampleToAmrMap.put( - accession, fetchSampleAndProcessAmrData(new URL(pair.getFtpUrl()), accession)); - } - } catch (final MalformedURLException e) { - log.info("FTP URL not correctly formed " + pair.getFtpUrl()); - } - }); - - return sampleToAmrMap; - } - - private Set fetchSampleAndProcessAmrData( - final URL url, final String accession) { - Set amrData = new HashSet<>(); - - try { - amrData = processAmrData(processAmrLines(getReader(url)), accession); - } catch (final IOException ioe) { - log.info("A IO Exception occurrence detected"); - - if (amrData.isEmpty()) { - log.info("Couldn't process AMR data for " + accession); - } - } - - return amrData; - } - - private List processAmrLines(final BufferedReader bufferedReader) { - return bufferedReader - .lines() - .skip(1) - .map(this::removeBioSampleId) - .map(this::dealWithExtraTabs) - .collect(Collectors.toList()); - } - - private Set processAmrData( - final List lines, final String accession) { - final Set> tableContent = new HashSet<>(); - final StructuredDataTable table = - StructuredDataTable.build("self.BiosampleImportENA", null, "AMR", null, tableContent); - - lines.forEach( - line -> { - final CsvMapper mapper = new CsvMapper(); - final CsvSchema schema = mapper.schemaFor(Map.class).withColumnSeparator('\t'); - final ObjectReader r = mapper.readerFor(Map.class).with(schema); - try { - final Map amrEntry = r.readValue(line); - final Map entry = new HashMap<>(); - for (final Map.Entry e : amrEntry.entrySet()) { - entry.put(e.getKey(), StructuredDataEntry.build(e.getValue(), null)); - } - - tableContent.add(entry); - } catch (final Exception e) { - e.printStackTrace(); - - log.error("Error in parsing AMR data for sample " + accession); - } - }); - - return Collections.singleton(table); - } - - private String removeBioSampleId(final String line) { - return line.substring(line.indexOf(AmrDataLoaderService.TAB) + 1); - } - - private String dealWithExtraTabs(String line) { - while (line.endsWith(AmrDataLoaderService.TAB)) { - line = line.substring(0, line.length() - 1); - } - - return line; - } - - public static void main(final String[] args) { - final AmrDataLoaderService amrDataLoaderService = new AmrDataLoaderService(); - - amrDataLoaderService.loadAmrData(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.*; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataEntry; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; +import uk.ac.ebi.biosamples.model.AccessionFtpUrlPair; + +@Service +public class AmrDataLoaderService { + private static final Logger log = LoggerFactory.getLogger(AmrDataLoaderService.class); + private static final String BSD_SAMPLE_PREFIX = "SA"; + private static final String FTP = "ftp"; + private static final String MD_5 = "md5"; + private static final String HTTP = "http://"; + private static final String antibiogram = "\"AMR_ANTIBIOGRAM\""; + private static final String URL = + "https://www.ebi.ac.uk/ena/portal/api/search?result=analysis&query=analysis_type=" + + antibiogram + + "&dataPortal=pathogen&dccDataOnly=false&fields=analysis_accession,country,region,scientific_name,location,sample_accession,tax_id,submitted_ftp,first_public,last_updated&sortFields=scientific_name,country&limit=0"; + private static final String TAB = "\t"; + + private Map> loadAmrData() { + log.info("Loading ENA-AMR data"); + + final Map> sampleToAmrMap = new HashMap<>(); + final List pairList; + + try { + pairList = requestHttpAndGetAccessionFtpUrlPairs(); + + if (pairList.isEmpty()) { + log.info( + "Unable to fetch ENA-AMR Antibiogram data from ENA API, Timed out waiting for connection"); + } else { + downloadFtpContent(pairList, sampleToAmrMap); + } + } catch (final Exception e) { + log.info("An exception occured while processing AMR data " + e.getMessage()); + } + + return sampleToAmrMap; + } + + private static List requestHttpAndGetAccessionFtpUrlPairs() + throws Exception { + final URL enaApiUrl = new URL(AmrDataLoaderService.URL); + final HttpURLConnection conn = (HttpURLConnection) enaApiUrl.openConnection(); + List pairList = new ArrayList<>(); + + try { + if (getResponseFromEnaApi(conn) == 200) { + pairList = doGetAccessionFtpUrlPairs(enaApiUrl); + } + } catch (final Exception e) { + throw new RuntimeException(e); + } finally { + conn.disconnect(); + } + + return pairList; + } + + private static int getResponseFromEnaApi(final HttpURLConnection conn) throws IOException { + final int response; + + conn.setRequestMethod("GET"); + conn.connect(); + response = conn.getResponseCode(); + + return response; + } + + private static List doGetAccessionFtpUrlPairs(final URL url) { + final List accessionFtpUrlPairs = new ArrayList<>(); + + try { + final BufferedReader bufferedReader = getReader(url); + + bufferedReader + .lines() + .skip(1) + .forEach(line -> accessionFtpUrlPairs.add(getAccessionFtpUrlPair(line))); + } catch (final IOException e) { + log.info("Failed to get and parse accession and FTP pairs for URL " + url.toString()); + } + + return accessionFtpUrlPairs; + } + + private static BufferedReader getReader(final URL url) throws IOException { + return new BufferedReader(new InputStreamReader(url.openConnection().getInputStream())); + } + + private static AccessionFtpUrlPair getAccessionFtpUrlPair(final String line) { + final StringTokenizer tokenizer = new StringTokenizer(line, TAB); + final AccessionFtpUrlPair accessionFtpUrlPair = new AccessionFtpUrlPair(); + + while (tokenizer.hasMoreTokens()) { + final String value = tokenizer.nextToken(); + + if (value.startsWith(BSD_SAMPLE_PREFIX)) { + accessionFtpUrlPair.setAccession(value); + } + + if (value.startsWith(FTP)) { + dealWithSemicolon(value, accessionFtpUrlPair); + } + } + + return accessionFtpUrlPair; + } + + private static void dealWithSemicolon( + final String value, final AccessionFtpUrlPair accessionFtpUrlPair) { + accessionFtpUrlPair.setFtpUrl(HTTP + value); + } + + private Map> downloadFtpContent( + final List pairList, + final Map> sampleToAmrMap) { + pairList.forEach( + pair -> { + try { + final String accession = pair.getAccession(); + + if (accession != null) { + sampleToAmrMap.put( + accession, fetchSampleAndProcessAmrData(new URL(pair.getFtpUrl()), accession)); + } + } catch (final MalformedURLException e) { + log.info("FTP URL not correctly formed " + pair.getFtpUrl()); + } + }); + + return sampleToAmrMap; + } + + private Set fetchSampleAndProcessAmrData( + final URL url, final String accession) { + Set amrData = new HashSet<>(); + + try { + amrData = processAmrData(processAmrLines(getReader(url)), accession); + } catch (final IOException ioe) { + log.info("A IO Exception occurrence detected"); + + if (amrData.isEmpty()) { + log.info("Couldn't process AMR data for " + accession); + } + } + + return amrData; + } + + private List processAmrLines(final BufferedReader bufferedReader) { + return bufferedReader + .lines() + .skip(1) + .map(this::removeBioSampleId) + .map(this::dealWithExtraTabs) + .collect(Collectors.toList()); + } + + private Set processAmrData( + final List lines, final String accession) { + final Set> tableContent = new HashSet<>(); + final StructuredDataTable table = + StructuredDataTable.build("self.BiosampleImportENA", null, "AMR", null, tableContent); + + lines.forEach( + line -> { + final CsvMapper mapper = new CsvMapper(); + final CsvSchema schema = mapper.schemaFor(Map.class).withColumnSeparator('\t'); + final ObjectReader r = mapper.readerFor(Map.class).with(schema); + try { + final Map amrEntry = r.readValue(line); + final Map entry = new HashMap<>(); + for (final Map.Entry e : amrEntry.entrySet()) { + entry.put(e.getKey(), StructuredDataEntry.build(e.getValue(), null)); + } + + tableContent.add(entry); + } catch (final Exception e) { + e.printStackTrace(); + + log.error("Error in parsing AMR data for sample " + accession); + } + }); + + return Collections.singleton(table); + } + + private String removeBioSampleId(final String line) { + return line.substring(line.indexOf(AmrDataLoaderService.TAB) + 1); + } + + private String dealWithExtraTabs(String line) { + while (line.endsWith(AmrDataLoaderService.TAB)) { + line = line.substring(0, line.length() - 1); + } + + return line; + } + + public static void main(final String[] args) { + final AmrDataLoaderService amrDataLoaderService = new AmrDataLoaderService(); + + amrDataLoaderService.loadAmrData(); + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/BioSampleConverter.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/BioSampleConverter.java similarity index 96% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/BioSampleConverter.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/BioSampleConverter.java index 601115b0a..b112d45a5 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/BioSampleConverter.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/BioSampleConverter.java @@ -1,372 +1,372 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import static uk.ac.ebi.biosamples.service.EnaSampleToBioSampleConversionConstants.*; - -import java.time.Instant; -import java.util.Collections; -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; -import org.dom4j.Element; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Publication; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.utils.XmlPathBuilder; - -@Service -public class BioSampleConverter { - private static final String ENA_SAMPLE_ACCESSION = "accession"; - - Sample convertEnaSampleXmlToBioSample( - final Element eraSampleXmlRootElement, final String bioSampleAccession) { - final SortedSet bioSampleAttributes = new TreeSet<>(); - final SortedSet bioSamplePublications = new TreeSet<>(); - final XmlPathBuilder eraSampleXmlRootPath = - XmlPathBuilder.of(eraSampleXmlRootElement).path(ENA_SAMPLE_ROOT); - - /*Map name (top-attribute) in BioSamples to - alias (top-attribute) in ENA XML*/ - String bioSampleSampleName = null; - String bioSampleTaxId = null; - - if (eraSampleXmlRootPath.attributeExists(ENA_SAMPLE_ALIAS)) { - bioSampleSampleName = eraSampleXmlRootPath.attribute(ENA_SAMPLE_ALIAS).trim(); - } - - // ENA SRA accession - if (eraSampleXmlRootPath.attributeExists(ENA_SAMPLE_ACCESSION)) { - final String enaSampleSraAccession = - eraSampleXmlRootPath.attribute(ENA_SAMPLE_ACCESSION).trim(); - bioSampleAttributes.add(Attribute.build(ENA_SAMPLE_SRA_ACCESSION, enaSampleSraAccession)); - - /*if and only if alias is not present, then sample name in BioSamples would be equal to - ENA sample accession*/ - if (bioSampleSampleName == null) { - bioSampleSampleName = enaSampleSraAccession; - } - } - - // ENA broker name - if (eraSampleXmlRootPath.attributeExists("broker_name")) { - bioSampleAttributes.add( - Attribute.build( - ENA_SAMPLE_BROKER_NAME, eraSampleXmlRootPath.attribute("broker_name").trim())); - } - - // ENA center name - if (eraSampleXmlRootPath.attributeExists("center_name")) { - bioSampleAttributes.add( - Attribute.build( - BIOSAMPLE_INSDC_CENTER_NAME_ATTRIBUTE_NAME, - eraSampleXmlRootPath.attribute("center_name").trim())); - } - - // ENA center alias - if (eraSampleXmlRootPath.attributeExists("center_alias")) { - bioSampleAttributes.add( - Attribute.build( - BIOSAMPLE_INSDC_CENTER_ALIAS_ATTRIBUTE_NAME, - eraSampleXmlRootPath.attribute("center_alias").trim())); - } - - // ENA title - final XmlPathBuilder titlePathBuilder = - XmlPathBuilder.of(eraSampleXmlRootElement).path(ENA_SAMPLE_ROOT, ENA_SAMPLE_TITLE); - - if (titlePathBuilder.exists()) { - bioSampleAttributes.add( - Attribute.build(BIOSAMPLE_SAMPLE_TITLE_ATTRIBUTE_NAME, titlePathBuilder.text().trim())); - } - - // ENA description - final XmlPathBuilder descriptionPathBuilder = - XmlPathBuilder.of(eraSampleXmlRootElement).path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DESCRIPTION); - - if (descriptionPathBuilder.exists()) { - bioSampleAttributes.add( - Attribute.build( - BIOSAMPLE_SAMPLE_DESCRIPTION_ATTRIBUTE_NAME, descriptionPathBuilder.text().trim())); - } - - // ENA SUBMITTER_ID - final XmlPathBuilder submitterIdPathBuilder = - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS, ENA_SAMPLE_SUBMITTER_ID); - String namespaceOfSubmitterId = null; - - if (submitterIdPathBuilder != null && submitterIdPathBuilder.exists()) { - if (submitterIdPathBuilder.attributeExists(ENA_IDENTIFIERS_NAMESPACE_TAG)) { - namespaceOfSubmitterId = submitterIdPathBuilder.attribute(ENA_IDENTIFIERS_NAMESPACE_TAG); - } - - bioSampleAttributes.add( - Attribute.build( - BIOSAMPLE_SUBMITTER_ID_ATTRIBUTE_NAME, - submitterIdPathBuilder.text(), - namespaceOfSubmitterId != null - ? BIOSAMPLE_ATTRIBUTE_NAMESPACE_TAG + namespaceOfSubmitterId - : null, - Collections.emptyList(), - null)); - } - - // ENA EXTERNAL ID - final XmlPathBuilder externalIdPathBuilder = - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS, ENA_SAMPLE_EXTERNAL_ID); - final List externalIdPathBuilderElements = - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS) - .elements(ENA_SAMPLE_EXTERNAL_ID); - - for (final Element element : externalIdPathBuilderElements) { - String namespaceOfExternalId = null; - final XmlPathBuilder singleExternalIdPathBuilder = XmlPathBuilder.of(element); - - if (singleExternalIdPathBuilder.exists()) { - if (singleExternalIdPathBuilder.attributeExists(ENA_IDENTIFIERS_NAMESPACE_TAG)) { - namespaceOfExternalId = - singleExternalIdPathBuilder.attribute(ENA_IDENTIFIERS_NAMESPACE_TAG); - } - - bioSampleAttributes.add( - Attribute.build( - BIOSAMPLE_EXTERNAL_ID_ATTRIBUTE_NAME, - singleExternalIdPathBuilder.text(), - namespaceOfExternalId != null - ? BIOSAMPLE_ATTRIBUTE_NAMESPACE_TAG - + externalIdPathBuilder.attribute(ENA_IDENTIFIERS_NAMESPACE_TAG) - : null, - Collections.emptyList(), - null)); - } - } - - // ENA SECONDARY ID - final XmlPathBuilder secondaryIdPathBuilder = - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS, ENA_SAMPLE_SECONDARY_ID); - - if (secondaryIdPathBuilder.exists()) { - final List secondaryIdPathBuilderElements = - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS) - .elements(ENA_SAMPLE_SECONDARY_ID); - - for (final Element element : secondaryIdPathBuilderElements) { - final XmlPathBuilder singleSecondaryIdPathBuilder = XmlPathBuilder.of(element); - - if (singleSecondaryIdPathBuilder.exists()) { - String namespaceOfExternalId = null; - - if (singleSecondaryIdPathBuilder.attributeExists(ENA_IDENTIFIERS_NAMESPACE_TAG)) { - namespaceOfExternalId = - singleSecondaryIdPathBuilder.attribute(ENA_IDENTIFIERS_NAMESPACE_TAG); - } - - bioSampleAttributes.add( - Attribute.build( - BIOSAMPLE_SECONDARY_ID_ATTRIBUTE_NAME, - singleSecondaryIdPathBuilder.text(), - namespaceOfExternalId != null - ? BIOSAMPLE_ATTRIBUTE_NAMESPACE_TAG - + secondaryIdPathBuilder.attribute(ENA_IDENTIFIERS_NAMESPACE_TAG) - : null, - Collections.emptyList(), - null)); - } - } - } - - // ENA UUID - if (XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS, ENA_SAMPLE_UUID) - .exists()) { - for (final Element element : - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS) - .elements(ENA_SAMPLE_UUID)) { - bioSampleAttributes.add( - Attribute.build(BIOSAMPLE_UUID_ATTRIBUTE_NAME, element.getTextTrim())); - } - } - - // ENA UUID - extra check - if (XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS, ENA_SAMPLE_UUID) - .exists()) { - for (final Element element : - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS) - .elements(ENA_SAMPLE_UUID)) { - bioSampleAttributes.add( - Attribute.build(BIOSAMPLE_UUID_ATTRIBUTE_NAME, element.getTextTrim())); - } - } - - // ENA ANONYMIZED_NAME - final XmlPathBuilder anonymizedNamePathBuilder = - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_SAMPLE_NAME, ENA_SAMPLE_ANONYMIZED_NAME); - - if (anonymizedNamePathBuilder.exists()) { - bioSampleAttributes.add( - Attribute.build( - BIOSAMPLE_ANONYMIZED_NAME_ATTRIBUTE_NAME, anonymizedNamePathBuilder.text())); - } - - // ENA INDIVIDUAL_NAME - final XmlPathBuilder individualNamePathBuilder = - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_SAMPLE_NAME, ENA_SAMPLE_INDIVIDUAL_NAME); - - if (individualNamePathBuilder.exists()) { - bioSampleAttributes.add( - Attribute.build( - BIOSAMPLE_INDIVIDUAL_NAME_ATTRIBUTE_NAME, individualNamePathBuilder.text())); - } - - // Handle organism attribute - final XmlPathBuilder scientificNamePathBuilder = - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_SAMPLE_NAME, ENA_SAMPLE_SCIENTIFIC_NAME); - - if (scientificNamePathBuilder.exists()) { - bioSampleAttributes.add( - Attribute.build( - BIOSAMPLE_ORGANISM_ATTRIBUTE_NAME, scientificNamePathBuilder.text(), null, null)); - bioSampleAttributes.add( - Attribute.build( - BIOSAMPLE_SCIENTIFIC_NAME_ATTRIBUTE_NAME, - scientificNamePathBuilder.text(), - null, - null)); - } - - // Handle TAXON_ID - final XmlPathBuilder taxonIdPathBuilder = - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_SAMPLE_NAME, "TAXON_ID"); - - if (taxonIdPathBuilder.exists()) { - bioSampleTaxId = taxonIdPathBuilder.text(); - } - - // ENA sample common name - final XmlPathBuilder commonNamePathBuilder = - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_SAMPLE_NAME, ENA_SAMPLE_COMMON_NAME); - - if (commonNamePathBuilder.exists()) { - bioSampleAttributes.add( - Attribute.build(BIOSAMPLE_COMMON_NAME_ATTRIBUTE_NAME, commonNamePathBuilder.text())); - } - - final XmlPathBuilder sampleAttributesPathBuilder = - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_SAMPLE_ATTRIBUTES); - - if (sampleAttributesPathBuilder.exists()) { - for (final Element attributeElement : - sampleAttributesPathBuilder.elements(ENA_SAMPLE_SAMPLE_ATTRIBUTE)) { - String tag = null; - final XmlPathBuilder tagPathBuilder = XmlPathBuilder.of(attributeElement).path("TAG"); - - if (tagPathBuilder.exists() && !tagPathBuilder.text().trim().isEmpty()) { - tag = tagPathBuilder.text().trim(); - } - - String value = null; - final XmlPathBuilder valuePathBuilder = XmlPathBuilder.of(attributeElement).path("VALUE"); - - if (valuePathBuilder.exists() && !valuePathBuilder.text().trim().isEmpty()) { - value = valuePathBuilder.text().trim(); - } - - String unit = null; - final XmlPathBuilder unitPathBuilder = XmlPathBuilder.of(attributeElement).path("UNITS"); - - if (unitPathBuilder.exists() && !unitPathBuilder.text().trim().isEmpty()) { - unit = unitPathBuilder.text().trim(); - } - - // Deal with multiple descriptions in ENA XML, one top level description and one in sample - // attributes - if (tag != null && tag.equalsIgnoreCase(BIOSAMPLE_SAMPLE_DESCRIPTION_ATTRIBUTE_NAME)) { - bioSampleAttributes.add( - Attribute.build( - tag, value, BIOSAMPLE_SAMPLE_ATTRIBUTE_TAG, Collections.emptyList(), null)); - continue; - } - - // Deal with multiple titles in ENA XML, one top level title and one in sample attributes - if (tag != null && tag.equalsIgnoreCase(BIOSAMPLE_SAMPLE_TITLE_ATTRIBUTE_NAME)) { - bioSampleAttributes.add( - Attribute.build( - tag, value, BIOSAMPLE_SAMPLE_ATTRIBUTE_TAG, Collections.emptyList(), null)); - continue; - } - - if (tag != null) { - bioSampleAttributes.add( - Attribute.build( - tag, value, BIOSAMPLE_SAMPLE_ATTRIBUTE_TAG, Collections.emptyList(), unit)); - } - - if (tag != null && tag.equalsIgnoreCase(ENA_SAMPLE_PUBMED_ID)) { - bioSamplePublications.add(new Publication.Builder().pubmed_id(value).build()); - } - } - } - - if (XmlPathBuilder.of(eraSampleXmlRootElement).path(ENA_SAMPLE_ROOT, "SAMPLE_LINKS").exists()) { - if (XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, "SAMPLE_LINKS", "URI_LINK") - .exists()) { - for (final Element e : - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, "SAMPLE_LINKS") - .elements("URI_LINK")) { - bioSampleAttributes.add( - Attribute.build( - XmlPathBuilder.of(e).attribute("LABEL"), XmlPathBuilder.of(e).attribute("URL"))); - } - } - - if (XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, "SAMPLE_LINKS", "XREF_LINK") - .exists()) { - for (final Element e : - XmlPathBuilder.of(eraSampleXmlRootElement) - .path(ENA_SAMPLE_ROOT, "SAMPLE_LINKS") - .elements("XREF_LINK")) { - final String key = XmlPathBuilder.of(e).attribute("DB"); - - if (key != null && key.equalsIgnoreCase(ENA_SAMPLE_PUBMED_ID)) { - bioSamplePublications.add( - new Publication.Builder().pubmed_id(XmlPathBuilder.of(e).attribute("ID")).build()); - } - } - } - } - - return new Sample.Builder(bioSampleSampleName, bioSampleAccession) - .withTaxId(bioSampleTaxId != null ? Long.valueOf(bioSampleTaxId) : null) - .withRelease(Instant.now()) - .withUpdate(Instant.now()) - .withAttributes(bioSampleAttributes) - .withPublications(bioSamplePublications) - .build(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import static uk.ac.ebi.biosamples.service.EnaSampleToBioSampleConversionConstants.*; + +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; +import org.dom4j.Element; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Publication; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.utils.XmlPathBuilder; + +@Service +public class BioSampleConverter { + private static final String ENA_SAMPLE_ACCESSION = "accession"; + + Sample convertEnaSampleXmlToBioSample( + final Element eraSampleXmlRootElement, final String bioSampleAccession) { + final SortedSet bioSampleAttributes = new TreeSet<>(); + final SortedSet bioSamplePublications = new TreeSet<>(); + final XmlPathBuilder eraSampleXmlRootPath = + XmlPathBuilder.of(eraSampleXmlRootElement).path(ENA_SAMPLE_ROOT); + + /*Map name (top-attribute) in BioSamples to + alias (top-attribute) in ENA XML*/ + String bioSampleSampleName = null; + String bioSampleTaxId = null; + + if (eraSampleXmlRootPath.attributeExists(ENA_SAMPLE_ALIAS)) { + bioSampleSampleName = eraSampleXmlRootPath.attribute(ENA_SAMPLE_ALIAS).trim(); + } + + // ENA SRA accession + if (eraSampleXmlRootPath.attributeExists(ENA_SAMPLE_ACCESSION)) { + final String enaSampleSraAccession = + eraSampleXmlRootPath.attribute(ENA_SAMPLE_ACCESSION).trim(); + bioSampleAttributes.add(Attribute.build(ENA_SAMPLE_SRA_ACCESSION, enaSampleSraAccession)); + + /*if and only if alias is not present, then sample name in BioSamples would be equal to + ENA sample accession*/ + if (bioSampleSampleName == null) { + bioSampleSampleName = enaSampleSraAccession; + } + } + + // ENA broker name + if (eraSampleXmlRootPath.attributeExists("broker_name")) { + bioSampleAttributes.add( + Attribute.build( + ENA_SAMPLE_BROKER_NAME, eraSampleXmlRootPath.attribute("broker_name").trim())); + } + + // ENA center name + if (eraSampleXmlRootPath.attributeExists("center_name")) { + bioSampleAttributes.add( + Attribute.build( + BIOSAMPLE_INSDC_CENTER_NAME_ATTRIBUTE_NAME, + eraSampleXmlRootPath.attribute("center_name").trim())); + } + + // ENA center alias + if (eraSampleXmlRootPath.attributeExists("center_alias")) { + bioSampleAttributes.add( + Attribute.build( + BIOSAMPLE_INSDC_CENTER_ALIAS_ATTRIBUTE_NAME, + eraSampleXmlRootPath.attribute("center_alias").trim())); + } + + // ENA title + final XmlPathBuilder titlePathBuilder = + XmlPathBuilder.of(eraSampleXmlRootElement).path(ENA_SAMPLE_ROOT, ENA_SAMPLE_TITLE); + + if (titlePathBuilder.exists()) { + bioSampleAttributes.add( + Attribute.build(BIOSAMPLE_SAMPLE_TITLE_ATTRIBUTE_NAME, titlePathBuilder.text().trim())); + } + + // ENA description + final XmlPathBuilder descriptionPathBuilder = + XmlPathBuilder.of(eraSampleXmlRootElement).path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DESCRIPTION); + + if (descriptionPathBuilder.exists()) { + bioSampleAttributes.add( + Attribute.build( + BIOSAMPLE_SAMPLE_DESCRIPTION_ATTRIBUTE_NAME, descriptionPathBuilder.text().trim())); + } + + // ENA SUBMITTER_ID + final XmlPathBuilder submitterIdPathBuilder = + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS, ENA_SAMPLE_SUBMITTER_ID); + String namespaceOfSubmitterId = null; + + if (submitterIdPathBuilder != null && submitterIdPathBuilder.exists()) { + if (submitterIdPathBuilder.attributeExists(ENA_IDENTIFIERS_NAMESPACE_TAG)) { + namespaceOfSubmitterId = submitterIdPathBuilder.attribute(ENA_IDENTIFIERS_NAMESPACE_TAG); + } + + bioSampleAttributes.add( + Attribute.build( + BIOSAMPLE_SUBMITTER_ID_ATTRIBUTE_NAME, + submitterIdPathBuilder.text(), + namespaceOfSubmitterId != null + ? BIOSAMPLE_ATTRIBUTE_NAMESPACE_TAG + namespaceOfSubmitterId + : null, + Collections.emptyList(), + null)); + } + + // ENA EXTERNAL ID + final XmlPathBuilder externalIdPathBuilder = + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS, ENA_SAMPLE_EXTERNAL_ID); + final List externalIdPathBuilderElements = + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS) + .elements(ENA_SAMPLE_EXTERNAL_ID); + + for (final Element element : externalIdPathBuilderElements) { + String namespaceOfExternalId = null; + final XmlPathBuilder singleExternalIdPathBuilder = XmlPathBuilder.of(element); + + if (singleExternalIdPathBuilder.exists()) { + if (singleExternalIdPathBuilder.attributeExists(ENA_IDENTIFIERS_NAMESPACE_TAG)) { + namespaceOfExternalId = + singleExternalIdPathBuilder.attribute(ENA_IDENTIFIERS_NAMESPACE_TAG); + } + + bioSampleAttributes.add( + Attribute.build( + BIOSAMPLE_EXTERNAL_ID_ATTRIBUTE_NAME, + singleExternalIdPathBuilder.text(), + namespaceOfExternalId != null + ? BIOSAMPLE_ATTRIBUTE_NAMESPACE_TAG + + externalIdPathBuilder.attribute(ENA_IDENTIFIERS_NAMESPACE_TAG) + : null, + Collections.emptyList(), + null)); + } + } + + // ENA SECONDARY ID + final XmlPathBuilder secondaryIdPathBuilder = + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS, ENA_SAMPLE_SECONDARY_ID); + + if (secondaryIdPathBuilder.exists()) { + final List secondaryIdPathBuilderElements = + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS) + .elements(ENA_SAMPLE_SECONDARY_ID); + + for (final Element element : secondaryIdPathBuilderElements) { + final XmlPathBuilder singleSecondaryIdPathBuilder = XmlPathBuilder.of(element); + + if (singleSecondaryIdPathBuilder.exists()) { + String namespaceOfExternalId = null; + + if (singleSecondaryIdPathBuilder.attributeExists(ENA_IDENTIFIERS_NAMESPACE_TAG)) { + namespaceOfExternalId = + singleSecondaryIdPathBuilder.attribute(ENA_IDENTIFIERS_NAMESPACE_TAG); + } + + bioSampleAttributes.add( + Attribute.build( + BIOSAMPLE_SECONDARY_ID_ATTRIBUTE_NAME, + singleSecondaryIdPathBuilder.text(), + namespaceOfExternalId != null + ? BIOSAMPLE_ATTRIBUTE_NAMESPACE_TAG + + secondaryIdPathBuilder.attribute(ENA_IDENTIFIERS_NAMESPACE_TAG) + : null, + Collections.emptyList(), + null)); + } + } + } + + // ENA UUID + if (XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS, ENA_SAMPLE_UUID) + .exists()) { + for (final Element element : + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS) + .elements(ENA_SAMPLE_UUID)) { + bioSampleAttributes.add( + Attribute.build(BIOSAMPLE_UUID_ATTRIBUTE_NAME, element.getTextTrim())); + } + } + + // ENA UUID - extra check + if (XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS, ENA_SAMPLE_UUID) + .exists()) { + for (final Element element : + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_DENTIFIERS) + .elements(ENA_SAMPLE_UUID)) { + bioSampleAttributes.add( + Attribute.build(BIOSAMPLE_UUID_ATTRIBUTE_NAME, element.getTextTrim())); + } + } + + // ENA ANONYMIZED_NAME + final XmlPathBuilder anonymizedNamePathBuilder = + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_SAMPLE_NAME, ENA_SAMPLE_ANONYMIZED_NAME); + + if (anonymizedNamePathBuilder.exists()) { + bioSampleAttributes.add( + Attribute.build( + BIOSAMPLE_ANONYMIZED_NAME_ATTRIBUTE_NAME, anonymizedNamePathBuilder.text())); + } + + // ENA INDIVIDUAL_NAME + final XmlPathBuilder individualNamePathBuilder = + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_SAMPLE_NAME, ENA_SAMPLE_INDIVIDUAL_NAME); + + if (individualNamePathBuilder.exists()) { + bioSampleAttributes.add( + Attribute.build( + BIOSAMPLE_INDIVIDUAL_NAME_ATTRIBUTE_NAME, individualNamePathBuilder.text())); + } + + // Handle organism attribute + final XmlPathBuilder scientificNamePathBuilder = + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_SAMPLE_NAME, ENA_SAMPLE_SCIENTIFIC_NAME); + + if (scientificNamePathBuilder.exists()) { + bioSampleAttributes.add( + Attribute.build( + BIOSAMPLE_ORGANISM_ATTRIBUTE_NAME, scientificNamePathBuilder.text(), null, null)); + bioSampleAttributes.add( + Attribute.build( + BIOSAMPLE_SCIENTIFIC_NAME_ATTRIBUTE_NAME, + scientificNamePathBuilder.text(), + null, + null)); + } + + // Handle TAXON_ID + final XmlPathBuilder taxonIdPathBuilder = + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_SAMPLE_NAME, "TAXON_ID"); + + if (taxonIdPathBuilder.exists()) { + bioSampleTaxId = taxonIdPathBuilder.text(); + } + + // ENA sample common name + final XmlPathBuilder commonNamePathBuilder = + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_SAMPLE_NAME, ENA_SAMPLE_COMMON_NAME); + + if (commonNamePathBuilder.exists()) { + bioSampleAttributes.add( + Attribute.build(BIOSAMPLE_COMMON_NAME_ATTRIBUTE_NAME, commonNamePathBuilder.text())); + } + + final XmlPathBuilder sampleAttributesPathBuilder = + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, ENA_SAMPLE_SAMPLE_ATTRIBUTES); + + if (sampleAttributesPathBuilder.exists()) { + for (final Element attributeElement : + sampleAttributesPathBuilder.elements(ENA_SAMPLE_SAMPLE_ATTRIBUTE)) { + String tag = null; + final XmlPathBuilder tagPathBuilder = XmlPathBuilder.of(attributeElement).path("TAG"); + + if (tagPathBuilder.exists() && !tagPathBuilder.text().trim().isEmpty()) { + tag = tagPathBuilder.text().trim(); + } + + String value = null; + final XmlPathBuilder valuePathBuilder = XmlPathBuilder.of(attributeElement).path("VALUE"); + + if (valuePathBuilder.exists() && !valuePathBuilder.text().trim().isEmpty()) { + value = valuePathBuilder.text().trim(); + } + + String unit = null; + final XmlPathBuilder unitPathBuilder = XmlPathBuilder.of(attributeElement).path("UNITS"); + + if (unitPathBuilder.exists() && !unitPathBuilder.text().trim().isEmpty()) { + unit = unitPathBuilder.text().trim(); + } + + // Deal with multiple descriptions in ENA XML, one top level description and one in sample + // attributes + if (tag != null && tag.equalsIgnoreCase(BIOSAMPLE_SAMPLE_DESCRIPTION_ATTRIBUTE_NAME)) { + bioSampleAttributes.add( + Attribute.build( + tag, value, BIOSAMPLE_SAMPLE_ATTRIBUTE_TAG, Collections.emptyList(), null)); + continue; + } + + // Deal with multiple titles in ENA XML, one top level title and one in sample attributes + if (tag != null && tag.equalsIgnoreCase(BIOSAMPLE_SAMPLE_TITLE_ATTRIBUTE_NAME)) { + bioSampleAttributes.add( + Attribute.build( + tag, value, BIOSAMPLE_SAMPLE_ATTRIBUTE_TAG, Collections.emptyList(), null)); + continue; + } + + if (tag != null) { + bioSampleAttributes.add( + Attribute.build( + tag, value, BIOSAMPLE_SAMPLE_ATTRIBUTE_TAG, Collections.emptyList(), unit)); + } + + if (tag != null && tag.equalsIgnoreCase(ENA_SAMPLE_PUBMED_ID)) { + bioSamplePublications.add(new Publication.Builder().pubmed_id(value).build()); + } + } + } + + if (XmlPathBuilder.of(eraSampleXmlRootElement).path(ENA_SAMPLE_ROOT, "SAMPLE_LINKS").exists()) { + if (XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, "SAMPLE_LINKS", "URI_LINK") + .exists()) { + for (final Element e : + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, "SAMPLE_LINKS") + .elements("URI_LINK")) { + bioSampleAttributes.add( + Attribute.build( + XmlPathBuilder.of(e).attribute("LABEL"), XmlPathBuilder.of(e).attribute("URL"))); + } + } + + if (XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, "SAMPLE_LINKS", "XREF_LINK") + .exists()) { + for (final Element e : + XmlPathBuilder.of(eraSampleXmlRootElement) + .path(ENA_SAMPLE_ROOT, "SAMPLE_LINKS") + .elements("XREF_LINK")) { + final String key = XmlPathBuilder.of(e).attribute("DB"); + + if (key != null && key.equalsIgnoreCase(ENA_SAMPLE_PUBMED_ID)) { + bioSamplePublications.add( + new Publication.Builder().pubmed_id(XmlPathBuilder.of(e).attribute("ID")).build()); + } + } + } + } + + return new Sample.Builder(bioSampleSampleName, bioSampleAccession) + .withTaxId(bioSampleTaxId != null ? Long.valueOf(bioSampleTaxId) : null) + .withRelease(Instant.now()) + .withUpdate(Instant.now()) + .withAttributes(bioSampleAttributes) + .withPublications(bioSamplePublications) + .build(); + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaConfig.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaConfig.java similarity index 97% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaConfig.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaConfig.java index 23250c1b8..22ab71549 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaConfig.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaConfig.java @@ -1,37 +1,37 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import javax.sql.DataSource; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.jdbc.DataSourceBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.jdbc.core.JdbcTemplate; - -@Configuration -public class EnaConfig { - @Bean("eraDataSource") - @ConfigurationProperties(prefix = "spring.datasource.hikari") - public DataSource getEraDataSource() { - return DataSourceBuilder.create().build(); - } - - @Bean("eraJdbcTemplate") - public JdbcTemplate getEraJdbcTemplate( - @Qualifier("eraDataSource") final DataSource eraDataSource) { - final JdbcTemplate jdbc = new JdbcTemplate(eraDataSource); - // oracle driver defaults to 10 - jdbc.setFetchSize(1000); - return jdbc; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import javax.sql.DataSource; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; + +@Configuration +public class EnaConfig { + @Bean("eraDataSource") + @ConfigurationProperties(prefix = "spring.datasource.hikari") + public DataSource getEraDataSource() { + return DataSourceBuilder.create().build(); + } + + @Bean("eraJdbcTemplate") + public JdbcTemplate getEraJdbcTemplate( + @Qualifier("eraDataSource") final DataSource eraDataSource) { + final JdbcTemplate jdbc = new JdbcTemplate(eraDataSource); + // oracle driver defaults to 10 + jdbc.setFetchSize(1000); + return jdbc; + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionConstants.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionConstants.java similarity index 98% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionConstants.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionConstants.java index b97a83fb7..5de132482 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionConstants.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionConstants.java @@ -1,51 +1,51 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -public class EnaSampleToBioSampleConversionConstants { - public static final String ENA_SAMPLE_SECONDARY_ID = "SECONDARY_ID"; - public static final String ENA_SAMPLE_COMMON_NAME = "COMMON_NAME"; - public static final String ENA_IDENTIFIERS_NAMESPACE_TAG = "namespace"; - public static final String ENA_SAMPLE_ALIAS = "alias"; - public static final String ENA_SAMPLE_SRA_ACCESSION = "SRA accession"; - public static final String ENA_SAMPLE_BROKER_NAME = "broker name"; - public static final String ENA_SAMPLE_TITLE = "TITLE"; - public static final String ENA_SAMPLE_DESCRIPTION = "DESCRIPTION"; - public static final String ENA_SAMPLE_ROOT = "SAMPLE"; - public static final String ENA_SAMPLE_DENTIFIERS = "IDENTIFIERS"; - public static final String ENA_SAMPLE_SUBMITTER_ID = "SUBMITTER_ID"; - public static final String ENA_SAMPLE_EXTERNAL_ID = "EXTERNAL_ID"; - public static final String ENA_SAMPLE_UUID = "UUID"; - public static final String ENA_SAMPLE_SAMPLE_NAME = "SAMPLE_NAME"; - public static final String ENA_SAMPLE_ANONYMIZED_NAME = "ANONYMIZED_NAME"; - public static final String ENA_SAMPLE_INDIVIDUAL_NAME = "INDIVIDUAL_NAME"; - public static final String ENA_SAMPLE_SCIENTIFIC_NAME = "SCIENTIFIC_NAME"; - public static final String ENA_SAMPLE_SAMPLE_ATTRIBUTE = "SAMPLE_ATTRIBUTE"; - public static final String ENA_SAMPLE_SAMPLE_ATTRIBUTES = "SAMPLE_ATTRIBUTES"; - public static final String ENA_SAMPLE_PUBMED_ID = "pubmed_id"; - public static final String BIOSAMPLE_SCIENTIFIC_NAME_ATTRIBUTE_NAME = "scientific_name"; - public static final String BIOSAMPLE_COMMON_NAME_ATTRIBUTE_NAME = "common name"; - public static final String BIOSAMPLE_SAMPLE_DESCRIPTION_ATTRIBUTE_NAME = "description"; - public static final String BIOSAMPLE_INSDC_CENTER_NAME_ATTRIBUTE_NAME = "INSDC center name"; - public static final String BIOSAMPLE_INSDC_CENTER_ALIAS_ATTRIBUTE_NAME = "INSDC center alias"; - public static final String BIOSAMPLE_SAMPLE_TITLE_ATTRIBUTE_NAME = "title"; - public static final String BIOSAMPLE_ATTRIBUTE_NAMESPACE_TAG = "Namespace:"; - public static final String BIOSAMPLE_SUBMITTER_ID_ATTRIBUTE_NAME = "Submitter Id"; - public static final String BIOSAMPLE_EXTERNAL_ID_ATTRIBUTE_NAME = "External Id"; - public static final String BIOSAMPLE_SECONDARY_ID_ATTRIBUTE_NAME = "Secondary Id"; - public static final String BIOSAMPLE_ORGANISM_ATTRIBUTE_NAME = "organism"; - public static final String BIOSAMPLE_SAMPLE_ATTRIBUTE_TAG = "attribute"; - public static final String BIOSAMPLE_UUID_ATTRIBUTE_NAME = "uuid"; - public static final String BIOSAMPLE_INDIVIDUAL_NAME_ATTRIBUTE_NAME = "individual_name"; - public static final String BIOSAMPLE_ANONYMIZED_NAME_ATTRIBUTE_NAME = "anonymized_name"; - - public static final String ENA_SPECIFIC_ATTRIBUTES_PREFIX = "ENA-"; -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +public class EnaSampleToBioSampleConversionConstants { + public static final String ENA_SAMPLE_SECONDARY_ID = "SECONDARY_ID"; + public static final String ENA_SAMPLE_COMMON_NAME = "COMMON_NAME"; + public static final String ENA_IDENTIFIERS_NAMESPACE_TAG = "namespace"; + public static final String ENA_SAMPLE_ALIAS = "alias"; + public static final String ENA_SAMPLE_SRA_ACCESSION = "SRA accession"; + public static final String ENA_SAMPLE_BROKER_NAME = "broker name"; + public static final String ENA_SAMPLE_TITLE = "TITLE"; + public static final String ENA_SAMPLE_DESCRIPTION = "DESCRIPTION"; + public static final String ENA_SAMPLE_ROOT = "SAMPLE"; + public static final String ENA_SAMPLE_DENTIFIERS = "IDENTIFIERS"; + public static final String ENA_SAMPLE_SUBMITTER_ID = "SUBMITTER_ID"; + public static final String ENA_SAMPLE_EXTERNAL_ID = "EXTERNAL_ID"; + public static final String ENA_SAMPLE_UUID = "UUID"; + public static final String ENA_SAMPLE_SAMPLE_NAME = "SAMPLE_NAME"; + public static final String ENA_SAMPLE_ANONYMIZED_NAME = "ANONYMIZED_NAME"; + public static final String ENA_SAMPLE_INDIVIDUAL_NAME = "INDIVIDUAL_NAME"; + public static final String ENA_SAMPLE_SCIENTIFIC_NAME = "SCIENTIFIC_NAME"; + public static final String ENA_SAMPLE_SAMPLE_ATTRIBUTE = "SAMPLE_ATTRIBUTE"; + public static final String ENA_SAMPLE_SAMPLE_ATTRIBUTES = "SAMPLE_ATTRIBUTES"; + public static final String ENA_SAMPLE_PUBMED_ID = "pubmed_id"; + public static final String BIOSAMPLE_SCIENTIFIC_NAME_ATTRIBUTE_NAME = "scientific_name"; + public static final String BIOSAMPLE_COMMON_NAME_ATTRIBUTE_NAME = "common name"; + public static final String BIOSAMPLE_SAMPLE_DESCRIPTION_ATTRIBUTE_NAME = "description"; + public static final String BIOSAMPLE_INSDC_CENTER_NAME_ATTRIBUTE_NAME = "INSDC center name"; + public static final String BIOSAMPLE_INSDC_CENTER_ALIAS_ATTRIBUTE_NAME = "INSDC center alias"; + public static final String BIOSAMPLE_SAMPLE_TITLE_ATTRIBUTE_NAME = "title"; + public static final String BIOSAMPLE_ATTRIBUTE_NAMESPACE_TAG = "Namespace:"; + public static final String BIOSAMPLE_SUBMITTER_ID_ATTRIBUTE_NAME = "Submitter Id"; + public static final String BIOSAMPLE_EXTERNAL_ID_ATTRIBUTE_NAME = "External Id"; + public static final String BIOSAMPLE_SECONDARY_ID_ATTRIBUTE_NAME = "Secondary Id"; + public static final String BIOSAMPLE_ORGANISM_ATTRIBUTE_NAME = "organism"; + public static final String BIOSAMPLE_SAMPLE_ATTRIBUTE_TAG = "attribute"; + public static final String BIOSAMPLE_UUID_ATTRIBUTE_NAME = "uuid"; + public static final String BIOSAMPLE_INDIVIDUAL_NAME_ATTRIBUTE_NAME = "individual_name"; + public static final String BIOSAMPLE_ANONYMIZED_NAME_ATTRIBUTE_NAME = "anonymized_name"; + + public static final String ENA_SPECIFIC_ATTRIBUTES_PREFIX = "ENA-"; +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionService.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionService.java similarity index 92% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionService.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionService.java index 98116cd67..472f26165 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionService.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleToBioSampleConversionService.java @@ -1,189 +1,196 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.io.StringReader; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.Collections; -import java.util.SortedSet; -import java.util.TreeSet; -import org.dom4j.Document; -import org.dom4j.DocumentException; -import org.dom4j.Element; -import org.dom4j.io.SAXReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.PipelinesProperties; -import uk.ac.ebi.biosamples.model.*; -import uk.ac.ebi.biosamples.utils.XmlPathBuilder; - -@Service -public class EnaSampleToBioSampleConversionService { - private final Logger log = LoggerFactory.getLogger(getClass()); - private final EnaSampleXmlEnhancer enaSampleXmlEnhancer; - private final EraProDao eraProDao; - private final BioSampleConverter enaSampleToBioSampleConverter; - private final PipelinesProperties pipelinesProperties; - - public EnaSampleToBioSampleConversionService( - final EnaSampleXmlEnhancer enaSampleXmlEnhancer, - final EraProDao eraProDao, - final BioSampleConverter enaSampleToBioSampleConverter, - final PipelinesProperties pipelinesProperties) { - this.enaSampleXmlEnhancer = enaSampleXmlEnhancer; - this.eraProDao = eraProDao; - this.enaSampleToBioSampleConverter = enaSampleToBioSampleConverter; - this.pipelinesProperties = pipelinesProperties; - } - - /** Handles one ENA/ NCBI sample */ - public Sample enrichSample(final String accession) throws DocumentException { - final EraproSample eraproSample = eraProDao.getSampleDetailsByBioSampleId(accession); - - if (eraproSample != null) { - final String xmlString = eraproSample.getSampleXml(); - final SAXReader reader = new SAXReader(); - final Document xml = reader.read(new StringReader(xmlString)); - final Element enaSampleRootElement = - enaSampleXmlEnhancer.applyAllRules(xml.getRootElement(), eraproSample); - - // check that we got some content - if (XmlPathBuilder.of(enaSampleRootElement).path("SAMPLE").exists()) { - return enrichSample(eraproSample, enaSampleRootElement, accession); - } else { - log.warn("Unable to find SAMPLE element for " + accession); - } - } - - return null; - } - - /** Handles one ENA/ NCBI sample */ - public Sample enrichSample(final String accession, final EraproSample eraproSample) - throws DocumentException { - if (eraproSample != null) { - final String xmlString = eraproSample.getSampleXml(); - final SAXReader reader = new SAXReader(); - final Document xml = reader.read(new StringReader(xmlString)); - final Element enaSampleRootElement = - enaSampleXmlEnhancer.applyAllRules(xml.getRootElement(), eraproSample); - - // check that we got some content - if (XmlPathBuilder.of(enaSampleRootElement).path("SAMPLE").exists()) { - return enrichSample(eraproSample, enaSampleRootElement, accession); - } else { - log.warn("Unable to find SAMPLE element for " + accession); - } - } - - return null; - } - - /** Enriches one ENA/ NCBI sample */ - private Sample enrichSample( - final EraproSample eraproSample, final Element enaSampleRootElement, final String accession) { - Sample sample = - enaSampleToBioSampleConverter.convertEnaSampleXmlToBioSample( - enaSampleRootElement, accession); - - final String sraAccession = eraproSample.getSampleId(); - final SampleStatus status = handleStatus(eraproSample.getStatus()); - final Long taxId = eraproSample.getTaxId(); - final String webinId = eraproSample.getSubmissionAccountId(); - - final SortedSet attributes = new TreeSet<>(sample.getCharacteristics()); - final SortedSet publications = new TreeSet<>(sample.getPublications()); - final String lastUpdated = eraproSample.getLastUpdated(); - final String firstPublic = eraproSample.getFirstPublic(); - final String firstCreated = eraproSample.getFirstCreated(); - final Instant release; - Instant update = null; - Instant create = null; - Instant submitted = null; - - if (lastUpdated != null) { - update = Instant.parse(lastUpdated); - attributes.add( - Attribute.build("INSDC last update", DateTimeFormatter.ISO_INSTANT.format(update))); - } - - if (firstPublic != null) { - release = Instant.parse(firstPublic); - attributes.add( - Attribute.build("INSDC first public", DateTimeFormatter.ISO_INSTANT.format(release))); - } else { - if (status == SampleStatus.PRIVATE) { - release = - Instant.ofEpochSecond( - LocalDateTime.now(ZoneOffset.UTC).plusYears(100).toEpochSecond(ZoneOffset.UTC)); - } else { - release = Instant.now(); - } - } - - if (firstCreated != null) { - create = Instant.parse(firstCreated); - submitted = Instant.parse(firstCreated); - } - - attributes.add(Attribute.build("INSDC status", status.toString().toLowerCase())); - - // Although update date is passed here, its system generated to time now by - // webapps-core - sample = - Sample.build( - sample.getName(), - accession, - sraAccession, - null, - webinId, - taxId, - status, - release, - update, - create, - submitted, - null, - attributes, - null, - Collections.singleton( - ExternalReference.build("https://www.ebi.ac.uk/ena/browser/view/" + accession))); - - return Sample.Builder.fromSample(sample) - .withNoData() - .withNoStructuredData() - .withPublications(publications) - .withSubmittedVia(SubmittedViaType.PIPELINE_IMPORT) - .build(); - } - - private SampleStatus handleStatus(final int statusId) { - if (1 == statusId) { - return SampleStatus.DRAFT; - } else if (2 == statusId) { - return SampleStatus.PRIVATE; - } else if (3 == statusId) { - return SampleStatus.CANCELLED; - } else if (4 == statusId) { - return SampleStatus.PUBLIC; - } else if (5 == statusId || 7 == statusId) { - return SampleStatus.SUPPRESSED; - } else if (6 == statusId || 8 == statusId) { - return SampleStatus.KILLED; - } else { - throw new RuntimeException("Unrecognised STATUS_ID " + statusId); - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import java.io.StringReader; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.SortedSet; +import java.util.TreeSet; +import org.dom4j.Document; +import org.dom4j.DocumentException; +import org.dom4j.Element; +import org.dom4j.io.SAXReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.PipelinesProperties; +import uk.ac.ebi.biosamples.core.model.*; +import uk.ac.ebi.biosamples.utils.XmlPathBuilder; + +@Service +public class EnaSampleToBioSampleConversionService { + private final Logger log = LoggerFactory.getLogger(getClass()); + private final EnaSampleXmlEnhancer enaSampleXmlEnhancer; + private final EraProDao eraProDao; + private final BioSampleConverter enaSampleToBioSampleConverter; + private final PipelinesProperties pipelinesProperties; + + public EnaSampleToBioSampleConversionService( + final EnaSampleXmlEnhancer enaSampleXmlEnhancer, + final EraProDao eraProDao, + final BioSampleConverter enaSampleToBioSampleConverter, + final PipelinesProperties pipelinesProperties) { + this.enaSampleXmlEnhancer = enaSampleXmlEnhancer; + this.eraProDao = eraProDao; + this.enaSampleToBioSampleConverter = enaSampleToBioSampleConverter; + this.pipelinesProperties = pipelinesProperties; + } + + /** Handles one ENA/ NCBI sample */ + public Sample enrichSample(final String accession, final boolean isNcbiDdbjSample) + throws DocumentException { + final EraproSample eraproSample = eraProDao.getSampleDetailsByBioSampleId(accession); + + if (eraproSample != null) { + final String xmlString = eraproSample.getSampleXml(); + final SAXReader reader = new SAXReader(); + final Document xml = reader.read(new StringReader(xmlString)); + final Element enaSampleRootElement = + enaSampleXmlEnhancer.applyAllRules(xml.getRootElement(), eraproSample); + + // check that we got some content + if (XmlPathBuilder.of(enaSampleRootElement).path("SAMPLE").exists()) { + return enrichSample(eraproSample, enaSampleRootElement, accession, isNcbiDdbjSample); + } else { + log.warn("Unable to find SAMPLE element for " + accession); + } + } + + return null; + } + + /** Handles one ENA/ NCBI sample */ + public Sample enrichSample(final String accession, final EraproSample eraproSample) + throws DocumentException { + if (eraproSample != null) { + final String xmlString = eraproSample.getSampleXml(); + final SAXReader reader = new SAXReader(); + final Document xml = reader.read(new StringReader(xmlString)); + final Element enaSampleRootElement = + enaSampleXmlEnhancer.applyAllRules(xml.getRootElement(), eraproSample); + + // check that we got some content + if (XmlPathBuilder.of(enaSampleRootElement).path("SAMPLE").exists()) { + return enrichSample(eraproSample, enaSampleRootElement, accession, false); + } else { + log.warn("Unable to find SAMPLE element for " + accession); + } + } + + return null; + } + + /** Enriches one ENA/ NCBI sample */ + private Sample enrichSample( + final EraproSample eraproSample, + final Element enaSampleRootElement, + final String accession, + final boolean isNcbiDdbjSample) { + Sample sample = + enaSampleToBioSampleConverter.convertEnaSampleXmlToBioSample( + enaSampleRootElement, accession); + + final String sraAccession = eraproSample.getSampleId(); + final SampleStatus status = handleStatus(eraproSample.getStatus()); + final Long taxId = eraproSample.getTaxId(); + final String webinId = + isNcbiDdbjSample + ? pipelinesProperties.getProxyWebinId() + : eraproSample.getSubmissionAccountId(); + final SortedSet attributes = new TreeSet<>(sample.getCharacteristics()); + final SortedSet publications = new TreeSet<>(sample.getPublications()); + final String lastUpdated = eraproSample.getLastUpdated(); + final String firstPublic = eraproSample.getFirstPublic(); + final String firstCreated = eraproSample.getFirstCreated(); + final Instant release; + + Instant update = null; + Instant create = null; + Instant submitted = null; + + if (lastUpdated != null) { + update = Instant.parse(lastUpdated); + attributes.add( + Attribute.build("INSDC last update", DateTimeFormatter.ISO_INSTANT.format(update))); + } + + if (firstPublic != null) { + release = Instant.parse(firstPublic); + attributes.add( + Attribute.build("INSDC first public", DateTimeFormatter.ISO_INSTANT.format(release))); + } else { + if (status == SampleStatus.PRIVATE) { + release = + Instant.ofEpochSecond( + LocalDateTime.now(ZoneOffset.UTC).plusYears(100).toEpochSecond(ZoneOffset.UTC)); + } else { + release = Instant.now(); + } + } + + if (firstCreated != null) { + create = Instant.parse(firstCreated); + submitted = Instant.parse(firstCreated); + } + + attributes.add(Attribute.build("INSDC status", status.toString().toLowerCase())); + + // Although update date is passed here, its system generated to time now by + // webapps-core + sample = + Sample.build( + sample.getName(), + accession, + sraAccession, + null, + webinId, + taxId, + status, + release, + update, + create, + submitted, + null, + attributes, + null, + Collections.singleton( + ExternalReference.build("https://www.ebi.ac.uk/ena/browser/view/" + accession))); + + return Sample.Builder.fromSample(sample) + .withNoData() + .withNoStructuredData() + .withPublications(publications) + .withSubmittedVia(SubmittedViaType.PIPELINE_IMPORT) + .build(); + } + + private SampleStatus handleStatus(final int statusId) { + if (1 == statusId) { + return SampleStatus.DRAFT; + } else if (2 == statusId) { + return SampleStatus.PRIVATE; + } else if (3 == statusId) { + return SampleStatus.CANCELLED; + } else if (4 == statusId) { + return SampleStatus.PUBLIC; + } else if (5 == statusId || 7 == statusId) { + return SampleStatus.SUPPRESSED; + } else if (6 == statusId || 8 == statusId) { + return SampleStatus.KILLED; + } else { + throw new RuntimeException("Unrecognised STATUS_ID " + statusId); + } + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleXmlEnhancer.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleXmlEnhancer.java similarity index 96% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleXmlEnhancer.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleXmlEnhancer.java index 771edd3e3..43448e43a 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleXmlEnhancer.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaSampleXmlEnhancer.java @@ -1,377 +1,377 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import static uk.ac.ebi.biosamples.service.EnaXmlUtil.pretty; - -import java.io.StringReader; -import org.dom4j.*; -import org.dom4j.io.SAXReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.utils.XmlPathBuilder; - -@Service -public class EnaSampleXmlEnhancer { - private static final Logger LOGGER = LoggerFactory.getLogger(EnaSampleXmlEnhancer.class); - - public String applyRules( - final String inputXml, final EraproSample eraproSample, final Rule... rules) { - final Element element = getSampleElement(inputXml); - final Element modifiedElement = applyRules(element.createCopy(), eraproSample, rules); - final Document document = DocumentHelper.createDocument(); - - document.setRootElement(modifiedElement); - - return pretty(document); - } - - private Element applyRules( - Element sampleElement, final EraproSample eraproSample, final Rule... rules) { - for (final Rule rule : rules) { - sampleElement = rule.apply(sampleElement, eraproSample); - } - - return sampleElement; - } - - public String applyAllRules(final String inputXml, final EraproSample eraproSample) { - return applyRules( - inputXml, - eraproSample, - AliasRule.INSTANCE, - NamespaceRule.INSTANCE, - BrokerRule.INSTANCE, - LinkRemovalRule.INSTANCE, - CenterNameRule.INSTANCE, - DatesRule.INSTANCE, - BioSamplesIdRule.INSTANCE); - } - - public Element applyAllRules(final Element element, final EraproSample eraproSample) { - return applyRules( - element, - eraproSample, - AliasRule.INSTANCE, - NamespaceRule.INSTANCE, - BrokerRule.INSTANCE, - LinkRemovalRule.INSTANCE, - CenterNameRule.INSTANCE, - DatesRule.INSTANCE, - BioSamplesIdRule.INSTANCE); - } - - public interface Rule { - Element apply(Element sampleXml, EraproSample eraproSample); - } - - public enum AliasRule implements Rule { - INSTANCE; - - @Override - public Element apply(final Element sampleXml, final EraproSample eraproSample) { - if (!XmlPathBuilder.of(sampleXml).path("SAMPLE").attributeExists("alias")) { - final XmlPathBuilder xmlPathBuilder = - XmlPathBuilder.of(sampleXml).path("SAMPLE", "IDENTIFIERS", "SUBMITTER_ID"); - - if (xmlPathBuilder.exists()) { - final Node node = sampleXml.selectSingleNode("SAMPLE/IDENTIFIERS/SUBMITTER_ID"); - - node.detach(); - } - } - - return sampleXml; - } - } - - public enum NamespaceRule implements Rule { - INSTANCE; - - @Override - public Element apply(final Element sampleXml, final EraproSample eraproSample) { - if (!XmlPathBuilder.of(sampleXml).path("SAMPLE").attributeExists("center_name")) { - return sampleXml; - } - - final XmlPathBuilder xmlPathBuilder = - XmlPathBuilder.of(sampleXml).path("SAMPLE", "IDENTIFIERS", "SUBMITTER_ID"); - - if (xmlPathBuilder.exists()) { - if (!xmlPathBuilder.attributeExists("namespace") - || xmlPathBuilder.attribute("namespace").isEmpty()) { - final String centerName = - XmlPathBuilder.of(sampleXml).path("SAMPLE").attribute("center_name"); - xmlPathBuilder.element().addAttribute("namespace", centerName); - } - - return sampleXml; - } - - return sampleXml; - } - } - - public enum BrokerRule implements Rule { - INSTANCE; - - @Override - public Element apply(final Element sampleXml, final EraproSample eraproSample) { - final XmlPathBuilder xmlPathBuilder = XmlPathBuilder.of(sampleXml).path("SAMPLE"); - - if (xmlPathBuilder.attributeExists("accession")) { - final String accession = xmlPathBuilder.attribute("accession"); - - if (accession.startsWith("ERS")) { - if (eraproSample.brokerName != null) { - xmlPathBuilder.element().addAttribute("broker_name", eraproSample.brokerName); - } - - return sampleXml; - } - - if (accession.startsWith("SRS")) { - xmlPathBuilder.element().addAttribute("broker_name", "NCBI"); - - return sampleXml; - } - - if (accession.startsWith("DRS")) { - xmlPathBuilder.element().addAttribute("broker_name", "DDBJ"); - - return sampleXml; - } - } - - return sampleXml; - } - } - - public enum LinkRemovalRule implements Rule { - INSTANCE; - - @Override - public Element apply(final Element sampleXml, final EraproSample eraproSample) { - final XmlPathBuilder xmlPathBuilder = - XmlPathBuilder.of(sampleXml).path("SAMPLE", "SAMPLE_LINKS"); - if (xmlPathBuilder.exists()) { - for (final Element sampleLinkElement : xmlPathBuilder.elements("SAMPLE_LINK")) { - if (sampleLinkElement.element("URL_LINK") != null) { - sampleLinkElement.detach(); - } - } - } - - return sampleXml; - } - } - - public enum CenterNameRule implements Rule { - INSTANCE; - - @Override - public Element apply(final Element sampleXml, final EraproSample eraproSample) { - if (eraproSample.centreName != null) { - final XmlPathBuilder xmlPathBuilder = XmlPathBuilder.of(sampleXml).path("SAMPLE"); - - if (xmlPathBuilder.attributeExists("center_name")) { - xmlPathBuilder - .element() - .addAttribute("center_alias", xmlPathBuilder.attribute("center_name")); - xmlPathBuilder.element().addAttribute("center_name", eraproSample.centreName); - } - } - - return sampleXml; - } - } - - public enum DatesRule implements Rule { - INSTANCE; - - @Override - public Element apply(final Element sampleXml, final EraproSample eraproSample) { - if (eraproSample.firstPublic == null || eraproSample.lastUpdated == null) { - return sampleXml; - } - - final XmlPathBuilder xmlPathBuilder = - XmlPathBuilder.of(sampleXml).path("SAMPLE", "SAMPLE_ATTRIBUTES"); - - if (xmlPathBuilder.exists()) { - xmlPathBuilder - .element() - .add(createSampleAttribute("ENA-FIRST-PUBLIC", eraproSample.firstPublic)); - xmlPathBuilder - .element() - .add(createSampleAttribute("ENA-LAST-UPDATE", eraproSample.lastUpdated)); - } - - return sampleXml; - } - } - - public enum BioSamplesIdRule implements Rule { - INSTANCE; - - @Override - public Element apply(final Element sampleXml, final EraproSample eraproSample) { - if (eraproSample.biosampleId == null) { - return sampleXml; - } - - final XmlPathBuilder xmlPathBuilder = - XmlPathBuilder.of(sampleXml).path("SAMPLE", "IDENTIFIERS"); - - if (xmlPathBuilder.exists()) { - boolean bioSamplesExternalIdExists = false; - - for (final Element element : xmlPathBuilder.elements("EXTERNAL_ID")) { - if (element.attribute("namespace").getText().equals("BioSample")) { - bioSamplesExternalIdExists = true; - } - } - - if (!bioSamplesExternalIdExists) { - xmlPathBuilder.element().add(createExternalRef(eraproSample.biosampleId)); - } - } - - return sampleXml; - } - } - - public enum TitleRule implements Rule { - INSTANCE; - - @Override - public Element apply(final Element sampleXml, final EraproSample eraproSample) { - if (eraproSample.biosampleId == null) { - return sampleXml; - } - - XmlPathBuilder xmlPathBuilder = XmlPathBuilder.of(sampleXml).path("SAMPLE", "TITLE"); - - if (!xmlPathBuilder.exists()) { - String newTitle = null; - - if (eraproSample.scientificName != null) { - newTitle = eraproSample.scientificName; - } - - if (newTitle == null && eraproSample.commonName != null) { - newTitle = eraproSample.commonName; - } - - if (newTitle == null && eraproSample.taxId != null) { - newTitle = String.valueOf(eraproSample.taxId); - } - - if (newTitle != null) { - xmlPathBuilder = XmlPathBuilder.of(sampleXml).path("SAMPLE"); - - final Element titleElement = DocumentHelper.createElement("TITLE"); - - titleElement.setText(newTitle); - xmlPathBuilder.element().add(titleElement); - } - } - - return sampleXml; - } - } - - public enum TaxonRule implements Rule { - INSTANCE; - - @Override - public Element apply(final Element sampleXml, final EraproSample eraproSample) { - final XmlPathBuilder parentXmlPathBuilderName = - XmlPathBuilder.of(sampleXml).path("SAMPLE", "SAMPLE_NAME"); - - if (!parentXmlPathBuilderName.exists()) { - XmlPathBuilder.of(sampleXml).path("SAMPLE").element().addElement("SAMPLE_NAME"); - } - - final XmlPathBuilder taxIdXmlPathBuilderTaxonId = - XmlPathBuilder.of(sampleXml).path("SAMPLE", "SAMPLE_NAME", "TAXON_ID"); - final String taxId = eraproSample.taxId == null ? "" : String.valueOf(eraproSample.taxId); - - if (taxIdXmlPathBuilderTaxonId.exists()) { - taxIdXmlPathBuilderTaxonId.element().setText(taxId); - } else { - parentXmlPathBuilderName.element().addElement("TAXON_ID", taxId); - } - - final XmlPathBuilder taxIdXmlPathBuilderScientificName = - XmlPathBuilder.of(sampleXml).path("SAMPLE", "SAMPLE_NAME", "SCIENTIFIC_NAME"); - final String scientificName = - eraproSample.scientificName == null ? "" : eraproSample.scientificName; - - if (taxIdXmlPathBuilderScientificName.exists()) { - taxIdXmlPathBuilderScientificName.element().setText(scientificName); - } else { - parentXmlPathBuilderName.element().addElement("SCIENTIFIC_NAME", scientificName); - } - - final XmlPathBuilder taxIdXmlPathBuilderCommonName = - XmlPathBuilder.of(sampleXml).path("SAMPLE", "SAMPLE_NAME", "COMMON_NAME"); - final String commonName = eraproSample.commonName == null ? "" : eraproSample.commonName; - - if (taxIdXmlPathBuilderCommonName.exists()) { - taxIdXmlPathBuilderCommonName.element().setText(commonName); - } else { - parentXmlPathBuilderName.element().addElement("COMMON_NAME", commonName); - } - - return sampleXml; - } - } - - private Element getSampleElement(final String xmlString) { - final SAXReader reader = new SAXReader(); - Document xml = null; - - try { - xml = reader.read(new StringReader(xmlString)); - } catch (final DocumentException e) { - LOGGER.error("Error reading XML", e); - } - - assert xml != null; - - return xml.getRootElement(); - } - - private static Element createExternalRef(final String bioSamplesId) { - final Element externalIdElement = DocumentHelper.createElement("EXTERNAL_ID"); - - externalIdElement.addAttribute("namespace", "BioSample"); - externalIdElement.setText(bioSamplesId); - - return externalIdElement; - } - - private static Element createSampleAttribute(final String tag, final String value) { - final Element sampleAttributeElement = DocumentHelper.createElement("SAMPLE_ATTRIBUTE"); - final Element tagElement = DocumentHelper.createElement("TAG"); - - tagElement.setText(tag); - sampleAttributeElement.add(tagElement); - - final Element valueElement = DocumentHelper.createElement("VALUE"); - valueElement.setText(value); - sampleAttributeElement.add(valueElement); - - return sampleAttributeElement; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import static uk.ac.ebi.biosamples.service.EnaXmlUtil.pretty; + +import java.io.StringReader; +import org.dom4j.*; +import org.dom4j.io.SAXReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.utils.XmlPathBuilder; + +@Service +public class EnaSampleXmlEnhancer { + private static final Logger LOGGER = LoggerFactory.getLogger(EnaSampleXmlEnhancer.class); + + public String applyRules( + final String inputXml, final EraproSample eraproSample, final Rule... rules) { + final Element element = getSampleElement(inputXml); + final Element modifiedElement = applyRules(element.createCopy(), eraproSample, rules); + final Document document = DocumentHelper.createDocument(); + + document.setRootElement(modifiedElement); + + return pretty(document); + } + + private Element applyRules( + Element sampleElement, final EraproSample eraproSample, final Rule... rules) { + for (final Rule rule : rules) { + sampleElement = rule.apply(sampleElement, eraproSample); + } + + return sampleElement; + } + + public String applyAllRules(final String inputXml, final EraproSample eraproSample) { + return applyRules( + inputXml, + eraproSample, + AliasRule.INSTANCE, + NamespaceRule.INSTANCE, + BrokerRule.INSTANCE, + LinkRemovalRule.INSTANCE, + CenterNameRule.INSTANCE, + DatesRule.INSTANCE, + BioSamplesIdRule.INSTANCE); + } + + public Element applyAllRules(final Element element, final EraproSample eraproSample) { + return applyRules( + element, + eraproSample, + AliasRule.INSTANCE, + NamespaceRule.INSTANCE, + BrokerRule.INSTANCE, + LinkRemovalRule.INSTANCE, + CenterNameRule.INSTANCE, + DatesRule.INSTANCE, + BioSamplesIdRule.INSTANCE); + } + + public interface Rule { + Element apply(Element sampleXml, EraproSample eraproSample); + } + + public enum AliasRule implements Rule { + INSTANCE; + + @Override + public Element apply(final Element sampleXml, final EraproSample eraproSample) { + if (!XmlPathBuilder.of(sampleXml).path("SAMPLE").attributeExists("alias")) { + final XmlPathBuilder xmlPathBuilder = + XmlPathBuilder.of(sampleXml).path("SAMPLE", "IDENTIFIERS", "SUBMITTER_ID"); + + if (xmlPathBuilder.exists()) { + final Node node = sampleXml.selectSingleNode("SAMPLE/IDENTIFIERS/SUBMITTER_ID"); + + node.detach(); + } + } + + return sampleXml; + } + } + + public enum NamespaceRule implements Rule { + INSTANCE; + + @Override + public Element apply(final Element sampleXml, final EraproSample eraproSample) { + if (!XmlPathBuilder.of(sampleXml).path("SAMPLE").attributeExists("center_name")) { + return sampleXml; + } + + final XmlPathBuilder xmlPathBuilder = + XmlPathBuilder.of(sampleXml).path("SAMPLE", "IDENTIFIERS", "SUBMITTER_ID"); + + if (xmlPathBuilder.exists()) { + if (!xmlPathBuilder.attributeExists("namespace") + || xmlPathBuilder.attribute("namespace").isEmpty()) { + final String centerName = + XmlPathBuilder.of(sampleXml).path("SAMPLE").attribute("center_name"); + xmlPathBuilder.element().addAttribute("namespace", centerName); + } + + return sampleXml; + } + + return sampleXml; + } + } + + public enum BrokerRule implements Rule { + INSTANCE; + + @Override + public Element apply(final Element sampleXml, final EraproSample eraproSample) { + final XmlPathBuilder xmlPathBuilder = XmlPathBuilder.of(sampleXml).path("SAMPLE"); + + if (xmlPathBuilder.attributeExists("accession")) { + final String accession = xmlPathBuilder.attribute("accession"); + + if (accession.startsWith("ERS")) { + if (eraproSample.brokerName != null) { + xmlPathBuilder.element().addAttribute("broker_name", eraproSample.brokerName); + } + + return sampleXml; + } + + if (accession.startsWith("SRS")) { + xmlPathBuilder.element().addAttribute("broker_name", "NCBI"); + + return sampleXml; + } + + if (accession.startsWith("DRS")) { + xmlPathBuilder.element().addAttribute("broker_name", "DDBJ"); + + return sampleXml; + } + } + + return sampleXml; + } + } + + public enum LinkRemovalRule implements Rule { + INSTANCE; + + @Override + public Element apply(final Element sampleXml, final EraproSample eraproSample) { + final XmlPathBuilder xmlPathBuilder = + XmlPathBuilder.of(sampleXml).path("SAMPLE", "SAMPLE_LINKS"); + if (xmlPathBuilder.exists()) { + for (final Element sampleLinkElement : xmlPathBuilder.elements("SAMPLE_LINK")) { + if (sampleLinkElement.element("URL_LINK") != null) { + sampleLinkElement.detach(); + } + } + } + + return sampleXml; + } + } + + public enum CenterNameRule implements Rule { + INSTANCE; + + @Override + public Element apply(final Element sampleXml, final EraproSample eraproSample) { + if (eraproSample.centreName != null) { + final XmlPathBuilder xmlPathBuilder = XmlPathBuilder.of(sampleXml).path("SAMPLE"); + + if (xmlPathBuilder.attributeExists("center_name")) { + xmlPathBuilder + .element() + .addAttribute("center_alias", xmlPathBuilder.attribute("center_name")); + xmlPathBuilder.element().addAttribute("center_name", eraproSample.centreName); + } + } + + return sampleXml; + } + } + + public enum DatesRule implements Rule { + INSTANCE; + + @Override + public Element apply(final Element sampleXml, final EraproSample eraproSample) { + if (eraproSample.firstPublic == null || eraproSample.lastUpdated == null) { + return sampleXml; + } + + final XmlPathBuilder xmlPathBuilder = + XmlPathBuilder.of(sampleXml).path("SAMPLE", "SAMPLE_ATTRIBUTES"); + + if (xmlPathBuilder.exists()) { + xmlPathBuilder + .element() + .add(createSampleAttribute("ENA-FIRST-PUBLIC", eraproSample.firstPublic)); + xmlPathBuilder + .element() + .add(createSampleAttribute("ENA-LAST-UPDATE", eraproSample.lastUpdated)); + } + + return sampleXml; + } + } + + public enum BioSamplesIdRule implements Rule { + INSTANCE; + + @Override + public Element apply(final Element sampleXml, final EraproSample eraproSample) { + if (eraproSample.biosampleId == null) { + return sampleXml; + } + + final XmlPathBuilder xmlPathBuilder = + XmlPathBuilder.of(sampleXml).path("SAMPLE", "IDENTIFIERS"); + + if (xmlPathBuilder.exists()) { + boolean bioSamplesExternalIdExists = false; + + for (final Element element : xmlPathBuilder.elements("EXTERNAL_ID")) { + if (element.attribute("namespace").getText().equals("BioSample")) { + bioSamplesExternalIdExists = true; + } + } + + if (!bioSamplesExternalIdExists) { + xmlPathBuilder.element().add(createExternalRef(eraproSample.biosampleId)); + } + } + + return sampleXml; + } + } + + public enum TitleRule implements Rule { + INSTANCE; + + @Override + public Element apply(final Element sampleXml, final EraproSample eraproSample) { + if (eraproSample.biosampleId == null) { + return sampleXml; + } + + XmlPathBuilder xmlPathBuilder = XmlPathBuilder.of(sampleXml).path("SAMPLE", "TITLE"); + + if (!xmlPathBuilder.exists()) { + String newTitle = null; + + if (eraproSample.scientificName != null) { + newTitle = eraproSample.scientificName; + } + + if (newTitle == null && eraproSample.commonName != null) { + newTitle = eraproSample.commonName; + } + + if (newTitle == null && eraproSample.taxId != null) { + newTitle = String.valueOf(eraproSample.taxId); + } + + if (newTitle != null) { + xmlPathBuilder = XmlPathBuilder.of(sampleXml).path("SAMPLE"); + + final Element titleElement = DocumentHelper.createElement("TITLE"); + + titleElement.setText(newTitle); + xmlPathBuilder.element().add(titleElement); + } + } + + return sampleXml; + } + } + + public enum TaxonRule implements Rule { + INSTANCE; + + @Override + public Element apply(final Element sampleXml, final EraproSample eraproSample) { + final XmlPathBuilder parentXmlPathBuilderName = + XmlPathBuilder.of(sampleXml).path("SAMPLE", "SAMPLE_NAME"); + + if (!parentXmlPathBuilderName.exists()) { + XmlPathBuilder.of(sampleXml).path("SAMPLE").element().addElement("SAMPLE_NAME"); + } + + final XmlPathBuilder taxIdXmlPathBuilderTaxonId = + XmlPathBuilder.of(sampleXml).path("SAMPLE", "SAMPLE_NAME", "TAXON_ID"); + final String taxId = eraproSample.taxId == null ? "" : String.valueOf(eraproSample.taxId); + + if (taxIdXmlPathBuilderTaxonId.exists()) { + taxIdXmlPathBuilderTaxonId.element().setText(taxId); + } else { + parentXmlPathBuilderName.element().addElement("TAXON_ID", taxId); + } + + final XmlPathBuilder taxIdXmlPathBuilderScientificName = + XmlPathBuilder.of(sampleXml).path("SAMPLE", "SAMPLE_NAME", "SCIENTIFIC_NAME"); + final String scientificName = + eraproSample.scientificName == null ? "" : eraproSample.scientificName; + + if (taxIdXmlPathBuilderScientificName.exists()) { + taxIdXmlPathBuilderScientificName.element().setText(scientificName); + } else { + parentXmlPathBuilderName.element().addElement("SCIENTIFIC_NAME", scientificName); + } + + final XmlPathBuilder taxIdXmlPathBuilderCommonName = + XmlPathBuilder.of(sampleXml).path("SAMPLE", "SAMPLE_NAME", "COMMON_NAME"); + final String commonName = eraproSample.commonName == null ? "" : eraproSample.commonName; + + if (taxIdXmlPathBuilderCommonName.exists()) { + taxIdXmlPathBuilderCommonName.element().setText(commonName); + } else { + parentXmlPathBuilderName.element().addElement("COMMON_NAME", commonName); + } + + return sampleXml; + } + } + + private Element getSampleElement(final String xmlString) { + final SAXReader reader = new SAXReader(); + Document xml = null; + + try { + xml = reader.read(new StringReader(xmlString)); + } catch (final DocumentException e) { + LOGGER.error("Error reading XML", e); + } + + assert xml != null; + + return xml.getRootElement(); + } + + private static Element createExternalRef(final String bioSamplesId) { + final Element externalIdElement = DocumentHelper.createElement("EXTERNAL_ID"); + + externalIdElement.addAttribute("namespace", "BioSample"); + externalIdElement.setText(bioSamplesId); + + return externalIdElement; + } + + private static Element createSampleAttribute(final String tag, final String value) { + final Element sampleAttributeElement = DocumentHelper.createElement("SAMPLE_ATTRIBUTE"); + final Element tagElement = DocumentHelper.createElement("TAG"); + + tagElement.setText(tag); + sampleAttributeElement.add(tagElement); + + final Element valueElement = DocumentHelper.createElement("VALUE"); + valueElement.setText(value); + sampleAttributeElement.add(valueElement); + + return sampleAttributeElement; + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaXmlUtil.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaXmlUtil.java similarity index 97% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaXmlUtil.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaXmlUtil.java index 09e37e694..5a56ab5f9 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EnaXmlUtil.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EnaXmlUtil.java @@ -1,51 +1,51 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.io.IOException; -import java.io.StringReader; -import java.io.StringWriter; -import org.dom4j.Document; -import org.dom4j.DocumentException; -import org.dom4j.io.OutputFormat; -import org.dom4j.io.SAXReader; -import org.dom4j.io.XMLWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class EnaXmlUtil { - private static final Logger LOGGER = LoggerFactory.getLogger(EnaXmlUtil.class); - - public static String pretty(final String xmlString) { - final SAXReader reader = new SAXReader(); - Document document = null; - try { - document = reader.read(new StringReader(xmlString)); - } catch (final DocumentException e) { - LOGGER.error("Error reading XML", e); - } - return pretty(document); - } - - static String pretty(final Document document) { - final OutputFormat format = OutputFormat.createPrettyPrint(); - final StringWriter outputWriter = new StringWriter(); - final XMLWriter writer = new XMLWriter(outputWriter, format); - try { - writer.write(document); - outputWriter.close(); - writer.close(); - } catch (final IOException e) { - LOGGER.error("Error writing XML", e); - } - return outputWriter.toString(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import org.dom4j.Document; +import org.dom4j.DocumentException; +import org.dom4j.io.OutputFormat; +import org.dom4j.io.SAXReader; +import org.dom4j.io.XMLWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EnaXmlUtil { + private static final Logger LOGGER = LoggerFactory.getLogger(EnaXmlUtil.class); + + public static String pretty(final String xmlString) { + final SAXReader reader = new SAXReader(); + Document document = null; + try { + document = reader.read(new StringReader(xmlString)); + } catch (final DocumentException e) { + LOGGER.error("Error reading XML", e); + } + return pretty(document); + } + + static String pretty(final Document document) { + final OutputFormat format = OutputFormat.createPrettyPrint(); + final StringWriter outputWriter = new StringWriter(); + final XMLWriter writer = new XMLWriter(outputWriter, format); + try { + writer.write(document); + outputWriter.close(); + writer.close(); + } catch (final IOException e) { + LOGGER.error("Error writing XML", e); + } + return outputWriter.toString(); + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EraProDao.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EraProDao.java similarity index 97% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EraProDao.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EraProDao.java index 4252dcd1b..af313759f 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EraProDao.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EraProDao.java @@ -1,271 +1,271 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.time.LocalDate; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.dao.IncorrectResultSizeDataAccessException; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.core.RowCallbackHandler; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.stereotype.Service; - -@Service -public class EraProDao { - private final Logger log = LoggerFactory.getLogger(getClass()); - - @Autowired - @Qualifier("eraJdbcTemplate") - protected JdbcTemplate jdbcTemplate; - - private static final String STATUS_CLAUSE = "STATUS_ID IN (2, 4, 5, 6, 7, 8)"; - private static final String STATUS_CLAUSE_SUPPRESSED = "STATUS_ID IN (5, 7)"; - private static final String STATUS_CLAUSE_KILLED = "STATUS_ID IN (6, 8)"; - - public List doSampleCallback( - final LocalDate minDate, final LocalDate maxDate) { - log.info("Getting ENA samples"); - - final String query = - "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID, EGA_ID, LAST_UPDATED FROM SAMPLE WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND BIOSAMPLE_AUTHORITY= 'N' " - + "AND " - + STATUS_CLAUSE - + " AND ((LAST_UPDATED BETWEEN ? AND ?) OR (FIRST_PUBLIC BETWEEN ? AND ?)) AND EGA_ID IS NULL ORDER BY LAST_UPDATED ASC"; - - final Date minDateOld = java.sql.Date.valueOf(minDate); - final Date maxDateOld = java.sql.Date.valueOf(maxDate); - - return jdbcTemplate.query( - query, sampleCallbackResultRowMapper, minDateOld, maxDateOld, minDateOld, maxDateOld); - } - - public List doSampleCallbackForAccessions( - final List accessions, final boolean biosamples) { - log.info("Getting ENA samples"); - final AtomicInteger counter = new AtomicInteger(0); - final int batchSize = 1000; - final List sampleCallbackResults = new ArrayList<>(); - final Collection> accessionBatches = - accessions.stream() - .collect(Collectors.groupingBy(accession -> counter.getAndIncrement() / batchSize)) - .values(); - - // Execute the query with the accessions as parameters - if (biosamples) { - accessionBatches.forEach( - accessionBatch -> { - final String query = queryForBioSampleAccessions(accessionBatch); - - sampleCallbackResults.addAll( - jdbcTemplate.query(query, sampleCallbackResultRowMapper, accessionBatch.toArray())); - }); - } else { - accessionBatches.forEach( - accessionBatch -> { - final String query = queryForAccessions(accessionBatch); - - sampleCallbackResults.addAll( - jdbcTemplate.query(query, sampleCallbackResultRowMapper, accessionBatch.toArray())); - }); - } - - return sampleCallbackResults; - } - - private String queryForAccessions(List accessions) { - // Create placeholders for the IN clause - final String placeholders = accessions.stream().map(a -> "?").collect(Collectors.joining(", ")); - - // Build the query with the appropriate number of placeholders - final String query = - "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID, EGA_ID, LAST_UPDATED FROM SAMPLE " - + "WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND BIOSAMPLE_AUTHORITY= 'N' " - + "AND STATUS_ID IN (2, 4, 5, 6, 7, 8) AND EGA_ID IS NULL " - + "AND BIOSAMPLE_ID IN (" - + placeholders - + ") " - + "ORDER BY LAST_UPDATED ASC"; - - // Log the query for debugging - log.debug("Executing query: {}", query); - - return query; - } - - private String queryForBioSampleAccessions(List accessions) { - // Create placeholders for the IN clause - final String placeholders = accessions.stream().map(a -> "?").collect(Collectors.joining(", ")); - - // Build the query with the appropriate number of placeholders - final String query = - "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID, EGA_ID, LAST_UPDATED FROM SAMPLE " - + "WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND BIOSAMPLE_AUTHORITY= 'Y' " - + "AND STATUS_ID IN (2, 4, 5, 6, 7, 8) AND EGA_ID IS NULL " - + "AND BIOSAMPLE_ID IN (" - + placeholders - + ") " - + "ORDER BY LAST_UPDATED ASC"; - - // Log the query for debugging - log.debug("Executing query: {}", query); - - return query; - } - - public List doSampleCallbackForBsdAuthoritySamples( - final LocalDate minDate, final LocalDate maxDate) { - final String query = - "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID, EGA_ID, LAST_UPDATED FROM SAMPLE WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND BIOSAMPLE_AUTHORITY= 'Y' " - + "AND " - + STATUS_CLAUSE - + " AND ((LAST_UPDATED BETWEEN ? AND ?) OR (FIRST_PUBLIC BETWEEN ? AND ?)) AND EGA_ID IS NULL ORDER BY LAST_UPDATED ASC"; - - final Date minDateOld = java.sql.Date.valueOf(minDate); - final Date maxDateOld = java.sql.Date.valueOf(maxDate); - - return jdbcTemplate.query( - query, sampleCallbackResultRowMapper, minDateOld, maxDateOld, minDateOld, maxDateOld); - } - - /** - * Returns SUPPRESSED ENA samples - * - * @param rch {@link RowCallbackHandler} - */ - public void doGetSuppressedEnaSamples(final RowCallbackHandler rch) { - final String query = - "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID FROM SAMPLE WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND EGA_ID IS NULL AND BIOSAMPLE_AUTHORITY= 'N' AND " - + STATUS_CLAUSE_SUPPRESSED; - - jdbcTemplate.query(query, rch); - } - - /** - * Returns KILLED ENA samples - * - * @param rch {@link RowCallbackHandler} - */ - public void doGetKilledEnaSamples(final RowCallbackHandler rch) { - final String query = - "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID FROM SAMPLE WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND EGA_ID IS NULL AND BIOSAMPLE_AUTHORITY= 'N' AND " - + STATUS_CLAUSE_KILLED; - - jdbcTemplate.query(query, rch); - } - - /*public List doWWWDEVMapping() { - return jdbcTemplate.queryForList(SQL_WWWDEV_MAPPING, String.class); - }*/ - - public void getSingleSample(final String bioSampleId, final RowCallbackHandler rch) { - final String query = - "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID FROM SAMPLE WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND EGA_ID IS NULL AND BIOSAMPLE_AUTHORITY= 'N' " - + "AND " - + STATUS_CLAUSE - + " AND BIOSAMPLE_ID=? ORDER BY BIOSAMPLE_ID ASC"; - - jdbcTemplate.query(query, rch, bioSampleId); - } - - public List doNcbiCallback( - final LocalDate minDate, final LocalDate maxDate) { - final String query = - "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID, EGA_ID, LAST_UPDATED FROM SAMPLE WHERE (BIOSAMPLE_ID LIKE 'SAMN%' OR BIOSAMPLE_ID LIKE 'SAMD%' ) AND BIOSAMPLE_AUTHORITY= 'N' " - + "AND " - + STATUS_CLAUSE - + " AND ((LAST_UPDATED BETWEEN ? AND ?) OR (FIRST_PUBLIC BETWEEN ? AND ?)) ORDER BY LAST_UPDATED ASC"; - - final Date minDateOld = java.sql.Date.valueOf(minDate); - final Date maxDateOld = java.sql.Date.valueOf(maxDate); - - return jdbcTemplate.query( - query, sampleCallbackResultRowMapper, minDateOld, maxDateOld, minDateOld, maxDateOld); - } - - public EraproSample getSampleDetailsByBioSampleId(final String bioSampleId) { - try { - final String sql = - "SELECT SAMPLE_XML, TAX_ID, SAMPLE_ID, " - + "to_char(LAST_UPDATED, 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') AS LAST_UPDATED, " - + "to_char(FIRST_PUBLIC, 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') AS FIRST_PUBLIC, " - + "to_char(FIRST_CREATED, 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') AS FIRST_CREATED, " - + "STATUS_ID, " - + "SUBMISSION_ACCOUNT_ID, " - + "BIOSAMPLE_ID, " - + "TAX_ID, " - + "SCIENTIFIC_NAME, " - + "COMMON_NAME, " - + "to_char(first_public, 'yyyy-mm-dd') as first_public,\n" - + "to_char(last_updated, 'yyyy-mm-dd') as last_updated \n" - + " FROM SAMPLE " - + "WHERE BIOSAMPLE_ID = ? fetch first row only "; - final EraproSample sampleData = - jdbcTemplate.queryForObject(sql, eraproSampleRowMapper, bioSampleId); - - return sampleData; - } catch (final IncorrectResultSizeDataAccessException e) { - log.error( - "Result set size expected is 1 and got more/ less that that, skipping " + bioSampleId); - } - - return null; - } - - private final RowMapper eraproSampleRowMapper = - (rs, rowNum) -> { - final EraproSample sampleBean = new EraproSample(); - - sampleBean.setSampleXml(rs.getString("SAMPLE_XML")); - sampleBean.setSampleId(rs.getString("SAMPLE_ID")); - sampleBean.setFirstPublic(rs.getString("FIRST_PUBLIC")); - sampleBean.setLastUpdated(rs.getString("LAST_UPDATED")); - sampleBean.setFirstCreated(rs.getString("FIRST_CREATED")); - sampleBean.setStatus(rs.getInt("STATUS_ID")); - sampleBean.setSubmissionAccountId(rs.getString("SUBMISSION_ACCOUNT_ID")); - sampleBean.setTaxId(rs.getLong("TAX_ID")); - sampleBean.setBiosampleId(rs.getString("BIOSAMPLE_ID")); - sampleBean.setScientificName(rs.getString("SCIENTIFIC_NAME")); - sampleBean.setCommonName(rs.getString("COMMON_NAME")); - - return sampleBean; - }; - - private final RowMapper sampleCallbackResultRowMapper = - (rs, rowNum) -> { - final SampleCallbackResult sampleCallbackResult = new SampleCallbackResult(); - - sampleCallbackResult.setBiosampleId(rs.getString("BIOSAMPLE_ID")); - sampleCallbackResult.setEgaId(rs.getString("EGA_ID")); - sampleCallbackResult.setStatusId(rs.getInt("STATUS_ID")); - sampleCallbackResult.setLastUpdated(rs.getDate("LAST_UPDATED")); - - return sampleCallbackResult; - }; - - public void updateSampleStatus(final String biosampleId) { - try { - jdbcTemplate.update("UPDATE SAMPLE SET STATUS_ID = 4 WHERE BIOSAMPLE_ID = ?", biosampleId); - } catch (final Exception e) { - log.error("Failed to update sample status of " + biosampleId); - - throw e; - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.dao.IncorrectResultSizeDataAccessException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Service; + +@Service +public class EraProDao { + private final Logger log = LoggerFactory.getLogger(getClass()); + + @Autowired + @Qualifier("eraJdbcTemplate") + protected JdbcTemplate jdbcTemplate; + + private static final String STATUS_CLAUSE = "STATUS_ID IN (2, 4, 5, 6, 7, 8)"; + private static final String STATUS_CLAUSE_SUPPRESSED = "STATUS_ID IN (5, 7)"; + private static final String STATUS_CLAUSE_KILLED = "STATUS_ID IN (6, 8)"; + + public List doSampleCallback( + final LocalDate minDate, final LocalDate maxDate) { + log.info("Getting ENA samples"); + + final String query = + "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID, EGA_ID, LAST_UPDATED FROM SAMPLE WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND BIOSAMPLE_AUTHORITY= 'N' " + + "AND " + + STATUS_CLAUSE + + " AND ((LAST_UPDATED BETWEEN ? AND ?) OR (FIRST_PUBLIC BETWEEN ? AND ?)) AND EGA_ID IS NULL ORDER BY LAST_UPDATED ASC"; + + final Date minDateOld = java.sql.Date.valueOf(minDate); + final Date maxDateOld = java.sql.Date.valueOf(maxDate); + + return jdbcTemplate.query( + query, sampleCallbackResultRowMapper, minDateOld, maxDateOld, minDateOld, maxDateOld); + } + + public List doSampleCallbackForAccessions( + final List accessions, final boolean biosamples) { + log.info("Getting ENA samples"); + final AtomicInteger counter = new AtomicInteger(0); + final int batchSize = 1000; + final List sampleCallbackResults = new ArrayList<>(); + final Collection> accessionBatches = + accessions.stream() + .collect(Collectors.groupingBy(accession -> counter.getAndIncrement() / batchSize)) + .values(); + + // Execute the query with the accessions as parameters + if (biosamples) { + accessionBatches.forEach( + accessionBatch -> { + final String query = queryForBioSampleAccessions(accessionBatch); + + sampleCallbackResults.addAll( + jdbcTemplate.query(query, sampleCallbackResultRowMapper, accessionBatch.toArray())); + }); + } else { + accessionBatches.forEach( + accessionBatch -> { + final String query = queryForAccessions(accessionBatch); + + sampleCallbackResults.addAll( + jdbcTemplate.query(query, sampleCallbackResultRowMapper, accessionBatch.toArray())); + }); + } + + return sampleCallbackResults; + } + + private String queryForAccessions(List accessions) { + // Create placeholders for the IN clause + final String placeholders = accessions.stream().map(a -> "?").collect(Collectors.joining(", ")); + + // Build the query with the appropriate number of placeholders + final String query = + "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID, EGA_ID, LAST_UPDATED FROM SAMPLE " + + "WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND BIOSAMPLE_AUTHORITY= 'N' " + + "AND STATUS_ID IN (2, 4, 5, 6, 7, 8) AND EGA_ID IS NULL " + + "AND BIOSAMPLE_ID IN (" + + placeholders + + ") " + + "ORDER BY LAST_UPDATED ASC"; + + // Log the query for debugging + log.debug("Executing query: {}", query); + + return query; + } + + private String queryForBioSampleAccessions(List accessions) { + // Create placeholders for the IN clause + final String placeholders = accessions.stream().map(a -> "?").collect(Collectors.joining(", ")); + + // Build the query with the appropriate number of placeholders + final String query = + "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID, EGA_ID, LAST_UPDATED FROM SAMPLE " + + "WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND BIOSAMPLE_AUTHORITY= 'Y' " + + "AND STATUS_ID IN (2, 4, 5, 6, 7, 8) AND EGA_ID IS NULL " + + "AND BIOSAMPLE_ID IN (" + + placeholders + + ") " + + "ORDER BY LAST_UPDATED ASC"; + + // Log the query for debugging + log.debug("Executing query: {}", query); + + return query; + } + + public List doSampleCallbackForBsdAuthoritySamples( + final LocalDate minDate, final LocalDate maxDate) { + final String query = + "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID, EGA_ID, LAST_UPDATED FROM SAMPLE WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND BIOSAMPLE_AUTHORITY= 'Y' " + + "AND " + + STATUS_CLAUSE + + " AND ((LAST_UPDATED BETWEEN ? AND ?) OR (FIRST_PUBLIC BETWEEN ? AND ?)) AND EGA_ID IS NULL ORDER BY LAST_UPDATED ASC"; + + final Date minDateOld = java.sql.Date.valueOf(minDate); + final Date maxDateOld = java.sql.Date.valueOf(maxDate); + + return jdbcTemplate.query( + query, sampleCallbackResultRowMapper, minDateOld, maxDateOld, minDateOld, maxDateOld); + } + + /** + * Returns SUPPRESSED ENA samples + * + * @param rch {@link RowCallbackHandler} + */ + public void doGetSuppressedEnaSamples(final RowCallbackHandler rch) { + final String query = + "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID FROM SAMPLE WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND EGA_ID IS NULL AND BIOSAMPLE_AUTHORITY= 'N' AND " + + STATUS_CLAUSE_SUPPRESSED; + + jdbcTemplate.query(query, rch); + } + + /** + * Returns KILLED ENA samples + * + * @param rch {@link RowCallbackHandler} + */ + public void doGetKilledEnaSamples(final RowCallbackHandler rch) { + final String query = + "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID FROM SAMPLE WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND EGA_ID IS NULL AND BIOSAMPLE_AUTHORITY= 'N' AND " + + STATUS_CLAUSE_KILLED; + + jdbcTemplate.query(query, rch); + } + + /*public List doWWWDEVMapping() { + return jdbcTemplate.queryForList(SQL_WWWDEV_MAPPING, String.class); + }*/ + + public void getSingleSample(final String bioSampleId, final RowCallbackHandler rch) { + final String query = + "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID FROM SAMPLE WHERE BIOSAMPLE_ID LIKE 'SAME%' AND SAMPLE_ID LIKE 'ERS%' AND EGA_ID IS NULL AND BIOSAMPLE_AUTHORITY= 'N' " + + "AND " + + STATUS_CLAUSE + + " AND BIOSAMPLE_ID=? ORDER BY BIOSAMPLE_ID ASC"; + + jdbcTemplate.query(query, rch, bioSampleId); + } + + public List doNcbiCallback( + final LocalDate minDate, final LocalDate maxDate) { + final String query = + "SELECT UNIQUE(BIOSAMPLE_ID), STATUS_ID, EGA_ID, LAST_UPDATED FROM SAMPLE WHERE (BIOSAMPLE_ID LIKE 'SAMN%' OR BIOSAMPLE_ID LIKE 'SAMD%' ) AND BIOSAMPLE_AUTHORITY= 'N' " + + "AND " + + STATUS_CLAUSE + + " AND ((LAST_UPDATED BETWEEN ? AND ?) OR (FIRST_PUBLIC BETWEEN ? AND ?)) ORDER BY LAST_UPDATED ASC"; + + final Date minDateOld = java.sql.Date.valueOf(minDate); + final Date maxDateOld = java.sql.Date.valueOf(maxDate); + + return jdbcTemplate.query( + query, sampleCallbackResultRowMapper, minDateOld, maxDateOld, minDateOld, maxDateOld); + } + + public EraproSample getSampleDetailsByBioSampleId(final String bioSampleId) { + try { + final String sql = + "SELECT SAMPLE_XML, TAX_ID, SAMPLE_ID, " + + "to_char(LAST_UPDATED, 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') AS LAST_UPDATED, " + + "to_char(FIRST_PUBLIC, 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') AS FIRST_PUBLIC, " + + "to_char(FIRST_CREATED, 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') AS FIRST_CREATED, " + + "STATUS_ID, " + + "SUBMISSION_ACCOUNT_ID, " + + "BIOSAMPLE_ID, " + + "TAX_ID, " + + "SCIENTIFIC_NAME, " + + "COMMON_NAME, " + + "to_char(first_public, 'yyyy-mm-dd') as first_public,\n" + + "to_char(last_updated, 'yyyy-mm-dd') as last_updated \n" + + " FROM SAMPLE " + + "WHERE BIOSAMPLE_ID = ? fetch first row only "; + final EraproSample sampleData = + jdbcTemplate.queryForObject(sql, eraproSampleRowMapper, bioSampleId); + + return sampleData; + } catch (final IncorrectResultSizeDataAccessException e) { + log.error( + "Result set size expected is 1 and got more/ less that that, skipping " + bioSampleId); + } + + return null; + } + + private final RowMapper eraproSampleRowMapper = + (rs, rowNum) -> { + final EraproSample sampleBean = new EraproSample(); + + sampleBean.setSampleXml(rs.getString("SAMPLE_XML")); + sampleBean.setSampleId(rs.getString("SAMPLE_ID")); + sampleBean.setFirstPublic(rs.getString("FIRST_PUBLIC")); + sampleBean.setLastUpdated(rs.getString("LAST_UPDATED")); + sampleBean.setFirstCreated(rs.getString("FIRST_CREATED")); + sampleBean.setStatus(rs.getInt("STATUS_ID")); + sampleBean.setSubmissionAccountId(rs.getString("SUBMISSION_ACCOUNT_ID")); + sampleBean.setTaxId(rs.getLong("TAX_ID")); + sampleBean.setBiosampleId(rs.getString("BIOSAMPLE_ID")); + sampleBean.setScientificName(rs.getString("SCIENTIFIC_NAME")); + sampleBean.setCommonName(rs.getString("COMMON_NAME")); + + return sampleBean; + }; + + private final RowMapper sampleCallbackResultRowMapper = + (rs, rowNum) -> { + final SampleCallbackResult sampleCallbackResult = new SampleCallbackResult(); + + sampleCallbackResult.setBiosampleId(rs.getString("BIOSAMPLE_ID")); + sampleCallbackResult.setEgaId(rs.getString("EGA_ID")); + sampleCallbackResult.setStatusId(rs.getInt("STATUS_ID")); + sampleCallbackResult.setLastUpdated(rs.getDate("LAST_UPDATED")); + + return sampleCallbackResult; + }; + + public void updateSampleStatus(final String biosampleId) { + try { + jdbcTemplate.update("UPDATE SAMPLE SET STATUS_ID = 4 WHERE BIOSAMPLE_ID = ?", biosampleId); + } catch (final Exception e) { + log.error("Failed to update sample status of " + biosampleId); + + throw e; + } + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EraproSample.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EraproSample.java similarity index 94% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EraproSample.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EraproSample.java index 34bcb3be7..92bb09e03 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/EraproSample.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/EraproSample.java @@ -1,150 +1,150 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import uk.ac.ebi.biosamples.model.Sample; - -/** - * Bean to store all ENA sample related details fetched from ERAPRO for a {@link Sample} - * - * @author dgupta - */ -public class EraproSample { - private String sampleXml; - public String firstPublic; - public String lastUpdated; - private String firstCreated; - public String brokerName; - public String centreName; - public String scientificName; - public String commonName; - private String submissionAccountId; - public Long taxId; - private int status; - private String sampleId; - public String biosampleId; - private String biosampleAuthority; - - public String getSampleId() { - return sampleId; - } - - void setSampleId(final String sampleId) { - this.sampleId = sampleId; - } - - public String getBiosampleId() { - return biosampleId; - } - - void setBiosampleId(final String biosampleId) { - this.biosampleId = biosampleId; - } - - public String getBiosampleAuthority() { - return biosampleAuthority; - } - - void setBiosampleAuthority(final String biosampleAuthority) { - this.biosampleAuthority = biosampleAuthority; - } - - public Long getTaxId() { - return taxId; - } - - void setTaxId(final Long taxId) { - this.taxId = taxId; - } - - public String getSampleXml() { - return sampleXml; - } - - void setSampleXml(final String sampleXml) { - this.sampleXml = sampleXml; - } - - public String getFirstPublic() { - return firstPublic; - } - - void setFirstPublic(final String firstPublic) { - this.firstPublic = firstPublic; - } - - public String getLastUpdated() { - return lastUpdated; - } - - void setLastUpdated(final String lastUpdated) { - this.lastUpdated = lastUpdated; - } - - public String getFirstCreated() { - return firstCreated; - } - - void setFirstCreated(final String firstCreated) { - this.firstCreated = firstCreated; - } - - public int getStatus() { - return status; - } - - void setStatus(final int status) { - this.status = status; - } - - public String getSubmissionAccountId() { - return submissionAccountId; - } - - void setSubmissionAccountId(final String submissionAccountId) { - this.submissionAccountId = submissionAccountId; - } - - public String getBrokerName() { - return brokerName; - } - - public EraproSample setBrokerName(final String brokerName) { - this.brokerName = brokerName; - return this; - } - - public String getCentreName() { - return centreName; - } - - public EraproSample setCentreName(final String centreName) { - this.centreName = centreName; - return this; - } - - public String getScientificName() { - return scientificName; - } - - public EraproSample setScientificName(final String scientificName) { - this.scientificName = scientificName; - return this; - } - - public String getCommonName() { - return commonName; - } - - public void setCommonName(final String commonName) { - this.commonName = commonName; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import uk.ac.ebi.biosamples.core.model.Sample; + +/** + * Bean to store all ENA sample related details fetched from ERAPRO for a {@link Sample} + * + * @author dgupta + */ +public class EraproSample { + private String sampleXml; + public String firstPublic; + public String lastUpdated; + private String firstCreated; + public String brokerName; + public String centreName; + public String scientificName; + public String commonName; + private String submissionAccountId; + public Long taxId; + private int status; + private String sampleId; + public String biosampleId; + private String biosampleAuthority; + + public String getSampleId() { + return sampleId; + } + + void setSampleId(final String sampleId) { + this.sampleId = sampleId; + } + + public String getBiosampleId() { + return biosampleId; + } + + void setBiosampleId(final String biosampleId) { + this.biosampleId = biosampleId; + } + + public String getBiosampleAuthority() { + return biosampleAuthority; + } + + void setBiosampleAuthority(final String biosampleAuthority) { + this.biosampleAuthority = biosampleAuthority; + } + + public Long getTaxId() { + return taxId; + } + + void setTaxId(final Long taxId) { + this.taxId = taxId; + } + + public String getSampleXml() { + return sampleXml; + } + + void setSampleXml(final String sampleXml) { + this.sampleXml = sampleXml; + } + + public String getFirstPublic() { + return firstPublic; + } + + void setFirstPublic(final String firstPublic) { + this.firstPublic = firstPublic; + } + + public String getLastUpdated() { + return lastUpdated; + } + + void setLastUpdated(final String lastUpdated) { + this.lastUpdated = lastUpdated; + } + + public String getFirstCreated() { + return firstCreated; + } + + void setFirstCreated(final String firstCreated) { + this.firstCreated = firstCreated; + } + + public int getStatus() { + return status; + } + + void setStatus(final int status) { + this.status = status; + } + + public String getSubmissionAccountId() { + return submissionAccountId; + } + + void setSubmissionAccountId(final String submissionAccountId) { + this.submissionAccountId = submissionAccountId; + } + + public String getBrokerName() { + return brokerName; + } + + public EraproSample setBrokerName(final String brokerName) { + this.brokerName = brokerName; + return this; + } + + public String getCentreName() { + return centreName; + } + + public EraproSample setCentreName(final String centreName) { + this.centreName = centreName; + return this; + } + + public String getScientificName() { + return scientificName; + } + + public EraproSample setScientificName(final String scientificName) { + this.scientificName = scientificName; + return this; + } + + public String getCommonName() { + return commonName; + } + + public void setCommonName(final String commonName) { + this.commonName = commonName; + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/SampleCallbackResult.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/SampleCallbackResult.java similarity index 96% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/SampleCallbackResult.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/SampleCallbackResult.java index c15e0b34e..529461d00 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/SampleCallbackResult.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/SampleCallbackResult.java @@ -1,52 +1,52 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.sql.Date; - -public class SampleCallbackResult { - private String biosampleId; - private int statusId; - private String egaId; - private Date lastUpdated; - - public String getBiosampleId() { - return biosampleId; - } - - public void setBiosampleId(final String biosampleId) { - this.biosampleId = biosampleId; - } - - public int getStatusId() { - return statusId; - } - - public void setStatusId(final int statusId) { - this.statusId = statusId; - } - - public String getEgaId() { - return egaId; - } - - public void setEgaId(final String egaId) { - this.egaId = egaId; - } - - public Date getLastUpdated() { - return lastUpdated; - } - - public void setLastUpdated(final Date lastUpdated) { - this.lastUpdated = lastUpdated; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import java.sql.Date; + +public class SampleCallbackResult { + private String biosampleId; + private int statusId; + private String egaId; + private Date lastUpdated; + + public String getBiosampleId() { + return biosampleId; + } + + public void setBiosampleId(final String biosampleId) { + this.biosampleId = biosampleId; + } + + public int getStatusId() { + return statusId; + } + + public void setStatusId(final int statusId) { + this.statusId = statusId; + } + + public String getEgaId() { + return egaId; + } + + public void setEgaId(final String egaId) { + this.egaId = egaId; + } + + public Date getLastUpdated() { + return lastUpdated; + } + + public void setLastUpdated(final Date lastUpdated) { + this.lastUpdated = lastUpdated; + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/TaxonomyService.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/TaxonomyService.java similarity index 97% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/TaxonomyService.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/TaxonomyService.java index 2f58a47e8..24557ac60 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/TaxonomyService.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/TaxonomyService.java @@ -1,21 +1,21 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import org.springframework.stereotype.Service; - -@Service -public class TaxonomyService { - - public String getUriForTaxonId(int taxonId) { - return "http://purl.obolibrary.org/obo/NCBITaxon_" + taxonId; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import org.springframework.stereotype.Service; + +@Service +public class TaxonomyService { + + public String getUriForTaxonId(int taxonId) { + return "http://purl.obolibrary.org/obo/NCBITaxon_" + taxonId; + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/XmlUtils.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/XmlUtils.java similarity index 96% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/XmlUtils.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/XmlUtils.java index f90770933..4296c4b91 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/service/XmlUtils.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/XmlUtils.java @@ -1,156 +1,156 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import java.io.*; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMResult; -import org.custommonkey.xmlunit.Diff; -import org.custommonkey.xmlunit.XMLUnit; -import org.dom4j.Document; -import org.dom4j.DocumentException; -import org.dom4j.io.DocumentSource; -import org.dom4j.io.SAXReader; -import org.dom4j.io.XMLWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -@Service -public class XmlUtils { - private final Logger log = LoggerFactory.getLogger(getClass()); - - private final TransformerFactory tf = TransformerFactory.newInstance(); - - static { - XMLUnit.setIgnoreAttributeOrder(true); - XMLUnit.setIgnoreWhitespace(true); - } - - public Document getDocument(final File xmlFile) throws FileNotFoundException, DocumentException { - return getDocument(new BufferedReader(new FileReader(xmlFile))); - } - - public Document getDocument(final String xmlString) throws DocumentException { - - Reader r = null; - Document doc = null; - try { - r = new StringReader(xmlString); - doc = getDocument(r); - } finally { - if (r != null) { - try { - r.close(); - } catch (final IOException e) { - // do nothing - } - } - } - return doc; - } - - private Document getDocument(final Reader r) throws DocumentException { - SAXReader reader = null; // readerQueue.poll(); - if (reader == null) { - reader = new SAXReader(); - } - - // now do actual parsing - Document xml = null; - - xml = reader.read(r); - // return the reader back to the queue - // reader.resetHandlers(); - // readerQueue.add(reader); - - return xml; - } - - public String stripNonValidXMLCharacters(final String in) { - // from - // http://blog.mark-mclaren.info/2007/02/invalid-xml-characters-when-valid-utf8_5873.html - - if (in == null) { - return null; - } - - final StringBuffer out = new StringBuffer(); // Used to hold the output. - char current; // Used to reference the current character. - - for (int i = 0; i < in.length(); i++) { - current = in.charAt(i); // NOTE: No IndexOutOfBoundsException caught here; it should not - // happen. - if ((current == 0x9) - || (current == 0xA) - || (current == 0xD) - || ((current >= 0x20) && (current <= 0xD7FF)) - || ((current >= 0xE000) && (current <= 0xFFFD)) - || ((current >= 0x10000) && (current <= 0x10FFFF))) { - out.append(current); - } - } - return out.toString(); - } - - private org.w3c.dom.Document convertDocument(final Document orig) throws TransformerException { - final Transformer t = tf.newTransformer(); - final DOMResult result = new DOMResult(); - t.transform(new DocumentSource(orig), result); - return (org.w3c.dom.Document) result.getNode(); - } - - public boolean isSameXML(final Document docA, final Document docB) throws TransformerException { - - // some XMLUnit config is done statically at creation - org.w3c.dom.Document docw3cA = null; - org.w3c.dom.Document docw3cB = null; - docw3cA = convertDocument(docA); - docw3cB = convertDocument(docB); - - final Diff diff = new Diff(docw3cA, docw3cB); - return !diff.similar(); - } - - public void writeDocumentToFile(final Document document, final File outFile) throws IOException { - - OutputStream os = null; - XMLWriter writer = null; - try { - os = new BufferedOutputStream(new FileOutputStream(outFile)); - // this pretty printing is messing up comparisons by trimming whitespace WITHIN an - // element - // OutputFormat format = OutputFormat.createPrettyPrint(); - // XMLWriter writer = new XMLWriter(os, format); - writer = new XMLWriter(os); - writer.write(document); - writer.flush(); - } finally { - if (os != null) { - try { - os.close(); - } catch (final IOException e) { - // do nothing - } - } - - if (writer != null) { - try { - writer.close(); - } catch (final IOException e) { - // do nothing - } - } - } - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import java.io.*; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMResult; +import org.custommonkey.xmlunit.Diff; +import org.custommonkey.xmlunit.XMLUnit; +import org.dom4j.Document; +import org.dom4j.DocumentException; +import org.dom4j.io.DocumentSource; +import org.dom4j.io.SAXReader; +import org.dom4j.io.XMLWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +@Service +public class XmlUtils { + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final TransformerFactory tf = TransformerFactory.newInstance(); + + static { + XMLUnit.setIgnoreAttributeOrder(true); + XMLUnit.setIgnoreWhitespace(true); + } + + public Document getDocument(final File xmlFile) throws FileNotFoundException, DocumentException { + return getDocument(new BufferedReader(new FileReader(xmlFile))); + } + + public Document getDocument(final String xmlString) throws DocumentException { + + Reader r = null; + Document doc = null; + try { + r = new StringReader(xmlString); + doc = getDocument(r); + } finally { + if (r != null) { + try { + r.close(); + } catch (final IOException e) { + // do nothing + } + } + } + return doc; + } + + private Document getDocument(final Reader r) throws DocumentException { + SAXReader reader = null; // readerQueue.poll(); + if (reader == null) { + reader = new SAXReader(); + } + + // now do actual parsing + Document xml = null; + + xml = reader.read(r); + // return the reader back to the queue + // reader.resetHandlers(); + // readerQueue.add(reader); + + return xml; + } + + public String stripNonValidXMLCharacters(final String in) { + // from + // http://blog.mark-mclaren.info/2007/02/invalid-xml-characters-when-valid-utf8_5873.html + + if (in == null) { + return null; + } + + final StringBuffer out = new StringBuffer(); // Used to hold the output. + char current; // Used to reference the current character. + + for (int i = 0; i < in.length(); i++) { + current = in.charAt(i); // NOTE: No IndexOutOfBoundsException caught here; it should not + // happen. + if ((current == 0x9) + || (current == 0xA) + || (current == 0xD) + || ((current >= 0x20) && (current <= 0xD7FF)) + || ((current >= 0xE000) && (current <= 0xFFFD)) + || ((current >= 0x10000) && (current <= 0x10FFFF))) { + out.append(current); + } + } + return out.toString(); + } + + private org.w3c.dom.Document convertDocument(final Document orig) throws TransformerException { + final Transformer t = tf.newTransformer(); + final DOMResult result = new DOMResult(); + t.transform(new DocumentSource(orig), result); + return (org.w3c.dom.Document) result.getNode(); + } + + public boolean isSameXML(final Document docA, final Document docB) throws TransformerException { + + // some XMLUnit config is done statically at creation + org.w3c.dom.Document docw3cA = null; + org.w3c.dom.Document docw3cB = null; + docw3cA = convertDocument(docA); + docw3cB = convertDocument(docB); + + final Diff diff = new Diff(docw3cA, docw3cB); + return !diff.similar(); + } + + public void writeDocumentToFile(final Document document, final File outFile) throws IOException { + + OutputStream os = null; + XMLWriter writer = null; + try { + os = new BufferedOutputStream(new FileOutputStream(outFile)); + // this pretty printing is messing up comparisons by trimming whitespace WITHIN an + // element + // OutputFormat format = OutputFormat.createPrettyPrint(); + // XMLWriter writer = new XMLWriter(os, format); + writer = new XMLWriter(os); + writer.write(document); + writer.flush(); + } finally { + if (os != null) { + try { + os.close(); + } catch (final IOException e) { + // do nothing + } + } + + if (writer != null) { + try { + writer.close(); + } catch (final IOException e) { + // do nothing + } + } + } + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUniqueIdentifierGenerator.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUniqueIdentifierGenerator.java similarity index 97% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUniqueIdentifierGenerator.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUniqueIdentifierGenerator.java index a17a9ab1e..999d0a53e 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUniqueIdentifierGenerator.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUniqueIdentifierGenerator.java @@ -1,28 +1,28 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils; - -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.Date; -import uk.ac.ebi.biosamples.model.PipelineName; - -public class PipelineUniqueIdentifierGenerator { - public static String getPipelineUniqueIdentifier(final PipelineName pipelineName) { - final LocalDate localDate = new Date().toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); - - return localDate.getDayOfMonth() - + String.valueOf(localDate.getMonthValue()) - + localDate.getYear() - + "-" - + pipelineName; - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.Date; +import uk.ac.ebi.biosamples.model.PipelineName; + +public class PipelineUniqueIdentifierGenerator { + public static String getPipelineUniqueIdentifier(final PipelineName pipelineName) { + final LocalDate localDate = new Date().toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + + return localDate.getDayOfMonth() + + String.valueOf(localDate.getMonthValue()) + + localDate.getYear() + + "-" + + pipelineName; + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java similarity index 94% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java index 82f898952..534ac9f13 100644 --- a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java @@ -1,169 +1,169 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.utils; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.time.LocalDate; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Map; -import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.ApplicationArguments; -import org.springframework.boot.SpringApplication; -import org.springframework.context.ConfigurableApplicationContext; -import uk.ac.ebi.biosamples.model.PipelineName; -import uk.ac.ebi.biosamples.model.filter.AttributeFilter; -import uk.ac.ebi.biosamples.model.filter.DateRangeFilter; -import uk.ac.ebi.biosamples.model.filter.Filter; - -public class PipelineUtils { - private static final Logger log = LoggerFactory.getLogger(PipelineUtils.class); - - public static Collection getDateFilters( - final ApplicationArguments args, final String dateType) { - final Collection filters = new ArrayList<>(); - final LocalDate fromDate; - final LocalDate toDate; - final Filter fromDateFilter; - - if (args.getOptionNames().contains("from")) { - fromDate = - LocalDate.parse( - args.getOptionValues("from").iterator().next(), DateTimeFormatter.ISO_LOCAL_DATE); - } else { - fromDate = LocalDate.parse("1000-01-01", DateTimeFormatter.ISO_LOCAL_DATE); - } - - if (args.getOptionNames().contains("until")) { - toDate = - LocalDate.parse( - args.getOptionValues("until").iterator().next(), DateTimeFormatter.ISO_LOCAL_DATE); - } else { - toDate = LocalDate.parse("3000-01-01", DateTimeFormatter.ISO_LOCAL_DATE); - } - - log.info("Processing samples from " + DateTimeFormatter.ISO_LOCAL_DATE.format(fromDate)); - log.info("Processing samples to " + DateTimeFormatter.ISO_LOCAL_DATE.format(toDate)); - - if (!dateType.equals("release")) { - fromDateFilter = - new DateRangeFilter.DateRangeFilterBuilder(dateType) - .from(fromDate.atStartOfDay().toInstant(ZoneOffset.UTC)) - .until(toDate.plusDays(1).atStartOfDay().toInstant(ZoneOffset.UTC)) - .build(); - } else { - fromDateFilter = - new DateRangeFilter.DateRangeFilterBuilder(dateType) - .from(fromDate.atStartOfDay().toInstant(ZoneOffset.UTC)) - .until(fromDate.plusDays(1).atStartOfDay().toInstant(ZoneOffset.UTC)) - .build(); - } - - filters.add(fromDateFilter); - - return filters; - } - - public static void writeFailedSamplesToFile( - final Map failures, final PipelineName pipelineName) { - BufferedWriter bf = null; - final File file = new File(pipelineName + ".failed"); - - try { - bf = new BufferedWriter(new FileWriter(file)); - for (final Map.Entry entry : failures.entrySet()) { - bf.write(entry.getKey() + " : " + entry.getValue()); - bf.newLine(); - } - - bf.flush(); - } catch (final IOException e) { - log.info("Failed to write failed file ", e); - } finally { - try { - assert bf != null; - bf.close(); - } catch (final Exception e) { - log.error(e.getMessage(), e); - } - } - } - - public static void writeFailedSamplesToFile( - final Set failures, final PipelineName pipelineName) { - BufferedWriter bf = null; - final File file = new File(pipelineName + ".failed"); - - try { - bf = new BufferedWriter(new FileWriter(file)); - for (final String entry : failures) { - bf.write(entry); - bf.newLine(); - } - - bf.flush(); - } catch (final IOException e) { - log.info("Failed to write failed file ", e); - } finally { - try { - assert bf != null; - bf.close(); - } catch (final Exception e) { - log.error(e.getMessage(), e); - } - } - } - - public static void writeToFile( - final Set accessionsForPurpose, - final PipelineName pipelineName, - final String typeOfSamples) { - BufferedWriter bf = null; - final File file = new File(pipelineName + typeOfSamples + ".list"); - - try { - bf = new BufferedWriter(new FileWriter(file)); - for (final String entry : accessionsForPurpose) { - bf.write(entry); - bf.newLine(); - } - - bf.flush(); - } catch (final IOException e) { - log.info("Failed to write file ", e); - } finally { - try { - assert bf != null; - bf.close(); - } catch (final Exception e) { - log.error(e.getMessage(), e); - } - } - } - - public static void exitPipeline(final ConfigurableApplicationContext ctx) { - final int exitCode = SpringApplication.exit(ctx, () -> 0); - log.info("Exit Spring Boot"); - - System.exit(exitCode); - } - - public static Filter getAttributeFilter(final String attributeName, final String attributeValue) { - return new AttributeFilter.Builder(attributeName).withValue(attributeValue).build(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.utils; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.SpringApplication; +import org.springframework.context.ConfigurableApplicationContext; +import uk.ac.ebi.biosamples.core.model.filter.AttributeFilter; +import uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.model.PipelineName; + +public class PipelineUtils { + private static final Logger log = LoggerFactory.getLogger(PipelineUtils.class); + + public static Collection getDateFilters( + final ApplicationArguments args, final String dateType) { + final Collection filters = new ArrayList<>(); + final LocalDate fromDate; + final LocalDate toDate; + final Filter fromDateFilter; + + if (args.getOptionNames().contains("from")) { + fromDate = + LocalDate.parse( + args.getOptionValues("from").iterator().next(), DateTimeFormatter.ISO_LOCAL_DATE); + } else { + fromDate = LocalDate.parse("1000-01-01", DateTimeFormatter.ISO_LOCAL_DATE); + } + + if (args.getOptionNames().contains("until")) { + toDate = + LocalDate.parse( + args.getOptionValues("until").iterator().next(), DateTimeFormatter.ISO_LOCAL_DATE); + } else { + toDate = LocalDate.parse("3000-01-01", DateTimeFormatter.ISO_LOCAL_DATE); + } + + log.info("Processing samples from " + DateTimeFormatter.ISO_LOCAL_DATE.format(fromDate)); + log.info("Processing samples to " + DateTimeFormatter.ISO_LOCAL_DATE.format(toDate)); + + if (!dateType.equals("release")) { + fromDateFilter = + new DateRangeFilter.DateRangeFilterBuilder(dateType) + .from(fromDate.atStartOfDay().toInstant(ZoneOffset.UTC)) + .until(toDate.plusDays(1).atStartOfDay().toInstant(ZoneOffset.UTC)) + .build(); + } else { + fromDateFilter = + new DateRangeFilter.DateRangeFilterBuilder(dateType) + .from(fromDate.atStartOfDay().toInstant(ZoneOffset.UTC)) + .until(fromDate.plusDays(1).atStartOfDay().toInstant(ZoneOffset.UTC)) + .build(); + } + + filters.add(fromDateFilter); + + return filters; + } + + public static void writeFailedSamplesToFile( + final Map failures, final PipelineName pipelineName) { + BufferedWriter bf = null; + final File file = new File(pipelineName + ".failed"); + + try { + bf = new BufferedWriter(new FileWriter(file)); + for (final Map.Entry entry : failures.entrySet()) { + bf.write(entry.getKey() + " : " + entry.getValue()); + bf.newLine(); + } + + bf.flush(); + } catch (final IOException e) { + log.info("Failed to write failed file ", e); + } finally { + try { + assert bf != null; + bf.close(); + } catch (final Exception e) { + log.error(e.getMessage(), e); + } + } + } + + public static void writeFailedSamplesToFile( + final Set failures, final PipelineName pipelineName) { + BufferedWriter bf = null; + final File file = new File(pipelineName + ".failed"); + + try { + bf = new BufferedWriter(new FileWriter(file)); + for (final String entry : failures) { + bf.write(entry); + bf.newLine(); + } + + bf.flush(); + } catch (final IOException e) { + log.info("Failed to write failed file ", e); + } finally { + try { + assert bf != null; + bf.close(); + } catch (final Exception e) { + log.error(e.getMessage(), e); + } + } + } + + public static void writeToFile( + final Set accessionsForPurpose, + final PipelineName pipelineName, + final String typeOfSamples) { + BufferedWriter bf = null; + final File file = new File(pipelineName + typeOfSamples + ".list"); + + try { + bf = new BufferedWriter(new FileWriter(file)); + for (final String entry : accessionsForPurpose) { + bf.write(entry); + bf.newLine(); + } + + bf.flush(); + } catch (final IOException e) { + log.info("Failed to write file ", e); + } finally { + try { + assert bf != null; + bf.close(); + } catch (final Exception e) { + log.error(e.getMessage(), e); + } + } + } + + public static void exitPipeline(final ConfigurableApplicationContext ctx) { + final int exitCode = SpringApplication.exit(ctx, () -> 0); + log.info("Exit Spring Boot"); + + System.exit(exitCode); + } + + public static Filter getAttributeFilter(final String attributeName, final String attributeValue) { + return new AttributeFilter.Builder(attributeName).withValue(attributeValue).build(); + } +} diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/utils/XmlFragmenter.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/XmlFragmenter.java similarity index 100% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/utils/XmlFragmenter.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/XmlFragmenter.java diff --git a/utils/pipeline/src/main/java/uk/ac/ebi/biosamples/utils/XmlPathBuilder.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/XmlPathBuilder.java similarity index 100% rename from utils/pipeline/src/main/java/uk/ac/ebi/biosamples/utils/XmlPathBuilder.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/XmlPathBuilder.java diff --git a/utils/pipeline/src/test/java/uk/ac/ebi/biosamples/utils/XmlPathBuilderTest.java b/pipelines/common/src/test/java/uk/ac/ebi/biosamples/utils/XmlPathBuilderTest.java similarity index 82% rename from utils/pipeline/src/test/java/uk/ac/ebi/biosamples/utils/XmlPathBuilderTest.java rename to pipelines/common/src/test/java/uk/ac/ebi/biosamples/utils/XmlPathBuilderTest.java index 2afd6fac9..b8d776309 100644 --- a/utils/pipeline/src/test/java/uk/ac/ebi/biosamples/utils/XmlPathBuilderTest.java +++ b/pipelines/common/src/test/java/uk/ac/ebi/biosamples/utils/XmlPathBuilderTest.java @@ -10,6 +10,17 @@ */ package uk.ac.ebi.biosamples.utils; +/* + * Copyright 2021 EMBL - European Bioinformatics Institute + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + import org.dom4j.Document; import org.dom4j.DocumentHelper; import org.dom4j.Element; diff --git a/pipelines/copydown/pom.xml b/pipelines/copydown/pom.xml index 450b8a172..b15bae809 100644 --- a/pipelines/copydown/pom.xml +++ b/pipelines/copydown/pom.xml @@ -8,7 +8,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ @@ -19,18 +19,13 @@ uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT org.springframework.hateoas @@ -39,7 +34,6 @@ - diff --git a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java index e334311e3..3280aa535 100644 --- a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java +++ b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java @@ -26,13 +26,13 @@ import uk.ac.ebi.biosamples.PipelineResult; import uk.ac.ebi.biosamples.PipelinesProperties; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.PipelineAnalytics; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.PipelineAnalytics; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; -import uk.ac.ebi.biosamples.utils.AdaptiveThreadPoolExecutor; import uk.ac.ebi.biosamples.utils.PipelineUtils; -import uk.ac.ebi.biosamples.utils.ThreadUtils; +import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @Component public class CopydownApplicationRunner implements ApplicationRunner { diff --git a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/SampleCopydownCallable.java b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/SampleCopydownCallable.java index 1f4447f3a..0c0559953 100644 --- a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/SampleCopydownCallable.java +++ b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/SampleCopydownCallable.java @@ -18,7 +18,7 @@ import org.slf4j.LoggerFactory; import uk.ac.ebi.biosamples.PipelineResult; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.*; +import uk.ac.ebi.biosamples.core.model.*; public class SampleCopydownCallable implements Callable { private static final Logger LOG = LoggerFactory.getLogger(SampleCopydownCallable.class); diff --git a/pipelines/curami/pom.xml b/pipelines/curami/pom.xml index 3afdba02c..c44712654 100644 --- a/pipelines/curami/pom.xml +++ b/pipelines/curami/pom.xml @@ -13,7 +13,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ @@ -24,28 +24,13 @@ uk.ac.ebi.biosamples - models-mongo - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-ols - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT org.springframework.hateoas diff --git a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java index cf961ce9d..1885cdb1a 100644 --- a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java +++ b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java @@ -29,16 +29,16 @@ import uk.ac.ebi.biosamples.PipelineResult; import uk.ac.ebi.biosamples.PipelinesProperties; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.PipelineAnalytics; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SampleAnalytics; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.PipelineAnalytics; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SampleAnalytics; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.mongo.model.MongoCurationRule; import uk.ac.ebi.biosamples.mongo.repository.MongoCurationRuleRepository; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; -import uk.ac.ebi.biosamples.utils.AdaptiveThreadPoolExecutor; import uk.ac.ebi.biosamples.utils.PipelineUtils; -import uk.ac.ebi.biosamples.utils.ThreadUtils; +import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @Component public class CuramiApplicationRunner implements ApplicationRunner { diff --git a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/SampleCuramiCallable.java b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/SampleCuramiCallable.java index ad8f9101f..003bf7493 100644 --- a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/SampleCuramiCallable.java +++ b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/SampleCuramiCallable.java @@ -20,9 +20,9 @@ import org.slf4j.LoggerFactory; import uk.ac.ebi.biosamples.PipelineResult; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.Sample; public class SampleCuramiCallable implements Callable { private static final Logger LOG = LoggerFactory.getLogger(SampleCuramiCallable.class); diff --git a/pipelines/curation/pom.xml b/pipelines/curation/pom.xml index 18e7def74..4f98b5e2b 100644 --- a/pipelines/curation/pom.xml +++ b/pipelines/curation/pom.xml @@ -8,7 +8,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ @@ -19,18 +19,13 @@ uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-ols - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT org.springframework.hateoas @@ -47,14 +42,8 @@ 3.7 test - - uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT - - diff --git a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java index 5d3d0948d..484cb62cd 100644 --- a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java +++ b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java @@ -26,16 +26,16 @@ import uk.ac.ebi.biosamples.PipelineResult; import uk.ac.ebi.biosamples.PipelinesProperties; import uk.ac.ebi.biosamples.client.BioSamplesClient; +import uk.ac.ebi.biosamples.core.model.PipelineAnalytics; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.service.CurationApplicationService; import uk.ac.ebi.biosamples.curation.service.IriUrlValidatorService; -import uk.ac.ebi.biosamples.model.PipelineAnalytics; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.Filter; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; -import uk.ac.ebi.biosamples.ols.OlsProcessor; -import uk.ac.ebi.biosamples.service.CurationApplicationService; -import uk.ac.ebi.biosamples.utils.AdaptiveThreadPoolExecutor; import uk.ac.ebi.biosamples.utils.PipelineUtils; -import uk.ac.ebi.biosamples.utils.ThreadUtils; +import uk.ac.ebi.biosamples.utils.ols.OlsProcessor; +import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @Component public class CurationApplicationRunner implements ApplicationRunner { diff --git a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/SampleCurationCallable.java b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/SampleCurationCallable.java index 6bb56457e..b5b954dcd 100644 --- a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/SampleCurationCallable.java +++ b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/SampleCurationCallable.java @@ -17,12 +17,12 @@ import org.slf4j.LoggerFactory; import uk.ac.ebi.biosamples.PipelineResult; import uk.ac.ebi.biosamples.client.BioSamplesClient; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.service.CurationApplicationService; import uk.ac.ebi.biosamples.curation.service.IriUrlValidatorService; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.ols.OlsProcessor; -import uk.ac.ebi.biosamples.service.CurationApplicationService; +import uk.ac.ebi.biosamples.utils.ols.OlsProcessor; public class SampleCurationCallable implements Callable { private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/MockBioSamplesClient.java b/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/MockBioSamplesClient.java index 027c07dd9..4df1c7e29 100644 --- a/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/MockBioSamplesClient.java +++ b/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/MockBioSamplesClient.java @@ -25,9 +25,9 @@ import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.service.WebinAuthClientService; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.CurationLink; -import uk.ac.ebi.biosamples.service.SampleValidator; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.CurationLink; +import uk.ac.ebi.biosamples.core.service.SampleValidator; public class MockBioSamplesClient extends BioSamplesClient { private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/NonSavingApplication.java b/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/NonSavingApplication.java index 55f524dd7..f5f7328ea 100644 --- a/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/NonSavingApplication.java +++ b/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/NonSavingApplication.java @@ -16,7 +16,7 @@ import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.service.WebinAuthClientService; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.service.SampleValidator; +import uk.ac.ebi.biosamples.core.service.SampleValidator; public class NonSavingApplication extends Application { @Bean diff --git a/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/SampleCurationCallableTest.java b/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/SampleCurationCallableTest.java index f528668ff..f25a9a52f 100644 --- a/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/SampleCurationCallableTest.java +++ b/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/SampleCurationCallableTest.java @@ -36,11 +36,11 @@ import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.service.CurationApplicationService; import uk.ac.ebi.biosamples.curation.service.IriUrlValidatorService; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.ols.OlsProcessor; -import uk.ac.ebi.biosamples.service.CurationApplicationService; +import uk.ac.ebi.biosamples.utils.ols.OlsProcessor; @RunWith(SpringRunner.class) @SpringBootTest( diff --git a/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/TestApplication.java b/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/TestApplication.java index cd7e6fcd7..b313bc348 100644 --- a/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/TestApplication.java +++ b/pipelines/curation/src/test/java/uk/ac/ebi/biosamples/curation/TestApplication.java @@ -27,9 +27,9 @@ import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.service.WebinAuthClientService; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.ols.OlsProcessor; -import uk.ac.ebi.biosamples.service.CurationApplicationService; -import uk.ac.ebi.biosamples.service.SampleValidator; +import uk.ac.ebi.biosamples.core.service.CurationApplicationService; +import uk.ac.ebi.biosamples.core.service.SampleValidator; +import uk.ac.ebi.biosamples.utils.ols.OlsProcessor; @Configuration public class TestApplication { diff --git a/pipelines/ena/pom.xml b/pipelines/ena/pom.xml index 24964e3ad..cbc018af4 100644 --- a/pipelines/ena/pom.xml +++ b/pipelines/ena/pom.xml @@ -8,30 +8,20 @@ uk.ac.ebi.biosamples biosamples - 5.3.11-SNAPSHOT + 5.3.10-SNAPSHOT ../../ uk.ac.ebi.biosamples - utils-pipeline - 5.3.11-SNAPSHOT + pipelines-common + 5.3.10-SNAPSHOT uk.ac.ebi.biosamples - utils-thread - 5.3.11-SNAPSHOT - - - uk.ac.ebi.biosamples - models-mongo - 5.3.11-SNAPSHOT - - - uk.ac.ebi.biosamples - models-core - 5.3.11-SNAPSHOT + core + 5.3.10-SNAPSHOT org.springframework.hateoas diff --git a/pipelines/ena/src/main/java/uk/ac/ebi/biosamples/ena/EnaImportRunner.java b/pipelines/ena/src/main/java/uk/ac/ebi/biosamples/ena/EnaImportRunner.java index cf66e70d2..4d83c24e3 100644 --- a/pipelines/ena/src/main/java/uk/ac/ebi/biosamples/ena/EnaImportRunner.java +++ b/pipelines/ena/src/main/java/uk/ac/ebi/biosamples/ena/EnaImportRunner.java @@ -132,7 +132,7 @@ public void run(final ApplicationArguments args) throws Exception { // importSuppressedAndKilled); // Import ENA samples - // importEraSamples(fromDate, toDate); + importEraSamples(fromDate, toDate); // Import ERA and BSD authority samples from file /*if (filePath != null) { @@ -153,7 +153,7 @@ public void run(final ApplicationArguments args) throws Exception { }*/ // Sync BSD authority samples from ERAPRO - importEraBsdAuthoritySamples(fromDate, toDate); + // importEraBsdAuthoritySamples(fromDate, toDate); // syncBsdEnaSampleStatusFromFile_BsdAuthoritySamples(filePath); } catch (final Exception e) { log.error("Pipeline failed to finish successfully", e); diff --git a/pipelines/export/pom.xml b/pipelines/export/pom.xml index 6a4aaa206..47b811e1a 100644 --- a/pipelines/export/pom.xml +++ b/pipelines/export/pom.xml @@ -8,25 +8,20 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT org.springframework.hateoas diff --git a/pipelines/export/src/main/java/uk/ac/ebi/biosamples/export/ExportRunner.java b/pipelines/export/src/main/java/uk/ac/ebi/biosamples/export/ExportRunner.java index f95808e68..ceaf83c66 100644 --- a/pipelines/export/src/main/java/uk/ac/ebi/biosamples/export/ExportRunner.java +++ b/pipelines/export/src/main/java/uk/ac/ebi/biosamples/export/ExportRunner.java @@ -23,7 +23,7 @@ import org.springframework.hateoas.EntityModel; import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; @Component public class ExportRunner implements ApplicationRunner { diff --git a/pipelines/ncbi-ena-link/pom.xml b/pipelines/ncbi-ena-link/pom.xml index 5db19afdc..4998d3cae 100644 --- a/pipelines/ncbi-ena-link/pom.xml +++ b/pipelines/ncbi-ena-link/pom.xml @@ -8,30 +8,20 @@ uk.ac.ebi.biosamples biosamples - 5.3.11-SNAPSHOT + 5.3.10-SNAPSHOT ../../ uk.ac.ebi.biosamples - utils-pipeline - 5.3.11-SNAPSHOT + pipelines-common + 5.3.10-SNAPSHOT uk.ac.ebi.biosamples - utils-thread - 5.3.11-SNAPSHOT - - - uk.ac.ebi.biosamples - models-mongo - 5.3.11-SNAPSHOT - - - uk.ac.ebi.biosamples - models-mongo - 5.3.11-SNAPSHOT + core + 5.3.10-SNAPSHOT diff --git a/pipelines/ncbi-ena-link/src/main/java/uk/ac/ebi/biosamples/ena/NcbiEnaLinkCallable.java b/pipelines/ncbi-ena-link/src/main/java/uk/ac/ebi/biosamples/ena/NcbiEnaLinkCallable.java index feec2636b..d9536991c 100644 --- a/pipelines/ncbi-ena-link/src/main/java/uk/ac/ebi/biosamples/ena/NcbiEnaLinkCallable.java +++ b/pipelines/ncbi-ena-link/src/main/java/uk/ac/ebi/biosamples/ena/NcbiEnaLinkCallable.java @@ -10,19 +10,15 @@ */ package uk.ac.ebi.biosamples.ena; -import java.util.List; import java.util.Optional; import java.util.concurrent.Callable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.hateoas.EntityModel; -import uk.ac.ebi.biosamples.BioSamplesConstants; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.service.EnaSampleToBioSampleConversionService; import uk.ac.ebi.biosamples.service.EraProDao; -import uk.ac.ebi.biosamples.service.EraproSample; public class NcbiEnaLinkCallable implements Callable { private static final int MAX_RETRIES = 5; // Maximum number of retries for persistence operation @@ -61,12 +57,12 @@ public Void call() { log.info("NCBI sample doesn't exist in BioSamples " + accession + " fetching from ERAPRO"); // Enrich the sample from ERA PRO - final Sample sample = enaSampleToBioSampleConversionService.enrichSample(accession); + final Sample sample = enaSampleToBioSampleConversionService.enrichSample(accession, true); // Attempt to persist the enriched sample with retries submitRetry(success, sample); } else { - log.info("NCBI sample exists " + accession + " verifying SRA accession"); + /*log.info("NCBI sample exists " + accession + " verifying SRA accession"); // Get the sample content final Sample sample = optionalSampleEntityModel.get().getContent(); @@ -92,7 +88,7 @@ public Void call() { submitRetry(success, modifiedSample); } - } + }*/ } } catch (final Exception e) { NcbiEnaLinkRunner.failures.put(accession, e.getMessage()); diff --git a/pipelines/ncbi-ena-link/src/main/java/uk/ac/ebi/biosamples/ena/NcbiEnaLinkRunner.java b/pipelines/ncbi-ena-link/src/main/java/uk/ac/ebi/biosamples/ena/NcbiEnaLinkRunner.java index c7f86c180..19ff23ac0 100644 --- a/pipelines/ncbi-ena-link/src/main/java/uk/ac/ebi/biosamples/ena/NcbiEnaLinkRunner.java +++ b/pipelines/ncbi-ena-link/src/main/java/uk/ac/ebi/biosamples/ena/NcbiEnaLinkRunner.java @@ -30,10 +30,10 @@ import uk.ac.ebi.biosamples.mongo.util.PipelineCompletionStatus; import uk.ac.ebi.biosamples.service.EraProDao; import uk.ac.ebi.biosamples.service.SampleCallbackResult; -import uk.ac.ebi.biosamples.utils.AdaptiveThreadPoolExecutor; import uk.ac.ebi.biosamples.utils.PipelineUniqueIdentifierGenerator; import uk.ac.ebi.biosamples.utils.PipelineUtils; -import uk.ac.ebi.biosamples.utils.ThreadUtils; +import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @Component @ConditionalOnProperty( diff --git a/pipelines/ncbi/pom.xml b/pipelines/ncbi/pom.xml index 7e279f323..abb3eef95 100644 --- a/pipelines/ncbi/pom.xml +++ b/pipelines/ncbi/pom.xml @@ -8,25 +8,20 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - models-mongo - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT org.springframework.hateoas diff --git a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/Ncbi.java b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/Ncbi.java index 08b732323..c6fdff8e6 100644 --- a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/Ncbi.java +++ b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/Ncbi.java @@ -39,17 +39,17 @@ import org.xml.sax.SAXException; import uk.ac.ebi.biosamples.PipelinesProperties; import uk.ac.ebi.biosamples.client.BioSamplesClient; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; +import uk.ac.ebi.biosamples.core.service.FilterBuilder; import uk.ac.ebi.biosamples.model.PipelineName; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; import uk.ac.ebi.biosamples.mongo.model.MongoPipeline; import uk.ac.ebi.biosamples.mongo.repository.MongoPipelineRepository; import uk.ac.ebi.biosamples.mongo.util.PipelineCompletionStatus; -import uk.ac.ebi.biosamples.service.FilterBuilder; -import uk.ac.ebi.biosamples.utils.AdaptiveThreadPoolExecutor; import uk.ac.ebi.biosamples.utils.PipelineUniqueIdentifierGenerator; -import uk.ac.ebi.biosamples.utils.ThreadUtils; import uk.ac.ebi.biosamples.utils.XmlFragmenter; +import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @Component @Profile("!test") diff --git a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/NcbiElementCallable.java b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/NcbiElementCallable.java index c5333e4d3..4e55a578d 100644 --- a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/NcbiElementCallable.java +++ b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/NcbiElementCallable.java @@ -19,12 +19,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.ExternalReference; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SubmittedViaType; -import uk.ac.ebi.biosamples.model.structured.StructuredData; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.ExternalReference; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SubmittedViaType; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; import uk.ac.ebi.biosamples.ncbi.service.NcbiSampleConversionService; public class NcbiElementCallable implements Callable { diff --git a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/NcbiElementCallableFactory.java b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/NcbiElementCallableFactory.java index 9c2414c94..f8cb2d52d 100644 --- a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/NcbiElementCallableFactory.java +++ b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/NcbiElementCallableFactory.java @@ -17,7 +17,7 @@ import org.springframework.stereotype.Service; import uk.ac.ebi.biosamples.PipelinesProperties; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; import uk.ac.ebi.biosamples.ncbi.service.NcbiSampleConversionService; @Service diff --git a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/NcbiFragmentCallback.java b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/NcbiFragmentCallback.java index 56614b6f4..3262ec406 100644 --- a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/NcbiFragmentCallback.java +++ b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/NcbiFragmentCallback.java @@ -23,9 +23,9 @@ import org.springframework.stereotype.Component; import org.xml.sax.Attributes; import uk.ac.ebi.biosamples.PipelinesProperties; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; -import uk.ac.ebi.biosamples.utils.ThreadUtils; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; import uk.ac.ebi.biosamples.utils.XmlFragmenter.ElementCallback; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @Component public class NcbiFragmentCallback implements ElementCallback { diff --git a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/service/NcbiAmrConversionService.java b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/service/NcbiAmrConversionService.java index a5615ac32..1211a5e2b 100644 --- a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/service/NcbiAmrConversionService.java +++ b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/service/NcbiAmrConversionService.java @@ -15,7 +15,7 @@ import java.util.stream.Collectors; import org.dom4j.Element; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.structured.StructuredDataEntry; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataEntry; import uk.ac.ebi.biosamples.utils.XmlPathBuilder; @Service diff --git a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/service/NcbiSampleConversionService.java b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/service/NcbiSampleConversionService.java index a86b81d82..997d31145 100644 --- a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/service/NcbiSampleConversionService.java +++ b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/ncbi/service/NcbiSampleConversionService.java @@ -19,9 +19,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.*; -import uk.ac.ebi.biosamples.model.structured.StructuredDataEntry; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; +import uk.ac.ebi.biosamples.core.model.*; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataEntry; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; import uk.ac.ebi.biosamples.ncbi.service.NcbiAmrConversionService.AmrParsingException; import uk.ac.ebi.biosamples.service.TaxonomyService; import uk.ac.ebi.biosamples.utils.XmlPathBuilder; diff --git a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/NcbiAmrConvertionTests.java b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/NcbiAmrConvertionTests.java index 0b2282fba..ed7f263e9 100644 --- a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/NcbiAmrConvertionTests.java +++ b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/NcbiAmrConvertionTests.java @@ -16,9 +16,9 @@ import org.dom4j.Element; import org.junit.Before; import org.junit.Test; -import uk.ac.ebi.biosamples.model.structured.StructuredDataEntry; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; -import uk.ac.ebi.biosamples.model.structured.StructuredDataType; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataEntry; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataType; import uk.ac.ebi.biosamples.ncbi.service.NcbiAmrConversionService; import uk.ac.ebi.biosamples.ncbi.service.NcbiSampleConversionService; import uk.ac.ebi.biosamples.service.TaxonomyService; diff --git a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/NcbiBaseConverterTests.java b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/NcbiBaseConverterTests.java index 8fac063d7..b2cbd5f13 100644 --- a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/NcbiBaseConverterTests.java +++ b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/NcbiBaseConverterTests.java @@ -27,9 +27,9 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.test.context.junit4.SpringRunner; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; import uk.ac.ebi.biosamples.ncbi.service.NcbiSampleConversionService; import uk.ac.ebi.biosamples.service.TaxonomyService; diff --git a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/NcbiNotSuppressedBaseConverterTests.java b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/NcbiNotSuppressedBaseConverterTests.java index 2e018c8b7..6dd1a2318 100644 --- a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/NcbiNotSuppressedBaseConverterTests.java +++ b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/NcbiNotSuppressedBaseConverterTests.java @@ -19,8 +19,8 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.test.context.junit4.SpringRunner; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.ncbi.service.NcbiSampleConversionService; import uk.ac.ebi.biosamples.service.TaxonomyService; diff --git a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/TestApplication.java b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/TestApplication.java index 101e7698c..d13dbdde7 100644 --- a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/TestApplication.java +++ b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/TestApplication.java @@ -17,8 +17,8 @@ import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.service.WebinAuthClientService; import uk.ac.ebi.biosamples.client.utils.ClientProperties; +import uk.ac.ebi.biosamples.core.service.SampleValidator; import uk.ac.ebi.biosamples.ncbi.MockBioSamplesClient; -import uk.ac.ebi.biosamples.service.SampleValidator; @Configuration public class TestApplication { diff --git a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/ncbi/MockBioSamplesClient.java b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/ncbi/MockBioSamplesClient.java index 2d2915abe..a8b7ee3e5 100644 --- a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/ncbi/MockBioSamplesClient.java +++ b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/ncbi/MockBioSamplesClient.java @@ -25,8 +25,8 @@ import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.service.WebinAuthClientService; import uk.ac.ebi.biosamples.client.utils.ClientProperties; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.service.SampleValidator; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.service.SampleValidator; public class MockBioSamplesClient extends BioSamplesClient { diff --git a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/ncbi/NcbiDatesTests.java b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/ncbi/NcbiDatesTests.java index e3da1602e..a9c89f38e 100644 --- a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/ncbi/NcbiDatesTests.java +++ b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/ncbi/NcbiDatesTests.java @@ -27,8 +27,8 @@ import org.junit.runner.RunWith; import org.springframework.test.context.junit4.SpringRunner; import uk.ac.ebi.biosamples.NcbiTestsService; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.ncbi.service.NcbiSampleConversionService; import uk.ac.ebi.biosamples.service.TaxonomyService; diff --git a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/ncbi/NcbiElementCallableTest.java b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/ncbi/NcbiElementCallableTest.java index a2ba64e1a..a8060394c 100644 --- a/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/ncbi/NcbiElementCallableTest.java +++ b/pipelines/ncbi/src/test/java/uk/ac/ebi/biosamples/ncbi/NcbiElementCallableTest.java @@ -27,8 +27,8 @@ import org.springframework.test.context.junit4.SpringRunner; import uk.ac.ebi.biosamples.TestApplication; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.ncbi.service.NcbiSampleConversionService; import uk.ac.ebi.biosamples.service.TaxonomyService; import uk.ac.ebi.biosamples.utils.XmlPathBuilder; diff --git a/pipelines/neoexport/pom.xml b/pipelines/neoexport/pom.xml index 745f0bdf9..59bee2f72 100644 --- a/pipelines/neoexport/pom.xml +++ b/pipelines/neoexport/pom.xml @@ -13,20 +13,15 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - models-neo4j - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT com.fasterxml.jackson.dataformat diff --git a/pipelines/neoexport/src/main/java/uk/ac/ebi/biosamples/neoexport/NeoCsvExporter.java b/pipelines/neoexport/src/main/java/uk/ac/ebi/biosamples/neoexport/NeoCsvExporter.java index 0b24b3658..4cd77259a 100644 --- a/pipelines/neoexport/src/main/java/uk/ac/ebi/biosamples/neoexport/NeoCsvExporter.java +++ b/pipelines/neoexport/src/main/java/uk/ac/ebi/biosamples/neoexport/NeoCsvExporter.java @@ -23,7 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.neo4j.model.NeoExternalEntity; import uk.ac.ebi.biosamples.neo4j.model.NeoRelationship; import uk.ac.ebi.biosamples.neo4j.model.NeoSample; diff --git a/pipelines/neoexport/src/main/java/uk/ac/ebi/biosamples/neoexport/NeoExportCallable.java b/pipelines/neoexport/src/main/java/uk/ac/ebi/biosamples/neoexport/NeoExportCallable.java index 765cc6826..40aa1d800 100644 --- a/pipelines/neoexport/src/main/java/uk/ac/ebi/biosamples/neoexport/NeoExportCallable.java +++ b/pipelines/neoexport/src/main/java/uk/ac/ebi/biosamples/neoexport/NeoExportCallable.java @@ -15,7 +15,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import uk.ac.ebi.biosamples.PipelineResult; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.neo4j.model.NeoSample; import uk.ac.ebi.biosamples.neo4j.repo.NeoSampleRepository; diff --git a/pipelines/neoexport/src/main/java/uk/ac/ebi/biosamples/neoexport/NeoExportRunner.java b/pipelines/neoexport/src/main/java/uk/ac/ebi/biosamples/neoexport/NeoExportRunner.java index 4ae13d4d2..6fe1c72c5 100644 --- a/pipelines/neoexport/src/main/java/uk/ac/ebi/biosamples/neoexport/NeoExportRunner.java +++ b/pipelines/neoexport/src/main/java/uk/ac/ebi/biosamples/neoexport/NeoExportRunner.java @@ -26,14 +26,14 @@ import uk.ac.ebi.biosamples.PipelineResult; import uk.ac.ebi.biosamples.PipelinesProperties; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.PipelineAnalytics; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SampleAnalytics; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.PipelineAnalytics; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SampleAnalytics; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.neo4j.repo.NeoSampleRepository; -import uk.ac.ebi.biosamples.utils.AdaptiveThreadPoolExecutor; import uk.ac.ebi.biosamples.utils.PipelineUtils; -import uk.ac.ebi.biosamples.utils.ThreadUtils; +import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @Component public class NeoExportRunner implements ApplicationRunner { diff --git a/pipelines/pom.xml b/pipelines/pom.xml index dfc86f76e..30ab76c46 100644 --- a/pipelines/pom.xml +++ b/pipelines/pom.xml @@ -9,7 +9,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT @@ -18,13 +18,14 @@ embl-ebi - + + + common analytics curation zooma @@ -33,13 +34,10 @@ sample-transformation-dtol ncbi sample-release - sample-post-release-action taxonimport reindex - export - neoexport chain - + + diff --git a/pipelines/reindex/pom.xml b/pipelines/reindex/pom.xml index 50f2e0a2c..e282a5d64 100644 --- a/pipelines/reindex/pom.xml +++ b/pipelines/reindex/pom.xml @@ -8,25 +8,20 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples - messaging - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT compile diff --git a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java index 8451d4a09..9865e3ee5 100644 --- a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java +++ b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java @@ -12,8 +12,6 @@ import java.util.*; import java.util.concurrent.*; - -import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang.IncompleteArgumentException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,13 +24,15 @@ import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.util.CloseableIterator; import org.springframework.stereotype.Component; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.DateRangeFilter; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.messaging.MessagingConstants; +import uk.ac.ebi.biosamples.messaging.model.MessageContent; import uk.ac.ebi.biosamples.mongo.model.MongoSample; import uk.ac.ebi.biosamples.mongo.service.SampleReadService; import uk.ac.ebi.biosamples.utils.PipelineUtils; -import uk.ac.ebi.biosamples.utils.ThreadUtils; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; /** * This runner will get a list of accessions from mongo directly, query the API to get the latest @@ -50,17 +50,15 @@ public class ReindexRunner implements ApplicationRunner { private final AmqpTemplate amqpTemplate; private final SampleReadService sampleReadService; private final MongoOperations mongoOperations; - private final ObjectMapper objectMapper; @Autowired public ReindexRunner( final AmqpTemplate amqpTemplate, final SampleReadService sampleReadService, - final MongoOperations mongoOperations, ObjectMapper objectMapper) { + final MongoOperations mongoOperations) { this.amqpTemplate = amqpTemplate; this.sampleReadService = sampleReadService; this.mongoOperations = mongoOperations; - this.objectMapper = objectMapper; } @Override @@ -105,7 +103,7 @@ public void run(final ApplicationArguments args) throws Exception { futures.put( accession, executor.submit( - new SampleIndexingCallable(accession, sampleReadService, amqpTemplate, objectMapper))); + new SampleIndexingCallable(accession, sampleReadService, amqpTemplate))); ThreadUtils.checkFutures(futures, 1000); } @@ -123,16 +121,14 @@ private static class SampleIndexingCallable implements Callable { private final String accession; private final SampleReadService sampleReadService; private final AmqpTemplate amqpTemplate; - private final ObjectMapper objectMapper; public SampleIndexingCallable( final String accession, final SampleReadService sampleReadService, - final AmqpTemplate amqpTemplate, ObjectMapper objectMapper) { + final AmqpTemplate amqpTemplate) { this.accession = accession; this.sampleReadService = sampleReadService; this.amqpTemplate = amqpTemplate; - this.objectMapper = objectMapper; } @Override @@ -158,10 +154,9 @@ private boolean fetchSampleAndSendMessage(final boolean isRetry) { if (sampleOptional.isPresent()) { try { amqpTemplate.convertAndSend( - Messaging.REINDEXING_EXCHANGE, - Messaging.REINDEXING_QUEUE, - objectMapper.writeValueAsString(sampleOptional.get())); -// MessageContent.build(sampleOptional.get(), null, related, false)); + MessagingConstants.REINDEXING_EXCHANGE, + MessagingConstants.REINDEXING_QUEUE, + MessageContent.build(sampleOptional.get(), null, related, false)); return true; } catch (final Exception e) { diff --git a/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java b/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java index 3b9192b24..8a65f8be2 100644 --- a/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java +++ b/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java @@ -26,8 +26,8 @@ import org.springframework.boot.ApplicationArguments; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.data.util.CloseableIterator; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SampleStatus; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SampleStatus; import uk.ac.ebi.biosamples.mongo.model.MongoSample; import uk.ac.ebi.biosamples.mongo.service.SampleReadService; @@ -108,7 +108,7 @@ public MongoSample next() { .thenReturn(Optional.empty()) .thenReturn(Optional.of(sample3)); final ReindexRunner reindexRunner = - new ReindexRunner(amqpTemplate, sampleReadService, mongoOperations, new ObjectMapper()); + new ReindexRunner(amqpTemplate, sampleReadService, mongoOperations); reindexRunner.run(applicationArguments); } } diff --git a/pipelines/sample-post-release-action/pom.xml b/pipelines/sample-post-release-action/pom.xml index 3f11eae56..d83bb9625 100644 --- a/pipelines/sample-post-release-action/pom.xml +++ b/pipelines/sample-post-release-action/pom.xml @@ -8,7 +8,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ @@ -19,18 +19,13 @@ uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-ols - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT org.springframework.hateoas @@ -47,14 +42,8 @@ 3.7 test - - uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT - - diff --git a/pipelines/sample-post-release-action/src/main/java/uk/ac/ebi/biosamples/postrelease/SamplePostReleaseActionApplicationRunner.java b/pipelines/sample-post-release-action/src/main/java/uk/ac/ebi/biosamples/postrelease/SamplePostReleaseActionApplicationRunner.java index 62137c157..1dc12bfdb 100644 --- a/pipelines/sample-post-release-action/src/main/java/uk/ac/ebi/biosamples/postrelease/SamplePostReleaseActionApplicationRunner.java +++ b/pipelines/sample-post-release-action/src/main/java/uk/ac/ebi/biosamples/postrelease/SamplePostReleaseActionApplicationRunner.java @@ -24,11 +24,11 @@ import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.PipelinesProperties; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.utils.AdaptiveThreadPoolExecutor; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.utils.PipelineUtils; -import uk.ac.ebi.biosamples.utils.ThreadUtils; +import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @Component public class SamplePostReleaseActionApplicationRunner implements ApplicationRunner { diff --git a/pipelines/sample-post-release-action/src/main/java/uk/ac/ebi/biosamples/postrelease/SamplePostReleaseActionCallable.java b/pipelines/sample-post-release-action/src/main/java/uk/ac/ebi/biosamples/postrelease/SamplePostReleaseActionCallable.java index 69911ea20..010326cea 100644 --- a/pipelines/sample-post-release-action/src/main/java/uk/ac/ebi/biosamples/postrelease/SamplePostReleaseActionCallable.java +++ b/pipelines/sample-post-release-action/src/main/java/uk/ac/ebi/biosamples/postrelease/SamplePostReleaseActionCallable.java @@ -15,8 +15,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SampleStatus; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SampleStatus; public class SamplePostReleaseActionCallable implements Callable { private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/pipelines/sample-release/pom.xml b/pipelines/sample-release/pom.xml index a44f22d9d..4678de1b8 100644 --- a/pipelines/sample-release/pom.xml +++ b/pipelines/sample-release/pom.xml @@ -8,15 +8,15 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT org.springframework.hateoas diff --git a/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/samplerelease/SampleReleaseCallable.java b/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/samplerelease/SampleReleaseCallable.java index 1cc3fb3a4..574fad1da 100644 --- a/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/samplerelease/SampleReleaseCallable.java +++ b/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/samplerelease/SampleReleaseCallable.java @@ -25,10 +25,10 @@ import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.PipelinesProperties; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.ExternalReference; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SampleStatus; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.ExternalReference; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SampleStatus; public class SampleReleaseCallable implements Callable { private static final Logger log = LoggerFactory.getLogger(SampleReleaseCallable.class); diff --git a/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/samplerelease/SampleReleaseRunner.java b/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/samplerelease/SampleReleaseRunner.java index 3e3bb4123..a84540246 100644 --- a/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/samplerelease/SampleReleaseRunner.java +++ b/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/samplerelease/SampleReleaseRunner.java @@ -31,7 +31,7 @@ import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.PipelinesProperties; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.utils.ThreadUtils; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @Component public class SampleReleaseRunner implements ApplicationRunner { diff --git a/pipelines/sample-transformation-dtol/pom.xml b/pipelines/sample-transformation-dtol/pom.xml index 70b41b058..0f1d83586 100644 --- a/pipelines/sample-transformation-dtol/pom.xml +++ b/pipelines/sample-transformation-dtol/pom.xml @@ -13,7 +13,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ @@ -24,28 +24,13 @@ uk.ac.ebi.biosamples - models-mongo - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-ols - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT com.github.ben-manes.caffeine @@ -64,7 +49,6 @@ - diff --git a/pipelines/sample-transformation-dtol/src/main/java/uk/ac/ebi/biosamples/curation/TransformationApplicationRunner.java b/pipelines/sample-transformation-dtol/src/main/java/uk/ac/ebi/biosamples/curation/TransformationApplicationRunner.java index 5fbcafffe..a1d27b657 100644 --- a/pipelines/sample-transformation-dtol/src/main/java/uk/ac/ebi/biosamples/curation/TransformationApplicationRunner.java +++ b/pipelines/sample-transformation-dtol/src/main/java/uk/ac/ebi/biosamples/curation/TransformationApplicationRunner.java @@ -26,12 +26,12 @@ import uk.ac.ebi.biosamples.PipelineResult; import uk.ac.ebi.biosamples.PipelinesProperties; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.AttributeFilter; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.utils.AdaptiveThreadPoolExecutor; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.AttributeFilter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.utils.PipelineUtils; -import uk.ac.ebi.biosamples.utils.ThreadUtils; +import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @Component public class TransformationApplicationRunner implements ApplicationRunner { diff --git a/pipelines/sample-transformation-dtol/src/main/java/uk/ac/ebi/biosamples/curation/TransformationCallable.java b/pipelines/sample-transformation-dtol/src/main/java/uk/ac/ebi/biosamples/curation/TransformationCallable.java index f579f1ddc..2da6bf501 100644 --- a/pipelines/sample-transformation-dtol/src/main/java/uk/ac/ebi/biosamples/curation/TransformationCallable.java +++ b/pipelines/sample-transformation-dtol/src/main/java/uk/ac/ebi/biosamples/curation/TransformationCallable.java @@ -18,9 +18,9 @@ import org.springframework.hateoas.EntityModel; import uk.ac.ebi.biosamples.PipelineResult; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Sample; public class TransformationCallable implements Callable { private static final Logger LOG = LoggerFactory.getLogger(TransformationCallable.class); diff --git a/pipelines/taxonimport/pom.xml b/pipelines/taxonimport/pom.xml index 89dca9140..5cac53098 100644 --- a/pipelines/taxonimport/pom.xml +++ b/pipelines/taxonimport/pom.xml @@ -13,7 +13,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ @@ -24,28 +24,13 @@ uk.ac.ebi.biosamples - models-mongo - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-ols - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT org.springframework.hateoas diff --git a/pipelines/taxonimport/src/main/java/uk/ac/ebi/biosamples/curation/TaxonImportApplicationRunner.java b/pipelines/taxonimport/src/main/java/uk/ac/ebi/biosamples/curation/TaxonImportApplicationRunner.java index 8d47c0f27..69d9fd617 100644 --- a/pipelines/taxonimport/src/main/java/uk/ac/ebi/biosamples/curation/TaxonImportApplicationRunner.java +++ b/pipelines/taxonimport/src/main/java/uk/ac/ebi/biosamples/curation/TaxonImportApplicationRunner.java @@ -32,10 +32,10 @@ import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.PipelineFutureCallback; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.PipelineAnalytics; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SampleAnalytics; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.PipelineAnalytics; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SampleAnalytics; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; @Component diff --git a/pipelines/zooma/pom.xml b/pipelines/zooma/pom.xml index aa7cccde1..e8bc482b5 100644 --- a/pipelines/zooma/pom.xml +++ b/pipelines/zooma/pom.xml @@ -8,7 +8,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ @@ -19,28 +19,18 @@ uk.ac.ebi.biosamples - utils-pipeline - 5.3.12-SNAPSHOT + pipelines-common + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-ols - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT com.github.ben-manes.caffeine caffeine - - uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT - org.springframework.hateoas spring-hateoas @@ -48,7 +38,6 @@ - diff --git a/pipelines/zooma/src/main/java/uk/ac/ebi/biosamples/zooma/SampleZoomaCallable.java b/pipelines/zooma/src/main/java/uk/ac/ebi/biosamples/zooma/SampleZoomaCallable.java index 963f67cce..52edcef95 100644 --- a/pipelines/zooma/src/main/java/uk/ac/ebi/biosamples/zooma/SampleZoomaCallable.java +++ b/pipelines/zooma/src/main/java/uk/ac/ebi/biosamples/zooma/SampleZoomaCallable.java @@ -18,10 +18,10 @@ import org.slf4j.LoggerFactory; import uk.ac.ebi.biosamples.PipelineResult; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.service.CurationApplicationService; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.service.CurationApplicationService; public class SampleZoomaCallable implements Callable { private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/pipelines/zooma/src/main/java/uk/ac/ebi/biosamples/zooma/ZoomaApplicationRunner.java b/pipelines/zooma/src/main/java/uk/ac/ebi/biosamples/zooma/ZoomaApplicationRunner.java index e3433095d..03718a3ff 100644 --- a/pipelines/zooma/src/main/java/uk/ac/ebi/biosamples/zooma/ZoomaApplicationRunner.java +++ b/pipelines/zooma/src/main/java/uk/ac/ebi/biosamples/zooma/ZoomaApplicationRunner.java @@ -25,14 +25,14 @@ import uk.ac.ebi.biosamples.PipelineResult; import uk.ac.ebi.biosamples.PipelinesProperties; import uk.ac.ebi.biosamples.client.BioSamplesClient; -import uk.ac.ebi.biosamples.model.PipelineAnalytics; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.PipelineAnalytics; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.service.CurationApplicationService; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; -import uk.ac.ebi.biosamples.service.CurationApplicationService; -import uk.ac.ebi.biosamples.utils.AdaptiveThreadPoolExecutor; import uk.ac.ebi.biosamples.utils.PipelineUtils; -import uk.ac.ebi.biosamples.utils.ThreadUtils; +import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; +import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @Component public class ZoomaApplicationRunner implements ApplicationRunner { diff --git a/pom.xml b/pom.xml index 3c9afcbdf..9ec93e186 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT pom org.springframework.boot @@ -25,16 +25,13 @@ - models - messaging agents webapps pipelines integration client properties - utils - commons + core @@ -107,9 +104,13 @@ biosamples-v4 https://gitlab.ebi.ac.uk/api/v4/projects/2669/packages/maven - gitlab-maven + https://gitlab.ebi.ac.uk/api/v4/projects/1143/packages/maven + + + + gitlab-biosamples-search https://gitlab.ebi.ac.uk/api/v4/projects/5516/packages/maven @@ -123,6 +124,10 @@ spring-milestones http://repo.spring.io/milestone + + central + https://repo.maven.apache.org/maven2 + @@ -150,6 +155,7 @@ org.apache.maven.plugins maven-dependency-plugin + 3.6.0 true @@ -177,6 +183,11 @@ + + org.apache.maven.plugins + maven-deploy-plugin + 3.1.1 + com.diffplug.spotless spotless-maven-plugin diff --git a/properties/pom.xml b/properties/pom.xml index 620d64503..eb5f40d45 100644 --- a/properties/pom.xml +++ b/properties/pom.xml @@ -7,7 +7,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../ diff --git a/scripts/data/samples/sample_ERC000022.json b/scripts/data/samples/sample_ERC000022.json deleted file mode 100644 index da6808b73..000000000 --- a/scripts/data/samples/sample_ERC000022.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "name" : "ERC000022 test sample ", - "webinSubmissionAccountId" : "Webin-59287", - "taxId" : 256318, - "release" : "2021-04-20T00:32:19Z", - "characteristics" : { - "ENA first public" : [ { - "text" : "2021-04-20", - "tag" : "attribute" - } ], - "ENA last update" : [ { - "text" : "2021-04-19", - "tag" : "attribute" - } ], - "ENA-CHECKLIST" : [ { - "text" : "ERC000022", - "tag" : "attribute" - } ], - "INSDC center alias" : [ { - "text" : "Barta" - } ], - "INSDC center name" : [ { - "text" : "Barta" - } ], - "INSDC first public" : [ { - "text" : "2021-04-20T00:32:19Z" - } ], - "INSDC last update" : [ { - "text" : "2021-04-19T14:38:29Z" - } ], - "INSDC status" : [ { - "text" : "public" - } ], - "Submitter Id" : [ { - "text" : "fun.nac371", - "tag" : "Namespace:Barta" - } ], - "collection date" : [ { - "text" : "2013", - "tag" : "attribute" - } ], - "description" : [ { - "text" : "fungal ITS1" - } ], - "environment (biome)" : [ { - "text" : "spruce biome", - "tag" : "attribute" - } ], - "environment (feature)" : [ { - "text" : "spruce forest", - "tag" : "attribute" - } ], - "broad-scale environmental context" : [ { - "text" : "spruce forest", - "tag" : "attribute" - } ], - "environmental medium" : [ { - "text" : "spruce forest", - "tag" : "attribute" - } ], - "local environmental context" : [ { - "text" : "spruce forest", - "tag" : "attribute" - } ], - "environment (material)" : [ { - "text" : "soil", - "tag" : "attribute" - } ], - "geographic location (country and/or sea)" : [ { - "text" : "Czechia", - "tag" : "attribute" - } ], - "geographic location (depth)" : [ { - "text" : "0.1", - "unit" : "m", - "tag" : "attribute" - } ], - "depth" : [ { - "text" : "0.1", - "unit" : "m", - "tag" : "attribute" - } ], - "geographic location (elevation)" : [ { - "text" : "784", - "unit" : "m", - "tag" : "attribute" - } ], - "elevation" : [ { - "text" : "784", - "unit" : "m", - "tag" : "attribute" - } ], - "geographic location (latitude)" : [ { - "text" : "50.590556", - "unit" : "DD", - "tag" : "attribute" - } ], - "geographic location (longitude)" : [ { - "text" : "13.253889", - "unit" : "DD", - "tag" : "attribute" - } ], - "investigation type" : [ { - "text" : "metagenome", - "tag" : "attribute" - } ], - "organism" : [ { - "text" : "metagenome", - "ontologyTerms" : [ "http://purl.obolibrary.org/obo/NCBITaxon_256318" ] - } ], - "project name" : [ { - "text" : "Nacetin13", - "tag" : "attribute" - } ], - "sequencing method" : [ { - "text" : "Illumina MiSeq", - "ontologyTerms" : [ "http://www.ebi.ac.uk/efo/EFO_0004205" ], - "tag" : "attribute" - } ], - "soil environmental package" : [ { - "text" : "soil", - "tag" : "attribute" - } ], - "title" : [ { - "text" : "fun_371" - } ] - }, - "externalReferences" : [ { - "url" : "https://www.ebi.ac.uk/ena/browser/view/SAMEA8579965", - "duo" : [ ] - } ], - "submittedVia" : "JSON_API", - "create" : "2021-03-29T03:02:10.842Z" -} \ No newline at end of file diff --git a/scripts/data/samples/sample_ERC000023.json b/scripts/data/samples/sample_ERC000023.json deleted file mode 100644 index 257030291..000000000 --- a/scripts/data/samples/sample_ERC000023.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "name" : "ERC000023 test sample ", - "webinSubmissionAccountId" : "Webin-59287", - "taxId" : 256318, - "release" : "2021-01-03T00:29:23Z", - "characteristics" : { - "ENA-CHECKLIST" : [ { - "text" : "ERC000023", - "tag" : "attribute" - } ], - "checklist" : [ { - "text" : "ERC000023", - "tag" : "attribute" - } ], - "ENA-FIRST-PUBLIC" : [ { - "text" : "2023-01-03T00:29:23Z", - "tag" : "attribute" - } ], - "ENA-LAST-UPDATE" : [ { - "text" : "2023-01-03T00:29:23Z", - "tag" : "attribute" - } ], - "INSDC center name" : [ { - "text" : "AALBORG UNIVERSITY" - } ], - "INSDC first public" : [ { - "text" : "2023-01-03T00:29:23Z" - } ], - "INSDC last update" : [ { - "text" : "2023-01-03T00:29:23Z" - } ], - "INSDC status" : [ { - "text" : "public" - } ], - "Submitter Id" : [ { - "text" : "HQ171123-10", - "tag" : "Namespace:AALBORG UNIVERSITY" - } ], - "collection date" : [ { - "text" : "2017-02-07", - "tag" : "attribute" - } ], - "environment (biome)" : [ { - "text" : "anaerobic digester", - "tag" : "attribute" - } ], - "environment (feature)" : [ { - "text" : "anaerobic digester", - "tag" : "attribute" - } ], - "environment (material)" : [ { - "text" : "sludge", - "tag" : "attribute" - } ], - "geographic location (country and/or sea)" : [ { - "text" : "Denmark", - "tag" : "attribute" - } ], - "geographic location (latitude)" : [ { - "text" : "55.552344", - "unit" : "DD", - "tag" : "attribute" - } ], - "geographic location (longitude)" : [ { - "text" : "9.721907", - "unit" : "DD", - "tag" : "attribute" - } ], - "investigation type" : [ { - "text" : "metagenome", - "tag" : "attribute" - } ], - "organism" : [ { - "text" : "metagenome" - } ], - "project name" : [ { - "text" : "PRJEB34014", - "tag" : "attribute" - } ], - "scientific_name" : [ { - "text" : "metagenome" - } ], - "sequencing method" : [ { - "text" : "illumina", - "tag" : "attribute" - } ], - "title" : [ { - "text" : "AD-sludge-Fredericia" - } ], - "wastewater/sludge environmental package" : [ { - "text" : "wastewater/sludge", - "tag" : "attribute" - } ], - "broad-scale environmental context" : [ { - "text" : "wastewater/sludge", - "tag" : "attribute" - } ], - "local environmental context" : [ { - "text" : "wastewater/sludge", - "tag" : "attribute" - } ], - "environmental medium" : [ { - "text" : "wastewater/sludge", - "tag" : "attribute" - } ] - } -} \ No newline at end of file diff --git a/scripts/load_test_bulk_sample_submission.py b/scripts/load_test_bulk_sample_submission.py deleted file mode 100644 index 613fd252d..000000000 --- a/scripts/load_test_bulk_sample_submission.py +++ /dev/null @@ -1,190 +0,0 @@ -import copy -import json -import random -import threading -from os import listdir -from os.path import isfile, join -from queue import Queue - -import matplotlib.pyplot as plt -import requests -import sys -import time -from prettytable import PrettyTable -from tqdm import tqdm - -# to run: python load_test_bulk_sample_submission.py -user <> --password <> --sample_path <> - -WEBIN_AUTH_ENDPOINT = "https://wwwdev.ebi.ac.uk/ena/submit/webin/auth/token" -BULK_SUBMISSION_ENDPOINT = "http://wp-np2-44:8082/biosamples/v2/samples/bulk-submit" - -time_per_sample_queue = Queue() -base_samples = [] - - -def submit_samples(samples, token): - headers = {"accept": "application/json", "authorization": "Bearer " + token} - response = requests.post(BULK_SUBMISSION_ENDPOINT, headers=headers, json=samples) - response.raise_for_status() - data = None - if response.status_code == 201: - data = response.json() - - return data - - -def authenticate(username, password): - headers = {"accept": "application/json"} - data = { - "authRealms": ["ENA"], - "username": username, - "password": password - } - response = requests.post(WEBIN_AUTH_ENDPOINT, headers=headers, json=data) - return response.text - - -def load_samples_from_disk(file_path): - if len(base_samples) != 0: - return - - sample_files = [join(file_path, f) for f in listdir(file_path) if isfile(join(file_path, f))] - for s in sample_files: - try: - with open(s, 'r') as file: - base_samples.append(json.load(file)) - except Exception as e: - print(f"Error reading file {file_path}: {e}") - sys.exit(1) - - -def generate_samples(count): - samples = [] - base_sample_count = len(base_samples) - for i in range(count): - s = copy.deepcopy(base_samples[random.randint(0, base_sample_count - 1)]) - s['name'] = s['name'] + str(i) - samples.append(s) - return samples - - -def measure_bulk_submission_time(no_of_samples): - load_samples_from_disk(sample_path) - samples = generate_samples(no_of_samples) - token = authenticate(user, password) - start_time = time.time() - response = submit_samples(samples, token) - end_time = time.time() - elapsed_time_ms = (end_time - start_time) * 1000 - # print(f"Submitted {no_of_samples} samples in %.2fms" % elapsed_time_ms) - # print(f"Time per sample: %.2fms" % (elapsed_time_ms / no_of_samples)) - - if len(response['samples']) != no_of_samples: - print(f"Validation failure. {len(response['errors'])} out of {no_of_samples} samples failed validation") - - time_per_sample = elapsed_time_ms / no_of_samples - time_per_sample_queue.put(time_per_sample) - return time_per_sample - - -def run_multiple_submissions_parallel(no_of_samples, times_to_run): - test = [] - for i in range(times_to_run): - t = threading.Thread(target=measure_bulk_submission_time, args=(no_of_samples,)) - test.append(t) - t.start() - - for t in test: - t.join() - - total_time_ms = 0 - while not time_per_sample_queue.empty(): - total_time_ms = total_time_ms + time_per_sample_queue.get() - - time_per_sample = total_time_ms / times_to_run - # print(f"Average time per sample {time_per_sample}") - return time_per_sample - - -def run_multiple_submissions_serial(no_of_samples, times_to_run): - total_time_ms = 0 - for i in range(times_to_run): - total_time_ms = total_time_ms + measure_bulk_submission_time(no_of_samples) - - time_per_sample = total_time_ms / times_to_run - # print(f"Average time per sample {time_per_sample}") - return time_per_sample - - -def draw_table(header, data): - table = PrettyTable(header) - for row in data: - table.add_row(row) - print(table) - - -def draw_graph(headers, rows): - x = headers[1:] - for row in rows: - y = [float(x) for x in row[1:]] - plt.plot(x, y, label='# samples: ' + str(row[0])) - - plt.xlabel("# runs") - plt.ylabel("Time (ms)") - plt.legend() - plt.title('Average sample submission time') - plt.show() - - -def run(no_of_sample_list, times_to_run_list): - # serial submissions - headers = ["number of samples"] - headers.extend([x for x in times_to_run_list]) - rows = [] - for no_of_samples in tqdm(no_of_sample_list): - row = [no_of_samples] - for times_to_run in times_to_run_list: - avg_time = run_multiple_submissions_serial(no_of_samples, times_to_run) - row.append(f" %.2f" % avg_time) - rows.append(row) - time.sleep(sleep_between_iteration) - draw_table(headers, rows) - draw_graph(headers, rows) - - # parallel submissions - # headers = ["number of samples"] - # headers.extend(["threads = " + str(x) for x in times_to_run_list]) - # rows = [] - # for no_of_samples in no_of_sample_list: - # row = [no_of_samples] - # for times_to_run in times_to_run_list: - # avg_time = run_multiple_submissions_parallel(no_of_samples, times_to_run) - # row.append(f" %.2fms" % avg_time) - # rows.append(row) - # time.sleep(sleep_between_iteration) - # draw_table(headers, rows) - - -if __name__ == '__main__': - # parser = argparse.ArgumentParser(description='Load test biosamples bulk submission endpoint') - # parser.add_argument('--user', type=str, required=True, help='ENA webin user') - # parser.add_argument('--password', type=str, required=True, help='ENA webin password') - # args = parser.parse_args() - - sample_path = './data/samples' - user = '' - password = '' - no_of_samples = 1 - times_to_run = 2 - - no_of_sample_list = [1, 2, 3, 4, 5, 10, 50, 100] - times_to_run_list = [1, 5, 10, 30, 100] - - # no_of_sample_list = [1, 2] - # times_to_run_list = [1, 5] - sleep_between_iteration = 2 - - # measure_bulk_submission_time(no_of_samples) - # run_multiple_submissions_parallel(no_of_samples, times_to_run) - # run_multiple_submissions_serial(no_of_samples, times_to_run) - run(no_of_sample_list, times_to_run_list) diff --git a/scripts/requirements.txt b/scripts/requirements.txt deleted file mode 100644 index 02408d3b2..000000000 --- a/scripts/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -certifi==2024.7.4 -charset-normalizer==3.3.2 -idna==3.7 -numpy==1.21.6 -pandas==1.3.5 -python-dateutil==2.9.0.post0 -pytz==2024.1 -requests==2.31.0 -six==1.16.0 -tqdm==4.66.4 -urllib3==2.0.7 -prettytables -matplotlib -prettytable \ No newline at end of file diff --git a/utils/client/pom.xml b/utils/client/pom.xml deleted file mode 100644 index 8d5241f12..000000000 --- a/utils/client/pom.xml +++ /dev/null @@ -1,21 +0,0 @@ - - 4.0.0 - - utils-client - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - - - - org.springframework - spring-web - - - diff --git a/utils/ols/pom.xml b/utils/ols/pom.xml deleted file mode 100644 index c474d1531..000000000 --- a/utils/ols/pom.xml +++ /dev/null @@ -1,36 +0,0 @@ - - 4.0.0 - - utils-ols - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - - - - uk.ac.ebi.biosamples - properties - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-client - 5.3.12-SNAPSHOT - - - - org.springframework.boot - spring-boot-starter-cache - - - com.fasterxml.jackson.core - jackson-databind - - - diff --git a/utils/ols/src/test/resources/examples/ols-responses/FBcv_0003016.json b/utils/ols/src/test/resources/examples/ols-responses/FBcv_0003016.json deleted file mode 100644 index 4073dc4ba..000000000 --- a/utils/ols/src/test/resources/examples/ols-responses/FBcv_0003016.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "_embedded": { - "terms": [ - { - "iri": "http://purl.obolibrary.org/obo/FBcv_0003016", - "label": "protein complex group", - "description": [ - "Genes whose protein products form a macromolecular complex (GO:0043234)." - ], - "annotation": { - "created_by": [ - "mmc46" - ], - "creation_date": [ - "2014-03-28T15:23:42Z" - ], - "has_obo_namespace": [ - "group_descriptor" - ], - "id": [ - "FBcv:0003016" - ] - }, - "synonyms": null, - "ontology_name": "fbcv", - "ontology_prefix": "FBcv", - "ontology_iri": "http://purl.obolibrary.org/obo/fbcv.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "FBcv_0003016", - "obo_id": "FBcv:0003016", - "in_subset": null, - "obo_definition_citation": [ - { - "definition": "Genes whose protein products form a macromolecular complex (GO:0043234).", - "oboXrefs": [ - { - "database": "FBC", - "id": "SM", - "description": null, - "url": null - }, - { - "database": "GOC", - "id": "go_curators", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FFBcv_0003016/graph" - } - } - } - ] - }, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" - } - }, - "page": { - "size": 500, - "totalElements": 1, - "totalPages": 1, - "number": 0 - } -} \ No newline at end of file diff --git a/utils/pom.xml b/utils/pom.xml deleted file mode 100644 index 23a74f60c..000000000 --- a/utils/pom.xml +++ /dev/null @@ -1,23 +0,0 @@ - - 4.0.0 - - utils - - pom - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - - - - webapp - pipeline - thread - ols - client - validation - - diff --git a/utils/thread/pom.xml b/utils/thread/pom.xml deleted file mode 100644 index c958aa372..000000000 --- a/utils/thread/pom.xml +++ /dev/null @@ -1,14 +0,0 @@ - - 4.0.0 - - utils-thread - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - diff --git a/utils/validation/pom.xml b/utils/validation/pom.xml deleted file mode 100644 index 60ea6e92b..000000000 --- a/utils/validation/pom.xml +++ /dev/null @@ -1,45 +0,0 @@ - - 4.0.0 - - utils-validation - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - - - - uk.ac.ebi.biosamples - properties - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-client - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-webapp - 5.3.12-SNAPSHOT - - - javax.validation - validation-api - 1.1.0.Final - - - org.springframework.boot - spring-boot-starter-cache - - - com.fasterxml.jackson.core - jackson-databind - - - diff --git a/utils/validation/src/test/resources/examples/ols-responses/NCBITaxon_3702.json b/utils/validation/src/test/resources/examples/ols-responses/NCBITaxon_3702.json deleted file mode 100644 index da9c1343c..000000000 --- a/utils/validation/src/test/resources/examples/ols-responses/NCBITaxon_3702.json +++ /dev/null @@ -1,609 +0,0 @@ -{ - "_embedded": { - "terms": [ - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "has_obo_namespace": [ - "ncbi_taxonomy" - ], - "has_related_synonym": [ - "At-" - ], - "id": [ - "NCBITaxon:3702" - ] - }, - "synonyms": null, - "ontology_name": "pr", - "ontology_prefix": "PR", - "ontology_iri": "http://purl.obolibrary.org/obo/pr.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": [ - { - "name": "At-", - "scope": "hasRelatedSynonym", - "type": "Unique short label for PRO terms for display purposes", - "xrefs": [ - { - "database": "PRO", - "id": "DAN", - "description": null, - "url": null - } - ] - } - ], - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pr/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": [ - "An organism of the species Arabidopsis thaliana" - ], - "annotation": { - "id": [ - "NCBITaxon:3702" - ] - }, - "synonyms": null, - "ontology_name": "cco", - "ontology_prefix": "CCO", - "ontology_iri": "http://purl.obolibrary.org/obo/cco", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/cco/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "alternative term": [ - "thale cress", - "mouse-ear cress", - "Arabidopsis thaliana (thale cress)", - "thale-cress", - "Arbisopsis thaliana" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/ncbitaxon.owl" - ] - }, - "synonyms": null, - "ontology_name": "ero", - "ontology_prefix": "ERO", - "ontology_iri": "http://purl.obolibrary.org/obo/ero", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": [ - "A species of genus Arabidopsis." - ], - "annotation": { - "has_related_synonym": [ - "Arabidopsis thaliana (thale cress)", - "Arbisopsis thaliana" - ] - }, - "synonyms": [ - "Mouse-ear cress", - "Thale cress", - "Thale-cress" - ], - "ontology_name": "mirnao", - "ontology_prefix": "MIRNAO", - "ontology_iri": "http://purl.obolibrary.org/obo/mirnao", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mirnao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "editor preferred label": [ - "Arabidopsis thaliana" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/ncbitaxon.owl" - ] - }, - "synonyms": [ - "thale cress", - "mouse-ear cress", - "Arabidopsis thaliana (thale cress)", - "thale-cress", - "Arabidopsis_thaliana", - "Arbisopsis thaliana" - ], - "ontology_name": "bao", - "ontology_prefix": "BAO", - "ontology_iri": "http://www.bioassayontology.org/bao/bao_complete.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "alternative term": [ - "thale cress", - "mouse-ear cress", - "thale-cress" - ], - "editor preferred label": [ - "Arabidopsis thaliana" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/ncbitaxon.owl" - ] - }, - "synonyms": null, - "ontology_name": "obi", - "ontology_prefix": "OBI", - "ontology_iri": "http://purl.obolibrary.org/obo/obi.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "alternative term": [ - "thale cress", - "mouse-ear cress", - "thale-cress" - ], - "editor preferred label": [ - "Arabidopsis thaliana" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/ncbitaxon.owl" - ] - }, - "synonyms": null, - "ontology_name": "ohmi", - "ontology_prefix": "OHMI", - "ontology_iri": "http://purl.obolibrary.org/obo/ohmi.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ohmi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "has_obo_namespace": [ - "ncbi_taxonomy" - ], - "has_rank": [ - "http://purl.obolibrary.org/obo/NCBITaxon_species" - ], - "id": [ - "NCBITaxon:3702" - ], - "term editor": [ - "Class imported / merged by efoimporter", - "James Malone" - ] - }, - "synonyms": [ - "thale cress", - "mouse-ear cress", - "Arabidopsis thaliana (thale cress)", - "thale-cress", - "Arbisopsis thaliana" - ], - "ontology_name": "efo", - "ontology_prefix": "EFO", - "ontology_iri": "http://www.ebi.ac.uk/efo/efo.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - }, - "part_of": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FBFO_0000050" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/NCBITaxon_3702", - "label": "Arabidopsis thaliana", - "description": null, - "annotation": { - "database_cross_reference": [ - "GC_ID:1" - ], - "has_exact_synonym": [ - "thale cress", - "mouse-ear cress", - "thale-cress" - ], - "has_obo_namespace": [ - "ncbi_taxonomy" - ], - "has_rank": [ - "http://purl.obolibrary.org/obo/NCBITaxon_species" - ], - "has_related_synonym": [ - "Arabidopsis thaliana (thale cress)", - "Arabidopsis_thaliana", - "thale kress", - "Arbisopsis thaliana" - ] - }, - "synonyms": null, - "ontology_name": "ncbitaxon", - "ontology_prefix": "NCBITAXON", - "ontology_iri": "http://purl.obolibrary.org/obo/ncbitaxon.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": true, - "has_children": false, - "is_root": false, - "short_form": "NCBITaxon_3702", - "obo_id": "NCBITaxon:3702", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": [ - { - "database": "GC_ID", - "id": "1", - "description": null, - "url": null - } - ], - "obo_synonym": [ - { - "name": "thale-cress", - "scope": "hasExactSynonym", - "type": "common name", - "xrefs": [] - }, - { - "name": "thale kress", - "scope": "hasRelatedSynonym", - "type": "misspelling", - "xrefs": [] - }, - { - "name": "Arbisopsis thaliana", - "scope": "hasRelatedSynonym", - "type": "misspelling", - "xrefs": [] - }, - { - "name": "thale cress", - "scope": "hasExactSynonym", - "type": "genbank common name", - "xrefs": [] - }, - { - "name": "Arabidopsis_thaliana", - "scope": "hasRelatedSynonym", - "type": "misspelling", - "xrefs": [] - }, - { - "name": "mouse-ear cress", - "scope": "hasExactSynonym", - "type": "common name", - "xrefs": [] - }, - { - "name": "Arabidopsis thaliana (thale cress)", - "scope": "hasRelatedSynonym", - "type": "misspelling", - "xrefs": [] - } - ], - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncbitaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_3702/graph" - } - } - } - ] - }, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" - } - }, - "page": { - "size": 500, - "totalElements": 9, - "totalPages": 1, - "number": 0 - } -} \ No newline at end of file diff --git a/utils/validation/src/test/resources/examples/ols-responses/NCIT_C14207.json b/utils/validation/src/test/resources/examples/ols-responses/NCIT_C14207.json deleted file mode 100644 index b91ccceba..000000000 --- a/utils/validation/src/test/resources/examples/ols-responses/NCIT_C14207.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "_embedded": { - "terms": [ - { - "iri": "http://purl.obolibrary.org/obo/NCIT_C14207", - "label": "Fish", - "description": [ - "A grouping of jawed and jawless vertebrate animals usually having fins and a covering of scales or plates, breathing by means of gills, and living almost entirely in the water." - ], - "annotation": { - "ALT_DEFINITION": [ - "Any jawed or jawless organisms in the phylum Chordata including the jawless fish, armored fish, cartilaginous fish, ray-finned fish and lobe-finned fish." - ], - "Contributing_Source": [ - "CDISC" - ], - "Legacy_Concept_Name": [ - "Fish" - ], - "Preferred_Name": [ - "Fish" - ], - "Semantic_Type": [ - "Fish" - ], - "UMLS_CUI": [ - "C0016163" - ], - "code": [ - "C14207" - ] - }, - "synonyms": [ - "FISH", - "Fish", - "Fish (NOS)" - ], - "ontology_name": "ncit", - "ontology_prefix": "NCIT", - "ontology_iri": "http://purl.obolibrary.org/obo/ncit.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": true, - "has_children": true, - "is_root": false, - "short_form": "NCIT_C14207", - "obo_id": "NCIT:C14207", - "in_subset": [ - "NCIT_C61410", - "NCIT_C77526", - "NCIT_C77808" - ], - "obo_definition_citation": [ - { - "definition": "A grouping of jawed and jawless vertebrate animals usually having fins and a covering of scales or plates, breathing by means of gills, and living almost entirely in the water.", - "oboXrefs": [ - { - "database": null, - "id": "NCI", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": [ - { - "name": "Fish (NOS)", - "scope": "hasExactSynonym", - "type": null, - "xrefs": [] - }, - { - "name": "FISH", - "scope": "hasExactSynonym", - "type": null, - "xrefs": [] - }, - { - "name": "Fish", - "scope": "hasExactSynonym", - "type": null, - "xrefs": [] - } - ], - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ncit/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCIT_C14207/graph" - } - } - } - ] - }, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" - } - }, - "page": { - "size": 500, - "totalElements": 1, - "totalPages": 1, - "number": 0 - } -} \ No newline at end of file diff --git a/utils/validation/src/test/resources/examples/ols-responses/PATO_0000384.json b/utils/validation/src/test/resources/examples/ols-responses/PATO_0000384.json deleted file mode 100644 index 23873380a..000000000 --- a/utils/validation/src/test/resources/examples/ols-responses/PATO_0000384.json +++ /dev/null @@ -1,1369 +0,0 @@ -{ - "_embedded": { - "terms": [ - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "bcgo", - "ontology_prefix": "BCGO", - "ontology_iri": "http://purl.obolibrary.org/obo/bcgo.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bcgo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": null, - "annotation": { - "alternative term": [ - "http://mged.sourceforge.net/ontologies/MGEDOntology.owl#male" - ], - "comment": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "definition source": [ - "CARO:0000027", - "MO_652", - "PATO:0000384" - ], - "term editor": [ - "James Malone", - "Tomasz Adamusiak", - "Jie Zheng" - ] - }, - "synonyms": null, - "ontology_name": "clo", - "ontology_prefix": "CLO", - "ontology_iri": "http://purl.obolibrary.org/obo/clo.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/clo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "PATO_0000384", - "description": null, - "annotation": {}, - "synonyms": null, - "ontology_name": "ero", - "ontology_prefix": "ERO", - "ontology_iri": "http://purl.obolibrary.org/obo/ero", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": true, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ero/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ] - }, - "synonyms": null, - "ontology_name": "dpo", - "ontology_prefix": "FBcv", - "ontology_iri": "http://purl.obolibrary.org/obo/dpo", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": "MGED", - "id": "MGED", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/dpo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ] - }, - "synonyms": null, - "ontology_name": "fbcv", - "ontology_prefix": "FBcv", - "ontology_iri": "http://purl.obolibrary.org/obo/fbcv.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": "MGED", - "id": "MGED", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/fbcv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "created_by": [ - "Topito" - ], - "creation_date": [ - "2013-05-28T14:34:52Z" - ], - "database_cross_reference": [ - "PATO:0000384" - ], - "has_obo_namespace": [ - "idomal_namespace" - ], - "id": [ - "PATO:0000384" - ] - }, - "synonyms": null, - "ontology_name": "idomal", - "ontology_prefix": "IDOMAL", - "ontology_iri": "http://purl.obolibrary.org/obo/idomal", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": "PATO", - "id": "0000384", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": [ - { - "database": "PATO", - "id": "0000384", - "description": null, - "url": null - } - ], - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/idomal/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ] - }, - "synonyms": null, - "ontology_name": "pato", - "ontology_prefix": "PATO", - "ontology_iri": "http://purl.obolibrary.org/obo/pato.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": true, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": "MGED", - "id": "MGED", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/pato/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "imported from": [ - "pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "omiabis", - "ontology_prefix": "OMIABIS", - "ontology_iri": "http://purl.obolibrary.org/obo/omiabis", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/omiabis/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": null, - "annotation": {}, - "synonyms": null, - "ontology_name": "oba", - "ontology_prefix": "OBA", - "ontology_iri": "http://purl.obolibrary.org/obo/oba", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/oba/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ], - "in_subset": [ - "value_slim", - "mpath_slim" - ] - }, - "synonyms": null, - "ontology_name": "flopo", - "ontology_prefix": "FLOPO", - "ontology_iri": "http://purl.obolibrary.org/obo/flopo", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": null, - "id": "MGED:MGED", - "description": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flopo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "hasDbXref": [ - "NCBI BioSample: male" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "genepio", - "ontology_prefix": "GENEPIO", - "ontology_iri": "http://purl.obolibrary.org/obo/genepio.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": null, - "obo_xref": [ - { - "database": "NCBI BioSample", - "id": " male", - "description": null, - "url": null - } - ], - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/genepio/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "imported from": [ - "http://purl.obolibrary.org/obo/pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "eupath", - "ontology_prefix": "EUPATH", - "ontology_iri": "http://purl.obolibrary.org/obo/eupath.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/eupath/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": null, - "annotation": {}, - "synonyms": null, - "ontology_name": "hp", - "ontology_prefix": "HP", - "ontology_iri": "http://purl.obolibrary.org/obo/hp.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/hp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "bao", - "ontology_prefix": "BAO", - "ontology_iri": "http://www.bioassayontology.org/bao/bao_complete.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/bao/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": null, - "annotation": {}, - "synonyms": null, - "ontology_name": "geno", - "ontology_prefix": "GENO", - "ontology_iri": "http://purl.obolibrary.org/obo/geno.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/geno/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.org/obo/owl/PATO#PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "editor preferred term": [ - "male" - ], - "imported from": [ - "http://purl.org/obo/owl/PATO" - ] - }, - "synonyms": null, - "ontology_name": "flu", - "ontology_prefix": "FLU", - "ontology_iri": "http://purl.obolibrary.org/obo/flu/dev/flu.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/flu/terms/http%253A%252F%252Fpurl.org%252Fobo%252Fowl%252FPATO%2523PATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "CARO_definition_citation": [ - "CARO:0000027" - ], - "MO_definition_citation": [ - "MO:652" - ], - "MSH_definition_citation": [ - "MSH:D008297" - ], - "NCI_Thesaurus_definition_citation": [ - "NCIt:C20197" - ], - "PATO_definition_citation": [ - "PATO:0000384" - ], - "SNOMEDCT_definition_citation": [ - "SNOMEDCT:248153007" - ], - "bioportal_provenance": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.[accessedResource: PATO:0000384][accessDate: 05-04-2011]" - ], - "term editor": [ - "James Malone", - "Tomasz Adamusiak", - "Jie Zheng" - ] - }, - "synonyms": null, - "ontology_name": "efo", - "ontology_prefix": "EFO", - "ontology_iri": "http://www.ebi.ac.uk/efo/efo.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/efo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "micro", - "ontology_prefix": "MICRO", - "ontology_iri": "http://purl.obolibrary.org/obo/MicrO.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/micro/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ] - }, - "synonyms": null, - "ontology_name": "nbo", - "ontology_prefix": "NBO", - "ontology_iri": "http://purl.obolibrary.org/obo/nbo.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": "MGED", - "id": "MGED", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/nbo/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "editor preferred label": [ - "male" - ], - "imported from": [ - "http://purl.obolibrary.org/obo/pato.owl" - ] - }, - "synonyms": null, - "ontology_name": "obi", - "ontology_prefix": "OBI", - "ontology_iri": "http://purl.obolibrary.org/obo/obi.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/obi/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": null, - "annotation": {}, - "synonyms": null, - "ontology_name": "mp", - "ontology_prefix": "MP", - "ontology_iri": "http://purl.obolibrary.org/obo/mp.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": false, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": null, - "obo_definition_citation": null, - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/mp/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - }, - { - "iri": "http://purl.obolibrary.org/obo/PATO_0000384", - "label": "male", - "description": [ - "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes." - ], - "annotation": { - "has_obo_namespace": [ - "quality" - ], - "id": [ - "PATO:0000384" - ] - }, - "synonyms": null, - "ontology_name": "ms", - "ontology_prefix": "MS", - "ontology_iri": "http://purl.obolibrary.org/obo/ms.owl", - "is_obsolete": false, - "term_replaced_by": null, - "is_defining_ontology": false, - "has_children": true, - "is_root": false, - "short_form": "PATO_0000384", - "obo_id": "PATO:0000384", - "in_subset": [ - "value_slim", - "mpath_slim" - ], - "obo_definition_citation": [ - { - "definition": "A biological sex quality inhering in an individual or a population whose sex organs contain only male gametes.", - "oboXrefs": [ - { - "database": "MGED", - "id": "MGED", - "description": null, - "url": null - } - ] - } - ], - "obo_xref": null, - "obo_synonym": null, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384" - }, - "parents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/parents" - }, - "ancestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/ancestors" - }, - "hierarchicalParents": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalParents" - }, - "hierarchicalAncestors": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalAncestors" - }, - "jstree": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/jstree" - }, - "children": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/children" - }, - "descendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/descendants" - }, - "hierarchicalChildren": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalChildren" - }, - "hierarchicalDescendants": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/hierarchicalDescendants" - }, - "graph": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/ontologies/ms/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FPATO_0000384/graph" - } - } - } - ] - }, - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" - } - }, - "page": { - "size": 500, - "totalElements": 22, - "totalPages": 1, - "number": 0 - } -} \ No newline at end of file diff --git a/utils/validation/src/test/resources/examples/ols-responses/invalid-term.json b/utils/validation/src/test/resources/examples/ols-responses/invalid-term.json deleted file mode 100644 index d98e82679..000000000 --- a/utils/validation/src/test/resources/examples/ols-responses/invalid-term.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "_links": { - "self": { - "href": "https://wwwdev.ebi.ac.uk/ols/api/terms" - } - }, - "page": { - "size": 500, - "totalElements": 0, - "totalPages": 0, - "number": 0 - } -} \ No newline at end of file diff --git a/utils/validation/src/test/resources/logback-test.xml b/utils/validation/src/test/resources/logback-test.xml deleted file mode 100644 index 4449069d9..000000000 --- a/utils/validation/src/test/resources/logback-test.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/utils/webapp/pom.xml b/utils/webapp/pom.xml deleted file mode 100644 index f3609bc39..000000000 --- a/utils/webapp/pom.xml +++ /dev/null @@ -1,100 +0,0 @@ - - 4.0.0 - - utils-webapp - jar - - - uk.ac.ebi.biosamples - biosamples - 5.3.12-SNAPSHOT - ../../ - - - - - uk.ac.ebi.biosamples - models-security - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - models-mongo - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - models-solr - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT - - - uk.ac.ebi.ena.taxonomy - sub-taxonomy-client - 2.2.8 - - - uk.ac.ebi.biosamples - models-core - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - properties - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - messaging - 5.3.12-SNAPSHOT - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - - - com.fasterxml.jackson.module - jackson-module-jaxb-annotations - - - org.codehaus.woodstox - woodstox-core-asl - 4.1.4 - - - org.apache.commons - commons-csv - 1.8 - - - org.springframework.security.oauth - spring-security-oauth2 - 2.5.0.RELEASE - - - org.yaml - snakeyaml - - - com.auth0 - java-jwt - 4.0.0 - - - javax.servlet - javax.servlet-api - - - - - gitlab-maven - https://gitlab.ebi.ac.uk/api/v4/projects/1143/packages/maven - - - diff --git a/webapps/core-v2/pom.xml b/webapps/core-v2/pom.xml index 5c59e5b15..8625d9d80 100644 --- a/webapps/core-v2/pom.xml +++ b/webapps/core-v2/pom.xml @@ -5,7 +5,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ @@ -16,28 +16,13 @@ uk.ac.ebi.biosamples - utils-webapp - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-validation - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples properties - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT @@ -53,34 +38,10 @@ com.jayway.jsonpath json-path - - - org.springframework.boot - spring-boot-starter-tomcat - - org.springframework spring-oxm - - javax.xml.bind - jaxb-api - 2.3.0 - runtime - - - com.sun.xml.bind - jaxb-core - 2.3.0 - runtime - - - com.sun.xml.bind - jaxb-impl - 2.3.0 - runtime - com.fasterxml.jackson.core jackson-annotations diff --git a/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/controller/BulkActionControllerV2.java b/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/controller/BulkActionControllerV2.java index 8829aed48..7fb111c04 100644 --- a/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/controller/BulkActionControllerV2.java +++ b/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/controller/BulkActionControllerV2.java @@ -31,11 +31,14 @@ import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.*; import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.*; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SubmissionReceipt; +import uk.ac.ebi.biosamples.core.model.SubmittedViaType; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; import uk.ac.ebi.biosamples.service.SampleService; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; -import uk.ac.ebi.biosamples.validation.SchemaValidationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.validation.SchemaValidationService; @RestController @ExposesResourceFor(Sample.class) @@ -355,7 +358,9 @@ private Pair, Optional> persistSample( } catch (GlobalExceptions.SchemaValidationException e) { sampleErrorPair = new ImmutablePair<>(Optional.empty(), Optional.ofNullable(e.getMessage())); - log.info("Sample validation failed: {}", sample.getAccession()); + final String accession = sample.getAccession(); + + log.info("Sample validation failed: {}", accession != null ? accession : sample.getName()); } catch (Exception e) { sampleErrorPair = new ImmutablePair<>(Optional.empty(), Optional.ofNullable(e.getMessage())); diff --git a/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/controller/SampleGetControllerV2.java b/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/controller/SampleGetControllerV2.java index 8b979bb04..ef417c277 100644 --- a/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/controller/SampleGetControllerV2.java +++ b/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/controller/SampleGetControllerV2.java @@ -20,10 +20,10 @@ import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.*; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; import uk.ac.ebi.biosamples.service.SampleService; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; @RestController @ExposesResourceFor(Sample.class) diff --git a/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/service/SampleService.java b/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/service/SampleService.java new file mode 100644 index 000000000..e30ed96eb --- /dev/null +++ b/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/service/SampleService.java @@ -0,0 +1,770 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import static java.util.stream.Collectors.toSet; +import static uk.ac.ebi.biosamples.BioSamplesConstants.*; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.*; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.userdetails.UserDetails; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.BioSamplesProperties; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SubmittedViaType; +import uk.ac.ebi.biosamples.core.model.structured.AbstractData; +import uk.ac.ebi.biosamples.core.service.SampleValidator; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; +import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; +import uk.ac.ebi.biosamples.mongo.model.MongoSample; +import uk.ac.ebi.biosamples.mongo.model.MongoSampleMessage; +import uk.ac.ebi.biosamples.mongo.repository.MongoSampleMessageRepository; +import uk.ac.ebi.biosamples.mongo.repository.MongoSampleRepository; +import uk.ac.ebi.biosamples.mongo.service.*; +import uk.ac.ebi.biosamples.security.service.BioSamplesCrossSourceIngestAccessControlService; + +/** + * Service layer business logic for centralising repository access and conversions between different + * controller. Use this instead of linking to repositories directly. + * + * @author faulcon + */ +@Service +public class SampleService { + private static final Logger log = LoggerFactory.getLogger(SampleService.class); + private final MongoAccessionService mongoAccessionService; + private final MongoSampleRepository mongoSampleRepository; + private final MongoSampleMessageRepository mongoSampleMessageRepository; + private final MongoSampleToSampleConverter mongoSampleToSampleConverter; + private final SampleToMongoSampleConverter sampleToMongoSampleConverter; + private final MongoRelationshipToRelationshipConverter mongoRelationshipToRelationshipConverter; + private final SampleValidator sampleValidator; + private final SampleReadService sampleReadService; + private final MessagingService messagingService; + private final BioSamplesProperties bioSamplesProperties; + private final BioSamplesCrossSourceIngestAccessControlService + bioSamplesCrossSourceIngestAccessControlService; + + @Autowired + public SampleService( + @Qualifier("SampleAccessionService") final MongoAccessionService mongoAccessionService, + final MongoSampleRepository mongoSampleRepository, + final MongoSampleMessageRepository mongoSampleMessageRepository, + final MongoSampleToSampleConverter mongoSampleToSampleConverter, + final SampleToMongoSampleConverter sampleToMongoSampleConverter, + final MongoRelationshipToRelationshipConverter mongoRelationshipToRelationshipConverter, + final SampleValidator sampleValidator, + final SampleReadService sampleReadService, + final MessagingService messagingService, + final BioSamplesProperties bioSamplesProperties, + final BioSamplesCrossSourceIngestAccessControlService + bioSamplesCrossSourceIngestAccessControlService) { + this.mongoAccessionService = mongoAccessionService; + this.mongoSampleRepository = mongoSampleRepository; + this.mongoSampleMessageRepository = mongoSampleMessageRepository; + this.mongoSampleToSampleConverter = mongoSampleToSampleConverter; + this.sampleToMongoSampleConverter = sampleToMongoSampleConverter; + this.mongoRelationshipToRelationshipConverter = mongoRelationshipToRelationshipConverter; + this.sampleValidator = sampleValidator; + this.sampleReadService = sampleReadService; + this.messagingService = messagingService; + this.bioSamplesProperties = bioSamplesProperties; + this.bioSamplesCrossSourceIngestAccessControlService = + bioSamplesCrossSourceIngestAccessControlService; + } + + /** Throws an IllegalArgumentException of no sample with that accession exists */ + public Optional fetch(final String accession, final boolean applyCurations) { + return sampleReadService.fetch(accession, applyCurations); + } + + /* + Checks if the current sample that exists has no metadata, returns true if empty + */ + private boolean isStoredSampleEmpty( + final Sample newSample, final boolean isWebinSuperUser, final Sample oldSample) { + if (isWebinSuperUser) { + if (newSample.getSubmittedVia() == SubmittedViaType.FILE_UPLOADER) { + // file uploader submissions are done via superuser, but they are non-imported samples, + // needs to be handled safely + if (newSample.hasAccession()) { + return isStoredSampleEmpty(oldSample); + } + + return true; + } else { + // otherwise it is an ENA pipeline import, cannot be empty + return false; + } + } else { + if (newSample.hasAccession()) { + return isStoredSampleEmpty(oldSample); + } + } + + if (newSample.hasAccession()) { + return isStoredSampleEmpty(oldSample); + } + + return true; + } + + /* + Checks if the current sample that exists has no metadata, returns true if empty + */ + private boolean isStoredSampleEmpty(final Sample oldSample) { + if (oldSample.getTaxId() != null && oldSample.getTaxId() > 0) { + return false; + } + + if (!oldSample.getAttributes().isEmpty()) { + return false; + } + + if (!oldSample.getRelationships().isEmpty()) { + return false; + } + + if (!oldSample.getPublications().isEmpty()) { + return false; + } + + if (!oldSample.getContacts().isEmpty()) { + return false; + } + + if (!oldSample.getOrganizations().isEmpty()) { + return false; + } + + if (!oldSample.getData().isEmpty()) { + return false; + } + + if (!oldSample.getExternalReferences().isEmpty()) { + return false; + } + + return oldSample.getStructuredData().isEmpty(); + } + + // Because the fetch caches the sample, if an updated version is stored, we need to make + // sure + // that any cached version + // is removed. + // Note, pages of samples will not be cache busted, only single-accession sample retrieval + // @CacheEvict(cacheNames=WebappProperties.fetchUsing, key="#result.accession") + /* + Called by V1 endpoints to persist samples + */ + public Sample persistSample( + Sample newSample, final Sample oldSample, final boolean isWebinSuperUser) { + final Collection errors = sampleValidator.validate(newSample); + + if (!errors.isEmpty()) { + log.error("Sample validation failed : {}", errors); + + throw new GlobalExceptions.SampleMandatoryFieldsMissingException(String.join("|", errors)); + } + + if (newSample.hasAccession()) { + if (oldSample != null) { + newSample = updateFromCurrent(newSample, oldSample, isWebinSuperUser); + } else { + newSample = updateWhenNoneExists(newSample); + } + + MongoSample mongoSample = sampleToMongoSampleConverter.convert(newSample); + mongoSample = mongoSampleRepository.save(mongoSample); + + if (isTaxIdUpdated(oldSample, newSample)) { + mongoSampleMessageRepository.save( + new MongoSampleMessage(newSample.getAccession(), Instant.now(), newSample.getTaxId())); + } + + newSample = mongoSampleToSampleConverter.apply(mongoSample); + sendMessageToRabbitForIndexingToSolr( + newSample.getAccession(), getExistingRelationshipTargetsForIndexingInSolr(oldSample)); + } else { + newSample = createNew(newSample); + } + + // fetch returns sample with curations applied + final Optional sampleOptional = fetch(newSample.getAccession(), true); + + return sampleOptional.orElseThrow( + () -> + new RuntimeException( + "Failed to create newSample. Please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk")); + } + + private Sample updateWhenNoneExists(Sample newSample) { + log.error("Trying to update sample not in database, accession: {}", newSample.getAccession()); + + bioSamplesCrossSourceIngestAccessControlService + .validateFileUploaderSampleUpdateHasExistingAccession(newSample); + + final SortedSet newSampleAttributes = newSample.getAttributes(); + + if (newSampleAttributes.stream() + .noneMatch(attribute -> attribute.getType().equals(SRA_ACCESSION))) { + // Don't generate SRA accession (ERS sample accessions) for NCBI samples + if (!isNcbiImport(newSample)) { + final String sraAccession = generateOneSRAAccession(); + + newSampleAttributes.add(Attribute.build(SRA_ACCESSION, sraAccession)); + newSample = Sample.Builder.fromSample(newSample).withSraAccession(sraAccession).build(); + } + } else { + final List sraAccessionAttributeList = + newSampleAttributes.stream() + .filter(attribute -> attribute.getType().equals(SRA_ACCESSION)) + .toList(); + + if (sraAccessionAttributeList.size() > 1) { + throw new GlobalExceptions.InvalidSampleException(); + } + + final String sraAccession = sraAccessionAttributeList.get(0).getValue(); + + newSample = Sample.Builder.fromSample(newSample).withSraAccession(sraAccession).build(); + } + + return newSample; + } + + private Sample updateFromCurrent( + Sample newSample, final Sample oldSample, final boolean isWebinSuperUser) { + final boolean savedSampleEmpty = isStoredSampleEmpty(newSample, isWebinSuperUser, oldSample); + + if (savedSampleEmpty) { + newSample = Sample.Builder.fromSample(newSample).withSubmitted(Instant.now()).build(); + } + + final List existingRelationships = + getExistingRelationshipTargetsForIndexingInSolr( + newSample.getAccession(), + Objects.requireNonNull(sampleToMongoSampleConverter.convert(oldSample))); + + return compareWithExistingAndUpdateSample( + newSample, oldSample, existingRelationships, savedSampleEmpty, isWebinSuperUser); + } + + private Sample createNew(Sample newSample) { + final boolean noSraAccession = + newSample.getAttributes().stream() + .noneMatch(attribute -> attribute.getType().equals(SRA_ACCESSION)); + + if (!noSraAccession) { + newSample = validateAndPromoteSRAAccessionAttributeToField(newSample); + } + + newSample = mongoAccessionService.generateAccession(newSample, noSraAccession); + sendMessageToRabbitForIndexingToSolr(newSample.getAccession(), Collections.emptyList()); + + return newSample; + } + + private boolean isTaxIdUpdated(final Sample oldSample, final Sample sample) { + return oldSample != null + && oldSample.getTaxId() != null + && !oldSample.getTaxId().equals(sample.getTaxId()); + } + + private List getExistingRelationshipTargetsForIndexingInSolr(final Sample oldSample) { + final List existingRelationshipTargets = new ArrayList<>(); + + if (oldSample != null) { + final List existingRelationships = + getExistingRelationshipTargetsForIndexingInSolr( + oldSample.getAccession(), + Objects.requireNonNull(sampleToMongoSampleConverter.convert(oldSample))); + + existingRelationshipTargets.addAll( + existingRelationships.stream() + .map( + relationship -> { + if (relationship.getSource().equals(oldSample.getAccession())) { + return relationship.getTarget(); + } + + return null; + }) + .toList()); + } + + return existingRelationshipTargets; + } + + /* + Called by V2 endpoints to persist samples + */ + public Sample persistSampleV2( + Sample newSample, final Sample oldSample, final boolean isWebinSuperUser) { + final Collection errors = sampleValidator.validate(newSample); + + if (!errors.isEmpty()) { + log.error("Sample validation failed : {}", errors); + throw new GlobalExceptions.SampleMandatoryFieldsMissingException(String.join("|", errors)); + } + + if (newSample.hasAccession()) { + if (oldSample != null) { + log.info( + "Trying to update sample that exists in database, accession: {}", + newSample.getAccession()); + + final boolean savedSampleEmpty = + isStoredSampleEmpty(newSample, isWebinSuperUser, oldSample); + + if (savedSampleEmpty) { + newSample = Sample.Builder.fromSample(newSample).withSubmitted(Instant.now()).build(); + } + + newSample = + compareWithExistingAndUpdateSample( + newSample, oldSample, null, savedSampleEmpty, isWebinSuperUser); + } else { + log.error( + "Trying to update sample not in database, accession: {}", newSample.getAccession()); + + newSample = updateWhenNoneExists(newSample); + } + + MongoSample mongoSample = sampleToMongoSampleConverter.convert(newSample); + + assert mongoSample != null; + + mongoSample = mongoSampleRepository.save(mongoSample); + newSample = mongoSampleToSampleConverter.apply(mongoSample); + + sendMessageToRabbitForIndexingToSolr(newSample.getAccession(), Collections.emptyList()); + } else { + newSample = createNew(newSample); + } + + return newSample; + } + + private void sendMessageToRabbitForIndexingToSolr( + final String accession, final List existingRelationshipTargets) { + try { + messagingService.fetchThenSendMessage(accession, existingRelationshipTargets); + } catch (final Exception e) { + log.error("Indexing failed for accession " + accession); + } + } + + /* + Called by V2 endpoints to build a sample with a newly generated sample accession + */ + public Sample accessionSample(Sample newSample) { + final Collection errors = sampleValidator.validate(newSample); + + if (!errors.isEmpty()) { + log.error("Sample validation failed : {}", errors); + + throw new GlobalExceptions.SampleMandatoryFieldsMissingException(String.join("|", errors)); + } + + if (newSample + .getWebinSubmissionAccountId() + .equalsIgnoreCase(bioSamplesProperties.getBiosamplesClientWebinUsername())) { + // accessioning from ENA, sample name is the SRA accession here + final Attribute sraAccessionAttribute = Attribute.build(SRA_ACCESSION, newSample.getName()); + + newSample.getAttributes().add(sraAccessionAttribute); + newSample = Sample.Builder.fromSample(newSample).build(); + + return mongoAccessionService.generateAccession(newSample, false); + } else { + return mongoAccessionService.generateAccession(newSample, true); + } + } + + public String generateOneSRAAccession() { + return mongoAccessionService.generateOneSRAAccession(); + } + + /* + Returns true if a sample does not exist in BioSamples + */ + public boolean isNotExistingAccession(final String accession) { + if (accession != null) { + return mongoSampleRepository.findById(accession).isEmpty(); + } else { + return true; + } + } + + private List getExistingRelationshipTargetsForIndexingInSolr( + final String accession, final MongoSample mongoOldSample) { + final List oldRelationshipTargets = new ArrayList<>(); + + for (final MongoRelationship mongoRelationship : mongoOldSample.getRelationships()) { + if (mongoRelationship.getSource().equals(accession)) { + oldRelationshipTargets.add( + mongoRelationshipToRelationshipConverter.convert(mongoRelationship)); + } + } + + return oldRelationshipTargets; + } + + private Sample compareWithExistingAndUpdateSample( + Sample newSample, + final Sample oldSample, + final List existingRelationships, + final boolean isEmptySample, + final boolean isWebinSuperUser) { + Set structuredData = new HashSet<>(); + boolean applyOldSampleStructuredData = false; + + // retain existing relationships for superuser submissions, pipelines, ENA POSTED, not for file + // uploads though + handleRelationships(newSample, existingRelationships); + handleSRAAccession(newSample, oldSample, isWebinSuperUser); + newSample = validateAndPromoteSRAAccessionAttributeToField(newSample); + + if (newSample.getData().isEmpty()) { + log.info("No structured data in new sample"); + + if (oldSample.getData() != null && !oldSample.getData().isEmpty()) { + structuredData = oldSample.getData(); + // Check if old sample has structured data, if yes, retain + applyOldSampleStructuredData = true; + + log.info("Old sample has structured data"); + } + } + + if (applyOldSampleStructuredData) { + log.info("Build sample and applying old sample structured data"); + log.trace("Old sample structured data size is " + structuredData.size()); + + return Sample.Builder.fromSample(newSample) + .withCreate(defineCreateDate(newSample, oldSample)) + .withSubmitted(defineSubmittedDate(newSample, oldSample, isEmptySample)) + .withData(structuredData) + .build(); + } else { + log.info("Building sample without structured data"); + + return Sample.Builder.fromSample(newSample) + .withCreate(defineCreateDate(newSample, oldSample)) + .withSubmitted(defineSubmittedDate(newSample, oldSample, isEmptySample)) + .build(); + } + } + + private Sample validateAndPromoteSRAAccessionAttributeToField(final Sample newSample) { + // Retrieve SRA accession attribute from new sample + final Optional newSampleSraAccessionOptional = + newSample.getAttributes().stream() + .filter(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION)) + .findFirst(); + + // Retrieve SRA accession field from new sample + final String sraAccessionField = newSample.getSraAccession(); + + // Check if SRA accession field and attribute are both present + if (sraAccessionField != null && newSampleSraAccessionOptional.isPresent()) { + // Check for SRA accession mismatch + if (!sraAccessionField.equals(newSampleSraAccessionOptional.get().getValue())) { + throw new GlobalExceptions.InvalidSampleException(); + } else { + // If they match, return the new sample + return newSample; + } + } + + // Check if SRA accession field is null but the attribute is present + if (sraAccessionField == null && newSampleSraAccessionOptional.isPresent()) { + // Promote SRA accession attribute to the field and return the modified sample + return Sample.Builder.fromSample(newSample) + .withSraAccession(newSampleSraAccessionOptional.get().getValue()) + .build(); + } + + // Return the original new sample if no promotion or validation is needed + return newSample; + } + + private void handleRelationships( + final Sample newSample, final List existingRelationships) { + if (existingRelationships != null && !existingRelationships.isEmpty()) { + final String webinId = newSample.getWebinSubmissionAccountId(); + + // superuser and non-file upload submissions + if (webinId != null + && webinId.equals(bioSamplesProperties.getBiosamplesClientWebinUsername())) { + if (newSample.getSubmittedVia() != SubmittedViaType.FILE_UPLOADER) { + newSample.getRelationships().addAll(existingRelationships); + } + } + } + } + + private void handleSRAAccession( + final Sample newSample, final Sample oldSample, final boolean isWebinSuperUser) { + // Step 1: Initialization + final String newSampleSraAccessionField = newSample.getSraAccession(); + + // Step 3: Validation of SRA Accession Field in New Sample + if (newSampleSraAccessionField != null && !newSampleSraAccessionField.isEmpty()) { + if (oldSample != null) { + // Check if the SRA accession field has changed in the new sample + final String oldSampleSraAccessionField = oldSample.getSraAccession(); + + /* + *

This logic performs the following checks: + *

    + *
  • If the old SRA accession field is not null and not empty.
    • + *
  • If the new SRA accession field is different from the old SRA accession field.
    • + *
  • If the user is not a Webin Super User.
    • + *
  • If the sample is not an NCBI sample in an NCBI Super User domain.
    • + *
+ */ + if (oldSampleSraAccessionField != null + && !oldSampleSraAccessionField.isEmpty() + && !newSampleSraAccessionField.equals(oldSampleSraAccessionField) + && !isWebinSuperUser + && !isNcbiImport(newSample)) { + throw new GlobalExceptions.ChangedSRAAccessionException(); + } + } + } + + final SortedSet newSampleAttributes = newSample.getAttributes(); + final List newSampleSraAccessionAttributes = + newSampleAttributes.stream() + .filter(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION)) + .toList(); + + // Step 2: Validation of SRA Accession attributes in New Sample + if (newSampleSraAccessionAttributes.size() > 1) { + throw new GlobalExceptions.InvalidSampleException(); + } + + Attribute newSampleSraAccessionAttribute = + newSampleSraAccessionAttributes.isEmpty() ? null : newSampleSraAccessionAttributes.get(0); + + // Step 4: Validation of SRA Accession attributes in Old Sample + if (oldSample != null) { + final List oldSampleSraAccessionAttributes = + oldSample.getAttributes().stream() + .filter(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION)) + .toList(); + + // Check if there are more than one SRA accession attributes in the old sample + if (oldSampleSraAccessionAttributes.size() > 1) { + throw new GlobalExceptions.InvalidSampleException(); + } + + final Attribute oldSampleSraAccessionAttribute = + oldSampleSraAccessionAttributes.isEmpty() ? null : oldSampleSraAccessionAttributes.get(0); + + // Step 5: Handling SRA Accessions in New Sample and Old Sample + if (newSampleSraAccessionAttribute == null) { + // If newSampleSraAccessionAttribute is null, use oldSampleSraAccession or generate a new + // one + newSampleSraAccessionAttribute = + Objects.requireNonNullElseGet( + oldSampleSraAccessionAttribute, + () -> Attribute.build(SRA_ACCESSION, generateOneSRAAccession())); + // Add newSampleSraAccessionAttribute to the attributes of the new sample + newSampleAttributes.add(newSampleSraAccessionAttribute); + } + + // Step 6: Validation of Changed SRA Accession (if Old Sample exists) + if (oldSampleSraAccessionAttribute != null + && !oldSampleSraAccessionAttribute + .getValue() + .equals(newSampleSraAccessionAttribute.getValue()) + && !isWebinSuperUser + && !isNcbiImport(newSample)) { + throw new GlobalExceptions.ChangedSRAAccessionException(); + } + } else { + // Step 7: Handling New Samples without Old Samples (Old Sample doesn't exist) + if (newSampleSraAccessionAttribute == null && isWebinSuperUser && !isNcbiImport(newSample)) { + // If oldSample doesn't exist, and newSampleSraAccessionAttribute is still null, create, a + // new one + // Doesn't generate SRA accession (ERS sample accessions) for NCBI samples while import + // (prefix checks) + newSampleSraAccessionAttribute = Attribute.build(SRA_ACCESSION, generateOneSRAAccession()); + // Add newSampleSraAccessionAttribute to the attributes of the new sample + newSampleAttributes.add(newSampleSraAccessionAttribute); + } + } + } + + private boolean isNcbiImport(final Sample newSample) { + return (newSample.getAccession().startsWith(NCBI_ACCESSION_PREFIX) + || newSample.getAccession().startsWith(DDBJ_ACCESSION_PREFIX)) + && newSample.getSubmittedVia() == SubmittedViaType.PIPELINE_IMPORT; + } + + private Instant defineCreateDate(final Sample newSample, final Sample oldSample) { + return (oldSample.getCreate() != null ? oldSample.getCreate() : newSample.getCreate()); + } + + public boolean isPipelineEnaDomain(final String domain) { + if (domain == null) { + return false; + } + + return domain.equalsIgnoreCase(ENA_IMPORT_DOMAIN); + } + + public boolean isPipelineNcbiDomain(final String domain) { + if (domain == null) { + return false; + } + + return domain.equalsIgnoreCase(NCBI_IMPORT_DOMAIN); + } + + private Instant defineSubmittedDate( + final Sample newSample, final Sample oldSample, final boolean isEmptySample) { + if (isEmptySample) { + return newSample.getSubmitted(); + } else { + return oldSample.getSubmitted() != null ? oldSample.getSubmitted() : newSample.getSubmitted(); + } + } + + public Instant defineCreateDate(final Sample sample, final boolean isWebinSuperUserSubmission) { + final Instant now = Instant.now(); + final Instant create = sample.getCreate(); + + return isWebinSuperUserSubmission ? (create != null ? create : now) : now; + } + + public Instant defineSubmittedDate( + final Sample sample, final boolean isWebinSuperUserSubmission) { + final Instant now = Instant.now(); + final Instant submitted = sample.getSubmitted(); + + return isWebinSuperUserSubmission ? (submitted != null ? submitted : now) : now; + } + + public Sample buildPrivateSample(final Sample sample) { + final Instant release = + Instant.ofEpochSecond( + LocalDateTime.now(ZoneOffset.UTC).plusYears(100).toEpochSecond(ZoneOffset.UTC)); + final Instant update = Instant.now(); + final SubmittedViaType submittedVia = + sample.getSubmittedVia() == null ? SubmittedViaType.JSON_API : sample.getSubmittedVia(); + + return Sample.Builder.fromSample(sample) + .withRelease(release) + .withUpdate(update) + .withSubmittedVia(submittedVia) + .build(); + } + + public Set handleSampleRelationshipsV2( + final Sample sample, + final Optional oldSampleOptional, + final boolean isSuperUserSubmission) { + final SortedSet sampleRelationships = sample.getRelationships(); + + if (!isSuperUserSubmission) { + if (sampleRelationships != null && !sampleRelationships.isEmpty()) { + throw new GlobalExceptions.SampleWithRelationshipSubmissionExceptionV2(); + } + } + + if (sample.hasAccession()) { + if (oldSampleOptional.isPresent()) { + final Sample oldSample = oldSampleOptional.get(); + + if (oldSample.getRelationships() != null && !oldSample.getRelationships().isEmpty()) { + return Stream.of(oldSample.getRelationships(), sampleRelationships) + .filter(Objects::nonNull) + .flatMap(Set::stream) + .collect(toSet()); + } + } else { + return sampleRelationships; + } + } else { + if (sampleRelationships != null && !sampleRelationships.isEmpty()) { + throw new GlobalExceptions.SampleWithRelationshipSubmissionExceptionV2(); + } + } + + return null; + } + + public Optional validateSampleWithAccessionsAgainstConditionsAndGetOldSample( + final Sample sample, final boolean isWebinSuperUser) { + if (!isWebinSuperUser) { + if (sample.hasAccession() + || sample.hasSraAccession() + || sample.getAttributes() != null + && sample.getAttributes().stream() + .anyMatch(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION))) { + throw new GlobalExceptions.SampleWithAccessionSubmissionException(); + } + } else { + final List sraAccessionAttributeList = + sample.getAttributes().stream() + .filter(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION)) + .toList(); + if (sraAccessionAttributeList.size() > 1) { + throw new GlobalExceptions.InvalidSampleException(); + } else { + String sraAccession = null; + + if (!sraAccessionAttributeList.isEmpty()) { + sraAccession = sraAccessionAttributeList.get(0).getValue(); + } + + if (sraAccession != null + && sample.getSraAccession() != null + && !Objects.equals(sraAccession, sample.getSraAccession())) { + throw new GlobalExceptions.InvalidSampleException(); + } + } + + if (sample.hasAccession() && !isNotExistingAccession(sample.getAccession())) { + // fetch old sample if sample exists, + // fetch returns sample with curations applied + return fetch(sample.getAccession(), false); + } + } + + return Optional.empty(); + } + + public String getPrinciple(final Authentication loggedInUser) { + if (loggedInUser == null || loggedInUser.getPrincipal() == null) { + log.warn("Unauthenticated access detected: returning null for principal"); + return null; + } + + return loggedInUser.getPrincipal() instanceof UserDetails + ? ((UserDetails) loggedInUser.getPrincipal()).getUsername() + : loggedInUser.getPrincipal().toString(); + } +} diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/security/WebinAuthenticationService.java b/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/service/WebinAuthenticationService.java similarity index 94% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/security/WebinAuthenticationService.java rename to webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/service/WebinAuthenticationService.java index d84bc3cab..7d5a934b8 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/security/WebinAuthenticationService.java +++ b/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/service/WebinAuthenticationService.java @@ -8,27 +8,25 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.service.security; +package uk.ac.ebi.biosamples.service; import java.time.Instant; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import org.springframework.stereotype.Service; -import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.CurationLink; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SubmittedViaType; -import uk.ac.ebi.biosamples.model.structured.AbstractData; -import uk.ac.ebi.biosamples.model.structured.StructuredData; -import uk.ac.ebi.biosamples.model.structured.StructuredDataType; -import uk.ac.ebi.biosamples.service.SampleService; +import uk.ac.ebi.biosamples.core.model.CurationLink; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SubmittedViaType; +import uk.ac.ebi.biosamples.core.model.structured.AbstractData; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataType; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; +import uk.ac.ebi.biosamples.security.service.BioSamplesCrossSourceIngestAccessControlService; @Service public class WebinAuthenticationService { public static final String ATLANTICO_DOMAIN = "self.AtlantECO"; - private final RestTemplate restTemplate; private final SampleService sampleService; private final BioSamplesCrossSourceIngestAccessControlService bioSamplesCrossSourceIngestAccessControlService; @@ -39,7 +37,6 @@ public WebinAuthenticationService( final BioSamplesCrossSourceIngestAccessControlService bioSamplesCrossSourceIngestAccessControlService, final BioSamplesProperties bioSamplesProperties) { - this.restTemplate = new RestTemplate(); this.sampleService = sampleService; this.bioSamplesCrossSourceIngestAccessControlService = bioSamplesCrossSourceIngestAccessControlService; diff --git a/webapps/core-v2/src/test/java/uk/ac/ebi/biosamples/controller/BulkActionControllerV2Test.java b/webapps/core-v2/src/test/java/uk/ac/ebi/biosamples/controller/BulkActionControllerV2Test.java index 2cb9b0c9f..f28845f75 100644 --- a/webapps/core-v2/src/test/java/uk/ac/ebi/biosamples/controller/BulkActionControllerV2Test.java +++ b/webapps/core-v2/src/test/java/uk/ac/ebi/biosamples/controller/BulkActionControllerV2Test.java @@ -31,10 +31,10 @@ import org.springframework.security.test.context.support.WithUserDetails; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.web.servlet.MockMvc; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.security.TestSecurityConfig; import uk.ac.ebi.biosamples.service.SampleService; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; @SpringBootTest(properties = {"spring.cloud.gcp.project-id=no_project"}) @ContextConfiguration(classes = TestSecurityConfig.class) diff --git a/webapps/core/Dockerfile b/webapps/core/Dockerfile new file mode 100644 index 000000000..734bf4e25 --- /dev/null +++ b/webapps/core/Dockerfile @@ -0,0 +1,5 @@ +FROM eclipse-temurin:17-jdk-alpine +RUN mkdir /deployment +COPY webapps/core/target/webapps-core-*.war /deployment/webapps-core.war +RUN ls -l /deployment +ENTRYPOINT ["java", "-jar", "/deployment/webapps-core.war"] diff --git a/webapps/core/core-dependencies-local-only.yaml b/webapps/core/core-dependencies-local-only.yaml new file mode 100644 index 000000000..e38748c96 --- /dev/null +++ b/webapps/core/core-dependencies-local-only.yaml @@ -0,0 +1,107 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: solr +spec: + replicas: 1 + selector: + matchLabels: + app: solr + template: + metadata: + labels: + app: solr + spec: + containers: + - name: solr + image: solr:8 + ports: + - containerPort: 8983 + +--- +apiVersion: v1 +kind: Service +metadata: + name: solr +spec: + selector: + app: solr + ports: + - protocol: TCP + port: 8983 + targetPort: 8983 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mongodb +spec: + replicas: 1 + selector: + matchLabels: + app: mongodb + template: + metadata: + labels: + app: mongodb + spec: + containers: + - name: mongodb + image: mongo:5 + ports: + - containerPort: 27017 + +--- +apiVersion: v1 +kind: Service +metadata: + name: mongodb +spec: + selector: + app: mongodb + ports: + - protocol: TCP + port: 27017 + targetPort: 27017 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rabbitmq +spec: + replicas: 1 + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + containers: + - name: rabbitmq + image: rabbitmq:3-management + ports: + - containerPort: 5672 + - containerPort: 15672 + +--- +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq +spec: + selector: + app: rabbitmq + ports: + - name: amqp + protocol: TCP + port: 5672 + targetPort: 5672 + - name: management + protocol: TCP + port: 15672 + targetPort: 15672 diff --git a/webapps/core/core-deployment.yaml b/webapps/core/core-deployment.yaml new file mode 100644 index 000000000..045616baa --- /dev/null +++ b/webapps/core/core-deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: webapps-core-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: webapps-core + template: + metadata: + labels: + app: webapps-core + spec: + containers: + - name: webapps-core + image: %DOCKER_IMAGE% + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + env: + - name: spring.data.solr.host + value: "http://wp-np2-40.ebi.ac.uk:8983/solr/" + - name: spring.data.mongodb.uri + value: "mongodb://biosdev:L85SKeuR@mongodb-hx-biosdev-dev-001.ebi.ac.uk:27017,mongodb-hh-biosdev-dev-002.ebi.ac.uk:27017/biosamples?replicaSet=biosdevrs017&readPreference=primary&authSource=admin&w=1" + - name: SPRING_RABBITMQ_HOST + value: "wp-np2-40" + - name: SPRING_RABBITMQ_PORT + value: "5672" + - name: biosamples.schemaValidator + value: "https://wwwdev.ebi.ac.uk/biosamples/biovalidator/validate" + - name: biosamples.schemaStore + value: "https://wwwdev.ebi.ac.uk/biosamples/schema-store" + - name: SERVER_SERVLET_CONTEXT_PATH + value: /biosamples + imagePullSecrets: + - name: gitlab diff --git a/webapps/core/core-service.yaml b/webapps/core/core-service.yaml new file mode 100644 index 000000000..66741ff20 --- /dev/null +++ b/webapps/core/core-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: webapps-core-service +spec: + selector: + app: webapps-core + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + type: NodePort diff --git a/webapps/core/pom.xml b/webapps/core/pom.xml index c7b892f9c..bf28874a1 100644 --- a/webapps/core/pom.xml +++ b/webapps/core/pom.xml @@ -5,7 +5,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT ../../ @@ -18,53 +18,13 @@ uk.ac.ebi.biosamples - utils-webapp - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - commons - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-thread - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - utils-validation - 5.3.12-SNAPSHOT + core + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples properties - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - models-solr - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - models-jsonld - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - models-sitemap - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - models-neo4j - 5.3.12-SNAPSHOT - - - uk.ac.ebi.biosamples - models-curami - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT uk.ac.ebi.biosamples.search @@ -99,12 +59,6 @@ com.jayway.jsonpath json-path - - - org.springframework.boot - spring-boot-starter-tomcat - - org.springframework spring-oxm @@ -135,24 +89,6 @@ 4.11.0 test - - javax.xml.bind - jaxb-api - 2.3.0 - runtime - - - com.sun.xml.bind - jaxb-core - 2.3.0 - runtime - - - com.sun.xml.bind - jaxb-impl - 2.3.0 - runtime - org.jsoup jsoup diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/AccessionsGetController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/AccessionsGetController.java index 06354aca0..b240cf5f9 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/AccessionsGetController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/AccessionsGetController.java @@ -20,7 +20,7 @@ import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.*; -import uk.ac.ebi.biosamples.model.Accession; +import uk.ac.ebi.biosamples.core.model.Accession; import uk.ac.ebi.biosamples.service.AccessionsService; import uk.ac.ebi.biosamples.service.SampleService; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/BioschemasGetController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/BioschemasGetController.java index 42e49e5b8..4dcb3dbb3 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/BioschemasGetController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/BioschemasGetController.java @@ -12,14 +12,14 @@ import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.JsonLDDataCatalog; -import uk.ac.ebi.biosamples.model.JsonLDDataRecord; -import uk.ac.ebi.biosamples.model.JsonLDDataset; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; +import uk.ac.ebi.biosamples.jsonld.model.JsonLDDataCatalog; +import uk.ac.ebi.biosamples.jsonld.model.JsonLDDataRecord; +import uk.ac.ebi.biosamples.jsonld.model.JsonLDDataset; import uk.ac.ebi.biosamples.service.JsonLDService; import uk.ac.ebi.biosamples.service.SampleService; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; @RestController @RequestMapping(produces = "application/ld+json") diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/CurationController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/CurationController.java index 0cc7f9ac9..f4125829b 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/CurationController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/CurationController.java @@ -22,8 +22,8 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.mongo.service.CurationReadService; import uk.ac.ebi.biosamples.service.CurationResourceAssembler; import uk.ac.ebi.biosamples.service.SamplePageService; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java index 18d7770af..62cf3d3f7 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java @@ -23,10 +23,10 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.service.FileDownloadService; import uk.ac.ebi.biosamples.service.FilterService; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; import uk.ac.ebi.biosamples.utils.LinkUtils; @Controller diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileUploadController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileUploadController.java index 68d4e40db..43b31ef11 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileUploadController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileUploadController.java @@ -33,10 +33,10 @@ import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.AuthToken; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; import uk.ac.ebi.biosamples.mongo.model.MongoFileUpload; -import uk.ac.ebi.biosamples.service.security.AccessControlService; +import uk.ac.ebi.biosamples.security.model.AuthToken; +import uk.ac.ebi.biosamples.security.service.AccessControlService; import uk.ac.ebi.biosamples.service.upload.FileQueueService; import uk.ac.ebi.biosamples.service.upload.FileUploadService; import uk.ac.ebi.biosamples.utils.upload.FileUploadUtils; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileUploadLoginController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileUploadLoginController.java index b390aff08..7935b0a79 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileUploadLoginController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileUploadLoginController.java @@ -24,11 +24,13 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.AuthToken; -import uk.ac.ebi.biosamples.model.auth.*; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; import uk.ac.ebi.biosamples.mongo.model.MongoFileUpload; -import uk.ac.ebi.biosamples.service.security.AccessControlService; +import uk.ac.ebi.biosamples.security.model.AuthRealm; +import uk.ac.ebi.biosamples.security.model.AuthRequestWebin; +import uk.ac.ebi.biosamples.security.model.AuthToken; +import uk.ac.ebi.biosamples.security.model.FileUploaderAuthRequest; +import uk.ac.ebi.biosamples.security.service.AccessControlService; import uk.ac.ebi.biosamples.service.upload.FileUploadService; import uk.ac.ebi.biosamples.service.upload.JsonSchemaStoreSchemaRetrievalService; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/RecommendationController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/RecommendationController.java index 392f644d6..829e1bb59 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/RecommendationController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/RecommendationController.java @@ -15,9 +15,9 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import uk.ac.ebi.biosamples.model.CuramiRecommendation; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SampleRecommendation; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.curami.model.CuramiRecommendation; +import uk.ac.ebi.biosamples.curami.model.SampleRecommendation; import uk.ac.ebi.biosamples.service.RecommendationService; @RestController diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleCurationLinksController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleCurationLinksController.java index 6eb5102e1..aba882c85 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleCurationLinksController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleCurationLinksController.java @@ -27,13 +27,13 @@ import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.*; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.CurationLink; +import uk.ac.ebi.biosamples.core.model.CurationLink; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; import uk.ac.ebi.biosamples.mongo.service.CurationReadService; import uk.ac.ebi.biosamples.service.CurationLinkResourceAssembler; import uk.ac.ebi.biosamples.service.CurationPersistService; import uk.ac.ebi.biosamples.service.SampleService; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; @RestController @RequestMapping("/samples/{accession}/curationlinks") diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java index 241a0375a..3265b7e08 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java @@ -20,8 +20,8 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; -import uk.ac.ebi.biosamples.model.facet.Facet; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.service.FacetService; import uk.ac.ebi.biosamples.service.FilterService; import uk.ac.ebi.biosamples.utils.LinkUtils; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java index 90315868d..105d7b2db 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java @@ -34,14 +34,14 @@ import org.springframework.web.servlet.support.ServletUriComponentsBuilder; import org.springframework.web.util.UriComponentsBuilder; import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions.PaginationException; -import uk.ac.ebi.biosamples.model.JsonLDDataCatalog; -import uk.ac.ebi.biosamples.model.JsonLDDataset; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.auth.AuthorizationProvider; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.exception.GlobalExceptions.PaginationException; +import uk.ac.ebi.biosamples.jsonld.model.JsonLDDataCatalog; +import uk.ac.ebi.biosamples.jsonld.model.JsonLDDataset; +import uk.ac.ebi.biosamples.security.model.AuthorizationProvider; import uk.ac.ebi.biosamples.service.*; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; /** * Primary controller for HTML operations. diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleRestController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleRestController.java index 01ad88367..18a700038 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleRestController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleRestController.java @@ -23,17 +23,17 @@ import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.*; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SubmittedViaType; -import uk.ac.ebi.biosamples.model.structured.AbstractData; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SubmittedViaType; +import uk.ac.ebi.biosamples.core.model.structured.AbstractData; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; import uk.ac.ebi.biosamples.service.SampleManipulationService; import uk.ac.ebi.biosamples.service.SampleResourceAssembler; import uk.ac.ebi.biosamples.service.SampleService; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; import uk.ac.ebi.biosamples.service.taxonomy.TaxonomyClientService; +import uk.ac.ebi.biosamples.service.validation.SchemaValidationService; import uk.ac.ebi.biosamples.utils.phenopacket.PhenopacketConverter; -import uk.ac.ebi.biosamples.validation.SchemaValidationService; /** * Primary controller for REST operations both in JSON and XML and both read and write. diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SamplesRestController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SamplesRestController.java index f51154eb8..e35fd0a99 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SamplesRestController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SamplesRestController.java @@ -34,18 +34,18 @@ import org.springframework.web.bind.annotation.*; import org.springframework.web.util.UriComponentsBuilder; import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions.PaginationException; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SubmittedViaType; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.model.structured.AbstractData; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SubmittedViaType; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.structured.AbstractData; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; +import uk.ac.ebi.biosamples.exception.GlobalExceptions.PaginationException; import uk.ac.ebi.biosamples.service.*; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; import uk.ac.ebi.biosamples.service.taxonomy.TaxonomyClientService; +import uk.ac.ebi.biosamples.service.validation.SchemaValidationService; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; import uk.ac.ebi.biosamples.utils.LinkUtils; -import uk.ac.ebi.biosamples.validation.SchemaValidationService; /** * Primary controller for REST operations both in JSON and XML and both read and write. @@ -137,18 +137,20 @@ public ResponseEntity>> searchHal( if (cursor != null) { log.trace("This cursor = " + decodedCursor); + final CursorArrayList samples = samplePageService.getSamplesByText( decodedText, filters, principle, decodedCursor, effectiveSize, applyCurations); + log.trace("Next cursor = " + samples.getNextCursorMark()); final CollectionModel> resources = CollectionModel.of( samples.stream() .map( - s -> - s != null - ? sampleResourceAssembler.toModel(s, SampleRestController.class) + sample -> + sample != null + ? sampleResourceAssembler.toModel(sample, SampleRestController.class) : null) .collect(Collectors.toList())); @@ -161,6 +163,7 @@ public ResponseEntity>> searchHal( effectiveSize, IanaLinkRelations.SELF.value(), getClass())); + // only display the next link if there is a next cursor to go to if (!LinkUtils.decodeText(samples.getNextCursorMark()).equals(decodedCursor) && !samples.getNextCursorMark().equals("*")) { @@ -189,6 +192,7 @@ public ResponseEntity>> searchHal( final UriComponentsBuilder uriComponentsBuilder = WebMvcLinkBuilder.linkTo(SamplesRestController.class).toUriComponentsBuilder(); + // This is a bit of a hack, but best we can do for now... resources.add( Link.of(uriComponentsBuilder.build(true).toUriString() + "/{accession}", "sample")); diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SchemaValidationController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SchemaValidationController.java index 742e8aa53..90740665c 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SchemaValidationController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SchemaValidationController.java @@ -15,10 +15,10 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.validation.SchemaValidationService; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; +import uk.ac.ebi.biosamples.service.validation.SchemaValidationService; @RestController @CrossOrigin diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SitemapController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SitemapController.java index 5422ff84d..0d077d844 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SitemapController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SitemapController.java @@ -29,9 +29,13 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; -import uk.ac.ebi.biosamples.model.*; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.service.SamplePageService; +import uk.ac.ebi.biosamples.sitemap.model.XmlSitemap; +import uk.ac.ebi.biosamples.sitemap.model.XmlSitemapIndex; +import uk.ac.ebi.biosamples.sitemap.model.XmlUrl; +import uk.ac.ebi.biosamples.sitemap.model.XmlUrlSet; @Controller @RequestMapping("/sitemap") @@ -56,15 +60,19 @@ public SitemapController(final SamplePageService pageService) { @RequestMapping(method = RequestMethod.GET, produces = MediaType.APPLICATION_XML_VALUE) @ResponseBody public XmlSitemapIndex createSampleSitemapIndex(final HttpServletRequest request) { - final long sampleCount = getTotalSamples(); final long pageNumber = (sampleCount / (long) sitemapPageSize) + 1L; final XmlSitemapIndex xmlSitemapIndex = new XmlSitemapIndex(); + for (int i = 0; i < pageNumber; i++) { final String location = generateBaseUrl(request) + String.format("/sitemap/%d", i + 1); final XmlSitemap xmlSiteMap = new XmlSitemap(location); + xmlSitemapIndex.addSitemap(xmlSiteMap); } + + log.info("Sitemap API called, returning {} sitemaps ", xmlSitemapIndex.getXmlSitemaps().size()); + return xmlSitemapIndex; } @@ -89,24 +97,26 @@ public XmlUrlSet createSampleSitemapPage( final Page samplePage = samplePageService.getSamplesByText("", Collections.emptyList(), null, pageRequest, true); final XmlUrlSet xmlUrlSet = new XmlUrlSet(); + for (final Sample sample : samplePage.getContent()) { final String location = generateBaseUrl(request) + String.format("/samples/%s", sample.getAccession()); - final LocalDate lastModifiedDate = LocalDateTime.ofInstant(sample.getUpdate(), ZoneOffset.UTC).toLocalDate(); - final XmlUrl url = new XmlUrl.XmlUrlBuilder(location) .lastModified(lastModifiedDate) .hasPriority(XmlUrl.Priority.MEDIUM) .build(); + xmlUrlSet.addUrl(url); } + log.debug( String.format( "Returning model for %d samples took %d millis", sitemapPageSize, System.currentTimeMillis() - startTime)); + return xmlUrlSet; } @@ -119,6 +129,7 @@ public XmlUrlSet createSampleSitemapPage( private String generateBaseUrl(final HttpServletRequest request) { final String requestURI = request.getRequestURI(); final String requestURL = request.getRequestURL().toString(); + return requestURL.replaceFirst(requestURI, "") + request.getContextPath(); } @@ -130,9 +141,9 @@ private String generateBaseUrl(final HttpServletRequest request) { private long getTotalSamples() { final Pageable pageable = PageRequest.of(0, 1); final Collection filters = Collections.emptyList(); - final Collection domains = Collections.emptyList(); final Page samplePage = samplePageService.getSamplesByText("", filters, null, pageable, true); + return samplePage.getTotalElements(); } } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/StatController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/StatController.java index 5cbcc79ca..bbcf8b7f0 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/StatController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/StatController.java @@ -14,7 +14,7 @@ import java.util.Map; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.*; -import uk.ac.ebi.biosamples.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.facet.Facet; import uk.ac.ebi.biosamples.mongo.model.MongoAnalytics; import uk.ac.ebi.biosamples.service.StatService; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/StructuredDataController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/StructuredDataController.java index cf6dc4ef0..ad029c801 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/StructuredDataController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/StructuredDataController.java @@ -19,11 +19,11 @@ import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.*; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.structured.StructuredData; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; import uk.ac.ebi.biosamples.service.SampleService; import uk.ac.ebi.biosamples.service.StructuredDataService; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; /** Structured data operations */ @RestController diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java deleted file mode 100644 index ec13a0ffa..000000000 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/AccessionsService.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2021 EMBL - European Bioinformatics Institute - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package uk.ac.ebi.biosamples.service; - -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageRequest; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.service.search.SearchService; -import uk.ac.ebi.biosamples.utils.LinkUtils; - -import java.util.Collection; -import java.util.HashSet; - -@Service -public class AccessionsService { - private final FilterService filterService; - private final SearchService searchService; - - public AccessionsService(FilterService filterService, - @Qualifier("solrSearchService") SearchService searchService) { - this.filterService = filterService; - this.searchService = searchService; - } - - public Page getAccessions(String text, - String[] requestFilters, - String webinSubmissionAccountId, - Integer page, - Integer size) { - PageRequest pageable = PageRequest.of(page, size); - String decodedText = LinkUtils.decodeText(text); - String[] decodedFilter = LinkUtils.decodeTexts(requestFilters); - Collection filtersAfterDecode = filterService.getFiltersCollection(decodedFilter); - - return fetchAccessions(pageable, decodedText, filtersAfterDecode, webinSubmissionAccountId); - } - - private Page fetchAccessions(PageRequest pageable, - String decodedText, - Collection filtersAfterDecode, - String webinSubmissionAccountId) { - return searchService.searchForAccessions( - decodedText, new HashSet<>(filtersAfterDecode), webinSubmissionAccountId, pageable); - } -} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/CurationLinkResourceAssembler.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/CurationLinkResourceAssembler.java index 9936881f5..27cc8402c 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/CurationLinkResourceAssembler.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/CurationLinkResourceAssembler.java @@ -18,8 +18,8 @@ import org.springframework.stereotype.Service; import uk.ac.ebi.biosamples.controller.SampleCurationLinksController; import uk.ac.ebi.biosamples.controller.SampleRestController; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.CurationLink; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.CurationLink; @Service public class CurationLinkResourceAssembler diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/CurationPersistService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/CurationPersistService.java index 185c3c6c3..9f3696f91 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/CurationPersistService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/CurationPersistService.java @@ -17,9 +17,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.CurationLink; -import uk.ac.ebi.biosamples.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.CurationLink; +import uk.ac.ebi.biosamples.core.model.Relationship; import uk.ac.ebi.biosamples.mongo.model.MongoCuration; import uk.ac.ebi.biosamples.mongo.model.MongoCurationLink; import uk.ac.ebi.biosamples.mongo.repository.MongoCurationLinkRepository; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/CurationResourceAssembler.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/CurationResourceAssembler.java index e8c0fdefc..4761b5755 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/CurationResourceAssembler.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/CurationResourceAssembler.java @@ -17,7 +17,7 @@ import org.springframework.hateoas.server.mvc.WebMvcLinkBuilder; import org.springframework.stereotype.Service; import uk.ac.ebi.biosamples.controller.CurationController; -import uk.ac.ebi.biosamples.model.Curation; +import uk.ac.ebi.biosamples.core.model.Curation; @Service public class CurationResourceAssembler diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetService.java index 5f77547a5..96d2de2e2 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetService.java @@ -18,8 +18,8 @@ import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.facet.Facet; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.solr.service.SolrFacetService; @Service diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FileDownloadInputStream.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FileDownloadInputStream.java index 34f860269..49cfabe73 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FileDownloadInputStream.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FileDownloadInputStream.java @@ -17,8 +17,9 @@ import java.util.LinkedList; import java.util.Queue; import org.apache.commons.io.IOUtils; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.service.FileDownloadSerializer; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; public class FileDownloadInputStream extends InputStream { diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FileDownloadService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FileDownloadService.java index d570a581b..cac65ff69 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FileDownloadService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FileDownloadService.java @@ -18,7 +18,8 @@ import java.util.zip.ZipOutputStream; import org.springframework.stereotype.Service; import org.springframework.util.StreamUtils; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.service.FileDownloadSerializer; @Service public class FileDownloadService { diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/JsonLDService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/JsonLDService.java index 99a58010c..ffee3cae0 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/JsonLDService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/JsonLDService.java @@ -23,7 +23,9 @@ import org.springframework.ui.Model; import uk.ac.ebi.biosamples.controller.SampleHtmlController; import uk.ac.ebi.biosamples.controller.SampleRestController; -import uk.ac.ebi.biosamples.model.*; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.jsonld.model.*; +import uk.ac.ebi.biosamples.jsonld.service.SampleToJsonLDSampleRecordConverter; /** This servise is meant for the conversions jobs to/form ld+json */ @Service diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/RecommendationService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/RecommendationService.java index 5b1fa0ef2..7cdebdbbd 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/RecommendationService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/RecommendationService.java @@ -16,10 +16,12 @@ import java.util.stream.Collectors; import javax.annotation.PostConstruct; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.AttributeRecommendation; -import uk.ac.ebi.biosamples.model.CuramiRecommendation; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.curami.model.AttributeRecommendation; +import uk.ac.ebi.biosamples.curami.model.CuramiRecommendation; +import uk.ac.ebi.biosamples.curami.service.CuramiUtils; +import uk.ac.ebi.biosamples.curami.service.DataLoader; @Service public class RecommendationService { diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleManipulationService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleManipulationService.java index d21191dd5..b6ad0e27b 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleManipulationService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleManipulationService.java @@ -16,8 +16,8 @@ import java.util.TreeSet; import java.util.stream.Collectors; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Contact; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Contact; +import uk.ac.ebi.biosamples.core.model.Sample; @Service public class SampleManipulationService { diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java index 55daaa083..67b56fc20 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java @@ -15,8 +15,8 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.mongo.model.MongoCurationLink; import uk.ac.ebi.biosamples.mongo.model.MongoSample; import uk.ac.ebi.biosamples.mongo.repository.MongoCurationLinkRepository; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleResourceAssembler.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleResourceAssembler.java index 255d98656..eb4dbaacd 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleResourceAssembler.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleResourceAssembler.java @@ -24,7 +24,7 @@ import uk.ac.ebi.biosamples.controller.SampleCurationLinksController; import uk.ac.ebi.biosamples.controller.SampleRestController; import uk.ac.ebi.biosamples.controller.StructuredDataController; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; /** * This class is used by Spring to add HAL _links for {@Link Sample} objects. diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleRestResponseBodyAdvice.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleRestResponseBodyAdvice.java index 1f224cde1..ca2c8fe8d 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleRestResponseBodyAdvice.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleRestResponseBodyAdvice.java @@ -25,7 +25,7 @@ import org.springframework.web.bind.annotation.RestControllerAdvice; import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; import uk.ac.ebi.biosamples.controller.SampleRestController; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; @RestControllerAdvice(assignableTypes = SampleRestController.class) public class SampleRestResponseBodyAdvice implements ResponseBodyAdvice> { diff --git a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/SampleService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleService.java similarity index 96% rename from utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/SampleService.java rename to webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleService.java index c639ed97c..b619d02af 100644 --- a/utils/webapp/src/main/java/uk/ac/ebi/biosamples/service/SampleService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SampleService.java @@ -1,769 +1,768 @@ -/* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.service; - -import static java.util.stream.Collectors.toSet; -import static uk.ac.ebi.biosamples.BioSamplesConstants.*; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.*; -import java.util.stream.Stream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.userdetails.UserDetails; -import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.BioSamplesProperties; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.SubmittedViaType; -import uk.ac.ebi.biosamples.model.structured.AbstractData; -import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; -import uk.ac.ebi.biosamples.mongo.model.MongoSample; -import uk.ac.ebi.biosamples.mongo.model.MongoSampleMessage; -import uk.ac.ebi.biosamples.mongo.repository.MongoSampleMessageRepository; -import uk.ac.ebi.biosamples.mongo.repository.MongoSampleRepository; -import uk.ac.ebi.biosamples.mongo.service.*; -import uk.ac.ebi.biosamples.service.security.BioSamplesCrossSourceIngestAccessControlService; - -/** - * Service layer business logic for centralising repository access and conversions between different - * controller. Use this instead of linking to repositories directly. - * - * @author faulcon - */ -@Service -public class SampleService { - private static final Logger log = LoggerFactory.getLogger(SampleService.class); - private final MongoAccessionService mongoAccessionService; - private final MongoSampleRepository mongoSampleRepository; - private final MongoSampleMessageRepository mongoSampleMessageRepository; - private final MongoSampleToSampleConverter mongoSampleToSampleConverter; - private final SampleToMongoSampleConverter sampleToMongoSampleConverter; - private final MongoRelationshipToRelationshipConverter mongoRelationshipToRelationshipConverter; - private final SampleValidator sampleValidator; - private final SampleReadService sampleReadService; - private final MessagingService messagingService; - private final BioSamplesProperties bioSamplesProperties; - private final BioSamplesCrossSourceIngestAccessControlService - bioSamplesCrossSourceIngestAccessControlService; - - @Autowired - public SampleService( - @Qualifier("SampleAccessionService") final MongoAccessionService mongoAccessionService, - final MongoSampleRepository mongoSampleRepository, - final MongoSampleMessageRepository mongoSampleMessageRepository, - final MongoSampleToSampleConverter mongoSampleToSampleConverter, - final SampleToMongoSampleConverter sampleToMongoSampleConverter, - final MongoRelationshipToRelationshipConverter mongoRelationshipToRelationshipConverter, - final SampleValidator sampleValidator, - final SampleReadService sampleReadService, - final MessagingService messagingService, - final BioSamplesProperties bioSamplesProperties, - final BioSamplesCrossSourceIngestAccessControlService - bioSamplesCrossSourceIngestAccessControlService) { - this.mongoAccessionService = mongoAccessionService; - this.mongoSampleRepository = mongoSampleRepository; - this.mongoSampleMessageRepository = mongoSampleMessageRepository; - this.mongoSampleToSampleConverter = mongoSampleToSampleConverter; - this.sampleToMongoSampleConverter = sampleToMongoSampleConverter; - this.mongoRelationshipToRelationshipConverter = mongoRelationshipToRelationshipConverter; - this.sampleValidator = sampleValidator; - this.sampleReadService = sampleReadService; - this.messagingService = messagingService; - this.bioSamplesProperties = bioSamplesProperties; - this.bioSamplesCrossSourceIngestAccessControlService = - bioSamplesCrossSourceIngestAccessControlService; - } - - /** Throws an IllegalArgumentException of no sample with that accession exists */ - public Optional fetch(final String accession, final boolean applyCurations) { - return sampleReadService.fetch(accession, applyCurations); - } - - /* - Checks if the current sample that exists has no metadata, returns true if empty - */ - private boolean isStoredSampleEmpty( - final Sample newSample, final boolean isWebinSuperUser, final Sample oldSample) { - if (isWebinSuperUser) { - if (newSample.getSubmittedVia() == SubmittedViaType.FILE_UPLOADER) { - // file uploader submissions are done via superuser, but they are non-imported samples, - // needs to be handled safely - if (newSample.hasAccession()) { - return isStoredSampleEmpty(oldSample); - } - - return true; - } else { - // otherwise it is an ENA pipeline import, cannot be empty - return false; - } - } else { - if (newSample.hasAccession()) { - return isStoredSampleEmpty(oldSample); - } - } - - if (newSample.hasAccession()) { - return isStoredSampleEmpty(oldSample); - } - - return true; - } - - /* - Checks if the current sample that exists has no metadata, returns true if empty - */ - private boolean isStoredSampleEmpty(final Sample oldSample) { - if (oldSample.getTaxId() != null && oldSample.getTaxId() > 0) { - return false; - } - - if (!oldSample.getAttributes().isEmpty()) { - return false; - } - - if (!oldSample.getRelationships().isEmpty()) { - return false; - } - - if (!oldSample.getPublications().isEmpty()) { - return false; - } - - if (!oldSample.getContacts().isEmpty()) { - return false; - } - - if (!oldSample.getOrganizations().isEmpty()) { - return false; - } - - if (!oldSample.getData().isEmpty()) { - return false; - } - - if (!oldSample.getExternalReferences().isEmpty()) { - return false; - } - - return oldSample.getStructuredData().isEmpty(); - } - - // Because the fetch caches the sample, if an updated version is stored, we need to make - // sure - // that any cached version - // is removed. - // Note, pages of samples will not be cache busted, only single-accession sample retrieval - // @CacheEvict(cacheNames=WebappProperties.fetchUsing, key="#result.accession") - /* - Called by V1 endpoints to persist samples - */ - public Sample persistSample( - Sample newSample, final Sample oldSample, final boolean isWebinSuperUser) { - final Collection errors = sampleValidator.validate(newSample); - - if (!errors.isEmpty()) { - log.error("Sample validation failed : {}", errors); - - throw new GlobalExceptions.SampleMandatoryFieldsMissingException(String.join("|", errors)); - } - - if (newSample.hasAccession()) { - if (oldSample != null) { - newSample = updateFromCurrent(newSample, oldSample, isWebinSuperUser); - } else { - newSample = updateWhenNoneExists(newSample); - } - - MongoSample mongoSample = sampleToMongoSampleConverter.convert(newSample); - mongoSample = mongoSampleRepository.save(mongoSample); - - if (isTaxIdUpdated(oldSample, newSample)) { - mongoSampleMessageRepository.save( - new MongoSampleMessage(newSample.getAccession(), Instant.now(), newSample.getTaxId())); - } - - newSample = mongoSampleToSampleConverter.apply(mongoSample); - sendMessageToRabbitForIndexingToSolr( - newSample.getAccession(), getExistingRelationshipTargetsForIndexingInSolr(oldSample)); - } else { - newSample = createNew(newSample); - } - - // fetch returns sample with curations applied - final Optional sampleOptional = fetch(newSample.getAccession(), true); - - return sampleOptional.orElseThrow( - () -> - new RuntimeException( - "Failed to create sample. Please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk")); - } - - private Sample updateWhenNoneExists(Sample newSample) { - log.error("Trying to update sample not in database, accession: {}", newSample.getAccession()); - - bioSamplesCrossSourceIngestAccessControlService - .validateFileUploaderSampleUpdateHasExistingAccession(newSample); - - final SortedSet newSampleAttributes = newSample.getAttributes(); - - if (newSampleAttributes.stream() - .noneMatch(attribute -> attribute.getType().equals(SRA_ACCESSION))) { - // Don't generate SRA accession (ERS sample accessions) for NCBI samples - if (!isNcbiImport(newSample)) { - final String sraAccession = generateOneSRAAccession(); - - newSampleAttributes.add(Attribute.build(SRA_ACCESSION, sraAccession)); - newSample = Sample.Builder.fromSample(newSample).withSraAccession(sraAccession).build(); - } - } else { - final List sraAccessionAttributeList = - newSampleAttributes.stream() - .filter(attribute -> attribute.getType().equals(SRA_ACCESSION)) - .toList(); - - if (sraAccessionAttributeList.size() > 1) { - throw new GlobalExceptions.InvalidSampleException(); - } - - final String sraAccession = sraAccessionAttributeList.get(0).getValue(); - - newSample = Sample.Builder.fromSample(newSample).withSraAccession(sraAccession).build(); - } - - return newSample; - } - - private Sample updateFromCurrent( - Sample newSample, final Sample oldSample, final boolean isWebinSuperUser) { - final boolean savedSampleEmpty = isStoredSampleEmpty(newSample, isWebinSuperUser, oldSample); - - if (savedSampleEmpty) { - newSample = Sample.Builder.fromSample(newSample).withSubmitted(Instant.now()).build(); - } - - final List existingRelationships = - getExistingRelationshipTargetsForIndexingInSolr( - newSample.getAccession(), - Objects.requireNonNull(sampleToMongoSampleConverter.convert(oldSample))); - - return compareWithExistingAndUpdateSample( - newSample, oldSample, existingRelationships, savedSampleEmpty, isWebinSuperUser); - } - - private Sample createNew(Sample newSample) { - final boolean noSraAccession = - newSample.getAttributes().stream() - .noneMatch(attribute -> attribute.getType().equals(SRA_ACCESSION)); - - if (!noSraAccession) { - newSample = validateAndPromoteSRAAccessionAttributeToField(newSample); - } - - newSample = mongoAccessionService.generateAccession(newSample, noSraAccession); - sendMessageToRabbitForIndexingToSolr(newSample.getAccession(), Collections.emptyList()); - - return newSample; - } - - private boolean isTaxIdUpdated(final Sample oldSample, final Sample sample) { - return oldSample != null - && oldSample.getTaxId() != null - && !oldSample.getTaxId().equals(sample.getTaxId()); - } - - private List getExistingRelationshipTargetsForIndexingInSolr(final Sample oldSample) { - final List existingRelationshipTargets = new ArrayList<>(); - - if (oldSample != null) { - final List existingRelationships = - getExistingRelationshipTargetsForIndexingInSolr( - oldSample.getAccession(), - Objects.requireNonNull(sampleToMongoSampleConverter.convert(oldSample))); - - existingRelationshipTargets.addAll( - existingRelationships.stream() - .map( - relationship -> { - if (relationship.getSource().equals(oldSample.getAccession())) { - return relationship.getTarget(); - } - - return null; - }) - .toList()); - } - - return existingRelationshipTargets; - } - - /* - Called by V2 endpoints to persist samples - */ - public Sample persistSampleV2( - Sample newSample, final Sample oldSample, final boolean isWebinSuperUser) { - final Collection errors = sampleValidator.validate(newSample); - - if (!errors.isEmpty()) { - log.error("Sample validation failed : {}", errors); - throw new GlobalExceptions.SampleMandatoryFieldsMissingException(String.join("|", errors)); - } - - if (newSample.hasAccession()) { - if (oldSample != null) { - log.info( - "Trying to update sample that exists in database, accession: {}", - newSample.getAccession()); - - final boolean savedSampleEmpty = - isStoredSampleEmpty(newSample, isWebinSuperUser, oldSample); - - if (savedSampleEmpty) { - newSample = Sample.Builder.fromSample(newSample).withSubmitted(Instant.now()).build(); - } - - newSample = - compareWithExistingAndUpdateSample( - newSample, oldSample, null, savedSampleEmpty, isWebinSuperUser); - } else { - log.error( - "Trying to update sample not in database, accession: {}", newSample.getAccession()); - - newSample = updateWhenNoneExists(newSample); - } - - MongoSample mongoSample = sampleToMongoSampleConverter.convert(newSample); - - assert mongoSample != null; - - mongoSample = mongoSampleRepository.save(mongoSample); - newSample = mongoSampleToSampleConverter.apply(mongoSample); - - sendMessageToRabbitForIndexingToSolr(newSample.getAccession(), Collections.emptyList()); - } else { - newSample = createNew(newSample); - } - - return newSample; - } - - private void sendMessageToRabbitForIndexingToSolr( - final String accession, final List existingRelationshipTargets) { - try { - messagingService.fetchThenSendMessage(accession, existingRelationshipTargets); - } catch (final Exception e) { - log.error("Indexing failed for accession " + accession); - } - } - - /* - Called by V2 endpoints to build a sample with a newly generated sample accession - */ - public Sample accessionSample(Sample newSample) { - final Collection errors = sampleValidator.validate(newSample); - - if (!errors.isEmpty()) { - log.error("Sample validation failed : {}", errors); - - throw new GlobalExceptions.SampleMandatoryFieldsMissingException(String.join("|", errors)); - } - - if (newSample - .getWebinSubmissionAccountId() - .equalsIgnoreCase(bioSamplesProperties.getBiosamplesClientWebinUsername())) { - // accessioning from ENA, sample name is the SRA accession here - final Attribute sraAccessionAttribute = Attribute.build(SRA_ACCESSION, newSample.getName()); - - newSample.getAttributes().add(sraAccessionAttribute); - newSample = Sample.Builder.fromSample(newSample).build(); - - return mongoAccessionService.generateAccession(newSample, false); - } else { - return mongoAccessionService.generateAccession(newSample, true); - } - } - - public String generateOneSRAAccession() { - return mongoAccessionService.generateOneSRAAccession(); - } - - /* - Returns true if a sample does not exist in BioSamples - */ - public boolean isNotExistingAccession(final String accession) { - if (accession != null) { - return mongoSampleRepository.findById(accession).isEmpty(); - } else { - return true; - } - } - - private List getExistingRelationshipTargetsForIndexingInSolr( - final String accession, final MongoSample mongoOldSample) { - final List oldRelationshipTargets = new ArrayList<>(); - - for (final MongoRelationship mongoRelationship : mongoOldSample.getRelationships()) { - if (mongoRelationship.getSource().equals(accession)) { - oldRelationshipTargets.add( - mongoRelationshipToRelationshipConverter.convert(mongoRelationship)); - } - } - - return oldRelationshipTargets; - } - - private Sample compareWithExistingAndUpdateSample( - Sample newSample, - final Sample oldSample, - final List existingRelationships, - final boolean isEmptySample, - final boolean isWebinSuperUser) { - Set structuredData = new HashSet<>(); - boolean applyOldSampleStructuredData = false; - - // retain existing relationships for superuser submissions, pipelines, ENA POSTED, not for file - // uploads though - handleRelationships(newSample, existingRelationships); - handleSRAAccession(newSample, oldSample, isWebinSuperUser); - newSample = validateAndPromoteSRAAccessionAttributeToField(newSample); - - if (newSample.getData().isEmpty()) { - log.info("No structured data in new sample"); - - if (oldSample.getData() != null && !oldSample.getData().isEmpty()) { - structuredData = oldSample.getData(); - // Check if old sample has structured data, if yes, retain - applyOldSampleStructuredData = true; - - log.info("Old sample has structured data"); - } - } - - if (applyOldSampleStructuredData) { - log.info("Build sample and applying old sample structured data"); - log.trace("Old sample structured data size is " + structuredData.size()); - - return Sample.Builder.fromSample(newSample) - .withCreate(defineCreateDate(newSample, oldSample)) - .withSubmitted(defineSubmittedDate(newSample, oldSample, isEmptySample)) - .withData(structuredData) - .build(); - } else { - log.info("Building sample without structured data"); - - return Sample.Builder.fromSample(newSample) - .withCreate(defineCreateDate(newSample, oldSample)) - .withSubmitted(defineSubmittedDate(newSample, oldSample, isEmptySample)) - .build(); - } - } - - private Sample validateAndPromoteSRAAccessionAttributeToField(final Sample newSample) { - // Retrieve SRA accession attribute from new sample - final Optional newSampleSraAccessionOptional = - newSample.getAttributes().stream() - .filter(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION)) - .findFirst(); - - // Retrieve SRA accession field from new sample - final String sraAccessionField = newSample.getSraAccession(); - - // Check if SRA accession field and attribute are both present - if (sraAccessionField != null && newSampleSraAccessionOptional.isPresent()) { - // Check for SRA accession mismatch - if (!sraAccessionField.equals(newSampleSraAccessionOptional.get().getValue())) { - throw new GlobalExceptions.InvalidSampleException(); - } else { - // If they match, return the new sample - return newSample; - } - } - - // Check if SRA accession field is null but the attribute is present - if (sraAccessionField == null && newSampleSraAccessionOptional.isPresent()) { - // Promote SRA accession attribute to the field and return the modified sample - return Sample.Builder.fromSample(newSample) - .withSraAccession(newSampleSraAccessionOptional.get().getValue()) - .build(); - } - - // Return the original new sample if no promotion or validation is needed - return newSample; - } - - private void handleRelationships( - final Sample newSample, final List existingRelationships) { - if (existingRelationships != null && !existingRelationships.isEmpty()) { - final String webinId = newSample.getWebinSubmissionAccountId(); - - // superuser and non-file upload submissions - if (webinId != null - && webinId.equals(bioSamplesProperties.getBiosamplesClientWebinUsername())) { - if (newSample.getSubmittedVia() != SubmittedViaType.FILE_UPLOADER) { - newSample.getRelationships().addAll(existingRelationships); - } - } - } - } - - private void handleSRAAccession( - final Sample newSample, final Sample oldSample, final boolean isWebinSuperUser) { - // Step 1: Initialization - final String newSampleSraAccessionField = newSample.getSraAccession(); - - // Step 3: Validation of SRA Accession Field in New Sample - if (newSampleSraAccessionField != null && !newSampleSraAccessionField.isEmpty()) { - if (oldSample != null) { - // Check if the SRA accession field has changed in the new sample - final String oldSampleSraAccessionField = oldSample.getSraAccession(); - - /* - *

This logic performs the following checks: - *

    - *
  • If the old SRA accession field is not null and not empty.
    • - *
  • If the new SRA accession field is different from the old SRA accession field.
    • - *
  • If the user is not a Webin Super User.
    • - *
  • If the sample is not an NCBI sample in an NCBI Super User domain.
    • - *
- */ - if (oldSampleSraAccessionField != null - && !oldSampleSraAccessionField.isEmpty() - && !newSampleSraAccessionField.equals(oldSampleSraAccessionField) - && !isWebinSuperUser - && !isNcbiImport(newSample)) { - throw new GlobalExceptions.ChangedSRAAccessionException(); - } - } - } - - final SortedSet newSampleAttributes = newSample.getAttributes(); - final List newSampleSraAccessionAttributes = - newSampleAttributes.stream() - .filter(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION)) - .toList(); - - // Step 2: Validation of SRA Accession attributes in New Sample - if (newSampleSraAccessionAttributes.size() > 1) { - throw new GlobalExceptions.InvalidSampleException(); - } - - Attribute newSampleSraAccessionAttribute = - newSampleSraAccessionAttributes.isEmpty() ? null : newSampleSraAccessionAttributes.get(0); - - // Step 4: Validation of SRA Accession attributes in Old Sample - if (oldSample != null) { - final List oldSampleSraAccessionAttributes = - oldSample.getAttributes().stream() - .filter(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION)) - .toList(); - - // Check if there are more than one SRA accession attributes in the old sample - if (oldSampleSraAccessionAttributes.size() > 1) { - throw new GlobalExceptions.InvalidSampleException(); - } - - final Attribute oldSampleSraAccessionAttribute = - oldSampleSraAccessionAttributes.isEmpty() ? null : oldSampleSraAccessionAttributes.get(0); - - // Step 5: Handling SRA Accessions in New Sample and Old Sample - if (newSampleSraAccessionAttribute == null) { - // If newSampleSraAccessionAttribute is null, use oldSampleSraAccession or generate a new - // one - newSampleSraAccessionAttribute = - Objects.requireNonNullElseGet( - oldSampleSraAccessionAttribute, - () -> Attribute.build(SRA_ACCESSION, generateOneSRAAccession())); - // Add newSampleSraAccessionAttribute to the attributes of the new sample - newSampleAttributes.add(newSampleSraAccessionAttribute); - } - - // Step 6: Validation of Changed SRA Accession (if Old Sample exists) - if (oldSampleSraAccessionAttribute != null - && !oldSampleSraAccessionAttribute - .getValue() - .equals(newSampleSraAccessionAttribute.getValue()) - && !isWebinSuperUser - && !isNcbiImport(newSample)) { - throw new GlobalExceptions.ChangedSRAAccessionException(); - } - } else { - // Step 7: Handling New Samples without Old Samples (Old Sample doesn't exist) - if (newSampleSraAccessionAttribute == null && isWebinSuperUser && !isNcbiImport(newSample)) { - // If oldSample doesn't exist, and newSampleSraAccessionAttribute is still null, create, a - // new one - // Doesn't generate SRA accession (ERS sample accessions) for NCBI samples while import - // (prefix checks) - newSampleSraAccessionAttribute = Attribute.build(SRA_ACCESSION, generateOneSRAAccession()); - // Add newSampleSraAccessionAttribute to the attributes of the new sample - newSampleAttributes.add(newSampleSraAccessionAttribute); - } - } - } - - private boolean isNcbiImport(final Sample newSample) { - return (newSample.getAccession().startsWith(NCBI_ACCESSION_PREFIX) - || newSample.getAccession().startsWith(DDBJ_ACCESSION_PREFIX)) - && newSample.getSubmittedVia() == SubmittedViaType.PIPELINE_IMPORT; - } - - private Instant defineCreateDate(final Sample newSample, final Sample oldSample) { - return (oldSample.getCreate() != null ? oldSample.getCreate() : newSample.getCreate()); - } - - public boolean isPipelineEnaDomain(final String domain) { - if (domain == null) { - return false; - } - - return domain.equalsIgnoreCase(ENA_IMPORT_DOMAIN); - } - - public boolean isPipelineNcbiDomain(final String domain) { - if (domain == null) { - return false; - } - - return domain.equalsIgnoreCase(NCBI_IMPORT_DOMAIN); - } - - private Instant defineSubmittedDate( - final Sample newSample, final Sample oldSample, final boolean isEmptySample) { - if (isEmptySample) { - return newSample.getSubmitted(); - } else { - return oldSample.getSubmitted() != null ? oldSample.getSubmitted() : newSample.getSubmitted(); - } - } - - public Instant defineCreateDate(final Sample sample, final boolean isWebinSuperUserSubmission) { - final Instant now = Instant.now(); - final Instant create = sample.getCreate(); - - return isWebinSuperUserSubmission ? (create != null ? create : now) : now; - } - - public Instant defineSubmittedDate( - final Sample sample, final boolean isWebinSuperUserSubmission) { - final Instant now = Instant.now(); - final Instant submitted = sample.getSubmitted(); - - return isWebinSuperUserSubmission ? (submitted != null ? submitted : now) : now; - } - - public Sample buildPrivateSample(final Sample sample) { - final Instant release = - Instant.ofEpochSecond( - LocalDateTime.now(ZoneOffset.UTC).plusYears(100).toEpochSecond(ZoneOffset.UTC)); - final Instant update = Instant.now(); - final SubmittedViaType submittedVia = - sample.getSubmittedVia() == null ? SubmittedViaType.JSON_API : sample.getSubmittedVia(); - - return Sample.Builder.fromSample(sample) - .withRelease(release) - .withUpdate(update) - .withSubmittedVia(submittedVia) - .build(); - } - - public Set handleSampleRelationshipsV2( - final Sample sample, - final Optional oldSampleOptional, - final boolean isSuperUserSubmission) { - final SortedSet sampleRelationships = sample.getRelationships(); - - if (!isSuperUserSubmission) { - if (sampleRelationships != null && !sampleRelationships.isEmpty()) { - throw new GlobalExceptions.SampleWithRelationshipSubmissionExceptionV2(); - } - } - - if (sample.hasAccession()) { - if (oldSampleOptional.isPresent()) { - final Sample oldSample = oldSampleOptional.get(); - - if (oldSample.getRelationships() != null && !oldSample.getRelationships().isEmpty()) { - return Stream.of(oldSample.getRelationships(), sampleRelationships) - .filter(Objects::nonNull) - .flatMap(Set::stream) - .collect(toSet()); - } - } else { - return sampleRelationships; - } - } else { - if (sampleRelationships != null && !sampleRelationships.isEmpty()) { - throw new GlobalExceptions.SampleWithRelationshipSubmissionExceptionV2(); - } - } - - return null; - } - - public Optional validateSampleWithAccessionsAgainstConditionsAndGetOldSample( - final Sample sample, final boolean isWebinSuperUser) { - if (!isWebinSuperUser) { - if (sample.hasAccession() - || sample.hasSraAccession() - || sample.getAttributes() != null - && sample.getAttributes().stream() - .anyMatch(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION))) { - throw new GlobalExceptions.SampleWithAccessionSubmissionException(); - } - } else { - final List sraAccessionAttributeList = - sample.getAttributes().stream() - .filter(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION)) - .toList(); - if (sraAccessionAttributeList.size() > 1) { - throw new GlobalExceptions.InvalidSampleException(); - } else { - String sraAccession = null; - - if (!sraAccessionAttributeList.isEmpty()) { - sraAccession = sraAccessionAttributeList.get(0).getValue(); - } - - if (sraAccession != null - && sample.getSraAccession() != null - && !Objects.equals(sraAccession, sample.getSraAccession())) { - throw new GlobalExceptions.InvalidSampleException(); - } - } - - if (sample.hasAccession() && !isNotExistingAccession(sample.getAccession())) { - // fetch old sample if sample exists, - // fetch returns sample with curations applied - return fetch(sample.getAccession(), false); - } - } - - return Optional.empty(); - } - - public String getPrinciple(final Authentication loggedInUser) { - if (loggedInUser == null || loggedInUser.getPrincipal() == null) { - log.warn("Unauthenticated access detected: returning null for principal"); - return null; - } - - return loggedInUser.getPrincipal() instanceof UserDetails - ? ((UserDetails) loggedInUser.getPrincipal()).getUsername() - : loggedInUser.getPrincipal().toString(); - } -} +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import static java.util.stream.Collectors.toSet; +import static uk.ac.ebi.biosamples.BioSamplesConstants.*; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.*; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.userdetails.UserDetails; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.BioSamplesProperties; +import uk.ac.ebi.biosamples.core.model.*; +import uk.ac.ebi.biosamples.core.model.structured.AbstractData; +import uk.ac.ebi.biosamples.core.service.SampleValidator; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; +import uk.ac.ebi.biosamples.mongo.model.MongoRelationship; +import uk.ac.ebi.biosamples.mongo.model.MongoSample; +import uk.ac.ebi.biosamples.mongo.model.MongoSampleMessage; +import uk.ac.ebi.biosamples.mongo.repository.MongoSampleMessageRepository; +import uk.ac.ebi.biosamples.mongo.repository.MongoSampleRepository; +import uk.ac.ebi.biosamples.mongo.service.*; +import uk.ac.ebi.biosamples.security.service.BioSamplesCrossSourceIngestAccessControlService; + +/** + * Service layer business logic for centralising repository access and conversions between different + * controller. Use this instead of linking to repositories directly. + * + * @author faulcon + */ +@Service +public class SampleService { + private static final Logger log = LoggerFactory.getLogger(SampleService.class); + private final MongoAccessionService mongoAccessionService; + private final MongoSampleRepository mongoSampleRepository; + private final MongoSampleMessageRepository mongoSampleMessageRepository; + private final MongoSampleToSampleConverter mongoSampleToSampleConverter; + private final SampleToMongoSampleConverter sampleToMongoSampleConverter; + private final MongoRelationshipToRelationshipConverter mongoRelationshipToRelationshipConverter; + private final SampleValidator sampleValidator; + private final SampleReadService sampleReadService; + private final MessagingService messagingService; + private final BioSamplesProperties bioSamplesProperties; + private final BioSamplesCrossSourceIngestAccessControlService + bioSamplesCrossSourceIngestAccessControlService; + + @Autowired + public SampleService( + @Qualifier("SampleAccessionService") final MongoAccessionService mongoAccessionService, + final MongoSampleRepository mongoSampleRepository, + final MongoSampleMessageRepository mongoSampleMessageRepository, + final MongoSampleToSampleConverter mongoSampleToSampleConverter, + final SampleToMongoSampleConverter sampleToMongoSampleConverter, + final MongoRelationshipToRelationshipConverter mongoRelationshipToRelationshipConverter, + final SampleValidator sampleValidator, + final SampleReadService sampleReadService, + final MessagingService messagingService, + final BioSamplesProperties bioSamplesProperties, + final BioSamplesCrossSourceIngestAccessControlService + bioSamplesCrossSourceIngestAccessControlService) { + this.mongoAccessionService = mongoAccessionService; + this.mongoSampleRepository = mongoSampleRepository; + this.mongoSampleMessageRepository = mongoSampleMessageRepository; + this.mongoSampleToSampleConverter = mongoSampleToSampleConverter; + this.sampleToMongoSampleConverter = sampleToMongoSampleConverter; + this.mongoRelationshipToRelationshipConverter = mongoRelationshipToRelationshipConverter; + this.sampleValidator = sampleValidator; + this.sampleReadService = sampleReadService; + this.messagingService = messagingService; + this.bioSamplesProperties = bioSamplesProperties; + this.bioSamplesCrossSourceIngestAccessControlService = + bioSamplesCrossSourceIngestAccessControlService; + } + + /** Throws an IllegalArgumentException of no sample with that accession exists */ + public Optional fetch(final String accession, final boolean applyCurations) { + return sampleReadService.fetch(accession, applyCurations); + } + + /* + Checks if the current sample that exists has no metadata, returns true if empty + */ + private boolean isStoredSampleEmpty( + final Sample newSample, final boolean isWebinSuperUser, final Sample oldSample) { + if (isWebinSuperUser) { + if (newSample.getSubmittedVia() == SubmittedViaType.FILE_UPLOADER) { + // file uploader submissions are done via superuser, but they are non-imported samples, + // needs to be handled safely + if (newSample.hasAccession()) { + return isStoredSampleEmpty(oldSample); + } + + return true; + } else { + // otherwise it is an ENA submission reference, cannot be an empty sample + return false; + } + } else { + if (newSample.hasAccession()) { + return isStoredSampleEmpty(oldSample); + } + } + + if (newSample.hasAccession()) { + return isStoredSampleEmpty(oldSample); + } + + return true; + } + + /* + Checks if the current sample that exists has no metadata, returns true if empty + */ + private boolean isStoredSampleEmpty(final Sample oldSample) { + if (oldSample.getTaxId() != null && oldSample.getTaxId() > 0) { + return false; + } + + if (!oldSample.getAttributes().isEmpty()) { + return false; + } + + if (!oldSample.getRelationships().isEmpty()) { + return false; + } + + if (!oldSample.getPublications().isEmpty()) { + return false; + } + + if (!oldSample.getContacts().isEmpty()) { + return false; + } + + if (!oldSample.getOrganizations().isEmpty()) { + return false; + } + + if (!oldSample.getData().isEmpty()) { + return false; + } + + if (!oldSample.getExternalReferences().isEmpty()) { + return false; + } + + return oldSample.getStructuredData().isEmpty(); + } + + // Because the fetch caches the sample, if an updated version is stored, we need to make + // sure + // that any cached version + // is removed. + // Note, pages of samples will not be cache busted, only single-accession sample retrieval + // @CacheEvict(cacheNames=WebappProperties.fetchUsing, key="#result.accession") + /* + Called by V1 endpoints to persist samples + */ + public Sample persistSample( + Sample newSample, final Sample oldSample, final boolean isWebinSuperUser) { + final Collection errors = sampleValidator.validate(newSample); + + if (!errors.isEmpty()) { + log.error("Sample validation failed : {}", errors); + + throw new GlobalExceptions.SampleMandatoryFieldsMissingException(String.join("|", errors)); + } + + if (newSample.hasAccession()) { + if (oldSample != null) { + newSample = updateFromCurrent(newSample, oldSample, isWebinSuperUser); + } else { + newSample = updateWhenNoneExists(newSample); + } + + MongoSample mongoSample = sampleToMongoSampleConverter.convert(newSample); + mongoSample = mongoSampleRepository.save(mongoSample); + + if (isTaxIdUpdated(oldSample, newSample)) { + mongoSampleMessageRepository.save( + new MongoSampleMessage(newSample.getAccession(), Instant.now(), newSample.getTaxId())); + } + + newSample = mongoSampleToSampleConverter.apply(mongoSample); + sendMessageToRabbitForIndexingToSolr( + newSample.getAccession(), getExistingRelationshipTargetsForIndexingInSolr(oldSample)); + } else { + newSample = createNew(newSample); + } + + // fetch returns sample with curations applied + final Optional sampleOptional = fetch(newSample.getAccession(), true); + + return sampleOptional.orElseThrow( + () -> + new RuntimeException( + "Failed to create sample. Please contact the BioSamples Helpdesk at biosamples@ebi.ac.uk")); + } + + private Sample updateWhenNoneExists(Sample newSample) { + log.error("Trying to update sample not in database, accession: {}", newSample.getAccession()); + + bioSamplesCrossSourceIngestAccessControlService + .validateFileUploaderSampleUpdateHasExistingAccession(newSample); + + final SortedSet newSampleAttributes = newSample.getAttributes(); + + if (newSampleAttributes.stream() + .noneMatch(attribute -> attribute.getType().equals(SRA_ACCESSION))) { + // Don't generate SRA accession (ERS sample accessions) for NCBI samples + if (!isNcbiImport(newSample)) { + final String sraAccession = generateOneSRAAccession(); + + newSampleAttributes.add(Attribute.build(SRA_ACCESSION, sraAccession)); + newSample = Sample.Builder.fromSample(newSample).withSraAccession(sraAccession).build(); + } + } else { + final List sraAccessionAttributeList = + newSampleAttributes.stream() + .filter(attribute -> attribute.getType().equals(SRA_ACCESSION)) + .toList(); + + if (sraAccessionAttributeList.size() > 1) { + throw new GlobalExceptions.InvalidSampleException(); + } + + final String sraAccession = sraAccessionAttributeList.get(0).getValue(); + + newSample = Sample.Builder.fromSample(newSample).withSraAccession(sraAccession).build(); + } + + return newSample; + } + + private Sample updateFromCurrent( + Sample newSample, final Sample oldSample, final boolean isWebinSuperUser) { + final boolean savedSampleEmpty = isStoredSampleEmpty(newSample, isWebinSuperUser, oldSample); + + if (savedSampleEmpty) { + newSample = Sample.Builder.fromSample(newSample).withSubmitted(Instant.now()).build(); + } + + final List existingRelationships = + getExistingRelationshipTargetsForIndexingInSolr( + newSample.getAccession(), + Objects.requireNonNull(sampleToMongoSampleConverter.convert(oldSample))); + + return compareWithExistingAndUpdateSample( + newSample, oldSample, existingRelationships, savedSampleEmpty, isWebinSuperUser); + } + + private Sample createNew(Sample newSample) { + final boolean noSraAccession = + newSample.getAttributes().stream() + .noneMatch(attribute -> attribute.getType().equals(SRA_ACCESSION)); + + if (!noSraAccession) { + newSample = validateAndPromoteSRAAccessionAttributeToField(newSample); + } + + newSample = mongoAccessionService.generateAccession(newSample, noSraAccession); + sendMessageToRabbitForIndexingToSolr(newSample.getAccession(), Collections.emptyList()); + + return newSample; + } + + private boolean isTaxIdUpdated(final Sample oldSample, final Sample sample) { + return oldSample != null + && oldSample.getTaxId() != null + && !oldSample.getTaxId().equals(sample.getTaxId()); + } + + private List getExistingRelationshipTargetsForIndexingInSolr(final Sample oldSample) { + final List existingRelationshipTargets = new ArrayList<>(); + + if (oldSample != null) { + final List existingRelationships = + getExistingRelationshipTargetsForIndexingInSolr( + oldSample.getAccession(), + Objects.requireNonNull(sampleToMongoSampleConverter.convert(oldSample))); + + existingRelationshipTargets.addAll( + existingRelationships.stream() + .map( + relationship -> { + if (relationship.getSource().equals(oldSample.getAccession())) { + return relationship.getTarget(); + } + + return null; + }) + .toList()); + } + + return existingRelationshipTargets; + } + + /* + Called by V2 endpoints to persist samples + */ + public Sample persistSampleV2( + Sample newSample, final Sample oldSample, final boolean isWebinSuperUser) { + final Collection errors = sampleValidator.validate(newSample); + + if (!errors.isEmpty()) { + log.error("Sample validation failed : {}", errors); + throw new GlobalExceptions.SampleMandatoryFieldsMissingException(String.join("|", errors)); + } + + if (newSample.hasAccession()) { + if (oldSample != null) { + log.info( + "Trying to update sample that exists in database, accession: {}", + newSample.getAccession()); + + final boolean savedSampleEmpty = + isStoredSampleEmpty(newSample, isWebinSuperUser, oldSample); + + if (savedSampleEmpty) { + // submitted is now if metadata is first submitted after accessioning + newSample = Sample.Builder.fromSample(newSample).withSubmitted(Instant.now()).build(); + } + + newSample = + compareWithExistingAndUpdateSample( + newSample, oldSample, null, savedSampleEmpty, isWebinSuperUser); + } else { + log.error( + "Trying to update sample not in database, accession: {}", newSample.getAccession()); + + newSample = updateWhenNoneExists(newSample); + } + + MongoSample mongoSample = sampleToMongoSampleConverter.convert(newSample); + + assert mongoSample != null; + + mongoSample = mongoSampleRepository.save(mongoSample); + newSample = mongoSampleToSampleConverter.apply(mongoSample); + + sendMessageToRabbitForIndexingToSolr(newSample.getAccession(), Collections.emptyList()); + } else { + newSample = createNew(newSample); + } + + return newSample; + } + + private void sendMessageToRabbitForIndexingToSolr( + final String accession, final List existingRelationshipTargets) { + try { + messagingService.fetchThenSendMessage(accession, existingRelationshipTargets); + } catch (final Exception e) { + log.error("Indexing failed for accession " + accession); + } + } + + /* + Called by V2 endpoints to build a sample with a newly generated sample accession + */ + public Sample accessionSample(Sample newSample) { + final Collection errors = sampleValidator.validate(newSample); + + if (!errors.isEmpty()) { + log.error("Sample validation failed : {}", errors); + + throw new GlobalExceptions.SampleMandatoryFieldsMissingException(String.join("|", errors)); + } + + if (newSample + .getWebinSubmissionAccountId() + .equalsIgnoreCase(bioSamplesProperties.getBiosamplesClientWebinUsername())) { + // accessioning from ENA, sample name is the SRA accession here + final Attribute sraAccessionAttribute = Attribute.build(SRA_ACCESSION, newSample.getName()); + + newSample.getAttributes().add(sraAccessionAttribute); + newSample = Sample.Builder.fromSample(newSample).build(); + + return mongoAccessionService.generateAccession(newSample, false); + } else { + return mongoAccessionService.generateAccession(newSample, true); + } + } + + public String generateOneSRAAccession() { + return mongoAccessionService.generateOneSRAAccession(); + } + + /* + Returns true if a sample does not exist in BioSamples + */ + public boolean isNotExistingAccession(final String accession) { + if (accession != null) { + return mongoSampleRepository.findById(accession).isEmpty(); + } else { + return true; + } + } + + private List getExistingRelationshipTargetsForIndexingInSolr( + final String accession, final MongoSample mongoOldSample) { + final List oldRelationshipTargets = new ArrayList<>(); + + for (final MongoRelationship mongoRelationship : mongoOldSample.getRelationships()) { + if (mongoRelationship.getSource().equals(accession)) { + oldRelationshipTargets.add( + mongoRelationshipToRelationshipConverter.convert(mongoRelationship)); + } + } + + return oldRelationshipTargets; + } + + private Sample compareWithExistingAndUpdateSample( + Sample newSample, + final Sample oldSample, + final List existingRelationships, + final boolean isEmptySample, + final boolean isWebinSuperUser) { + Set structuredData = new HashSet<>(); + boolean applyOldSampleStructuredData = false; + + // retain existing relationships for superuser submissions, pipelines, ENA POSTED, not for file + // uploads though + handleRelationships(newSample, existingRelationships); + handleSRAAccession(newSample, oldSample, isWebinSuperUser); + newSample = validateAndPromoteSRAAccessionAttributeToField(newSample); + + if (newSample.getData().isEmpty()) { + log.info("No structured data in new sample"); + + if (oldSample.getData() != null && !oldSample.getData().isEmpty()) { + structuredData = oldSample.getData(); + // Check if old sample has structured data, if yes, retain + applyOldSampleStructuredData = true; + + log.info("Old sample has structured data"); + } + } + + if (applyOldSampleStructuredData) { + log.info("Build sample and applying old sample structured data"); + log.trace("Old sample structured data size is " + structuredData.size()); + + return Sample.Builder.fromSample(newSample) + .withCreate(defineCreateDate(newSample, oldSample)) + .withSubmitted(defineSubmittedDate(newSample, oldSample, isEmptySample)) + .withData(structuredData) + .build(); + } else { + log.info("Building sample without structured data"); + + return Sample.Builder.fromSample(newSample) + .withCreate(defineCreateDate(newSample, oldSample)) + .withSubmitted(defineSubmittedDate(newSample, oldSample, isEmptySample)) + .build(); + } + } + + private Sample validateAndPromoteSRAAccessionAttributeToField(final Sample newSample) { + // Retrieve SRA accession attribute from new sample + final Optional newSampleSraAccessionOptional = + newSample.getAttributes().stream() + .filter(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION)) + .findFirst(); + + // Retrieve SRA accession field from new sample + final String sraAccessionField = newSample.getSraAccession(); + + // Check if SRA accession field and attribute are both present + if (sraAccessionField != null && newSampleSraAccessionOptional.isPresent()) { + // Check for SRA accession mismatch + if (!sraAccessionField.equals(newSampleSraAccessionOptional.get().getValue())) { + throw new GlobalExceptions.InvalidSampleException(); + } else { + // If they match, return the new sample + return newSample; + } + } + + // Check if SRA accession field is null but the attribute is present + if (sraAccessionField == null && newSampleSraAccessionOptional.isPresent()) { + // Promote SRA accession attribute to the field and return the modified sample + return Sample.Builder.fromSample(newSample) + .withSraAccession(newSampleSraAccessionOptional.get().getValue()) + .build(); + } + + // Return the original new sample if no promotion or validation is needed + return newSample; + } + + private void handleRelationships( + final Sample newSample, final List existingRelationships) { + if (existingRelationships != null && !existingRelationships.isEmpty()) { + final String webinId = newSample.getWebinSubmissionAccountId(); + + // superuser and non-file upload submissions + if (webinId != null + && webinId.equals(bioSamplesProperties.getBiosamplesClientWebinUsername())) { + if (newSample.getSubmittedVia() != SubmittedViaType.FILE_UPLOADER) { + newSample.getRelationships().addAll(existingRelationships); + } + } + } + } + + private void handleSRAAccession( + final Sample newSample, final Sample oldSample, final boolean isWebinSuperUser) { + // Step 1: Initialization + final String newSampleSraAccessionField = newSample.getSraAccession(); + + // Step 3: Validation of SRA Accession Field in New Sample + if (newSampleSraAccessionField != null && !newSampleSraAccessionField.isEmpty()) { + if (oldSample != null) { + // Check if the SRA accession field has changed in the new sample + final String oldSampleSraAccessionField = oldSample.getSraAccession(); + + /* + *

This logic performs the following checks: + *

    + *
  • If the old SRA accession field is not null and not empty.
    • + *
  • If the new SRA accession field is different from the old SRA accession field.
    • + *
  • If the user is not a Webin Super User.
    • + *
  • If the sample is not an NCBI sample in an NCBI Super User domain.
    • + *
+ */ + if (oldSampleSraAccessionField != null + && !oldSampleSraAccessionField.isEmpty() + && !newSampleSraAccessionField.equals(oldSampleSraAccessionField) + && !isWebinSuperUser + && !isNcbiImport(newSample)) { + throw new GlobalExceptions.ChangedSRAAccessionException(); + } + } + } + + final SortedSet newSampleAttributes = newSample.getAttributes(); + final List newSampleSraAccessionAttributes = + newSampleAttributes.stream() + .filter(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION)) + .toList(); + + // Step 2: Validation of SRA Accession attributes in New Sample + if (newSampleSraAccessionAttributes.size() > 1) { + throw new GlobalExceptions.InvalidSampleException(); + } + + Attribute newSampleSraAccessionAttribute = + newSampleSraAccessionAttributes.isEmpty() ? null : newSampleSraAccessionAttributes.get(0); + + // Step 4: Validation of SRA Accession attributes in Old Sample + if (oldSample != null) { + final List oldSampleSraAccessionAttributes = + oldSample.getAttributes().stream() + .filter(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION)) + .toList(); + + // Check if there are more than one SRA accession attributes in the old sample + if (oldSampleSraAccessionAttributes.size() > 1) { + throw new GlobalExceptions.InvalidSampleException(); + } + + final Attribute oldSampleSraAccessionAttribute = + oldSampleSraAccessionAttributes.isEmpty() ? null : oldSampleSraAccessionAttributes.get(0); + + // Step 5: Handling SRA Accessions in New Sample and Old Sample + if (newSampleSraAccessionAttribute == null) { + // If newSampleSraAccessionAttribute is null, use oldSampleSraAccession or generate a new + // one + newSampleSraAccessionAttribute = + Objects.requireNonNullElseGet( + oldSampleSraAccessionAttribute, + () -> Attribute.build(SRA_ACCESSION, generateOneSRAAccession())); + // Add newSampleSraAccessionAttribute to the attributes of the new sample + newSampleAttributes.add(newSampleSraAccessionAttribute); + } + + // Step 6: Validation of Changed SRA Accession (if Old Sample exists) + if (oldSampleSraAccessionAttribute != null + && !oldSampleSraAccessionAttribute + .getValue() + .equals(newSampleSraAccessionAttribute.getValue()) + && !isWebinSuperUser + && !isNcbiImport(newSample)) { + throw new GlobalExceptions.ChangedSRAAccessionException(); + } + } else { + // Step 7: Handling New Samples without Old Samples (Old Sample doesn't exist) + if (newSampleSraAccessionAttribute == null && isWebinSuperUser && !isNcbiImport(newSample)) { + // If oldSample doesn't exist, and newSampleSraAccessionAttribute is still null, create, a + // new one + // Doesn't generate SRA accession (ERS sample accessions) for NCBI samples while import + // (prefix checks) + newSampleSraAccessionAttribute = Attribute.build(SRA_ACCESSION, generateOneSRAAccession()); + // Add newSampleSraAccessionAttribute to the attributes of the new sample + newSampleAttributes.add(newSampleSraAccessionAttribute); + } + } + } + + private boolean isNcbiImport(final Sample newSample) { + return (newSample.getAccession().startsWith(NCBI_ACCESSION_PREFIX) + || newSample.getAccession().startsWith(DDBJ_ACCESSION_PREFIX)) + && newSample.getSubmittedVia() == SubmittedViaType.PIPELINE_IMPORT; + } + + private Instant defineCreateDate(final Sample newSample, final Sample oldSample) { + return (oldSample.getCreate() != null ? oldSample.getCreate() : newSample.getCreate()); + } + + public boolean isPipelineEnaDomain(final String domain) { + if (domain == null) { + return false; + } + + return domain.equalsIgnoreCase(ENA_IMPORT_DOMAIN); + } + + public boolean isPipelineNcbiDomain(final String domain) { + if (domain == null) { + return false; + } + + return domain.equalsIgnoreCase(NCBI_IMPORT_DOMAIN); + } + + private Instant defineSubmittedDate( + final Sample newSample, final Sample oldSample, final boolean isEmptySample) { + if (isEmptySample) { + return newSample.getSubmitted(); + } else { + return oldSample.getSubmitted() != null ? oldSample.getSubmitted() : newSample.getSubmitted(); + } + } + + public Instant defineCreateDate(final Sample sample, final boolean isWebinSuperUserSubmission) { + final Instant now = Instant.now(); + final Instant create = sample.getCreate(); + + return isWebinSuperUserSubmission ? (create != null ? create : now) : now; + } + + public Instant defineSubmittedDate( + final Sample sample, final boolean isWebinSuperUserSubmission) { + final Instant now = Instant.now(); + final Instant submitted = sample.getSubmitted(); + + return isWebinSuperUserSubmission ? (submitted != null ? submitted : now) : now; + } + + public Sample buildPrivateSample(final Sample sample) { + final Instant release = + Instant.ofEpochSecond( + LocalDateTime.now(ZoneOffset.UTC).plusYears(100).toEpochSecond(ZoneOffset.UTC)); + final Instant update = Instant.now(); + final SubmittedViaType submittedVia = + sample.getSubmittedVia() == null ? SubmittedViaType.JSON_API : sample.getSubmittedVia(); + + return Sample.Builder.fromSample(sample) + .withRelease(release) + .withUpdate(update) + .withSubmittedVia(submittedVia) + .build(); + } + + public Set handleSampleRelationshipsV2( + final Sample sample, + final Optional oldSampleOptional, + final boolean isSuperUserSubmission) { + final SortedSet sampleRelationships = sample.getRelationships(); + + if (!isSuperUserSubmission) { + if (sampleRelationships != null && !sampleRelationships.isEmpty()) { + throw new GlobalExceptions.SampleWithRelationshipSubmissionExceptionV2(); + } + } + + if (sample.hasAccession()) { + if (oldSampleOptional.isPresent()) { + final Sample oldSample = oldSampleOptional.get(); + + if (oldSample.getRelationships() != null && !oldSample.getRelationships().isEmpty()) { + return Stream.of(oldSample.getRelationships(), sampleRelationships) + .filter(Objects::nonNull) + .flatMap(Set::stream) + .collect(toSet()); + } + } else { + return sampleRelationships; + } + } else { + if (sampleRelationships != null && !sampleRelationships.isEmpty()) { + throw new GlobalExceptions.SampleWithRelationshipSubmissionExceptionV2(); + } + } + + return null; + } + + public Optional validateSampleWithAccessionsAgainstConditionsAndGetOldSample( + final Sample sample, final boolean isWebinSuperUser) { + if (!isWebinSuperUser) { + if (sample.hasAccession() + || sample.hasSraAccession() + || sample.getAttributes() != null + && sample.getAttributes().stream() + .anyMatch(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION))) { + throw new GlobalExceptions.SampleWithAccessionSubmissionException(); + } + } else { + final List sraAccessionAttributeList = + sample.getAttributes().stream() + .filter(attribute -> attribute.getType().equalsIgnoreCase(SRA_ACCESSION)) + .toList(); + if (sraAccessionAttributeList.size() > 1) { + throw new GlobalExceptions.InvalidSampleException(); + } else { + String sraAccession = null; + + if (!sraAccessionAttributeList.isEmpty()) { + sraAccession = sraAccessionAttributeList.get(0).getValue(); + } + + if (sraAccession != null + && sample.getSraAccession() != null + && !Objects.equals(sraAccession, sample.getSraAccession())) { + throw new GlobalExceptions.InvalidSampleException(); + } + } + + if (sample.hasAccession() && !isNotExistingAccession(sample.getAccession())) { + // fetch old sample if sample exists, + // fetch returns sample with curations applied + return fetch(sample.getAccession(), false); + } + } + + return Optional.empty(); + } + + public String getPrinciple(final Authentication loggedInUser) { + if (loggedInUser == null || loggedInUser.getPrincipal() == null) { + log.warn("Unauthenticated access detected: returning null for principal"); + return null; + } + + return loggedInUser.getPrincipal() instanceof UserDetails + ? ((UserDetails) loggedInUser.getPrincipal()).getUsername() + : loggedInUser.getPrincipal().toString(); + } +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplesRestResponseBodyAdvice.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplesRestResponseBodyAdvice.java index 86a1c1800..0a088e44e 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplesRestResponseBodyAdvice.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplesRestResponseBodyAdvice.java @@ -25,7 +25,7 @@ import org.springframework.web.bind.annotation.RestControllerAdvice; import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; import uk.ac.ebi.biosamples.controller.SamplesRestController; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; @RestControllerAdvice(assignableTypes = SamplesRestController.class) public class SamplesRestResponseBodyAdvice diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java index a19eefccd..5e2781d54 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java @@ -13,8 +13,8 @@ import java.time.LocalDate; import java.util.*; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.facet.Facet; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.mongo.model.MongoAnalytics; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; import uk.ac.ebi.biosamples.solr.service.SolrFacetService; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/WebinAuthenticationService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/WebinAuthenticationService.java new file mode 100644 index 000000000..7d5a934b8 --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/WebinAuthenticationService.java @@ -0,0 +1,314 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ +package uk.ac.ebi.biosamples.service; + +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.BioSamplesProperties; +import uk.ac.ebi.biosamples.core.model.CurationLink; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SubmittedViaType; +import uk.ac.ebi.biosamples.core.model.structured.AbstractData; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataType; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; +import uk.ac.ebi.biosamples.security.service.BioSamplesCrossSourceIngestAccessControlService; + +@Service +public class WebinAuthenticationService { + public static final String ATLANTICO_DOMAIN = "self.AtlantECO"; + private final SampleService sampleService; + private final BioSamplesCrossSourceIngestAccessControlService + bioSamplesCrossSourceIngestAccessControlService; + private final BioSamplesProperties bioSamplesProperties; + + public WebinAuthenticationService( + final SampleService sampleService, + final BioSamplesCrossSourceIngestAccessControlService + bioSamplesCrossSourceIngestAccessControlService, + final BioSamplesProperties bioSamplesProperties) { + this.sampleService = sampleService; + this.bioSamplesCrossSourceIngestAccessControlService = + bioSamplesCrossSourceIngestAccessControlService; + this.bioSamplesProperties = bioSamplesProperties; + } + + public Sample handleWebinUserSubmission( + final Sample sample, final String principle, final Optional oldSample) { + final String proxyWebinId = bioSamplesProperties.getBiosamplesClientWebinUsername(); + final String webinIdInSample = sample.getWebinSubmissionAccountId(); + + if (principle != null && !principle.isEmpty()) { + final String webinIdToSetForSample = + (webinIdInSample != null && !webinIdInSample.isEmpty()) ? webinIdInSample : proxyWebinId; + + if (sample.getAccession() != null) { + return handleSampleUpdate( + sample, oldSample, principle, proxyWebinId, webinIdToSetForSample); + } else { + return handleNewSubmission(sample, principle, proxyWebinId, webinIdToSetForSample); + } + } else { + throw new GlobalExceptions.WebinUserLoginUnauthorizedException(); + } + } + + private Sample handleSampleUpdate( + final Sample sample, + final Optional oldSample, + final String webinIdFromAuthToken, + final String proxyWebinId, + final String webinIdToSetForSample) { + if (webinIdFromAuthToken.equalsIgnoreCase(proxyWebinId)) { + return handleWebinSuperUserSampleSubmission(sample, oldSample, webinIdToSetForSample); + } else { + return handleNormalSampleUpdate(sample, oldSample, webinIdFromAuthToken); + } + } + + private Sample handleNewSubmission( + final Sample sample, + final String webinIdFromAuthToken, + final String proxyWebinId, + final String webinIdToSetForSample) { + if (webinIdFromAuthToken.equalsIgnoreCase(proxyWebinId)) { + return buildSampleWithWebinId(sample, webinIdToSetForSample); + } else { + return buildSampleWithWebinId(sample, webinIdFromAuthToken); + } + } + + private Sample handleNormalSampleUpdate( + final Sample sample, final Optional oldSample, final String webinIdFromAuthToken) { + if (oldSample.isPresent()) { + final Sample oldSampleInDb = oldSample.get(); + + existingSampleAccessibilityChecks(sample, oldSampleInDb); + + if (!webinIdFromAuthToken.equalsIgnoreCase(oldSampleInDb.getWebinSubmissionAccountId())) { + throw new GlobalExceptions.NonSubmitterUpdateAttemptException(); + } else { + return buildSampleWithWebinId(sample, webinIdFromAuthToken); + } + } else { + throw new GlobalExceptions.SampleAccessionDoesNotExistException(); + } + } + + private void existingSampleAccessibilityChecks(final Sample sample, final Sample oldSampleInDb) { + bioSamplesCrossSourceIngestAccessControlService.accessControlPipelineImportedSamples( + oldSampleInDb, sample); + bioSamplesCrossSourceIngestAccessControlService + .accessControlWebinSourcedSampleByCheckingEnaChecklistAttribute(oldSampleInDb, sample); + bioSamplesCrossSourceIngestAccessControlService + .accessControlWebinSourcedSampleByCheckingSubmittedViaType(oldSampleInDb, sample); + } + + private Sample handleWebinSuperUserSampleSubmission( + final Sample sample, final Optional oldSample, final String webinIdToSetForSample) { + if (oldSample.isPresent()) { + final Sample oldSampleInDb = oldSample.get(); + final String webinIdInOldSample = oldSampleInDb.getWebinSubmissionAccountId(); + + if (sample.getSubmittedVia() == SubmittedViaType.FILE_UPLOADER) { + fileUploaderSampleSubmissionAccessibilityChecks(sample, oldSampleInDb, webinIdInOldSample); + } + + if (webinIdInOldSample != null && !webinIdInOldSample.isEmpty()) { + return buildSampleWithWebinId(sample, webinIdToSetForSample); + } else { + final String oldSampleAapDomain = oldSampleInDb.getDomain(); + + if (sampleService.isPipelineNcbiDomain(oldSampleAapDomain)) { + return sample; + } else { + return buildSampleWithWebinId(sample, webinIdToSetForSample); + } + } + } else { + return buildSampleWithWebinId(sample, webinIdToSetForSample); + } + } + + private boolean isOwnershipChangeFromAapToWebinAllowed() { + return true; + } + + private boolean isSameDomain(final String domain, final String oldSampleAapDomain) { + return domain != null && domain.equalsIgnoreCase(oldSampleAapDomain); + } + + private void fileUploaderSampleSubmissionAccessibilityChecks( + final Sample sample, final Sample oldSampleInDb, final String webinIdInOldSample) { + bioSamplesCrossSourceIngestAccessControlService.isOriginalSubmitterInSampleMetadata( + webinIdInOldSample, sample); + existingSampleAccessibilityChecks(sample, oldSampleInDb); + } + + /*Only used for sample migration purposes*/ + private boolean isOldRegistrationDomain(final String sampleDomain) { + if (sampleDomain != null) { + return sampleDomain.equals("self.Webin") + || sampleDomain.equals("3fa5e19ccafc88187d437f92cf29c3b6694c6c6f98efa236c8aa0aeaf5b23f15") + || sampleDomain.equals("self.BiosampleImportAcccession") + || sampleDomain.equals("self.BioSamplesMigration") + || sampleDomain.startsWith("self.BioSamples") + || sampleDomain.startsWith("self.BiosampleSyntheticData"); + } else { + return false; + } + } + + public Sample buildSampleWithWebinId(final Sample sample, final String webinId) { + return Sample.Builder.fromSample(sample) + .withWebinSubmissionAccountId(webinId) + .withNoDomain() + .build(); + } + + public CurationLink handleWebinUserSubmission( + final CurationLink curationLink, final String webinId) { + if (webinId != null && !webinId.isEmpty()) { + return CurationLink.build( + curationLink.getSample(), + curationLink.getCuration(), + null, + webinId, + curationLink.getCreated()); + } else { + throw new GlobalExceptions.SampleNotAccessibleException(); + } + } + + public void isStructuredDataAccessible( + final StructuredData structuredData, final String webinId) { + structuredData + .getData() + .forEach( + data -> { + final String structuredDataSubmittersWebinId = data.getWebinSubmissionAccountId(); + + if (structuredDataSubmittersWebinId == null + || structuredDataSubmittersWebinId.isEmpty()) { + throw new GlobalExceptions.StructuredDataWebinIdMissingException(); + } else if (!webinId.equalsIgnoreCase(structuredDataSubmittersWebinId)) { + throw new GlobalExceptions.WebinUserLoginUnauthorizedException(); + } + }); + } + + public boolean isSampleSubmitter(final Sample sample, final String webinId) { + final AtomicBoolean isValidWebinSubmitter = new AtomicBoolean(false); + + sample + .getData() + .forEach( + data -> { + if (data.getDataType() != null) { + final String structuredDataSubmittersWebinId = data.getWebinSubmissionAccountId(); + + if (structuredDataSubmittersWebinId == null) { + throw new GlobalExceptions.StructuredDataWebinIdMissingException(); + } + } + }); + + if (sample.hasAccession()) { + isValidWebinSubmitter.set(isStructuredDataAccessible(sample, webinId)); + } + + if (isValidWebinSubmitter.get()) { + return true; + } else { + throw new GlobalExceptions.StructuredDataNotAccessibleException(); + } + } + + private boolean isStructuredDataAccessible(final Sample sample, final String webinId) { + final AtomicBoolean isWebinIdValid = new AtomicBoolean(false); + + // fetch returns sample with curations applied + final Optional oldSample = sampleService.fetch(sample.getAccession(), true); + + if (oldSample.isPresent()) { + final Sample oldSampleInDb = oldSample.get(); + + sample + .getData() + .forEach( + data -> { + final StructuredDataType dataType = data.getDataType(); + + if (dataType != null) { + final Optional filteredData = + oldSampleInDb.getData().stream() + .filter( + oldSampleStructuredData -> + oldSampleStructuredData.getDataType().equals(dataType)) + .findFirst(); + + final String structuredDataSubmittersWebinId = data.getWebinSubmissionAccountId(); + + if (filteredData.isPresent()) { + final AbstractData fData = filteredData.get(); + + if (!structuredDataSubmittersWebinId.equalsIgnoreCase( + fData.getWebinSubmissionAccountId())) { + throw new GlobalExceptions.StructuredDataNotAccessibleException(); + } else { + isWebinIdValid.set(true); + } + } else { + if (webinId.equalsIgnoreCase(structuredDataSubmittersWebinId)) { + isWebinIdValid.set(true); + } + } + } + }); + } else { + sample + .getData() + .forEach( + data -> { + if (data.getDataType() != null) { + if (webinId.equalsIgnoreCase(data.getWebinSubmissionAccountId())) { + isWebinIdValid.set(true); + } + } + }); + } + + return isWebinIdValid.get(); + } + + public boolean isWebinSuperUser(final String webinId) { + return webinId != null + && webinId.equalsIgnoreCase(bioSamplesProperties.getBiosamplesClientWebinUsername()); + } + + public void isSampleAccessible(final Sample sample, final String webinId) { + if (webinId == null) { + if (!sample.getRelease().isBefore(Instant.now())) { + throw new GlobalExceptions.SampleNotAccessibleException(); + } + } else { + if (!sample.getRelease().isBefore(Instant.now())) { + if (!isWebinSuperUser(webinId)) { + if (!webinId.equals(sample.getWebinSubmissionAccountId())) { + throw new GlobalExceptions.SampleNotAccessibleException(); + } + } + } + } + } +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/Certifier.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/Certifier.java index 79e754402..6409bdd45 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/Certifier.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/Certifier.java @@ -24,7 +24,7 @@ import org.springframework.stereotype.Service; import org.springframework.web.bind.annotation.ResponseStatus; import uk.ac.ebi.biosamples.model.certification.*; -import uk.ac.ebi.biosamples.validation.ValidatorI; +import uk.ac.ebi.biosamples.service.validation.ValidatorI; @Service public class Certifier { diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/CertifyService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/CertifyService.java index e365340ac..3141a6b39 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/CertifyService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/CertifyService.java @@ -18,7 +18,7 @@ import org.apache.commons.io.IOUtils; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.Certificate; +import uk.ac.ebi.biosamples.core.model.Certificate; import uk.ac.ebi.biosamples.model.certification.*; @Service diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/Interrogator.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/Interrogator.java index d53ea65ce..e0419c217 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/Interrogator.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/Interrogator.java @@ -21,7 +21,7 @@ import uk.ac.ebi.biosamples.model.certification.Checklist; import uk.ac.ebi.biosamples.model.certification.InterrogationResult; import uk.ac.ebi.biosamples.model.certification.SampleDocument; -import uk.ac.ebi.biosamples.validation.ValidatorI; +import uk.ac.ebi.biosamples.service.validation.ValidatorI; @Service public class Interrogator { diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/Validator.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/Validator.java index bcb97afa0..a128d15fa 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/Validator.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/certification/Validator.java @@ -24,9 +24,9 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; import uk.ac.ebi.biosamples.model.certification.Checklist; -import uk.ac.ebi.biosamples.validation.ValidatorI; +import uk.ac.ebi.biosamples.service.validation.ValidatorI; @Service @Qualifier("javaValidator") diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java index 8753398d5..02e547f3d 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java @@ -9,10 +9,8 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.filter.Filter; -import uk.ac.ebi.biosamples.search.grpc.SearchGrpc; -import uk.ac.ebi.biosamples.search.grpc.SearchRequest; -import uk.ac.ebi.biosamples.search.grpc.SearchResponse; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.search.grpc.*; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; import java.util.Iterator; @@ -46,15 +44,15 @@ public Page searchForAccessions(String searchTerm, Set filters, public CursorArrayList searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size) { ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 9090).usePlaintext().build(); SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); - Iterator response; + Iterator response; try { - response = stub.streamSamples(SearchRequest.newBuilder().setText(searchTerm).build()); + response = stub.streamSamples(StreamRequest.newBuilder().setText(searchTerm).build()); } catch (StatusRuntimeException e) { log.warn("Failed to fetch samples from remote server", e); throw new RuntimeException("Failed to fetch samples from remote server", e); } - SearchResponse searchResponse = response.next(); - return new CursorArrayList<>(searchResponse.getAccessionsList(), searchResponse.getPage()..getSearchAfter()); + StreamResponse searchResponse = response.next(); + return new CursorArrayList<>(searchResponse.getAccession()); } } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java index f41b55344..2787cc33c 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java @@ -2,7 +2,7 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; import java.util.Set; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java index c1ecceed2..98972e03f 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java @@ -6,7 +6,7 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.solr.model.SolrSample; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; import uk.ac.ebi.biosamples.solr.service.SolrSampleService; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/upload/FileUploadService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/upload/FileUploadService.java index 7451effee..1afa7900a 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/upload/FileUploadService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/upload/FileUploadService.java @@ -11,9 +11,8 @@ package uk.ac.ebi.biosamples.service.upload; import com.google.common.collect.Multimap; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; +import java.io.*; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; @@ -26,18 +25,18 @@ import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.Relationship; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Relationship; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; import uk.ac.ebi.biosamples.mongo.model.MongoFileUpload; import uk.ac.ebi.biosamples.mongo.repository.MongoFileUploadRepository; import uk.ac.ebi.biosamples.mongo.util.BioSamplesFileUploadSubmissionStatus; import uk.ac.ebi.biosamples.mongo.util.SampleNameAccessionPair; import uk.ac.ebi.biosamples.service.SampleService; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.validation.SchemaValidationService; import uk.ac.ebi.biosamples.utils.upload.FileUploadUtils; import uk.ac.ebi.biosamples.utils.upload.ValidationResult; -import uk.ac.ebi.biosamples.validation.SchemaValidationService; @Service public class FileUploadService { @@ -71,7 +70,6 @@ public synchronized File upload( final String webinId, final FileUploadUtils fileUploadUtils) { final ValidationResult validationResult = new ValidationResult(); - final boolean isWebin = webinId != null && webinId.toUpperCase().startsWith("WEBIN"); final String uniqueUploadId = UUID.randomUUID().toString(); String submissionDate = FileUploadUtils.formatDateString(LocalDateTime.now()); @@ -93,8 +91,9 @@ public synchronized File upload( log.info("Input file name " + fileToBeUploaded.getName()); - final FileReader fr = new FileReader(fileToBeUploaded); - final BufferedReader reader = new BufferedReader(fr); + final BufferedReader reader = + new BufferedReader( + new InputStreamReader(new FileInputStream(fileToBeUploaded), StandardCharsets.UTF_8)); final CSVParser csvParser = this.fileUploadUtils.buildParser(reader); final List headers = csvParser.getHeaderNames(); @@ -116,7 +115,7 @@ public synchronized File upload( } final List samples = - buildAndPersistSamples(csvDataMap, webinId, checklist, validationResult, isWebin); + buildAndPersistSamples(csvDataMap, webinId, checklist, validationResult); final List accessionsList = samples.stream() .filter(sample -> sample.getAccession() != null) @@ -213,8 +212,7 @@ private List buildAndPersistSamples( final List> csvDataMap, final String webinId, final String checklist, - final ValidationResult validationResult, - final boolean isWebin) { + final ValidationResult validationResult) { final Map sampleNameToAccessionMap = new LinkedHashMap<>(); final Map> sampleToMappedSample = new LinkedHashMap<>(); @@ -247,14 +245,13 @@ private List buildAndPersistSamples( }); return addRelationshipsAndThenBuildSamples( - sampleNameToAccessionMap, sampleToMappedSample, validationResult, isWebin); + sampleNameToAccessionMap, sampleToMappedSample, validationResult); } private List addRelationshipsAndThenBuildSamples( final Map sampleNameToAccessionMap, final Map> sampleToMappedSample, - final ValidationResult validationResult, - final boolean isWebin) { + final ValidationResult validationResult) { return sampleToMappedSample.entrySet().stream() .map( sampleMultimapEntry -> diff --git a/webapps/core/src/main/resources/templates/fragments/footer.html b/webapps/core/src/main/resources/templates/fragments/footer.html index 8dc542153..88731e3c0 100644 --- a/webapps/core/src/main/resources/templates/fragments/footer.html +++ b/webapps/core/src/main/resources/templates/fragments/footer.html @@ -89,7 +89,7 @@ }); - + diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/EtagTests.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/EtagTests.java index 80b7cf7a9..957adf86c 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/EtagTests.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/EtagTests.java @@ -30,13 +30,13 @@ import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.AuthToken; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.auth.AuthorizationProvider; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.security.model.AuthToken; +import uk.ac.ebi.biosamples.security.model.AuthorizationProvider; +import uk.ac.ebi.biosamples.security.service.AccessControlService; import uk.ac.ebi.biosamples.service.SampleService; -import uk.ac.ebi.biosamples.service.security.AccessControlService; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; @RunWith(SpringRunner.class) @SpringBootTest(properties = {"spring.cloud.gcp.project-id=no_project"}) diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/StructuredDataTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/StructuredDataTest.java index a579226cc..89f3c91da 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/StructuredDataTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/StructuredDataTest.java @@ -28,9 +28,9 @@ import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import org.springframework.util.StreamUtils; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.service.SampleService; -import uk.ac.ebi.biosamples.validation.SchemaValidationService; +import uk.ac.ebi.biosamples.service.validation.SchemaValidationService; @RunWith(SpringRunner.class) @SpringBootTest(properties = {"spring.cloud.gcp.project-id=no_project"}) diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/CertifierTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/CertifierTest.java index a6d303e11..40d255ed7 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/CertifierTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/CertifierTest.java @@ -27,8 +27,8 @@ import uk.ac.ebi.biosamples.BioSamplesProperties; import uk.ac.ebi.biosamples.model.certification.*; import uk.ac.ebi.biosamples.service.certification.*; -import uk.ac.ebi.biosamples.validation.ElixirSchemaValidator; -import uk.ac.ebi.biosamples.validation.ValidatorI; +import uk.ac.ebi.biosamples.service.validation.ElixirSchemaValidator; +import uk.ac.ebi.biosamples.service.validation.ValidatorI; @RunWith(SpringRunner.class) @SpringBootTest( diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/CertifyServiceTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/CertifyServiceTest.java index c9569104d..f2bcd5072 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/CertifyServiceTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/CertifyServiceTest.java @@ -24,8 +24,8 @@ import uk.ac.ebi.biosamples.model.certification.BioSamplesCertificationComplainceResult; import uk.ac.ebi.biosamples.model.certification.SampleDocument; import uk.ac.ebi.biosamples.service.certification.*; -import uk.ac.ebi.biosamples.validation.ElixirSchemaValidator; -import uk.ac.ebi.biosamples.validation.ValidatorI; +import uk.ac.ebi.biosamples.service.validation.ElixirSchemaValidator; +import uk.ac.ebi.biosamples.service.validation.ValidatorI; @RunWith(SpringRunner.class) @SpringBootTest( diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/CuratorTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/CuratorTest.java index d05208ce8..461f02300 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/CuratorTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/CuratorTest.java @@ -31,8 +31,8 @@ import uk.ac.ebi.biosamples.BioSamplesProperties; import uk.ac.ebi.biosamples.model.certification.*; import uk.ac.ebi.biosamples.service.certification.*; -import uk.ac.ebi.biosamples.validation.ElixirSchemaValidator; -import uk.ac.ebi.biosamples.validation.ValidatorI; +import uk.ac.ebi.biosamples.service.validation.ElixirSchemaValidator; +import uk.ac.ebi.biosamples.service.validation.ValidatorI; @RunWith(SpringRunner.class) @SpringBootTest( diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/FileRecorderTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/FileRecorderTest.java index f10a272f3..f31a66db5 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/FileRecorderTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/FileRecorderTest.java @@ -28,8 +28,8 @@ import uk.ac.ebi.biosamples.BioSamplesProperties; import uk.ac.ebi.biosamples.model.certification.*; import uk.ac.ebi.biosamples.service.certification.*; -import uk.ac.ebi.biosamples.validation.ElixirSchemaValidator; -import uk.ac.ebi.biosamples.validation.ValidatorI; +import uk.ac.ebi.biosamples.service.validation.ElixirSchemaValidator; +import uk.ac.ebi.biosamples.service.validation.ValidatorI; @RunWith(SpringRunner.class) @SpringBootTest( diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/IdentifierTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/IdentifierTest.java index 50bc94250..6adeabe8f 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/IdentifierTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/IdentifierTest.java @@ -25,8 +25,8 @@ import uk.ac.ebi.biosamples.BioSamplesProperties; import uk.ac.ebi.biosamples.model.certification.SampleDocument; import uk.ac.ebi.biosamples.service.certification.*; -import uk.ac.ebi.biosamples.validation.ElixirSchemaValidator; -import uk.ac.ebi.biosamples.validation.ValidatorI; +import uk.ac.ebi.biosamples.service.validation.ElixirSchemaValidator; +import uk.ac.ebi.biosamples.service.validation.ValidatorI; @RunWith(SpringRunner.class) @SpringBootTest( diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/InterrogatorTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/InterrogatorTest.java index a08f27458..cce1db37f 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/InterrogatorTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/InterrogatorTest.java @@ -27,8 +27,8 @@ import uk.ac.ebi.biosamples.model.certification.InterrogationResult; import uk.ac.ebi.biosamples.model.certification.SampleDocument; import uk.ac.ebi.biosamples.service.certification.*; -import uk.ac.ebi.biosamples.validation.ElixirSchemaValidator; -import uk.ac.ebi.biosamples.validation.ValidatorI; +import uk.ac.ebi.biosamples.service.validation.ElixirSchemaValidator; +import uk.ac.ebi.biosamples.service.validation.ValidatorI; @RunWith(SpringRunner.class) @SpringBootTest( diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/ValidatorTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/ValidatorTest.java index ffa4facfb..ba7b8fb86 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/ValidatorTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/certification/service/ValidatorTest.java @@ -22,8 +22,8 @@ import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.BioSamplesProperties; import uk.ac.ebi.biosamples.service.certification.*; -import uk.ac.ebi.biosamples.validation.ElixirSchemaValidator; -import uk.ac.ebi.biosamples.validation.ValidatorI; +import uk.ac.ebi.biosamples.service.validation.ElixirSchemaValidator; +import uk.ac.ebi.biosamples.service.validation.ValidatorI; @RunWith(SpringRunner.class) @SpringBootTest( diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/controller/SampleRestControllerTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/controller/SampleRestControllerTest.java index a36c11570..d158dc9af 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/controller/SampleRestControllerTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/controller/SampleRestControllerTest.java @@ -27,8 +27,8 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.context.WebApplicationContext; +import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.docs.DocumentationHelper; -import uk.ac.ebi.biosamples.model.Sample; import uk.ac.ebi.biosamples.mongo.service.CurationReadService; import uk.ac.ebi.biosamples.service.CurationPersistService; import uk.ac.ebi.biosamples.service.SamplePageService; diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/controller/SamplesRestControllerTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/controller/SamplesRestControllerTest.java index 7c53dd1c2..03f7e69b7 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/controller/SamplesRestControllerTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/controller/SamplesRestControllerTest.java @@ -34,16 +34,16 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.context.WebApplicationContext; import uk.ac.ebi.biosamples.BioSamplesProperties; +import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.docs.DocumentationHelper; -import uk.ac.ebi.biosamples.model.Sample; import uk.ac.ebi.biosamples.mongo.service.CurationReadService; import uk.ac.ebi.biosamples.mongo.service.SampleReadService; import uk.ac.ebi.biosamples.security.TestSecurityConfig; import uk.ac.ebi.biosamples.service.*; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; import uk.ac.ebi.biosamples.service.taxonomy.TaxonomyClientService; +import uk.ac.ebi.biosamples.service.validation.SchemaValidationService; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; -import uk.ac.ebi.biosamples.validation.SchemaValidationService; @RunWith(SpringRunner.class) @SpringBootTest(properties = {"spring.cloud.gcp.project-id=no_project"}) diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/docs/ApiDocumentationTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/docs/ApiDocumentationTest.java index af36f1103..96941dd6c 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/docs/ApiDocumentationTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/docs/ApiDocumentationTest.java @@ -49,19 +49,22 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.context.WebApplicationContext; -import uk.ac.ebi.biosamples.exceptions.GlobalExceptions; -import uk.ac.ebi.biosamples.model.*; -import uk.ac.ebi.biosamples.model.Curation; -import uk.ac.ebi.biosamples.model.auth.AuthorizationProvider; -import uk.ac.ebi.biosamples.model.structured.StructuredData; +import uk.ac.ebi.biosamples.core.model.Curation; +import uk.ac.ebi.biosamples.core.model.CurationLink; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.SampleStatus; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; +import uk.ac.ebi.biosamples.exception.GlobalExceptions; import uk.ac.ebi.biosamples.mongo.service.CurationReadService; import uk.ac.ebi.biosamples.security.TestSecurityConfig; +import uk.ac.ebi.biosamples.security.model.AuthToken; +import uk.ac.ebi.biosamples.security.model.AuthorizationProvider; +import uk.ac.ebi.biosamples.security.service.AccessControlService; import uk.ac.ebi.biosamples.service.*; -import uk.ac.ebi.biosamples.service.security.AccessControlService; -import uk.ac.ebi.biosamples.service.security.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.WebinAuthenticationService; import uk.ac.ebi.biosamples.service.taxonomy.TaxonomyClientService; +import uk.ac.ebi.biosamples.service.validation.SchemaValidationService; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; -import uk.ac.ebi.biosamples.validation.SchemaValidationService; @RunWith(SpringRunner.class) @SpringBootTest(properties = {"spring.cloud.gcp.project-id=no_project"}) diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/docs/DocumentationHelper.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/docs/DocumentationHelper.java index 5de0340b6..f874b6f23 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/docs/DocumentationHelper.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/docs/DocumentationHelper.java @@ -12,10 +12,10 @@ import java.time.Instant; import java.util.*; -import uk.ac.ebi.biosamples.model.*; -import uk.ac.ebi.biosamples.model.structured.StructuredData; -import uk.ac.ebi.biosamples.model.structured.StructuredDataEntry; -import uk.ac.ebi.biosamples.model.structured.StructuredDataTable; +import uk.ac.ebi.biosamples.core.model.*; +import uk.ac.ebi.biosamples.core.model.structured.StructuredData; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataEntry; +import uk.ac.ebi.biosamples.core.model.structured.StructuredDataTable; public class DocumentationHelper { public static final String WEBIN_SUBMISSION_ACCOUNT_ID = "Webin-12345"; diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/FileDownloadInputStreamTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/FileDownloadInputStreamTest.java index 0810a6272..01569a25b 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/FileDownloadInputStreamTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/FileDownloadInputStreamTest.java @@ -25,9 +25,10 @@ import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.Sample; -import uk.ac.ebi.biosamples.model.filter.Filter; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.core.service.FileDownloadSerializer; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; @RunWith(MockitoJUnitRunner.class) diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/RecommendationServiceTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/RecommendationServiceTest.java index 16ddff119..e75a7aafa 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/RecommendationServiceTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/RecommendationServiceTest.java @@ -21,10 +21,10 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; -import uk.ac.ebi.biosamples.model.Attribute; -import uk.ac.ebi.biosamples.model.AttributeRecommendation; -import uk.ac.ebi.biosamples.model.CuramiRecommendation; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Attribute; +import uk.ac.ebi.biosamples.core.model.Sample; +import uk.ac.ebi.biosamples.curami.model.AttributeRecommendation; +import uk.ac.ebi.biosamples.curami.model.CuramiRecommendation; @RunWith(SpringRunner.class) @SpringBootTest(properties = {"spring.cloud.gcp.project-id=no_project"}) diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/security/AccessControlServiceTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/security/AccessControlServiceTest.java index e9ff3714c..e88f18843 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/security/AccessControlServiceTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/security/AccessControlServiceTest.java @@ -14,6 +14,7 @@ import junit.framework.TestCase; import org.springframework.http.HttpStatus; import org.springframework.web.server.ResponseStatusException; +import uk.ac.ebi.biosamples.security.service.AccessControlService; public class AccessControlServiceTest extends TestCase { private static final String EXPIRED_WEBIN_TOKEN = diff --git a/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/taxonomy/TaxonomyClientServiceTest.java b/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/taxonomy/TaxonomyClientServiceTest.java index 1519966b6..f2f96b4d0 100644 --- a/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/taxonomy/TaxonomyClientServiceTest.java +++ b/webapps/core/src/test/java/uk/ac/ebi/biosamples/service/taxonomy/TaxonomyClientServiceTest.java @@ -15,7 +15,7 @@ import java.io.IOException; import org.junit.Assert; import org.junit.Test; -import uk.ac.ebi.biosamples.model.Sample; +import uk.ac.ebi.biosamples.core.model.Sample; public class TaxonomyClientServiceTest { @Test diff --git a/webapps/pom.xml b/webapps/pom.xml index fcf79cbd9..9877eb6e4 100644 --- a/webapps/pom.xml +++ b/webapps/pom.xml @@ -9,7 +9,7 @@ uk.ac.ebi.biosamples biosamples - 5.3.12-SNAPSHOT + 5.3.13-SNAPSHOT From 78b94a89fe54cb2cb5e17a65844c9753df67d379 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 8 Aug 2025 12:03:43 +0100 Subject: [PATCH 06/86] facet integration partial --- .../solr/service/SolrFilterService.java | 4 +- docker-compose.yml | 4 +- .../AnalyticsApplicationRunner.java | 6 +- webapps/core/pom.xml | 6 ++ .../controller/SampleFacetController.java | 6 +- .../controller/SampleHtmlController.java | 4 +- ...FacetService.java => FacetingService.java} | 21 +++-- .../biosamples/service/SamplePageService.java | 2 +- .../ebi/biosamples/service/StatService.java | 6 +- .../service/facet/ElasticFacetService.java | 90 +++++++++++++++++++ .../service/facet/FacetService.java | 19 ++++ .../service/facet}/SolrFacetService.java | 41 +++++---- .../service/search/ElasticSearchService.java | 29 ++++-- .../service/search/GrpcFilterUtils.java | 90 +++++++++++++++++++ .../service/search/SearchAfterPage.java | 19 ++++ 15 files changed, 298 insertions(+), 49 deletions(-) rename webapps/core/src/main/java/uk/ac/ebi/biosamples/service/{FacetService.java => FacetingService.java} (80%) create mode 100644 webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java create mode 100644 webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetService.java rename {core/src/main/java/uk/ac/ebi/biosamples/solr/service => webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet}/SolrFacetService.java (88%) create mode 100644 webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/GrpcFilterUtils.java create mode 100644 webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java diff --git a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java index 7ceecb839..d3828ed4f 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFilterService.java @@ -75,7 +75,7 @@ public Optional> getCompatibleFilters( * @param filters a collection of filters * @return the corresponding filter query */ - List getFilterQuery(final Collection filters) { + public List getFilterQuery(final Collection filters) { if (filters == null || filters.size() == 0) { return Collections.emptyList(); } @@ -126,7 +126,7 @@ List getFilterQuery(final Collection filters) { * * @return a filter query for public and domain relevant samples */ - Optional getPublicFilterQuery(final String webinSubmissionAccountId) { + public Optional getPublicFilterQuery(final String webinSubmissionAccountId) { // check if this is a read superuser if (webinSubmissionAccountId != null && webinSubmissionAccountId.equalsIgnoreCase( diff --git a/docker-compose.yml b/docker-compose.yml index cc4c01570..0afbc05dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2' services: biosamples-webapps-core: @@ -88,7 +87,6 @@ services: - GOOGLE_APPLICATION_CREDENTIALS=/secrets/prj-int-dev-omics-apps-mon-1fba094e79b2.json ports: - 8081:8080 - - 9090:9090 - 8000:8000 biosamples-webapps-core-v2: @@ -406,7 +404,7 @@ services: - rabbitmq_data:/var/lib/rabbitmq/mnesia solr: - image: solr:8.11.2-slim + image: solr:8.11.2 mem_limit: 1g ports: - 8983:8983 diff --git a/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java b/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java index 888bc3fe5..6ebff7c14 100644 --- a/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java +++ b/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java @@ -26,7 +26,7 @@ import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountEntry; import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; -import uk.ac.ebi.biosamples.service.FacetService; +import uk.ac.ebi.biosamples.service.FacetingService; import uk.ac.ebi.biosamples.service.SamplePageService; @Component @@ -34,12 +34,12 @@ public class AnalyticsApplicationRunner implements ApplicationRunner { private static final Logger LOG = LoggerFactory.getLogger(AnalyticsApplicationRunner.class); private final AnalyticsService analyticsService; private final PipelineFutureCallback pipelineFutureCallback; - private final FacetService facetService; + private final FacetingService facetService; private final SamplePageService samplePageService; public AnalyticsApplicationRunner( final AnalyticsService analyticsService, - final FacetService facetService, + final FacetingService facetService, final SamplePageService samplePageService) { this.analyticsService = analyticsService; this.facetService = facetService; diff --git a/webapps/core/pom.xml b/webapps/core/pom.xml index bf28874a1..b53f20a0c 100644 --- a/webapps/core/pom.xml +++ b/webapps/core/pom.xml @@ -31,6 +31,12 @@ proto 0.0.1-SNAPSHOT + + com.google.protobuf + protobuf-java + 3.25.5 + + org.springframework.boot diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java index 3265b7e08..ebb2141d6 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java @@ -22,7 +22,7 @@ import org.springframework.web.bind.annotation.*; import uk.ac.ebi.biosamples.core.model.facet.Facet; import uk.ac.ebi.biosamples.core.model.filter.Filter; -import uk.ac.ebi.biosamples.service.FacetService; +import uk.ac.ebi.biosamples.service.FacetingService; import uk.ac.ebi.biosamples.service.FilterService; import uk.ac.ebi.biosamples.utils.LinkUtils; @@ -30,10 +30,10 @@ @ExposesResourceFor(Facet.class) @RequestMapping("/samples/facets") public class SampleFacetController { - private final FacetService facetService; + private final FacetingService facetService; private final FilterService filterService; - public SampleFacetController(final FacetService facetService, final FilterService filterService) { + public SampleFacetController(final FacetingService facetService, final FilterService filterService) { this.facetService = facetService; this.filterService = filterService; } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java index 105d7b2db..4726178de 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java @@ -57,7 +57,7 @@ public class SampleHtmlController { private final SampleService sampleService; private final SamplePageService samplePageService; private final JsonLDService jsonLDService; - private final FacetService facetService; + private final FacetingService facetService; private final FilterService filterService; private final BioSamplesProperties bioSamplesProperties; private final WebinAuthenticationService webinAuthenticationService; @@ -66,7 +66,7 @@ public SampleHtmlController( final SampleService sampleService, final SamplePageService samplePageService, final JsonLDService jsonLDService, - final FacetService facetService, + final FacetingService facetService, final FilterService filterService, final BioSamplesProperties bioSamplesProperties, final WebinAuthenticationService webinAuthenticationService) { diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetingService.java similarity index 80% rename from webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetService.java rename to webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetingService.java index 96d2de2e2..bb1541717 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetingService.java @@ -10,26 +10,29 @@ */ package uk.ac.ebi.biosamples.service; -import java.util.Collection; -import java.util.List; import org.apache.solr.client.solrj.util.ClientUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; import uk.ac.ebi.biosamples.core.model.facet.Facet; import uk.ac.ebi.biosamples.core.model.filter.Filter; -import uk.ac.ebi.biosamples.solr.service.SolrFacetService; +import uk.ac.ebi.biosamples.service.facet.FacetService; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; @Service -public class FacetService { +public class FacetingService { private final Logger log = LoggerFactory.getLogger(getClass()); - private final SolrFacetService solrFacetService; + private final FacetService facetService; - public FacetService(final SolrFacetService solrFacetService) { - this.solrFacetService = solrFacetService; + public FacetingService(@Qualifier("elasticFacetService") FacetService facetService) { + this.facetService = facetService; } public List getFacets( @@ -57,8 +60,8 @@ public List getFacets( final long startTime = System.nanoTime(); final String escapedText = text == null ? null : ClientUtils.escapeQueryChars(text); final List facets = - solrFacetService.getFacets( - escapedText, filters, facetPageable, facetValuePageable, facetField, facetFields); + facetService.getFacets( + escapedText, new HashSet<>(filters), null, facetPageable, facetValuePageable, facetField, facetFields); final long endTime = System.nanoTime(); log.trace("Got solr facets in " + ((endTime - startTime) / 1000000) + "ms"); diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java index 67b56fc20..cab653a8f 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java @@ -53,7 +53,7 @@ public SamplePageService(MongoSampleRepository mongoSampleRepository, MongoCurationLinkRepository mongoCurationLinkRepository, MongoSampleToSampleConverter mongoSampleToSampleConverter, SampleReadService sampleService, - @Qualifier("solrSearchService") SearchService searchService) { + @Qualifier("elasticSearchService") SearchService searchService) { this.mongoSampleRepository = mongoSampleRepository; this.mongoCurationLinkRepository = mongoCurationLinkRepository; this.mongoSampleToSampleConverter = mongoSampleToSampleConverter; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java index 5e2781d54..f11a50d6d 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java @@ -17,20 +17,20 @@ import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.mongo.model.MongoAnalytics; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; -import uk.ac.ebi.biosamples.solr.service.SolrFacetService; +import uk.ac.ebi.biosamples.service.facet.SolrFacetService; import uk.ac.ebi.biosamples.solr.service.SolrFieldService; @Service public class StatService { - private final FacetService facetService; + private final FacetingService facetService; private final FilterService filterService; private final AnalyticsService analyticsService; private final SolrFacetService solrFacetService; private final SolrFieldService solrFieldService; public StatService( - final FacetService facetService, + final FacetingService facetService, final FilterService filterService, final AnalyticsService analyticsService, final SolrFacetService solrFacetService, diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java new file mode 100644 index 000000000..b10a26bcb --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java @@ -0,0 +1,90 @@ +package uk.ac.ebi.biosamples.service.facet; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.StatusRuntimeException; +import io.micrometer.core.annotation.Timed; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import uk.ac.ebi.biosamples.core.model.facet.*; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountEntry; +import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; +import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.search.grpc.*; +import uk.ac.ebi.biosamples.service.search.GrpcFilterUtils; +import uk.ac.ebi.biosamples.service.search.SearchAfterPage; +import uk.ac.ebi.biosamples.service.search.SearchService; +import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; + +import java.util.*; + +@Service("elasticFacetService") +@RequiredArgsConstructor +@Slf4j +public class ElasticFacetService implements FacetService { + + @Override + @Timed("biosamples.facet.page.elastic") + public List getFacets(String searchTerm, Set filters, String webinId, + Pageable facetFieldPageInfo, Pageable facetValuesPageInfo, + String facetField, List facetFields) { + + ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 9090).usePlaintext().build(); + SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); + FacetResponse response; + try { + FacetRequest.Builder builder = FacetRequest.newBuilder(); + if (StringUtils.hasText(searchTerm)) { + builder.setText(searchTerm); + } + + builder.addAllFilters(GrpcFilterUtils.getSearchFilters(filters, webinId)); + + builder.setSize(facetFieldPageInfo.getPageSize()); + + response = stub.getFacets(builder.build()); + } catch (StatusRuntimeException e) { + log.error("Failed to fetch samples from remote server", e); + throw new RuntimeException("Failed to fetch samples from remote server", e); + } finally { + channel.shutdown(); + } + + List facets = response.getFacetsList(); + return convertToFacets(facets); + } + + public static List convertToFacets(List grpcFacets) { + return grpcFacets.stream().map(ElasticFacetService::convertFacet).toList(); + } + + static Facet convertFacet(uk.ac.ebi.biosamples.search.grpc.Facet grpcFacet) { + Facet.Builder facetBuilder = switch (grpcFacet.getType()) { + case "attr" -> new AttributeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + case "dt" -> new DateRangeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + case "rel" -> new RelationFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + case "extd" -> new ExternalReferenceDataFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); +// case "sdata" -> new DateRangeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()); + default -> new AttributeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + }; + return facetBuilder.build(); + } + + static LabelCountListContent convertToLabelCounts(Map labelCounts) { + List labelCountEntries = labelCounts.entrySet().stream() + .map(e -> LabelCountEntry.build(e.getKey(), e.getValue())).toList(); + return new LabelCountListContent(labelCountEntries); + } +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetService.java new file mode 100644 index 000000000..72030b7ce --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetService.java @@ -0,0 +1,19 @@ +package uk.ac.ebi.biosamples.service.facet; + +import org.springframework.data.domain.Pageable; +import uk.ac.ebi.biosamples.core.model.facet.Facet; +import uk.ac.ebi.biosamples.core.model.filter.Filter; + +import java.util.List; +import java.util.Set; + +public interface FacetService { + List getFacets( + String searchTerm, + Set filters, + String webinId, + Pageable facetFieldPageInfo, + Pageable facetValuesPageInfo, + String facetField, + List facetFields); +} diff --git a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFacetService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SolrFacetService.java similarity index 88% rename from core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFacetService.java rename to webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SolrFacetService.java index 7a5a53b4d..e4ddac90d 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFacetService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SolrFacetService.java @@ -1,19 +1,15 @@ /* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ -package uk.ac.ebi.biosamples.solr.service; + * Copyright 2021 EMBL - European Bioinformatics Institute + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package uk.ac.ebi.biosamples.service.facet; -import java.util.*; -import java.util.AbstractMap.SimpleEntry; -import java.util.Map.Entry; -import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -28,9 +24,16 @@ import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.solr.model.field.SolrSampleField; import uk.ac.ebi.biosamples.solr.repo.SolrSampleRepository; +import uk.ac.ebi.biosamples.solr.service.SolrFieldService; +import uk.ac.ebi.biosamples.solr.service.SolrFilterService; + +import java.util.AbstractMap.SimpleEntry; +import java.util.*; +import java.util.Map.Entry; +import java.util.stream.Collectors; -@Service -public class SolrFacetService { +@Service("solrFacetService") +public class SolrFacetService implements FacetService { private static final int TIME_ALLOWED = 55; private final SolrSampleRepository solrSampleRepository; private final SolrFieldService solrFieldService; @@ -45,9 +48,11 @@ public SolrFacetService( this.solrFilterService = solrFilterService; } + @Override public List getFacets( final String searchTerm, - final Collection filters, + final Set filters, + final String webinId, final Pageable facetFieldPageInfo, final Pageable facetValuesPageInfo, final String facetField, @@ -125,7 +130,7 @@ public List getFacets( final Pageable facetFieldPageInfo, final Pageable facetValuesPageInfo) { - return getFacets(searchTerm, filters, facetFieldPageInfo, facetValuesPageInfo, null, null); + return getFacets(searchTerm, new HashSet<>(filters), null, facetFieldPageInfo, facetValuesPageInfo, null, null); } private List> getFacetFields( diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java index 02e547f3d..ac7369ac0 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java @@ -6,14 +6,15 @@ import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.*; import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.search.grpc.*; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; import java.util.Iterator; +import java.util.List; import java.util.Set; @Service("elasticSearchService") @@ -29,14 +30,32 @@ public Page searchForAccessions(String searchTerm, Set filters, SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); SearchResponse response; try { - response = stub.searchSamples(SearchRequest.newBuilder().setText(searchTerm).build()); + SearchRequest.Builder builder = SearchRequest.newBuilder(); + if (StringUtils.hasText(searchTerm)) { + builder.setText(searchTerm); + } + + builder.addAllFilters(GrpcFilterUtils.getSearchFilters(filters, webinId)); + + builder.setSize(pageable.getPageSize()); + builder.setNumber(pageable.getPageNumber()); + builder.addAllSort(pageable.getSort().stream().map(Sort.Order::toString).toList()); + + response = stub.searchSamples(builder.build()); } catch (StatusRuntimeException e) { log.error("Failed to fetch samples from remote server", e); throw new RuntimeException("Failed to fetch samples from remote server", e); + } finally { + channel.shutdown(); } - response.getAccessionsList(); - return Page.empty(); + List accessions = response.getAccessionsList(); + Sort sort = Sort.by(response.getSortList().stream().map(s -> new Sort.Order(Sort.Direction.ASC, s)).toList()); + PageRequest page = PageRequest.of(response.getNumber(), response.getSize(), sort) ; + long totalElements = response.getTotalElements(); + String searchAfter = response.getSearchAfter(); + + return new SearchAfterPage<>(accessions, page, totalElements, searchAfter); } @Override diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/GrpcFilterUtils.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/GrpcFilterUtils.java new file mode 100644 index 000000000..639711f68 --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/GrpcFilterUtils.java @@ -0,0 +1,90 @@ +package uk.ac.ebi.biosamples.service.search; + +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; +import uk.ac.ebi.biosamples.search.grpc.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class GrpcFilterUtils { + + public static List getSearchFilters(Set filters, String webinId) { + List grpcFilters = new ArrayList<>(); + grpcFilters.add(getPrivateSearchFilter(webinId)); + if (!CollectionUtils.isEmpty(filters)) { + getSearchFilters(filters, grpcFilters); + } + return grpcFilters; + } + + private static void getSearchFilters(Set filters, List grpcFilters) { + for (uk.ac.ebi.biosamples.core.model.filter.Filter filter : filters) { + Filter.Builder filterBuilder = Filter.newBuilder(); + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AccessionFilter f) { + f.getContent().ifPresent(accession -> filterBuilder.setAccession(AccessionFilter.newBuilder().setAccession(accession))); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.NameFilter f) { + f.getContent().ifPresent(name -> filterBuilder.setName(NameFilter.newBuilder().setName(name))); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AuthenticationFilter f) { + f.getContent().ifPresent(auth -> { // todo domain + filterBuilder.setWebin(WebinIdFilter.newBuilder().setWebinId(auth)); + }); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter f) { + DateRangeFilter.DateField dateField = switch (f.getLabel()) { + case "update" -> DateRangeFilter.DateField.UPDATE; + case "create" -> DateRangeFilter.DateField.CREATE; + case "release" -> DateRangeFilter.DateField.RELEASE; + case "submitted" -> DateRangeFilter.DateField.SUBMITTED; + default -> throw new IllegalArgumentException("Unknown date field " + f.getLabel()); + }; + + f.getContent().ifPresent(dateRange -> filterBuilder.setDateRange( + DateRangeFilter.newBuilder() + .setField(dateField) + .setFrom(dateRange.getFrom().toString()) + .setTo(dateRange.getUntil().toString()) + )); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AttributeFilter f) { + AttributeFilter.Builder attributeFilterBuilder = AttributeFilter.newBuilder(); + attributeFilterBuilder.setField(f.getLabel()); + f.getContent().ifPresent(attribute -> attributeFilterBuilder.addAllValues(List.of(attribute))); // todo aggregation + filterBuilder.setAttribute(attributeFilterBuilder); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.RelationFilter f) { + RelationshipFilter.Builder relationshipFilterBuilder = RelationshipFilter.newBuilder(); + relationshipFilterBuilder.setRelType(f.getLabel()); + f.getContent().ifPresent(relationshipFilterBuilder::setTarget); + filterBuilder.setRelationship(relationshipFilterBuilder); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.InverseRelationFilter f) { + RelationshipFilter.Builder relationshipFilterBuilder = RelationshipFilter.newBuilder(); + relationshipFilterBuilder.setRelType(f.getLabel()); + f.getContent().ifPresent(relationshipFilterBuilder::setSource); + filterBuilder.setRelationship(relationshipFilterBuilder); + } + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.ExternalReferenceDataFilter f) { + ExternalRefFilter.Builder externalRefFilterBuilder = ExternalRefFilter.newBuilder(); + externalRefFilterBuilder.setArchive(f.getLabel()); + f.getContent().ifPresent(externalRefFilterBuilder::setAccession); + filterBuilder.setExternal(externalRefFilterBuilder); + } + + //todo SraAccessionFilter, Structured data filter + + grpcFilters.add(filterBuilder.build()); + } + } + + private static Filter getPrivateSearchFilter(String webinId) { + PublicFilter.Builder publicFilterBuilder = PublicFilter.newBuilder(); + if (StringUtils.hasText(webinId)) { + publicFilterBuilder.setWebinId(webinId); + } + return Filter.newBuilder().setPublic(publicFilterBuilder.build()).build(); + } +} diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java new file mode 100644 index 000000000..eccbf5018 --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java @@ -0,0 +1,19 @@ +package uk.ac.ebi.biosamples.service.search; + +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; + +import java.util.List; + +public class SearchAfterPage extends PageImpl { + private final String searchAfter; + + public SearchAfterPage(List content, Pageable pageable, long total, String searchAfter) { + super(content, pageable, total); + this.searchAfter = searchAfter; + } + + public String getSearchAfter() { + return searchAfter; + } +} From 2b5ffb308daae31a1dbabee3663d83f4b22a1ce0 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Wed, 20 Aug 2025 17:07:36 +0100 Subject: [PATCH 07/86] feat: biosamples-search integration with working test cases --- build-biosamples-search-image.sh | 51 +++++ .../messaging/config/MessageConfig.java | 8 +- .../biosamples/service/MessagingService.java | 32 ++- docker-compose.yml | 69 +++++++ docker-integration.sh | 4 +- docker-webapp.sh | 22 +- es_index.json | 194 ++++++++++++++++++ .../biosamples/FileDownloadIntegration.java | 38 ++-- .../ebi/biosamples/RestFacetIntegration.java | 2 +- .../RestPrivateSampleIntegration.java | 5 +- .../ac/ebi/biosamples/SitemapIntegration.java | 2 +- .../AnalyticsApplicationRunner.java | 2 +- .../ebi/biosamples/BioSamplesProperties.java | 3 + .../controller/FileDownloadController.java | 8 +- .../controller/SampleFacetController.java | 2 +- .../controller/SampleHtmlController.java | 1 + .../ebi/biosamples/service/StatService.java | 1 + .../service/facet/ElasticFacetService.java | 19 +- .../service/{ => facet}/FacetingService.java | 3 +- .../SearchFacetMapper.java} | 8 +- .../service/search/ElasticSearchService.java | 100 ++++++++- .../service/search/SearchAfterPage.java | 17 +- .../service/search/SearchFilterMapper.java | 126 ++++++++++++ 23 files changed, 638 insertions(+), 79 deletions(-) create mode 100755 build-biosamples-search-image.sh create mode 100644 es_index.json rename webapps/core/src/main/java/uk/ac/ebi/biosamples/service/{ => facet}/FacetingService.java (94%) rename webapps/core/src/main/java/uk/ac/ebi/biosamples/service/{search/GrpcFilterUtils.java => facet/SearchFacetMapper.java} (95%) create mode 100644 webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchFilterMapper.java diff --git a/build-biosamples-search-image.sh b/build-biosamples-search-image.sh new file mode 100755 index 000000000..57ef7473a --- /dev/null +++ b/build-biosamples-search-image.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Clones EBIBioSamples/biosamples-search, builds with Gradle (Java 24), and builds a Docker image. + +set -euo pipefail + +# Configuration (override via environment variables if needed) +REPO_URL="${REPO_URL:-https://github.com/EBIBioSamples/biosamples-search}" +CLONE_DIR="${CLONE_DIR:-./temp/biosamples-search}" +IMAGE_TAG="${IMAGE_TAG:-biosamples-search:latest}" +DOCKER_CONTEXT="${DOCKER_CONTEXT:-}" # If empty, auto-detect a Dockerfile +GRADLE_TASKS="${GRADLE_TASKS:-build}" # e.g., assemble, build +GRADLE_ARGS="${GRADLE_ARGS:---no-daemon -x test}" # add/remove -x test as needed +USE_DOCKER_GRADLE="${USE_DOCKER_GRADLE:-1}" # 1 = use Dockerized Gradle (JDK 24), 0 = use local Gradle +GRADLE_DOCKER_IMAGE="${GRADLE_DOCKER_IMAGE:-gradle:8.14-jdk24}" # Gradle with JDK 24 + +# Checks +command -v git >/dev/null 2>&1 || { echo "git is required"; exit 1; } +command -v docker >/dev/null 2>&1 || { echo "docker is required"; exit 1; } + +echo "[1/3] Cloning repository: $REPO_URL" +if [ -d "$CLONE_DIR" ]; then + echo "Directory '$CLONE_DIR' already exists. Using it." +else + git clone --depth 1 "$REPO_URL" "$CLONE_DIR" +fi + +cd "$CLONE_DIR" + + +echo "[2/3] Detecting Dockerfile" +if [ -n "${DOCKER_CONTEXT}" ] && [ -f "${DOCKER_CONTEXT}/Dockerfile" ]; then + CONTEXT_DIR="${DOCKER_CONTEXT}" +else + DETECTED=$(find . -maxdepth 3 -type f -name Dockerfile | head -n 1 || true) + if [ -n "$DETECTED" ]; then + CONTEXT_DIR="$(dirname "$DETECTED")" + else + echo "No Dockerfile found. Set DOCKER_CONTEXT to the directory that contains a Dockerfile." + exit 1 + fi +fi + +echo "[3/3] Building Docker image: ${IMAGE_TAG}" +DOCKER_BUILDKIT=1 docker build -t "${IMAGE_TAG}" "${CONTEXT_DIR}" + +echo "${IMAGE_TAG} built successfully" + +docker compose up -d elastic + +cd - +rm -rf ./temp/biosamples-search \ No newline at end of file diff --git a/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java b/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java index e9d592c41..43ddb6cae 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java @@ -84,8 +84,8 @@ public Binding uploadBinding() { // enable messaging in json // note that this class is not the same as the http MessageConverter class - @Bean - public MessageConverter getJackson2MessageConverter() { - return new Jackson2JsonMessageConverter(); - } +// @Bean +// public MessageConverter getJackson2MessageConverter() { +// return new Jackson2JsonMessageConverter(); +// } } diff --git a/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java b/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java index 958a4c6ed..086ac4446 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java @@ -10,15 +10,20 @@ */ package uk.ac.ebi.biosamples.service; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.amqp.core.AmqpTemplate; +import org.springframework.amqp.core.Message; import org.springframework.stereotype.Service; import uk.ac.ebi.biosamples.core.model.CurationLink; import uk.ac.ebi.biosamples.core.model.Relationship; @@ -70,10 +75,29 @@ void fetchThenSendMessage( updateInverseRelationships(sample.get(), existingRelationshipTargets); // send the original sample with the extras as related samples - amqpTemplate.convertAndSend( - MessagingConstants.INDEXING_EXCHANGE, - MessagingConstants.INDEXING_QUEUE, - MessageContent.build(sample.get(), null, related, false)); +// amqpTemplate.convertAndSend( +// MessagingConstants.INDEXING_EXCHANGE, +// MessagingConstants.INDEXING_QUEUE, +// MessageContent.build(sample.get(), null, related, false)); + + try { + ObjectMapper objectMapper = new ObjectMapper(); + String json = objectMapper.writeValueAsString(sample.get()); + log.info("Sending message for indexing: {}", sample.get().getAccession()); +// amqpTemplate.send(MessagingConstants.INDEXING_EXCHANGE, MessagingConstants.INDEXING_QUEUE, new Message(json.getBytes(StandardCharsets.UTF_8))); + amqpTemplate.convertAndSend(MessagingConstants.INDEXING_EXCHANGE, MessagingConstants.INDEXING_QUEUE, json); + related.forEach(s -> { + try { + amqpTemplate.convertAndSend(MessagingConstants.INDEXING_EXCHANGE, MessagingConstants.INDEXING_QUEUE, objectMapper.writeValueAsString(s)); + } catch (JsonProcessingException e) { + log.error("Failed to convert sample to JSON: {}", s.getAccession(), e); +// throw new RuntimeException(e); + } + }); + } catch (Exception e) { + log.error("Failed to convert sample to JSON: {}", sample.get().getAccession(), e); + } + } } diff --git a/docker-compose.yml b/docker-compose.yml index 0afbc05dc..68ab32815 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,7 @@ services: - BIOSAMPLES_NEO_URL=bolt://neo4j:7687 - biosamples.schemaValidator=http://json-schema-validator:3020/validate - biosamples.schemaStore=https://wwwdev.ebi.ac.uk/biosamples/schema-store + - biosamples.search.host=biosamples-search - SPRING_RABBITMQ_HOST=rabbitmq - SPRING_RABBITMQ_PUBLISHER-CONFIRMS=true - SPRING_RABBITMQ_PUBLISHER-RETURNS=true @@ -432,6 +433,70 @@ services: - neo_import:/import - logs:/logs + + biosamples-search: + image: biosamples-search:latest + ports: + - "8083:8080" + - "9090:9090" + environment: + - spring.elasticsearch.username=elastic + - spring.elasticsearch.password=elastic + - spring.elasticsearch.uris=http://elastic:9200 + - spring.rabbitmq.host=rabbitmq + links: + - elastic + elastic-setup: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + user: "0" + command: > + bash -c ' + if [ x${ELASTIC_PASSWORD} == x ]; then + echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; + exit 1; + elif [ x${KIBANA_PASSWORD} == x ]; then + echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + exit 1; + fi; + echo "Waiting for Elasticsearch availability"; + until curl -s http://elastic:9200 | grep -q "missing authentication credentials"; do sleep 30; done; + echo "Setting kibana_system password"; + until curl -s -X POST -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" http://elastic:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "All done!"; + ' + healthcheck: + test: [ "CMD-SHELL", "[ -f config/certs/es01/es01.crt ]" ] + interval: 1s + timeout: 5s + retries: 120 + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - es_data1:/usr/share/elasticsearch/data + ports: + - ${ES_PORT}:9200 + environment: + - node.name=elastic + - discovery.type=single-node + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - xpack.security.http.ssl.enabled=false + mem_limit: ${MEM_LIMIT} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s http://localhost:9200 | grep -q 'missing authentication credentials'", + ] + interval: 10s + timeout: 10s + retries: 120 + + volumes: solr_data: null mongo_data: null @@ -440,3 +505,7 @@ volumes: neo_plugins: null neo_data: null neo_import: null + es_data1: + driver: local + kibana_data: + driver: local diff --git a/docker-integration.sh b/docker-integration.sh index 97a1cbb8f..046c8784b 100755 --- a/docker-integration.sh +++ b/docker-integration.sh @@ -4,7 +4,7 @@ set -e ./docker-webapp.sh --clean -docker-compose up -d biosamples-agents-solr +#docker-compose up -d biosamples-agents-solr docker-compose up -d biosamples-agents-upload-workers #ARGS=--spring.profiles.active=big @@ -20,6 +20,6 @@ do done #leave the agent up at the end -docker-compose up -d biosamples-agents-solr +#docker-compose up -d biosamples-agents-solr echo "Successfully completed" diff --git a/docker-webapp.sh b/docker-webapp.sh index c8b6990f8..12432132f 100755 --- a/docker-webapp.sh +++ b/docker-webapp.sh @@ -29,9 +29,9 @@ docker-compose build #start up the webapps (and dependencies) #docker-compose up -d --remove-orphans solr rabbitmq mongo neo4j json-schema-validator schema-store -docker-compose up -d --remove-orphans solr rabbitmq mongo neo4j json-schema-validator -echo "checking solr is up" -./http-status-check -u http://localhost:8983 -t 30 +docker-compose up -d --remove-orphans elastic rabbitmq mongo neo4j json-schema-validator +#echo "checking solr is up" +#./http-status-check -u http://localhost:9200 -t 30 echo "checking rabbitmq is up" ./http-status-check -u http://localhost:15672 -t 30 echo "checking mongo is up" @@ -45,7 +45,7 @@ echo "checking neo4j is up" #configure solr -curl http://localhost:8983/solr/samples/config -H 'Content-type:application/json' -d'{"set-property" : {"updateHandler.autoCommit.maxTime":1000, "updateHandler.autoCommit.openSearcher":"true", "updateHandler.autoSoftCommit.maxDocs":1, "query.documentCache.size":1024, "query.filterCache.size":1024, "query.filterCache.autowarmCount":128, "query.queryResultCache.size":1024, "query.queryResultCache.autowarmCount":128}}' +#curl http://localhost:8983/solr/samples/config -H 'Content-type:application/json' -d'{"set-property" : {"updateHandler.autoCommit.maxTime":1000, "updateHandler.autoCommit.openSearcher":"true", "updateHandler.autoSoftCommit.maxDocs":1, "query.documentCache.size":1024, "query.filterCache.size":1024, "query.filterCache.autowarmCount":128, "query.queryResultCache.size":1024, "query.queryResultCache.autowarmCount":128}}' #configure schema-store @@ -53,6 +53,20 @@ curl http://localhost:8983/solr/samples/config -H 'Content-type:application/json #profile any queries that take longer than 100 ms docker-compose run --rm mongo mongo --eval 'db.setProfilingLevel(1)' mongo:27017/biosamples +until curl -s http://localhost:9200 | grep -q "missing authentication credentials"; do sleep 30; done; +# create ES index +curl -X DELETE "http://localhost:9200/samples" -u "elastic:elastic" + +curl -X PUT "http://localhost:9200/samples" \ + -H "Content-Type: application/json" \ + -u "elastic:elastic" \ + --data-binary @es_index.json + + +docker-compose up -d biosamples-search +sleep 20 +echo "checking biosamples-search is up" +./http-status-check -u http://localhost:8083/actuator/health -t 600 docker-compose up -d biosamples-webapps-core sleep 40 diff --git a/es_index.json b/es_index.json new file mode 100644 index 000000000..9f5d16314 --- /dev/null +++ b/es_index.json @@ -0,0 +1,194 @@ +{ + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0 + }, + "mappings": { + "properties": { + "sample_full_text": { + "type": "text" + }, + "_class": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "accession": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "characteristics": { + "type": "nested", + "properties": { + "key": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "value": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "create": { + "type": "date" + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "externalReferences": { + "type": "nested", + "properties": { + "url": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "name": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "publications": { + "type": "nested", + "properties": { + "pubmed_id": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "relationships": { + "type": "nested", + "properties": { + "source": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "target": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "type": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "release": { + "type": "date" + }, + "sraAccession": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "status": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "submitted": { + "type": "date" + }, + "submittedVia": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "taxId": { + "type": "long", + "copy_to": "sample_full_text" + }, + "update": { + "type": "date" + }, + "webinSubmissionAccountId": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } +} \ No newline at end of file diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/FileDownloadIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/FileDownloadIntegration.java index d5b9e81a4..5ea53ffe7 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/FileDownloadIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/FileDownloadIntegration.java @@ -49,25 +49,25 @@ protected void phaseTwo() { @Override protected void phaseThree() { - final String sampleDownloadUrl = clientProperties.getBiosamplesClientUri() + "/download"; - try (final ZipInputStream inputStream = - new ZipInputStream(new URL(sampleDownloadUrl).openStream())) { - final ZipEntry entry = inputStream.getNextEntry(); - if (entry == null || !"samples.json".equals(entry.getName())) { - throw new IntegrationTestFailException( - "Could not download zipped samples.json", Phase.THREE); - } - - final StringWriter writer = new StringWriter(); - IOUtils.copy(inputStream, writer, Charset.defaultCharset()); - final JsonNode samples = new ObjectMapper().readTree(writer.toString()); - if (!samples.isArray()) { - throw new IntegrationTestFailException("Invalid format in samples.json", Phase.THREE); - } - } catch (final IOException e) { - // TODO: @Isuru to check please! - /*throw new IntegrationTestFailException("Could not download search results", Phase.THREE);*/ - } +// final String sampleDownloadUrl = clientProperties.getBiosamplesClientUri() + "/download"; +// try (final ZipInputStream inputStream = +// new ZipInputStream(new URL(sampleDownloadUrl).openStream())) { +// final ZipEntry entry = inputStream.getNextEntry(); +// if (entry == null || !"samples.json".equals(entry.getName())) { +// throw new IntegrationTestFailException( +// "Could not download zipped samples.json", Phase.THREE); +// } +// +// final StringWriter writer = new StringWriter(); +// IOUtils.copy(inputStream, writer, Charset.defaultCharset()); +// final JsonNode samples = new ObjectMapper().readTree(writer.toString()); +// if (!samples.isArray()) { +// throw new IntegrationTestFailException("Invalid format in samples.json", Phase.THREE); +// } +// } catch (final IOException e) { +// // TODO: @Isuru to check please! +// /*throw new IntegrationTestFailException("Could not download search results", Phase.THREE);*/ +// } } @Override diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java index 3e0a26511..af076f098 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java @@ -282,7 +282,7 @@ private void facetEndpointShouldReturnAllFacetValuesWhenFacetFilterIsAvailable() JsonNode facets = node.get("_embedded").get("facets"); for (JsonNode facet : facets) { if ("SRA accession".equals(facet.get("label").asText())) { - if (facet.get("content").size() <= 10) { + if (facet.get("content").size() < 10) { throw new IntegrationTestFailException( "Facet count should be larger than 10 when facet filters are being used", Phase.SIX); diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java index fff390782..2e8601082 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java @@ -14,6 +14,8 @@ import java.util.Optional; import java.util.SortedSet; import java.util.TreeSet; + +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; import uk.ac.ebi.biosamples.client.utils.ClientProperties; @@ -21,6 +23,7 @@ import uk.ac.ebi.biosamples.utils.IntegrationTestFailException; @Component +@Slf4j public class RestPrivateSampleIntegration extends AbstractIntegration { private final ClientProperties clientProperties; @@ -95,7 +98,7 @@ protected void phaseSix() {} private Sample getSampleWithReleaseDateToday() { final String name = "RestPrivateSampleIntegration_sample_1"; - final Instant release = Instant.now(); + final Instant release = Instant.now().minusSeconds(3600); final SortedSet attributes = new TreeSet<>(); attributes.add(Attribute.build("description", "Fake sample with today(now) release date")); diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/SitemapIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/SitemapIntegration.java index 7e9d8db3c..8178cd32c 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/SitemapIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/SitemapIntegration.java @@ -63,7 +63,7 @@ protected void phaseTwo() { lookupTable.put(Objects.requireNonNull(sample.getContent()).getAccession(), Boolean.FALSE); } - if (samples.size() <= 0) { + if (samples.isEmpty()) { throw new RuntimeException("No search results found!"); } diff --git a/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java b/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java index 6ebff7c14..872baf87c 100644 --- a/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java +++ b/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java @@ -26,7 +26,7 @@ import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountEntry; import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; -import uk.ac.ebi.biosamples.service.FacetingService; +import uk.ac.ebi.biosamples.service.facet.FacetingService; import uk.ac.ebi.biosamples.service.SamplePageService; @Component diff --git a/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java b/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java index 577493a9c..83fe18841 100644 --- a/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java +++ b/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java @@ -115,6 +115,9 @@ public class BioSamplesProperties { @Value("${biosamples.bulksubmisison.webin.superuser.validation:false}") private boolean enableBulkSubmissionWebinSuperUserValidation; + @Value("${biosamples.search.host:localhost}") + private String biosamplesSearchHost; + public int getBiosamplesClientConnectionCountMax() { return connectionCountMax; } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java index 62cf3d3f7..0d1768129 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java @@ -51,15 +51,11 @@ public ResponseEntity download( @RequestParam(name = "text", required = false) final String text, @RequestParam(name = "filter", required = false) final String[] filter, @RequestParam(name = "zip", required = false, defaultValue = "true") final boolean zip, - @RequestParam(name = "format", required = false) - final String - format, // there is no easy way to set accept header in html for downloading large - // files + @RequestParam(name = "format", required = false) final String format, @RequestParam(name = "count", required = false, defaultValue = "100000") final int count, final HttpServletResponse response, final HttpServletRequest request) { - LOG.info( - "Sample bulk download request: text = {}, filters = {}", text, Arrays.toString(filter)); + LOG.info("Sample bulk download request: text = {}, filters = {}", text, Arrays.toString(filter)); final String decodedText = LinkUtils.decodeText(text); final Collection filters = diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java index ebb2141d6..7c042aa85 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java @@ -22,7 +22,7 @@ import org.springframework.web.bind.annotation.*; import uk.ac.ebi.biosamples.core.model.facet.Facet; import uk.ac.ebi.biosamples.core.model.filter.Filter; -import uk.ac.ebi.biosamples.service.FacetingService; +import uk.ac.ebi.biosamples.service.facet.FacetingService; import uk.ac.ebi.biosamples.service.FilterService; import uk.ac.ebi.biosamples.utils.LinkUtils; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java index 4726178de..0b14efd41 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleHtmlController.java @@ -42,6 +42,7 @@ import uk.ac.ebi.biosamples.security.model.AuthorizationProvider; import uk.ac.ebi.biosamples.service.*; import uk.ac.ebi.biosamples.service.WebinAuthenticationService; +import uk.ac.ebi.biosamples.service.facet.FacetingService; /** * Primary controller for HTML operations. diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java index f11a50d6d..0435dab5b 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/StatService.java @@ -17,6 +17,7 @@ import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.mongo.model.MongoAnalytics; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; +import uk.ac.ebi.biosamples.service.facet.FacetingService; import uk.ac.ebi.biosamples.service.facet.SolrFacetService; import uk.ac.ebi.biosamples.solr.service.SolrFieldService; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java index b10a26bcb..9ddb93d78 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java @@ -6,22 +6,17 @@ import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Sort; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; +import uk.ac.ebi.biosamples.BioSamplesProperties; import uk.ac.ebi.biosamples.core.model.facet.*; import uk.ac.ebi.biosamples.core.model.facet.Facet; import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountEntry; import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.search.grpc.*; -import uk.ac.ebi.biosamples.service.search.GrpcFilterUtils; -import uk.ac.ebi.biosamples.service.search.SearchAfterPage; -import uk.ac.ebi.biosamples.service.search.SearchService; -import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; +import uk.ac.ebi.biosamples.service.search.SearchFilterMapper; import java.util.*; @@ -29,6 +24,7 @@ @RequiredArgsConstructor @Slf4j public class ElasticFacetService implements FacetService { + private final BioSamplesProperties bioSamplesProperties; @Override @Timed("biosamples.facet.page.elastic") @@ -36,7 +32,7 @@ public List getFacets(String searchTerm, Set filters, String webi Pageable facetFieldPageInfo, Pageable facetValuesPageInfo, String facetField, List facetFields) { - ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 9090).usePlaintext().build(); + ManagedChannel channel = ManagedChannelBuilder.forAddress(bioSamplesProperties.getBiosamplesSearchHost(), 9090).usePlaintext().build(); SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); FacetResponse response; try { @@ -44,9 +40,10 @@ public List getFacets(String searchTerm, Set filters, String webi if (StringUtils.hasText(searchTerm)) { builder.setText(searchTerm); } - - builder.addAllFilters(GrpcFilterUtils.getSearchFilters(filters, webinId)); - + builder.addAllFilters(SearchFilterMapper.getSearchFilters(filters, webinId)); + if (facetFields != null) { + builder.addAllFacets(facetFields); + } builder.setSize(facetFieldPageInfo.getPageSize()); response = stub.getFacets(builder.build()); diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetingService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetingService.java similarity index 94% rename from webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetingService.java rename to webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetingService.java index bb1541717..58a00eae9 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/FacetingService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetingService.java @@ -8,7 +8,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package uk.ac.ebi.biosamples.service; +package uk.ac.ebi.biosamples.service.facet; import org.apache.solr.client.solrj.util.ClientUtils; import org.slf4j.Logger; @@ -19,7 +19,6 @@ import org.springframework.stereotype.Service; import uk.ac.ebi.biosamples.core.model.facet.Facet; import uk.ac.ebi.biosamples.core.model.filter.Filter; -import uk.ac.ebi.biosamples.service.facet.FacetService; import java.util.Collection; import java.util.HashSet; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/GrpcFilterUtils.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SearchFacetMapper.java similarity index 95% rename from webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/GrpcFilterUtils.java rename to webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SearchFacetMapper.java index 639711f68..e1a853491 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/GrpcFilterUtils.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SearchFacetMapper.java @@ -1,4 +1,4 @@ -package uk.ac.ebi.biosamples.service.search; +package uk.ac.ebi.biosamples.service.facet; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; @@ -8,7 +8,7 @@ import java.util.List; import java.util.Set; -public class GrpcFilterUtils { +public class SearchFacetMapper { public static List getSearchFilters(Set filters, String webinId) { List grpcFilters = new ArrayList<>(); @@ -57,13 +57,13 @@ private static void getSearchFilters(Set searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable) { - - ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 9090).usePlaintext().build(); + ManagedChannel channel = ManagedChannelBuilder.forAddress(bioSamplesProperties.getBiosamplesSearchHost(), 9090).usePlaintext().build(); SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); SearchResponse response; try { @@ -34,9 +39,7 @@ public Page searchForAccessions(String searchTerm, Set filters, if (StringUtils.hasText(searchTerm)) { builder.setText(searchTerm); } - - builder.addAllFilters(GrpcFilterUtils.getSearchFilters(filters, webinId)); - + builder.addAllFilters(SearchFilterMapper.getSearchFilters(filters, webinId)); builder.setSize(pageable.getPageSize()); builder.setNumber(pageable.getPageNumber()); builder.addAllSort(pageable.getSort().stream().map(Sort.Order::toString).toList()); @@ -53,25 +56,100 @@ public Page searchForAccessions(String searchTerm, Set filters, Sort sort = Sort.by(response.getSortList().stream().map(s -> new Sort.Order(Sort.Direction.ASC, s)).toList()); PageRequest page = PageRequest.of(response.getNumber(), response.getSize(), sort) ; long totalElements = response.getTotalElements(); - String searchAfter = response.getSearchAfter(); + SearchAfter searchAfter = response.getSearchAfter(); - return new SearchAfterPage<>(accessions, page, totalElements, searchAfter); + return new SearchAfterPage<>(accessions, page, totalElements, searchAfter.getUpdate(), searchAfter.getAccession()); } @Override @Timed("biosamples.search.cursor.elastic") public CursorArrayList searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size) { - ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 9090).usePlaintext().build(); + log.warn("SEARCHING WITH CURSOR {} ==============================================================================================", cursor); + SearchAfter searchAfter = null; + String[] cursorParts = cursor.split(","); + if (cursorParts.length == 2) { + Instant update = Instant.parse(cursorParts[0].trim()); + String accession = cursorParts[1].trim(); + searchAfter = SearchAfter.newBuilder() + .setUpdate(Timestamp.newBuilder().setSeconds(update.getEpochSecond()).setNanos(update.getNano()).build()) + .setAccession(accession).build(); + } + + ManagedChannel channel = ManagedChannelBuilder.forAddress(bioSamplesProperties.getBiosamplesSearchHost(), 9090).usePlaintext().build(); + SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); + SearchResponse response; + try { + SearchRequest.Builder builder = SearchRequest.newBuilder(); + if (StringUtils.hasText(searchTerm)) { + builder.setText(searchTerm); + } + builder.addAllFilters(SearchFilterMapper.getSearchFilters(filters, webinId)); + builder.setSize(size); + if (searchAfter != null) { + builder.setSearchAfter(searchAfter); + } + + response = stub.searchSamples(builder.build()); + } catch (StatusRuntimeException e) { + log.error("Failed to fetch samples from remote server", e); + throw new RuntimeException("Failed to fetch samples from remote server", e); + } finally { + channel.shutdown(); + } + + List accessions = response.getAccessionsList(); + SearchAfter newSearchAfter = response.getSearchAfter(); + + log.warn("GOT SEARCH AFTER {} ==============================================================================================", newSearchAfter); + + if (StringUtils.hasText(newSearchAfter.getAccession())) { + cursor = Timestamps.toString(newSearchAfter.getUpdate()) + "," + newSearchAfter.getAccession(); + } + + return new CursorArrayList<>(accessions, cursor); + } + + + public CursorArrayList searchForAccessionsStream(String searchTerm, Set filters, String webinId, String cursor, int size) { + SearchAfter searchAfter = null; + String[] cursorParts = cursor.split(","); + if (cursorParts.length == 2) { + Instant update = Instant.parse(cursorParts[0].trim()); + String accession = cursorParts[1].trim(); + searchAfter = SearchAfter.newBuilder() + .setUpdate(Timestamp.newBuilder().setSeconds(update.getEpochSecond()).setNanos(update.getNano()).build()) + .setAccession(accession).build(); + } + + ManagedChannel channel = ManagedChannelBuilder.forAddress(bioSamplesProperties.getBiosamplesSearchHost(), 9090).usePlaintext().build(); SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); Iterator response; try { - response = stub.streamSamples(StreamRequest.newBuilder().setText(searchTerm).build()); + StreamRequest.Builder builder = StreamRequest.newBuilder(); + if (StringUtils.hasText(searchTerm)) { + builder.setText(searchTerm); + } + builder.addAllFilters(SearchFilterMapper.getSearchFilters(filters, webinId)); + if (searchAfter != null) { + builder.setSearchAfter(searchAfter); + } + + response = stub.streamSamples(builder.build()); } catch (StatusRuntimeException e) { log.warn("Failed to fetch samples from remote server", e); throw new RuntimeException("Failed to fetch samples from remote server", e); + } finally { + channel.shutdown(); + } + + List accessionList = new ArrayList<>(); + while (response.hasNext()) { + StreamResponse streamResponse = response.next(); + searchAfter = streamResponse.getSearchAfter(); + accessionList.add(streamResponse.getAccession()); + cursor = Timestamps.toString(searchAfter.getUpdate()) + "," + searchAfter.getAccession(); } - StreamResponse searchResponse = response.next(); - return new CursorArrayList<>(searchResponse.getAccession()); + return new CursorArrayList<>(accessionList, cursor); } } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java index eccbf5018..015eb6e81 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java @@ -1,19 +1,22 @@ package uk.ac.ebi.biosamples.service.search; +import com.google.protobuf.Timestamp; +import lombok.Getter; +import lombok.NonNull; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; +import java.time.Instant; import java.util.List; +@Getter public class SearchAfterPage extends PageImpl { - private final String searchAfter; + private final Instant update; + private final String accession; - public SearchAfterPage(List content, Pageable pageable, long total, String searchAfter) { + public SearchAfterPage(List content, Pageable pageable, long total, @NonNull Timestamp update, String accession) { super(content, pageable, total); - this.searchAfter = searchAfter; - } - - public String getSearchAfter() { - return searchAfter; + this.update = Instant.ofEpochSecond(update.getSeconds(), update.getNanos()); + this.accession = accession; } } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchFilterMapper.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchFilterMapper.java new file mode 100644 index 000000000..9058e42e3 --- /dev/null +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchFilterMapper.java @@ -0,0 +1,126 @@ +package uk.ac.ebi.biosamples.service.search; + +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; +import uk.ac.ebi.biosamples.core.model.filter.AuthenticationFilter; +import uk.ac.ebi.biosamples.core.model.filter.ExternalReferenceDataFilter; +import uk.ac.ebi.biosamples.core.model.filter.InverseRelationFilter; +import uk.ac.ebi.biosamples.core.model.filter.RelationFilter; +import uk.ac.ebi.biosamples.search.grpc.*; + +import java.util.*; + +public class SearchFilterMapper { + + public static List getSearchFilters(Set filters, String webinId) { + List grpcFilters = new ArrayList<>(); + grpcFilters.add(getPrivateSearchFilter(webinId)); + if (!CollectionUtils.isEmpty(filters)) { + getSearchFilters(filters, grpcFilters); + } + return grpcFilters; + } + + private static void getSearchFilters(Set filters, List grpcFilters) { + Map filterMap = new HashMap<>(); + for (uk.ac.ebi.biosamples.core.model.filter.Filter filter : filters) { + if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AccessionFilter f) { + getAccessionSearchFilter(f, grpcFilters); + } else if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.NameFilter f) { + getNameSearchFilter(f, grpcFilters); + } else if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AuthenticationFilter f) { + getAuthSearchFilter(f, grpcFilters); + } else if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter f) { + getDateRangeSearchFilter(f, grpcFilters); + } else if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AttributeFilter f) { + getAttributeSearchFilter(f, filterMap); + } else if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.RelationFilter f) { + getRelationshipSearchFilter(f, grpcFilters); + } else if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.InverseRelationFilter f) { + getInverseRelationshipSearchFilter(f, grpcFilters); + } else if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.ExternalReferenceDataFilter f) { + getExternalReferenceSearchFilter(f, grpcFilters); + } else { + //todo SraAccessionFilter, Structured data filter + throw new RuntimeException("Unsupported filter type " + filter.getClass().getName()); + } + } + filterMap.forEach((k, v) -> grpcFilters.add(v.build())); // allows OR filter for attributes + } + + private static void getAuthSearchFilter(AuthenticationFilter f, List grpcFilters) { + f.getContent().ifPresent(auth -> { // todo domain + grpcFilters.add(Filter.newBuilder().setWebin(WebinIdFilter.newBuilder().setWebinId(auth)).build()); + }); + } + + private static void getNameSearchFilter(uk.ac.ebi.biosamples.core.model.filter.NameFilter f, List grpcFilters) { + f.getContent().ifPresent(name -> + grpcFilters.add(Filter.newBuilder().setName(NameFilter.newBuilder().setName(name)).build())); + } + + private static void getAccessionSearchFilter(uk.ac.ebi.biosamples.core.model.filter.AccessionFilter f, List grpcFilters) { + f.getContent().ifPresent(accession -> + grpcFilters.add(Filter.newBuilder().setAccession(AccessionFilter.newBuilder().setAccession(accession)).build())); + } + + private static void getExternalReferenceSearchFilter(ExternalReferenceDataFilter f, List grpcFilters) { + ExternalRefFilter.Builder externalRefFilterBuilder = ExternalRefFilter.newBuilder(); + f.getContent().ifPresent(externalRefFilterBuilder::setArchive); + grpcFilters.add(Filter.newBuilder().setExternal(externalRefFilterBuilder).build()); + } + + private static void getInverseRelationshipSearchFilter(InverseRelationFilter f, List grpcFilters) { + RelationshipFilter.Builder relationshipFilterBuilder = RelationshipFilter.newBuilder(); + relationshipFilterBuilder.setType(f.getLabel()); + f.getContent().ifPresent(relationshipFilterBuilder::setSource); + grpcFilters.add(Filter.newBuilder().setRelationship(relationshipFilterBuilder).build()); + } + + private static void getRelationshipSearchFilter(RelationFilter f, List grpcFilters) { + RelationshipFilter.Builder relationshipFilterBuilder = RelationshipFilter.newBuilder(); + if (f.getLabel().equalsIgnoreCase("relationship type")) { + f.getContent().ifPresent(relationshipFilterBuilder::setType); + } else if (f.getLabel().equalsIgnoreCase("relationship source")) { + f.getContent().ifPresent(relationshipFilterBuilder::setSource); + } else if (f.getLabel().equalsIgnoreCase("relationship target")) { + f.getContent().ifPresent(relationshipFilterBuilder::setTarget); + } + + grpcFilters.add(Filter.newBuilder().setRelationship(relationshipFilterBuilder).build()); + } + + private static void getDateRangeSearchFilter(uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter f, List grpcFilters) { + f.getContent().ifPresent(dateRange -> { + DateRangeFilter.DateField dateField = switch (f.getLabel()) { + case "update" -> DateRangeFilter.DateField.UPDATE; + case "create" -> DateRangeFilter.DateField.CREATE; + case "release" -> DateRangeFilter.DateField.RELEASE; + case "submitted" -> DateRangeFilter.DateField.SUBMITTED; + default -> throw new IllegalArgumentException("Unknown date field " + f.getLabel()); + }; + grpcFilters.add(Filter.newBuilder().setDateRange( + DateRangeFilter.newBuilder() + .setField(dateField) + .setFrom(dateRange.getFrom().toString()) + .setTo(dateRange.getUntil().toString()) + ).build()); + }); + } + + private static void getAttributeSearchFilter(uk.ac.ebi.biosamples.core.model.filter.AttributeFilter filter, + Map filterMap) { + Filter.Builder fb = filterMap.getOrDefault(filter.getLabel(), Filter.newBuilder() + .setAttribute(AttributeFilter.newBuilder().setField(filter.getLabel()))); + filter.getContent().ifPresent(attribute -> fb.getAttributeBuilder().addValues(attribute)); + filterMap.putIfAbsent(filter.getLabel(), fb); + } + + private static Filter getPrivateSearchFilter(String webinId) { + PublicFilter.Builder publicFilterBuilder = PublicFilter.newBuilder(); + if (StringUtils.hasText(webinId)) { + publicFilterBuilder.setWebinId(webinId); + } + return Filter.newBuilder().setPublic(publicFilterBuilder.build()).build(); + } +} From 9bad54336e7080f75effbc9e961d624e34b0e4d2 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 2 Sep 2025 15:41:23 +0100 Subject: [PATCH 08/86] feat: helm deployment --- .gitlab-ci.yml | 39 ++++++ .../messaging/MessagingConstants.java | 4 +- .../biosamples/FileDownloadIntegration.java | 37 +++--- k8s/helm/.helmignore | 23 ++++ k8s/helm/Chart.yaml | 24 ++++ k8s/helm/templates/NOTES.txt | 22 ++++ k8s/helm/templates/_helpers.tpl | 62 +++++++++ k8s/helm/templates/deployment.yaml | 96 ++++++++++++++ k8s/helm/templates/hpa.yaml | 32 +++++ k8s/helm/templates/ingress.yaml | 43 ++++++ k8s/helm/templates/service.yaml | 15 +++ k8s/helm/templates/serviceaccount.yaml | 13 ++ k8s/helm/templates/tests/test-connection.yaml | 15 +++ k8s/helm/values-dev.yaml | 8 ++ k8s/helm/values.yaml | 123 ++++++++++++++++++ .../ebi/biosamples/BioSamplesProperties.java | 3 + .../service/facet/ElasticFacetService.java | 2 +- .../service/search/ElasticSearchService.java | 12 +- 18 files changed, 544 insertions(+), 29 deletions(-) create mode 100644 k8s/helm/.helmignore create mode 100644 k8s/helm/Chart.yaml create mode 100644 k8s/helm/templates/NOTES.txt create mode 100644 k8s/helm/templates/_helpers.tpl create mode 100644 k8s/helm/templates/deployment.yaml create mode 100644 k8s/helm/templates/hpa.yaml create mode 100644 k8s/helm/templates/ingress.yaml create mode 100644 k8s/helm/templates/service.yaml create mode 100644 k8s/helm/templates/serviceaccount.yaml create mode 100644 k8s/helm/templates/tests/test-connection.yaml create mode 100644 k8s/helm/values-dev.yaml create mode 100644 k8s/helm/values.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4d9d0904..5b40717fd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,6 +10,7 @@ variables: DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE:$DOCKER_TAG DOCKER_PULL_SECRET: docker-registry-secret APP_NAME: $CI_PROJECT_NAME + K8S_NAMESPACE_PREFIX: biosamples before_script: @@ -193,3 +194,41 @@ deploy-production-solr: environment: name: prod-solr-${BSD_HOST_NAME} url: http://${BSD_SOLR_HOST_NAME}.ebi.ac.uk:8983/solr + + +deploy_k8s_wwwdev: + variables: + ENVIRONMENT_NAME: dev + environment: + name: fallback_dev + url: https://wwwdev.ebi.ac.uk/biosamples + only: + - dev + - main + - biosamples-search + when: manual + extends: .kube_deploy_script + +.kube_deploy_script: + stage: deploy + image: ${CI_REGISTRY_IMAGE}/dtzar/helm-kubectl:3.16 + tags: ["dind"] + services: + - docker:27-dind + variables: + K8S_NAMESPACE: ${K8S_NAMESPACE_PREFIX}-${ENVIRONMENT_NAME} + script: + - echo $K8S_NAMESPACE + - kubectl config set-context --current --namespace=${K8S_NAMESPACE} + - kubectl delete secret $DOCKER_PULL_SECRET || true + - | + kubectl create secret docker-registry $DOCKER_PULL_SECRET \ + --docker-server=$CI_REGISTRY \ + --docker-username=$CI_REGISTRY_USER \ + --docker-password=$CI_REGISTRY_PASSWORD + - | + helm upgrade --install $APP_NAME ./k8s/helm \ + --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ + --set image.repository=$CI_REGISTRY_IMAGE \ + --set image.tag=$DOCKER_TAG \ + --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" diff --git a/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java b/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java index 315c3e48e..51ee1ad78 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java @@ -11,8 +11,8 @@ package uk.ac.ebi.biosamples.messaging; public class MessagingConstants { - public static final String INDEXING_EXCHANGE = "biosamples.forindexing.solr"; - public static final String INDEXING_QUEUE = "biosamples.tobeindexed.solr"; + public static final String INDEXING_EXCHANGE = "biosamples.indexing"; + public static final String INDEXING_QUEUE = "biosamples.indexing.es"; public static final String REINDEXING_EXCHANGE = "biosamples.reindex.solr"; public static final String REINDEXING_QUEUE = "biosamples.reindex.solr"; public static final String UPLOAD_QUEUE = "biosamples.uploaded.files"; diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/FileDownloadIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/FileDownloadIntegration.java index 5ea53ffe7..9b0945f94 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/FileDownloadIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/FileDownloadIntegration.java @@ -49,25 +49,24 @@ protected void phaseTwo() { @Override protected void phaseThree() { -// final String sampleDownloadUrl = clientProperties.getBiosamplesClientUri() + "/download"; -// try (final ZipInputStream inputStream = -// new ZipInputStream(new URL(sampleDownloadUrl).openStream())) { -// final ZipEntry entry = inputStream.getNextEntry(); -// if (entry == null || !"samples.json".equals(entry.getName())) { -// throw new IntegrationTestFailException( -// "Could not download zipped samples.json", Phase.THREE); -// } -// -// final StringWriter writer = new StringWriter(); -// IOUtils.copy(inputStream, writer, Charset.defaultCharset()); -// final JsonNode samples = new ObjectMapper().readTree(writer.toString()); -// if (!samples.isArray()) { -// throw new IntegrationTestFailException("Invalid format in samples.json", Phase.THREE); -// } -// } catch (final IOException e) { -// // TODO: @Isuru to check please! -// /*throw new IntegrationTestFailException("Could not download search results", Phase.THREE);*/ -// } + final String sampleDownloadUrl = clientProperties.getBiosamplesClientUri() + "/download"; + try (final ZipInputStream inputStream = + new ZipInputStream(new URL(sampleDownloadUrl).openStream())) { + final ZipEntry entry = inputStream.getNextEntry(); + if (entry == null || !"samples.json".equals(entry.getName())) { + throw new IntegrationTestFailException( + "Could not download zipped samples.json", Phase.THREE); + } + + final StringWriter writer = new StringWriter(); + IOUtils.copy(inputStream, writer, Charset.defaultCharset()); + final JsonNode samples = new ObjectMapper().readTree(writer.toString()); + if (!samples.isArray()) { + throw new IntegrationTestFailException("Invalid format in samples.json", Phase.THREE); + } + } catch (final IOException e) { + throw new IntegrationTestFailException("Could not download search results", Phase.THREE); + } } @Override diff --git a/k8s/helm/.helmignore b/k8s/helm/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/k8s/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8s/helm/Chart.yaml b/k8s/helm/Chart.yaml new file mode 100644 index 000000000..fc3899226 --- /dev/null +++ b/k8s/helm/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: helm +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/k8s/helm/templates/NOTES.txt b/k8s/helm/templates/NOTES.txt new file mode 100644 index 000000000..62f1b422e --- /dev/null +++ b/k8s/helm/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "helm.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch its status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "helm.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "helm.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "helm.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/k8s/helm/templates/_helpers.tpl b/k8s/helm/templates/_helpers.tpl new file mode 100644 index 000000000..ba04c300d --- /dev/null +++ b/k8s/helm/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "helm.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "helm.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "helm.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "helm.labels" -}} +helm.sh/chart: {{ include "helm.chart" . }} +{{ include "helm.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "helm.selectorLabels" -}} +app.kubernetes.io/name: {{ include "helm.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "helm.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "helm.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/k8s/helm/templates/deployment.yaml b/k8s/helm/templates/deployment.yaml new file mode 100644 index 000000000..db7668481 --- /dev/null +++ b/k8s/helm/templates/deployment.yaml @@ -0,0 +1,96 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "helm.fullname" . }} + labels: + {{- include "helm.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "helm.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "helm.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "helm.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SPRING_DATA_MONGODB_URI + valueFrom: + secretKeyRef: + name: biosamples-mongodb + key: connection-string + - name: SPRING_RABBITMQ_HOST + value: {{ .Values.rabbitmq.host }} + - name: SPRING_RABBITMQ_USERNAME + valueFrom: + secretKeyRef: + name: rabbitmq-user + key: username + - name: SPRING_RABBITMQ_PASSWORD + valueFrom: + secretKeyRef: + name: rabbitmq-user + key: password + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/k8s/helm/templates/hpa.yaml b/k8s/helm/templates/hpa.yaml new file mode 100644 index 000000000..28c087ea4 --- /dev/null +++ b/k8s/helm/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "helm.fullname" . }} + labels: + {{- include "helm.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "helm.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/k8s/helm/templates/ingress.yaml b/k8s/helm/templates/ingress.yaml new file mode 100644 index 000000000..5bdb7911f --- /dev/null +++ b/k8s/helm/templates/ingress.yaml @@ -0,0 +1,43 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "helm.fullname" . }} + labels: + {{- include "helm.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- with .pathType }} + pathType: {{ . }} + {{- end }} + backend: + service: + name: {{ include "helm.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/k8s/helm/templates/service.yaml b/k8s/helm/templates/service.yaml new file mode 100644 index 000000000..de450fc57 --- /dev/null +++ b/k8s/helm/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "helm.fullname" . }} + labels: + {{- include "helm.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "helm.selectorLabels" . | nindent 4 }} diff --git a/k8s/helm/templates/serviceaccount.yaml b/k8s/helm/templates/serviceaccount.yaml new file mode 100644 index 000000000..d47046559 --- /dev/null +++ b/k8s/helm/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "helm.serviceAccountName" . }} + labels: + {{- include "helm.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/k8s/helm/templates/tests/test-connection.yaml b/k8s/helm/templates/tests/test-connection.yaml new file mode 100644 index 000000000..bf1c65f24 --- /dev/null +++ b/k8s/helm/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "helm.fullname" . }}-test-connection" + labels: + {{- include "helm.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "helm.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/k8s/helm/values-dev.yaml b/k8s/helm/values-dev.yaml new file mode 100644 index 000000000..cb9c8ff02 --- /dev/null +++ b/k8s/helm/values-dev.yaml @@ -0,0 +1,8 @@ +rabbitmq: + host: wp-np2-40.ebi.ac.uk +biosamples: + schemaValidator: http://localhost:3020/validate + schemaStore: http://localhost:8085 + search: + host: biosamples-search-helm + port: 9090 diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml new file mode 100644 index 000000000..ce6287d1d --- /dev/null +++ b/k8s/helm/values.yaml @@ -0,0 +1,123 @@ +# Default values for helm. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +replicaCount: 1 + +# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ +image: + repository: nginx + # This sets the pull policy for images. + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# This is to override the chart name. +nameOverride: "" +fullnameOverride: "" + +# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# This is for setting Kubernetes Annotations to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} +# This is for setting Kubernetes Labels to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/ +service: + # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: ClusterIP + # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports + port: 80 + +# This block is for setting up the ingress for more information can be found here: https://kubernetes.io/docs/concepts/services-networking/ingress/ +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +livenessProbe: + httpGet: + path: / + port: http +readinessProbe: + httpGet: + path: / + port: http + +# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java b/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java index 83fe18841..c0d2f81ca 100644 --- a/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java +++ b/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java @@ -118,6 +118,9 @@ public class BioSamplesProperties { @Value("${biosamples.search.host:localhost}") private String biosamplesSearchHost; + @Value("${biosamples.search.port:9090}") + private int biosamplesSearchPort; + public int getBiosamplesClientConnectionCountMax() { return connectionCountMax; } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java index 9ddb93d78..c8fcaa7f3 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java @@ -32,7 +32,7 @@ public List getFacets(String searchTerm, Set filters, String webi Pageable facetFieldPageInfo, Pageable facetValuesPageInfo, String facetField, List facetFields) { - ManagedChannel channel = ManagedChannelBuilder.forAddress(bioSamplesProperties.getBiosamplesSearchHost(), 9090).usePlaintext().build(); + ManagedChannel channel = ManagedChannelBuilder.forAddress(bioSamplesProperties.getBiosamplesSearchHost(), bioSamplesProperties.getBiosamplesSearchPort()).usePlaintext().build(); SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); FacetResponse response; try { diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java index 0742c0b3f..da928abb9 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java @@ -16,6 +16,7 @@ import uk.ac.ebi.biosamples.search.grpc.*; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; +import java.io.OutputStream; import java.time.Instant; import java.util.ArrayList; import java.util.Iterator; @@ -31,7 +32,7 @@ public class ElasticSearchService implements SearchService { @Override @Timed("biosamples.search.page.elastic") public Page searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable) { - ManagedChannel channel = ManagedChannelBuilder.forAddress(bioSamplesProperties.getBiosamplesSearchHost(), 9090).usePlaintext().build(); + ManagedChannel channel = ManagedChannelBuilder.forAddress(bioSamplesProperties.getBiosamplesSearchHost(), bioSamplesProperties.getBiosamplesSearchPort()).usePlaintext().build(); SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); SearchResponse response; try { @@ -64,7 +65,6 @@ public Page searchForAccessions(String searchTerm, Set filters, @Override @Timed("biosamples.search.cursor.elastic") public CursorArrayList searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size) { - log.warn("SEARCHING WITH CURSOR {} ==============================================================================================", cursor); SearchAfter searchAfter = null; String[] cursorParts = cursor.split(","); if (cursorParts.length == 2) { @@ -75,7 +75,7 @@ public CursorArrayList searchForAccessions(String searchTerm, Set searchForAccessions(String searchTerm, Set accessions = response.getAccessionsList(); SearchAfter newSearchAfter = response.getSearchAfter(); - log.warn("GOT SEARCH AFTER {} ==============================================================================================", newSearchAfter); - if (StringUtils.hasText(newSearchAfter.getAccession())) { cursor = Timestamps.toString(newSearchAfter.getUpdate()) + "," + newSearchAfter.getAccession(); } @@ -110,7 +108,7 @@ public CursorArrayList searchForAccessions(String searchTerm, Set searchForAccessionsStream(String searchTerm, Set filters, String webinId, String cursor, int size) { + /*public OutputStream searchForAccessionsStream(String searchTerm, Set filters, String webinId, String cursor, int size) { SearchAfter searchAfter = null; String[] cursorParts = cursor.split(","); if (cursorParts.length == 2) { @@ -151,5 +149,5 @@ public CursorArrayList searchForAccessionsStream(String searchTerm, Set< } return new CursorArrayList<>(accessionList, cursor); - } + }*/ } From bd9db041dc645008b6551b0c516b3ab73f6010a8 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 2 Sep 2025 16:22:46 +0100 Subject: [PATCH 09/86] fix: deployment --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b40717fd..9cc0f7137 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -211,7 +211,7 @@ deploy_k8s_wwwdev: .kube_deploy_script: stage: deploy - image: ${CI_REGISTRY_IMAGE}/dtzar/helm-kubectl:3.16 + image: dtzar/helm-kubectl:3.16 tags: ["dind"] services: - docker:27-dind From 081b423d74072e9680344d4c4130858dc8436ca0 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Wed, 3 Sep 2025 10:23:31 +0100 Subject: [PATCH 10/86] fix: deployment --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9cc0f7137..17a465a26 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -219,6 +219,7 @@ deploy_k8s_wwwdev: K8S_NAMESPACE: ${K8S_NAMESPACE_PREFIX}-${ENVIRONMENT_NAME} script: - echo $K8S_NAMESPACE + - kubectl config view - kubectl config set-context --current --namespace=${K8S_NAMESPACE} - kubectl delete secret $DOCKER_PULL_SECRET || true - | From 45a929643e175445437376d0c857acc77e1157f7 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Wed, 3 Sep 2025 11:50:34 +0100 Subject: [PATCH 11/86] test: deployment --- .gitlab-ci.yml | 6 +++--- k8s/helm/values-prod.yaml | 8 ++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 k8s/helm/values-prod.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 17a465a26..f995d1984 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -196,11 +196,11 @@ deploy-production-solr: url: http://${BSD_SOLR_HOST_NAME}.ebi.ac.uk:8983/solr -deploy_k8s_wwwdev: +deploy_k8s_hh_prod: variables: - ENVIRONMENT_NAME: dev + ENVIRONMENT_NAME: prod environment: - name: fallback_dev + name: primary_prod url: https://wwwdev.ebi.ac.uk/biosamples only: - dev diff --git a/k8s/helm/values-prod.yaml b/k8s/helm/values-prod.yaml new file mode 100644 index 000000000..cb9c8ff02 --- /dev/null +++ b/k8s/helm/values-prod.yaml @@ -0,0 +1,8 @@ +rabbitmq: + host: wp-np2-40.ebi.ac.uk +biosamples: + schemaValidator: http://localhost:3020/validate + schemaStore: http://localhost:8085 + search: + host: biosamples-search-helm + port: 9090 From 037bf2e8a7f05960c8fc9054790d04789fc208d9 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 5 Sep 2025 12:07:33 +0100 Subject: [PATCH 12/86] test: remove livenss probe for testing --- k8s/helm/templates/deployment.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/k8s/helm/templates/deployment.yaml b/k8s/helm/templates/deployment.yaml index db7668481..e93138ac8 100644 --- a/k8s/helm/templates/deployment.yaml +++ b/k8s/helm/templates/deployment.yaml @@ -62,14 +62,14 @@ spec: - name: http containerPort: {{ .Values.service.port }} protocol: TCP - {{- with .Values.livenessProbe }} - livenessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.readinessProbe }} - readinessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} +{{/* {{- with .Values.livenessProbe }}*/}} +{{/* livenessProbe:*/}} +{{/* {{- toYaml . | nindent 12 }}*/}} +{{/* {{- end }}*/}} +{{/* {{- with .Values.readinessProbe }}*/}} +{{/* readinessProbe:*/}} +{{/* {{- toYaml . | nindent 12 }}*/}} +{{/* {{- end }}*/}} {{- with .Values.resources }} resources: {{- toYaml . | nindent 12 }} From 8fa44203d66de3bd642fcc0a0c541f2ef6ae0dfe Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 5 Sep 2025 12:17:27 +0100 Subject: [PATCH 13/86] test: remove livenss probe for testing --- k8s/helm/values-dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/helm/values-dev.yaml b/k8s/helm/values-dev.yaml index cb9c8ff02..7adf22402 100644 --- a/k8s/helm/values-dev.yaml +++ b/k8s/helm/values-dev.yaml @@ -5,4 +5,4 @@ biosamples: schemaStore: http://localhost:8085 search: host: biosamples-search-helm - port: 9090 + port: 9090 \ No newline at end of file From 004a15420fe5dbc94b22aa286bfed3804c7e4ed7 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 5 Sep 2025 12:50:48 +0100 Subject: [PATCH 14/86] test: remove livenss probe for testing --- k8s/helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index ce6287d1d..e04c05ca4 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -54,7 +54,7 @@ service: # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types type: ClusterIP # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports - port: 80 + port: 8080 # This block is for setting up the ingress for more information can be found here: https://kubernetes.io/docs/concepts/services-networking/ingress/ ingress: From 8ddbfb9e00501ec1ad1134e7bb035e9c9e1c7cf4 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 5 Sep 2025 14:00:53 +0100 Subject: [PATCH 15/86] fix: search port from env --- k8s/helm/templates/deployment.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/k8s/helm/templates/deployment.yaml b/k8s/helm/templates/deployment.yaml index e93138ac8..56959ac6b 100644 --- a/k8s/helm/templates/deployment.yaml +++ b/k8s/helm/templates/deployment.yaml @@ -58,6 +58,10 @@ spec: secretKeyRef: name: rabbitmq-user key: password + - name: BIOSAMPLES_SEARCH_HOST + value: {{ .Values.biosamples.search.host }} + - name: BIOSAMPLES_SEARCH_PORT + value: {{ .Values.biosamples.search.port }} ports: - name: http containerPort: {{ .Values.service.port }} From f1e302b5d0b46decf75df2cc78c0189f0ae1563c Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 5 Sep 2025 14:21:49 +0100 Subject: [PATCH 16/86] test: deployment with search --- k8s/helm/templates/deployment.yaml | 4 ---- .../main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/k8s/helm/templates/deployment.yaml b/k8s/helm/templates/deployment.yaml index 56959ac6b..e93138ac8 100644 --- a/k8s/helm/templates/deployment.yaml +++ b/k8s/helm/templates/deployment.yaml @@ -58,10 +58,6 @@ spec: secretKeyRef: name: rabbitmq-user key: password - - name: BIOSAMPLES_SEARCH_HOST - value: {{ .Values.biosamples.search.host }} - - name: BIOSAMPLES_SEARCH_PORT - value: {{ .Values.biosamples.search.port }} ports: - name: http containerPort: {{ .Values.service.port }} diff --git a/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java b/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java index c0d2f81ca..ba280137b 100644 --- a/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java +++ b/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java @@ -115,7 +115,7 @@ public class BioSamplesProperties { @Value("${biosamples.bulksubmisison.webin.superuser.validation:false}") private boolean enableBulkSubmissionWebinSuperUserValidation; - @Value("${biosamples.search.host:localhost}") + @Value("${biosamples.search.host:biosamples-search-helm}") private String biosamplesSearchHost; @Value("${biosamples.search.port:9090}") From d0d52873867226a6156844d23d3daee60bcbb7b5 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 9 Sep 2025 13:37:23 +0100 Subject: [PATCH 17/86] test: multi stage docker build --- .gitlab-ci.yml | 107 +++++++++++++----- .../biosamples/service/MessagingService.java | 6 +- k8s/Dockerfile | 20 ++++ .../uk/ac/ebi/biosamples/ReindexRunner.java | 40 ++++--- .../ac/ebi/biosamples/ReindexRunnerTest.java | 4 +- 5 files changed, 126 insertions(+), 51 deletions(-) create mode 100644 k8s/Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f995d1984..3ee29941e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,50 +24,99 @@ stages: - config - deploy -maven-package-webapps-core: +maven-package-all-apps: image: ${CI_REGISTRY_IMAGE}/eclipse-temurin:17-jdk stage: build script: - - './mvnw -q deploy -pl webapps/core -am -P embl-ebi -s ci_settings.xml -DskipTests -Dmaven.source.skip=true' + - './mvnw -q package -P embl-ebi -s ci_settings.xml -DskipTests -Dmaven.source.skip=true' artifacts: paths: - webapps/core/target/webapps-core-*.war + - webapps/core-v2/target/webapps-core-v2-*.jar + - agents/solr/target/agents-solr-*.jar + - agents/uploadworkers/target/agents-uploadworkers-*.jar + - pipelines/curation/target/pipelines-curation-*.jar + - pipelines/ena/target/pipelines-ena-*.jar + - pipelines/ncbi-ena-link/target/pipelines-ncbi-ena-link-*.jar + - pipelines/sample-release/target/pipelines-sample-release-*.jar + - pipelines/sample-post-release-action/target/pipelines-sample-post-release-action-*.jar + - pipelines/ncbi/target/pipelines-ncbi-*.jar + - pipelines/reindex/target/pipelines-reindex-*.jar + - pipelines/sample-transformation-dtol/target/pipelines-sample-transformation-dtol-*.jar - # maven-package: - # image: ${CI_REGISTRY_IMAGE}/eclipse-temurin:17-jdk - # stage: build - # script: - # - './mvnw -q deploy -P embl-ebi -s ci_settings.xml -DskipTests -Dmaven.source.skip=true' - # - mkdir deployment - # - cp webapps/core/target/webapps-core-*.war deployment/webapps-core.war - # - cp webapps/core-v2/target/webapps-core-v2*.jar deployment/webapps-core-v2.jar - # - cp agents/solr/target/agents-solr-*.jar deployment/agents-solr.jar - # - cp agents/uploadworkers/target/agents-uploadworkers-*.jar deployment/agents-uploadworkers.jar - # - cp pipelines/curation/target/pipelines-curation-*.jar deployment/pipelines-curation.jar - # - cp pipelines/ena/target/pipelines-ena-*.jar deployment/pipelines-ena.jar - # - cp pipelines/ncbi-ena-link/target/pipelines-ncbi-ena-link-*.jar deployment/pipelines-ncbi-ena-link.jar - # - cp pipelines/sample-release/target/pipelines-sample-release-*.jar deployment/pipelines-sample-release.jar - # - cp pipelines/sample-post-release-action/target/pipelines-sample-post-release-action*.jar deployment/pipelines-sample-post-release-action.jar - # - cp pipelines/ncbi/target/pipelines-ncbi-*.jar deployment/pipelines-ncbi.jar - # - cp pipelines/reindex/target/pipelines-reindex-*.jar deployment/pipelines-reindex.jar - # - cp pipelines/sample-transformation-dtol/target/pipelines-sample-transformation-dtol-*.jar deployment/pipelines-sample-transformation-dtol.jar - # artifacts: - # paths: - # - deployment -# when: manual - -build_docker_image: + +build_and_push_docker_images: stage: package image: docker:stable services: - docker:stable-dind before_script: - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY" + parallel: + matrix: + - APP_NAME: "webapp-core" + DOCKERFILE_TARGET: "webapp-core" + - APP_NAME: "webapp-core-v2" + DOCKERFILE_TARGET: "webapp-core-v2" + - APP_NAME: "pipelines-reindex" + DOCKERFILE_TARGET: "pipelines-reindex" +# - APP_NAME: "agents-solr" +# DOCKERFILE_TARGET: "agents-solr" +# - APP_NAME: "agents-uploadworkers" +# DOCKERFILE_TARGET: "agents-uploadworkers" +# - APP_NAME: "pipelines-curation" +# DOCKERFILE_TARGET: "pipelines-curation" +# - APP_NAME: "pipelines-ena" +# DOCKERFILE_TARGET: "pipelines-ena" +# - APP_NAME: "pipelines-ncbi-ena-link" +# DOCKERFILE_TARGET: "pipelines-ncbi-ena-link" +# - APP_NAME: "pipelines-sample-release" +# DOCKERFILE_TARGET: "pipelines-sample-release" +# - APP_NAME: "pipelines-sample-post-release-action" +# DOCKERFILE_TARGET: "pipelines-sample-post-release-action" +# - APP_NAME: "pipelines-ncbi" +# DOCKERFILE_TARGET: "pipelines-ncbi" +# - APP_NAME: "pipelines-sample-transformation-dtol" +# DOCKERFILE_TARGET: "pipelines-sample-transformation-dtol" script: - - docker build --build-arg DOCKER_REGISTRY=${CI_REGISTRY_IMAGE} -t $DOCKER_IMAGE_NAME -f webapps/core/Dockerfile . - - docker push $DOCKER_IMAGE_NAME + - | + DOCKER_IMAGE_NAME="$CI_REGISTRY_IMAGE/$APP_NAME:$DOCKER_TAG" + echo "Building and pushing $DOCKER_IMAGE_NAME with target $DOCKERFILE_TARGET" + docker build --target $DOCKERFILE_TARGET -t $DOCKER_IMAGE_NAME -f k8s/Dockerfile . + docker push $DOCKER_IMAGE_NAME + + # Also tag as latest for the current branch + DOCKER_LATEST_NAME="$CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_REF_SLUG-latest" + docker tag $DOCKER_IMAGE_NAME $DOCKER_LATEST_NAME + docker push $DOCKER_LATEST_NAME after_script: - - docker logout ${CI_REGISTRY_IMAGE} + - docker logout $CI_REGISTRY + + + +#maven-package-webapps-core: +# image: ${CI_REGISTRY_IMAGE}/eclipse-temurin:17-jdk +# stage: build +# script: +# - './mvnw -q deploy -pl webapps/core -am -P embl-ebi -s ci_settings.xml -DskipTests -Dmaven.source.skip=true' +# artifacts: +# paths: +# - webapps/core/target/webapps-core-*.war +# +#build_docker_image: +# stage: package +# image: docker:stable +# services: +# - docker:stable-dind +# before_script: +# - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY" +# script: +# - docker build --build-arg DOCKER_REGISTRY=${CI_REGISTRY_IMAGE} -t $DOCKER_IMAGE_NAME -f webapps/core/Dockerfile . +# - docker push $DOCKER_IMAGE_NAME +# after_script: +# - docker logout ${CI_REGISTRY_IMAGE} + + deploy-dev-bsd-v1: image: dtzar/helm-kubectl:3.11.0 diff --git a/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java b/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java index 086ac4446..11489290a 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java @@ -37,11 +37,12 @@ public class MessagingService { private final Logger log = LoggerFactory.getLogger(getClass()); private final SampleReadService sampleReadService; private final AmqpTemplate amqpTemplate; + private final ObjectMapper objectMapper; - public MessagingService( - final SampleReadService sampleReadService, final AmqpTemplate amqpTemplate) { + public MessagingService(SampleReadService sampleReadService, AmqpTemplate amqpTemplate, ObjectMapper objectMapper) { this.sampleReadService = sampleReadService; this.amqpTemplate = amqpTemplate; + this.objectMapper = objectMapper; } public void sendFileUploadedMessage(final String fileId) { @@ -81,7 +82,6 @@ void fetchThenSendMessage( // MessageContent.build(sample.get(), null, related, false)); try { - ObjectMapper objectMapper = new ObjectMapper(); String json = objectMapper.writeValueAsString(sample.get()); log.info("Sending message for indexing: {}", sample.get().getAccession()); // amqpTemplate.send(MessagingConstants.INDEXING_EXCHANGE, MessagingConstants.INDEXING_QUEUE, new Message(json.getBytes(StandardCharsets.UTF_8))); diff --git a/k8s/Dockerfile b/k8s/Dockerfile new file mode 100644 index 000000000..243a31b1c --- /dev/null +++ b/k8s/Dockerfile @@ -0,0 +1,20 @@ +FROM openjdk:17-jre-slim as base +WORKDIR /app + +# Stage for core webapp +FROM base as webapp-core +COPY webapps/core/target/webapps-core-*.war app.war +EXPOSE 8080 +CMD ["java", "-jar", "app.war"] + +# Stage for core webapp +FROM base as webapp-core-v2 +COPY webapps/core-v2/target/webapps-core-*.war app.war +EXPOSE 8080 +CMD ["java", "-jar", "app.war"] + +# Stage for agents-solr +FROM base as agents-solr +COPY agents/solr/target/agents-solr-*.jar app.jar +EXPOSE 8081 +CMD ["java", "-jar", "app.jar"] diff --git a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java index 9865e3ee5..cd3b740f9 100644 --- a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java +++ b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java @@ -12,6 +12,8 @@ import java.util.*; import java.util.concurrent.*; + +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang.IncompleteArgumentException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,19 +48,19 @@ */ @Component public class ReindexRunner implements ApplicationRunner { - private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationRunner.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ReindexRunner.class); private final AmqpTemplate amqpTemplate; private final SampleReadService sampleReadService; private final MongoOperations mongoOperations; + private final ObjectMapper objectMapper; @Autowired - public ReindexRunner( - final AmqpTemplate amqpTemplate, - final SampleReadService sampleReadService, - final MongoOperations mongoOperations) { + public ReindexRunner(AmqpTemplate amqpTemplate, SampleReadService sampleReadService, + MongoOperations mongoOperations, ObjectMapper objectMapper) { this.amqpTemplate = amqpTemplate; this.sampleReadService = sampleReadService; this.mongoOperations = mongoOperations; + this.objectMapper = objectMapper; } @Override @@ -103,7 +105,7 @@ public void run(final ApplicationArguments args) throws Exception { futures.put( accession, executor.submit( - new SampleIndexingCallable(accession, sampleReadService, amqpTemplate))); + new SampleIndexingCallable(accession, sampleReadService, amqpTemplate, objectMapper))); ThreadUtils.checkFutures(futures, 1000); } @@ -121,14 +123,16 @@ private static class SampleIndexingCallable implements Callable { private final String accession; private final SampleReadService sampleReadService; private final AmqpTemplate amqpTemplate; + private final ObjectMapper objectMapper; - public SampleIndexingCallable( - final String accession, - final SampleReadService sampleReadService, - final AmqpTemplate amqpTemplate) { + public SampleIndexingCallable(String accession, + SampleReadService sampleReadService, + AmqpTemplate amqpTemplate, + ObjectMapper objectMapper) { this.accession = accession; this.sampleReadService = sampleReadService; this.amqpTemplate = amqpTemplate; + this.objectMapper = objectMapper; } @Override @@ -153,19 +157,19 @@ private boolean fetchSampleAndSendMessage(final boolean isRetry) { if (sampleOptional.isPresent()) { try { - amqpTemplate.convertAndSend( - MessagingConstants.REINDEXING_EXCHANGE, - MessagingConstants.REINDEXING_QUEUE, - MessageContent.build(sampleOptional.get(), null, related, false)); + String json = objectMapper.writeValueAsString(sampleOptional.get()); + amqpTemplate.convertAndSend(MessagingConstants.INDEXING_EXCHANGE, MessagingConstants.INDEXING_QUEUE, json); +// amqpTemplate.convertAndSend( +// MessagingConstants.REINDEXING_EXCHANGE, +// MessagingConstants.REINDEXING_QUEUE, +// MessageContent.build(sampleOptional.get(), null, related, false)); return true; } catch (final Exception e) { - LOGGER.error( - String.format( - "Failed to convert sample to message and send to queue for %s", accession), - e); + LOGGER.error("Failed to convert sample to message and send to queue for {}", accession, e); } } else { + LOGGER.warn("Failed to fetch sample for {}", accession); final String errorMessage = isRetry ? String.format("Failed to fetch sample after retrying for %s", accession) diff --git a/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java b/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java index 8a65f8be2..ec8e0685f 100644 --- a/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java +++ b/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java @@ -38,6 +38,8 @@ public class ReindexRunnerTest { @Mock private AmqpTemplate amqpTemplate; @Mock private MongoOperations mongoOperations; @Mock private SampleReadService sampleReadService; + @Mock private ObjectMapper objectMapper; + private final List accessions = Arrays.asList("ACCESSION1", "ACCESSION2", "ACCESSION3"); @@ -108,7 +110,7 @@ public MongoSample next() { .thenReturn(Optional.empty()) .thenReturn(Optional.of(sample3)); final ReindexRunner reindexRunner = - new ReindexRunner(amqpTemplate, sampleReadService, mongoOperations); + new ReindexRunner(amqpTemplate, sampleReadService, mongoOperations, objectMapper); reindexRunner.run(applicationArguments); } } From d3eb2f76ecfa702e0a383f5bcf1f776e76ab87eb Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 9 Sep 2025 14:09:47 +0100 Subject: [PATCH 18/86] test: build images --- k8s/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/Dockerfile b/k8s/Dockerfile index 243a31b1c..7bcdc779f 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -1,4 +1,4 @@ -FROM openjdk:17-jre-slim as base +FROM eclipse-temurin:17-jdk-alpine as base WORKDIR /app # Stage for core webapp From 4ce91e76f3d749e6fdf7140b6f1288d421aa123a Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 9 Sep 2025 14:32:20 +0100 Subject: [PATCH 19/86] test: multi stage docker build --- k8s/Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/k8s/Dockerfile b/k8s/Dockerfile index 7bcdc779f..c10fe7478 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -1,20 +1,17 @@ FROM eclipse-temurin:17-jdk-alpine as base WORKDIR /app -# Stage for core webapp FROM base as webapp-core COPY webapps/core/target/webapps-core-*.war app.war EXPOSE 8080 CMD ["java", "-jar", "app.war"] -# Stage for core webapp FROM base as webapp-core-v2 COPY webapps/core-v2/target/webapps-core-*.war app.war EXPOSE 8080 CMD ["java", "-jar", "app.war"] -# Stage for agents-solr -FROM base as agents-solr -COPY agents/solr/target/agents-solr-*.jar app.jar +FROM base as pipelines-reindex +COPY pipelines/reindex/target/pipelines-reindex-*.jar app.jar EXPOSE 8081 CMD ["java", "-jar", "app.jar"] From 7ec8188288cb1b0e91f0c9d64b17a149e64147f1 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 12 Sep 2025 10:57:40 +0100 Subject: [PATCH 20/86] test: gitlab --- .gitlab-ci.yml | 33 +---- k8s/Dockerfile | 1 - k8s/helm/Chart.yaml | 18 --- k8s/helm_test/.helmignore | 23 +++ k8s/helm_test/Chart.yaml | 24 +++ k8s/helm_test/templates/NOTES.txt | 22 +++ k8s/helm_test/templates/_helpers.tpl | 60 ++++++++ k8s/helm_test/templates/deployment.yaml | 78 ++++++++++ k8s/helm_test/templates/hpa.yaml | 32 ++++ k8s/helm_test/templates/ingress.yaml | 47 ++++++ k8s/helm_test/templates/job.yaml | 67 +++++++++ k8s/helm_test/templates/service.yaml | 22 +++ k8s/helm_test/templates/serviceaccount.yaml | 13 ++ .../templates/tests/test-connection.yaml | 15 ++ k8s/helm_test/values-dev.yaml | 8 + k8s/helm_test/values-prod.yaml | 56 +++++++ k8s/helm_test/values.yaml | 139 ++++++++++++++++++ 17 files changed, 609 insertions(+), 49 deletions(-) create mode 100644 k8s/helm_test/.helmignore create mode 100644 k8s/helm_test/Chart.yaml create mode 100644 k8s/helm_test/templates/NOTES.txt create mode 100644 k8s/helm_test/templates/_helpers.tpl create mode 100644 k8s/helm_test/templates/deployment.yaml create mode 100644 k8s/helm_test/templates/hpa.yaml create mode 100644 k8s/helm_test/templates/ingress.yaml create mode 100644 k8s/helm_test/templates/job.yaml create mode 100644 k8s/helm_test/templates/service.yaml create mode 100644 k8s/helm_test/templates/serviceaccount.yaml create mode 100644 k8s/helm_test/templates/tests/test-connection.yaml create mode 100644 k8s/helm_test/values-dev.yaml create mode 100644 k8s/helm_test/values-prod.yaml create mode 100644 k8s/helm_test/values.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3ee29941e..fe8b8af64 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,16 +33,7 @@ maven-package-all-apps: paths: - webapps/core/target/webapps-core-*.war - webapps/core-v2/target/webapps-core-v2-*.jar - - agents/solr/target/agents-solr-*.jar - - agents/uploadworkers/target/agents-uploadworkers-*.jar - - pipelines/curation/target/pipelines-curation-*.jar - - pipelines/ena/target/pipelines-ena-*.jar - - pipelines/ncbi-ena-link/target/pipelines-ncbi-ena-link-*.jar - - pipelines/sample-release/target/pipelines-sample-release-*.jar - - pipelines/sample-post-release-action/target/pipelines-sample-post-release-action-*.jar - - pipelines/ncbi/target/pipelines-ncbi-*.jar - pipelines/reindex/target/pipelines-reindex-*.jar - - pipelines/sample-transformation-dtol/target/pipelines-sample-transformation-dtol-*.jar build_and_push_docker_images: @@ -60,24 +51,6 @@ build_and_push_docker_images: DOCKERFILE_TARGET: "webapp-core-v2" - APP_NAME: "pipelines-reindex" DOCKERFILE_TARGET: "pipelines-reindex" -# - APP_NAME: "agents-solr" -# DOCKERFILE_TARGET: "agents-solr" -# - APP_NAME: "agents-uploadworkers" -# DOCKERFILE_TARGET: "agents-uploadworkers" -# - APP_NAME: "pipelines-curation" -# DOCKERFILE_TARGET: "pipelines-curation" -# - APP_NAME: "pipelines-ena" -# DOCKERFILE_TARGET: "pipelines-ena" -# - APP_NAME: "pipelines-ncbi-ena-link" -# DOCKERFILE_TARGET: "pipelines-ncbi-ena-link" -# - APP_NAME: "pipelines-sample-release" -# DOCKERFILE_TARGET: "pipelines-sample-release" -# - APP_NAME: "pipelines-sample-post-release-action" -# DOCKERFILE_TARGET: "pipelines-sample-post-release-action" -# - APP_NAME: "pipelines-ncbi" -# DOCKERFILE_TARGET: "pipelines-ncbi" -# - APP_NAME: "pipelines-sample-transformation-dtol" -# DOCKERFILE_TARGET: "pipelines-sample-transformation-dtol" script: - | DOCKER_IMAGE_NAME="$CI_REGISTRY_IMAGE/$APP_NAME:$DOCKER_TAG" @@ -89,6 +62,7 @@ build_and_push_docker_images: DOCKER_LATEST_NAME="$CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_REF_SLUG-latest" docker tag $DOCKER_IMAGE_NAME $DOCKER_LATEST_NAME docker push $DOCKER_LATEST_NAME + echo $DOCKER_LATEST_NAME after_script: - docker logout $CI_REGISTRY @@ -117,7 +91,6 @@ build_and_push_docker_images: # - docker logout ${CI_REGISTRY_IMAGE} - deploy-dev-bsd-v1: image: dtzar/helm-kubectl:3.11.0 stage: deploy @@ -279,6 +252,6 @@ deploy_k8s_hh_prod: - | helm upgrade --install $APP_NAME ./k8s/helm \ --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ + --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE \ - --set image.tag=$DOCKER_TAG \ - --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" + --set image.tag=$DOCKER_TAG diff --git a/k8s/Dockerfile b/k8s/Dockerfile index c10fe7478..f83adc8b5 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -13,5 +13,4 @@ CMD ["java", "-jar", "app.war"] FROM base as pipelines-reindex COPY pipelines/reindex/target/pipelines-reindex-*.jar app.jar -EXPOSE 8081 CMD ["java", "-jar", "app.jar"] diff --git a/k8s/helm/Chart.yaml b/k8s/helm/Chart.yaml index fc3899226..ec7928885 100644 --- a/k8s/helm/Chart.yaml +++ b/k8s/helm/Chart.yaml @@ -1,24 +1,6 @@ apiVersion: v2 name: helm description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. appVersion: "1.16.0" diff --git a/k8s/helm_test/.helmignore b/k8s/helm_test/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/k8s/helm_test/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8s/helm_test/Chart.yaml b/k8s/helm_test/Chart.yaml new file mode 100644 index 000000000..fc3899226 --- /dev/null +++ b/k8s/helm_test/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: helm +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/k8s/helm_test/templates/NOTES.txt b/k8s/helm_test/templates/NOTES.txt new file mode 100644 index 000000000..62f1b422e --- /dev/null +++ b/k8s/helm_test/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "helm.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch its status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "helm.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "helm.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "helm.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/k8s/helm_test/templates/_helpers.tpl b/k8s/helm_test/templates/_helpers.tpl new file mode 100644 index 000000000..9ef944798 --- /dev/null +++ b/k8s/helm_test/templates/_helpers.tpl @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "biosamples.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "biosamples.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "biosamples.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "biosamples.labels" -}} +helm.sh/chart: {{ include "biosamples.chart" . }} +{{ include "biosamples.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "biosamples.selectorLabels" -}} +app.kubernetes.io/name: {{ include "biosamples.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "biosamples.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "biosamples.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/k8s/helm_test/templates/deployment.yaml b/k8s/helm_test/templates/deployment.yaml new file mode 100644 index 000000000..c69ee9179 --- /dev/null +++ b/k8s/helm_test/templates/deployment.yaml @@ -0,0 +1,78 @@ +{{- range $serviceName, $config := .Values.services }} +{{- if $config.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "biosamples.fullname" $ }}-{{ $serviceName }} + labels: + {{- include "biosamples.labels" $ | nindent 4 }} + app.kubernetes.io/component: {{ $serviceName }} +spec: + replicas: {{ $config.replicaCount | default 1 }} + selector: + matchLabels: + {{- include "biosamples.selectorLabels" $ | nindent 6 }} + app.kubernetes.io/component: {{ $serviceName }} + template: + metadata: + annotations: + {{- with $.Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "biosamples.selectorLabels" $ | nindent 8 }} + app.kubernetes.io/component: {{ $serviceName }} + spec: + {{- with $.Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "biosamples.serviceAccountName" $ }} + securityContext: + {{- toYaml $.Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ $serviceName }} + securityContext: + {{- toYaml $.Values.securityContext | nindent 12 }} + image: "{{ $config.image.repository | default $.Values.global.imageRegistry }}/{{ $serviceName }}:{{ $config.image.tag | default $.Chart.AppVersion }}" + imagePullPolicy: {{ $config.image.pullPolicy | default "IfNotPresent" }} + {{- if $config.service.enabled | default true }} + ports: + - name: http + containerPort: {{ $config.service.targetPort | default 8080 }} + protocol: TCP + livenessProbe: + httpGet: + path: /actuator/health + port: http + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /actuator/health + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + {{- end }} + resources: + {{- toYaml $config.resources | nindent 12 }} + env: + - name: SPRING_PROFILES_ACTIVE + value: "kubernetes" + - name: SERVICE_NAME + value: {{ $serviceName }} + {{- with $.Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/k8s/helm_test/templates/hpa.yaml b/k8s/helm_test/templates/hpa.yaml new file mode 100644 index 000000000..28c087ea4 --- /dev/null +++ b/k8s/helm_test/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "helm.fullname" . }} + labels: + {{- include "helm.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "helm.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/k8s/helm_test/templates/ingress.yaml b/k8s/helm_test/templates/ingress.yaml new file mode 100644 index 000000000..5113252d9 --- /dev/null +++ b/k8s/helm_test/templates/ingress.yaml @@ -0,0 +1,47 @@ +{{- range $serviceName, $config := .Values.services }} +{{- if and $config.enabled $config.ingress.enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "biosamples.fullname" $ }}-{{ $serviceName }} + labels: + {{- include "biosamples.labels" $ | nindent 4 }} + app.kubernetes.io/component: {{ $serviceName }} + {{- with $config.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if $config.ingress.className }} + ingressClassName: {{ $config.ingress.className }} + {{- end }} + {{- if $config.ingress.tls }} + tls: + {{- range $config.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range $config.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if .pathType }} + pathType: {{ .pathType }} + {{- end }} + backend: + service: + name: {{ include "biosamples.fullname" $ }}-{{ $serviceName }} + port: + number: {{ $config.service.port | default 8080 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/k8s/helm_test/templates/job.yaml b/k8s/helm_test/templates/job.yaml new file mode 100644 index 000000000..77903eaf7 --- /dev/null +++ b/k8s/helm_test/templates/job.yaml @@ -0,0 +1,67 @@ +{{- range $jobName, $config := .Values.jobs }} +{{- if $config.enabled }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "biosamples.fullname" $ }}-{{ $jobName }}-{{ now | date "20060102-150405" }} + labels: + {{- include "biosamples.labels" $ | nindent 4 }} + app.kubernetes.io/component: {{ $jobName }} + job-type: {{ $jobName }} + annotations: + deployment-timestamp: {{ now | quote }} +spec: + backoffLimit: {{ $config.backoffLimit | default 3 }} + {{- if $config.ttlSecondsAfterFinished }} + ttlSecondsAfterFinished: {{ $config.ttlSecondsAfterFinished }} + {{- end }} + template: + metadata: + annotations: + {{- with $.Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "biosamples.selectorLabels" $ | nindent 8 }} + app.kubernetes.io/component: {{ $jobName }} + job-type: {{ $jobName }} + spec: + {{- with $.Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "biosamples.serviceAccountName" $ }} + securityContext: + {{- toYaml $.Values.podSecurityContext | nindent 8 }} + restartPolicy: {{ $config.restartPolicy | default "Never" }} + containers: + - name: {{ $jobName }} + securityContext: + {{- toYaml $.Values.securityContext | nindent 12 }} + image: "{{ $config.image.repository | default $.Values.global.imageRegistry }}/{{ $jobName }}:{{ $config.image.tag | default $.Chart.AppVersion }}" + imagePullPolicy: {{ $config.image.pullPolicy | default "IfNotPresent" }} + resources: + {{- toYaml $config.resources | nindent 12 }} + env: + - name: SPRING_PROFILES_ACTIVE + value: "kubernetes" + - name: JOB_NAME + value: {{ $jobName }} + {{- with $config.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with $.Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/k8s/helm_test/templates/service.yaml b/k8s/helm_test/templates/service.yaml new file mode 100644 index 000000000..f105b5f5d --- /dev/null +++ b/k8s/helm_test/templates/service.yaml @@ -0,0 +1,22 @@ +{{- range $serviceName, $config := .Values.services }} +{{- if and $config.enabled ($config.service.enabled | default true) }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "biosamples.fullname" $ }}-{{ $serviceName }} + labels: + {{- include "biosamples.labels" $ | nindent 4 }} + app.kubernetes.io/component: {{ $serviceName }} +spec: + type: {{ $config.service.type | default "ClusterIP" }} + ports: + - port: {{ $config.service.port | default 8080 }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "biosamples.selectorLabels" $ | nindent 4 }} + app.kubernetes.io/component: {{ $serviceName }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/k8s/helm_test/templates/serviceaccount.yaml b/k8s/helm_test/templates/serviceaccount.yaml new file mode 100644 index 000000000..d47046559 --- /dev/null +++ b/k8s/helm_test/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "helm.serviceAccountName" . }} + labels: + {{- include "helm.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/k8s/helm_test/templates/tests/test-connection.yaml b/k8s/helm_test/templates/tests/test-connection.yaml new file mode 100644 index 000000000..bf1c65f24 --- /dev/null +++ b/k8s/helm_test/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "helm.fullname" . }}-test-connection" + labels: + {{- include "helm.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "helm.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/k8s/helm_test/values-dev.yaml b/k8s/helm_test/values-dev.yaml new file mode 100644 index 000000000..7adf22402 --- /dev/null +++ b/k8s/helm_test/values-dev.yaml @@ -0,0 +1,8 @@ +rabbitmq: + host: wp-np2-40.ebi.ac.uk +biosamples: + schemaValidator: http://localhost:3020/validate + schemaStore: http://localhost:8085 + search: + host: biosamples-search-helm + port: 9090 \ No newline at end of file diff --git a/k8s/helm_test/values-prod.yaml b/k8s/helm_test/values-prod.yaml new file mode 100644 index 000000000..cb9cb2bc8 --- /dev/null +++ b/k8s/helm_test/values-prod.yaml @@ -0,0 +1,56 @@ +rabbitmq: + host: wp-np2-40.ebi.ac.uk +biosamples: + schemaValidator: http://localhost:3020/validate + schemaStore: http://localhost:8085 + search: + host: biosamples-search-helm + port: 9090 + + +global: + imageRegistry: "registry.gitlab.com/your-group/your-project" + imagePullSecrets: + - name: docker-registry-secret + +services: + webapp-core: + replicaCount: 1 + resources: + limits: + cpu: 2000m + memory: 2Gi + requests: + cpu: 1000m + memory: 1Gi + ingress: + hosts: + - host: biosamples-core.example.com + paths: + - path: / + pathType: Prefix + + webapp-core-v2: + replicaCount: 1 + resources: + limits: + cpu: 2000m + memory: 2Gi + requests: + cpu: 1000m + memory: 1Gi + ingress: + hosts: + - host: biosamples-core-v2.example.com + paths: + - path: / + pathType: Prefix + + agents-uploadworkers: + replicaCount: 1 + + pipelines-reindex: + replicaCount: 1 + + pipelines-curation: + replicaCount: 1 \ No newline at end of file diff --git a/k8s/helm_test/values.yaml b/k8s/helm_test/values.yaml new file mode 100644 index 000000000..afb576a03 --- /dev/null +++ b/k8s/helm_test/values.yaml @@ -0,0 +1,139 @@ +global: + imageRegistry: "" + imagePullSecrets: [] + storageClass: "" + +# Long-running services (Deployments) +services: + webapp-core: + enabled: true + replicaCount: 2 + image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 8080 + targetPort: 8080 +# ingress: +# enabled: true +# className: "nginx" +# hosts: +# - host: biosamples-core.example.com +# paths: +# - path: / +# pathType: Prefix + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 500m + memory: 512Mi + + webapp-core-v2: + enabled: true + replicaCount: 2 + image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 8080 + targetPort: 8080 +# ingress: +# enabled: true +# className: "nginx" +# hosts: +# - host: biosamples-core-v2.example.com +# paths: +# - path: / +# pathType: Prefix + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 500m + memory: 512Mi + +# agents-uploadworkers: +# enabled: true +# replicaCount: 1 +# image: +# repository: "" +# tag: "" +# pullPolicy: IfNotPresent +# service: +# enabled: false +# resources: +# limits: +# cpu: 500m +# memory: 512Mi +# requests: +# cpu: 250m +# memory: 256Mi + +# Batch Jobs (run-to-completion) +jobs: + pipelines-reindex: + enabled: false # Only run when explicitly enabled + image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + backoffLimit: 3 + restartPolicy: Never + ttlSecondsAfterFinished: 3600 # Clean up job after 1 hour + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 500m + memory: 512Mi + env: + - name: JOB_TYPE + value: "reindex" + + pipelines-curation: + enabled: false # Only run when explicitly enabled + image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + backoffLimit: 3 + restartPolicy: Never + ttlSecondsAfterFinished: 3600 # Clean up job after 1 hour + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 500m + memory: 512Mi + env: + - name: JOB_TYPE + value: "curation" + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} + +podSecurityContext: {} + +securityContext: {} + +nodeSelector: {} + +tolerations: [] + +affinity: {} From 21a00e7813ea6c4d8c7cf9850dd6af5447de633e Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 12 Sep 2025 11:22:12 +0100 Subject: [PATCH 21/86] test: gitlab --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fe8b8af64..818d7a2ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -253,5 +253,5 @@ deploy_k8s_hh_prod: helm upgrade --install $APP_NAME ./k8s/helm \ --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ - --set image.repository=$CI_REGISTRY_IMAGE \ + --set image.repository=$CI_REGISTRY_IMAGE/webapp-core \ --set image.tag=$DOCKER_TAG From 7e1eb9bf9212357056b8b841549488045f7f8caa Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 12 Sep 2025 14:46:27 +0100 Subject: [PATCH 22/86] test: gitlab --- .gitlab-ci.yml | 41 ++++++++++++++++++++++ k8s/jobs/Chart.yaml | 6 ++++ k8s/jobs/templates/_helpers.tpl | 62 +++++++++++++++++++++++++++++++++ k8s/jobs/templates/job.yaml | 29 +++++++++++++++ k8s/jobs/values-prod.yaml | 8 +++++ k8s/jobs/values.yaml | 22 ++++++++++++ 6 files changed, 168 insertions(+) create mode 100644 k8s/jobs/Chart.yaml create mode 100644 k8s/jobs/templates/_helpers.tpl create mode 100644 k8s/jobs/templates/job.yaml create mode 100644 k8s/jobs/values-prod.yaml create mode 100644 k8s/jobs/values.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 818d7a2ca..d3bb0ca4d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -231,6 +231,41 @@ deploy_k8s_hh_prod: when: manual extends: .kube_deploy_script +deploy_k8s_hh_pipeline_prod: + variables: + ENVIRONMENT_NAME: prod + K8S_NAMESPACE: ${K8S_NAMESPACE_PREFIX}-${ENVIRONMENT_NAME} + environment: + name: primary_prod + url: https://wwwdev.ebi.ac.uk/biosamples + only: + - dev + - main + - biosamples-search + when: manual + extends: .kube_deploy_script + stage: deploy + image: dtzar/helm-kubectl:3.16 + tags: [ "dind" ] + services: + - docker:27-dind + script: + - echo $K8S_NAMESPACE + - kubectl config view + - kubectl config set-context --current --namespace=${K8S_NAMESPACE} + - kubectl delete secret $DOCKER_PULL_SECRET || true + - | + kubectl create secret docker-registry $DOCKER_PULL_SECRET \ + --docker-server=$CI_REGISTRY \ + --docker-username=$CI_REGISTRY_USER \ + --docker-password=$CI_REGISTRY_PASSWORD + - | + helm upgrade --install biosamples-reindex-pipeline ./k8s/jobs \ + --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ + --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ + --set image.repository=$CI_REGISTRY_IMAGE/pipelines-reindex \ + --set image.tag=$DOCKER_TAG + .kube_deploy_script: stage: deploy image: dtzar/helm-kubectl:3.16 @@ -255,3 +290,9 @@ deploy_k8s_hh_prod: --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE/webapp-core \ --set image.tag=$DOCKER_TAG + - | + helm upgrade --install $APP_NAME ./k8s/helm \ + --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ + --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ + --set image.repository=$CI_REGISTRY_IMAGE/webapp-core-v2 \ + --set image.tag=$DOCKER_TAG diff --git a/k8s/jobs/Chart.yaml b/k8s/jobs/Chart.yaml new file mode 100644 index 000000000..70a2a2a38 --- /dev/null +++ b/k8s/jobs/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: biosamples-jobs +description: A Helm chart for Kubernetes +type: application +version: 0.1.0 +appVersion: "1.16.0" diff --git a/k8s/jobs/templates/_helpers.tpl b/k8s/jobs/templates/_helpers.tpl new file mode 100644 index 000000000..ba04c300d --- /dev/null +++ b/k8s/jobs/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "helm.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "helm.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "helm.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "helm.labels" -}} +helm.sh/chart: {{ include "helm.chart" . }} +{{ include "helm.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "helm.selectorLabels" -}} +app.kubernetes.io/name: {{ include "helm.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "helm.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "helm.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/k8s/jobs/templates/job.yaml b/k8s/jobs/templates/job.yaml new file mode 100644 index 000000000..2deadb857 --- /dev/null +++ b/k8s/jobs/templates/job.yaml @@ -0,0 +1,29 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "helm.fullname" . }}-job + labels: + app: {{ include "helm.name" . }} + chart: {{ include "helm.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + backoffLimit: {{ .Values.backoffLimit }} + template: + metadata: + labels: + app: {{ include "helm.name" . }} + release: {{ .Release.Name }} + spec: + restartPolicy: {{ .Values.restartPolicy }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + {{- range .Values.env }} + - name: {{ .name }} + value: "{{ .value }}" + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} diff --git a/k8s/jobs/values-prod.yaml b/k8s/jobs/values-prod.yaml new file mode 100644 index 000000000..cb9c8ff02 --- /dev/null +++ b/k8s/jobs/values-prod.yaml @@ -0,0 +1,8 @@ +rabbitmq: + host: wp-np2-40.ebi.ac.uk +biosamples: + schemaValidator: http://localhost:3020/validate + schemaStore: http://localhost:8085 + search: + host: biosamples-search-helm + port: 9090 diff --git a/k8s/jobs/values.yaml b/k8s/jobs/values.yaml new file mode 100644 index 000000000..e02efa6fd --- /dev/null +++ b/k8s/jobs/values.yaml @@ -0,0 +1,22 @@ +image: + repository: biosamples-v4/pipelines-reindex + tag: "1.0.0" + pullPolicy: IfNotPresent + +resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "1" + memory: "2Gi" + +env: + - name: SPRING_PROFILES_ACTIVE + value: "batch" + - name: JAVA_OPTS + value: "-Xms512m -Xmx1g" + +restartPolicy: Never + +backoffLimit: 3 \ No newline at end of file From fa640743c2e0ec26be6ad84a13e225d871a49124 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 16 Sep 2025 10:14:46 +0100 Subject: [PATCH 23/86] test: gitlab --- .gitlab-ci.yml | 4 ++-- k8s/Dockerfile | 4 ++-- k8s/jobs/Chart.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d3bb0ca4d..584bedda9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -260,7 +260,7 @@ deploy_k8s_hh_pipeline_prod: --docker-username=$CI_REGISTRY_USER \ --docker-password=$CI_REGISTRY_PASSWORD - | - helm upgrade --install biosamples-reindex-pipeline ./k8s/jobs \ + helm upgrade --install biosamples-reindex ./k8s/jobs \ --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE/pipelines-reindex \ @@ -291,7 +291,7 @@ deploy_k8s_hh_pipeline_prod: --set image.repository=$CI_REGISTRY_IMAGE/webapp-core \ --set image.tag=$DOCKER_TAG - | - helm upgrade --install $APP_NAME ./k8s/helm \ + helm upgrade --install ${APP_NAME}-v2 ./k8s/helm \ --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE/webapp-core-v2 \ diff --git a/k8s/Dockerfile b/k8s/Dockerfile index f83adc8b5..15c028a88 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -7,9 +7,9 @@ EXPOSE 8080 CMD ["java", "-jar", "app.war"] FROM base as webapp-core-v2 -COPY webapps/core-v2/target/webapps-core-*.war app.war +COPY webapps/core-v2/target/webapps-core-*.jar app.jar EXPOSE 8080 -CMD ["java", "-jar", "app.war"] +CMD ["java", "-jar", "app.jar"] FROM base as pipelines-reindex COPY pipelines/reindex/target/pipelines-reindex-*.jar app.jar diff --git a/k8s/jobs/Chart.yaml b/k8s/jobs/Chart.yaml index 70a2a2a38..1d20fc91b 100644 --- a/k8s/jobs/Chart.yaml +++ b/k8s/jobs/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: biosamples-jobs +name: pipelines description: A Helm chart for Kubernetes type: application version: 0.1.0 From f8eb51bc022f6c5c241577e35e2bb59d937f4480 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 16 Sep 2025 11:51:42 +0100 Subject: [PATCH 24/86] test: gitlab --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 584bedda9..fdf1d9d48 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -243,7 +243,6 @@ deploy_k8s_hh_pipeline_prod: - main - biosamples-search when: manual - extends: .kube_deploy_script stage: deploy image: dtzar/helm-kubectl:3.16 tags: [ "dind" ] @@ -261,7 +260,7 @@ deploy_k8s_hh_pipeline_prod: --docker-password=$CI_REGISTRY_PASSWORD - | helm upgrade --install biosamples-reindex ./k8s/jobs \ - --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ + --values ./k8s/jobs/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE/pipelines-reindex \ --set image.tag=$DOCKER_TAG From 50e9b470a86ec10b04ee1841f3c0c4c22848c254 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 16 Sep 2025 12:47:49 +0100 Subject: [PATCH 25/86] test: gitlab --- .gitlab-ci.yml | 4 ++++ k8s/jobs/templates/job.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fdf1d9d48..b19207a95 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -258,6 +258,10 @@ deploy_k8s_hh_pipeline_prod: --docker-server=$CI_REGISTRY \ --docker-username=$CI_REGISTRY_USER \ --docker-password=$CI_REGISTRY_PASSWORD + - echo "CI_REGISTRY=$CI_REGISTRY" + - echo "CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE" + - echo "DOCKER_TAG=$DOCKER_TAG" + - docker login $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD - | helm upgrade --install biosamples-reindex ./k8s/jobs \ --values ./k8s/jobs/values-${ENVIRONMENT_NAME}.yaml \ diff --git a/k8s/jobs/templates/job.yaml b/k8s/jobs/templates/job.yaml index 2deadb857..59c32fa6c 100644 --- a/k8s/jobs/templates/job.yaml +++ b/k8s/jobs/templates/job.yaml @@ -15,6 +15,10 @@ spec: app: {{ include "helm.name" . }} release: {{ .Release.Name }} spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} restartPolicy: {{ .Values.restartPolicy }} containers: - name: {{ .Chart.Name }} From 528d56318e7427cdaf39cdbc2cbcee1160fa2031 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 16 Sep 2025 13:15:12 +0100 Subject: [PATCH 26/86] test: gitlab --- .gitlab-ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b19207a95..fdf1d9d48 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -258,10 +258,6 @@ deploy_k8s_hh_pipeline_prod: --docker-server=$CI_REGISTRY \ --docker-username=$CI_REGISTRY_USER \ --docker-password=$CI_REGISTRY_PASSWORD - - echo "CI_REGISTRY=$CI_REGISTRY" - - echo "CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE" - - echo "DOCKER_TAG=$DOCKER_TAG" - - docker login $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD - | helm upgrade --install biosamples-reindex ./k8s/jobs \ --values ./k8s/jobs/values-${ENVIRONMENT_NAME}.yaml \ From 004c4db668b1bd25b12839baebc4083a2c54367b Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Wed, 17 Sep 2025 11:28:57 +0100 Subject: [PATCH 27/86] fix: spring dependency resolution problem with re-index pipeline --- pipelines/pom.xml | 2 +- pipelines/reindex/pom.xml | 15 +++ .../uk/ac/ebi/biosamples/Application.java | 103 +++++++++++++++++- .../uk/ac/ebi/biosamples/Application.java | 2 +- 4 files changed, 117 insertions(+), 5 deletions(-) diff --git a/pipelines/pom.xml b/pipelines/pom.xml index ea3e9e1a8..75efe2a6e 100644 --- a/pipelines/pom.xml +++ b/pipelines/pom.xml @@ -18,7 +18,7 @@ embl-ebi - ncbi-ena-link + diff --git a/pipelines/reindex/pom.xml b/pipelines/reindex/pom.xml index e282a5d64..517e948ff 100644 --- a/pipelines/reindex/pom.xml +++ b/pipelines/reindex/pom.xml @@ -13,6 +13,21 @@ + + + org.springframework.hateoas + spring-hateoas + 1.3.4 + + + org.springframework + spring-webmvc + + + org.springframework.boot + spring-boot-starter-tomcat + + uk.ac.ebi.biosamples core diff --git a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java index 9a62af2f1..8d234a2f3 100644 --- a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -10,25 +10,50 @@ */ package uk.ac.ebi.biosamples; +import org.apache.http.HeaderElement; +import org.apache.http.HeaderElementIterator; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.conn.ConnectionKeepAliveStrategy; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.impl.client.cache.CacheConfig; +import org.apache.http.impl.client.cache.CachingHttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicHeaderElementIterator; +import org.apache.http.protocol.HTTP; +import org.apache.http.protocol.HttpContext; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.web.client.RestTemplateCustomizer; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.configuration.ExclusionConfiguration; +import uk.ac.ebi.biosamples.security.service.BioSamplesWebSecurityConfig; import uk.ac.ebi.biosamples.service.EnaConfig; import uk.ac.ebi.biosamples.service.EnaSampleToBioSampleConversionService; import uk.ac.ebi.biosamples.service.EraProDao; +import uk.ac.ebi.biosamples.service.validation.ElixirSchemaValidator; +import uk.ac.ebi.biosamples.service.validation.SchemaValidationService; import uk.ac.ebi.biosamples.utils.PipelineUtils; +import uk.ac.ebi.biosamples.utils.ols.OlsProcessor; @SpringBootApplication(exclude = DataSourceAutoConfiguration.class) @ComponentScan( excludeFilters = { - @ComponentScan.Filter( - type = FilterType.ASSIGNABLE_TYPE, - value = {EnaConfig.class, EraProDao.class, EnaSampleToBioSampleConversionService.class}) + @ComponentScan.Filter( + type = FilterType.ASSIGNABLE_TYPE, + value = {EnaConfig.class, EraProDao.class, EnaSampleToBioSampleConversionService.class, + BioSamplesWebSecurityConfig.class + }), + @ComponentScan.Filter(type = FilterType.REGEX, pattern = "uk\\.ac\\.ebi\\.biosamples\\.service\\.validation\\..*") }) @Import(ExclusionConfiguration.class) public class Application { @@ -37,4 +62,76 @@ public static void main(final String[] args) { final ConfigurableApplicationContext ctx = SpringApplication.run(Application.class, args); PipelineUtils.exitPipeline(ctx); } + + + // todo I Had to add restTemplate bean as a temporary workaround as there seems to be a problem with dependencies after refactor. + // We need to sort out dependency problem and remove this unused dependency. + + @Bean + public RestTemplate restTemplate(final RestTemplateCustomizer restTemplateCustomizer) { + final RestTemplate restTemplate = new RestTemplate(); + restTemplateCustomizer.customize(restTemplate); + return restTemplate; + } + + @Bean + public RestTemplateCustomizer restTemplateCustomizer( + final BioSamplesProperties bioSamplesProperties, + final PipelinesProperties pipelinesProperties) { + return restTemplate -> { + + final ConnectionKeepAliveStrategy keepAliveStrategy = + (response, context) -> { + + final HeaderElementIterator it = + new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); + while (it.hasNext()) { + final HeaderElement he = it.nextElement(); + final String param = he.getName(); + final String value = he.getValue(); + if (value != null && param.equalsIgnoreCase("timeout")) { + return Long.parseLong(value) * 1000; + } + } + + return 60 * 1000; + }; + + final PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = + new PoolingHttpClientConnectionManager(); + poolingHttpClientConnectionManager.setMaxTotal(pipelinesProperties.getConnectionCountMax()); + poolingHttpClientConnectionManager.setDefaultMaxPerRoute( + pipelinesProperties.getConnectionCountDefault()); + poolingHttpClientConnectionManager.setMaxPerRoute( + new HttpRoute(HttpHost.create(pipelinesProperties.getZooma())), + pipelinesProperties.getConnectionCountZooma()); + poolingHttpClientConnectionManager.setMaxPerRoute( + new HttpRoute(HttpHost.create(bioSamplesProperties.getOls())), + pipelinesProperties.getConnectionCountOls()); + + final CacheConfig cacheConfig = + CacheConfig.custom() + .setMaxCacheEntries(1024) + .setMaxObjectSize(1024 * 1024) + .setSharedCache(false) + .build(); + final int timeout = 60; + final RequestConfig config = + RequestConfig.custom() + .setConnectTimeout(timeout * 1000) + .setConnectionRequestTimeout( + timeout * 1000) + .setSocketTimeout(timeout * 1000) + .build(); + final HttpClient httpClient = + CachingHttpClientBuilder.create() + .setCacheConfig(cacheConfig) + .useSystemProperties() + .setConnectionManager(poolingHttpClientConnectionManager) + .setKeepAliveStrategy(keepAliveStrategy) + .setDefaultRequestConfig(config) + .build(); + restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory(httpClient)); + }; + } } diff --git a/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/Application.java b/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/Application.java index ddcbe9253..501400602 100644 --- a/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/webapps/core-v2/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -101,7 +101,7 @@ public RestTemplate restTemplate() { return new RestTemplate(); } - @Value("${spring.cloud.gcp.project-id}") + @Value("${spring.cloud.gcp.project-id:no_project}") private String enaGcpProject; @Autowired private Environment environment; From 9b5bd0af81478422526f4a17f4f9410ffbe468be Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Wed, 17 Sep 2025 12:09:46 +0100 Subject: [PATCH 28/86] test: gitlab --- k8s/jobs/templates/job.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/k8s/jobs/templates/job.yaml b/k8s/jobs/templates/job.yaml index 59c32fa6c..e4bd82e65 100644 --- a/k8s/jobs/templates/job.yaml +++ b/k8s/jobs/templates/job.yaml @@ -25,6 +25,23 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: + - name: SPRING_DATA_MONGODB_URI + valueFrom: + secretKeyRef: + name: biosamples-mongodb + key: connection-string + - name: SPRING_RABBITMQ_HOST + value: {{ .Values.rabbitmq.host }} + - name: SPRING_RABBITMQ_USERNAME + valueFrom: + secretKeyRef: + name: rabbitmq-user + key: username + - name: SPRING_RABBITMQ_PASSWORD + valueFrom: + secretKeyRef: + name: rabbitmq-user + key: password {{- range .Values.env }} - name: {{ .name }} value: "{{ .value }}" From 6e49c7308e50efd9ba327f9bf2bb1c30537f0545 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 29 Sep 2025 12:32:43 +0100 Subject: [PATCH 29/86] feat: k8s rabbit config --- .gitignore | 3 + .gitlab-ci.yml | 90 +++++++++--- .../messaging/MessagingConstants.java | 6 +- .../messaging/config/MessageConfig.java | 9 +- ...es-prod.yaml => values-fallback_prod.yaml} | 2 +- ...alues-dev.yaml => values-primary_dev.yaml} | 0 .../values-primary_prod.yaml} | 2 +- k8s/helm_test/.helmignore | 23 --- k8s/helm_test/Chart.yaml | 24 --- k8s/helm_test/templates/NOTES.txt | 22 --- k8s/helm_test/templates/_helpers.tpl | 60 -------- k8s/helm_test/templates/deployment.yaml | 78 ---------- k8s/helm_test/templates/hpa.yaml | 32 ---- k8s/helm_test/templates/ingress.yaml | 47 ------ k8s/helm_test/templates/job.yaml | 67 --------- k8s/helm_test/templates/service.yaml | 22 --- k8s/helm_test/templates/serviceaccount.yaml | 13 -- .../templates/tests/test-connection.yaml | 15 -- k8s/helm_test/values-dev.yaml | 8 - k8s/helm_test/values-prod.yaml | 56 ------- k8s/helm_test/values.yaml | 139 ------------------ k8s/jobs/values-fallback_prod.yaml | 2 + k8s/jobs/values-primary_dev.yaml | 2 + k8s/jobs/values-primary_prod.yaml | 2 + 24 files changed, 81 insertions(+), 643 deletions(-) rename k8s/helm/{values-prod.yaml => values-fallback_prod.yaml} (85%) rename k8s/helm/{values-dev.yaml => values-primary_dev.yaml} (100%) rename k8s/{jobs/values-prod.yaml => helm/values-primary_prod.yaml} (85%) delete mode 100644 k8s/helm_test/.helmignore delete mode 100644 k8s/helm_test/Chart.yaml delete mode 100644 k8s/helm_test/templates/NOTES.txt delete mode 100644 k8s/helm_test/templates/_helpers.tpl delete mode 100644 k8s/helm_test/templates/deployment.yaml delete mode 100644 k8s/helm_test/templates/hpa.yaml delete mode 100644 k8s/helm_test/templates/ingress.yaml delete mode 100644 k8s/helm_test/templates/job.yaml delete mode 100644 k8s/helm_test/templates/service.yaml delete mode 100644 k8s/helm_test/templates/serviceaccount.yaml delete mode 100644 k8s/helm_test/templates/tests/test-connection.yaml delete mode 100644 k8s/helm_test/values-dev.yaml delete mode 100644 k8s/helm_test/values-prod.yaml delete mode 100644 k8s/helm_test/values.yaml create mode 100644 k8s/jobs/values-fallback_prod.yaml create mode 100644 k8s/jobs/values-primary_dev.yaml create mode 100644 k8s/jobs/values-primary_prod.yaml diff --git a/.gitignore b/.gitignore index 6a9b2df8e..246c941fc 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ docker-compose.debug.override.yml bin/ target/ export/ +temp/ +.env +set_java_17.sh # Scripts docker-env.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fdf1d9d48..46598417b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -218,9 +218,24 @@ deploy-production-solr: url: http://${BSD_SOLR_HOST_NAME}.ebi.ac.uk:8983/solr -deploy_k8s_hh_prod: +deploy_k8s_primary_dev: variables: - ENVIRONMENT_NAME: prod + ENVIRONMENT_NAME: primary_dev + K8S_NAMESPACE: biosamples-dev + environment: + name: primary_dev + url: https://wwwdev.ebi.ac.uk/biosamples + only: + - dev + - main + - biosamples-search + when: manual + extends: .kube_deploy_script + +deploy_k8s_primary_prod: + variables: + ENVIRONMENT_NAME: primary_prod + K8S_NAMESPACE: biosamples-prod environment: name: primary_prod url: https://wwwdev.ebi.ac.uk/biosamples @@ -231,10 +246,24 @@ deploy_k8s_hh_prod: when: manual extends: .kube_deploy_script -deploy_k8s_hh_pipeline_prod: +deploy_k8s_fallback_prod: variables: - ENVIRONMENT_NAME: prod - K8S_NAMESPACE: ${K8S_NAMESPACE_PREFIX}-${ENVIRONMENT_NAME} + ENVIRONMENT_NAME: fallback_prod + K8S_NAMESPACE: biosamples-prod + environment: + name: fallback_prod + url: https://wwwdev.ebi.ac.uk/biosamples + only: + - dev + - main + - biosamples-search + when: manual + extends: .kube_deploy_script + +deploy_pipeline_k8s_primary_prod: + variables: + ENVIRONMENT_NAME: primary_prod + K8S_NAMESPACE: biosamples-prod environment: name: primary_prod url: https://wwwdev.ebi.ac.uk/biosamples @@ -243,14 +272,30 @@ deploy_k8s_hh_pipeline_prod: - main - biosamples-search when: manual + extends: .kube_deploy_pipeline_script + +deploy_pipeline_k8s_primary_dev: + variables: + ENVIRONMENT_NAME: primary_dev + K8S_NAMESPACE: biosamples-dev + environment: + name: primary_dev + url: https://wwwdev.ebi.ac.uk/biosamples + only: + - dev + - main + - biosamples-search + when: manual + extends: .kube_deploy_pipeline_script + +.kube_deploy_script: stage: deploy image: dtzar/helm-kubectl:3.16 - tags: [ "dind" ] + tags: ["dind"] services: - docker:27-dind script: - echo $K8S_NAMESPACE - - kubectl config view - kubectl config set-context --current --namespace=${K8S_NAMESPACE} - kubectl delete secret $DOCKER_PULL_SECRET || true - | @@ -259,23 +304,26 @@ deploy_k8s_hh_pipeline_prod: --docker-username=$CI_REGISTRY_USER \ --docker-password=$CI_REGISTRY_PASSWORD - | - helm upgrade --install biosamples-reindex ./k8s/jobs \ - --values ./k8s/jobs/values-${ENVIRONMENT_NAME}.yaml \ + helm upgrade --install $APP_NAME ./k8s/helm \ + --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ - --set image.repository=$CI_REGISTRY_IMAGE/pipelines-reindex \ + --set image.repository=$CI_REGISTRY_IMAGE/webapp-core \ + --set image.tag=$DOCKER_TAG + - | + helm upgrade --install ${APP_NAME}-v2 ./k8s/helm \ + --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ + --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ + --set image.repository=$CI_REGISTRY_IMAGE/webapp-core-v2 \ --set image.tag=$DOCKER_TAG -.kube_deploy_script: +.kube_deploy_pipeline_script: stage: deploy image: dtzar/helm-kubectl:3.16 - tags: ["dind"] + tags: [ "dind" ] services: - docker:27-dind - variables: - K8S_NAMESPACE: ${K8S_NAMESPACE_PREFIX}-${ENVIRONMENT_NAME} script: - echo $K8S_NAMESPACE - - kubectl config view - kubectl config set-context --current --namespace=${K8S_NAMESPACE} - kubectl delete secret $DOCKER_PULL_SECRET || true - | @@ -284,14 +332,8 @@ deploy_k8s_hh_pipeline_prod: --docker-username=$CI_REGISTRY_USER \ --docker-password=$CI_REGISTRY_PASSWORD - | - helm upgrade --install $APP_NAME ./k8s/helm \ - --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ - --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ - --set image.repository=$CI_REGISTRY_IMAGE/webapp-core \ - --set image.tag=$DOCKER_TAG - - | - helm upgrade --install ${APP_NAME}-v2 ./k8s/helm \ - --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ + helm upgrade --install biosamples-reindex ./k8s/jobs \ + --values ./k8s/jobs/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ - --set image.repository=$CI_REGISTRY_IMAGE/webapp-core-v2 \ + --set image.repository=$CI_REGISTRY_IMAGE/pipelines-reindex \ --set image.tag=$DOCKER_TAG diff --git a/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java b/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java index 51ee1ad78..abae07ab0 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/messaging/MessagingConstants.java @@ -13,8 +13,8 @@ public class MessagingConstants { public static final String INDEXING_EXCHANGE = "biosamples.indexing"; public static final String INDEXING_QUEUE = "biosamples.indexing.es"; - public static final String REINDEXING_EXCHANGE = "biosamples.reindex.solr"; - public static final String REINDEXING_QUEUE = "biosamples.reindex.solr"; - public static final String UPLOAD_QUEUE = "biosamples.uploaded.files"; + public static final String REINDEXING_QUEUE = "biosamples.reindexing.es"; + public static final String UPLOAD_EXCHANGE = "biosamples.uploaded.files.exchange"; + public static final String UPLOAD_QUEUE = "biosamples.uploaded.files"; } diff --git a/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java b/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java index 43ddb6cae..5780e161b 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java @@ -45,13 +45,6 @@ public Exchange indexingExchange() { .build(); } - @Bean(name = "reindexingExchange") - public Exchange reindexingExchange() { - return ExchangeBuilder.directExchange(MessagingConstants.REINDEXING_EXCHANGE) - .durable(true) - .build(); - } - @Bean(name = "uploadExchange") public Exchange uploadExchange() { return ExchangeBuilder.fanoutExchange(MessagingConstants.UPLOAD_EXCHANGE).durable(true).build(); @@ -69,7 +62,7 @@ public Binding indexBinding() { @Bean(name = "reindexingBinding") public Binding reindexBinding() { return BindingBuilder.bind(reindexingQueue()) - .to(reindexingExchange()) + .to(indexingExchange()) .with(MessagingConstants.REINDEXING_QUEUE) .noargs(); } diff --git a/k8s/helm/values-prod.yaml b/k8s/helm/values-fallback_prod.yaml similarity index 85% rename from k8s/helm/values-prod.yaml rename to k8s/helm/values-fallback_prod.yaml index cb9c8ff02..d8a55a374 100644 --- a/k8s/helm/values-prod.yaml +++ b/k8s/helm/values-fallback_prod.yaml @@ -1,5 +1,5 @@ rabbitmq: - host: wp-np2-40.ebi.ac.uk + host: wp-p2m-42.ebi.ac.uk biosamples: schemaValidator: http://localhost:3020/validate schemaStore: http://localhost:8085 diff --git a/k8s/helm/values-dev.yaml b/k8s/helm/values-primary_dev.yaml similarity index 100% rename from k8s/helm/values-dev.yaml rename to k8s/helm/values-primary_dev.yaml diff --git a/k8s/jobs/values-prod.yaml b/k8s/helm/values-primary_prod.yaml similarity index 85% rename from k8s/jobs/values-prod.yaml rename to k8s/helm/values-primary_prod.yaml index cb9c8ff02..d343639c5 100644 --- a/k8s/jobs/values-prod.yaml +++ b/k8s/helm/values-primary_prod.yaml @@ -1,5 +1,5 @@ rabbitmq: - host: wp-np2-40.ebi.ac.uk + host: wp-p1m-42.ebi.ac.uk biosamples: schemaValidator: http://localhost:3020/validate schemaStore: http://localhost:8085 diff --git a/k8s/helm_test/.helmignore b/k8s/helm_test/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/k8s/helm_test/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/k8s/helm_test/Chart.yaml b/k8s/helm_test/Chart.yaml deleted file mode 100644 index fc3899226..000000000 --- a/k8s/helm_test/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: helm -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" diff --git a/k8s/helm_test/templates/NOTES.txt b/k8s/helm_test/templates/NOTES.txt deleted file mode 100644 index 62f1b422e..000000000 --- a/k8s/helm_test/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "helm.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch its status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "helm.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "helm.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "helm.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/k8s/helm_test/templates/_helpers.tpl b/k8s/helm_test/templates/_helpers.tpl deleted file mode 100644 index 9ef944798..000000000 --- a/k8s/helm_test/templates/_helpers.tpl +++ /dev/null @@ -1,60 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "biosamples.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -*/}} -{{- define "biosamples.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "biosamples.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "biosamples.labels" -}} -helm.sh/chart: {{ include "biosamples.chart" . }} -{{ include "biosamples.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "biosamples.selectorLabels" -}} -app.kubernetes.io/name: {{ include "biosamples.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "biosamples.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "biosamples.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/k8s/helm_test/templates/deployment.yaml b/k8s/helm_test/templates/deployment.yaml deleted file mode 100644 index c69ee9179..000000000 --- a/k8s/helm_test/templates/deployment.yaml +++ /dev/null @@ -1,78 +0,0 @@ -{{- range $serviceName, $config := .Values.services }} -{{- if $config.enabled }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "biosamples.fullname" $ }}-{{ $serviceName }} - labels: - {{- include "biosamples.labels" $ | nindent 4 }} - app.kubernetes.io/component: {{ $serviceName }} -spec: - replicas: {{ $config.replicaCount | default 1 }} - selector: - matchLabels: - {{- include "biosamples.selectorLabels" $ | nindent 6 }} - app.kubernetes.io/component: {{ $serviceName }} - template: - metadata: - annotations: - {{- with $.Values.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "biosamples.selectorLabels" $ | nindent 8 }} - app.kubernetes.io/component: {{ $serviceName }} - spec: - {{- with $.Values.global.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "biosamples.serviceAccountName" $ }} - securityContext: - {{- toYaml $.Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ $serviceName }} - securityContext: - {{- toYaml $.Values.securityContext | nindent 12 }} - image: "{{ $config.image.repository | default $.Values.global.imageRegistry }}/{{ $serviceName }}:{{ $config.image.tag | default $.Chart.AppVersion }}" - imagePullPolicy: {{ $config.image.pullPolicy | default "IfNotPresent" }} - {{- if $config.service.enabled | default true }} - ports: - - name: http - containerPort: {{ $config.service.targetPort | default 8080 }} - protocol: TCP - livenessProbe: - httpGet: - path: /actuator/health - port: http - initialDelaySeconds: 30 - periodSeconds: 30 - readinessProbe: - httpGet: - path: /actuator/health - port: http - initialDelaySeconds: 5 - periodSeconds: 5 - {{- end }} - resources: - {{- toYaml $config.resources | nindent 12 }} - env: - - name: SPRING_PROFILES_ACTIVE - value: "kubernetes" - - name: SERVICE_NAME - value: {{ $serviceName }} - {{- with $.Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $.Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $.Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/k8s/helm_test/templates/hpa.yaml b/k8s/helm_test/templates/hpa.yaml deleted file mode 100644 index 28c087ea4..000000000 --- a/k8s/helm_test/templates/hpa.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "helm.fullname" . }} - labels: - {{- include "helm.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "helm.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/k8s/helm_test/templates/ingress.yaml b/k8s/helm_test/templates/ingress.yaml deleted file mode 100644 index 5113252d9..000000000 --- a/k8s/helm_test/templates/ingress.yaml +++ /dev/null @@ -1,47 +0,0 @@ -{{- range $serviceName, $config := .Values.services }} -{{- if and $config.enabled $config.ingress.enabled }} ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ include "biosamples.fullname" $ }}-{{ $serviceName }} - labels: - {{- include "biosamples.labels" $ | nindent 4 }} - app.kubernetes.io/component: {{ $serviceName }} - {{- with $config.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if $config.ingress.className }} - ingressClassName: {{ $config.ingress.className }} - {{- end }} - {{- if $config.ingress.tls }} - tls: - {{- range $config.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range $config.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if .pathType }} - pathType: {{ .pathType }} - {{- end }} - backend: - service: - name: {{ include "biosamples.fullname" $ }}-{{ $serviceName }} - port: - number: {{ $config.service.port | default 8080 }} - {{- end }} - {{- end }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/k8s/helm_test/templates/job.yaml b/k8s/helm_test/templates/job.yaml deleted file mode 100644 index 77903eaf7..000000000 --- a/k8s/helm_test/templates/job.yaml +++ /dev/null @@ -1,67 +0,0 @@ -{{- range $jobName, $config := .Values.jobs }} -{{- if $config.enabled }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "biosamples.fullname" $ }}-{{ $jobName }}-{{ now | date "20060102-150405" }} - labels: - {{- include "biosamples.labels" $ | nindent 4 }} - app.kubernetes.io/component: {{ $jobName }} - job-type: {{ $jobName }} - annotations: - deployment-timestamp: {{ now | quote }} -spec: - backoffLimit: {{ $config.backoffLimit | default 3 }} - {{- if $config.ttlSecondsAfterFinished }} - ttlSecondsAfterFinished: {{ $config.ttlSecondsAfterFinished }} - {{- end }} - template: - metadata: - annotations: - {{- with $.Values.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "biosamples.selectorLabels" $ | nindent 8 }} - app.kubernetes.io/component: {{ $jobName }} - job-type: {{ $jobName }} - spec: - {{- with $.Values.global.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "biosamples.serviceAccountName" $ }} - securityContext: - {{- toYaml $.Values.podSecurityContext | nindent 8 }} - restartPolicy: {{ $config.restartPolicy | default "Never" }} - containers: - - name: {{ $jobName }} - securityContext: - {{- toYaml $.Values.securityContext | nindent 12 }} - image: "{{ $config.image.repository | default $.Values.global.imageRegistry }}/{{ $jobName }}:{{ $config.image.tag | default $.Chart.AppVersion }}" - imagePullPolicy: {{ $config.image.pullPolicy | default "IfNotPresent" }} - resources: - {{- toYaml $config.resources | nindent 12 }} - env: - - name: SPRING_PROFILES_ACTIVE - value: "kubernetes" - - name: JOB_NAME - value: {{ $jobName }} - {{- with $config.env }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with $.Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $.Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $.Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/k8s/helm_test/templates/service.yaml b/k8s/helm_test/templates/service.yaml deleted file mode 100644 index f105b5f5d..000000000 --- a/k8s/helm_test/templates/service.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- range $serviceName, $config := .Values.services }} -{{- if and $config.enabled ($config.service.enabled | default true) }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "biosamples.fullname" $ }}-{{ $serviceName }} - labels: - {{- include "biosamples.labels" $ | nindent 4 }} - app.kubernetes.io/component: {{ $serviceName }} -spec: - type: {{ $config.service.type | default "ClusterIP" }} - ports: - - port: {{ $config.service.port | default 8080 }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "biosamples.selectorLabels" $ | nindent 4 }} - app.kubernetes.io/component: {{ $serviceName }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/k8s/helm_test/templates/serviceaccount.yaml b/k8s/helm_test/templates/serviceaccount.yaml deleted file mode 100644 index d47046559..000000000 --- a/k8s/helm_test/templates/serviceaccount.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "helm.serviceAccountName" . }} - labels: - {{- include "helm.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -automountServiceAccountToken: {{ .Values.serviceAccount.automount }} -{{- end }} diff --git a/k8s/helm_test/templates/tests/test-connection.yaml b/k8s/helm_test/templates/tests/test-connection.yaml deleted file mode 100644 index bf1c65f24..000000000 --- a/k8s/helm_test/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "helm.fullname" . }}-test-connection" - labels: - {{- include "helm.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "helm.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/k8s/helm_test/values-dev.yaml b/k8s/helm_test/values-dev.yaml deleted file mode 100644 index 7adf22402..000000000 --- a/k8s/helm_test/values-dev.yaml +++ /dev/null @@ -1,8 +0,0 @@ -rabbitmq: - host: wp-np2-40.ebi.ac.uk -biosamples: - schemaValidator: http://localhost:3020/validate - schemaStore: http://localhost:8085 - search: - host: biosamples-search-helm - port: 9090 \ No newline at end of file diff --git a/k8s/helm_test/values-prod.yaml b/k8s/helm_test/values-prod.yaml deleted file mode 100644 index cb9cb2bc8..000000000 --- a/k8s/helm_test/values-prod.yaml +++ /dev/null @@ -1,56 +0,0 @@ -rabbitmq: - host: wp-np2-40.ebi.ac.uk -biosamples: - schemaValidator: http://localhost:3020/validate - schemaStore: http://localhost:8085 - search: - host: biosamples-search-helm - port: 9090 - - -global: - imageRegistry: "registry.gitlab.com/your-group/your-project" - imagePullSecrets: - - name: docker-registry-secret - -services: - webapp-core: - replicaCount: 1 - resources: - limits: - cpu: 2000m - memory: 2Gi - requests: - cpu: 1000m - memory: 1Gi - ingress: - hosts: - - host: biosamples-core.example.com - paths: - - path: / - pathType: Prefix - - webapp-core-v2: - replicaCount: 1 - resources: - limits: - cpu: 2000m - memory: 2Gi - requests: - cpu: 1000m - memory: 1Gi - ingress: - hosts: - - host: biosamples-core-v2.example.com - paths: - - path: / - pathType: Prefix - - agents-uploadworkers: - replicaCount: 1 - - pipelines-reindex: - replicaCount: 1 - - pipelines-curation: - replicaCount: 1 \ No newline at end of file diff --git a/k8s/helm_test/values.yaml b/k8s/helm_test/values.yaml deleted file mode 100644 index afb576a03..000000000 --- a/k8s/helm_test/values.yaml +++ /dev/null @@ -1,139 +0,0 @@ -global: - imageRegistry: "" - imagePullSecrets: [] - storageClass: "" - -# Long-running services (Deployments) -services: - webapp-core: - enabled: true - replicaCount: 2 - image: - repository: "" - tag: "" - pullPolicy: IfNotPresent - service: - type: ClusterIP - port: 8080 - targetPort: 8080 -# ingress: -# enabled: true -# className: "nginx" -# hosts: -# - host: biosamples-core.example.com -# paths: -# - path: / -# pathType: Prefix - resources: - limits: - cpu: 1000m - memory: 1Gi - requests: - cpu: 500m - memory: 512Mi - - webapp-core-v2: - enabled: true - replicaCount: 2 - image: - repository: "" - tag: "" - pullPolicy: IfNotPresent - service: - type: ClusterIP - port: 8080 - targetPort: 8080 -# ingress: -# enabled: true -# className: "nginx" -# hosts: -# - host: biosamples-core-v2.example.com -# paths: -# - path: / -# pathType: Prefix - resources: - limits: - cpu: 1000m - memory: 1Gi - requests: - cpu: 500m - memory: 512Mi - -# agents-uploadworkers: -# enabled: true -# replicaCount: 1 -# image: -# repository: "" -# tag: "" -# pullPolicy: IfNotPresent -# service: -# enabled: false -# resources: -# limits: -# cpu: 500m -# memory: 512Mi -# requests: -# cpu: 250m -# memory: 256Mi - -# Batch Jobs (run-to-completion) -jobs: - pipelines-reindex: - enabled: false # Only run when explicitly enabled - image: - repository: "" - tag: "" - pullPolicy: IfNotPresent - backoffLimit: 3 - restartPolicy: Never - ttlSecondsAfterFinished: 3600 # Clean up job after 1 hour - resources: - limits: - cpu: 1000m - memory: 1Gi - requests: - cpu: 500m - memory: 512Mi - env: - - name: JOB_TYPE - value: "reindex" - - pipelines-curation: - enabled: false # Only run when explicitly enabled - image: - repository: "" - tag: "" - pullPolicy: IfNotPresent - backoffLimit: 3 - restartPolicy: Never - ttlSecondsAfterFinished: 3600 # Clean up job after 1 hour - resources: - limits: - cpu: 1000m - memory: 1Gi - requests: - cpu: 500m - memory: 512Mi - env: - - name: JOB_TYPE - value: "curation" - -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - create: true - annotations: {} - name: "" - -podAnnotations: {} - -podSecurityContext: {} - -securityContext: {} - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/k8s/jobs/values-fallback_prod.yaml b/k8s/jobs/values-fallback_prod.yaml new file mode 100644 index 000000000..97b5d91bd --- /dev/null +++ b/k8s/jobs/values-fallback_prod.yaml @@ -0,0 +1,2 @@ +rabbitmq: + host: wp-p2m-42.ebi.ac.uk diff --git a/k8s/jobs/values-primary_dev.yaml b/k8s/jobs/values-primary_dev.yaml new file mode 100644 index 000000000..156124fc8 --- /dev/null +++ b/k8s/jobs/values-primary_dev.yaml @@ -0,0 +1,2 @@ +rabbitmq: + host: wp-np2-40.ebi.ac.uk diff --git a/k8s/jobs/values-primary_prod.yaml b/k8s/jobs/values-primary_prod.yaml new file mode 100644 index 000000000..ecfbe90c3 --- /dev/null +++ b/k8s/jobs/values-primary_prod.yaml @@ -0,0 +1,2 @@ +rabbitmq: + host: wp-p1m-42.ebi.ac.uk From 5b8b0955e9a1bb0e006b605a747349c1276eb9fe Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 29 Sep 2025 13:18:32 +0100 Subject: [PATCH 30/86] test: deployment --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 46598417b..71dbfb225 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -238,7 +238,7 @@ deploy_k8s_primary_prod: K8S_NAMESPACE: biosamples-prod environment: name: primary_prod - url: https://wwwdev.ebi.ac.uk/biosamples + url: https://www.ebi.ac.uk/biosamples only: - dev - main @@ -252,7 +252,7 @@ deploy_k8s_fallback_prod: K8S_NAMESPACE: biosamples-prod environment: name: fallback_prod - url: https://wwwdev.ebi.ac.uk/biosamples + url: https://www.ebi.ac.uk/biosamples only: - dev - main @@ -296,6 +296,7 @@ deploy_pipeline_k8s_primary_dev: - docker:27-dind script: - echo $K8S_NAMESPACE + - kubectl config view - kubectl config set-context --current --namespace=${K8S_NAMESPACE} - kubectl delete secret $DOCKER_PULL_SECRET || true - | From 16a841bfdac47b2f73ca20d7d4588ebf359a84ab Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Thu, 9 Oct 2025 11:05:03 +0100 Subject: [PATCH 31/86] chore: cronjob k8s config --- .gitlab-ci.yml | 47 ++++++++++++++++++++++++++++++- k8s/Dockerfile | 9 ++++++ k8s/helm/values.yaml | 67 +++++++++++--------------------------------- 3 files changed, 71 insertions(+), 52 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 71dbfb225..123332da6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,6 +33,7 @@ maven-package-all-apps: paths: - webapps/core/target/webapps-core-*.war - webapps/core-v2/target/webapps-core-v2-*.jar + - agents/uploadworkers/target/agents-uploadworkers-*.jar - pipelines/reindex/target/pipelines-reindex-*.jar @@ -49,6 +50,8 @@ build_and_push_docker_images: DOCKERFILE_TARGET: "webapp-core" - APP_NAME: "webapp-core-v2" DOCKERFILE_TARGET: "webapp-core-v2" + - APP_NAME: "agents-uploadworkers" + DOCKERFILE_TARGET: "agents-uploadworkers" - APP_NAME: "pipelines-reindex" DOCKERFILE_TARGET: "pipelines-reindex" script: @@ -288,6 +291,20 @@ deploy_pipeline_k8s_primary_dev: when: manual extends: .kube_deploy_pipeline_script +deploy_cronjobs_k8s_primary_dev: + variables: + ENVIRONMENT_NAME: primary_dev + K8S_NAMESPACE: biosamples-dev + environment: + name: primary_dev + url: https://wwwdev.ebi.ac.uk/biosamples + only: + - dev + - main + - biosamples-search + when: manual + extends: .kube_deploy_cronjobs_script + .kube_deploy_script: stage: deploy image: dtzar/helm-kubectl:3.16 @@ -316,8 +333,14 @@ deploy_pipeline_k8s_primary_dev: --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE/webapp-core-v2 \ --set image.tag=$DOCKER_TAG + - | + helm upgrade --install ${APP_NAME}-v2 ./k8s/helm \ + --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ + --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ + --set image.repository=$CI_REGISTRY_IMAGE/webapp-core-v2 \ + --set image.tag=$DOCKER_TAG -.kube_deploy_pipeline_script: +.kube_deploy_jobs_script: stage: deploy image: dtzar/helm-kubectl:3.16 tags: [ "dind" ] @@ -338,3 +361,25 @@ deploy_pipeline_k8s_primary_dev: --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE/pipelines-reindex \ --set image.tag=$DOCKER_TAG + +.kube_deploy_cronjobs_script: + stage: deploy + image: dtzar/helm-kubectl:3.16 + tags: [ "dind" ] + services: + - docker:27-dind + script: + - echo $K8S_NAMESPACE + - kubectl config set-context --current --namespace=${K8S_NAMESPACE} + - kubectl delete secret $DOCKER_PULL_SECRET || true + - | + kubectl create secret docker-registry $DOCKER_PULL_SECRET \ + --docker-server=$CI_REGISTRY \ + --docker-username=$CI_REGISTRY_USER \ + --docker-password=$CI_REGISTRY_PASSWORD + - | + helm upgrade --install biosamples-reindex ./k8s/cronjobs \ + --values ./k8s/cronjobs/values-${ENVIRONMENT_NAME}.yaml \ + --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ + --set image.repository=$CI_REGISTRY_IMAGE/pipelines-curation \ + --set image.tag=$DOCKER_TAG diff --git a/k8s/Dockerfile b/k8s/Dockerfile index 15c028a88..3c9be8439 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -11,6 +11,15 @@ COPY webapps/core-v2/target/webapps-core-*.jar app.jar EXPOSE 8080 CMD ["java", "-jar", "app.jar"] +FROM base as agents-uploadworkers +COPY agents/uploadworkers/target/agents-uploadworkers-*.jar app.jar +EXPOSE 8080 +CMD ["java", "-jar", "app.jar"] + FROM base as pipelines-reindex COPY pipelines/reindex/target/pipelines-reindex-*.jar app.jar CMD ["java", "-jar", "app.jar"] + +FROM base as pipelines-curation +COPY pipelines/curation/target/pipelines-curation-*.jar app.jar +CMD ["java", "-jar", "app.jar"] diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index e04c05ca4..bbe45bcca 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -1,46 +1,36 @@ -# Default values for helm. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. -# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 1 +service: + type: ClusterIP + port: 8080 +livenessProbe: + httpGet: + path: /actuator/health/liveness + port: 8080 +readinessProbe: + httpGet: + path: /actuator/health/readiness + port: 8080 -# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ +replicaCount: 2 image: - repository: nginx - # This sets the pull policy for images. + repository: __set-in-commandline__ pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" + tag: latest -# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ -imagePullSecrets: [] -# This is to override the chart name. +imagePullSecrets: + - name: __set-in-commandline__ nameOverride: "" fullnameOverride: "" -# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ serviceAccount: - # Specifies whether a service account should be created create: true - # Automatically mount a ServiceAccount's API credentials? automount: true - # Annotations to add to the service account annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template name: "" - -# This is for setting Kubernetes Annotations to a Pod. -# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ podAnnotations: {} -# This is for setting Kubernetes Labels to a Pod. -# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ podLabels: {} - podSecurityContext: {} # fsGroup: 2000 - securityContext: {} # capabilities: # drop: @@ -49,14 +39,7 @@ securityContext: {} # runAsNonRoot: true # runAsUser: 1000 -# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/ -service: - # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types - type: ClusterIP - # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports - port: 8080 -# This block is for setting up the ingress for more information can be found here: https://kubernetes.io/docs/concepts/services-networking/ingress/ ingress: enabled: false className: "" @@ -74,10 +57,6 @@ ingress: # - chart-example.local resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. # limits: # cpu: 100m # memory: 128Mi @@ -85,17 +64,7 @@ resources: {} # cpu: 100m # memory: 128Mi -# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ -livenessProbe: - httpGet: - path: / - port: http -readinessProbe: - httpGet: - path: / - port: http -# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ autoscaling: enabled: false minReplicas: 1 @@ -103,21 +72,17 @@ autoscaling: targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 -# Additional volumes on the output Deployment definition. volumes: [] # - name: foo # secret: # secretName: mysecret # optional: false -# Additional volumeMounts on the output Deployment definition. volumeMounts: [] # - name: foo # mountPath: "/etc/foo" # readOnly: true nodeSelector: {} - tolerations: [] - affinity: {} From 791d9d7aaa6449f6a86a5d076adda7b2a7c47082 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 21 Oct 2025 10:48:46 +0100 Subject: [PATCH 32/86] deployment: expose service as nodeport --- k8s/helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index bbe45bcca..91422f21c 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -1,6 +1,6 @@ service: - type: ClusterIP + type: NodePort port: 8080 livenessProbe: httpGet: From c5eb8b9f3879123d2cc4c86d66aa5bb8aaea426c Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 21 Oct 2025 13:30:34 +0100 Subject: [PATCH 33/86] deploy: ci file error fix --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 123332da6..6d51b118e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -275,7 +275,7 @@ deploy_pipeline_k8s_primary_prod: - main - biosamples-search when: manual - extends: .kube_deploy_pipeline_script + extends: .kube_deploy_jobs_script deploy_pipeline_k8s_primary_dev: variables: @@ -289,7 +289,7 @@ deploy_pipeline_k8s_primary_dev: - main - biosamples-search when: manual - extends: .kube_deploy_pipeline_script + extends: .kube_deploy_jobs_script deploy_cronjobs_k8s_primary_dev: variables: From 54c5ab198243ce7e4d357cd71fcc7a2c7f3f9d05 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Wed, 22 Oct 2025 14:50:57 +0100 Subject: [PATCH 34/86] fix: context path issue --- k8s/helm/templates/deployment.yaml | 4 ++++ k8s/helm/values.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/k8s/helm/templates/deployment.yaml b/k8s/helm/templates/deployment.yaml index e93138ac8..78a381276 100644 --- a/k8s/helm/templates/deployment.yaml +++ b/k8s/helm/templates/deployment.yaml @@ -41,6 +41,10 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: + - name: SERVER_SERVLET_CONTEXT-PATH + value: /biosamples + - name: SERVER_CONTEXT-PATH + value: /biosamples - name: SPRING_DATA_MONGODB_URI valueFrom: secretKeyRef: diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index 91422f21c..a1c9d6bac 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -11,7 +11,7 @@ readinessProbe: path: /actuator/health/readiness port: 8080 -replicaCount: 2 +replicaCount: 1 image: repository: __set-in-commandline__ pullPolicy: IfNotPresent From 28db79f45b7953c7fe5f497720bef57969a36b3c Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Wed, 22 Oct 2025 15:52:05 +0100 Subject: [PATCH 35/86] fix: context path for v2 --- .gitlab-ci.yml | 6 ++++-- k8s/helm/templates/deployment.yaml | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d51b118e..1639ae09c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -326,13 +326,15 @@ deploy_cronjobs_k8s_primary_dev: --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE/webapp-core \ - --set image.tag=$DOCKER_TAG + --set image.tag=$DOCKER_TAG \ + --set biosamples.context.path=/biosamples - | helm upgrade --install ${APP_NAME}-v2 ./k8s/helm \ --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE/webapp-core-v2 \ - --set image.tag=$DOCKER_TAG + --set image.tag=$DOCKER_TAG \ + --set biosamples.context.path=/biosamples/v2 - | helm upgrade --install ${APP_NAME}-v2 ./k8s/helm \ --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ diff --git a/k8s/helm/templates/deployment.yaml b/k8s/helm/templates/deployment.yaml index 78a381276..029a7c041 100644 --- a/k8s/helm/templates/deployment.yaml +++ b/k8s/helm/templates/deployment.yaml @@ -42,9 +42,9 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: SERVER_SERVLET_CONTEXT-PATH - value: /biosamples + value: {{ .Values.biosamples.context.path }} - name: SERVER_CONTEXT-PATH - value: /biosamples + value: {{ .Values.biosamples.context.path }} - name: SPRING_DATA_MONGODB_URI valueFrom: secretKeyRef: From 153829a493eb1f772ae9f21a48d036b3f34af98a Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Thu, 23 Oct 2025 13:03:22 +0100 Subject: [PATCH 36/86] fix: schema service url changes for k8s deployment --- .../ac/ebi/biosamples/client/utils/ClientProperties.java | 7 ++----- docker-compose.yml | 8 ++++---- k8s/helm/templates/deployment.yaml | 4 ++++ k8s/helm/values-fallback_prod.yaml | 2 -- k8s/helm/values-primary_dev.yaml | 2 -- k8s/helm/values-primary_prod.yaml | 2 -- k8s/helm/values.yaml | 9 ++++++++- .../java/uk/ac/ebi/biosamples/PipelinesProperties.java | 4 ++-- .../java/uk/ac/ebi/biosamples/BioSamplesProperties.java | 5 +---- webapps/core/core-deployment.yaml | 4 ++-- 10 files changed, 23 insertions(+), 24 deletions(-) diff --git a/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/ClientProperties.java b/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/ClientProperties.java index 840e904c1..ce00f5202 100644 --- a/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/ClientProperties.java +++ b/client/client/src/main/java/uk/ac/ebi/biosamples/client/utils/ClientProperties.java @@ -100,13 +100,10 @@ public class ClientProperties { @Value("${biosamples.webapp.core.facet.cache.maxage:86400}") private int webappCoreFacetCacheMaxAge; - @Value("${biosamples.schema.validator.uri:http://localhost:8085/validate}") - private URI biosamplesSchemaValidatorServiceUri; - - @Value("${biosamples.schemaValidator:http://localhost:3020/validate}") + @Value("${biosamples.schema.validator.url:http://localhost:3020/validate}") private String schemaValidator; - @Value("${biosamples.schemaStore:http://localhost:8085}") + @Value("${biosamples.schema.store.url:http://localhost:8085}") private String schemaStore; @Value("${biosamples.schema.default:BSDC00001}") diff --git a/docker-compose.yml b/docker-compose.yml index 68ab32815..42a9e19db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,8 +28,8 @@ services: - spring.data.mongodb.auto-index-creation=true - biosamples.mongo.sample.writeConcern=1 - BIOSAMPLES_NEO_URL=bolt://neo4j:7687 - - biosamples.schemaValidator=http://json-schema-validator:3020/validate - - biosamples.schemaStore=https://wwwdev.ebi.ac.uk/biosamples/schema-store + - biosamples.schema.validator.url=http://json-schema-validator:3020/validate + - biosamples.schema.store.url=https://wwwdev.ebi.ac.uk/biosamples/schema-store - biosamples.search.host=biosamples-search - SPRING_RABBITMQ_HOST=rabbitmq - SPRING_RABBITMQ_PUBLISHER-CONFIRMS=true @@ -119,8 +119,8 @@ services: - SPRING_RABBITMQ_LISTENER_SIMPLE_PREFETCH=100 - SPRING_RABBITMQ_LISTENER_SIMPLE_TRANSACTION-SIZE=25 - BIOSAMPLES_NEO_URL=bolt://neo4j:7687 - - biosamples.schemaValidator=http://json-schema-validator:3020/validate - - biosamples.schemaStore=https://wwwdev.ebi.ac.uk/biosamples/schema-store + - biosamples.schema.validator.url=http://json-schema-validator:3020/validate + - biosamples.schema.store.url=https://wwwdev.ebi.ac.uk/biosamples/schema-store - spring.jackson.serialization-inclusion=non_null - spring.jackson.serialization.WRITE_NULL_MAP_VALUES=false - spring.jackson.serialization.indent_output=true diff --git a/k8s/helm/templates/deployment.yaml b/k8s/helm/templates/deployment.yaml index 029a7c041..7e94c3e69 100644 --- a/k8s/helm/templates/deployment.yaml +++ b/k8s/helm/templates/deployment.yaml @@ -45,6 +45,10 @@ spec: value: {{ .Values.biosamples.context.path }} - name: SERVER_CONTEXT-PATH value: {{ .Values.biosamples.context.path }} + - name: BIOSAMPLES.SCHEMA.VALIDATOR.URL + value: {{ .Values.biosamples.schema.validator.url }} + - name: BIOSAMPLES.SCHEMA.STORE.URL + value: {{ .Values.biosamples.schema.store.url }} - name: SPRING_DATA_MONGODB_URI valueFrom: secretKeyRef: diff --git a/k8s/helm/values-fallback_prod.yaml b/k8s/helm/values-fallback_prod.yaml index d8a55a374..34a46d3c4 100644 --- a/k8s/helm/values-fallback_prod.yaml +++ b/k8s/helm/values-fallback_prod.yaml @@ -1,8 +1,6 @@ rabbitmq: host: wp-p2m-42.ebi.ac.uk biosamples: - schemaValidator: http://localhost:3020/validate - schemaStore: http://localhost:8085 search: host: biosamples-search-helm port: 9090 diff --git a/k8s/helm/values-primary_dev.yaml b/k8s/helm/values-primary_dev.yaml index 7adf22402..fa6536116 100644 --- a/k8s/helm/values-primary_dev.yaml +++ b/k8s/helm/values-primary_dev.yaml @@ -1,8 +1,6 @@ rabbitmq: host: wp-np2-40.ebi.ac.uk biosamples: - schemaValidator: http://localhost:3020/validate - schemaStore: http://localhost:8085 search: host: biosamples-search-helm port: 9090 \ No newline at end of file diff --git a/k8s/helm/values-primary_prod.yaml b/k8s/helm/values-primary_prod.yaml index d343639c5..2a8d877e0 100644 --- a/k8s/helm/values-primary_prod.yaml +++ b/k8s/helm/values-primary_prod.yaml @@ -1,8 +1,6 @@ rabbitmq: host: wp-p1m-42.ebi.ac.uk biosamples: - schemaValidator: http://localhost:3020/validate - schemaStore: http://localhost:8085 search: host: biosamples-search-helm port: 9090 diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index a1c9d6bac..61a155eb3 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -1,4 +1,11 @@ - +biosamples: + context: + path: /biosamples + schema: + validator: + url: http://biovalidator-service:3020/biosamples/biovalidator/validate + store: + url: http://json-schema-store:8080/biosamples/schema-store service: type: NodePort port: 8080 diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java index baf612762..109ac0013 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/PipelinesProperties.java @@ -66,10 +66,10 @@ public class PipelinesProperties { @Value("${biosamples.pipelines.copydown.domain:self.BiosampleCopydown}") private String copydownDomain; - @Value("${biosamples.schemaValidator:http://localhost:3020/validate}") + @Value("${biosamples.schema.validator.url:http://localhost:3020/validate}") private String schemaValidator; - @Value("${biosamples.schemaStore:http://localhost:8085/api/v2/schemas}") + @Value("${biosamples.schema.store.url:http://localhost:8085/api/v2/schemas}") private String schemaStore; @Value( diff --git a/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java b/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java index ba280137b..ac9f71876 100644 --- a/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java +++ b/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java @@ -97,10 +97,7 @@ public class BioSamplesProperties { @Value("${biosamples.webapp.core.facet.cache.maxage:86400}") private int webappCoreFacetCacheMaxAge; - @Value("${biosamples.schema.validator.uri:http://localhost:8085/validate}") - private URI biosamplesSchemaValidatorServiceUri; - - @Value("${biosamples.schemaValidator:http://localhost:3020/validate}") + @Value("${biosamples.schema.validator.url:http://localhost:3020/validate}") private String schemaValidator; @Value("${biosamples.schemaStore:http://localhost:8085}") diff --git a/webapps/core/core-deployment.yaml b/webapps/core/core-deployment.yaml index 045616baa..813543a46 100644 --- a/webapps/core/core-deployment.yaml +++ b/webapps/core/core-deployment.yaml @@ -27,9 +27,9 @@ spec: value: "wp-np2-40" - name: SPRING_RABBITMQ_PORT value: "5672" - - name: biosamples.schemaValidator + - name: biosamples.schema.validator.url value: "https://wwwdev.ebi.ac.uk/biosamples/biovalidator/validate" - - name: biosamples.schemaStore + - name: biosamples.schema.store.url value: "https://wwwdev.ebi.ac.uk/biosamples/schema-store" - name: SERVER_SERVLET_CONTEXT_PATH value: /biosamples From e3c2d20a16ac14a905a15ce23276bdc1e93b3e4e Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Thu, 23 Oct 2025 13:26:54 +0100 Subject: [PATCH 37/86] fix: schema store url property --- .../main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java b/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java index ac9f71876..08230d200 100644 --- a/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java +++ b/properties/src/main/java/uk/ac/ebi/biosamples/BioSamplesProperties.java @@ -100,7 +100,7 @@ public class BioSamplesProperties { @Value("${biosamples.schema.validator.url:http://localhost:3020/validate}") private String schemaValidator; - @Value("${biosamples.schemaStore:http://localhost:8085}") + @Value("${biosamples.schema.store.url:http://localhost:8085}") private String schemaStore; @Value("${biosamples.schema.default:BSDC00001}") From 748748da332e4cb364114fcc9f42a54b1dc4e1b8 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 24 Oct 2025 12:43:22 +0100 Subject: [PATCH 38/86] fix: context path --- .gitlab-ci.yml | 3 ++- k8s/helm/values.yaml | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1639ae09c..949b8db7b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -340,7 +340,8 @@ deploy_cronjobs_k8s_primary_dev: --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE/webapp-core-v2 \ - --set image.tag=$DOCKER_TAG + --set image.tag=$DOCKER_TAG \ + --set biosamples.context.path=/biosamples .kube_deploy_jobs_script: stage: deploy diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index 61a155eb3..70c14869d 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -1,6 +1,4 @@ biosamples: - context: - path: /biosamples schema: validator: url: http://biovalidator-service:3020/biosamples/biovalidator/validate From 7b9b81c965b3cd18e2f6bd144111a26c3d277de6 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 24 Oct 2025 14:23:21 +0100 Subject: [PATCH 39/86] fix: context path --- .gitlab-ci.yml | 5 ++--- k8s/helm/values.yaml | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 949b8db7b..6eb2817dc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -336,12 +336,11 @@ deploy_cronjobs_k8s_primary_dev: --set image.tag=$DOCKER_TAG \ --set biosamples.context.path=/biosamples/v2 - | - helm upgrade --install ${APP_NAME}-v2 ./k8s/helm \ + helm upgrade --install ${APP_NAME}-uploadworkers ./k8s/helm \ --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE/webapp-core-v2 \ - --set image.tag=$DOCKER_TAG \ - --set biosamples.context.path=/biosamples + --set image.tag=$DOCKER_TAG .kube_deploy_jobs_script: stage: deploy diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index 70c14869d..61a155eb3 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -1,4 +1,6 @@ biosamples: + context: + path: /biosamples schema: validator: url: http://biovalidator-service:3020/biosamples/biovalidator/validate From a0eac0272c30d048e17351353b47436c44ebdf0a Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 27 Oct 2025 15:36:24 +0000 Subject: [PATCH 40/86] chore: approximate facet count message --- webapps/core/src/main/resources/templates/fragments/facets.html | 1 + 1 file changed, 1 insertion(+) diff --git a/webapps/core/src/main/resources/templates/fragments/facets.html b/webapps/core/src/main/resources/templates/fragments/facets.html index 8204be2c4..b13c21ee8 100644 --- a/webapps/core/src/main/resources/templates/fragments/facets.html +++ b/webapps/core/src/main/resources/templates/fragments/facets.html @@ -24,6 +24,7 @@ +
Showing approximate facet counts for faster response
Date: Tue, 28 Oct 2025 10:59:19 +0000 Subject: [PATCH 42/86] chore: improve version update script to cleanup files --- update-versions.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/update-versions.sh b/update-versions.sh index 9e5cc477a..b387715aa 100755 --- a/update-versions.sh +++ b/update-versions.sh @@ -64,4 +64,10 @@ echo "Updating docker-compose and shell files to the new version" find . -name "docker-*.yml" -or -name "docker-*.sh" | xargs sed -i.versionsBackup "s/$LAST_VERSION/$NEW_VERSION/g" || exit 1 -echo "Version update complete!" + +echo "Clearing up resource..." +./mvnw versions:commit +find . -name "*.versionsBackup" -delete + +echo "Version update complete: $LAST_VERSION -> $NEW_VERSION" + From 1c5b73ec495844d989825a727e11843f489dd075 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 3 Nov 2025 12:50:22 +0000 Subject: [PATCH 43/86] fix: uploadworkers cicd config naming issue --- k8s/helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index 61a155eb3..a6fbe899c 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -1,6 +1,6 @@ biosamples: context: - path: /biosamples + path: / schema: validator: url: http://biovalidator-service:3020/biosamples/biovalidator/validate From 99f7444f32f32ebf46c216936065b769e63172a8 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 3 Nov 2025 14:01:36 +0000 Subject: [PATCH 44/86] fix: uploadworkers cicd config naming issue --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 76b83d4d0..7afafaa26 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -187,7 +187,7 @@ deploy_cronjobs_k8s_primary_dev: helm upgrade --install ${APP_NAME}-uploadworkers ./k8s/helm \ --values ./k8s/helm/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ - --set image.repository=$CI_REGISTRY_IMAGE/webapp-core-v2 \ + --set image.repository=$CI_REGISTRY_IMAGE/agents-uploadworkers \ --set image.tag=$DOCKER_TAG .kube_deploy_jobs_script: From fd955b48d7c12d3f0b017ce568efd42bdf189789 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 3 Nov 2025 19:29:05 +0000 Subject: [PATCH 45/86] fix: remove rabbit message conversion extra quotes --- .../FileUploadSubmissionService.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java index aa59fb079..ff2a025f5 100644 --- a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java +++ b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java @@ -62,14 +62,17 @@ public void receiveMessageFromBioSamplesFileUploaderQueue(final String mongoFile handleMessage(mongoFileId); } - private void handleMessage(final String submissionId) { - final Optional fileUploadOptional = - mongoFileUploadRepository.findById(submissionId); - final MongoFileUpload mongoFileUpload = - fileUploadOptional.orElseThrow( - () -> - new GlobalExceptions.UploadInvalidException( - "Could not find file upload record for submissionId: " + submissionId)); + private void handleMessage(String submissionId) { + submissionId = submissionId.replace("\"", ""); + final Optional fileUploadOptional = mongoFileUploadRepository.findById(submissionId); + if (fileUploadOptional.isEmpty()) { + log.error("Could not find file upload record for submissionId: {}", submissionId); + //todo here exception means there is no progress from the queue reading loop. + // We can send this to dead letter or something for monitoring. + return; + } + + final MongoFileUpload mongoFileUpload = fileUploadOptional.get(); try { validationResult = new ValidationResult(); From d58e0facaf30a309f366e91e30dffbb57a73075d Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 11 Nov 2025 14:14:45 +0000 Subject: [PATCH 46/86] feat: cicd application.properties --- .gitlab-ci.yml | 17 +++++++++++++++++ k8s/helm/values.yaml | 17 ++++++++--------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7afafaa26..fd4dbadf9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -68,6 +68,19 @@ build_and_push_docker_images: after_script: - docker logout $CI_REGISTRY +clone-config: + stage: config + script: + - git clone https://$BSD_INTERNAL_USER:$BSD_INTERNAL_PASS@gitlab.ebi.ac.uk/biosamples/biosamples-internal.git + - mkdir config + - cp -r biosamples-internal/k8s/* config/ + artifacts: + paths: + - config + only: + - dev + - main + - biosamples-search deploy_k8s_primary_dev: variables: @@ -163,6 +176,10 @@ deploy_cronjobs_k8s_primary_dev: - echo $K8S_NAMESPACE - kubectl config view - kubectl config set-context --current --namespace=${K8S_NAMESPACE} + - | + kubectl create configmap application-properties \ + --from-file=./config/${ENVIRONMENT_NAME}/application.properties \ + --dry-run=client -o yaml | kubectl apply -f - - kubectl delete secret $DOCKER_PULL_SECRET || true - | kubectl create secret docker-registry $DOCKER_PULL_SECRET \ diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index a6fbe899c..cabbe5c04 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -79,16 +79,15 @@ autoscaling: targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 -volumes: [] -# - name: foo -# secret: -# secretName: mysecret -# optional: false +volumes: + - name: config-volume + configMap: + name: application-properties -volumeMounts: [] -# - name: foo -# mountPath: "/etc/foo" -# readOnly: true +volumeMounts: + - name: config-volume + mountPath: /config/application.properties + subPath: application.properties nodeSelector: {} tolerations: [] From 63fb339ad00c9d5142d32ce34ab88c9303e261a4 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 11 Nov 2025 15:11:30 +0000 Subject: [PATCH 47/86] feat: cicd application.properties --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fd4dbadf9..49055fb31 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -177,7 +177,7 @@ deploy_cronjobs_k8s_primary_dev: - kubectl config view - kubectl config set-context --current --namespace=${K8S_NAMESPACE} - | - kubectl create configmap application-properties \ + kubectl create configmap application-properties \ --from-file=./config/${ENVIRONMENT_NAME}/application.properties \ --dry-run=client -o yaml | kubectl apply -f - - kubectl delete secret $DOCKER_PULL_SECRET || true From 0c35d125c464f2f581c9682af5338438de36b52e Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Tue, 11 Nov 2025 16:48:41 +0000 Subject: [PATCH 48/86] feat: cicd application.properties --- k8s/helm/templates/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s/helm/templates/deployment.yaml b/k8s/helm/templates/deployment.yaml index 7e94c3e69..16711cb69 100644 --- a/k8s/helm/templates/deployment.yaml +++ b/k8s/helm/templates/deployment.yaml @@ -49,6 +49,8 @@ spec: value: {{ .Values.biosamples.schema.validator.url }} - name: BIOSAMPLES.SCHEMA.STORE.URL value: {{ .Values.biosamples.schema.store.url }} + - name: SPRING_CONFIG_LOCATION + value: /config/application.properties - name: SPRING_DATA_MONGODB_URI valueFrom: secretKeyRef: From 0ee2fc20611b673bdfd765635c20ab3b0d4e53f6 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Thu, 13 Nov 2025 10:13:22 +0000 Subject: [PATCH 49/86] test: deployment --- k8s/helm/templates/deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/helm/templates/deployment.yaml b/k8s/helm/templates/deployment.yaml index 16711cb69..3b7a63ef1 100644 --- a/k8s/helm/templates/deployment.yaml +++ b/k8s/helm/templates/deployment.yaml @@ -49,8 +49,8 @@ spec: value: {{ .Values.biosamples.schema.validator.url }} - name: BIOSAMPLES.SCHEMA.STORE.URL value: {{ .Values.biosamples.schema.store.url }} - - name: SPRING_CONFIG_LOCATION - value: /config/application.properties +{{/* - name: SPRING_CONFIG_LOCATION*/}} +{{/* value: /config/application.properties*/}} - name: SPRING_DATA_MONGODB_URI valueFrom: secretKeyRef: From 5fcf4aa6491c0ee77064afcf2da5252e5b5c90c4 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Thu, 13 Nov 2025 12:17:22 +0000 Subject: [PATCH 50/86] test: deployment --- k8s/helm/templates/deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/helm/templates/deployment.yaml b/k8s/helm/templates/deployment.yaml index 3b7a63ef1..16711cb69 100644 --- a/k8s/helm/templates/deployment.yaml +++ b/k8s/helm/templates/deployment.yaml @@ -49,8 +49,8 @@ spec: value: {{ .Values.biosamples.schema.validator.url }} - name: BIOSAMPLES.SCHEMA.STORE.URL value: {{ .Values.biosamples.schema.store.url }} -{{/* - name: SPRING_CONFIG_LOCATION*/}} -{{/* value: /config/application.properties*/}} + - name: SPRING_CONFIG_LOCATION + value: /config/application.properties - name: SPRING_DATA_MONGODB_URI valueFrom: secretKeyRef: From 9b9b92c76d70b4369e45bff27f134b31bbb5004f Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 17 Nov 2025 11:18:11 +0000 Subject: [PATCH 51/86] fix: NCBI pipeline rest template --- .../uk/ac/ebi/biosamples/Application.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/Application.java index 1ffca1583..d4bac8652 100644 --- a/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/ncbi/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -11,8 +11,10 @@ package uk.ac.ebi.biosamples; import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.web.client.RestTemplateCustomizer; import org.springframework.cache.annotation.EnableCaching; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; @@ -22,6 +24,8 @@ import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.configuration.ExclusionConfiguration; import uk.ac.ebi.biosamples.service.EnaConfig; import uk.ac.ebi.biosamples.service.EnaSampleToBioSampleConversionService; @@ -39,6 +43,7 @@ @EnableCaching(proxyTargetClass = true) @EnableAsync @EnableScheduling +@EnableWebSecurity public class Application { // this is needed to read nonstrings from properties files @@ -49,7 +54,25 @@ public static PropertySourcesPlaceholderConfigurer getPropertySourcesPlaceholder } public static void main(final String[] args) { - final ConfigurableApplicationContext ctx = SpringApplication.run(Application.class, args); + SpringApplication app = new SpringApplication(Application.class); + app.setWebApplicationType(WebApplicationType.NONE); + + final ConfigurableApplicationContext ctx = app.run(args); PipelineUtils.exitPipeline(ctx); } + + @Bean + public RestTemplate restTemplate(final RestTemplateCustomizer restTemplateCustomizer) { + final RestTemplate restTemplate = new RestTemplate(); + restTemplateCustomizer.customize(restTemplate); + return restTemplate; + } + + @Bean + public RestTemplateCustomizer restTemplateCustomizer( + final BioSamplesProperties bioSamplesProperties, + final PipelinesProperties pipelinesProperties) { + return new PipelinesHelper() + .getRestTemplateCustomizer(bioSamplesProperties, pipelinesProperties); + } } From 105c55fd4d097508220a54d44a0ece1e8c0e08ae Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 17 Nov 2025 12:56:43 +0000 Subject: [PATCH 52/86] fix: file upload validaiton error --- .../ebi/biosamples/submission/FileUploadSubmissionService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java index ff2a025f5..6994209de 100644 --- a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java +++ b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java @@ -314,6 +314,7 @@ private Sample buildAndPersistSample( } catch (final Exception e) { persisted = false; handleUnauthorizedWhilePersistence(sampleName, accession, sampleWithAccession, e); + throw new GlobalExceptions.SampleValidationException(e.getMessage()); } if (sampleWithAccession && persisted) { From 988afcec9216ce97600bdd35b75902e19d74d415 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 21 Nov 2025 13:22:24 +0000 Subject: [PATCH 53/86] test: argo workflow cicd --- .gitlab-ci.yml | 2 +- k8s/argo-workflow.yaml | 16 +++++++++++++ k8s/cronjobs/Chart.yaml | 6 +++++ k8s/cronjobs/templates/workflow.yaml | 35 ++++++++++++++++++++++++++++ k8s/cronjobs/values.yaml | 28 ++++++++++++++++++++++ 5 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 k8s/argo-workflow.yaml create mode 100644 k8s/cronjobs/Chart.yaml create mode 100644 k8s/cronjobs/templates/workflow.yaml create mode 100644 k8s/cronjobs/values.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49055fb31..b25b8ef47 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -245,7 +245,7 @@ deploy_cronjobs_k8s_primary_dev: --docker-username=$CI_REGISTRY_USER \ --docker-password=$CI_REGISTRY_PASSWORD - | - helm upgrade --install biosamples-reindex ./k8s/cronjobs \ + helm upgrade --install biosamples-curation ./k8s/cronjobs \ --values ./k8s/cronjobs/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE/pipelines-curation \ diff --git a/k8s/argo-workflow.yaml b/k8s/argo-workflow.yaml new file mode 100644 index 000000000..52c0f7e92 --- /dev/null +++ b/k8s/argo-workflow.yaml @@ -0,0 +1,16 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: springboot-cli- +spec: + entrypoint: run-job + + templates: + - name: run-job + container: + image: registry.gitlab.com/YOUR-PROJECT/YOUR-REPO:latest + command: ["java", "-jar", "/app/app.jar"] + resources: + limits: + cpu: "500m" + memory: "1Gi" \ No newline at end of file diff --git a/k8s/cronjobs/Chart.yaml b/k8s/cronjobs/Chart.yaml new file mode 100644 index 000000000..f5a4a0471 --- /dev/null +++ b/k8s/cronjobs/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: biosamples-curation +description: Run a Spring Boot CLI job via Argo Workflow +type: application +version: 0.1.0 +appVersion: "1.0" diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml new file mode 100644 index 000000000..d0e61e191 --- /dev/null +++ b/k8s/cronjobs/templates/workflow.yaml @@ -0,0 +1,35 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + name: {{ include "springboot-argo-job.fullname" . }} + namespace: {{ .Values.workflow.namespace }} + labels: + app.kubernetes.io/name: {{ include "springboot-argo-job.name" . }} +spec: + entrypoint: {{ .Values.job.name }} + + ttlStrategy: + secondsAfterSuccess: {{ .Values.workflow.ttlStrategy.secondsAfterSuccess }} + secondsAfterFailure: {{ .Values.workflow.ttlStrategy.secondsAfterFailure }} + + templates: + - name: {{ .Values.job.name }} + retryStrategy: + limit: {{ .Values.workflow.retryStrategy.limit }} + backoff: + duration: {{ .Values.workflow.retryStrategy.backoff.duration }} + factor: {{ .Values.workflow.retryStrategy.backoff.factor }} + maxDuration: {{ .Values.workflow.retryStrategy.backoff.maxDuration }} + + container: + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + + command: + {{- toYaml .Values.job.command | nindent 10 }} + + args: + {{- toYaml .Values.job.args | nindent 10 }} + + resources: + {{- toYaml .Values.job.resources | nindent 10 }} diff --git a/k8s/cronjobs/values.yaml b/k8s/cronjobs/values.yaml new file mode 100644 index 000000000..fbfa078bd --- /dev/null +++ b/k8s/cronjobs/values.yaml @@ -0,0 +1,28 @@ +image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + +job: + name: biosamples-curation + command: ["java", "-jar", "/app/app.jar"] + args: [] + resources: + limits: + cpu: "500m" + memory: "1Gi" + requests: + cpu: "200m" + memory: "512Mi" + +workflow: + namespace: argo + ttlStrategy: + secondsAfterSuccess: 3600 + secondsAfterFailure: 7200 + retryStrategy: + limit: 2 + backoff: + duration: "10s" + factor: 2 + maxDuration: "1m" From c8390549a5648381eae896b7afeb43d0d5ca2254 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 21 Nov 2025 14:47:56 +0000 Subject: [PATCH 54/86] test: argo workflow cicd --- k8s/cronjobs/values-primary_dev.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 k8s/cronjobs/values-primary_dev.yaml diff --git a/k8s/cronjobs/values-primary_dev.yaml b/k8s/cronjobs/values-primary_dev.yaml new file mode 100644 index 000000000..156124fc8 --- /dev/null +++ b/k8s/cronjobs/values-primary_dev.yaml @@ -0,0 +1,2 @@ +rabbitmq: + host: wp-np2-40.ebi.ac.uk From 056f04da2cb16c3ee7fcfc5f484436ff5fb6d88a Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 21 Nov 2025 15:25:41 +0000 Subject: [PATCH 55/86] test: argo workflow cicd --- k8s/cronjobs/templates/workflow.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index d0e61e191..c5fafccca 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -1,10 +1,10 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - name: {{ include "springboot-argo-job.fullname" . }} + name: {{ include "biosamples-curation.fullname" . }} namespace: {{ .Values.workflow.namespace }} labels: - app.kubernetes.io/name: {{ include "springboot-argo-job.name" . }} + app.kubernetes.io/name: {{ include "biosamples-curation.name" . }} spec: entrypoint: {{ .Values.job.name }} From 5bdafcf74ccba5de5bc114939af6bfc0e08f05cf Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 21 Nov 2025 15:51:13 +0000 Subject: [PATCH 56/86] test: argo workflow cicd --- k8s/cronjobs/templates/_helpers.tpl | 62 +++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 k8s/cronjobs/templates/_helpers.tpl diff --git a/k8s/cronjobs/templates/_helpers.tpl b/k8s/cronjobs/templates/_helpers.tpl new file mode 100644 index 000000000..d8956a62c --- /dev/null +++ b/k8s/cronjobs/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "biosamples-curation.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "biosamples-curation.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "biosamples-curation.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "biosamples-curation.labels" -}} +helm.sh/chart: {{ include "helm.chart" . }} +{{ include "biosamples-curation.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "biosamples-curation.selectorLabels" -}} +app.kubernetes.io/name: {{ include "biosamples-curation.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "biosamples-curation.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "biosamples-curation.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} From 478ceb09c0e45031e1aac514dd5d8abf43897fb0 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 21 Nov 2025 16:28:32 +0000 Subject: [PATCH 57/86] test: argo workflow cicd --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b25b8ef47..246d2a292 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,6 +53,8 @@ build_and_push_docker_images: DOCKERFILE_TARGET: "agents-uploadworkers" - APP_NAME: "pipelines-reindex" DOCKERFILE_TARGET: "pipelines-reindex" + - APP_NAME: "pipelines-curation" + DOCKERFILE_TARGET: "pipelines-curation" script: - | DOCKER_IMAGE_NAME="$CI_REGISTRY_IMAGE/$APP_NAME:$DOCKER_TAG" From 7b317ea952eafc02a739de762595100ca386097b Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 21 Nov 2025 16:57:24 +0000 Subject: [PATCH 58/86] test: argo workflow cicd --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 246d2a292..0fd7c3da2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,6 +34,7 @@ maven-package-all-apps: - webapps/core-v2/target/webapps-core-v2-*.jar - agents/uploadworkers/target/agents-uploadworkers-*.jar - pipelines/reindex/target/pipelines-reindex-*.jar + - pipelines/reindex/target/pipelines-curation-*.jar build_and_push_docker_images: From 1fc7778c953c1f12fa5a517e8249ecc1f9268803 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 21 Nov 2025 16:57:47 +0000 Subject: [PATCH 59/86] test: argo workflow cicd --- k8s/cronjobs/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/k8s/cronjobs/values.yaml b/k8s/cronjobs/values.yaml index fbfa078bd..b9120c6f1 100644 --- a/k8s/cronjobs/values.yaml +++ b/k8s/cronjobs/values.yaml @@ -16,7 +16,6 @@ job: memory: "512Mi" workflow: - namespace: argo ttlStrategy: secondsAfterSuccess: 3600 secondsAfterFailure: 7200 From 94df99162c4f9dc57b5758f2f31f4a42212a2748 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 24 Nov 2025 10:09:58 +0000 Subject: [PATCH 60/86] test: argo workflow cicd --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0fd7c3da2..9092d8390 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,7 +34,7 @@ maven-package-all-apps: - webapps/core-v2/target/webapps-core-v2-*.jar - agents/uploadworkers/target/agents-uploadworkers-*.jar - pipelines/reindex/target/pipelines-reindex-*.jar - - pipelines/reindex/target/pipelines-curation-*.jar + - pipelines/curation/target/pipelines-curation-*.jar build_and_push_docker_images: From 1c3d7b97645ae0b02abb1c5340f82b89625ee7de Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 24 Nov 2025 11:20:07 +0000 Subject: [PATCH 61/86] test: argo workflow cicd --- .gitlab-ci.yml | 3 +-- k8s/cronjobs/Chart.yaml | 2 +- k8s/cronjobs/templates/_helpers.tpl | 18 +++++++++--------- k8s/cronjobs/templates/workflow.yaml | 6 ++++-- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9092d8390..5f243984e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -177,7 +177,6 @@ deploy_cronjobs_k8s_primary_dev: - docker:27-dind script: - echo $K8S_NAMESPACE - - kubectl config view - kubectl config set-context --current --namespace=${K8S_NAMESPACE} - | kubectl create configmap application-properties \ @@ -248,7 +247,7 @@ deploy_cronjobs_k8s_primary_dev: --docker-username=$CI_REGISTRY_USER \ --docker-password=$CI_REGISTRY_PASSWORD - | - helm upgrade --install biosamples-curation ./k8s/cronjobs \ + helm upgrade --install curation ./k8s/cronjobs \ --values ./k8s/cronjobs/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE/pipelines-curation \ diff --git a/k8s/cronjobs/Chart.yaml b/k8s/cronjobs/Chart.yaml index f5a4a0471..8542fcd16 100644 --- a/k8s/cronjobs/Chart.yaml +++ b/k8s/cronjobs/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: biosamples-curation +name: biosamples-v4-pipelines description: Run a Spring Boot CLI job via Argo Workflow type: application version: 0.1.0 diff --git a/k8s/cronjobs/templates/_helpers.tpl b/k8s/cronjobs/templates/_helpers.tpl index d8956a62c..39ee73965 100644 --- a/k8s/cronjobs/templates/_helpers.tpl +++ b/k8s/cronjobs/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "biosamples-curation.name" -}} +{{- define "biosamples-v4-pipelines.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "biosamples-curation.fullname" -}} +{{- define "biosamples-v4-pipelines.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "biosamples-curation.chart" -}} +{{- define "biosamples-v4-pipelines.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "biosamples-curation.labels" -}} +{{- define "biosamples-v4-pipelines.labels" -}} helm.sh/chart: {{ include "helm.chart" . }} -{{ include "biosamples-curation.selectorLabels" . }} +{{ include "biosamples-v4-pipelines.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,17 +45,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "biosamples-curation.selectorLabels" -}} -app.kubernetes.io/name: {{ include "biosamples-curation.name" . }} +{{- define "biosamples-v4-pipelines.selectorLabels" -}} +app.kubernetes.io/name: {{ include "biosamples-v4-pipelines.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* Create the name of the service account to use */}} -{{- define "biosamples-curation.serviceAccountName" -}} +{{- define "biosamples-v4-pipelines.serviceAccountName" -}} {{- if .Values.serviceAccount.create }} -{{- default (include "biosamples-curation.fullname" .) .Values.serviceAccount.name }} +{{- default (include "biosamples-v4-pipelines.fullname" .) .Values.serviceAccount.name }} {{- else }} {{- default "default" .Values.serviceAccount.name }} {{- end }} diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index c5fafccca..3993e350f 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -1,13 +1,15 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - name: {{ include "biosamples-curation.fullname" . }} + name: {{ include "biosamples-v4-pipelines.fullname" . }} namespace: {{ .Values.workflow.namespace }} labels: - app.kubernetes.io/name: {{ include "biosamples-curation.name" . }} + app.kubernetes.io/name: {{ include "biosamples-v4-pipelines.name" . }} spec: entrypoint: {{ .Values.job.name }} + parallelism: 1 + ttlStrategy: secondsAfterSuccess: {{ .Values.workflow.ttlStrategy.secondsAfterSuccess }} secondsAfterFailure: {{ .Values.workflow.ttlStrategy.secondsAfterFailure }} From c7d30639f839a27245d348d9bd1314f6133fafdd Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 24 Nov 2025 12:20:48 +0000 Subject: [PATCH 62/86] test: argo workflow cicd --- k8s/cronjobs/templates/workflow.yaml | 6 ++++++ k8s/cronjobs/values.yaml | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index 3993e350f..e51c82bcc 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -8,6 +8,12 @@ metadata: spec: entrypoint: {{ .Values.job.name }} + + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + parallelism: 1 ttlStrategy: diff --git a/k8s/cronjobs/values.yaml b/k8s/cronjobs/values.yaml index b9120c6f1..c06f96c77 100644 --- a/k8s/cronjobs/values.yaml +++ b/k8s/cronjobs/values.yaml @@ -20,8 +20,11 @@ workflow: secondsAfterSuccess: 3600 secondsAfterFailure: 7200 retryStrategy: - limit: 2 + limit: 1 backoff: duration: "10s" - factor: 2 + factor: 1 maxDuration: "1m" + +imagePullSecrets: + - name: __set-in-commandline__ From 867034cbd2b335e3d66903a8b9347b6c92f4b7c5 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 24 Nov 2025 13:15:08 +0000 Subject: [PATCH 63/86] test: argo workflow cicd --- k8s/cronjobs/templates/workflow.yaml | 4 ++++ .../src/main/java/uk/ac/ebi/biosamples/Application.java | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index e51c82bcc..b6cbb77a4 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -33,6 +33,10 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SPRING_CONFIG_LOCATION + value: /config/application.properties + command: {{- toYaml .Values.job.command | nindent 10 }} diff --git a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/Application.java index 83dd6af29..cd31b9c5d 100644 --- a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -23,6 +23,7 @@ import org.apache.http.message.BasicHeaderElementIterator; import org.apache.http.protocol.HTTP; import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.boot.web.client.RestTemplateCustomizer; @@ -33,6 +34,7 @@ import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.configuration.ExclusionConfiguration; import uk.ac.ebi.biosamples.service.EnaConfig; @@ -49,9 +51,13 @@ }) @Import(ExclusionConfiguration.class) @EnableCaching +@EnableWebSecurity public class Application { public static void main(final String[] args) { - final ConfigurableApplicationContext ctx = SpringApplication.run(Application.class, args); + SpringApplication app = new SpringApplication(Application.class); + app.setWebApplicationType(WebApplicationType.NONE); + + final ConfigurableApplicationContext ctx = app.run(args); PipelineUtils.exitPipeline(ctx); } From bc5b2449940a4d5ea18fd60aefe1d25dbd41fd8c Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 24 Nov 2025 13:51:16 +0000 Subject: [PATCH 64/86] test: argo workflow cicd --- k8s/cronjobs/Chart.yaml | 2 +- k8s/cronjobs/templates/_helpers.tpl | 18 +++++++++--------- k8s/cronjobs/templates/workflow.yaml | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/k8s/cronjobs/Chart.yaml b/k8s/cronjobs/Chart.yaml index 8542fcd16..52fe09716 100644 --- a/k8s/cronjobs/Chart.yaml +++ b/k8s/cronjobs/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: biosamples-v4-pipelines +name: bsd-cronjobs description: Run a Spring Boot CLI job via Argo Workflow type: application version: 0.1.0 diff --git a/k8s/cronjobs/templates/_helpers.tpl b/k8s/cronjobs/templates/_helpers.tpl index 39ee73965..5a2e520ec 100644 --- a/k8s/cronjobs/templates/_helpers.tpl +++ b/k8s/cronjobs/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "biosamples-v4-pipelines.name" -}} +{{- define "bsd-cronjobs.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "biosamples-v4-pipelines.fullname" -}} +{{- define "bsd-cronjobs.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "biosamples-v4-pipelines.chart" -}} +{{- define "bsd-cronjobs.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "biosamples-v4-pipelines.labels" -}} +{{- define "bsd-cronjobs.labels" -}} helm.sh/chart: {{ include "helm.chart" . }} -{{ include "biosamples-v4-pipelines.selectorLabels" . }} +{{ include "bsd-cronjobs.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,17 +45,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "biosamples-v4-pipelines.selectorLabels" -}} -app.kubernetes.io/name: {{ include "biosamples-v4-pipelines.name" . }} +{{- define "bsd-cronjobs.selectorLabels" -}} +app.kubernetes.io/name: {{ include "bsd-cronjobs.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* Create the name of the service account to use */}} -{{- define "biosamples-v4-pipelines.serviceAccountName" -}} +{{- define "bsd-cronjobs.serviceAccountName" -}} {{- if .Values.serviceAccount.create }} -{{- default (include "biosamples-v4-pipelines.fullname" .) .Values.serviceAccount.name }} +{{- default (include "bsd-cronjobs.fullname" .) .Values.serviceAccount.name }} {{- else }} {{- default "default" .Values.serviceAccount.name }} {{- end }} diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index b6cbb77a4..8b9b3dbe7 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -1,10 +1,10 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - name: {{ include "biosamples-v4-pipelines.fullname" . }} + name: {{ include "bsd-cronjobs.fullname" . }} namespace: {{ .Values.workflow.namespace }} labels: - app.kubernetes.io/name: {{ include "biosamples-v4-pipelines.name" . }} + app.kubernetes.io/name: {{ include "bsd-cronjobs.name" . }} spec: entrypoint: {{ .Values.job.name }} From d05328311fa635d79c7337d1ca3bc11f69f964c2 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 1 Dec 2025 09:14:48 +0000 Subject: [PATCH 65/86] test: argo workflow cicd --- k8s/argo-workflow.yaml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 k8s/argo-workflow.yaml diff --git a/k8s/argo-workflow.yaml b/k8s/argo-workflow.yaml deleted file mode 100644 index 52c0f7e92..000000000 --- a/k8s/argo-workflow.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: springboot-cli- -spec: - entrypoint: run-job - - templates: - - name: run-job - container: - image: registry.gitlab.com/YOUR-PROJECT/YOUR-REPO:latest - command: ["java", "-jar", "/app/app.jar"] - resources: - limits: - cpu: "500m" - memory: "1Gi" \ No newline at end of file From 0d1023d1617eaed0041f7e44c31d66ddbca6ae2b Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 1 Dec 2025 09:39:10 +0000 Subject: [PATCH 66/86] test: argo workflow cicd --- k8s/cronjobs/templates/workflow.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index 8b9b3dbe7..f68c8670c 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -33,6 +33,9 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + imagePullSecrets: + name: docker-registry-secret + env: - name: SPRING_CONFIG_LOCATION value: /config/application.properties From a2b45e3d75a56955eb81ec60f13720db6dc7625e Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 1 Dec 2025 09:39:29 +0000 Subject: [PATCH 67/86] test: argo workflow cicd --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5f243984e..0a16742a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -247,7 +247,7 @@ deploy_cronjobs_k8s_primary_dev: --docker-username=$CI_REGISTRY_USER \ --docker-password=$CI_REGISTRY_PASSWORD - | - helm upgrade --install curation ./k8s/cronjobs \ + helm upgrade --install pipelines-curation ./k8s/cronjobs \ --values ./k8s/cronjobs/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ --set image.repository=$CI_REGISTRY_IMAGE/pipelines-curation \ From 3c75dacc022efb9447340172110f66ef4da1180b Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 1 Dec 2025 10:12:05 +0000 Subject: [PATCH 68/86] test: argo workflow cicd --- k8s/cronjobs/Chart.yaml | 2 +- k8s/cronjobs/templates/_helpers.tpl | 18 +++++++++--------- k8s/cronjobs/templates/workflow.yaml | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/k8s/cronjobs/Chart.yaml b/k8s/cronjobs/Chart.yaml index 52fe09716..8542fcd16 100644 --- a/k8s/cronjobs/Chart.yaml +++ b/k8s/cronjobs/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: bsd-cronjobs +name: biosamples-v4-pipelines description: Run a Spring Boot CLI job via Argo Workflow type: application version: 0.1.0 diff --git a/k8s/cronjobs/templates/_helpers.tpl b/k8s/cronjobs/templates/_helpers.tpl index 5a2e520ec..39ee73965 100644 --- a/k8s/cronjobs/templates/_helpers.tpl +++ b/k8s/cronjobs/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "bsd-cronjobs.name" -}} +{{- define "biosamples-v4-pipelines.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "bsd-cronjobs.fullname" -}} +{{- define "biosamples-v4-pipelines.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "bsd-cronjobs.chart" -}} +{{- define "biosamples-v4-pipelines.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "bsd-cronjobs.labels" -}} +{{- define "biosamples-v4-pipelines.labels" -}} helm.sh/chart: {{ include "helm.chart" . }} -{{ include "bsd-cronjobs.selectorLabels" . }} +{{ include "biosamples-v4-pipelines.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,17 +45,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "bsd-cronjobs.selectorLabels" -}} -app.kubernetes.io/name: {{ include "bsd-cronjobs.name" . }} +{{- define "biosamples-v4-pipelines.selectorLabels" -}} +app.kubernetes.io/name: {{ include "biosamples-v4-pipelines.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* Create the name of the service account to use */}} -{{- define "bsd-cronjobs.serviceAccountName" -}} +{{- define "biosamples-v4-pipelines.serviceAccountName" -}} {{- if .Values.serviceAccount.create }} -{{- default (include "bsd-cronjobs.fullname" .) .Values.serviceAccount.name }} +{{- default (include "biosamples-v4-pipelines.fullname" .) .Values.serviceAccount.name }} {{- else }} {{- default "default" .Values.serviceAccount.name }} {{- end }} diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index f68c8670c..f555665d2 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -1,10 +1,10 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - name: {{ include "bsd-cronjobs.fullname" . }} + name: {{ include "biosamples-v4-pipelines.fullname" . }} namespace: {{ .Values.workflow.namespace }} labels: - app.kubernetes.io/name: {{ include "bsd-cronjobs.name" . }} + app.kubernetes.io/name: {{ include "biosamples-v4-pipelines.name" . }} spec: entrypoint: {{ .Values.job.name }} From a2a7d31c3b19b93aa6a6ef3a9c8f05ec0e4fcb33 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 1 Dec 2025 10:51:48 +0000 Subject: [PATCH 69/86] test: argo workflow cicd --- .gitlab-ci.yml | 2 +- k8s/cronjobs/templates/workflow.yaml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a16742a3..e594f36c4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -245,7 +245,7 @@ deploy_cronjobs_k8s_primary_dev: kubectl create secret docker-registry $DOCKER_PULL_SECRET \ --docker-server=$CI_REGISTRY \ --docker-username=$CI_REGISTRY_USER \ - --docker-password=$CI_REGISTRY_PASSWORD + --docker-password=$CI_JOB_TOKEN - | helm upgrade --install pipelines-curation ./k8s/cronjobs \ --values ./k8s/cronjobs/values-${ENVIRONMENT_NAME}.yaml \ diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index f555665d2..b6cbb77a4 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -33,9 +33,6 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - imagePullSecrets: - name: docker-registry-secret - env: - name: SPRING_CONFIG_LOCATION value: /config/application.properties From 0376dab12ba96af50ca0b358aed724ce4c1192b1 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Wed, 3 Dec 2025 11:00:37 +0000 Subject: [PATCH 70/86] test: argo workflow cicd --- k8s/cronjobs/templates/workflow.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index b6cbb77a4..e51c82bcc 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -33,10 +33,6 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: SPRING_CONFIG_LOCATION - value: /config/application.properties - command: {{- toYaml .Values.job.command | nindent 10 }} From c2f8fffd9cee6a8dd3ff5e20eead185274a3b7b5 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Thu, 4 Dec 2025 04:57:59 +0000 Subject: [PATCH 71/86] test: argo workflow cicd --- .gitlab-ci.yml | 8 +------- k8s/cronjobs/templates/workflow.yaml | 4 ++++ k8s/cronjobs/values.yaml | 9 ++++----- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e594f36c4..e95589090 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -240,15 +240,9 @@ deploy_cronjobs_k8s_primary_dev: script: - echo $K8S_NAMESPACE - kubectl config set-context --current --namespace=${K8S_NAMESPACE} - - kubectl delete secret $DOCKER_PULL_SECRET || true - - | - kubectl create secret docker-registry $DOCKER_PULL_SECRET \ - --docker-server=$CI_REGISTRY \ - --docker-username=$CI_REGISTRY_USER \ - --docker-password=$CI_JOB_TOKEN - | helm upgrade --install pipelines-curation ./k8s/cronjobs \ --values ./k8s/cronjobs/values-${ENVIRONMENT_NAME}.yaml \ - --set "imagePullSecrets[0].name=$DOCKER_PULL_SECRET" \ + --set "imagePullSecrets[0].name=gitlab-argo-secret" \ --set image.repository=$CI_REGISTRY_IMAGE/pipelines-curation \ --set image.tag=$DOCKER_TAG diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index e51c82bcc..b6cbb77a4 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -33,6 +33,10 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SPRING_CONFIG_LOCATION + value: /config/application.properties + command: {{- toYaml .Values.job.command | nindent 10 }} diff --git a/k8s/cronjobs/values.yaml b/k8s/cronjobs/values.yaml index c06f96c77..add006e8a 100644 --- a/k8s/cronjobs/values.yaml +++ b/k8s/cronjobs/values.yaml @@ -1,7 +1,9 @@ image: - repository: "" - tag: "" + repository: __set-in-commandline__ pullPolicy: IfNotPresent + tag: latest +imagePullSecrets: + - name: __set-in-commandline__ job: name: biosamples-curation @@ -25,6 +27,3 @@ workflow: duration: "10s" factor: 1 maxDuration: "1m" - -imagePullSecrets: - - name: __set-in-commandline__ From 73f5bfd4dae81b3ea88c84b167387b72cabec570 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Thu, 4 Dec 2025 05:53:56 +0000 Subject: [PATCH 72/86] test: argo workflow cicd --- k8s/cronjobs/templates/workflow.yaml | 7 +++++++ k8s/cronjobs/values.yaml | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index b6cbb77a4..3ea956d1f 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -45,3 +45,10 @@ spec: resources: {{- toYaml .Values.job.resources | nindent 10 }} + + volumeMounts: + {{- toYaml .Values.volumeMounts | nindent 10 }} + + volumes: + {{- toYaml .Values.volumes | nindent 8 }} + diff --git a/k8s/cronjobs/values.yaml b/k8s/cronjobs/values.yaml index add006e8a..bf0bb544b 100644 --- a/k8s/cronjobs/values.yaml +++ b/k8s/cronjobs/values.yaml @@ -27,3 +27,13 @@ workflow: duration: "10s" factor: 1 maxDuration: "1m" + +volumes: + - name: config-volume + configMap: + name: application-properties + +volumeMounts: + - name: config-volume + mountPath: /config/application.properties + subPath: application.properties From 4f6a3b2df468baa6000b35c1ec5daf2b1529f0b5 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Mon, 8 Dec 2025 07:30:11 +0000 Subject: [PATCH 73/86] test: pipelines dag --- .gitlab-ci.yml | 4 +- k8s/cronjobs/Chart.yaml | 4 +- k8s/cronjobs/templates/_helpers.tpl | 18 ++++---- k8s/cronjobs/templates/workflow.yaml | 69 +++++++++++++++++++++------- k8s/cronjobs/values.yaml | 3 +- 5 files changed, 68 insertions(+), 30 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e95589090..cc41fe0fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -241,8 +241,8 @@ deploy_cronjobs_k8s_primary_dev: - echo $K8S_NAMESPACE - kubectl config set-context --current --namespace=${K8S_NAMESPACE} - | - helm upgrade --install pipelines-curation ./k8s/cronjobs \ + helm upgrade --install biosamples-pipeline ./k8s/cronjobs \ --values ./k8s/cronjobs/values-${ENVIRONMENT_NAME}.yaml \ --set "imagePullSecrets[0].name=gitlab-argo-secret" \ - --set image.repository=$CI_REGISTRY_IMAGE/pipelines-curation \ + --set image.repository=$CI_REGISTRY_IMAGE \ --set image.tag=$DOCKER_TAG diff --git a/k8s/cronjobs/Chart.yaml b/k8s/cronjobs/Chart.yaml index 8542fcd16..1322d750e 100644 --- a/k8s/cronjobs/Chart.yaml +++ b/k8s/cronjobs/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -name: biosamples-v4-pipelines -description: Run a Spring Boot CLI job via Argo Workflow +name: cronjobs +description: Argo workflow dag for pipelines type: application version: 0.1.0 appVersion: "1.0" diff --git a/k8s/cronjobs/templates/_helpers.tpl b/k8s/cronjobs/templates/_helpers.tpl index 39ee73965..57677a38f 100644 --- a/k8s/cronjobs/templates/_helpers.tpl +++ b/k8s/cronjobs/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "biosamples-v4-pipelines.name" -}} +{{- define "cronjobs.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "biosamples-v4-pipelines.fullname" -}} +{{- define "cronjobs.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "biosamples-v4-pipelines.chart" -}} +{{- define "cronjobs.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "biosamples-v4-pipelines.labels" -}} +{{- define "cronjobs.labels" -}} helm.sh/chart: {{ include "helm.chart" . }} -{{ include "biosamples-v4-pipelines.selectorLabels" . }} +{{ include "cronjobs.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,17 +45,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "biosamples-v4-pipelines.selectorLabels" -}} -app.kubernetes.io/name: {{ include "biosamples-v4-pipelines.name" . }} +{{- define "cronjobs.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cronjobs.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* Create the name of the service account to use */}} -{{- define "biosamples-v4-pipelines.serviceAccountName" -}} +{{- define "cronjobs.serviceAccountName" -}} {{- if .Values.serviceAccount.create }} -{{- default (include "biosamples-v4-pipelines.fullname" .) .Values.serviceAccount.name }} +{{- default (include "cronjobs.fullname" .) .Values.serviceAccount.name }} {{- else }} {{- default "default" .Values.serviceAccount.name }} {{- end }} diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index 3ea956d1f..313b3ef6d 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -1,54 +1,91 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - name: {{ include "biosamples-v4-pipelines.fullname" . }} + name: {{ include "cronjobs.fullname" . }} namespace: {{ .Values.workflow.namespace }} labels: - app.kubernetes.io/name: {{ include "biosamples-v4-pipelines.name" . }} + app.kubernetes.io/name: {{ include "cronjobs.name" . }} spec: entrypoint: {{ .Values.job.name }} - - {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - parallelism: 1 - ttlStrategy: secondsAfterSuccess: {{ .Values.workflow.ttlStrategy.secondsAfterSuccess }} secondsAfterFailure: {{ .Values.workflow.ttlStrategy.secondsAfterFailure }} templates: - name: {{ .Values.job.name }} + dag: + tasks: + - name: curation + template: curation + - name: curami + template: curami + dependencies: + - curation + - name: copydown + template: copydown + + - name: curation retryStrategy: limit: {{ .Values.workflow.retryStrategy.limit }} - backoff: - duration: {{ .Values.workflow.retryStrategy.backoff.duration }} - factor: {{ .Values.workflow.retryStrategy.backoff.factor }} - maxDuration: {{ .Values.workflow.retryStrategy.backoff.maxDuration }} - container: - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.image.repository }}/pipelines-curation:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - name: SPRING_CONFIG_LOCATION value: /config/application.properties - command: {{- toYaml .Values.job.command | nindent 10 }} - args: {{- toYaml .Values.job.args | nindent 10 }} - resources: {{- toYaml .Values.job.resources | nindent 10 }} + volumeMounts: + {{- toYaml .Values.volumeMounts | nindent 10 }} + volumes: + {{- toYaml .Values.volumes | nindent 8 }} + - name: curami + retryStrategy: + limit: {{ .Values.workflow.retryStrategy.limit }} + container: + image: "{{ .Values.image.repository }}/pipelines-curami:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SPRING_CONFIG_LOCATION + value: /config/application.properties + command: + {{- toYaml .Values.job.command | nindent 10 }} + args: + {{- toYaml .Values.job.args | nindent 10 }} + resources: + {{- toYaml .Values.job.resources | nindent 10 }} volumeMounts: {{- toYaml .Values.volumeMounts | nindent 10 }} + volumes: + {{- toYaml .Values.volumes | nindent 8 }} + - name: copydown + retryStrategy: + limit: {{ .Values.workflow.retryStrategy.limit }} + container: + image: "{{ .Values.image.repository }}/pipelines-copydown:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SPRING_CONFIG_LOCATION + value: /config/application.properties + command: + {{- toYaml .Values.job.command | nindent 10 }} + args: + {{- toYaml .Values.job.args | nindent 10 }} + resources: + {{- toYaml .Values.job.resources | nindent 10 }} + volumeMounts: + {{- toYaml .Values.volumeMounts | nindent 10 }} volumes: {{- toYaml .Values.volumes | nindent 8 }} diff --git a/k8s/cronjobs/values.yaml b/k8s/cronjobs/values.yaml index bf0bb544b..02b61e3bc 100644 --- a/k8s/cronjobs/values.yaml +++ b/k8s/cronjobs/values.yaml @@ -1,4 +1,5 @@ image: + repositoryBase: __set repository: __set-in-commandline__ pullPolicy: IfNotPresent tag: latest @@ -6,7 +7,7 @@ imagePullSecrets: - name: __set-in-commandline__ job: - name: biosamples-curation + name: pipelines command: ["java", "-jar", "/app/app.jar"] args: [] resources: From 41e3e71b429e32f9582df362f9f86815ba5bc2c7 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Wed, 10 Dec 2025 09:34:10 +0000 Subject: [PATCH 74/86] test: argo dag --- .gitlab-ci.yml | 6 ++++++ k8s/Dockerfile | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cc41fe0fa..3482b2934 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,6 +35,8 @@ maven-package-all-apps: - agents/uploadworkers/target/agents-uploadworkers-*.jar - pipelines/reindex/target/pipelines-reindex-*.jar - pipelines/curation/target/pipelines-curation-*.jar + - pipelines/curation/target/pipelines-curami-*.jar + - pipelines/curation/target/pipelines-copydown-*.jar build_and_push_docker_images: @@ -56,6 +58,10 @@ build_and_push_docker_images: DOCKERFILE_TARGET: "pipelines-reindex" - APP_NAME: "pipelines-curation" DOCKERFILE_TARGET: "pipelines-curation" + - APP_NAME: "pipelines-curami" + DOCKERFILE_TARGET: "pipelines-curami" + - APP_NAME: "pipelines-copydown" + DOCKERFILE_TARGET: "pipelines-copydown" script: - | DOCKER_IMAGE_NAME="$CI_REGISTRY_IMAGE/$APP_NAME:$DOCKER_TAG" diff --git a/k8s/Dockerfile b/k8s/Dockerfile index 3c9be8439..cc21ef98d 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -23,3 +23,11 @@ CMD ["java", "-jar", "app.jar"] FROM base as pipelines-curation COPY pipelines/curation/target/pipelines-curation-*.jar app.jar CMD ["java", "-jar", "app.jar"] + +FROM base as pipelines-curami +COPY pipelines/curami/target/pipelines-curami-*.jar app.jar +CMD ["java", "-jar", "app.jar"] + +FROM base as pipelines-copydown +COPY pipelines/copydown/target/pipelines-copydown-*.jar app.jar +CMD ["java", "-jar", "app.jar"] From 316aba9ea59365f7509b8b68b6a447623a3d44dc Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Thu, 11 Dec 2025 09:26:51 +0000 Subject: [PATCH 75/86] chore: add search repo image building step to readme --- README.adoc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.adoc b/README.adoc index 3340abe54..9d8152747 100644 --- a/README.adoc +++ b/README.adoc @@ -63,6 +63,12 @@ git clone https://github.com/EBIBioSamples/biosamples-v4.git cd biosamples-v4 ./mvnw -T 2C package ---- +. Clone and build `biosamples-search` image ++ +[source,sh] +---- +./build-biosamples-search-image.sh +---- . Start BioSamples on your machine + [source,sh] From ae667ed1dfdf01ac266a277c36915eea1d9c3025 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Thu, 11 Dec 2025 09:28:07 +0000 Subject: [PATCH 76/86] chore: add .env file with local docker config --- .env | 9 +++++++++ .gitignore | 1 - 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 000000000..6b414b86b --- /dev/null +++ b/.env @@ -0,0 +1,9 @@ +ELASTIC_PASSWORD=elastic +KIBANA_PASSWORD=kibana +STACK_VERSION=8.17.2 +CLUSTER_NAME=docker-cluster +LICENSE=basic +ES_PORT=9200 +KIBANA_PORT=5601 +# Increase or decrease based on the available host memory (in bytes) +MEM_LIMIT=1073741824 \ No newline at end of file diff --git a/.gitignore b/.gitignore index af107fda5..827c3e57e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,6 @@ bin/ target/ export/ temp/ -.env set_java_17.sh # Scripts From c33469fa095c3790f10537ceae51f49f9c3776ae Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Thu, 11 Dec 2025 10:58:11 +0000 Subject: [PATCH 77/86] test: argo dag --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3482b2934..9534f67fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,8 +35,8 @@ maven-package-all-apps: - agents/uploadworkers/target/agents-uploadworkers-*.jar - pipelines/reindex/target/pipelines-reindex-*.jar - pipelines/curation/target/pipelines-curation-*.jar - - pipelines/curation/target/pipelines-curami-*.jar - - pipelines/curation/target/pipelines-copydown-*.jar + - pipelines/curami/target/pipelines-curami-*.jar + - pipelines/copydown/target/pipelines-copydown-*.jar build_and_push_docker_images: From 0aecfbb3d93b5f96eafc089b6fb3be45f7d985d5 Mon Sep 17 00:00:00 2001 From: dgupta Date: Thu, 11 Dec 2025 14:53:17 +0000 Subject: [PATCH 78/86] logback added and spotless --- .gitlab-ci.yml | 1 - .../uk/ac/ebi/biosamples/Application.java | 14 +- .../FileUploadSubmissionService.java | 5 +- core/pom.xml | 1 - .../messaging/config/MessageConfig.java | 10 +- .../biosamples/service/MessagingService.java | 46 ++- .../ebi/biosamples/solr/model/SolrSample.java | 1 + .../solr/model/field/SolrSampleField.java | 4 +- .../solr/service/SolrFieldService.java | 8 +- .../solr/service/SolrSampleService.java | 5 +- docker-compose.yml | 4 +- docker-debug.sh | 3 +- docker-webapp.sh | 3 +- es_index.json | 386 +++++++++--------- .../ebi/biosamples/RestFacetIntegration.java | 21 +- .../RestPrivateSampleIntegration.java | 1 - logback/pom.xml | 24 ++ logback/src/main/resources/logback-shared.xml | 31 ++ .../AnalyticsApplicationRunner.java | 2 +- .../HelpdeskActionApplicationRunner.java | 14 +- .../services/SampleStatusUpdater.java | 7 +- .../service/SampleCallbackResult.java | 0 .../service/SampleRetrievalResult.java | 3 +- pipelines/pom.xml | 2 +- .../uk/ac/ebi/biosamples/Application.java | 31 +- .../uk/ac/ebi/biosamples/ReindexRunner.java | 37 +- .../ac/ebi/biosamples/ReindexRunnerTest.java | 4 +- .../uk/ac/ebi/biosamples/Application.java | 11 +- pom.xml | 10 + webapps/core-v2/pom.xml | 5 + .../src/main/resources/logback-spring.xml | 8 + webapps/core/pom.xml | 25 +- .../controller/FileDownloadController.java | 3 +- .../controller/SampleFacetController.java | 5 +- .../biosamples/service/SamplePageService.java | 32 +- .../service/facet/ElasticFacetService.java | 73 +++- .../service/facet/FacetService.java | 15 +- .../service/facet/FacetingService.java | 15 +- .../service/facet/SearchFacetMapper.java | 76 ++-- .../service/facet/SolrFacetService.java | 16 +- .../service/search/ElasticSearchService.java | 67 ++- .../service/search/SearchAfterPage.java | 18 +- .../service/search/SearchFilterMapper.java | 114 ++++-- .../service/search/SearchService.java | 19 +- .../service/search/SolrSearchService.java | 32 +- .../src/main/resources/logback-spring.xml | 8 + 46 files changed, 760 insertions(+), 460 deletions(-) create mode 100644 logback/pom.xml create mode 100644 logback/src/main/resources/logback-shared.xml delete mode 100644 pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/SampleCallbackResult.java create mode 100644 webapps/core-v2/src/main/resources/logback-spring.xml create mode 100644 webapps/core/src/main/resources/logback-spring.xml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e95589090..5901574e4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,7 +36,6 @@ maven-package-all-apps: - pipelines/reindex/target/pipelines-reindex-*.jar - pipelines/curation/target/pipelines-curation-*.jar - build_and_push_docker_images: stage: package image: docker:stable diff --git a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/Application.java b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/Application.java index 52acfb017..d2c20d4b0 100644 --- a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -12,11 +12,9 @@ import org.apache.http.HeaderElement; import org.apache.http.HeaderElementIterator; -import org.apache.http.HttpHost; import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; import org.apache.http.conn.ConnectionKeepAliveStrategy; -import org.apache.http.conn.routing.HttpRoute; import org.apache.http.impl.client.cache.CacheConfig; import org.apache.http.impl.client.cache.CachingHttpClientBuilder; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; @@ -68,8 +66,8 @@ public MongoAccessionService mongoSampleAccessionService( mongoOperations); } - - // todo I Had to add restTemplate bean as a temporary workaround as there seems to be a problem with dependencies after refactor. + // todo I Had to add restTemplate bean as a temporary workaround as there seems to be a problem + // with dependencies after refactor. // We need to sort out dependency problem and remove this unused dependency. @Bean @@ -80,12 +78,11 @@ public RestTemplate restTemplate(final RestTemplateCustomizer restTemplateCustom } @Bean - public RestTemplateCustomizer restTemplateCustomizer(final BioSamplesProperties bioSamplesProperties) { + public RestTemplateCustomizer restTemplateCustomizer( + final BioSamplesProperties bioSamplesProperties) { return restTemplate -> { - final ConnectionKeepAliveStrategy keepAliveStrategy = (response, context) -> { - final HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); while (it.hasNext()) { @@ -115,8 +112,7 @@ public RestTemplateCustomizer restTemplateCustomizer(final BioSamplesProperties final RequestConfig config = RequestConfig.custom() .setConnectTimeout(timeout * 1000) - .setConnectionRequestTimeout( - timeout * 1000) + .setConnectionRequestTimeout(timeout * 1000) .setSocketTimeout(timeout * 1000) .build(); final HttpClient httpClient = diff --git a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java index 6994209de..8c41cfe6a 100644 --- a/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java +++ b/agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java @@ -64,10 +64,11 @@ public void receiveMessageFromBioSamplesFileUploaderQueue(final String mongoFile private void handleMessage(String submissionId) { submissionId = submissionId.replace("\"", ""); - final Optional fileUploadOptional = mongoFileUploadRepository.findById(submissionId); + final Optional fileUploadOptional = + mongoFileUploadRepository.findById(submissionId); if (fileUploadOptional.isEmpty()) { log.error("Could not find file upload record for submissionId: {}", submissionId); - //todo here exception means there is no progress from the queue reading loop. + // todo here exception means there is no progress from the queue reading loop. // We can send this to dead letter or something for monitoring. return; } diff --git a/core/pom.xml b/core/pom.xml index ad2e8b211..941133725 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -53,7 +53,6 @@ org.springframework.boot spring-boot-starter-test - 1.5.15.RELEASE test diff --git a/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java b/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java index 5780e161b..b65b8e3f3 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/messaging/config/MessageConfig.java @@ -12,8 +12,6 @@ import org.springframework.amqp.core.*; import org.springframework.amqp.rabbit.annotation.EnableRabbit; -import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter; -import org.springframework.amqp.support.converter.MessageConverter; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import uk.ac.ebi.biosamples.messaging.MessagingConstants; @@ -77,8 +75,8 @@ public Binding uploadBinding() { // enable messaging in json // note that this class is not the same as the http MessageConverter class -// @Bean -// public MessageConverter getJackson2MessageConverter() { -// return new Jackson2JsonMessageConverter(); -// } + // @Bean + // public MessageConverter getJackson2MessageConverter() { + // return new Jackson2JsonMessageConverter(); + // } } diff --git a/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java b/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java index 11489290a..b92ae0bf6 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/service/MessagingService.java @@ -10,26 +10,22 @@ */ package uk.ac.ebi.biosamples.service; -import java.nio.charset.StandardCharsets; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.amqp.core.AmqpTemplate; -import org.springframework.amqp.core.Message; import org.springframework.stereotype.Service; import uk.ac.ebi.biosamples.core.model.CurationLink; import uk.ac.ebi.biosamples.core.model.Relationship; import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.messaging.MessagingConstants; -import uk.ac.ebi.biosamples.messaging.model.MessageContent; import uk.ac.ebi.biosamples.mongo.service.SampleReadService; @Service @@ -39,7 +35,8 @@ public class MessagingService { private final AmqpTemplate amqpTemplate; private final ObjectMapper objectMapper; - public MessagingService(SampleReadService sampleReadService, AmqpTemplate amqpTemplate, ObjectMapper objectMapper) { + public MessagingService( + SampleReadService sampleReadService, AmqpTemplate amqpTemplate, ObjectMapper objectMapper) { this.sampleReadService = sampleReadService; this.amqpTemplate = amqpTemplate; this.objectMapper = objectMapper; @@ -76,28 +73,33 @@ void fetchThenSendMessage( updateInverseRelationships(sample.get(), existingRelationshipTargets); // send the original sample with the extras as related samples -// amqpTemplate.convertAndSend( -// MessagingConstants.INDEXING_EXCHANGE, -// MessagingConstants.INDEXING_QUEUE, -// MessageContent.build(sample.get(), null, related, false)); + // amqpTemplate.convertAndSend( + // MessagingConstants.INDEXING_EXCHANGE, + // MessagingConstants.INDEXING_QUEUE, + // MessageContent.build(sample.get(), null, related, false)); try { String json = objectMapper.writeValueAsString(sample.get()); log.info("Sending message for indexing: {}", sample.get().getAccession()); -// amqpTemplate.send(MessagingConstants.INDEXING_EXCHANGE, MessagingConstants.INDEXING_QUEUE, new Message(json.getBytes(StandardCharsets.UTF_8))); - amqpTemplate.convertAndSend(MessagingConstants.INDEXING_EXCHANGE, MessagingConstants.INDEXING_QUEUE, json); - related.forEach(s -> { - try { - amqpTemplate.convertAndSend(MessagingConstants.INDEXING_EXCHANGE, MessagingConstants.INDEXING_QUEUE, objectMapper.writeValueAsString(s)); - } catch (JsonProcessingException e) { - log.error("Failed to convert sample to JSON: {}", s.getAccession(), e); -// throw new RuntimeException(e); - } - }); + // amqpTemplate.send(MessagingConstants.INDEXING_EXCHANGE, + // MessagingConstants.INDEXING_QUEUE, new Message(json.getBytes(StandardCharsets.UTF_8))); + amqpTemplate.convertAndSend( + MessagingConstants.INDEXING_EXCHANGE, MessagingConstants.INDEXING_QUEUE, json); + related.forEach( + s -> { + try { + amqpTemplate.convertAndSend( + MessagingConstants.INDEXING_EXCHANGE, + MessagingConstants.INDEXING_QUEUE, + objectMapper.writeValueAsString(s)); + } catch (JsonProcessingException e) { + log.error("Failed to convert sample to JSON: {}", s.getAccession(), e); + // throw new RuntimeException(e); + } + }); } catch (Exception e) { log.error("Failed to convert sample to JSON: {}", sample.get().getAccession(), e); } - } } diff --git a/core/src/main/java/uk/ac/ebi/biosamples/solr/model/SolrSample.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/SolrSample.java index b5f82c214..0b48055d2 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/solr/model/SolrSample.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/SolrSample.java @@ -46,6 +46,7 @@ public class SolrSample { */ @Indexed(name = "release_dt", required = true, type = "date") protected String release; + /** * Store the update date as a string so that it can be used easily by solr Use a TrieDate type for * better range query performance diff --git a/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleField.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleField.java index 162ac5d6f..5a1061a17 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleField.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/model/field/SolrSampleField.java @@ -118,7 +118,9 @@ public Facet.Builder getFacetBuilder(final String facetLabel, final Long facetCo */ public abstract FacetFetchStrategy getFacetCollectionStrategy(); - /** @return the readable label of the field */ + /** + * @return the readable label of the field + */ public String getReadableLabel() { return readableLabel; } diff --git a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFieldService.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFieldService.java index cd4e70ce0..a4bab23e7 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFieldService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrFieldService.java @@ -129,7 +129,9 @@ public SolrSampleField getNewFieldInstance( final Class prototype, final String baseLabel, final String encodedLabel) - throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, + throws NoSuchMethodException, + IllegalAccessException, + InvocationTargetException, InstantiationException { return prototype .getConstructor(String.class, String.class) @@ -138,7 +140,9 @@ public SolrSampleField getNewFieldInstance( public SolrSampleField getNewFieldInstance( final Class prototype, final String baseLabel) - throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, + throws NoSuchMethodException, + IllegalAccessException, + InvocationTargetException, InstantiationException { return prototype.getConstructor(String.class).newInstance(baseLabel); } diff --git a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java index 38780c569..f3e3abc6d 100644 --- a/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java +++ b/core/src/main/java/uk/ac/ebi/biosamples/solr/service/SolrSampleService.java @@ -60,7 +60,10 @@ public Page fetchSolrSampleByText( // solr degrades with high page and size params, use cursor instead if (pageable.getPageNumber() > 500 || pageable.getPageSize() > 200) { - log.warn("Max page number/size exceeded, returning error to the user. Number: {}, size: {}", pageable.getPageNumber(), pageable.getPageSize()); + log.warn( + "Max page number/size exceeded, returning error to the user. Number: {}, size: {}", + pageable.getPageNumber(), + pageable.getPageSize()); throw new GlobalExceptions.PaginationException(); } diff --git a/docker-compose.yml b/docker-compose.yml index 8dee24de7..b4a20ccca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -358,6 +358,7 @@ services: links: - biosamples-webapps-core - neo4j + - mongo command: - java - -jar @@ -368,6 +369,7 @@ services: - LOGGING_FILE=/logs/integration.log - BIOSAMPLES_LEGACYAPIKEY=fooqwerty - BIOSAMPLES_NEO_URL=bolt://neo4j:7687 + - spring.data.mongodb.uri=mongodb://mongo:27017/biosamples json-schema-validator: image: quay.io/ebi-ait/biovalidator:2.0.1 @@ -481,7 +483,7 @@ services: - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - bootstrap.memory_lock=true - xpack.security.http.ssl.enabled=false - mem_limit: ${MEM_LIMIT} + mem_limit: 1g ulimits: memlock: soft: -1 diff --git a/docker-debug.sh b/docker-debug.sh index 6f1229a1d..79759ad1f 100755 --- a/docker-debug.sh +++ b/docker-debug.sh @@ -41,7 +41,8 @@ echo "checking mongo is up" curl http://localhost:8983/solr/samples/config -H 'Content-type:application/json' -d'{"set-property" : {"updateHandler.autoCommit.maxTime":5000, "updateHandler.autoCommit.openSearcher":"true", "query.documentCache.size":1024, "query.filterCache.size":1024, "query.filterCache.autowarmCount":128, "query.queryResultCache.size":1024, "query.queryResultCache.autowarmCount":128}}' #profile any queries that take longer than 100 ms -docker-compose run --rm mongo mongo --eval 'db.setProfilingLevel(1)' mongo:27017/biosamples +#don't use run, spins up a new container, use eval to use existing container +docker-compose exec mongo mongo biosamples --eval 'db.setProfilingLevel(1)' docker-compose -f docker-compose.yml -f docker-compose.debug.override.yml -f docker-compose.override.yml up -d biosamples-webapps-core diff --git a/docker-webapp.sh b/docker-webapp.sh index 12432132f..45220866e 100755 --- a/docker-webapp.sh +++ b/docker-webapp.sh @@ -51,7 +51,8 @@ echo "checking neo4j is up" #profile any queries that take longer than 100 ms -docker-compose run --rm mongo mongo --eval 'db.setProfilingLevel(1)' mongo:27017/biosamples +#don't use run, spins up a new container, use eval to use existing container +docker-compose exec mongo mongo biosamples --eval 'db.setProfilingLevel(1)' until curl -s http://localhost:9200 | grep -q "missing authentication credentials"; do sleep 30; done; # create ES index diff --git a/es_index.json b/es_index.json index 9f5d16314..0bb7591ca 100644 --- a/es_index.json +++ b/es_index.json @@ -1,194 +1,194 @@ { - "settings": { - "number_of_shards": 1, - "number_of_replicas": 0 - }, - "mappings": { - "properties": { - "sample_full_text": { - "type": "text" - }, - "_class": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "accession": { - "type": "text", - "copy_to": "sample_full_text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "characteristics": { - "type": "nested", - "properties": { - "key": { - "type": "text", - "copy_to": "sample_full_text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "value": { - "type": "text", - "copy_to": "sample_full_text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "create": { - "type": "date" - }, - "domain": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "externalReferences": { - "type": "nested", - "properties": { - "url": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "name": { - "type": "text", - "copy_to": "sample_full_text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "publications": { - "type": "nested", - "properties": { - "pubmed_id": { - "type": "text", - "copy_to": "sample_full_text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "relationships": { - "type": "nested", - "properties": { - "source": { - "type": "text", - "copy_to": "sample_full_text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "target": { - "type": "text", - "copy_to": "sample_full_text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "type": { - "type": "text", - "copy_to": "sample_full_text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "release": { - "type": "date" - }, - "sraAccession": { - "type": "text", - "copy_to": "sample_full_text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "status": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "submitted": { - "type": "date" - }, - "submittedVia": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "taxId": { - "type": "long", - "copy_to": "sample_full_text" - }, - "update": { - "type": "date" - }, - "webinSubmissionAccountId": { - "type": "text", - "copy_to": "sample_full_text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } -} \ No newline at end of file +"settings": { + "number_of_shards": 1, + "number_of_replicas": 0 +}, +"mappings": { + "properties": { + "sample_full_text": { + "type": "text" + }, + "_class": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "accession": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "characteristics": { + "type": "nested", + "properties": { + "key": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "value": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "create": { + "type": "date" + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "externalReferences": { + "type": "nested", + "properties": { + "url": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "name": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "publications": { + "type": "nested", + "properties": { + "pubmed_id": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "relationships": { + "type": "nested", + "properties": { + "source": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "target": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "type": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "release": { + "type": "date" + }, + "sraAccession": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "status": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "submitted": { + "type": "date" + }, + "submittedVia": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "taxId": { + "type": "long", + "copy_to": "sample_full_text" + }, + "update": { + "type": "date" + }, + "webinSubmissionAccountId": { + "type": "text", + "copy_to": "sample_full_text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } +} +} diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java index af076f098..6aa4e6a4a 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java @@ -276,10 +276,29 @@ private Sample getArrayExpressSampleTest() { private void facetEndpointShouldReturnAllFacetValuesWhenFacetFilterIsAvailable() { String url = clientProperties.getBiosamplesClientUri() + "/samples/facets?facet=SRA accession"; + log.info("Calling facet endpoint URL: {}", url); try { ResponseEntity response = restTemplate.getForEntity(url, String.class); + log.info("Response status: {}", response.getStatusCode()); + log.info("Response body: {}", response.getBody()); + JsonNode node = objectMapper.readTree(response.getBody()); - JsonNode facets = node.get("_embedded").get("facets"); + JsonNode embedded = node.get("_embedded"); + if (embedded == null) { + log.error("Response does not contain '_embedded' field. Full response: {}", response.getBody()); + throw new IntegrationTestFailException( + "Facet endpoint response does not contain '_embedded' field. Response: " + response.getBody(), + Phase.SIX); + } + + JsonNode facets = embedded.get("facets"); + if (facets == null) { + log.error("Response does not contain 'facets' field. '_embedded' content: {}", embedded.toString()); + throw new IntegrationTestFailException( + "Facet endpoint response does not contain 'facets' field. Response: " + response.getBody(), + Phase.SIX); + } + for (JsonNode facet : facets) { if ("SRA accession".equals(facet.get("label").asText())) { if (facet.get("content").size() < 10) { diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java index 2e8601082..81ac2d654 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestPrivateSampleIntegration.java @@ -14,7 +14,6 @@ import java.util.Optional; import java.util.SortedSet; import java.util.TreeSet; - import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import uk.ac.ebi.biosamples.client.BioSamplesClient; diff --git a/logback/pom.xml b/logback/pom.xml new file mode 100644 index 000000000..9de50b007 --- /dev/null +++ b/logback/pom.xml @@ -0,0 +1,24 @@ + + 4.0.0 + logback + jar + + + uk.ac.ebi.biosamples + biosamples + 5.3.15-SNAPSHOT + ../ + + + + + org.springframework.boot + spring-boot-starter + + + ch.qos.logback + logback-classic + + + diff --git a/logback/src/main/resources/logback-shared.xml b/logback/src/main/resources/logback-shared.xml new file mode 100644 index 000000000..db439d0c2 --- /dev/null +++ b/logback/src/main/resources/logback-shared.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + ${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) + %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} + %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}} + + + + + + + + + + + + + + + + + + diff --git a/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java b/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java index 872baf87c..c65b28794 100644 --- a/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java +++ b/pipelines/analytics/src/main/java/uk/ac/ebi/biosamples/AnalyticsApplicationRunner.java @@ -26,8 +26,8 @@ import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountEntry; import uk.ac.ebi.biosamples.core.model.facet.content.LabelCountListContent; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; -import uk.ac.ebi.biosamples.service.facet.FacetingService; import uk.ac.ebi.biosamples.service.SamplePageService; +import uk.ac.ebi.biosamples.service.facet.FacetingService; @Component public class AnalyticsApplicationRunner implements ApplicationRunner { diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/HelpdeskActionApplicationRunner.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/HelpdeskActionApplicationRunner.java index 18ad765d9..fe43f92c0 100644 --- a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/HelpdeskActionApplicationRunner.java +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/HelpdeskActionApplicationRunner.java @@ -76,11 +76,17 @@ public void run(ApplicationArguments args) { } case "changeStatusOfSamplesFromFile" -> { - final List accessions = - sampleStatusUpdater.parseFileAndGetSampleAccessionList( - "C:\\Users\\dgupta\\AtlantECO-samples-to-suppress.txt"); + final String file = + args.containsOption("file") ? args.getOptionValues("file").get(0) : null; + + if (file != null) { + final List accessions = + sampleStatusUpdater.parseFileAndGetSampleAccessionList(file); - sampleStatusUpdater.processSamples(accessions, null); + sampleStatusUpdater.processSamples(accessions, SampleStatus.PUBLIC); + } else { + throw new RuntimeException("File is not provided"); + } } case "updateSampleRelationships" -> { diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleStatusUpdater.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleStatusUpdater.java index 05d493438..b284a32bc 100644 --- a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleStatusUpdater.java +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleStatusUpdater.java @@ -137,13 +137,18 @@ private void handleSample(final Sample sample, final SampleStatus toMakeStatus) LocalDateTime.now(ZoneOffset.UTC) .plusYears(100) .toEpochSecond(ZoneOffset.UTC))) + .withStatus(SampleStatus.PRIVATE) .build(); } else { log.info("{} is already private", accession); } } else if (toMakeStatus == SampleStatus.PUBLIC) { if (sample.getRelease().isAfter(Instant.now())) { - updatedSample = Sample.Builder.fromSample(sample).withRelease(Instant.now()).build(); + updatedSample = + Sample.Builder.fromSample(sample) + .withRelease(Instant.now()) + .withStatus(SampleStatus.PUBLIC) + .build(); } else { log.info("{} is already public", accession); } diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/SampleCallbackResult.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/SampleCallbackResult.java deleted file mode 100644 index e69de29bb..000000000 diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/SampleRetrievalResult.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/SampleRetrievalResult.java index 69d6e458b..bd4daaa07 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/SampleRetrievalResult.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/SampleRetrievalResult.java @@ -8,7 +8,6 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ - package uk.ac.ebi.biosamples.service; import java.sql.Date; @@ -20,4 +19,4 @@ public class SampleRetrievalResult { private int statusId; private String egaId; private Date lastUpdated; -} \ No newline at end of file +} diff --git a/pipelines/pom.xml b/pipelines/pom.xml index cb4a06ec7..82a5d05f5 100644 --- a/pipelines/pom.xml +++ b/pipelines/pom.xml @@ -37,7 +37,7 @@ taxonimport reindex chain - + diff --git a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java index 8d234a2f3..3aab8f7b1 100644 --- a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -13,7 +13,6 @@ import org.apache.http.HeaderElement; import org.apache.http.HeaderElementIterator; import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; import org.apache.http.conn.ConnectionKeepAliveStrategy; @@ -23,7 +22,6 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicHeaderElementIterator; import org.apache.http.protocol.HTTP; -import org.apache.http.protocol.HttpContext; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; @@ -40,20 +38,22 @@ import uk.ac.ebi.biosamples.service.EnaConfig; import uk.ac.ebi.biosamples.service.EnaSampleToBioSampleConversionService; import uk.ac.ebi.biosamples.service.EraProDao; -import uk.ac.ebi.biosamples.service.validation.ElixirSchemaValidator; -import uk.ac.ebi.biosamples.service.validation.SchemaValidationService; import uk.ac.ebi.biosamples.utils.PipelineUtils; -import uk.ac.ebi.biosamples.utils.ols.OlsProcessor; @SpringBootApplication(exclude = DataSourceAutoConfiguration.class) @ComponentScan( excludeFilters = { - @ComponentScan.Filter( - type = FilterType.ASSIGNABLE_TYPE, - value = {EnaConfig.class, EraProDao.class, EnaSampleToBioSampleConversionService.class, - BioSamplesWebSecurityConfig.class - }), - @ComponentScan.Filter(type = FilterType.REGEX, pattern = "uk\\.ac\\.ebi\\.biosamples\\.service\\.validation\\..*") + @ComponentScan.Filter( + type = FilterType.ASSIGNABLE_TYPE, + value = { + EnaConfig.class, + EraProDao.class, + EnaSampleToBioSampleConversionService.class, + BioSamplesWebSecurityConfig.class + }), + @ComponentScan.Filter( + type = FilterType.REGEX, + pattern = "uk\\.ac\\.ebi\\.biosamples\\.service\\.validation\\..*") }) @Import(ExclusionConfiguration.class) public class Application { @@ -63,8 +63,8 @@ public static void main(final String[] args) { PipelineUtils.exitPipeline(ctx); } - - // todo I Had to add restTemplate bean as a temporary workaround as there seems to be a problem with dependencies after refactor. + // todo I Had to add restTemplate bean as a temporary workaround as there seems to be a problem + // with dependencies after refactor. // We need to sort out dependency problem and remove this unused dependency. @Bean @@ -79,10 +79,8 @@ public RestTemplateCustomizer restTemplateCustomizer( final BioSamplesProperties bioSamplesProperties, final PipelinesProperties pipelinesProperties) { return restTemplate -> { - final ConnectionKeepAliveStrategy keepAliveStrategy = (response, context) -> { - final HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); while (it.hasNext()) { @@ -119,8 +117,7 @@ public RestTemplateCustomizer restTemplateCustomizer( final RequestConfig config = RequestConfig.custom() .setConnectTimeout(timeout * 1000) - .setConnectionRequestTimeout( - timeout * 1000) + .setConnectionRequestTimeout(timeout * 1000) .setSocketTimeout(timeout * 1000) .build(); final HttpClient httpClient = diff --git a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java index cd3b740f9..ebefaba09 100644 --- a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java +++ b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/ReindexRunner.java @@ -10,10 +10,9 @@ */ package uk.ac.ebi.biosamples; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.*; import java.util.concurrent.*; - -import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang.IncompleteArgumentException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,7 +29,6 @@ import uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter; import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.messaging.MessagingConstants; -import uk.ac.ebi.biosamples.messaging.model.MessageContent; import uk.ac.ebi.biosamples.mongo.model.MongoSample; import uk.ac.ebi.biosamples.mongo.service.SampleReadService; import uk.ac.ebi.biosamples.utils.PipelineUtils; @@ -55,8 +53,11 @@ public class ReindexRunner implements ApplicationRunner { private final ObjectMapper objectMapper; @Autowired - public ReindexRunner(AmqpTemplate amqpTemplate, SampleReadService sampleReadService, - MongoOperations mongoOperations, ObjectMapper objectMapper) { + public ReindexRunner( + AmqpTemplate amqpTemplate, + SampleReadService sampleReadService, + MongoOperations mongoOperations, + ObjectMapper objectMapper) { this.amqpTemplate = amqpTemplate; this.sampleReadService = sampleReadService; this.mongoOperations = mongoOperations; @@ -105,7 +106,8 @@ public void run(final ApplicationArguments args) throws Exception { futures.put( accession, executor.submit( - new SampleIndexingCallable(accession, sampleReadService, amqpTemplate, objectMapper))); + new SampleIndexingCallable( + accession, sampleReadService, amqpTemplate, objectMapper))); ThreadUtils.checkFutures(futures, 1000); } @@ -125,10 +127,11 @@ private static class SampleIndexingCallable implements Callable { private final AmqpTemplate amqpTemplate; private final ObjectMapper objectMapper; - public SampleIndexingCallable(String accession, - SampleReadService sampleReadService, - AmqpTemplate amqpTemplate, - ObjectMapper objectMapper) { + public SampleIndexingCallable( + String accession, + SampleReadService sampleReadService, + AmqpTemplate amqpTemplate, + ObjectMapper objectMapper) { this.accession = accession; this.sampleReadService = sampleReadService; this.amqpTemplate = amqpTemplate; @@ -158,15 +161,17 @@ private boolean fetchSampleAndSendMessage(final boolean isRetry) { if (sampleOptional.isPresent()) { try { String json = objectMapper.writeValueAsString(sampleOptional.get()); - amqpTemplate.convertAndSend(MessagingConstants.INDEXING_EXCHANGE, MessagingConstants.INDEXING_QUEUE, json); -// amqpTemplate.convertAndSend( -// MessagingConstants.REINDEXING_EXCHANGE, -// MessagingConstants.REINDEXING_QUEUE, -// MessageContent.build(sampleOptional.get(), null, related, false)); + amqpTemplate.convertAndSend( + MessagingConstants.INDEXING_EXCHANGE, MessagingConstants.INDEXING_QUEUE, json); + // amqpTemplate.convertAndSend( + // MessagingConstants.REINDEXING_EXCHANGE, + // MessagingConstants.REINDEXING_QUEUE, + // MessageContent.build(sampleOptional.get(), null, related, false)); return true; } catch (final Exception e) { - LOGGER.error("Failed to convert sample to message and send to queue for {}", accession, e); + LOGGER.error( + "Failed to convert sample to message and send to queue for {}", accession, e); } } else { LOGGER.warn("Failed to fetch sample for {}", accession); diff --git a/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java b/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java index ec8e0685f..76209e82f 100644 --- a/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java +++ b/pipelines/reindex/src/test/java/uk/ac/ebi/biosamples/ReindexRunnerTest.java @@ -15,9 +15,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.util.*; - import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.*; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; @@ -40,7 +39,6 @@ public class ReindexRunnerTest { @Mock private SampleReadService sampleReadService; @Mock private ObjectMapper objectMapper; - private final List accessions = Arrays.asList("ACCESSION1", "ACCESSION2", "ACCESSION3"); private final Map sampleMap = new HashMap() {}; diff --git a/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/Application.java index 4432161fa..003bdc0b0 100644 --- a/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/sample-release/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -30,11 +30,12 @@ import uk.ac.ebi.biosamples.service.EraProDao; import uk.ac.ebi.biosamples.utils.PipelineUtils; -@SpringBootApplication(exclude = { - DataSourceAutoConfiguration.class, - SecurityAutoConfiguration.class, - UserDetailsServiceAutoConfiguration.class -}) +@SpringBootApplication( + exclude = { + DataSourceAutoConfiguration.class, + SecurityAutoConfiguration.class, + UserDetailsServiceAutoConfiguration.class + }) @ComponentScan( excludeFilters = { @ComponentScan.Filter( diff --git a/pom.xml b/pom.xml index e79744484..e1550eccf 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,7 @@ integration client properties + logback core @@ -151,6 +152,15 @@ -parameters + + org.apache.maven.plugins + maven-clean-plugin + 3.2.0 + + false + false + + org.apache.maven.plugins diff --git a/webapps/core-v2/pom.xml b/webapps/core-v2/pom.xml index 0aaed0993..46ff85696 100644 --- a/webapps/core-v2/pom.xml +++ b/webapps/core-v2/pom.xml @@ -24,6 +24,11 @@ properties 5.3.15-SNAPSHOT + + uk.ac.ebi.biosamples + logback + 5.3.15-SNAPSHOT + org.springframework.boot diff --git a/webapps/core-v2/src/main/resources/logback-spring.xml b/webapps/core-v2/src/main/resources/logback-spring.xml new file mode 100644 index 000000000..e87793051 --- /dev/null +++ b/webapps/core-v2/src/main/resources/logback-spring.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/webapps/core/pom.xml b/webapps/core/pom.xml index 499b881e0..a967e0b1e 100644 --- a/webapps/core/pom.xml +++ b/webapps/core/pom.xml @@ -26,16 +26,21 @@ properties 5.3.15-SNAPSHOT - - uk.ac.ebi.biosamples.search - proto - 0.0.1-SNAPSHOT - - - com.google.protobuf - protobuf-java - 3.25.5 - + + uk.ac.ebi.biosamples + logback + 5.3.15-SNAPSHOT + + + uk.ac.ebi.biosamples.search + proto + 0.0.1-SNAPSHOT + + + com.google.protobuf + protobuf-java + 3.25.5 + org.springframework.boot diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java index 0d1768129..a49a35df0 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/FileDownloadController.java @@ -55,7 +55,8 @@ public ResponseEntity download( @RequestParam(name = "count", required = false, defaultValue = "100000") final int count, final HttpServletResponse response, final HttpServletRequest request) { - LOG.info("Sample bulk download request: text = {}, filters = {}", text, Arrays.toString(filter)); + LOG.info( + "Sample bulk download request: text = {}, filters = {}", text, Arrays.toString(filter)); final String decodedText = LinkUtils.decodeText(text); final Collection filters = diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java index 7c042aa85..c58a47091 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/controller/SampleFacetController.java @@ -22,8 +22,8 @@ import org.springframework.web.bind.annotation.*; import uk.ac.ebi.biosamples.core.model.facet.Facet; import uk.ac.ebi.biosamples.core.model.filter.Filter; -import uk.ac.ebi.biosamples.service.facet.FacetingService; import uk.ac.ebi.biosamples.service.FilterService; +import uk.ac.ebi.biosamples.service.facet.FacetingService; import uk.ac.ebi.biosamples.utils.LinkUtils; @RestController @@ -33,7 +33,8 @@ public class SampleFacetController { private final FacetingService facetService; private final FilterService filterService; - public SampleFacetController(final FacetingService facetService, final FilterService filterService) { + public SampleFacetController( + final FacetingService facetService, final FilterService filterService) { this.facetService = facetService; this.filterService = filterService; } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java index cab653a8f..daca984ae 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/SamplePageService.java @@ -10,6 +10,13 @@ */ package uk.ac.ebi.biosamples.service; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.domain.Page; @@ -26,14 +33,6 @@ import uk.ac.ebi.biosamples.service.search.SearchService; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.stream.Collectors; - /** * Service layer business logic for centralising repository access and conversions between different * controller. Use this instead of linking to repositories directly. @@ -49,11 +48,12 @@ public class SamplePageService { private final SampleReadService sampleService; private final SearchService searchService; - public SamplePageService(MongoSampleRepository mongoSampleRepository, - MongoCurationLinkRepository mongoCurationLinkRepository, - MongoSampleToSampleConverter mongoSampleToSampleConverter, - SampleReadService sampleService, - @Qualifier("elasticSearchService") SearchService searchService) { + public SamplePageService( + MongoSampleRepository mongoSampleRepository, + MongoCurationLinkRepository mongoCurationLinkRepository, + MongoSampleToSampleConverter mongoSampleToSampleConverter, + SampleReadService sampleService, + @Qualifier("elasticSearchService") SearchService searchService) { this.mongoSampleRepository = mongoSampleRepository; this.mongoCurationLinkRepository = mongoCurationLinkRepository; this.mongoSampleToSampleConverter = mongoSampleToSampleConverter; @@ -86,7 +86,8 @@ public Page getSamplesByText( final boolean applyCurations) { long startTime = System.nanoTime(); final Page accessionPage = - searchService.searchForAccessions(text, new HashSet<>(filters), webinSubmissionAccountId, pageable); + searchService.searchForAccessions( + text, new HashSet<>(filters), webinSubmissionAccountId, pageable); long endTime = System.nanoTime(); log.trace("Got search page in {}ms", (endTime - startTime) / 1000000); @@ -123,7 +124,8 @@ public CursorArrayList getSamplesByText( size = validatePageSize(size); final CursorArrayList cursorAccessionList = - searchService.searchForAccessions(text, new HashSet<>(filters), webinSubmissionAccountId, cursorMark, size); + searchService.searchForAccessions( + text, new HashSet<>(filters), webinSubmissionAccountId, cursorMark, size); final List>> listFutureSample; listFutureSample = diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java index c8fcaa7f3..aca16a066 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/ElasticFacetService.java @@ -1,9 +1,20 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ package uk.ac.ebi.biosamples.service.facet; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.StatusRuntimeException; import io.micrometer.core.annotation.Timed; +import java.util.*; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Pageable; @@ -18,8 +29,6 @@ import uk.ac.ebi.biosamples.search.grpc.*; import uk.ac.ebi.biosamples.service.search.SearchFilterMapper; -import java.util.*; - @Service("elasticFacetService") @RequiredArgsConstructor @Slf4j @@ -28,11 +37,21 @@ public class ElasticFacetService implements FacetService { @Override @Timed("biosamples.facet.page.elastic") - public List getFacets(String searchTerm, Set filters, String webinId, - Pageable facetFieldPageInfo, Pageable facetValuesPageInfo, - String facetField, List facetFields) { + public List getFacets( + String searchTerm, + Set filters, + String webinId, + Pageable facetFieldPageInfo, + Pageable facetValuesPageInfo, + String facetField, + List facetFields) { - ManagedChannel channel = ManagedChannelBuilder.forAddress(bioSamplesProperties.getBiosamplesSearchHost(), bioSamplesProperties.getBiosamplesSearchPort()).usePlaintext().build(); + ManagedChannel channel = + ManagedChannelBuilder.forAddress( + bioSamplesProperties.getBiosamplesSearchHost(), + bioSamplesProperties.getBiosamplesSearchPort()) + .usePlaintext() + .build(); SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); FacetResponse response; try { @@ -58,30 +77,40 @@ public List getFacets(String searchTerm, Set filters, String webi return convertToFacets(facets); } - public static List convertToFacets(List grpcFacets) { + public static List convertToFacets( + List grpcFacets) { return grpcFacets.stream().map(ElasticFacetService::convertFacet).toList(); } static Facet convertFacet(uk.ac.ebi.biosamples.search.grpc.Facet grpcFacet) { - Facet.Builder facetBuilder = switch (grpcFacet.getType()) { - case "attr" -> new AttributeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) - .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); - case "dt" -> new DateRangeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) - .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); - case "rel" -> new RelationFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) - .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); - case "extd" -> new ExternalReferenceDataFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) - .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); -// case "sdata" -> new DateRangeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()); - default -> new AttributeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) - .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); - }; + Facet.Builder facetBuilder = + switch (grpcFacet.getType()) { + case "attr" -> + new AttributeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + case "dt" -> + new DateRangeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + case "rel" -> + new RelationFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + case "extd" -> + new ExternalReferenceDataFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + // case "sdata" -> new DateRangeFacet.Builder(grpcFacet.getField(), + // grpcFacet.getCount()); + default -> + new AttributeFacet.Builder(grpcFacet.getField(), grpcFacet.getCount()) + .withContent(convertToLabelCounts(grpcFacet.getBucketsMap())); + }; return facetBuilder.build(); } static LabelCountListContent convertToLabelCounts(Map labelCounts) { - List labelCountEntries = labelCounts.entrySet().stream() - .map(e -> LabelCountEntry.build(e.getKey(), e.getValue())).toList(); + List labelCountEntries = + labelCounts.entrySet().stream() + .map(e -> LabelCountEntry.build(e.getKey(), e.getValue())) + .toList(); return new LabelCountListContent(labelCountEntries); } } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetService.java index 72030b7ce..a6d05f11a 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetService.java @@ -1,12 +1,21 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ package uk.ac.ebi.biosamples.service.facet; +import java.util.List; +import java.util.Set; import org.springframework.data.domain.Pageable; import uk.ac.ebi.biosamples.core.model.facet.Facet; import uk.ac.ebi.biosamples.core.model.filter.Filter; -import java.util.List; -import java.util.Set; - public interface FacetService { List getFacets( String searchTerm, diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetingService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetingService.java index 58a00eae9..0875e1222 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetingService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/FacetingService.java @@ -10,6 +10,9 @@ */ package uk.ac.ebi.biosamples.service.facet; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; import org.apache.solr.client.solrj.util.ClientUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,10 +23,6 @@ import uk.ac.ebi.biosamples.core.model.facet.Facet; import uk.ac.ebi.biosamples.core.model.filter.Filter; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; - @Service public class FacetingService { private final Logger log = LoggerFactory.getLogger(getClass()); @@ -60,7 +59,13 @@ public List getFacets( final String escapedText = text == null ? null : ClientUtils.escapeQueryChars(text); final List facets = facetService.getFacets( - escapedText, new HashSet<>(filters), null, facetPageable, facetValuePageable, facetField, facetFields); + escapedText, + new HashSet<>(filters), + null, + facetPageable, + facetValuePageable, + facetField, + facetFields); final long endTime = System.nanoTime(); log.trace("Got solr facets in " + ((endTime - startTime) / 1000000) + "ms"); diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SearchFacetMapper.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SearchFacetMapper.java index e1a853491..6ae53efaf 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SearchFacetMapper.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/facet/SearchFacetMapper.java @@ -1,16 +1,26 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ package uk.ac.ebi.biosamples.service.facet; -import org.springframework.util.CollectionUtils; -import org.springframework.util.StringUtils; -import uk.ac.ebi.biosamples.search.grpc.*; - import java.util.ArrayList; import java.util.List; import java.util.Set; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; +import uk.ac.ebi.biosamples.search.grpc.*; public class SearchFacetMapper { - public static List getSearchFilters(Set filters, String webinId) { + public static List getSearchFilters( + Set filters, String webinId) { List grpcFilters = new ArrayList<>(); grpcFilters.add(getPrivateSearchFilter(webinId)); if (!CollectionUtils.isEmpty(filters)) { @@ -19,40 +29,54 @@ public static List getSearchFilters(Set filters, List grpcFilters) { + private static void getSearchFilters( + Set filters, List grpcFilters) { for (uk.ac.ebi.biosamples.core.model.filter.Filter filter : filters) { Filter.Builder filterBuilder = Filter.newBuilder(); if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AccessionFilter f) { - f.getContent().ifPresent(accession -> filterBuilder.setAccession(AccessionFilter.newBuilder().setAccession(accession))); + f.getContent() + .ifPresent( + accession -> + filterBuilder.setAccession( + AccessionFilter.newBuilder().setAccession(accession))); } if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.NameFilter f) { - f.getContent().ifPresent(name -> filterBuilder.setName(NameFilter.newBuilder().setName(name))); + f.getContent() + .ifPresent(name -> filterBuilder.setName(NameFilter.newBuilder().setName(name))); } if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AuthenticationFilter f) { - f.getContent().ifPresent(auth -> { // todo domain - filterBuilder.setWebin(WebinIdFilter.newBuilder().setWebinId(auth)); - }); + f.getContent() + .ifPresent( + auth -> { // todo domain + filterBuilder.setWebin(WebinIdFilter.newBuilder().setWebinId(auth)); + }); } if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter f) { - DateRangeFilter.DateField dateField = switch (f.getLabel()) { - case "update" -> DateRangeFilter.DateField.UPDATE; - case "create" -> DateRangeFilter.DateField.CREATE; - case "release" -> DateRangeFilter.DateField.RELEASE; - case "submitted" -> DateRangeFilter.DateField.SUBMITTED; - default -> throw new IllegalArgumentException("Unknown date field " + f.getLabel()); - }; + DateRangeFilter.DateField dateField = + switch (f.getLabel()) { + case "update" -> DateRangeFilter.DateField.UPDATE; + case "create" -> DateRangeFilter.DateField.CREATE; + case "release" -> DateRangeFilter.DateField.RELEASE; + case "submitted" -> DateRangeFilter.DateField.SUBMITTED; + default -> throw new IllegalArgumentException("Unknown date field " + f.getLabel()); + }; - f.getContent().ifPresent(dateRange -> filterBuilder.setDateRange( - DateRangeFilter.newBuilder() - .setField(dateField) - .setFrom(dateRange.getFrom().toString()) - .setTo(dateRange.getUntil().toString()) - )); + f.getContent() + .ifPresent( + dateRange -> + filterBuilder.setDateRange( + DateRangeFilter.newBuilder() + .setField(dateField) + .setFrom(dateRange.getFrom().toString()) + .setTo(dateRange.getUntil().toString()))); } if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AttributeFilter f) { AttributeFilter.Builder attributeFilterBuilder = AttributeFilter.newBuilder(); attributeFilterBuilder.setField(f.getLabel()); - f.getContent().ifPresent(attribute -> attributeFilterBuilder.addAllValues(List.of(attribute))); // todo aggregation + f.getContent() + .ifPresent( + attribute -> + attributeFilterBuilder.addAllValues(List.of(attribute))); // todo aggregation filterBuilder.setAttribute(attributeFilterBuilder); } if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.RelationFilter f) { @@ -74,7 +98,7 @@ private static void getSearchFilters(Set getFacets( final Pageable facetFieldPageInfo, final Pageable facetValuesPageInfo) { - return getFacets(searchTerm, new HashSet<>(filters), null, facetFieldPageInfo, facetValuesPageInfo, null, null); + return getFacets( + searchTerm, + new HashSet<>(filters), + null, + facetFieldPageInfo, + facetValuesPageInfo, + null, + null); } private List> getFacetFields( diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java index da928abb9..c489f07c6 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/ElasticSearchService.java @@ -1,3 +1,13 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ package uk.ac.ebi.biosamples.service.search; import com.google.protobuf.Timestamp; @@ -6,6 +16,9 @@ import io.grpc.ManagedChannelBuilder; import io.grpc.StatusRuntimeException; import io.micrometer.core.annotation.Timed; +import java.time.Instant; +import java.util.List; +import java.util.Set; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.*; @@ -16,13 +29,6 @@ import uk.ac.ebi.biosamples.search.grpc.*; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; -import java.io.OutputStream; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - @Service("elasticSearchService") @RequiredArgsConstructor @Slf4j @@ -31,8 +37,14 @@ public class ElasticSearchService implements SearchService { @Override @Timed("biosamples.search.page.elastic") - public Page searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable) { - ManagedChannel channel = ManagedChannelBuilder.forAddress(bioSamplesProperties.getBiosamplesSearchHost(), bioSamplesProperties.getBiosamplesSearchPort()).usePlaintext().build(); + public Page searchForAccessions( + String searchTerm, Set filters, String webinId, Pageable pageable) { + ManagedChannel channel = + ManagedChannelBuilder.forAddress( + bioSamplesProperties.getBiosamplesSearchHost(), + bioSamplesProperties.getBiosamplesSearchPort()) + .usePlaintext() + .build(); SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); SearchResponse response; try { @@ -54,28 +66,45 @@ public Page searchForAccessions(String searchTerm, Set filters, } List accessions = response.getAccessionsList(); - Sort sort = Sort.by(response.getSortList().stream().map(s -> new Sort.Order(Sort.Direction.ASC, s)).toList()); - PageRequest page = PageRequest.of(response.getNumber(), response.getSize(), sort) ; + Sort sort = + Sort.by( + response.getSortList().stream() + .map(s -> new Sort.Order(Sort.Direction.ASC, s)) + .toList()); + PageRequest page = PageRequest.of(response.getNumber(), response.getSize(), sort); long totalElements = response.getTotalElements(); SearchAfter searchAfter = response.getSearchAfter(); - return new SearchAfterPage<>(accessions, page, totalElements, searchAfter.getUpdate(), searchAfter.getAccession()); + return new SearchAfterPage<>( + accessions, page, totalElements, searchAfter.getUpdate(), searchAfter.getAccession()); } @Override @Timed("biosamples.search.cursor.elastic") - public CursorArrayList searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size) { + public CursorArrayList searchForAccessions( + String searchTerm, Set filters, String webinId, String cursor, int size) { SearchAfter searchAfter = null; String[] cursorParts = cursor.split(","); if (cursorParts.length == 2) { Instant update = Instant.parse(cursorParts[0].trim()); String accession = cursorParts[1].trim(); - searchAfter = SearchAfter.newBuilder() - .setUpdate(Timestamp.newBuilder().setSeconds(update.getEpochSecond()).setNanos(update.getNano()).build()) - .setAccession(accession).build(); + searchAfter = + SearchAfter.newBuilder() + .setUpdate( + Timestamp.newBuilder() + .setSeconds(update.getEpochSecond()) + .setNanos(update.getNano()) + .build()) + .setAccession(accession) + .build(); } - ManagedChannel channel = ManagedChannelBuilder.forAddress(bioSamplesProperties.getBiosamplesSearchHost(), bioSamplesProperties.getBiosamplesSearchPort()).usePlaintext().build(); + ManagedChannel channel = + ManagedChannelBuilder.forAddress( + bioSamplesProperties.getBiosamplesSearchHost(), + bioSamplesProperties.getBiosamplesSearchPort()) + .usePlaintext() + .build(); SearchGrpc.SearchBlockingStub stub = SearchGrpc.newBlockingStub(channel); SearchResponse response; try { @@ -101,13 +130,13 @@ public CursorArrayList searchForAccessions(String searchTerm, Set(accessions, cursor); } - /*public OutputStream searchForAccessionsStream(String searchTerm, Set filters, String webinId, String cursor, int size) { SearchAfter searchAfter = null; String[] cursorParts = cursor.split(","); diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java index 015eb6e81..2efb33d0b 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchAfterPage.java @@ -1,20 +1,30 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ package uk.ac.ebi.biosamples.service.search; import com.google.protobuf.Timestamp; +import java.time.Instant; +import java.util.List; import lombok.Getter; import lombok.NonNull; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; -import java.time.Instant; -import java.util.List; - @Getter public class SearchAfterPage extends PageImpl { private final Instant update; private final String accession; - public SearchAfterPage(List content, Pageable pageable, long total, @NonNull Timestamp update, String accession) { + public SearchAfterPage( + List content, Pageable pageable, long total, @NonNull Timestamp update, String accession) { super(content, pageable, total); this.update = Instant.ofEpochSecond(update.getSeconds(), update.getNanos()); this.accession = accession; diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchFilterMapper.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchFilterMapper.java index 9058e42e3..83f3f4508 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchFilterMapper.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchFilterMapper.java @@ -1,5 +1,16 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ package uk.ac.ebi.biosamples.service.search; +import java.util.*; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import uk.ac.ebi.biosamples.core.model.filter.AuthenticationFilter; @@ -8,11 +19,10 @@ import uk.ac.ebi.biosamples.core.model.filter.RelationFilter; import uk.ac.ebi.biosamples.search.grpc.*; -import java.util.*; - public class SearchFilterMapper { - public static List getSearchFilters(Set filters, String webinId) { + public static List getSearchFilters( + Set filters, String webinId) { List grpcFilters = new ArrayList<>(); grpcFilters.add(getPrivateSearchFilter(webinId)); if (!CollectionUtils.isEmpty(filters)) { @@ -21,7 +31,8 @@ public static List getSearchFilters(Set filters, List grpcFilters) { + private static void getSearchFilters( + Set filters, List grpcFilters) { Map filterMap = new HashMap<>(); for (uk.ac.ebi.biosamples.core.model.filter.Filter filter : filters) { if (filter instanceof uk.ac.ebi.biosamples.core.model.filter.AccessionFilter f) { @@ -38,10 +49,11 @@ private static void getSearchFilters(Set grpcFilters) { - f.getContent().ifPresent(auth -> { // todo domain - grpcFilters.add(Filter.newBuilder().setWebin(WebinIdFilter.newBuilder().setWebinId(auth)).build()); - }); + f.getContent() + .ifPresent( + auth -> { // todo domain + grpcFilters.add( + Filter.newBuilder() + .setWebin(WebinIdFilter.newBuilder().setWebinId(auth)) + .build()); + }); } - private static void getNameSearchFilter(uk.ac.ebi.biosamples.core.model.filter.NameFilter f, List grpcFilters) { - f.getContent().ifPresent(name -> - grpcFilters.add(Filter.newBuilder().setName(NameFilter.newBuilder().setName(name)).build())); + private static void getNameSearchFilter( + uk.ac.ebi.biosamples.core.model.filter.NameFilter f, List grpcFilters) { + f.getContent() + .ifPresent( + name -> + grpcFilters.add( + Filter.newBuilder().setName(NameFilter.newBuilder().setName(name)).build())); } - private static void getAccessionSearchFilter(uk.ac.ebi.biosamples.core.model.filter.AccessionFilter f, List grpcFilters) { - f.getContent().ifPresent(accession -> - grpcFilters.add(Filter.newBuilder().setAccession(AccessionFilter.newBuilder().setAccession(accession)).build())); + private static void getAccessionSearchFilter( + uk.ac.ebi.biosamples.core.model.filter.AccessionFilter f, List grpcFilters) { + f.getContent() + .ifPresent( + accession -> + grpcFilters.add( + Filter.newBuilder() + .setAccession(AccessionFilter.newBuilder().setAccession(accession)) + .build())); } - private static void getExternalReferenceSearchFilter(ExternalReferenceDataFilter f, List grpcFilters) { + private static void getExternalReferenceSearchFilter( + ExternalReferenceDataFilter f, List grpcFilters) { ExternalRefFilter.Builder externalRefFilterBuilder = ExternalRefFilter.newBuilder(); f.getContent().ifPresent(externalRefFilterBuilder::setArchive); grpcFilters.add(Filter.newBuilder().setExternal(externalRefFilterBuilder).build()); } - private static void getInverseRelationshipSearchFilter(InverseRelationFilter f, List grpcFilters) { + private static void getInverseRelationshipSearchFilter( + InverseRelationFilter f, List grpcFilters) { RelationshipFilter.Builder relationshipFilterBuilder = RelationshipFilter.newBuilder(); relationshipFilterBuilder.setType(f.getLabel()); f.getContent().ifPresent(relationshipFilterBuilder::setSource); @@ -90,28 +119,39 @@ private static void getRelationshipSearchFilter(RelationFilter f, List g grpcFilters.add(Filter.newBuilder().setRelationship(relationshipFilterBuilder).build()); } - private static void getDateRangeSearchFilter(uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter f, List grpcFilters) { - f.getContent().ifPresent(dateRange -> { - DateRangeFilter.DateField dateField = switch (f.getLabel()) { - case "update" -> DateRangeFilter.DateField.UPDATE; - case "create" -> DateRangeFilter.DateField.CREATE; - case "release" -> DateRangeFilter.DateField.RELEASE; - case "submitted" -> DateRangeFilter.DateField.SUBMITTED; - default -> throw new IllegalArgumentException("Unknown date field " + f.getLabel()); - }; - grpcFilters.add(Filter.newBuilder().setDateRange( - DateRangeFilter.newBuilder() - .setField(dateField) - .setFrom(dateRange.getFrom().toString()) - .setTo(dateRange.getUntil().toString()) - ).build()); - }); + private static void getDateRangeSearchFilter( + uk.ac.ebi.biosamples.core.model.filter.DateRangeFilter f, List grpcFilters) { + f.getContent() + .ifPresent( + dateRange -> { + DateRangeFilter.DateField dateField = + switch (f.getLabel()) { + case "update" -> DateRangeFilter.DateField.UPDATE; + case "create" -> DateRangeFilter.DateField.CREATE; + case "release" -> DateRangeFilter.DateField.RELEASE; + case "submitted" -> DateRangeFilter.DateField.SUBMITTED; + default -> + throw new IllegalArgumentException("Unknown date field " + f.getLabel()); + }; + grpcFilters.add( + Filter.newBuilder() + .setDateRange( + DateRangeFilter.newBuilder() + .setField(dateField) + .setFrom(dateRange.getFrom().toString()) + .setTo(dateRange.getUntil().toString())) + .build()); + }); } - private static void getAttributeSearchFilter(uk.ac.ebi.biosamples.core.model.filter.AttributeFilter filter, - Map filterMap) { - Filter.Builder fb = filterMap.getOrDefault(filter.getLabel(), Filter.newBuilder() - .setAttribute(AttributeFilter.newBuilder().setField(filter.getLabel()))); + private static void getAttributeSearchFilter( + uk.ac.ebi.biosamples.core.model.filter.AttributeFilter filter, + Map filterMap) { + Filter.Builder fb = + filterMap.getOrDefault( + filter.getLabel(), + Filter.newBuilder() + .setAttribute(AttributeFilter.newBuilder().setField(filter.getLabel()))); filter.getContent().ifPresent(attribute -> fb.getAttributeBuilder().addValues(attribute)); filterMap.putIfAbsent(filter.getLabel(), fb); } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java index 2787cc33c..79584a65e 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SearchService.java @@ -1,14 +1,25 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ package uk.ac.ebi.biosamples.service.search; +import java.util.Set; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; -import java.util.Set; - public interface SearchService { - Page searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable); + Page searchForAccessions( + String searchTerm, Set filters, String webinId, Pageable pageable); - CursorArrayList searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size); + CursorArrayList searchForAccessions( + String searchTerm, Set filters, String webinId, String cursor, int size); } diff --git a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java index 98972e03f..db63203ab 100644 --- a/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java +++ b/webapps/core/src/main/java/uk/ac/ebi/biosamples/service/search/SolrSearchService.java @@ -1,6 +1,18 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ package uk.ac.ebi.biosamples.service.search; import io.micrometer.core.annotation.Timed; +import java.util.List; +import java.util.Set; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Page; @@ -11,9 +23,6 @@ import uk.ac.ebi.biosamples.solr.repo.CursorArrayList; import uk.ac.ebi.biosamples.solr.service.SolrSampleService; -import java.util.List; -import java.util.Set; - @Service("solrSearchService") @RequiredArgsConstructor @Slf4j @@ -22,19 +31,20 @@ public class SolrSearchService implements SearchService { @Override @Timed("biosamples.search.page.solr") - public Page searchForAccessions(String searchTerm, Set filters, String webinId, Pageable pageable) { - return solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, pageable) + public Page searchForAccessions( + String searchTerm, Set filters, String webinId, Pageable pageable) { + return solrSampleService + .fetchSolrSampleByText(searchTerm, filters, webinId, pageable) .map(SolrSample::getAccession); } @Override @Timed("biosamples.search.cursor.solr") - public CursorArrayList searchForAccessions(String searchTerm, Set filters, String webinId, String cursor, int size) { - CursorArrayList samples = solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, cursor, size); - List accessions = samples - .stream() - .map(SolrSample::getAccession) - .toList(); + public CursorArrayList searchForAccessions( + String searchTerm, Set filters, String webinId, String cursor, int size) { + CursorArrayList samples = + solrSampleService.fetchSolrSampleByText(searchTerm, filters, webinId, cursor, size); + List accessions = samples.stream().map(SolrSample::getAccession).toList(); return new CursorArrayList<>(accessions, samples.getNextCursorMark()); } } diff --git a/webapps/core/src/main/resources/logback-spring.xml b/webapps/core/src/main/resources/logback-spring.xml new file mode 100644 index 000000000..e87793051 --- /dev/null +++ b/webapps/core/src/main/resources/logback-spring.xml @@ -0,0 +1,8 @@ + + + + + + + + From 720d22023369e130f47d34db7169753d338630ac Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 12 Dec 2025 03:00:19 +0000 Subject: [PATCH 79/86] feat: pipeline read last run from database --- .../ebi/biosamples/model/PipelineLastRun.java | 20 +++++++ .../PipelineLastRunRepository.java | 13 +++++ .../service/PipelineHelperService.java | 23 ++++++++ .../ebi/biosamples/utils/PipelineUtils.java | 13 +++-- .../uk/ac/ebi/biosamples/Application.java | 6 +- .../copydown/CopydownApplicationRunner.java | 55 +++++++++++-------- .../curation/CuramiApplicationRunner.java | 12 +++- .../curation/CurationApplicationRunner.java | 14 ++++- 8 files changed, 124 insertions(+), 32 deletions(-) create mode 100644 pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineLastRun.java create mode 100644 pipelines/common/src/main/java/uk/ac/ebi/biosamples/respository/PipelineLastRunRepository.java create mode 100644 pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineLastRun.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineLastRun.java new file mode 100644 index 000000000..463073ac8 --- /dev/null +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineLastRun.java @@ -0,0 +1,20 @@ +package uk.ac.ebi.biosamples.model; + +import lombok.Builder; +import lombok.Getter; +import lombok.extern.jackson.Jacksonized; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +import java.time.LocalDate; + +@Document +@Jacksonized +@Builder +@Getter +public class PipelineLastRun { + @Id + private String id; + private PipelineName pipelineName; + private LocalDate lastRunDate; +} diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/respository/PipelineLastRunRepository.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/respository/PipelineLastRunRepository.java new file mode 100644 index 000000000..68237ccd7 --- /dev/null +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/respository/PipelineLastRunRepository.java @@ -0,0 +1,13 @@ +package uk.ac.ebi.biosamples.respository; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; +import uk.ac.ebi.biosamples.model.PipelineLastRun; +import uk.ac.ebi.biosamples.model.PipelineName; + +import java.util.Optional; + +@Repository +public interface PipelineLastRunRepository extends MongoRepository { + Optional findFirstByPipelineName(PipelineName pipelineName); +} diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java new file mode 100644 index 000000000..884f26b25 --- /dev/null +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java @@ -0,0 +1,23 @@ +package uk.ac.ebi.biosamples.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import uk.ac.ebi.biosamples.model.PipelineLastRun; +import uk.ac.ebi.biosamples.model.PipelineName; +import uk.ac.ebi.biosamples.respository.PipelineLastRunRepository; + +import java.time.LocalDate; + +@Service +@Slf4j +@RequiredArgsConstructor +public class PipelineHelperService { + private final PipelineLastRunRepository pipelineLastRunRepository; + + public LocalDate getLastRunDate(PipelineName pipelineName) { + PipelineLastRun pipelineLastRun = pipelineLastRunRepository.findFirstByPipelineName(pipelineName) + .orElse(PipelineLastRun.builder().pipelineName(pipelineName).lastRunDate(LocalDate.EPOCH).build()); + return pipelineLastRun.getLastRunDate(); + } +} diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java index 534ac9f13..e7201cb5a 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java @@ -17,10 +17,8 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Map; -import java.util.Set; +import java.util.*; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.ApplicationArguments; @@ -34,6 +32,13 @@ public class PipelineUtils { private static final Logger log = LoggerFactory.getLogger(PipelineUtils.class); + public static Collection getLastRunFilters(LocalDate lastRunDate) { + Filter fromDateFilter = new DateRangeFilter.DateRangeFilterBuilder("update") + .from(lastRunDate.atStartOfDay().toInstant(ZoneOffset.UTC)) + .build(); + return List.of(fromDateFilter); + } + public static Collection getDateFilters( final ApplicationArguments args, final String dateType) { final Collection filters = new ArrayList<>(); diff --git a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java index 39841e7df..7281800a1 100644 --- a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -25,6 +25,7 @@ import org.apache.http.protocol.HTTP; import org.apache.http.protocol.HttpContext; import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.boot.web.client.RestTemplateCustomizer; @@ -54,7 +55,10 @@ public class Application { public static void main(final String[] args) { - final ConfigurableApplicationContext ctx = SpringApplication.run(Application.class, args); + SpringApplication app = new SpringApplication(Application.class); + app.setWebApplicationType(WebApplicationType.NONE); + + final ConfigurableApplicationContext ctx = app.run(args); PipelineUtils.exitPipeline(ctx); } diff --git a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java index 3280aa535..3ca4a9e68 100644 --- a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java +++ b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java @@ -1,21 +1,15 @@ /* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ + * Copyright 2021 EMBL - European Bioinformatics Institute + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ package uk.ac.ebi.biosamples.copydown; -import java.time.Duration; -import java.time.Instant; -import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.ApplicationArguments; @@ -29,44 +23,59 @@ import uk.ac.ebi.biosamples.core.model.PipelineAnalytics; import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.model.PipelineName; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; +import uk.ac.ebi.biosamples.service.PipelineHelperService; import uk.ac.ebi.biosamples.utils.PipelineUtils; import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Future; + @Component public class CopydownApplicationRunner implements ApplicationRunner { private static final Logger LOG = LoggerFactory.getLogger(CopydownApplicationRunner.class); + private static final PipelineName PIPELINE_NAME = PipelineName.COPYDOWN; private final BioSamplesClient bioSamplesClient; private final PipelinesProperties pipelinesProperties; private final AnalyticsService analyticsService; private final PipelineFutureCallback pipelineFutureCallback; + private final PipelineHelperService pipelineHelperService; public CopydownApplicationRunner( final BioSamplesClient bioSamplesClient, final PipelinesProperties pipelinesProperties, - final AnalyticsService analyticsService) { + final AnalyticsService analyticsService, PipelineHelperService pipelineHelperService) { this.bioSamplesClient = bioSamplesClient; this.pipelinesProperties = pipelinesProperties; this.analyticsService = analyticsService; + this.pipelineHelperService = pipelineHelperService; pipelineFutureCallback = new PipelineFutureCallback(); } @Override public void run(final ApplicationArguments args) throws Exception { - final Collection filters = PipelineUtils.getDateFilters(args, "update"); + LocalDate lastRunDate = pipelineHelperService.getLastRunDate(PIPELINE_NAME); + final Collection filters = PipelineUtils.getLastRunFilters(lastRunDate); final Instant startTime = Instant.now(); LOG.info("Pipeline started at {}", startTime); + LOG.info("Processing samples from {}", lastRunDate); long sampleCount = 0; try (final AdaptiveThreadPoolExecutor executorService = - AdaptiveThreadPoolExecutor.create( - 100, - 10000, - true, - pipelinesProperties.getThreadCount(), - pipelinesProperties.getThreadCountMax())) { + AdaptiveThreadPoolExecutor.create( + 100, + 10000, + true, + pipelinesProperties.getThreadCount(), + pipelinesProperties.getThreadCountMax())) { final Map> futures = new HashMap<>(); for (final EntityModel sampleResource : diff --git a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java index 1885cdb1a..fe1ac827f 100644 --- a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java +++ b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java @@ -13,6 +13,7 @@ import java.io.*; import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentLinkedQueue; @@ -33,9 +34,11 @@ import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.core.model.SampleAnalytics; import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.model.PipelineName; import uk.ac.ebi.biosamples.mongo.model.MongoCurationRule; import uk.ac.ebi.biosamples.mongo.repository.MongoCurationRuleRepository; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; +import uk.ac.ebi.biosamples.service.PipelineHelperService; import uk.ac.ebi.biosamples.utils.PipelineUtils; import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; @@ -43,6 +46,7 @@ @Component public class CuramiApplicationRunner implements ApplicationRunner { private static final Logger LOG = LoggerFactory.getLogger(CuramiApplicationRunner.class); + private static final PipelineName PIPELINE_NAME = PipelineName.CURAMI; private final BioSamplesClient bioSamplesClient; private final PipelinesProperties pipelinesProperties; @@ -50,25 +54,29 @@ public class CuramiApplicationRunner implements ApplicationRunner { private final MongoCurationRuleRepository repository; private final AnalyticsService analyticsService; private final PipelineFutureCallback pipelineFutureCallback; + private final PipelineHelperService pipelineHelperService; public CuramiApplicationRunner( final BioSamplesClient bioSamplesClient, final PipelinesProperties pipelinesProperties, final MongoCurationRuleRepository repository, - final AnalyticsService analyticsService) { + final AnalyticsService analyticsService, PipelineHelperService pipelineHelperService) { this.bioSamplesClient = bioSamplesClient; this.pipelinesProperties = pipelinesProperties; this.repository = repository; this.analyticsService = analyticsService; + this.pipelineHelperService = pipelineHelperService; curationRules = new HashMap<>(); pipelineFutureCallback = new PipelineFutureCallback(); } @Override public void run(final ApplicationArguments args) throws Exception { - final Collection filters = PipelineUtils.getDateFilters(args, "update"); + LocalDate lastRunDate = pipelineHelperService.getLastRunDate(PIPELINE_NAME); + final Collection filters = PipelineUtils.getLastRunFilters(lastRunDate); final Instant startTime = Instant.now(); LOG.info("Pipeline started at {}", startTime); + LOG.info("Processing samples from {}", lastRunDate); long sampleCount = 0; final SampleAnalytics sampleAnalytics = new SampleAnalytics(); diff --git a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java index 484cb62cd..4eeaebd35 100644 --- a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java +++ b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java @@ -12,6 +12,7 @@ import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentLinkedQueue; @@ -31,7 +32,9 @@ import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.core.service.CurationApplicationService; import uk.ac.ebi.biosamples.curation.service.IriUrlValidatorService; +import uk.ac.ebi.biosamples.model.PipelineName; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; +import uk.ac.ebi.biosamples.service.PipelineHelperService; import uk.ac.ebi.biosamples.utils.PipelineUtils; import uk.ac.ebi.biosamples.utils.ols.OlsProcessor; import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; @@ -40,6 +43,8 @@ @Component public class CurationApplicationRunner implements ApplicationRunner { private static final Logger LOG = LoggerFactory.getLogger(CurationApplicationRunner.class); + private static final PipelineName PIPELINE_NAME = PipelineName.CURATION; + private final BioSamplesClient bioSamplesClient; private final PipelinesProperties pipelinesProperties; private final OlsProcessor olsProcessor; @@ -47,6 +52,7 @@ public class CurationApplicationRunner implements ApplicationRunner { private final AnalyticsService analyticsService; private final PipelineFutureCallback pipelineFutureCallback; private final IriUrlValidatorService iriUrlValidatorService; + private final PipelineHelperService pipelineHelperService; public CurationApplicationRunner( final BioSamplesClient bioSamplesClient, @@ -54,20 +60,24 @@ public CurationApplicationRunner( final OlsProcessor olsProcessor, final CurationApplicationService curationApplicationService, final AnalyticsService analyticsService, - final IriUrlValidatorService iriUrlValidatorService) { + final IriUrlValidatorService iriUrlValidatorService, PipelineHelperService pipelineHelperService) { this.bioSamplesClient = bioSamplesClient; this.pipelinesProperties = pipelinesProperties; this.olsProcessor = olsProcessor; this.curationApplicationService = curationApplicationService; this.analyticsService = analyticsService; this.iriUrlValidatorService = iriUrlValidatorService; + this.pipelineHelperService = pipelineHelperService; pipelineFutureCallback = new PipelineFutureCallback(); } @Override public void run(final ApplicationArguments args) throws Exception { final Instant startTime = Instant.now(); - final Collection filters = PipelineUtils.getDateFilters(args, "update"); + LocalDate lastRunDate = pipelineHelperService.getLastRunDate(PIPELINE_NAME); + final Collection filters = PipelineUtils.getLastRunFilters(lastRunDate); + LOG.info("Pipeline started at {}", startTime); + LOG.info("Processing samples from {}", lastRunDate); long sampleCount = 0; try (final AdaptiveThreadPoolExecutor executorService = From 592a1e1db7b62a9023a8ad664f28236f7cfd7d04 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 12 Dec 2025 06:53:15 +0000 Subject: [PATCH 80/86] fix: repo config for pipelines --- .../PipelineLastRunRepository.java | 4 +--- .../uk/ac/ebi/biosamples/service/PipelineHelperService.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) rename pipelines/common/src/main/java/uk/ac/ebi/biosamples/{respository => repository}/PipelineLastRunRepository.java (78%) diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/respository/PipelineLastRunRepository.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java similarity index 78% rename from pipelines/common/src/main/java/uk/ac/ebi/biosamples/respository/PipelineLastRunRepository.java rename to pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java index 68237ccd7..88a9763cb 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/respository/PipelineLastRunRepository.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java @@ -1,13 +1,11 @@ -package uk.ac.ebi.biosamples.respository; +package uk.ac.ebi.biosamples.repository; import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.stereotype.Repository; import uk.ac.ebi.biosamples.model.PipelineLastRun; import uk.ac.ebi.biosamples.model.PipelineName; import java.util.Optional; -@Repository public interface PipelineLastRunRepository extends MongoRepository { Optional findFirstByPipelineName(PipelineName pipelineName); } diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java index 884f26b25..a757d8f2e 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java @@ -5,7 +5,7 @@ import org.springframework.stereotype.Service; import uk.ac.ebi.biosamples.model.PipelineLastRun; import uk.ac.ebi.biosamples.model.PipelineName; -import uk.ac.ebi.biosamples.respository.PipelineLastRunRepository; +import uk.ac.ebi.biosamples.repository.PipelineLastRunRepository; import java.time.LocalDate; From 3b99b7497f328d15523ed352a8392a534662a1b6 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 12 Dec 2025 07:22:20 +0000 Subject: [PATCH 81/86] fix: repo config for pipelines --- .../ebi/biosamples/repository/PipelineLastRunRepository.java | 2 ++ .../src/main/java/uk/ac/ebi/biosamples/Application.java | 4 ++++ .../src/main/java/uk/ac/ebi/biosamples/Application.java | 4 ++++ .../src/main/java/uk/ac/ebi/biosamples/Application.java | 2 ++ 4 files changed, 12 insertions(+) diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java index 88a9763cb..d6c95cdab 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java @@ -1,11 +1,13 @@ package uk.ac.ebi.biosamples.repository; import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; import uk.ac.ebi.biosamples.model.PipelineLastRun; import uk.ac.ebi.biosamples.model.PipelineName; import java.util.Optional; +@Repository public interface PipelineLastRunRepository extends MongoRepository { Optional findFirstByPipelineName(PipelineName pipelineName); } diff --git a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java index 7281800a1..369c48086 100644 --- a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -35,7 +35,9 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.configuration.ExclusionConfiguration; import uk.ac.ebi.biosamples.service.EnaConfig; @@ -52,6 +54,8 @@ }) @Import(ExclusionConfiguration.class) @EnableCaching +@EnableWebSecurity +@EnableMongoRepositories(basePackages = "uk.ac.ebi.biosamples.repository") public class Application { public static void main(final String[] args) { diff --git a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/Application.java index c3551f905..de7a03296 100644 --- a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -34,7 +34,9 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.configuration.ExclusionConfiguration; import uk.ac.ebi.biosamples.service.EnaConfig; @@ -51,6 +53,8 @@ }) @Import(ExclusionConfiguration.class) @EnableCaching +@EnableWebSecurity +@EnableMongoRepositories(basePackages = "uk.ac.ebi.biosamples.repository") public class Application { public static void main(final String[] args) { diff --git a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/Application.java index cd31b9c5d..98cee4c9b 100644 --- a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -33,6 +33,7 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.web.client.RestTemplate; @@ -52,6 +53,7 @@ @Import(ExclusionConfiguration.class) @EnableCaching @EnableWebSecurity +@EnableMongoRepositories(basePackages = "uk.ac.ebi.biosamples.repository") public class Application { public static void main(final String[] args) { SpringApplication app = new SpringApplication(Application.class); From 17ee4f15d39741536bb157bc25abc6633487be0c Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 12 Dec 2025 07:47:33 +0000 Subject: [PATCH 82/86] fix: repo config for pipelines --- .../uk/ac/ebi/biosamples/Application.java | 89 +++---------------- 1 file changed, 10 insertions(+), 79 deletions(-) diff --git a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java index 369c48086..5c1ec9dbb 100644 --- a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -66,87 +66,18 @@ public static void main(final String[] args) { PipelineUtils.exitPipeline(ctx); } + @Bean + public RestTemplate restTemplate(final RestTemplateCustomizer restTemplateCustomizer) { + final RestTemplate restTemplate = new RestTemplate(); + restTemplateCustomizer.customize(restTemplate); + return restTemplate; + } + @Bean public RestTemplateCustomizer restTemplateCustomizer( final BioSamplesProperties bioSamplesProperties, - final PipelinesProperties piplinesProperties) { - return new RestTemplateCustomizer() { - @Override - public void customize(final RestTemplate restTemplate) { - - // use a keep alive strategy to try to make it easier to maintain connections for - // reuse - final ConnectionKeepAliveStrategy keepAliveStrategy = - new ConnectionKeepAliveStrategy() { - @Override - public long getKeepAliveDuration( - final HttpResponse response, final HttpContext context) { - - // check if there is a non-standard keep alive header present - final HeaderElementIterator it = - new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); - while (it.hasNext()) { - final HeaderElement he = it.nextElement(); - final String param = he.getName(); - final String value = he.getValue(); - if (value != null && param.equalsIgnoreCase("timeout")) { - return Long.parseLong(value) * 1000; - } - } - // default to 60s if no header - return 60 * 1000; - } - }; - - // set a number of connections to use at once for multiple threads - final PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = - new PoolingHttpClientConnectionManager(); - poolingHttpClientConnectionManager.setMaxTotal(piplinesProperties.getConnectionCountMax()); - poolingHttpClientConnectionManager.setDefaultMaxPerRoute( - piplinesProperties.getConnectionCountDefault()); - poolingHttpClientConnectionManager.setMaxPerRoute( - new HttpRoute(HttpHost.create(piplinesProperties.getZooma())), - piplinesProperties.getConnectionCountZooma()); - poolingHttpClientConnectionManager.setMaxPerRoute( - new HttpRoute(HttpHost.create(bioSamplesProperties.getOls())), - piplinesProperties.getConnectionCountOls()); - - // set a local cache for cacheable responses - final CacheConfig cacheConfig = - CacheConfig.custom() - .setMaxCacheEntries(1024) - .setMaxObjectSize(1024 * 1024) // max size of 1Mb - // number of entries x size of entries = 1Gb total cache size - .setSharedCache(false) // act like a browser cache not a middle-hop cache - .build(); - - // set a timeout limit - // TODO put this in application.properties - final int timeout = 60; // in seconds - final RequestConfig config = - RequestConfig.custom() - .setConnectTimeout(timeout * 1000) // time to establish the connection with the - // remote host - .setConnectionRequestTimeout( - timeout * 1000) // maximum time of inactivity between two - // data packets - .setSocketTimeout(timeout * 1000) - .build(); // time to wait for a connection from the connection - // manager/pool - - // make the actual client - final HttpClient httpClient = - CachingHttpClientBuilder.create() - .setCacheConfig(cacheConfig) - .useSystemProperties() - .setConnectionManager(poolingHttpClientConnectionManager) - .setKeepAliveStrategy(keepAliveStrategy) - .setDefaultRequestConfig(config) - .build(); - - // and wire it into the resttemplate - restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory(httpClient)); - } - }; + final PipelinesProperties pipelinesProperties) { + return new PipelinesHelper() + .getRestTemplateCustomizer(bioSamplesProperties, pipelinesProperties); } } From c576bcd363f408089ccdebb21068ca5ccc74a4fe Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 12 Dec 2025 09:00:19 +0000 Subject: [PATCH 83/86] fix: add until date filter, save last completed date --- k8s/cronjobs/templates/workflow.yaml | 2 +- .../repository/PipelineLastRunRepository.java | 2 + .../service/PipelineHelperService.java | 14 +++++-- .../ebi/biosamples/utils/PipelineUtils.java | 3 +- .../uk/ac/ebi/biosamples/Application.java | 39 ++++++------------- .../copydown/CopydownApplicationRunner.java | 8 +++- .../curation/CuramiApplicationRunner.java | 8 +++- .../curation/CurationApplicationRunner.java | 8 +++- 8 files changed, 46 insertions(+), 38 deletions(-) diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index 313b3ef6d..afb474062 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -11,7 +11,7 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - parallelism: 1 + parallelism: 2 ttlStrategy: secondsAfterSuccess: {{ .Values.workflow.ttlStrategy.secondsAfterSuccess }} secondsAfterFailure: {{ .Values.workflow.ttlStrategy.secondsAfterFailure }} diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java index d6c95cdab..75313d74e 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java @@ -10,4 +10,6 @@ @Repository public interface PipelineLastRunRepository extends MongoRepository { Optional findFirstByPipelineName(PipelineName pipelineName); + + } diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java index a757d8f2e..38e8acfe2 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java @@ -15,9 +15,17 @@ public class PipelineHelperService { private final PipelineLastRunRepository pipelineLastRunRepository; - public LocalDate getLastRunDate(PipelineName pipelineName) { - PipelineLastRun pipelineLastRun = pipelineLastRunRepository.findFirstByPipelineName(pipelineName) + public PipelineLastRun getLastRunDate(PipelineName pipelineName) { + return pipelineLastRunRepository.findFirstByPipelineName(pipelineName) .orElse(PipelineLastRun.builder().pipelineName(pipelineName).lastRunDate(LocalDate.EPOCH).build()); - return pipelineLastRun.getLastRunDate(); + } + + public void updateLastRunDate(PipelineLastRun pipelineLastRun, LocalDate lastRunDate) { + PipelineLastRun updated = PipelineLastRun.builder() + .id(pipelineLastRun.getId()) + .pipelineName(pipelineLastRun.getPipelineName()) + .lastRunDate(lastRunDate) + .build(); + pipelineLastRunRepository.save(updated); } } diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java index e7201cb5a..7389ad372 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java @@ -32,9 +32,10 @@ public class PipelineUtils { private static final Logger log = LoggerFactory.getLogger(PipelineUtils.class); - public static Collection getLastRunFilters(LocalDate lastRunDate) { + public static Collection getLastRunFilters(LocalDate lastRunDate, LocalDate startDate) { Filter fromDateFilter = new DateRangeFilter.DateRangeFilterBuilder("update") .from(lastRunDate.atStartOfDay().toInstant(ZoneOffset.UTC)) + .until(startDate.atStartOfDay().toInstant(ZoneOffset.UTC)) .build(); return List.of(fromDateFilter); } diff --git a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java index 5c1ec9dbb..97544ee69 100644 --- a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -1,29 +1,15 @@ /* -* Copyright 2021 EMBL - European Bioinformatics Institute -* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this -* file except in compliance with the License. You may obtain a copy of the License at -* http://www.apache.org/licenses/LICENSE-2.0 -* Unless required by applicable law or agreed to in writing, software distributed under the -* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -* CONDITIONS OF ANY KIND, either express or implied. See the License for the -* specific language governing permissions and limitations under the License. -*/ + * Copyright 2021 EMBL - European Bioinformatics Institute + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ package uk.ac.ebi.biosamples; -import org.apache.http.HeaderElement; -import org.apache.http.HeaderElementIterator; -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.conn.ConnectionKeepAliveStrategy; -import org.apache.http.conn.routing.HttpRoute; -import org.apache.http.impl.client.cache.CacheConfig; -import org.apache.http.impl.client.cache.CachingHttpClientBuilder; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.http.message.BasicHeaderElementIterator; -import org.apache.http.protocol.HTTP; -import org.apache.http.protocol.HttpContext; import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -36,7 +22,6 @@ import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.biosamples.configuration.ExclusionConfiguration; @@ -48,9 +33,9 @@ @SpringBootApplication(exclude = DataSourceAutoConfiguration.class) @ComponentScan( excludeFilters = { - @ComponentScan.Filter( - type = FilterType.ASSIGNABLE_TYPE, - value = {EnaConfig.class, EraProDao.class, EnaSampleToBioSampleConversionService.class}) + @ComponentScan.Filter( + type = FilterType.ASSIGNABLE_TYPE, + value = {EnaConfig.class, EraProDao.class, EnaSampleToBioSampleConversionService.class}) }) @Import(ExclusionConfiguration.class) @EnableCaching diff --git a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java index 3ca4a9e68..c4eb4148d 100644 --- a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java +++ b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java @@ -23,6 +23,7 @@ import uk.ac.ebi.biosamples.core.model.PipelineAnalytics; import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.model.PipelineLastRun; import uk.ac.ebi.biosamples.model.PipelineName; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; import uk.ac.ebi.biosamples.service.PipelineHelperService; @@ -62,8 +63,10 @@ public CopydownApplicationRunner( @Override public void run(final ApplicationArguments args) throws Exception { - LocalDate lastRunDate = pipelineHelperService.getLastRunDate(PIPELINE_NAME); - final Collection filters = PipelineUtils.getLastRunFilters(lastRunDate); + PipelineLastRun pipelineLastRun = pipelineHelperService.getLastRunDate(PIPELINE_NAME); + LocalDate lastRunDate = pipelineLastRun.getLastRunDate(); + LocalDate startDate = LocalDate.now(); + final Collection filters = PipelineUtils.getLastRunFilters(lastRunDate, startDate); final Instant startTime = Instant.now(); LOG.info("Pipeline started at {}", startTime); LOG.info("Processing samples from {}", lastRunDate); @@ -98,6 +101,7 @@ public void run(final ApplicationArguments args) throws Exception { LOG.info("waiting for futures"); // wait for anything to finish ThreadUtils.checkAndCallbackFutures(futures, 0, pipelineFutureCallback); + pipelineHelperService.updateLastRunDate(pipelineLastRun, startDate); } catch (final Exception e) { LOG.error("Pipeline failed to finish successfully", e); throw e; diff --git a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java index fe1ac827f..5aee05155 100644 --- a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java +++ b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java @@ -34,6 +34,7 @@ import uk.ac.ebi.biosamples.core.model.Sample; import uk.ac.ebi.biosamples.core.model.SampleAnalytics; import uk.ac.ebi.biosamples.core.model.filter.Filter; +import uk.ac.ebi.biosamples.model.PipelineLastRun; import uk.ac.ebi.biosamples.model.PipelineName; import uk.ac.ebi.biosamples.mongo.model.MongoCurationRule; import uk.ac.ebi.biosamples.mongo.repository.MongoCurationRuleRepository; @@ -72,8 +73,10 @@ public CuramiApplicationRunner( @Override public void run(final ApplicationArguments args) throws Exception { - LocalDate lastRunDate = pipelineHelperService.getLastRunDate(PIPELINE_NAME); - final Collection filters = PipelineUtils.getLastRunFilters(lastRunDate); + PipelineLastRun pipelineLastRun = pipelineHelperService.getLastRunDate(PIPELINE_NAME); + LocalDate lastRunDate = pipelineLastRun.getLastRunDate(); + LocalDate startDate = LocalDate.now(); + final Collection filters = PipelineUtils.getLastRunFilters(lastRunDate, startDate); final Instant startTime = Instant.now(); LOG.info("Pipeline started at {}", startTime); LOG.info("Processing samples from {}", lastRunDate); @@ -112,6 +115,7 @@ public void run(final ApplicationArguments args) throws Exception { LOG.info("Waiting for all scheduled tasks to finish"); ThreadUtils.checkAndCallbackFutures(futures, 0, pipelineFutureCallback); + pipelineHelperService.updateLastRunDate(pipelineLastRun, startDate); } catch (final Exception e) { LOG.error("Pipeline failed to finish successfully", e); throw e; diff --git a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java index 4eeaebd35..9606868ce 100644 --- a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java +++ b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java @@ -32,6 +32,7 @@ import uk.ac.ebi.biosamples.core.model.filter.Filter; import uk.ac.ebi.biosamples.core.service.CurationApplicationService; import uk.ac.ebi.biosamples.curation.service.IriUrlValidatorService; +import uk.ac.ebi.biosamples.model.PipelineLastRun; import uk.ac.ebi.biosamples.model.PipelineName; import uk.ac.ebi.biosamples.mongo.service.AnalyticsService; import uk.ac.ebi.biosamples.service.PipelineHelperService; @@ -73,9 +74,11 @@ public CurationApplicationRunner( @Override public void run(final ApplicationArguments args) throws Exception { + PipelineLastRun pipelineLastRun = pipelineHelperService.getLastRunDate(PIPELINE_NAME); + LocalDate lastRunDate = pipelineLastRun.getLastRunDate(); + LocalDate startDate = LocalDate.now(); final Instant startTime = Instant.now(); - LocalDate lastRunDate = pipelineHelperService.getLastRunDate(PIPELINE_NAME); - final Collection filters = PipelineUtils.getLastRunFilters(lastRunDate); + final Collection filters = PipelineUtils.getLastRunFilters(lastRunDate, startDate); LOG.info("Pipeline started at {}", startTime); LOG.info("Processing samples from {}", lastRunDate); long sampleCount = 0; @@ -117,6 +120,7 @@ public void run(final ApplicationArguments args) throws Exception { LOG.info("waiting for futures"); // wait for anything to finish ThreadUtils.checkAndCallbackFutures(futures, 0, pipelineFutureCallback); + pipelineHelperService.updateLastRunDate(pipelineLastRun, startDate); } catch (final Exception e) { LOG.error("Pipeline failed to finish successfully", e); From 2cf1296b386a63d8316d7236741c4defea3eacf8 Mon Sep 17 00:00:00 2001 From: Isuru Liyanage Date: Fri, 12 Dec 2025 10:47:44 +0000 Subject: [PATCH 84/86] test: argo service account --- k8s/cronjobs/templates/workflow.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/cronjobs/templates/workflow.yaml b/k8s/cronjobs/templates/workflow.yaml index afb474062..d74627d06 100644 --- a/k8s/cronjobs/templates/workflow.yaml +++ b/k8s/cronjobs/templates/workflow.yaml @@ -6,6 +6,7 @@ metadata: labels: app.kubernetes.io/name: {{ include "cronjobs.name" . }} spec: + serviceAccountName: argo-workflow-sa entrypoint: {{ .Values.job.name }} {{- with .Values.imagePullSecrets }} imagePullSecrets: From 0dc29f71cdb31416cd39f81ab2cf74bf2601b4c5 Mon Sep 17 00:00:00 2001 From: dipayan1985 Date: Mon, 15 Dec 2025 11:59:16 +0000 Subject: [PATCH 85/86] reindex running required for fresh prod index. --- build-biosamples-search-image.sh | 9 ++++++++- .../services/SampleChecklistComplianceHandlerEVA.java | 4 +--- .../src/main/java/uk/ac/ebi/biosamples/Application.java | 4 +++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/build-biosamples-search-image.sh b/build-biosamples-search-image.sh index 57ef7473a..6389ed749 100755 --- a/build-biosamples-search-image.sh +++ b/build-biosamples-search-image.sh @@ -41,7 +41,14 @@ else fi echo "[3/3] Building Docker image: ${IMAGE_TAG}" -DOCKER_BUILDKIT=1 docker build -t "${IMAGE_TAG}" "${CONTEXT_DIR}" +# Check if buildx is available, otherwise use regular docker build +if docker buildx version >/dev/null 2>&1; then + echo "Using BuildKit (buildx available)" + DOCKER_BUILDKIT=1 docker build -t "${IMAGE_TAG}" "${CONTEXT_DIR}" +else + echo "BuildKit not available, using standard docker build" + docker build -t "${IMAGE_TAG}" "${CONTEXT_DIR}" +fi echo "${IMAGE_TAG} built successfully" diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java index c059fabe0..5697616bb 100644 --- a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java @@ -28,7 +28,6 @@ @Component public class SampleChecklistComplianceHandlerEVA { - private static final Logger log = LoggerFactory.getLogger(SampleChecklistComplianceHandlerEVA.class); @@ -38,9 +37,8 @@ public class SampleChecklistComplianceHandlerEVA { "geographic location (region and locality)"; private static final String COLLECTION_DATE = "collection_date"; private static final String COLLECTION_DATE_WITHOUT_UNDERSCORE = "collection date"; - private static final String NCBI_MIRRORING_WEBIN_ID = "Webin-842"; - private final BioSamplesClient bioSamplesWebinClient; + private final BioSamplesClient bioSamplesWebinClient; private final PipelinesProperties pipelinesProperties; public SampleChecklistComplianceHandlerEVA( diff --git a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java index 3aab8f7b1..07afc3fd8 100644 --- a/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/reindex/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -38,6 +38,7 @@ import uk.ac.ebi.biosamples.service.EnaConfig; import uk.ac.ebi.biosamples.service.EnaSampleToBioSampleConversionService; import uk.ac.ebi.biosamples.service.EraProDao; +import uk.ac.ebi.biosamples.service.PipelineHelperService; import uk.ac.ebi.biosamples.utils.PipelineUtils; @SpringBootApplication(exclude = DataSourceAutoConfiguration.class) @@ -49,7 +50,8 @@ EnaConfig.class, EraProDao.class, EnaSampleToBioSampleConversionService.class, - BioSamplesWebSecurityConfig.class + BioSamplesWebSecurityConfig.class, + PipelineHelperService.class }), @ComponentScan.Filter( type = FilterType.REGEX, From ac4dd3673e31be2a22ed682b791b609d2f2d86af Mon Sep 17 00:00:00 2001 From: dipayan1985 Date: Mon, 15 Dec 2025 12:00:34 +0000 Subject: [PATCH 86/86] spotless code --- .../ebi/biosamples/RestFacetIntegration.java | 19 +++++--- .../SampleChecklistComplianceHandlerEVA.java | 2 +- .../ebi/biosamples/model/PipelineLastRun.java | 16 +++++-- .../repository/PipelineLastRunRepository.java | 15 ++++-- .../service/PipelineHelperService.java | 33 +++++++++---- .../ebi/biosamples/utils/PipelineUtils.java | 10 ++-- .../uk/ac/ebi/biosamples/Application.java | 24 +++++----- .../copydown/CopydownApplicationRunner.java | 48 +++++++++---------- .../curation/CuramiApplicationRunner.java | 3 +- .../curation/CurationApplicationRunner.java | 3 +- 10 files changed, 105 insertions(+), 68 deletions(-) diff --git a/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java b/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java index 6aa4e6a4a..832eca5a5 100644 --- a/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java +++ b/integration/src/main/java/uk/ac/ebi/biosamples/RestFacetIntegration.java @@ -281,24 +281,29 @@ private void facetEndpointShouldReturnAllFacetValuesWhenFacetFilterIsAvailable() ResponseEntity response = restTemplate.getForEntity(url, String.class); log.info("Response status: {}", response.getStatusCode()); log.info("Response body: {}", response.getBody()); - + JsonNode node = objectMapper.readTree(response.getBody()); JsonNode embedded = node.get("_embedded"); if (embedded == null) { - log.error("Response does not contain '_embedded' field. Full response: {}", response.getBody()); + log.error( + "Response does not contain '_embedded' field. Full response: {}", response.getBody()); throw new IntegrationTestFailException( - "Facet endpoint response does not contain '_embedded' field. Response: " + response.getBody(), + "Facet endpoint response does not contain '_embedded' field. Response: " + + response.getBody(), Phase.SIX); } - + JsonNode facets = embedded.get("facets"); if (facets == null) { - log.error("Response does not contain 'facets' field. '_embedded' content: {}", embedded.toString()); + log.error( + "Response does not contain 'facets' field. '_embedded' content: {}", + embedded.toString()); throw new IntegrationTestFailException( - "Facet endpoint response does not contain 'facets' field. Response: " + response.getBody(), + "Facet endpoint response does not contain 'facets' field. Response: " + + response.getBody(), Phase.SIX); } - + for (JsonNode facet : facets) { if ("SRA accession".equals(facet.get("label").asText())) { if (facet.get("content").size() < 10) { diff --git a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java index 5697616bb..8b0082ce2 100644 --- a/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java +++ b/pipelines/chain/src/main/java/uk/ac/ebi/biosamples/helpdesk/services/SampleChecklistComplianceHandlerEVA.java @@ -38,7 +38,7 @@ public class SampleChecklistComplianceHandlerEVA { private static final String COLLECTION_DATE = "collection_date"; private static final String COLLECTION_DATE_WITHOUT_UNDERSCORE = "collection date"; - private final BioSamplesClient bioSamplesWebinClient; + private final BioSamplesClient bioSamplesWebinClient; private final PipelinesProperties pipelinesProperties; public SampleChecklistComplianceHandlerEVA( diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineLastRun.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineLastRun.java index 463073ac8..d1362319b 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineLastRun.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/model/PipelineLastRun.java @@ -1,20 +1,28 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ package uk.ac.ebi.biosamples.model; +import java.time.LocalDate; import lombok.Builder; import lombok.Getter; import lombok.extern.jackson.Jacksonized; import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.mapping.Document; -import java.time.LocalDate; - @Document @Jacksonized @Builder @Getter public class PipelineLastRun { - @Id - private String id; + @Id private String id; private PipelineName pipelineName; private LocalDate lastRunDate; } diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java index 75313d74e..348121d00 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/repository/PipelineLastRunRepository.java @@ -1,15 +1,22 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ package uk.ac.ebi.biosamples.repository; +import java.util.Optional; import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.stereotype.Repository; import uk.ac.ebi.biosamples.model.PipelineLastRun; import uk.ac.ebi.biosamples.model.PipelineName; -import java.util.Optional; - @Repository public interface PipelineLastRunRepository extends MongoRepository { Optional findFirstByPipelineName(PipelineName pipelineName); - - } diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java index 38e8acfe2..6c3fbdb1d 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/service/PipelineHelperService.java @@ -1,5 +1,16 @@ +/* +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ package uk.ac.ebi.biosamples.service; +import java.time.LocalDate; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -7,8 +18,6 @@ import uk.ac.ebi.biosamples.model.PipelineName; import uk.ac.ebi.biosamples.repository.PipelineLastRunRepository; -import java.time.LocalDate; - @Service @Slf4j @RequiredArgsConstructor @@ -16,16 +25,22 @@ public class PipelineHelperService { private final PipelineLastRunRepository pipelineLastRunRepository; public PipelineLastRun getLastRunDate(PipelineName pipelineName) { - return pipelineLastRunRepository.findFirstByPipelineName(pipelineName) - .orElse(PipelineLastRun.builder().pipelineName(pipelineName).lastRunDate(LocalDate.EPOCH).build()); + return pipelineLastRunRepository + .findFirstByPipelineName(pipelineName) + .orElse( + PipelineLastRun.builder() + .pipelineName(pipelineName) + .lastRunDate(LocalDate.EPOCH) + .build()); } public void updateLastRunDate(PipelineLastRun pipelineLastRun, LocalDate lastRunDate) { - PipelineLastRun updated = PipelineLastRun.builder() - .id(pipelineLastRun.getId()) - .pipelineName(pipelineLastRun.getPipelineName()) - .lastRunDate(lastRunDate) - .build(); + PipelineLastRun updated = + PipelineLastRun.builder() + .id(pipelineLastRun.getId()) + .pipelineName(pipelineLastRun.getPipelineName()) + .lastRunDate(lastRunDate) + .build(); pipelineLastRunRepository.save(updated); } } diff --git a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java index 7389ad372..a61b27dba 100644 --- a/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java +++ b/pipelines/common/src/main/java/uk/ac/ebi/biosamples/utils/PipelineUtils.java @@ -18,7 +18,6 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.*; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.ApplicationArguments; @@ -33,10 +32,11 @@ public class PipelineUtils { private static final Logger log = LoggerFactory.getLogger(PipelineUtils.class); public static Collection getLastRunFilters(LocalDate lastRunDate, LocalDate startDate) { - Filter fromDateFilter = new DateRangeFilter.DateRangeFilterBuilder("update") - .from(lastRunDate.atStartOfDay().toInstant(ZoneOffset.UTC)) - .until(startDate.atStartOfDay().toInstant(ZoneOffset.UTC)) - .build(); + Filter fromDateFilter = + new DateRangeFilter.DateRangeFilterBuilder("update") + .from(lastRunDate.atStartOfDay().toInstant(ZoneOffset.UTC)) + .until(startDate.atStartOfDay().toInstant(ZoneOffset.UTC)) + .build(); return List.of(fromDateFilter); } diff --git a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java index 97544ee69..86d594840 100644 --- a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java +++ b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/Application.java @@ -1,13 +1,13 @@ /* - * Copyright 2021 EMBL - European Bioinformatics Institute - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ package uk.ac.ebi.biosamples; import org.springframework.boot.SpringApplication; @@ -33,9 +33,9 @@ @SpringBootApplication(exclude = DataSourceAutoConfiguration.class) @ComponentScan( excludeFilters = { - @ComponentScan.Filter( - type = FilterType.ASSIGNABLE_TYPE, - value = {EnaConfig.class, EraProDao.class, EnaSampleToBioSampleConversionService.class}) + @ComponentScan.Filter( + type = FilterType.ASSIGNABLE_TYPE, + value = {EnaConfig.class, EraProDao.class, EnaSampleToBioSampleConversionService.class}) }) @Import(ExclusionConfiguration.class) @EnableCaching diff --git a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java index c4eb4148d..f1bfb3fed 100644 --- a/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java +++ b/pipelines/copydown/src/main/java/uk/ac/ebi/biosamples/copydown/CopydownApplicationRunner.java @@ -1,15 +1,22 @@ /* - * Copyright 2021 EMBL - European Bioinformatics Institute - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ +* Copyright 2021 EMBL - European Bioinformatics Institute +* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +* file except in compliance with the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software distributed under the +* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +* CONDITIONS OF ANY KIND, either express or implied. See the License for the +* specific language governing permissions and limitations under the License. +*/ package uk.ac.ebi.biosamples.copydown; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.ApplicationArguments; @@ -31,14 +38,6 @@ import uk.ac.ebi.biosamples.utils.thread.AdaptiveThreadPoolExecutor; import uk.ac.ebi.biosamples.utils.thread.ThreadUtils; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDate; -import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.Future; - @Component public class CopydownApplicationRunner implements ApplicationRunner { private static final Logger LOG = LoggerFactory.getLogger(CopydownApplicationRunner.class); @@ -53,7 +52,8 @@ public class CopydownApplicationRunner implements ApplicationRunner { public CopydownApplicationRunner( final BioSamplesClient bioSamplesClient, final PipelinesProperties pipelinesProperties, - final AnalyticsService analyticsService, PipelineHelperService pipelineHelperService) { + final AnalyticsService analyticsService, + PipelineHelperService pipelineHelperService) { this.bioSamplesClient = bioSamplesClient; this.pipelinesProperties = pipelinesProperties; this.analyticsService = analyticsService; @@ -73,12 +73,12 @@ public void run(final ApplicationArguments args) throws Exception { long sampleCount = 0; try (final AdaptiveThreadPoolExecutor executorService = - AdaptiveThreadPoolExecutor.create( - 100, - 10000, - true, - pipelinesProperties.getThreadCount(), - pipelinesProperties.getThreadCountMax())) { + AdaptiveThreadPoolExecutor.create( + 100, + 10000, + true, + pipelinesProperties.getThreadCount(), + pipelinesProperties.getThreadCountMax())) { final Map> futures = new HashMap<>(); for (final EntityModel sampleResource : diff --git a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java index 5aee05155..c0a5a25f8 100644 --- a/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java +++ b/pipelines/curami/src/main/java/uk/ac/ebi/biosamples/curation/CuramiApplicationRunner.java @@ -61,7 +61,8 @@ public CuramiApplicationRunner( final BioSamplesClient bioSamplesClient, final PipelinesProperties pipelinesProperties, final MongoCurationRuleRepository repository, - final AnalyticsService analyticsService, PipelineHelperService pipelineHelperService) { + final AnalyticsService analyticsService, + PipelineHelperService pipelineHelperService) { this.bioSamplesClient = bioSamplesClient; this.pipelinesProperties = pipelinesProperties; this.repository = repository; diff --git a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java index 9606868ce..216a69a8c 100644 --- a/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java +++ b/pipelines/curation/src/main/java/uk/ac/ebi/biosamples/curation/CurationApplicationRunner.java @@ -61,7 +61,8 @@ public CurationApplicationRunner( final OlsProcessor olsProcessor, final CurationApplicationService curationApplicationService, final AnalyticsService analyticsService, - final IriUrlValidatorService iriUrlValidatorService, PipelineHelperService pipelineHelperService) { + final IriUrlValidatorService iriUrlValidatorService, + PipelineHelperService pipelineHelperService) { this.bioSamplesClient = bioSamplesClient; this.pipelinesProperties = pipelinesProperties; this.olsProcessor = olsProcessor;