();
- columnHeaders.addAll(result.getBindingNames());
- writer.startQueryResult(columnHeaders);
- while (result.hasNext()) {
- BindingSet b = result.next();
- writer.handleSolution(b);
- }
- writer.endQueryResult();
- resultJson = out.toString();
- resultJson = JenaResultSetFormatter.convertToSimplifiedJsonArray(resultJson).toString();
-
- } catch (IOException exc) {
- throw new JPSRuntimeException(exc);
- }
- }
- }
- LOGGER.debug(QueryPlanLog.getQueryPlan());
- MonitoringUtil.printMonitoringInformation(repository.getFederationContext());
-
- repository.shutDown();
- return resultJson;
- }
- };
-
- return impl;
- }
-
- /**
- * Executes a SPARQL 1.1 query with specified endpoints, e.g.
- *
- * PREFIX ...
- * SELECT ?exp ?yield_value ?yield_unit
- * WHERE {
- * SERVICE ?service1 {
- * ?exp ontorxn:hasYield ?yield.
- * ?yield om:hasValue ?measure.
- * ?measure om:hasNumericalValue ?yield_value; om:hasUnit ?yield_unit.
- * }
- * SERVICE ?service2 {
- * <https://www.example.com/triplestore/ontodoe/DoE_1/DoE_1> ontodoe:utilisesHistoricalData ?historical_data.
- * ?historical_data ontodoe:refersTo ?exp.
- * }
- * VALUES ?service1 { <http://172.17.0.3:8080/blazegraph/namespace/lab_1/sparql> <http://172.17.0.4:8080/blazegraph/namespace/lab_2/sparql> }
- * VALUES ?service2 { <http://172.17.0.3:8080/blazegraph/namespace/doe_chemrxn/sparql> }
- * }
- * Such a query can be executed either on RDF4J server or Blazegraph server.
- * Both cases require any dataset or namespace, resp. as a starting point for the query request.
- * This is an endpoint URL specified by fedEngineUrl, e.g.
- *
- * http://www.theworldavatar.com/blazegraph/namespace/somenamesapce/sparql
- * http://localhost:8080/rdf4j-server/repositories/somenamespace
- *
- *
- * @param fedEngineUrl endpoint URL as starting point for the request
- * @return
- */
- public static FederatedQueryInterface createForQueriesWithGivenEndpoints(String fedEngineUrl) {
- return create(fedEngineUrl, null);
- }
-
- /**
- * Executes a SPARQL 1.1 query with and without specified endpoints.
- * If data sources should be selected automatically,
- * the query must contain SERVICE clauses with service variables but without VALUES clauses.
- * As an examples, see the query in {@link #createForQueriesWithGivenEndpoints(String)} without the VALUES.
- *
- * The code selects automatically the relevant data sources for the basic triple patterns within a SERVICE clause
- * and adds a VALUES clause with corresponding endpoint URLs for each SERVICE clause.
- * Finally, it executes the completed SPARQL 1.1 query as in {@link #createForQueriesWithGivenEndpoints(String)}.
- *
- * @param fedEngineUrl endpoint URL as starting point for the request
- * @param servDescrIndexer contains the schema-level index for all federation members to allow automated data source selection
- * @param host2host if not null, host2host is used to convert host addresses;
- * conversion may necessary e.g. in a test environment with Docker containers where Blazegraph servers return endpoint URLs
- * with wrong host addresses (such as "localhost:8080")
- * @return
- */
- public static FederatedQueryInterface createWithEndpointSelection(
- String fedEngineUrl, ServiceDescriptionIndexer servDescrIndexer, Map host2host) {
-
- DataSourceSelector selector = new DataSourceSelector(servDescrIndexer, host2host);
- return create(fedEngineUrl, selector);
- }
-
- /**
- * A wrapper for four engine configurations:service descriptions .
- * Currently, this is the case for Blazegraph server but not for RDF4J server
- *
- * @param cached if true and the indexer was created before, the same indexer is returned;
- * otherwise an indexer is created from scratch
- * @param serviceUrls
- * @param endpointUrls
- * @return
- */
- public static ServiceDescriptionIndexer getIndexer(boolean cached, List serviceUrls, List endpointUrls) {
- if (cached && (indexer != null)) {
- return indexer;
- }
-
- // collect all endpoint Urls
- List allEndpoints = new ArrayList();
- if (serviceUrls != null) {
- for (String serviceUrl : serviceUrls) {
- BlazegraphRepositoryWrapper wrapper = new BlazegraphRepositoryWrapper(serviceUrl);
- List endpoints = wrapper.queryNamespaces(true);
- allEndpoints.addAll(endpoints);
- }
- }
- if (endpointUrls != null) {
- allEndpoints.addAll(endpointUrls);
- }
-
- if (allEndpoints.isEmpty()) {
- throw new JPSRuntimeException("At least one endpoint is required, serviceUrls=" + serviceUrls + ", endpointUrls=" + endpointUrls);
- }
-
- // read all service descriptions
- indexer = new ServiceDescriptionIndexer();
- for (String url : allEndpoints) {
- indexer.addServiceDescription(url);
- }
-
- return indexer;
- }
-}
diff --git a/src/main/java/uk/ac/cam/cares/jps/base/query/fed/FederatedQueryInterface.java b/src/main/java/uk/ac/cam/cares/jps/base/query/fed/FederatedQueryInterface.java
deleted file mode 100644
index f9b528dfb..000000000
--- a/src/main/java/uk/ac/cam/cares/jps/base/query/fed/FederatedQueryInterface.java
+++ /dev/null
@@ -1,31 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import uk.ac.cam.cares.jps.base.query.JenaResultSetFormatter;
-
-/**
- * A common interface for different engines / implementations for executing federated queries.
- *
- */
-public interface FederatedQueryInterface {
-
- /**
- * Executes the SPARQL query as federated query and returns the result set as JSON array string.
- * Each element of the JSON array represents a row in the result set and consists of key-value pairs.
- * Example of a JSON element:
- *
- * {"generation":"http://www.theworldavatar.com/ontology/ontoeip/powerplants/PowerPlant.owl#NaturalGasGeneration",
- * The above format is a simplified version of the official W3C JSON format
- * as described in {@link https://www.w3.org/TR/rdf-sparql-json-res/}.
- * Concerning the above example in the official W3C JSON format see
- * {@link JenaResultSetFormatter#convertToJSONW3CStandard(org.apache.jena.query.ResultSet)}.
- *
- * @param sparql
- * @return query result set
- * @throws Exception
- */
- public String executeFederatedQuery(String sparql);
-}
diff --git a/src/main/java/uk/ac/cam/cares/jps/base/query/fed/ParsedQueryTreeVisitor.java b/src/main/java/uk/ac/cam/cares/jps/base/query/fed/ParsedQueryTreeVisitor.java
deleted file mode 100644
index 1b3810c4e..000000000
--- a/src/main/java/uk/ac/cam/cares/jps/base/query/fed/ParsedQueryTreeVisitor.java
+++ /dev/null
@@ -1,173 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.eclipse.rdf4j.model.Value;
-import org.eclipse.rdf4j.model.impl.SimpleIRI;
-import org.eclipse.rdf4j.query.algebra.BindingSetAssignment;
-import org.eclipse.rdf4j.query.algebra.QueryModelNode;
-import org.eclipse.rdf4j.query.algebra.Service;
-import org.eclipse.rdf4j.query.algebra.StatementPattern;
-import org.eclipse.rdf4j.query.algebra.Var;
-import org.eclipse.rdf4j.query.algebra.helpers.AbstractQueryModelVisitor;
-
-import uk.ac.cam.cares.jps.base.query.sparql.PrefixToUrlMap;
-import uk.ac.cam.cares.jps.base.query.sparql.Prefixes;
-
-/**
- * This class visits all tree nodes of a parsed SPARQL query
- * and retrieves information about SERVICE and VALUES clauses.
- * It collects
- * (1) names and bounded values (i.e. endpoint URLs) of SERVICE variables
- * (2) IRIs of relations (used as predicates in basic triple patterns) and
- * (3) IRIs of classes (used as objects in basic triple patterns with predicate "rdf:type")
- *
- */
-public class ParsedQueryTreeVisitor extends AbstractQueryModelVisitor {
-
- static final Logger LOGGER = LogManager.getLogger(ParsedQueryTreeVisitor.class);
-
- /**
- * A helper class to store the information related to a query's SERVICE clause.
- *
- */
- public class ServiceGraphPatternSummary {
- public String serviceVarName = null;
- public String serviceVarValue = null;
- public List keys = null;
- }
-
- private static final String BLAZEGRAPH_GEO_PREFIX = PrefixToUrlMap.getPrefixUrl(Prefixes.BLAZEGRAPH_GEO);
- private List summaries = new ArrayList();
- private List allKeys = new ArrayList();
- /**
- * Relations and classes found in basic triple patterns of the currently visited SERVICE clause.
- */
- private List currentKeys = null;
- private Set bindingNames = new HashSet();
-
- public List getSummariesWithoutBindings() {
- List result = new ArrayList();
- for (ServiceGraphPatternSummary current : summaries) {
- if (!bindingNames.contains(current.serviceVarName)) {
- result.add(current);
- }
- }
-
- return result;
- }
-
- /**
- * Delegates if the node type is relevant for extracting information
- * (e.g. in case of StatementPattern node containing a basic triple pattern).
- */
- @Override
- protected void meetNode(QueryModelNode node) {
-
- LOGGER.debug("node=" + node.getSignature());
-
- if (node instanceof Service) {
- meetServiceNode((Service) node);
- } else if (node instanceof StatementPattern){
- meetStatementPatternNode((StatementPattern) node);
- } else if (node instanceof BindingSetAssignment) {
- meetBindingSetAssignmentNode((BindingSetAssignment) node);
- } else {
- super.meetNode(node);
- }
- }
-
- /**
- * Starts and stops to collect information for a SERVICE clause and its enclosed triple patterns.
- * @param node
- */
- protected void meetServiceNode(Service node) {
-
- Var serviceRef = node.getServiceRef();
- Value value = serviceRef.getValue();
- String svalue = (value == null)? null : value.toString();
- if (serviceRef.isAnonymous() && (value instanceof SimpleIRI)) {
- // i.e. the endpoint IRI for SERVICE is already given
- super.meetNode(node);
- LOGGER.debug("Service node: No keys for data source selection are collected since endpoint IRI is given!");
- return;
- } else if (svalue != null && svalue.startsWith(BLAZEGRAPH_GEO_PREFIX)) {
- // i.e. SERVICE keyword marks a SPARQL subquery
- super.meetNode(node);
- LOGGER.debug("Service node: No keys for data source selection are collected since Blazegraph's special geo service was identified!");
- return;
- }
- // else:
- // i.e. SERVICE keyword is used to mark Blazegraph's geospatial service
- // triple patterns within the SERVICE body should not be considered for data source selection
- // (the geo vocabulary is not part of the service description)
- // nothing to do, only remove the geo vocabulary from collected keys
-
-
- ServiceGraphPatternSummary summary = new ServiceGraphPatternSummary();
- summary.serviceVarName = node.getServiceRef().getName();
- if (svalue != null) {
- summary.serviceVarValue = svalue;
- }
- currentKeys = new ArrayList();
-
- super.meetNode(node);
-
- summary.keys = currentKeys;
- currentKeys = null;
-
- summaries.add(summary);
- LOGGER.debug("The following keys were collected: " + summary.keys);
- }
-
- private String addKey(Var variable) {
- Value value = variable.getValue();
- if (value != null) {
- String key = value.stringValue();
- if (key.startsWith(BLAZEGRAPH_GEO_PREFIX))
- // don't consider Blazegraph special vocabulary for each geo service
- return null;
- allKeys.add(key);
- if (currentKeys != null) {
- currentKeys.add(key);
- //LOGGER.debug("collected new key=" + key);
- return key;
- }
- }
- return null;
- }
-
- /**
- * Extracts relation and class IRIs from bounded predicates and objects of a basic triple pattern.
- *
- * @param node
- */
- protected void meetStatementPatternNode(StatementPattern node) {
- // Only add predicate and class IRIs as keys.
- // If getSubjectVar() is bounded to a value, the value is usually
- // not a class IRI and should not be added.
- // Similarly, íf getObjectVar() is bounded to a value and the predicate is not rdf:type.
- String key = addKey(node.getPredicateVar());
- if ((key != null) && (key == "http://www.w3.org/1999/02/22-rdf-syntax-ns#type")) {
- addKey(node.getObjectVar());
- }
-
- super.meetNode(node);
- }
-
- /**
- * Collects bounded values (i.e. endpoint URLs) from a VALUES clause.
- *
- * @param node
- */
- protected void meetBindingSetAssignmentNode(BindingSetAssignment node) {
- LOGGER.debug("binding names=" + node.getBindingNames());
- bindingNames.addAll(node.getBindingNames());
- super.meetNode(node);
- }
-}
diff --git a/src/main/java/uk/ac/cam/cares/jps/base/query/fed/ServiceDescriptionIndexer.java b/src/main/java/uk/ac/cam/cares/jps/base/query/fed/ServiceDescriptionIndexer.java
deleted file mode 100644
index 2ba581987..000000000
--- a/src/main/java/uk/ac/cam/cares/jps/base/query/fed/ServiceDescriptionIndexer.java
+++ /dev/null
@@ -1,407 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Function;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.jena.ontology.OntModel;
-import org.apache.jena.query.QuerySolution;
-import org.apache.jena.query.ResultSet;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.json.JSONArray;
-import org.json.JSONObject;
-
-import uk.ac.cam.cares.jps.base.exception.JPSRuntimeException;
-import uk.ac.cam.cares.jps.base.http.Http;
-import uk.ac.cam.cares.jps.base.query.JenaHelper;
-import uk.ac.cam.cares.jps.base.query.JenaResultSetFormatter;
-
-/**
- * This class creates a schema-level index from
- * service descriptions .
- * A service description is an RDF graph containing information about a SPARQL service and its SPARQL endpoints.
- * Among other things, the RDF graph usually contains features of the SPARQL service
- * and the number of occurrences of relations and classes in triples of the datasets.
- * This class uses the IRIs of relations and classes for indexing.
- * Each relation and class IRI is mapped to a "postings" list of endpoints / datasets.
- * An endpoint / dataset is added to the "postings" list for an IRI if it contains at least
- * one triple with this IRI as predicate or object, resp.
- * The created index allows to perform conjunctive keyword-based queries.
- * Since the keywords are relation and class IRIs,
- * the query result can be used for automated endpoint selection.
- *
- */
-public class ServiceDescriptionIndexer {
-
- static final Logger LOGGER = LogManager.getLogger(ServiceDescriptionIndexer.class);
-
- /**
- * A helper class to store relevant information filtered from a service description.
- *
- */
- public class ServiceDescriptionSummary {
- public Integer id = null;
- public String path = null;
- public String endpointURL = null;
- public Long ntriples = null;
- public Long nentities = null;
- public Long nproperties = null;
- public Long nclasses = null;
-
- public ServiceDescriptionSummary(int id) {
- this.id = id;
- }
-
- @Override
- public String toString() {
- return "Summary[#t=" + ntriples + ",#e=" +nentities
- + ",#c=" + nclasses + ",#p=" + nproperties + ",url=" + endpointURL + "]";
- }
- }
-
- public class PostingsListElement implements Comparable{
- /**
- * ID of the summary of the corresponding service description
- */
- public Integer id = null;
- /**
- * The number of triples in the corresponding datasets wrt. to the index key (relation or class IRI)
- */
- public Long tripleNumber = null;
-
- public PostingsListElement(Integer id, Long number) {
- this.id = id;
- this.tripleNumber = number;
- }
-
- /**
- * Elements in the postings list are ordered by their ID.
- */
- @Override
- public int compareTo(PostingsListElement element) {
- return this.id.compareTo(element.id);
- }
- }
-
- private final static String INDEX_NAME = "SCHEMA";
- /**
- * Each summary in the list represents the relevant information of a service description
- */
- private List summaries = new ArrayList();
- private SimpleMultiIndex index = new SimpleMultiIndex();
-
- public ServiceDescriptionIndexer() {
- }
-
- private Map> getSchemaIndex() {
- return index.getIndex(INDEX_NAME);
- }
-
- /**
- * Returns all indexed relation and class IRIs
- *
- * @return
- */
- public Set getKeys() {
- return getSchemaIndex().keySet();
- }
-
- public Set getPostingsList(String key) {
- return index.getPostingsList(INDEX_NAME, key);
- }
-
- /**
- * Returns a list of summaries for all added service descriptions.
- *
- * @return
- */
- public List getSummaries() {
- return summaries;
- }
-
- /**
- * Creates a map between IRIs (classes or properties depending on the given type)
- * and the number of triples in the service description.
- *
- * @param model contains the service description as RDF graph
- * @param sparql to query the RDF graph
- * @param type either class or property
- * @return
- */
- private static Map getNumberOfTriples(OntModel model, String sparql, String type) {
- ResultSet result = JenaHelper.query(model, sparql);
- JSONObject joOrig = JenaResultSetFormatter.convertToSimplifiedList(result);
- JSONArray list = joOrig.getJSONArray("results");
-
- Iterator it = list.iterator();
- Map map = new HashMap();
- while (it.hasNext()) {
- JSONObject current = (JSONObject) it.next();
- map.put(current.getString(type), current.getLong("ntriples"));
- }
- return map;
- }
-
- /**
- * Queries the service description for the number of triples wrt. classes.
- *
- * @param model contains the service description as RDF graph
- * @return
- */
- public static Map getNumberOfClassTriples(OntModel model) {
- String sparql = "SELECT * WHERE { \r\n"
- + " ?s ?class . \r\n"
- + " ?s ?ntriples . \r\n"
- + "}";
- return getNumberOfTriples(model, sparql, "class");
- }
-
- /**
- * Queries the service description for the number of triples wrt. properties.
- *
- * @param model contains the service description as RDF graph
- * @return
- */
- public static Map getNumberOfPropertyTriples(OntModel model) {
- String sparql = "SELECT * WHERE { \r\n"
- + " ?s ?property . \r\n"
- + " ?s ?ntriples . \r\n"
- + "}";
- return getNumberOfTriples(model, sparql, "property");
- }
-
- /**
- * Adds new service descriptions for indexing.
- * Three different types are allowed:
- *
- * (1) A local file given by its path:
- * The file represents a (serialized) service description stored locally.
- * SPARQL protocol ).
- *
- * @param dirFileorUrl a (mixed) list of directories, files or URLs
- */
- public void addServiceDescription(String... dirFileorUrl) {
-
- for (String path : dirFileorUrl) {
- File file = new File(path);
- if (file.isDirectory()) {
- Collection files = FileUtils.listFiles(file, null, true);
- for (File f : files) {
- add(f.getAbsolutePath());
- }
- } else {
- add(path);
- }
- }
- }
-
- /**
- * Requests the service description from the given URL
- *
- * @param url
- * @return
- */
- public static String queryServiceDescription(String url) {
- String accept = "application/rdf+xml";
- HttpGet request = Http.get(url, accept);
- return Http.execute(request);
- }
-
- /**
- * Reads the file or request the URL,
- * queries relevant information from the corresponding service description,
- * and adds it to index
- *
- * @param path a file or URL
- */
- private void add(String path) {
-
- OntModel model = null;
- if (path.startsWith("http")) {
- String descr = queryServiceDescription(path);
- model = JenaHelper.createModel();
- JenaHelper.readFromString(descr, model);
- } else {
- model = JenaHelper.createModel(path);
- }
-
- ServiceDescriptionSummary summary = createSummary();
- fillSummary(summary, path, model);
-
- // add classes and their counts to index
- Map map = getNumberOfClassTriples(model);
- for (String key : map.keySet()) {
- PostingsListElement elem = new PostingsListElement(summary.id, map.get(key));
- index.add(INDEX_NAME, key, elem);
- }
-
- // add properties and their counts to index
- map = getNumberOfPropertyTriples(model);
- for (String key : map.keySet()) {
- PostingsListElement elem = new PostingsListElement(summary.id, map.get(key));
- index.add(INDEX_NAME, key, elem);
- }
-
-
- LOGGER.debug("Added service description=" + summary + " from path=" + path);
- }
-
- /**
- * Queries the service description for information that is relevant for indexing
- * and conjunctive queries and adds the information to the given summary.
- *
- * @param summary empty summary to add relevant information
- * @param path file or URL
- * @param model contains the service description as RDF graph
- */
- public void fillSummary(ServiceDescriptionSummary summary, String path, OntModel model) {
-
- summary.path = path;
-
- String sparql = "PREFIX rdf: \r\n"
- + "PREFIX sd: \r\n"
- + "PREFIX void: \r\n"
- + "SELECT * WHERE {\r\n"
- + " ?dataset rdf:type sd:Dataset .\r\n"
- + " ?dataset sd:defaultGraph ?graph .\r\n"
- + " ?graph void:triples ?ntriples .\r\n"
- + " ?graph void:entities ?nentities .\r\n"
- + " ?graph void:properties ?nproperties .\r\n"
- + " ?graph void:classes ?nclasses .\r\n"
- + "}";
-
- ResultSet result = JenaHelper.query(model, sparql);
- JSONObject jo = JenaResultSetFormatter.convertToSimplifiedList(result);
- JSONArray list = jo.getJSONArray("results");
-
- Iterator it = list.iterator();
- JSONObject current = (JSONObject) it.next();
- if (it.hasNext()) {
- throw new JPSRuntimeException("The query result set contains more than one row, service description=" + path);
- }
-
-
-
- // the following "lambda method" get returns null if the key is not found
- // in contrast to current.get which throws an exception in this case
- Function get = ( s -> {
- if (current.has(s))
- {return current.getLong(s); };
- return null;
- });
-
- summary.ntriples = get.apply("ntriples");
- summary.nentities = get.apply("nentities");
- summary.nproperties = get.apply("nproperties");
- summary.nclasses = get.apply("nclasses");
-
-
- // query endpoint URL
- sparql = "PREFIX rdf: \r\n"
- + "PREFIX sd: \r\n"
- + "PREFIX void: \r\n"
- + "SELECT * WHERE {\r\n"
- + " ?service sd:endpoint ?endpoint .\r\n"
- + "}";
-
- result = JenaHelper.query(model, sparql);
- QuerySolution s = result.nextSolution();
- String endpointURL = s.get("endpoint").asResource().getURI();
- // HACK: The Blazegraph instance for citiesKG does not provide the correct endpoint URL
- // e.g. the port in http://localhost:9999/blazegraph/namespace/singaporeEPSG24500/sparql is incorrect
- // maybe it is not configured correctly
- if (endpointURL.startsWith("http://localhost:9999")) {
- int i = endpointURL.indexOf("namespace/");
- String namespace = endpointURL.substring(i, endpointURL.length());
- summary.endpointURL = "http://www.theworldavatar.com:83/citieskg/" + namespace;
- LOGGER.info("changed endpoint URL from " + endpointURL + " to " + summary.endpointURL);
- } else {
- summary.endpointURL = endpointURL;
- }
- }
-
- /**
- * Creates a new empty summary with an increased id and adds it to the list of all summaries.
- *
- * @return empty summary
- */
- public synchronized ServiceDescriptionSummary createSummary() {
- int id = summaries.size();
- ServiceDescriptionSummary summary = new ServiceDescriptionSummary(id);
- summaries.add(summary);
- return summary;
- }
-
- /**
- * Performs a conjunctive query for the given list of keywords.
- * The keywords must be relation or class IRIs.
- * The method uses the index previously created from service descriptions
- * to find all endpoints / datasets that contain at least one relevant triple
- * for each IRI in the list of keywords.
- *
- * @param keywords a list of relation and class IRIs
- * @return list of summaries containing query relevant endpoint URLs
- */
- public List conjunctiveQuery(List keywords) {
-
- List result = new ArrayList();
-
- // for each service descriptions, calculate the number of supported given keys
- Map mapId2Count = new HashMap();
- for (String key : keywords) {
- Set set = index.getPostingsList(INDEX_NAME, key);
- if (set == null) {
- LOGGER.warn("no postings list was found for key=" + key);
- break;
- }
- for (PostingsListElement elem : set) {
- int id = elem.id;
- if (mapId2Count.containsKey(id)) {
- int count = mapId2Count.get(id) + 1;
- mapId2Count.put(id, count);
- } else {
- mapId2Count.put(id, 1);
- }
- }
- }
-
- // find all service descriptions that support all given keys
- for (Integer id : mapId2Count.keySet()) {
- int count = mapId2Count.get(id);
- if (count == keywords.size()) {
- ServiceDescriptionSummary summary = getSummaries().get(id);
- result.add(summary);
- }
- }
-
- return result;
- }
-
- /**
- * Returns the endpoints URLs corresponding to all added service descriptions.
- *
- * @return endpoint URLs
- */
- public List getEndpointUrls() {
- List urls = new ArrayList();
- for (ServiceDescriptionSummary summary : getSummaries()) {
- urls.add(summary.endpointURL);
- }
- return urls;
- }
-}
diff --git a/src/main/java/uk/ac/cam/cares/jps/base/query/fed/SimpleMultiIndex.java b/src/main/java/uk/ac/cam/cares/jps/base/query/fed/SimpleMultiIndex.java
deleted file mode 100644
index dc088b574..000000000
--- a/src/main/java/uk/ac/cam/cares/jps/base/query/fed/SimpleMultiIndex.java
+++ /dev/null
@@ -1,46 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-/**
- * A very simple implementation of a multi-index with postings list.
- * An index is a map from an index key (of type K) to a set of entries (of type V).
- * The class can manage more than one index, e.g. an schema-level and an index-level index.
- * Different indexes are accessed by different index names.
- *
- * @param the type of the index, e.g. String for instance IRIs
- * @param the type of the entries in the postings list, e.g. String for endpoint URLs
- */
-public class SimpleMultiIndex {
-
- private Map>> indexNameMap = new HashMap>>();
-
- public void add(String indexName, K key, V value) {
- Map> index = indexNameMap.get(indexName);
- if (index == null) {
- index = new HashMap>();
- indexNameMap.put(indexName, index);
- }
- Set postingsList = index.get(key);
- if (postingsList == null) {
- postingsList = new TreeSet();
- index.put(key, postingsList);
- }
- postingsList.add(value);
- }
-
- public Map> getIndex(String indexName) {
- return indexNameMap.get(indexName);
- }
-
- public Set getPostingsList(String indexName, K key) {
- Map> index = getIndex(indexName);
- if (index == null) {
- return null;
- }
- return index.get(key);
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/BlazegraphRepositoryWrapperIntegrationTest.java b/src/test/java/uk/ac/cam/cares/jps/base/query/fed/BlazegraphRepositoryWrapperIntegrationTest.java
deleted file mode 100644
index 3de43ba6d..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/BlazegraphRepositoryWrapperIntegrationTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.jena.ontology.OntModel;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.json.JSONArray;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import junit.framework.TestCase;
-import uk.ac.cam.cares.jps.base.query.JenaHelper;
-
-@Ignore("The code this tests is not used and takes a long time to run.")
-public class BlazegraphRepositoryWrapperIntegrationTest extends TestCase {
-
- static final Logger LOGGER = LogManager.getLogger(BlazegraphRepositoryWrapperIntegrationTest.class);
-
- @Test
- public void testGetParts1() {
- String serviceUrl = "http://localhost:8080/blazegraph";
- String[] parts = BlazegraphRepositoryWrapper.getParts(serviceUrl);
- assertEquals(serviceUrl, parts[0]);
- assertNull(parts[1]);
- }
-
- @Test
- public void testGetParts2() {
- String serviceUrl = "http://localhost:8080/blazegraph";
- String namespace = "anynamespace";
- String endpointUrl = serviceUrl + BlazegraphRepositoryWrapper.getPathForBlazegraph(namespace);
- String[] parts = BlazegraphRepositoryWrapper.getParts(endpointUrl);
- assertEquals(serviceUrl, parts[0]);
- assertEquals(namespace, parts[1]);
- }
-
- @Test
- public void testCreateNamespace() {
-
- String serviceUrl = TripleStoreProvider.getServiceUrl(TripleStoreProvider.ID_BLAZEGRAPH_1);
- BlazegraphRepositoryWrapper wrapper = new BlazegraphRepositoryWrapper(serviceUrl);
-
- String newnamespace = "testEmpty";
- Properties props = TripleStoreProvider.readStandardNamespacePropertiesForBlazegraph();
- wrapper.createNamespace(newnamespace, props);
-
- // assert namespace
- List namespaces = wrapper.queryNamespaces(false);
- LOGGER.debug("found namespaces after creating new name space=" + namespaces.toString());
- String message = "new namespace was not found, namespace=" + newnamespace;
- assertTrue(message, namespaces.contains(newnamespace));
-
- // assert queries are possible and empty new dataset returns empty result
- String sparql = "SELECT * WHERE { ?s ?p ?o }";
- String result = wrapper.query(sparql, newnamespace);
- JSONArray array = new JSONArray(result);
- assertEquals(0, array.length());
- }
-
- @Test
- public void testQueryServiceDescription() {
-
- String serviceUrl = TripleStoreProvider.getServiceUrl(TripleStoreProvider.ID_BLAZEGRAPH_1);
- //String serviceUrl = "http://localhost:8091/blazegraph";
- BlazegraphRepositoryWrapper wrapper = new BlazegraphRepositoryWrapper(serviceUrl);
-
- String namespace = TripleStoreProvider.NAMESPACE_WIKIDATA_SMALL;
- String endpointUrl = TripleStoreProvider.getEndpointUrl(namespace);
-
- // assert that data can be queried and the count is correct
- String sparql = "PREFIX wdt: map = ServiceDescriptionIndexer.getNumberOfPropertyTriples(model);
- long actualTripleNumber = map.get("http://www.wikidata.org/prop/direct/P662");
- assertEquals(expectedCount, actualTripleNumber);
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/DataSourceSelectorTest.java b/src/test/java/uk/ac/cam/cares/jps/base/query/fed/DataSourceSelectorTest.java
deleted file mode 100644
index 42b1acb18..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/DataSourceSelectorTest.java
+++ /dev/null
@@ -1,130 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.junit.Ignore;
-
-import uk.ac.cam.cares.jps.base.query.fed.ParsedQueryTreeVisitor.ServiceGraphPatternSummary;
-
-@Ignore("The code this tests is not used and takes a long time to run.")
-public class DataSourceSelectorTest extends QueryProvider {
-
- static final Logger LOGGER = LogManager.getLogger(DataSourceSelectorTest.class);
-
- private ServiceDescriptionIndexer getSmallTestIndexer() {
- return ServiceDescriptionIndexerTest.getSmallTestIndexer();
- }
-
- private void assertKeySuffixes(List keys, int expectedNumber, String[] expectedSuffixes) {
- assertEquals(expectedNumber, keys.size());
- int count = 0;
- for (String suffix : expectedSuffixes) {
- for (String key : keys) {
- if (key.endsWith(suffix)) {
- count++;
- break;
- }
- }
- }
- assertEquals(expectedNumber, count);
- }
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- setServiceUrlParams("http://my.local.host:8080/myblazegraph", false);
- }
-
- public void testExtractKeysForSubqueries() {
- setQueryFormatParams(false, true, false);
- Query query = getSparqlDistributedLab_1();
- String sparql = query.sparql;
- List summaries = DataSourceSelector.extractKeysForSubqueries(sparql);
-
- assertEquals(2, summaries.size());
- assertKeySuffixes(summaries.get(0).keys, 4, new String[] {"hasYield", "hasValue", "hasNumericalValue", "hasUnit" });
- assertKeySuffixes(summaries.get(1).keys, 2, new String[] {"utilisesHistoricalData", "refersTo" });
- }
-
- public void testAddValuesClause_1() {
- setQueryFormatParams(false, true, false);
- Query query = getSparqlDistributedLab_1();
- String sparql = query.sparql;
- sparql += "ORDER BY ASC(?exp)\r\n" + "LIMIT 1\r\n";
-
- String varName = "service1";
- String[] varValues = new String[] {
- "http://localhost:8080/blazegraph/namespace/lab_1/sparql",
- "http://localhost:8080/blazegraph/namespace/lab_2/sparql"
- };
- sparql = DataSourceSelector.addValuesClause(sparql, varName, Arrays.asList(varValues));
- varName = "service2";
- varValues = new String[] {"http://localhost:8080/blazegraph/namespace/doe_chemrxn/sparql"};
- sparql = DataSourceSelector.addValuesClause(sparql, varName, Arrays.asList(varValues));
-
- assertTrue(sparql.contains("VALUES ?service1 { "
- + " }"));
- assertTrue(sparql.contains("VALUES ?service2 { }"));
- }
-
- public void testAddValuesClause_2() {
- setQueryFormatParams(false, true, false);
- Query query = getSparqlDistributedLab_2();
- String sparql = query.sparql;
-
- String varName = "service1";
- String[] varValues = new String[] {
- "http://localhost:8080/blazegraph/namespace/lab_1/sparql",
- "http://localhost:8080/blazegraph/namespace/lab_2/sparql"
- };
- sparql = DataSourceSelector.addValuesClause(sparql, varName, Arrays.asList(varValues));
-
- assertTrue(sparql.contains("VALUES ?service1 { "
- + " }"));
- }
-
- public void testAddValuesClause_3() {
- setQueryFormatParams(false, true, false);
- Query query = getSparqlBiodieselCityGML();
- String sparql = query.sparql;
-
- ServiceDescriptionIndexer indexer = getSmallTestIndexer();
- DataSourceSelector selector = new DataSourceSelector(indexer);
- sparql = selector.addValuesClauses(sparql);
-
- assertTrue(sparql.contains("VALUES ?service1 { }"));
- assertTrue(sparql.contains("VALUES ?service2 { }"));
- }
-
- public void testAddValuesClausesWithGivenServiceIRIs() throws Exception {
- setQueryFormatParams(false, true, false);
- Query query = getSparqlLocalWikidataWithPubChemCID887();
- String sparql = query.sparql;
-
- sparql = sparql.replace("?serva", "");
- sparql = sparql.replace("?servb", "");
- ServiceDescriptionIndexer indexer = getSmallTestIndexer();
- DataSourceSelector selector = new DataSourceSelector(indexer);
- String newSparql = selector.addValuesClauses(sparql);
-
- // assert that no VALUES clauses have been added
- assertFalse(newSparql.contains("VALUES"));
- assertEquals(sparql, newSparql);
- }
-
- public void testAddValuesClausesWithGivenValueClauses() {
- setQueryFormatParams(false, true, true);
- Query query = getSparqlLocalWikidataWithPubChemCID887();
- String sparql = query.sparql;
-
- ServiceDescriptionIndexer indexer = getSmallTestIndexer();
- DataSourceSelector selector = new DataSourceSelector(indexer);
- String newSparql = selector.addValuesClauses(sparql);
-
- // assert that no additional VALUES clauses have been added
- assertEquals(sparql, newSparql);
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/DatasetGenerator.java b/src/test/java/uk/ac/cam/cares/jps/base/query/fed/DatasetGenerator.java
deleted file mode 100644
index 424a47bab..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/DatasetGenerator.java
+++ /dev/null
@@ -1,194 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.function.Supplier;
-
-public class DatasetGenerator {
-
- public class Pair {
- public S x1;
- public T x2;
- public Pair(S x1, T x2) {
- this.x1 = x1;
- this.x2 = x2;
- }
- }
-
- private String prefixes = null;
- private List triplePatterns = new ArrayList();
- List>> generators = new ArrayList>>();
- private List> generatedValues = new ArrayList>();
- private StringBuffer b = new StringBuffer();
-
- public DatasetGenerator(String prefixes) {
- this.prefixes = prefixes;
- }
-
- public DatasetGenerator pattern(String triplePattern) {
- triplePatterns.add(triplePattern);
- return this;
- }
-
- public DatasetGenerator generator(String variableName, Supplier dataGenerator) {
- generators.add(new Pair>(variableName, dataGenerator));
- return this;
- }
-
- public DatasetGenerator generator(String variableName, Supplier dataGenerator, int maxBeforeNull) {
- if (maxBeforeNull < 0) {
- return generator(variableName, dataGenerator);
- }
- Supplier optionalSupplier = supplierOptional(dataGenerator, maxBeforeNull);
- generators.add(new Pair>(variableName, optionalSupplier));
- return this;
- }
-
- private Map generateVariableValues() {
- Map name2value = new HashMap();
- for (Pair> current : generators) {
- String name = current.x1;
- Supplier supplier = current.x2;
- Object value = supplier.get();
- String valueString = (value == null)? null : value.toString();
- name2value.put(name, valueString);
- }
- return name2value;
- }
-
- public DatasetGenerator generateVariableValues(int number) {
- for (int i=0; i generated = generateVariableValues();
- generatedValues.add(generated);
- }
- return this;
- }
-
- public List getGeneratedValues(String variableName, int maxNumber) {
- List result = new ArrayList();
- for (Map current : generatedValues) {
- String value = current.get(variableName);
- result.add(value);
- if (result.size() == maxNumber) {
- break;
- }
- }
- return result;
- }
-
- public List getOrderedVariableNames() {
-
- List varNames = new ArrayList();
- for (Pair> current : generators) {
- varNames.add(current.x1);
- }
-
- // variable names such as "?scfEnergy" "?scfEnergyABCValue" must be ordered
- // by decreasing String length such that "?scfEnergyABCValue" is replaced before "?scfEnergy";
- // otherwise we would obtain e.g. "-123.1ABCValue"
- Comparator comparator = new Comparator() {
-
- @Override
- public int compare(String o1, String o2) {
- return o2.length() - o1.length();
- }
- };
- Collections.sort(varNames, comparator);
- return varNames;
- }
-
- public String build() {
- b.append(prefixes);
- b.append("\r\n");
- b.append("\r\n");
-
- List varNames = getOrderedVariableNames();
-
- for (Map current : generatedValues) {
-
- for (String triplePattern : triplePatterns) {
- for (String variableName : varNames) {
- String value = current.get(variableName);
- if (value != null) {
- triplePattern = triplePattern.replace(variableName, value);
- }
- }
- if (triplePattern.indexOf("?") == -1) {
- // i.e. all variables are bounded
- b.append(triplePattern).append(" .\r\n");
- }
- }
- b.append("\r\n");
- }
- return b.toString();
- }
-
- public static Supplier supplierConstant(Object constant) {
- Supplier supplier = () -> constant;
- return supplier;
- }
-
- public static Supplier supplierUUID(String firstPart, String lastPart) {
- Supplier supplier = () -> firstPart + UUID.randomUUID() + lastPart;
- return supplier;
- }
-
- public static Supplier supplierList(List list) {
- Supplier supplier = new Supplier() {
-
- private int index = -1;
-
- @Override
- public Object get() {
- index++;
- return list.get(index);
- }
- };
- return supplier;
- }
-
- public static Supplier supplierJoin(List list, Supplier supplier) {
- Supplier joinSupplier = new Supplier() {
-
- private int index = -1;
-
- @Override
- public Object get() {
- index++;
- if (index < list.size()) {
- return list.get(index);
- }
- return supplier.get();
- }
- };
- return joinSupplier;
- }
-
- public static Supplier supplierInt(int min, int max) {
- int span = max - min + 1;
- Supplier supplier = () -> (int) (min + Math.floor(Math.random() * span));
- return supplier;
- }
-
- public static Supplier supplierOptional(Supplier supplier, int maxBeforeNull ) {
- Supplier optionalSupplier = new Supplier() {
-
- private int count = 0;
-
- @Override
- public Object get() {
- count++;
- if (count > maxBeforeNull) {
- return null;
- }
- return supplier.get();
- }
- };
- return optionalSupplier;
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/DatasetGeneratorTest.java b/src/test/java/uk/ac/cam/cares/jps/base/query/fed/DatasetGeneratorTest.java
deleted file mode 100644
index 951519a43..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/DatasetGeneratorTest.java
+++ /dev/null
@@ -1,155 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.function.Supplier;
-
-import org.apache.jena.ontology.OntModel;
-import org.apache.jena.query.ResultSet;
-import org.apache.jena.riot.Lang;
-import org.json.JSONArray;
-import org.junit.Ignore;
-
-import junit.framework.AssertionFailedError;
-import uk.ac.cam.cares.jps.base.query.JenaHelper;
-import uk.ac.cam.cares.jps.base.query.JenaResultSetFormatter;
-
-@Ignore("The code this tests is not used and takes a long time to run.")
-public class DatasetGeneratorTest extends QueryProvider {
-
- public void testSupplierList() {
- List list = Arrays.asList(10, 20, 30);
- Supplier supplier = DatasetGenerator.supplierList(list);
- assertEquals(10, supplier.get());
- assertEquals(20, supplier.get());
- assertEquals(30, supplier.get());
-
- try {
- supplier.get();
- throw new AssertionFailedError("should throw an ArrayIndexOutOfBoundsException");
- } catch (ArrayIndexOutOfBoundsException e) {
- }
- }
-
- public void testSupplierJoin() {
- List list1 = Arrays.asList(10, 20);
- List list2 = Arrays.asList("A", "B");
- Supplier supplier = DatasetGenerator.supplierList(list2);
- Supplier joinSupplier = DatasetGenerator.supplierJoin(list1, supplier);
- assertEquals(10, joinSupplier.get());
- assertEquals(20, joinSupplier.get());
- assertEquals("A", joinSupplier.get());
- assertEquals("B", joinSupplier.get());
- try {
- joinSupplier.get();
- throw new AssertionFailedError("should throw an ArrayIndexOutOfBoundsException");
- } catch (ArrayIndexOutOfBoundsException e) {
- }
- }
-
- public void testSupplierInt() {
- Supplier supplier = DatasetGenerator.supplierInt(-1, 2);
- Set set = new HashSet();
- for (int i=0; i<1000; i++) {
- set.add((Integer) supplier.get());
- }
- assertEquals(4, set.size());
- for (int i=-1; i<=2; i++) {
- assertTrue(set.contains(i));
- }
- }
-
- public void testSupplierOptional() {
- Supplier supplier = DatasetGenerator.supplierConstant("A");
- supplier = DatasetGenerator.supplierOptional(supplier, 2);
- assertEquals("A", supplier.get());
- assertEquals("A", supplier.get());
- assertNull(supplier.get());
- assertNull(supplier.get());
- }
-
- public void testGetOrderedVariableNames() {
-
- DatasetGenerator generator = new DatasetGenerator("").
- generator("?scfEnergy", DatasetGenerator.supplierConstant("constant1")).
- generator("?scfEnergyValue", DatasetGenerator.supplierConstant("constant2"));
-
- List varNames = generator.getOrderedVariableNames();
- assertEquals("?scfEnergyValue", varNames.get(0));
- assertEquals("?scfEnergy", varNames.get(1));
- }
-
- public void testBuild() {
- DatasetGenerator generator = new DatasetGenerator("").
- pattern("?species ontospecies:casRegistryID ?crid").
- pattern("?species ontospecies:hasStandardEnthalpyOfFormation ?enthalpy").
- pattern("?enthalpy ontospecies:value ?enthalpyOfFormationValue").
- generator("?species", DatasetGenerator.supplierConstant("SPECIES")).
- generator("?crid", DatasetGenerator.supplierConstant("CRID")).
- generator("?enthalpy", DatasetGenerator.supplierConstant("ENTHALPY")).
- generator("?enthalpyOfFormationValue", DatasetGenerator.supplierConstant("ENTHALPYVALUE"));
-
- String dataset = generator.generateVariableValues(1).build();
-
- String expected = "\r\n"
- + "\r\n"
- + "SPECIES ontospecies:casRegistryID CRID .\r\n"
- + "SPECIES ontospecies:hasStandardEnthalpyOfFormation ENTHALPY .\r\n"
- + "ENTHALPY ontospecies:value ENTHALPYVALUE .\r\n"
- + "\r\n";
-
- assertEquals(expected, dataset);
- }
-
- public void testBuildWithSupplierOptional() {
-
- DatasetGenerator generator = new DatasetGenerator("").
- pattern("?species ontospecies:casRegistryID ?crid").
- pattern("?species ontospecies:hasStandardEnthalpyOfFormation ?enthalpy").
- generator("?species", DatasetGenerator.supplierConstant("SPECIES")).
- generator("?crid", DatasetGenerator.supplierConstant("CRID"), 2). // 2 time CRID, then null
- generator("?enthalpy", DatasetGenerator.supplierConstant("ENTHALPY"));
-
- String dataset = generator.generateVariableValues(4).build();
-
- //System.out.println("XXX");
- //System.out.println(dataset);
- //System.out.println("YYY");
-
- String expected = "\r\n"
- + "\r\n"
- + "SPECIES ontospecies:casRegistryID CRID .\r\n"
- + "SPECIES ontospecies:hasStandardEnthalpyOfFormation ENTHALPY .\r\n"
- + "\r\n"
- + "SPECIES ontospecies:casRegistryID CRID .\r\n"
- + "SPECIES ontospecies:hasStandardEnthalpyOfFormation ENTHALPY .\r\n"
- + "\r\n"
- + "SPECIES ontospecies:hasStandardEnthalpyOfFormation ENTHALPY .\r\n"
- + "\r\n"
- + "SPECIES ontospecies:hasStandardEnthalpyOfFormation ENTHALPY .\r\n"
- + "\r\n";
-
- assertEquals(expected, dataset);
- }
-
- public void testGenerateOntoSpeciesOntoCompChem() {
-
- int sizeJoin = 25;
- String[] datasets = DatasetProvider.generateOntoSpeciesOntoCompChem(100, 200, sizeJoin, -1);
-
- // merge the triple of both datasets into one RDF graph
- OntModel model = JenaHelper.createModel();
- JenaHelper.readFromString(datasets[0], model, Lang.TURTLE);
- JenaHelper.readFromString(datasets[1], model, Lang.TURTLE);
-
- // assert that the non-federated query has result set of size sizeJoin
- setQueryFormatParams(false, false, false);
- Query query = getSparqlOntoSpeciesOntoCompChemLarge();
- ResultSet result = JenaHelper.query(model, query.sparql);
- String resultW3C = JenaResultSetFormatter.convertToJSONW3CStandard(result);
- JSONArray ja = JenaResultSetFormatter.convertToSimplifiedJsonArray(resultW3C);
- assertEquals(sizeJoin, ja.length());
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/DatasetProvider.java b/src/test/java/uk/ac/cam/cares/jps/base/query/fed/DatasetProvider.java
deleted file mode 100644
index 733abd798..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/DatasetProvider.java
+++ /dev/null
@@ -1,80 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.util.List;
-import java.util.function.Supplier;
-
-public class DatasetProvider {
-
- public static String[] generateOntoSpeciesOntoCompChem(int sizeOntoSpecies, int sizeOntoCompChem, int sizeJoin, int numbercrid) {
-
- DatasetGenerator genSpecies = createGeneratorOntoSpecies(numbercrid);
- genSpecies.generateVariableValues(sizeOntoSpecies);
- String datasetSpecies = genSpecies.build();
-
- List joinValues = genSpecies.getGeneratedValues("?species", sizeJoin);
- DatasetGenerator genCompChem = createGeneratorOntoCompChem(joinValues);
- genCompChem.generateVariableValues(sizeOntoCompChem);
- String datasetCompChem = genCompChem.build();
-
- return new String[] { datasetSpecies, datasetCompChem };
- }
-
- public static DatasetGenerator createGeneratorOntoSpecies(int numbercrid) {
- String prefixes = "@prefix ontospecies: .\r\n"
- + "@prefix ontocompchem: .\r\n"
- + "@prefix gc: joinValuesSpecies) {
- String prefixes = "@prefix ontospecies: .\r\n"
- + "@prefix ontocompchem: .\r\n"
- + "@prefix gc: joinsupplier = DatasetGenerator.supplierUUID("kbontospecies:Species_", ""); ;
- if (joinValuesSpecies != null) {
- joinsupplier = DatasetGenerator.supplierJoin(joinValuesSpecies, joinsupplier);
- }
-
- generator.generator("?species", joinsupplier).
- generator("?compchemspecies", DatasetGenerator.supplierUUID("kbontocompchem:spec_", "")).
- generator("?scfEnergy", DatasetGenerator.supplierUUID("kbontocompchem:scfenergy_", "")).
- generator("?scfElectronicEnergy", DatasetGenerator.supplierUUID("kbontocompchem:electronicenergy_", "")).
- generator("?scfEnergyValue", DatasetGenerator.supplierConstant(-314.455919684)).
- generator("?zeroEnergy", DatasetGenerator.supplierUUID("kbontocompchem:zeropointenergy_", "")).
- generator("?zeroElectronicEnergy", DatasetGenerator.supplierUUID("kbontocompchem:zeroelectronicenergy_", "")).
- generator("?zeroEnergyValue", DatasetGenerator.supplierConstant(0.219432));
-
- return generator;
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedQueryBlazegraphGivenEndpointsIntegrationTest.java b/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedQueryBlazegraphGivenEndpointsIntegrationTest.java
deleted file mode 100644
index fbec0cbf5..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedQueryBlazegraphGivenEndpointsIntegrationTest.java
+++ /dev/null
@@ -1,149 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.io.IOException;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.junit.Ignore;
-
-import uk.ac.cam.cares.jps.base.query.RemoteStoreClient;
-
-@Ignore("The code this tests is not used and takes a long time to run.")
-/**
- * One of multiple test configurations, see {@link QueryProvider} for details.
- */
-public class FedQueryBlazegraphGivenEndpointsIntegrationTest extends QueryProvider {
-
- private static final Logger LOGGER = LogManager.getLogger(FedQueryBlazegraphGivenEndpointsIntegrationTest.class);
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- TripleStoreProvider.getInstance();
- setQueryFormatParams(false, true, true);
- setServiceUrlParams(null, true);
- }
-
- protected void queryAndAssert(Query query) {
- queryAndAssert(query.sparql, query.result);
- }
-
- private void queryAndAssert(String sparql, String expectedResult) {
- LOGGER.debug("Federated query for Blazegraph with given service endpoints and without data selection");
- String fedEngineUrl = TripleStoreProvider.getEndpointUrl(TripleStoreProvider.NAMESPACE_BLAZEGRAPH_EMTPY);
- FederatedQueryInterface repo = FederatedQueryFactory.createForQueriesWithGivenEndpoints(fedEngineUrl);
- String actualJson = repo.executeFederatedQuery(sparql);
- assertQueryResult(expectedResult, actualJson);
- }
-
- public void testSparqlDistributedLab_1() {
- queryAndAssert(getSparqlDistributedLab_1());
- }
-
- public void testSparqlDistributedLab_2() {
- queryAndAssert(getSparqlDistributedLab_2());
- }
-
- public void testSparqlDistributedLab_3() {
- queryAndAssert(getSparqlDistributedLab_3());
- }
-
- public void testSparqlDistributedLab_3_inverted_service_order() {
- queryAndAssert(getSparqlDistributedLab_3_inverted_service_order());
- }
-
- public void testSparqlOntoSpeciesOntoCompChemSmall() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemSmall());
- }
-
- public void testSparqlOntoSpeciesOntoCompChemMedium() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemMedium());
- }
-
- public void testSparqlOntoSpeciesOntoCompChemLarge() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemLarge());
- }
-
- public void xxxtestRemoteSparqlBiodieselCityGML() {
- queryAndAssert(getSparqlBiodieselCityGML());
- }
-
- public void xxxtestRemoteSparqlWikidataDBpedia() {
- queryAndAssert(getSparqlWikidataDBpedia());
- }
-
- private String createEndpoint(String containerId, String namespace) {
- TripleStoreProvider provider = TripleStoreProvider.getInstance();
- String path = BlazegraphRepositoryWrapper.getPathForBlazegraph(namespace);
- provider.createDatasetBlazegraph(containerId, namespace, path, null);
- return TripleStoreProvider.getEndpointUrl(namespace);
- }
-
- public void testSelectDistinctSolvesProblemWithDuplicatedRows() throws IOException {
-
- String prefixes = "PREFIX twa: \r\n"
- + "PREFIX ontospecies: \r\n"
- + "PREFIX ontocompchem: \r\n";
-
- // create first endpoint and insert triples
- String namespace1 = "duplicate_1";
- String url1 = createEndpoint(TripleStoreProvider.ID_BLAZEGRAPH_1, namespace1);
- String sparql = prefixes
- + "INSERT DATA {\r\n"
- + " twa:1001 ontospecies:casRegistryID 1001 .\r\n"
- + " twa:1002 ontospecies:casRegistryID 1002 .\r\n"
- + " twa:1003 ontospecies:casRegistryID 1003 .\r\n"
- + " twa:1004 ontospecies:casRegistryID 1004 .\r\n"
- + "}";
- RemoteStoreClient client = new RemoteStoreClient(url1, url1);
- client.executeUpdate(sparql);
-
- //sparql = prefixes + "SELECT * WHERE { ?s ?p ?o }";
- //JSONArray result = client.executeQuery(sparql);
- //System.out.println("RESULT 1 = \n" + result);
-
- // create second endpoint and insert triples
- String namespace2 = "duplicate_2";
- String url2 = createEndpoint(TripleStoreProvider.ID_BLAZEGRAPH_2, namespace2);
- sparql = prefixes
- + "INSERT DATA {\r\n"
- + " twa:1001_1 ontocompchem:hasUniqueSpecies twa:1001 .\r\n" // + 1
- + " twa:1001_2 ontocompchem:hasUniqueSpecies twa:1001 .\r\n" // + 3
- + " twa:1001_3 ontocompchem:hasUniqueSpecies twa:1001 .\r\n" // + 5
- //+ " twa:1001_4 ontocompchem:hasUniqueSpecies twa:1001 .\r\n" // + 7
- + " twa:1002_1 ontocompchem:hasUniqueSpecies twa:1002 .\r\n" // + 1
- + " twa:1002_2 ontocompchem:hasUniqueSpecies twa:1002 .\r\n" // + 3
- + " twa:1003_1 ontocompchem:hasUniqueSpecies twa:1003 .\r\n" // + 1 (1003_1 with existing species in duplicate_1)
- + " twa:1003_1 ontocompchem:hasUniqueSpecies twa:1043 .\r\n" // + 0 (1003_1 with non-existing species in duplicate_1)
- + " twa:1043_1 ontocompchem:hasUniqueSpecies twa:1043 .\r\n" // + 0 (another IRI with non-existing species in duplicate_1)
- + "}";
- client = new RemoteStoreClient(url2, url2);
- client.executeUpdate(sparql);
-
- // execute the federated query
- String fedQuery = prefixes
- + "SELECT *\r\n"
- + "WHERE {\r\n"
- + " SERVICE <%s> {\r\n"
- + " ?compchemspecies ontocompchem:hasUniqueSpecies ?species .\r\n"
- + " }\r\n"
- + " SERVICE <%s> {\r\n"
- + " ?species ontospecies:casRegistryID ?crid .\r\n"
- + " }\r\n"
- + "}";
- String dockerUrl1 = TripleStoreProvider.getDockerEndpointUrl(namespace1);
- String dockerUrl2 = TripleStoreProvider.getDockerEndpointUrl(namespace2);
- fedQuery = String.format(fedQuery, dockerUrl2, dockerUrl1);
-
- //
- String expected = "6";
-
- // check that there 14 rows instead
- String actual = "14"; // = 9 + 4 + 1
- queryAndAssert(fedQuery, actual);
-
- // use DISTINCT to eliminate the duplicated rows
- fedQuery = fedQuery.replace("SELECT", "SELECT DISTINCT");
- queryAndAssert(fedQuery, expected);
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedQueryBlazegraphSourceSelectionIntegrationTest.java b/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedQueryBlazegraphSourceSelectionIntegrationTest.java
deleted file mode 100644
index 8eacbf895..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedQueryBlazegraphSourceSelectionIntegrationTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.util.Map;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.junit.Ignore;
-
-@Ignore("The code this tests is not used and takes a long time to run.")
-/**
- * One of multiple test configurations, see {@link QueryProvider} for details.
- */
-public class FedQueryBlazegraphSourceSelectionIntegrationTest extends QueryProvider {
-
- private static final Logger LOGGER = LogManager.getLogger(FedQueryBlazegraphSourceSelectionIntegrationTest.class);
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- TripleStoreProvider.getInstance();
- setQueryFormatParams(false, true, false);
- setServiceUrlParams(null, true);
- }
-
- protected void queryAndAssert(Query query) {
- queryAndAssert(query.sparql, query.result);
- }
-
- private void queryAndAssert(String sparql, String expectedResult) {
- LOGGER.debug("Federated query for Blazegraph with endpoint selection");
- String fedEngineUrl = TripleStoreProvider.getEndpointUrl(TripleStoreProvider.NAMESPACE_BLAZEGRAPH_EMTPY);
- ServiceDescriptionIndexer indexer = getIndexer();
- LOGGER.debug("number of endpoints=" + indexer.getEndpointUrls().size());
- Map host2host = TripleStoreProvider.getHostConversionMap();
- FederatedQueryInterface repo = FederatedQueryFactory.createWithEndpointSelection(fedEngineUrl, indexer, host2host);
- //FederatedQueryInterface repo = FederatedQueryFactory.createForQueriesWithGivenEndpoints(fedEngineUrl);
- String actualJson = repo.executeFederatedQuery(sparql);
- assertQueryResult(expectedResult, actualJson);
- }
-
- public void testSparqlDistributedLab_1() {
- queryAndAssert(getSparqlDistributedLab_1());
- }
-
- public void testSparqlDistributedLab_2() {
- queryAndAssert(getSparqlDistributedLab_2());
- }
-
- public void testSparqlDistributedLab_3() {
- queryAndAssert(getSparqlDistributedLab_3());
- }
-
- public void testSparqlDistributedLab_3_inverted_service_order() {
- queryAndAssert(getSparqlDistributedLab_3_inverted_service_order());
- }
-
- public void testSparqlOntoSpeciesOntoCompChemSmall() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemSmall());
- }
-
- public void testSparqlOntoSpeciesOntoCompChemMedium() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemMedium());
- }
-
- public void testSparqlOntoSpeciesOntoCompChemLarge() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemLarge());
- }
-
- public void xxxtestRemoteSparqlBiodieselCityGML() {
- queryAndAssert(getSparqlBiodieselCityGML());
- }
-
- public void xxxtestRemoteSparqlWikidataDBpedia() {
- queryAndAssert(getSparqlWikidataDBpedia());
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedQueryRdf4jGivenEndpointsIntegrationTest.java b/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedQueryRdf4jGivenEndpointsIntegrationTest.java
deleted file mode 100644
index 151aeaa81..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedQueryRdf4jGivenEndpointsIntegrationTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.junit.Ignore;
-
-@Ignore("The code this tests is not used and takes a long time to run.")
-/**
- * One of multiple test configurations, see {@link QueryProvider} for details.
- */
-public class FedQueryRdf4jGivenEndpointsIntegrationTest extends QueryProvider {
-
- private static final Logger LOGGER = LogManager.getLogger(FedQueryRdf4jGivenEndpointsIntegrationTest.class);
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- TripleStoreProvider.getInstance();
- setQueryFormatParams(false, true, true);
- setServiceUrlParams(null, true);
- }
-
- protected void queryAndAssert(Query query) {
- queryAndAssert(query.sparql, query.result);
- }
-
- private void queryAndAssert(String sparql, String expectedResult) {
- LOGGER.debug("Federated query for RDF4J with given service endpoints and without data selection");
- String fedEngineUrl = TripleStoreProvider.getEndpointUrl(TripleStoreProvider.NAMESPACE_RDF4J_EMPTY);
- FederatedQueryInterface repo = FederatedQueryFactory.createForQueriesWithGivenEndpoints(fedEngineUrl);
- String actualJson = repo.executeFederatedQuery(sparql);
- assertQueryResult(expectedResult, actualJson);
- }
-
- public void testSparqlDistributedLab_1() {
- queryAndAssert(getSparqlDistributedLab_1());
- }
-
- public void testSparqlDistributedLab_2() {
- queryAndAssert(getSparqlDistributedLab_2());
- }
-
- public void testSparqlDistributedLab_3() {
- queryAndAssert(getSparqlDistributedLab_3());
- }
-
- public void testSparqlDistributedLab_3_inverted_service_order() {
- queryAndAssert(getSparqlDistributedLab_3_inverted_service_order());
- }
-
- public void testSparqlOntoSpeciesOntoCompChemSmall() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemSmall());
- }
-
- public void testSparqlOntoSpeciesOntoCompChemMedium() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemMedium());
- }
-
- public void xxxtestSparqlOntoSpeciesOntoCompChemLarge() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemLarge());
- }
-
- public void xxxtestRemoteSparqlBiodieselCityGML() {
- queryAndAssert(getSparqlBiodieselCityGML());
- }
-
- public void xxxtestRemoteSparqlWikidataDBpedia() {
- queryAndAssert(getSparqlWikidataDBpedia());
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedQueryRdf4jSourceSelectionIntegrationTest.java b/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedQueryRdf4jSourceSelectionIntegrationTest.java
deleted file mode 100644
index 1dca145fe..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedQueryRdf4jSourceSelectionIntegrationTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.http.client.methods.HttpGet;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.json.JSONArray;
-import org.json.JSONObject;
-import org.junit.Ignore;
-
-import uk.ac.cam.cares.jps.base.http.Http;
-import uk.ac.cam.cares.jps.base.query.JenaResultSetFormatter;
-
-@Ignore("The code this tests is not used and takes a long time to run.")
-/**
- * One of multiple test configurations, see {@link QueryProvider} for details.
- */
-public class FedQueryRdf4jSourceSelectionIntegrationTest extends QueryProvider {
-
- private static final Logger LOGGER = LogManager.getLogger(FedQueryRdf4jSourceSelectionIntegrationTest.class);
- private static FederatedQueryInterface engine = null;
- private static List endpoints = null;
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- TripleStoreProvider.getInstance();
- setQueryFormatParams(false, true, false);
- setServiceUrlParams(null, true);
- if (engine == null) {
- LOGGER.info("Creating engine with endpoint selection");
- String fedEngineUrl = TripleStoreProvider.getEndpointUrl(TripleStoreProvider.NAMESPACE_RDF4J_EMPTY);
- ServiceDescriptionIndexer indexer = getIndexer();
- endpoints = indexer.getEndpointUrls();
- Map host2host = TripleStoreProvider.getHostConversionMap();
- engine = FederatedQueryFactory.createWithEndpointSelection(fedEngineUrl, indexer, host2host);
- }
- }
-
- protected void queryAndAssert(Query query) {
- queryAndAssert(query.sparql, query.result);
- }
-
- private void queryAndAssert(String sparql, String expectedResult) {
- LOGGER.debug("Federated query for RDF4J with endpoint selection");
- LOGGER.debug("number of endpoints=" + endpoints.size());
- String actualJson = engine.executeFederatedQuery(sparql);
- assertQueryResult(expectedResult, actualJson);
- }
-
- // https://rdf4j.org/documentation/reference/rest-api/#repository-list
- private List getRDF4JEndpoints(String serviceUrl) {
- List endpoints = new ArrayList();
-
- String url = serviceUrl + "/repositories";
- String accept = "application/sparql-results+json";
- HttpGet request = Http.get(url, accept);
- String result = Http.execute(request);
- JSONArray ja = JenaResultSetFormatter.convertToSimplifiedJsonArray(result);
- for (int i=0; i endpoints = getRDF4JEndpoints(serviceUrl);
- LOGGER.debug("endpoints=" + endpoints);
- boolean found = false;
- for (String endpoint : endpoints) {
- if (endpoint.endsWith(namespace)) {
- found = true;
- break;
- }
- }
- assertTrue("endpoint was not found", found);
- }
-
- public void testSparqlDistributedLab_1() {
- queryAndAssert(getSparqlDistributedLab_1());
- }
-
- public void testSparqlDistributedLab_2() {
- queryAndAssert(getSparqlDistributedLab_2());
- }
-
- public void testSparqlDistributedLab_3() {
- queryAndAssert(getSparqlDistributedLab_3());
- }
-
- public void testSparqlDistributedLab_3_inverted_service_order() {
- queryAndAssert(getSparqlDistributedLab_3_inverted_service_order());
- }
-
- public void testSparqlOntoSpeciesOntoCompChemSmall() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemSmall());
- }
-
- public void testSparqlOntoSpeciesOntoCompChemMedium() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemMedium());
- }
-
- public void xxxtestSparqlOntoSpeciesOntoCompChemLarge() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemLarge());
- }
-
- public void xxxtestRemoteSparqlBiodieselCityGML() {
- queryAndAssert(getSparqlBiodieselCityGML());
- }
-
- public void xxxtestRemoteSparqlWikidataDBpedia() {
- queryAndAssert(getSparqlWikidataDBpedia());
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedXIntegrationTest.java b/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedXIntegrationTest.java
deleted file mode 100644
index 16385ed29..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/FedXIntegrationTest.java
+++ /dev/null
@@ -1,72 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.util.List;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.junit.Ignore;
-
-@Ignore("The code this tests is not used and takes a long time to run.")
-// https://rdf4j.org/documentation/programming/federation/
-/**
- * One of multiple test configurations, see {@link QueryProvider} for details.
- */
-public class FedXIntegrationTest extends QueryProvider {
-
- private static final Logger LOGGER = LogManager.getLogger(FedXIntegrationTest.class);
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- TripleStoreProvider.getInstance();
- setQueryFormatParams(false, false, false);
- }
-
- protected void queryAndAssert(Query query) {
- queryAndAssert(query.sparql, query.getEndpoints(), query.result);
- }
-
- private void queryAndAssert(String sparql, List endpoints, String expectedResult) {
- endpoints = getIndexer().getEndpointUrls();
- LOGGER.debug("FedX with number of endpoints=" + endpoints.size());
- FederatedQueryInterface repo = FederatedQueryFactory.createFedX(endpoints);
- String actualJson = repo.executeFederatedQuery(sparql);
- assertQueryResult(expectedResult, actualJson);
- }
-
- public void testSparqlDistributedLab_1() {
- queryAndAssert(getSparqlDistributedLab_1());
- }
-
- public void testSparqlDistributedLab_2() {
- queryAndAssert(getSparqlDistributedLab_2());
- }
-
- public void testSparqlDistributedLab_3() {
- queryAndAssert(getSparqlDistributedLab_3());
- }
-
- public void testSparqlDistributedLab_3_inverted_service_order() {
- queryAndAssert(getSparqlDistributedLab_3_inverted_service_order());
- }
-
- public void testSparqlOntoSpeciesOntoCompChemSmall() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemSmall());
- }
-
- public void testSparqlOntoSpeciesOntoCompChemMedium() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemMedium());
- }
-
- public void testSparqlOntoSpeciesOntoCompChemLarge() {
- queryAndAssert(getSparqlOntoSpeciesOntoCompChemLarge());
- }
-
- public void xxxtestRemoteSparqlBiodieselCityGML() {
- queryAndAssert(getSparqlBiodieselCityGML());
- }
-
- public void xxxtestRemoteSparqlWikidataDBpedia() {
- queryAndAssert(getSparqlWikidataDBpedia());
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/QueryProvider.java b/src/test/java/uk/ac/cam/cares/jps/base/query/fed/QueryProvider.java
deleted file mode 100644
index 247c0a9be..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/QueryProvider.java
+++ /dev/null
@@ -1,601 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonParser;
-
-import junit.framework.TestCase;
-import uk.ac.cam.cares.jps.base.exception.JPSRuntimeException;
-import uk.ac.cam.cares.jps.base.query.sparql.PrefixToUrlMap;
-import uk.ac.cam.cares.jps.base.query.sparql.Prefixes;
-import uk.ac.cam.cares.jps.base.query.RemoteStoreClientTest;
-
-/**
- * This is the base class for multiple test configurations of integration tests for executing federated queries.
- * The current test configurations (subclasses) are:
- *
- * (1) FedX ({@link FedXIntegrationTest}): a federated query engine which is part of RDF4J.
- * It accepts SPARQL 1.0 queries without SERVICE clauses as federated queries.
- * Each test configuration uses the {@link FederatedQueryFactory} to create the corresponding engine
- * and the same method {@link FederatedQueryInterface#executeFederatedQuery(String)} to execute a federated query.
- * An federated query example is shown in {@link FederatedQueryFactory#createForQueriesWithGivenEndpoints(String)}.
- * While all test configurations use the same queries with the same basic triple patterns,
- * a query may and may not include SERVICE or VALUES clauses. For this reason, a helper class {@link Query} is defined
- * that allows to format the same query by means of three parameters,
- * see {@link Query#formatQuery(boolean, boolean, boolean)}.
- */
-public abstract class QueryProvider extends TestCase {
-
- /**
- * A helper class to format the "same" federated query in different ways
- * for different test configurations.
- *
- */
- public class Query {
-
- public class Pair {
- String paramName = null;
- List endpoints = new ArrayList();
-
- public Pair(String paramName, List endpoints) {
- this.paramName = paramName;
- this.endpoints = endpoints;
- }
- }
-
- /**
- * The plain SPARQL string containing some %s placeholder to insert SERVICE and VALUES clauses.
- */
- String sparql = null;
- List serviceList = new ArrayList();
- String result = null;
-
- public void addEndpoints(String paramName, String... endpoints) {
- serviceList.add(new Pair(paramName, Arrays.asList(endpoints)));
- }
-
- /**
- * Returns all relevant endpoint URLs.
- *
- * @return
- */
- public List getEndpoints() {
- Set endpoints = new HashSet();
- for (Pair pair : serviceList) {
- endpoints.addAll(pair.endpoints);
- }
- return new ArrayList(endpoints);
- }
-
- /**
- * Formats the "plain" SPARQL string containing some %s placeholders
- * that can be replaced by SERVICE and VALUES clauses.
- * Only the following combinations make sense:
- *
- * (1) false, false, false: returns a valid SPARQL 1.0 without SERVICE and VALUES clauses as used by FedX values = new ArrayList();
- String value = "";
- boolean serviceAdded = false;
- if (addServiceUrls || addServiceParams) {
- for (int i=0; i {\r\n";
- values.add(value);
- } else if (addServiceParams) {
- value = (i>0)? " }\r\n" : "";
- value += " SERVICE ?" + pair.paramName + " {\r\n";
- values.add(value);
- } else {
- values.add("");
- }
- }
- } else {
- for (int i=0; i ";
- }
- value += " VALUES ?" + pair.paramName + " { " + concat + "}\r\n";
- }
- }
- values.add(value);
-
- Query newquery = new Query();
- newquery.sparql = String.format(sparql, values.toArray());
- newquery.serviceList = serviceList;
- newquery.result = result;
-
-
- if (true) {
- // add DISTINCT to remove duplicated results rows (in case of Blazegraph)
- if (!newquery.sparql.contains("DISTINCT")) {
- newquery.sparql = newquery.sparql.replace("SELECT", "SELECT DISTINCT");
- }
- }
-
-
- return newquery;
- }
- }
-
- static final Logger LOGGER = LogManager.getLogger(QueryProvider.class);
-
- // the configuration parameters to format queries
- private boolean addServiceUrls = false;
- private boolean addServiceParams = false;
- private boolean addValuesClauses = false;
- // the parameters to manipulate endpoints URLs in DOCKER test environment
- private boolean useDockerServiceUrl = false;
- private String fixedServiceUrl = null;
-
- protected void setQueryFormatParams(boolean addServiceUrls, boolean addServiceParams, boolean addValuesClauses) {
- this.addServiceUrls = addServiceUrls;
- this.addServiceParams = addServiceParams;
- this.addValuesClauses = addValuesClauses;
- }
-
- protected void setServiceUrlParams(String fixedServiceUrl, boolean useDockerServiceUrl) {
- this.fixedServiceUrl = fixedServiceUrl;
- this.useDockerServiceUrl = useDockerServiceUrl;
- }
-
- protected Query format(Query query) {
- return query.formatQuery(addServiceUrls, addServiceParams, addValuesClauses);
- }
-
- /*
- private static ServiceDescriptionIndexer fullIndexer = null;
-
- public static ServiceDescriptionIndexer getFullIndexer() {
- if (fullIndexer == null) {
- LOGGER.debug("initializing full indexer");
- File dir = new File("D:/tmp/service_desciptions_test");
- String[] extensions = new String[] {"rdf"};
- Collection files = FileUtils.listFiles(dir, extensions, true);
- fullIndexer = new ServiceDescriptionIndexer();
- for (File file : files) {
- fullIndexer.addServiceDescription(file.getAbsolutePath());
- }
- LOGGER.debug("initialed full indexer, service descriptions=" + files.size());
- }
- return fullIndexer;
- }
-
- protected static ServiceDescriptionIndexer getOldIndexer(boolean useAllEndpoints) {
- if (useAllEndpoints) {
- return getFullIndexer();
- }
- return ServiceDescriptionIndexerTest.getSmallTestIndexer();
- }
- */
-
- public ServiceDescriptionIndexer getIndexer() {
- List serviceUrls = new ArrayList();
- serviceUrls.add(TripleStoreProvider.getServiceUrl(TripleStoreProvider.ID_BLAZEGRAPH_1));
- serviceUrls.add(TripleStoreProvider.getServiceUrl(TripleStoreProvider.ID_BLAZEGRAPH_2));
-
- // additional federation members for the test with remote queries
- List federationMembers = new ArrayList();
- //federationMembers.add("http://www.theworldavatar.com/blazegraph/namespace/sgbiodieselplants/sparql");
- //federationMembers.add("http://www.theworldavatar.com:83/citieskg/namespace/singaporeEPSG24500/sparql");
- //federationMembers.add("http://www.theworldavatar.com/blazegraph/namespace/ontospecies/sparql");
- //federationMembers.add("http://www.theworldavatar.com/blazegraph/namespace/ontocompchem/sparql");
-
- return FederatedQueryFactory.getIndexer(true, serviceUrls, federationMembers);
- }
-
- /**
- * Returns the complete endpoint URL for a given namespace / dataset.
- * If useDockerServiceUrl is true, the corresponding Docker container related endpoint URL is returned.
- *
- * @param namespace
- * @return
- */
- public String endpoint(String namespace) {
- if (fixedServiceUrl == null) {
- if (useDockerServiceUrl) {
- return TripleStoreProvider.getDockerEndpointUrl(namespace);
- } else {
- return TripleStoreProvider.getEndpointUrl(namespace);
- }
- }
- return fixedServiceUrl + "/namespace/" + namespace + "/sparql";
- }
-
- public void assertQueryResult(String expected, String actual) {
- LOGGER.debug("actual result=\n" + actual);
- LOGGER.debug("expected result=\n" + expected);
-
- JsonParser parser = new JsonParser();
- JsonArray actualJA = (JsonArray) parser.parse(actual);
- /*
- JsonElement actualJson = parser.parse(actual);
- JsonArray actualJA = null;
- if (actualJson.isJsonArray()) {
- actualJA = (JsonArray) actualJson;
- } else {
- actualJA = actualJson.getAsJsonObject().getAsJsonArray("results");
- }
- */
-
- // if expected is an integer only compare the result size
- try {
- int expectedSize = Integer.valueOf(expected);
- LOGGER.debug("result size expected=" + expectedSize + ", actual=" + actualJA.size());
- assertEquals(expectedSize, actualJA.size());
- return;
- } catch (NumberFormatException e) {
- }
-
- JsonArray expectedJA = (JsonArray) parser.parse(expected);
-
- LOGGER.debug("result size expected=" + expectedJA.size() + ", actual=" + actualJA.size());
- assertEquals(expectedJA.size(), actualJA.size());
-
- // although the implementation of GSON's equal method is not order-sensitive
- // the following assert method does not work correctly
- // if the same elements in the actual and expected result arrays are ordered differently
- //assertEquals(expectedJA, actualJA);
-
- for (int i=0; i\r\n"
- + "PREFIX ontorxn: \r\n"
- + "PREFIX om: \r\n"
- + "PREFIX ontolab: \r\n"
- + "PREFIX saref: \r\n"
- + "PREFIX ontospecies: "
- + "\r\n";
- }
-
- public Query getSparqlDistributedLab_1() {
- Query query = new Query();
- query.sparql = prefixes()
- + "SELECT ?exp ?yield_value ?yield_unit\r\n"
- + "WHERE {\r\n"
- + "%s"
- + " ?exp ontorxn:hasYield ?yield.\r\n"
- + " ?yield om:hasValue ?measure.\r\n"
- + " ?measure om:hasNumericalValue ?yield_value; om:hasUnit ?yield_unit.\r\n"
- + "%s"
- + " ontodoe:utilisesHistoricalData ?historical_data.\r\n"
- + " ?historical_data ontodoe:refersTo ?exp.\r\n"
- + "%s"
- + "}\r\n";
- //query.addEndpoints("service1", "http://localhost:8080/blazegraph/namespace/lab_1/sparql", "http://localhost:8080/blazegraph/namespace/lab_2/sparql");
- //query.addEndpoints("service2", "http://localhost:8080/blazegraph/namespace/doe_chemrxn/sparql");
- query.addEndpoints("service1", endpoint(TripleStoreProvider.NAMESPACE_LAB_1), endpoint(TripleStoreProvider.NAMESPACE_LAB_2));
- query.addEndpoints("service2", endpoint(TripleStoreProvider.NAMESPACE_DOE_CHEMRXN));
- query.result = "[{\"yield_value\":\"47.7\",\"yield_unit\":\"http://www.ontology-of-units-of-measure.org/resource/om-2/percent\",\"exp\":\"https://www.example.com/triplestore/ontorxn/ReactionExperiment_1/RxnExp_1\"},{\"yield_value\":\"40.0\",\"yield_unit\":\"http://www.ontology-of-units-of-measure.org/resource/om-2/percent\",\"exp\":\"https://www.example.com/triplestore/ontorxn/ReactionExperiment_2/RxnExp_1\"},{\"yield_value\":\"54.1\",\"yield_unit\":\"http://www.ontology-of-units-of-measure.org/resource/om-2/percent\",\"exp\":\"https://www.example.com/triplestore/ontorxn/ReactionExperiment_3/RxnExp_1\"},{\"yield_value\":\"8.7\",\"yield_unit\":\"http://www.ontology-of-units-of-measure.org/resource/om-2/percent\",\"exp\":\"https://www.example.com/triplestore/ontorxn/ReactionExperiment_4/RxnExp_1\"},{\"yield_value\":\"47.9\",\"yield_unit\":\"http://www.ontology-of-units-of-measure.org/resource/om-2/percent\",\"exp\":\"https://www.example.com/triplestore/ontorxn/ReactionExperiment_5/RxnExp_1\"}]";
- return format(query);
- }
-
- public Query getSparqlDistributedLab_2() {
- Query query = new Query();
- query.sparql = prefixes()
- + "SELECT ?rs400 ?lab\r\n"
- + "WHERE {\r\n"
- + "%s"
- + " ?rs400 a ontovapourtec:VapourtecRS400.\r\n"
- + " ?rs400 saref:hasState ontovapourtec:Idle.\r\n"
- + " ?rs400 ontolab:isContainedIn ?lab.\r\n"
- + "%s"
- + "}";
- //query.addEndpoints("service", "http://localhost:8080/blazegraph/namespace/lab_1/sparql", "http://localhost:8080/blazegraph/namespace/lab_2/sparql");
- query.addEndpoints("service", endpoint(TripleStoreProvider.NAMESPACE_LAB_1), endpoint(TripleStoreProvider.NAMESPACE_LAB_2));
- query.result = "[{\"rs400\":\"http://example.com/blazegraph/namespace/testlab/dummy_lab_1/VapourtecRS400_Dummy\",\"lab\":\"http://example.com/blazegraph/namespace/testlab/dummy_lab_1/Laboratory_Dummy\"},{\"rs400\":\"http://example.com/blazegraph/namespace/testlab/dummy_lab_2/VapourtecRS400_Dummy\",\"lab\":\"http://example.com/blazegraph/namespace/testlab/dummy_lab_2/Laboratory_Dummy\"}]";
- return format(query);
- }
-
- public Query getSparqlDistributedLab_3() {
- Query query = new Query();
- query.sparql = prefixes()
- + "SELECT ?species ?molar_weight_val ?molar_weight_unit\r\n"
- + "WHERE {\r\n"
- + "%s"
- + " ontorxn:isOccurenceOf ?rxn.\r\n"
- + "%s"
- + " ?rxn ontocaperxn:hasReactant ?reactant.\r\n"
- + " ?reactant ontospecies:hasUniqueSpecies ?species.\r\n"
- + " ?species ontospecies:hasMolecularWeight ?molar_weight.\r\n"
- + " ?molar_weight ontospecies:value ?molar_weight_val.\r\n"
- + " ?molar_weight ontospecies:units ?molar_weight_unit.\r\n"
- + "%s"
- + "}";
- //query.addEndpoints("servA", "http://localhost:8080/blazegraph/namespace/lab_1/sparql", "http://localhost:8080/blazegraph/namespace/lab_2/sparql");
- //query.addEndpoints("servB", "http://localhost:8080/blazegraph/namespace/doe_chemrxn/sparql");
- query.addEndpoints("servA", endpoint(TripleStoreProvider.NAMESPACE_LAB_1), endpoint(TripleStoreProvider.NAMESPACE_LAB_2));
- query.addEndpoints("servB", endpoint(TripleStoreProvider.NAMESPACE_DOE_CHEMRXN));
- query.result = "[{\"species\":\"http://www.theworldavatar.com/kb/ontospecies/Species_353d4667-e25d-476a-bd74-5c34723c8ea3\",\"molar_weight_unit\":\"g/mol\",\"molar_weight_val\":\"58.08\"},{\"species\":\"http://www.theworldavatar.com/kb/ontospecies/Species_54d8b46b-17bc-4bbd-a3cc-3b3a16d6ae4b\",\"molar_weight_unit\":\"g/mol\",\"molar_weight_val\":\"106.124\"}]";
- return format(query);
- }
-
- public Query getSparqlDistributedLab_3_inverted_service_order() {
- Query query = new Query();
- query.sparql = prefixes()
- + "SELECT ?species ?molar_weight_val ?molar_weight_unit\r\n"
- + "WHERE {\r\n"
- + "%s"
- + " ?rxn ontocaperxn:hasReactant ?reactant.\r\n"
- + " ?reactant ontospecies:hasUniqueSpecies ?species.\r\n"
- + " ?species ontospecies:hasMolecularWeight ?molar_weight.\r\n"
- + " ?molar_weight ontospecies:value ?molar_weight_val.\r\n"
- + " ?molar_weight ontospecies:units ?molar_weight_unit.\r\n"
- + "%s"
- + " ontorxn:isOccurenceOf ?rxn.\r\n"
- + "%s"
- + "}";
- //query.addEndpoints("servB", "http://localhost:8080/blazegraph/namespace/doe_chemrxn/sparql");
- //query.addEndpoints("servA", "http://localhost:8080/blazegraph/namespace/lab_1/sparql", "http://localhost:8080/blazegraph/namespace/lab_2/sparql");
- query.addEndpoints("servB", endpoint(TripleStoreProvider.NAMESPACE_DOE_CHEMRXN));
- query.addEndpoints("servA", endpoint(TripleStoreProvider.NAMESPACE_LAB_1), endpoint(TripleStoreProvider.NAMESPACE_LAB_2));
- query.result = "[{\"species\":\"http://www.theworldavatar.com/kb/ontospecies/Species_353d4667-e25d-476a-bd74-5c34723c8ea3\",\"molar_weight_unit\":\"g/mol\",\"molar_weight_val\":\"58.08\"},{\"species\":\"http://www.theworldavatar.com/kb/ontospecies/Species_54d8b46b-17bc-4bbd-a3cc-3b3a16d6ae4b\",\"molar_weight_unit\":\"g/mol\",\"molar_weight_val\":\"106.124\"}]";
- return format(query);
- }
-
- /**
- * A federated query constructed from two separate queries in
- * {@link https://github.com/cambridge-cares/TheWorldAvatar/blob/main/JPS_BIODIESELPLANT/src/uk/ac/cam/cares/jps/bio/CrossDomainQuery.java}.
- *
- * @return
- */
- public Query getSparqlBiodieselCityGML() {
- Query query = new Query();
- query.sparql = PrefixToUrlMap.getPrefixesForSPARQL(Prefixes.OCPSYST, Prefixes.OCPPERF, Prefixes.ONTOCITYGML, Prefixes.BLAZEGRAPH_GEO)
- + "SELECT ?IRIs ?Cost\r\n"
- + "WHERE {\r\n"
- + "%s"
- + " ?EquipC OCPPERF:isCostOfPlantItem ?Equip .\r\n"
- + " ?EquipC OCPSYST:hasValue ?EquipCost .\r\n"
- + " ?EquipCost OCPSYST:numericalValue ?Cost.\r\n"
- + " FILTER(?Cost > 30000)\r\n"
- + " BIND( STR(?Equip) AS ?IRIs)\r\n"
- + "%s"
- + " GRAPH .\r\n"
- + " ?cityObject BLAZEGEO:customFields \"X0#Y0#Z0#X1#Y1#Z1#X2#Y2#Z2#X3#Y3#Z3#X4#Y4#Z4\".\r\n"
- + " ?cityObject BLAZEGEO:customFieldsLowerBounds \"10428#26648#0#10428#26648#0#10428#26648#0#10428#26648#0#10428#26648#0\".\r\n"
- + " ?cityObject BLAZEGEO:customFieldsUpperBounds \"10880#26955#10#10880#26955#10#10880#26955#10#10880#26955#10#10880#26955#10\". \r\n"
- + " }\r\n"
- + " }\r\n"
- + "%s"
- + "}";
- query.addEndpoints("service1", "http://www.theworldavatar.com/blazegraph/namespace/sgbiodieselplants/sparql");
- query.addEndpoints("service2", "http://www.theworldavatar.com:83/citieskg/namespace/singaporeEPSG24500/sparql");
- query.result = "[{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/E-301.owl#E-301\",\"Cost\":\"63800.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/C-301.owl#C-301\",\"Cost\":\"900900.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/E-302.owl#E-302\",\"Cost\":\"76900.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/E-303.owl#E-303\",\"Cost\":\"64000.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/E-304.owl#E-304\",\"Cost\":\"73900.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/E-306.owl#E-306\",\"Cost\":\"171400.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/E-305.owl#E-305\",\"Cost\":\"72900.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/P-301.owl#P-301\",\"Cost\":\"33850.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/E-307.owl#E-307\",\"Cost\":\"102700.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/T-301.owl#T-301\",\"Cost\":\"242800.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/T-302.owl#T-302\",\"Cost\":\"178100.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/T-303.owl#T-303\",\"Cost\":\"1404900.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/R-301.owl#R-301\",\"Cost\":\"149400.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/R-302.owl#R-302\",\"Cost\":\"148300.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/R-303.owl#R-303\",\"Cost\":\"371400.0\"},{\"IRIs\":\"http://www.jparksimulator.com/kb/sgp/jurongisland/biodieselplant3/P-302.owl#P-302\",\"Cost\":\"31100.0\"}]";
- return format(query);
- }
-
- public Query getSparqlLocalWikidataWithPubChemCID887() {
- Query query = new Query();
- query.sparql = PrefixToUrlMap.getPrefixesForSPARQL(Prefixes.XSD, Prefixes.WIKIDATA, Prefixes.WIKIDATAT, Prefixes.ONTOSPECIES)
- + "SELECT ?species ?pid ?wdcompound\r\n"
- + "WHERE {\r\n"
- + "%s"
- + " ?species OSPECIES:pubChemCID ?pid .\r\n"
- + " FILTER ( ?pid = \"887\" )\r\n"
- + "%s"
- + " ?wdcompound wdt:P662 ?pid .\r\n"
- + "%s"
- + "}";
- //query.addEndpoints("serva", "http://www.theworldavatar.com/blazegraph/namespace/ontospecies/sparql", "http://localhost:8080/blazegraph/namespace/doe_chemrxn/sparql");
- //query.addEndpoints("servb", "http://localhost:8080/blazegraph/namespace/wikidata_small/sparql");
- query.addEndpoints("serva", "http://www.theworldavatar.com/blazegraph/namespace/ontospecies/sparql", endpoint(TripleStoreProvider.NAMESPACE_DOE_CHEMRXN));
- query.addEndpoints("servb", endpoint(TripleStoreProvider.NAMESPACE_WIKIDATA_SMALL));
-
- query.result = "[{\"species\":\"http://www.theworldavatar.com/kb/ontospecies/Species_bbfd8e83-6c58-4ff3-820c-2095d2ab6998\",\"pid\":\"887\",\"wdcompound\":\"http://www.wikidata.org/entity/Q14982\"},{\"species\":\"http://www.theworldavatar.com/kb/ontospecies/Species_f157c9d6-8656-4e1c-be52-314888caf917\",\"pid\":\"887\",\"wdcompound\":\"http://www.wikidata.org/entity/Q14982\"}]";
- return format(query);
- }
-
- /**
- * An federated query for dbpedia and wikidata used in from {@link https://rdf4j.org/documentation/programming/federation/} .
- * @return
- */
- public Query getSparqlWikidataDBpedia() {
- Query query = new Query();
- query.sparql = PrefixToUrlMap.getPrefixesForSPARQL(Prefixes.WIKIDATA, Prefixes.WIKIDATAT)
- + "SELECT ?country ?countrySameAs ?gdp\r\n"
- + "WHERE {\r\n"
- + "%s"
- + " ?country a .\r\n"
- + " ?country ?countrySameAs .\r\n"
- + "%s"
- + " ?countrySameAs wdt:P2131 ?gdp .\r\n"
- + "%s"
- + "}\r\n";
- query.addEndpoints("service1", "http://dbpedia.org/sparql");
- query.addEndpoints("service2", "https://query.wikidata.org/sparql");
- query.result = "[{\"country\":\"http://dbpedia.org/resource/Netherlands\",\"gdp\":\"830572618850\",\"countrySameAs\":\"http://www.wikidata.org/entity/Q29999\"},{\"country\":\"http://dbpedia.org/resource/Czech_Republic\",\"gdp\":\"250681000000\",\"countrySameAs\":\"http://www.wikidata.org/entity/Q213\"}]\r\n";
- return format(query);
- }
-
- private void addEndpointsForSparqlOntoSpeciesOntoCompChem(Query query) {
- //query.addEndpoints("service1", "http://www.theworldavatar.com/blazegraph/namespace/ontospecies/sparql");
- //query.addEndpoints("service2", "http://www.theworldavatar.com/blazegraph/namespace/ontocompchem/sparql");
- //query.addEndpoints("service1", "http://localhost:8080/blazegraph/namespace/ontospecies/sparql");
- //query.addEndpoints("service2", "http://localhost:8080/blazegraph/namespace/ontocompchemcloned/sparql");
- //query.addEndpoints("service2", "http://localhost:8080/blazegraph/namespace/ontocompchemhasuniqespecies/sparql");
- //query.addEndpoints("service2", "http://localhost:8080/blazegraph/namespace/ontocompchem/sparql");
- query.addEndpoints("service1", endpoint(TripleStoreProvider.NAMESPACE_ONTOSPECIES));
- query.addEndpoints("service2", endpoint(TripleStoreProvider.NAMESPACE_ONTOCOMPCHEM));
- }
-
- /**
- * A federated query as in {@link RemoteStoreClientTest#performMechanismCountQueryTest()}.
- *
- * @return
- */
- public Query getSparqlOntoSpeciesOntoCompChemLarge() {
-
- //jsonObject.put("scfEnergyValue", "-464.940687165");
- Query query = new Query();
- query.sparql = "PREFIX ontospecies: \r\n"
- + "PREFIX ontocompchem: \r\n"
- + "PREFIX gc: \r\n"
- + "PREFIX ontocompchem: \r\n"
- + "PREFIX gc: \r\n"
- + "PREFIX ontocompchem: \r\n"
- + "PREFIX gc: map = indexer.getNumberOfClassTriples(model);
- assertEquals(14, map.keySet().size());
- long number = map.get("http://www.theworldavatar.com/ontology/ontospecies/OntoSpecies.owl#MolecularWeight");
- assertEquals(5641, number);
- }
-
- public void testGetNumberOfPropertyTriples() {
- ServiceDescriptionIndexer indexer = new ServiceDescriptionIndexer();
- String path = getServiceDescrPath(0);
- OntModel model = JenaHelper.createModel(path);
- Map map = indexer.getNumberOfPropertyTriples(model);
- assertEquals(34, map.keySet().size());
- long number = map.get("http://www.theworldavatar.com/ontology/ontospecies/OntoSpecies.owl#casRegistryID");
- assertEquals(9690, number);
- }
-
- public void testAddOneServiceDescription() {
- ServiceDescriptionIndexer indexer = new ServiceDescriptionIndexer();
- String path = getServiceDescrPath(0);
- indexer.addServiceDescription(path);
-
- List summaries = indexer.getSummaries();
- assertEquals(1, summaries.size());
- assertEquals(path, summaries.get(0).path);
-
- Set keys = indexer.getKeys();
- assertEquals(48, keys.size());
-
- for (String key : keys) {
- Set postingsList = indexer.getPostingsList(key);
- assertEquals(1, postingsList.size());
- }
- }
-
- public void testAddTwoServiceDescriptions() {
- ServiceDescriptionIndexer indexer = new ServiceDescriptionIndexer();
- // add first description
- String path1 = getServiceDescrPath(1);
- indexer.addServiceDescription(path1);
-
- Set keys = indexer.getKeys();
- assertEquals(183, keys.size());
-
- // add second description
- String path2 = getServiceDescrPath(0);
- indexer.addServiceDescription(path2);
-
- keys = indexer.getKeys();
- // 15 common keys in OntoKin and OntoSpecies datasets
- // 216 = 183 + 48 - 15
- assertEquals(216 , keys.size());
-
- List summaries = indexer.getSummaries();
- assertEquals(2, summaries.size());
- assertEquals(path1, summaries.get(0).path);
- assertEquals(path2, summaries.get(1).path);
-
- // assert 15 common keys
- List commonKeys = new ArrayList();
- for (String key : keys) {
- Set postingsList = indexer.getPostingsList(key);
- if (postingsList.size() == 2) {
- commonKeys.add(key);
- }
- }
- assertEquals(15, commonKeys.size());
- LOGGER.debug("common keys (relations, types) between ontospecies and ontokin= " + commonKeys);
- assertTrue(commonKeys.contains("http://www.theworldavatar.com/ontology/ontokin/OntoKin.owl#GasPhase"));
- }
-
- public void testFillSummary() {
- ServiceDescriptionIndexer indexer = new ServiceDescriptionIndexer();
- ServiceDescriptionIndexer.ServiceDescriptionSummary summary = indexer.createSummary();
- String path = getServiceDescrPath(0);
- OntModel model = JenaHelper.createModel(path);
- indexer.fillSummary(summary, path, model);
- assertEquals(491495l, (long) summary.ntriples);
- assertEquals(134368l, (long) summary.nentities);
- assertEquals(34l, (long) summary.nproperties);
- assertEquals(14l, (long) summary.nclasses);
- assertEquals("http://localhost:8080/blazegraph/namespace/ontospecies/sparql", summary.endpointURL);
- }
-
- public void testConjunctiveQuery_1_FourKeysOntoSpeciesOntoKin() {
- ServiceDescriptionIndexer indexer = getSmallTestIndexer();
-
- // both service descriptions contain "GasPhase"
- String[] keys = new String[] {
- "http://www.theworldavatar.com/ontology/ontokin/OntoKin.owl#GasPhase"
- };
- List list = indexer.conjunctiveQuery(Arrays.asList(keys));
- assertEquals(2, list.size());
-
- // only the first service description contains "hasPhase"
- keys = new String[] {
- "http://www.theworldavatar.com/ontology/ontospecies/OntoSpecies.owl#hasPhase"
- };
- list = indexer.conjunctiveQuery(Arrays.asList(keys));
- assertEquals(1, list.size());
-
- // only the first service description contains the following four keys
- keys = new String[] {
- "http://www.theworldavatar.com/ontology/ontokin/OntoKin.owl#GasPhase",
- "http://www.theworldavatar.com/ontology/ontospecies/OntoSpecies.owl#hasPhase",
- "http://www.theworldavatar.com/ontology/ontokin/OntoKin.owl#Element",
- "http://www.w3.org/2000/01/rdf-schema#label"
- };
- list = indexer.conjunctiveQuery(Arrays.asList(keys));
- assertEquals(1, list.size());
- assertEquals(getServiceDescrPath(0), list.get(0).path);
- }
-
- public void testConjunctiveQuery_2_BiodieselCitiesKG() {
- ServiceDescriptionIndexer indexer = getSmallTestIndexer();
-
- // keys for the first subquery concerning Biodiesel plant in Singapore
- String[] keys = new String[] {
- "http://www.theworldavatar.com/ontology/ontocape/chemical_process_system/CPS_performance/economic_performance.owl#isCostOfPlantItem",
- "http://www.theworldavatar.com/ontology/ontocape/upper_level/system.owl#hasValue",
- "http://www.theworldavatar.com/ontology/ontocape/upper_level/system.owl#numericalValue"
- };
- List list = indexer.conjunctiveQuery(Arrays.asList(keys));
- assertEquals(1, list.size());
- assertEquals(getServiceDescrPath(3), list.get(0).path);
- }
-
- public void testConjunctiveQuery_3_BiodieselCitiesKG() {
- ServiceDescriptionIndexer indexer = getSmallTestIndexer();
-
- // keys for the second subquery concerning Biodiesel plant in Singapore
- String[] keys = new String[] {
- "http://www.theworldavatar.com/ontology/ontocitygml/citieskg/OntoCityGML.owl#attrName",
- "http://www.theworldavatar.com/ontology/ontocitygml/citieskg/OntoCityGML.owl#uriVal",
- "http://www.theworldavatar.com/ontology/ontocitygml/citieskg/OntoCityGML.owl#cityObjectId",
- "http://www.theworldavatar.com/ontology/ontocitygml/citieskg/OntoCityGML.owl#EnvelopeType",
- // the following predicates are Bigdata specific and used for its geo:search service
- // they don't appear explicitely as part of triples of any dataset
- // and their triple counts are thus zero
- //"http://www.bigdata.com/rdf/geospatial#predicate",
- //"http://www.bigdata.com/rdf/geospatial#searchDatatype",
- //"http://www.bigdata.com/rdf/geospatial#customFields",
- //"http://www.bigdata.com/rdf/geospatial#customFieldsLowerBounds",
- //"http://www.bigdata.com/rdf/geospatial#customFieldsUpperBounds"
- };
- List list = indexer.conjunctiveQuery(Arrays.asList(keys));
- assertEquals(1, list.size());
- assertEquals(getServiceDescrPath(2), list.get(0).path);
- }
-
- public void testConjunctiveQuery_4_Automated_Lab() {
- ServiceDescriptionIndexer indexer = getSmallTestIndexer();
-
- // keys for the first subquery concerning labs
- String[] keys = new String[] {
- "https://github.com/cambridge-cares/TheWorldAvatar/blob/develop/JPS_Ontology/ontology/ontorxn/OntoRxn.owl#hasYield",
- "http://www.ontology-of-units-of-measure.org/resource/om-2/hasValue",
- "http://www.ontology-of-units-of-measure.org/resource/om-2/hasNumericalValue",
- "http://www.ontology-of-units-of-measure.org/resource/om-2/hasUnit"
- };
- List list = indexer.conjunctiveQuery(Arrays.asList(keys));
- assertEquals(2, list.size());
- String firstPath = list.get(0).path;
- String secondPath = list.get(1).path;
- if (firstPath.contains("lab_1")) {
- assertEquals(getServiceDescrPath(5), firstPath);
- assertEquals(getServiceDescrPath(6), secondPath);
- } else {
- assertEquals(getServiceDescrPath(6), firstPath);
- assertEquals(getServiceDescrPath(5), secondPath);
- }
- }
-
- public void testConjunctiveQuery_5_Automated_Lab() {
- ServiceDescriptionIndexer indexer = getSmallTestIndexer();
-
- // keys for the second subquery concerning DoE
- String[] keys = new String[] {
- "https://github.com/cambridge-cares/TheWorldAvatar/blob/develop/JPS_Ontology/ontology/ontodoe/OntoDoE.owl#utilisesHistoricalData",
- "https://github.com/cambridge-cares/TheWorldAvatar/blob/develop/JPS_Ontology/ontology/ontodoe/OntoDoE.owl#refersTo"
- };
- List list = indexer.conjunctiveQuery(Arrays.asList(keys));
- assertEquals(1, list.size());
- assertEquals(getServiceDescrPath(4), list.get(0).path);
- }
-
- public void testOntoCompChemHasUniqueSpecies () {
- ServiceDescriptionIndexer indexer = getSmallTestIndexer();
-
- // keys for the second subquery concerning DoE
- String[] keys = new String[] {
- "http://www.theworldavatar.com/ontology/ontocompchem/ontocompchem.owl#hasUniqueSpecies"
- };
- List list = indexer.conjunctiveQuery(Arrays.asList(keys));
- assertEquals(1, list.size());
- assertEquals(getServiceDescrPath(7), list.get(0).path);
-
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/SimpleMultiIndexTest.java b/src/test/java/uk/ac/cam/cares/jps/base/query/fed/SimpleMultiIndexTest.java
deleted file mode 100644
index ee61ce11a..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/SimpleMultiIndexTest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import static org.junit.jupiter.api.Assertions.assertIterableEquals;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.junit.Ignore;
-
-import junit.framework.TestCase;
-
-@Ignore("The code this tests is not used and takes a long time to run.")
-public class SimpleMultiIndexTest extends TestCase {
-
- public void testOneIndexWithThreePostingLists() {
- SimpleMultiIndex index = new SimpleMultiIndex();
- index.add("properties", "rdfs:label", 12);
- index.add("properties", "rdfs:label", 5);
- index.add("properties", "foaf:friend", 50);
- index.add("properties", "skos:broader", 500);
- index.add("properties", "rdfs:label", 4);
- index.add("properties", "foaf:friend", 40);
- index.add("properties", "skos:broader", 400);
- index.add("properties", "rdfs:label", 9);
-
- Map> propIndex = index.getIndex("properties");
- assertEquals(3, propIndex.keySet().size());
- Set postingsList = index.getPostingsList("properties", "rdfs:label");
- assertEquals(4, postingsList.size());
- // check that the postings list is sorted by id
- List expected = Arrays.asList( new Integer[] {4, 5, 9, 12});
- assertIterableEquals(expected, postingsList);
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/TripleStoreProvider.java b/src/test/java/uk/ac/cam/cares/jps/base/query/fed/TripleStoreProvider.java
deleted file mode 100644
index e292bc3a4..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/query/fed/TripleStoreProvider.java
+++ /dev/null
@@ -1,491 +0,0 @@
-package uk.ac.cam.cares.jps.base.query.fed;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.testcontainers.containers.ContainerLaunchException;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.images.builder.ImageFromDockerfile;
-import org.testcontainers.utility.DockerImageName;
-
-import com.github.dockerjava.api.command.InspectContainerResponse;
-
-import uk.ac.cam.cares.jps.base.config.AgentLocator;
-import uk.ac.cam.cares.jps.base.exception.JPSRuntimeException;
-import uk.ac.cam.cares.jps.base.http.Http;
-import uk.ac.cam.cares.jps.base.query.RemoteStoreClient;
-import uk.ac.cam.cares.jps.base.util.FileUtil;
-
-/**
- * This class creates multiple Docker test containers on-the-fly.
- * Each container hosts a triple store including SPARQL endpoints and RDF data.
- * The created test environment can be used for integration tests for federated queries.
- * It allows to test various engines and configurations for executing federated SPARQL queries.
- *
- * The class uses the org.testcontainers framework which allows to create containers easily.
- * The framework also provides
- *
- * communication between containers .
- * However, it is not straight-forward to resolve a SPARQL endpoint URL
- * which may appear in a SPARQL 1.1 SERVICE clause or in a SPARQL service description.
- * This may lead to HTTP 4xx errors if a request for a SPARQL endpoint is send
- * from one container to another. The class solves this problem by converting localhost
- * IP addresses into Docker IP addresses.
- * For instance, the endpoint URL for Blazegraph has usually the following form:
- *
- * http://localhost:8080/blazegraph/namespace/lab_1/sparql
- *
- * where http://localhost:8080/blazegraph is the service URL and lab_1 is the namespace (dataset). The conversion would
- * replace localhost:8080 in the endpoint URL e.g. by 172.17.0.3:8080 which can be used internally for requests.
- *
- * The test environment can be extended by further containers (triple stores) and datasets by
- * extending the method {@link #init}. The test environment is created as singleton and the initialization is
- * started as soon as {@link #getInstance()} is called.
- */
-public class TripleStoreProvider {
-
- /**
- * A helper class that contains a created container and the SPARQL service URL.
- *
- */
- private class ContainerInfo {
-
- GenericContainer container = null;
- String serviceUrl = null;
-
- public ContainerInfo(GenericContainer container, String serviceUrl) {
- this.container = container;
- this.serviceUrl = serviceUrl;
- }
- }
-
- /**
- * A helper class that allows to construct the complete endpoint URL, e.g.
- *
- * http://localhost:54168/rdf4j-server/repositories/rdf4j_empty
- *
- * In this example, http://localhost:54168/rdf4j-server is the service URL
- * and /repositories/rdf4j_empty is the path.
- */
- private class EndpointInfo {
-
- String containerId = null;
- String path = null;
-
- public EndpointInfo(String containerId, String path) {
- this.containerId = containerId;
- this.path = path;
- }
- }
-
- static final Logger LOGGER = LogManager.getLogger(TripleStoreProvider.class);
- public final static String DIR_TEST_RESOURCES = "./src/test/resources";
-
- // container IDs
- public final static String ID_BLAZEGRAPH_1 = "blazegraph_1";
- public final static String ID_BLAZEGRAPH_2 = "blazegraph_2";
- public final static String ID_RDF4J_1 = "rdf4j_1";
-
- // namespaces (i.e. datasets, repos, etc.)
- public final static String NAMESPACE_LAB_1 = "lab_1";
- public final static String NAMESPACE_LAB_2 = "lab_2";
- public final static String NAMESPACE_DOE_CHEMRXN = "doe_chemrxn";
- public final static String NAMESPACE_WIKIDATA_SMALL = "wikidata_small";
- public final static String NAMESPACE_BLAZEGRAPH_EMTPY = "blazegraph_empty";
- public final static String NAMESPACE_ONTOSPECIES = "ontospecies";
- public final static String NAMESPACE_ONTOCOMPCHEM = "ontocompchem";
- public static final String NAMESPACE_RDF4J_EMPTY = "rdf4j_empty";
-
- private static TripleStoreProvider instance = null;
- /**
- * maps a container id to a container info object containing the created test container and the service URL
- */
- private Map id2container = new HashMap();
- /**
- * maps the namespace to an endpoint info object that allows to identify the triple store (container) and
- * to construct the full endpoint URL
- */
- private Map namespace2endpoint = new HashMap();
- /**
- * host2host is used to convert host addresses;
- * conversion may necessary e.g. in a test environment with Docker containers where Blazegraph servers return endpoint URLs
- * with wrong host addresses (such as "localhost:8080")
- */
- private Map host2host = new HashMap();
-
- private TripleStoreProvider() {
- }
-
- public synchronized static TripleStoreProvider getInstance() {
- if (instance == null) {
- instance = new TripleStoreProvider();
- instance.init();
- }
- return instance;
- }
-
- /**
- * Creates a container for a given image. If the image is not found in the local repository,
- * it is downloaded once.
- *
- * @param imageName
- * @return
- */
- private GenericContainer createContainerByImage(String imageName) {
- return new GenericContainer<>(DockerImageName.parse(imageName));
- }
-
- /**
- * Builds an image with a Blazegraph server by using the same commands as in the
- * corresponding CMCL Docker file and returns a container for this image.
- *
- * @param imageName if null image will be deleted after test; if not null the image can be reused when restarting the test
- * @return
- */
- private GenericContainer createBlazegraphContainerByBuilder(String imageName) {
- ImageFromDockerfile dockerfile = null;
- if (imageName == null) {
- dockerfile = new ImageFromDockerfile();
- } else {
- // the image is not deleted after test; this speeds up the setup when the test is restarted
- dockerfile = new ImageFromDockerfile(imageName, false);
- }
-
- dockerfile = dockerfile
- .withDockerfileFromBuilder(builder -> builder
- .from("tomcat:9.0-jre11-openjdk-slim-buster")
- .add("https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_2_1_6_RC/blazegraph.war", "/usr/local/tomcat/webapps/blazegraph.war")
- .workDir("/data")
- .build());
-
- return new GenericContainer<>(dockerfile);
- }
-
- /**
- * Creates a container with a Blazegraph server.
- *
- * @param port exposed container port
- * @return
- */
- private GenericContainer createBlazegraphTripleStore(int port) {
- String imageName = "ghcr.io/cambridge-cares/blazegraph:1.2.0";
- GenericContainer tripleStore = null;
- try {
- tripleStore = createContainerByImage(imageName).withExposedPorts(port);
- tripleStore.start();
- LOGGER.debug("Docker image was started, name=" + imageName + ", port=" + port);
- } catch (ContainerLaunchException exc) {
- LOGGER.debug("Docker image was not found, name=" + imageName + ", port=" + port);
- tripleStore = createBlazegraphContainerByBuilder(imageName).withExposedPorts(port);
- tripleStore.start();
- LOGGER.debug("Docker image was created and started, name=" + imageName + ", port=" + port);
- }
- return tripleStore;
- }
-
- /**
- * Creates a container with an RDF4j server and workbench.
- *
- * @param port exposed container port
- * @return
- */
- private GenericContainer createRDF4JServer(int port) {
- String imageName = "eclipse/rdf4j-workbench:3.7.7";
- GenericContainer tripleStore = createContainerByImage(imageName).withExposedPorts(port);
- tripleStore.start();
- LOGGER.debug("Docker image was started, name=" + imageName + ", port=" + port);
- return tripleStore;
- }
-
- private String getLocalhost(GenericContainer container) {
- return container.getHost() + ":" + container.getFirstMappedPort();
- }
-
- /**
- * Returns the "IP:port" where IP it the IP address internally used for the container
- * in the default Docker network.
- *
- * @param container
- * @param port
- * @return
- */
- private String getDockerIp(GenericContainer container, int port) {
- InspectContainerResponse info = container.getContainerInfo();
- String ipAddress = info.getNetworkSettings().getIpAddress();
- return ipAddress + ":" + port;
- }
-
- /**
- * Returns the service URL for a given dataset (namespace)
- *
- * @param namespace
- * @return
- */
- private static String getServiceUrlByNamespace(String namespace) {
- String id = getInstance().namespace2endpoint.get(namespace).containerId;
- return getInstance().id2container.get(id).serviceUrl;
- }
-
- public static String getServiceUrl(String containerId) {
- return getInstance().id2container.get(containerId).serviceUrl;
- }
-
- public static String getEndpointUrl(String serviceUrl, String namespace) {
- String path = getInstance().namespace2endpoint.get(namespace).path;
- return serviceUrl + path;
- }
-
- public static String getEndpointUrl(String namespace) {
- String serviceUrl = getServiceUrlByNamespace(namespace);
- return getEndpointUrl(serviceUrl, namespace);
- }
-
- /**
- * Returns the full endpoint URL for a given namespace where the IP address
- * is replaced by the IP address internally used by Docker.
- *
- * @param namespace
- * @return
- */
- public static String getDockerEndpointUrl(String namespace) {
- String serviceUrl = getServiceUrlByNamespace(namespace);
- Map host2host = getInstance().host2host;
- for (String host : host2host.keySet()) {
- if (serviceUrl.contains(host)) {
- String newhost = host2host.get(host);
- serviceUrl = serviceUrl.replace(host, newhost);
- break;
- }
- }
- return getEndpointUrl(serviceUrl, namespace);
- }
-
- public static Map getHostConversionMap() {
- return getInstance().host2host;
- }
-
- /**
- * Reads the default properties required for creating a new dataset on Blazegraph.
- *
- * @return default properties
- */
- public static Properties readStandardNamespacePropertiesForBlazegraph() {
- try (InputStream input = new FileInputStream("./src/test/resources/FedQuery/RWStore.properties")) {
- Properties props = new Properties();
- props.load(input);
- return props;
- } catch (IOException e) {
- throw new JPSRuntimeException(e.getMessage(), e);
- }
- }
-
- /**
- * Uploads the triples from a collection of files to the specified endpoint.
- *
- * @param endpointUrl
- * @param files
- * @return
- */
- public static int uploadFiles(String endpointUrl, Collection files) {
- RemoteStoreClient storeClient = new RemoteStoreClient(endpointUrl, endpointUrl);
- //int count = 0;
- //LOGGER.debug("uploading files to endpointURL=" + endpointUrl + ", number of files=" + files.size());
- for (File file : files) {
- //count += 1;
- //LOGGER.debug("uploading file (" + count + "/" + files.size() + ")=" + file.getName());
- storeClient.uploadFile(file);
- }
- LOGGER.debug("finished uploading files to endpointUrl=" + endpointUrl, ", number of files=" + files.size());
- return files.size();
- }
-
- /**
- * If a directory is given, all contained files are added to the returned collection.
- *
- * @param dataDirOrFiles a mixed list of file and directory paths
- * @return
- */
- public static Collection getFiles(String... dataDirOrFiles) {
- Collection