diff --git a/qendpoint-backend/pom.xml b/qendpoint-backend/pom.xml index 6b75e3315..2686d2fbe 100644 --- a/qendpoint-backend/pom.xml +++ b/qendpoint-backend/pom.xml @@ -46,7 +46,7 @@ 5.0.2 3.4.0 1.5.6 - + 2.11.0 UTF-8 UTF-8 @@ -112,6 +112,11 @@ + + com.google.code.gson + gson + ${gson.version} + commons-codec diff --git a/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/EndpointController.java b/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/EndpointController.java index e36497c18..09a2d6677 100644 --- a/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/EndpointController.java +++ b/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/EndpointController.java @@ -41,7 +41,7 @@ public record FormatReturn(String query) {} @Autowired Sparql sparql; - @RequestMapping(value = "/sparql") + @RequestMapping(value = "/sparql", method = { RequestMethod.GET, RequestMethod.POST }) public void sparqlEndpoint(@RequestParam(value = "query", required = false) final String query, @RequestParam(value = "update", required = false) final String updateQuery, @RequestParam(value = "format", defaultValue = "json") final String format, diff --git a/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java b/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java index 7b25ce0ca..fd572ac06 100644 --- a/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java +++ b/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java @@ -10,8 +10,12 @@ import com.the_qa_company.qendpoint.store.EndpointFiles; import com.the_qa_company.qendpoint.store.EndpointStore; import com.the_qa_company.qendpoint.store.EndpointStoreUtils; +import com.the_qa_company.qendpoint.store.HDTProps; import com.the_qa_company.qendpoint.utils.FileUtils; import com.the_qa_company.qendpoint.utils.RDFStreamUtils; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import org.apache.lucene.index.IndexReader; import org.eclipse.rdf4j.model.Statement; import org.eclipse.rdf4j.model.util.Values; import org.eclipse.rdf4j.repository.RepositoryConnection; @@ -28,14 +32,14 @@ import com.the_qa_company.qendpoint.core.util.io.CloseSuppressPath; import com.the_qa_company.qendpoint.core.util.io.IOUtil; import org.eclipse.rdf4j.sail.lucene.LuceneSail; +import org.eclipse.rdf4j.sail.lucene.SearchIndex; +import org.eclipse.rdf4j.sail.lucene.impl.LuceneIndex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.web.server.ServerWebInputException; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -53,6 +57,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -342,6 +347,46 @@ void initializeEndpointStore(boolean finishLoading) throws IOException { CompiledSailOptions opt = sparqlRepository.getOptions(); port = opt.getPort(); } + + if (endpoint != null) { + HDTProps props = endpoint.getHdtProps(); + long bnCount = props.getEndBlankObjects() - props.getStartBlankObjects() // obj + + props.getEndBlankShared() - props.getStartBlankShared() // shared + + props.getEndBlankSubjects() - props.getStartBlankSubjects(); // subj + long literals = props.getEndLiteral() - props.getStartLiteral(); + logger.info("Index props: Lit:{} bn:{}", literals, bnCount); + } + Set lcs = sparqlRepository.getLuceneSails(); + if (!lcs.isEmpty()) { + logger.info("Lucene sails ({})", lcs.size()); + + final int maxCount = 5; + Iterator it = lcs.iterator(); + for (int i = 0; i < Math.min(maxCount, lcs.size()); i++) { + if (!it.hasNext()) + break; + LuceneSail lc = it.next(); + + String id = lc.getParameter(LuceneSail.INDEX_ID); + if (id == null || id.isEmpty()) + id = ""; + SearchIndex lcIdx = lc.getLuceneIndex(); + String infoStr = lcIdx.getClass().getSimpleName(); + if (lcIdx instanceof LuceneIndex li) { + IndexReader reader = li.getIndexReader(); + int numDocs = reader.numDocs(); + infoStr += " numDocs*Fields:" + numDocs + "*" + li.getIndexWriter().getFieldNames().size(); + } else { + infoStr += " no data"; // add ES/Solr?? + } + logger.info("{} {}", id, infoStr); + } + if (lcs.size() > maxCount) { + logger.info("..."); + } + + } + } if (finishLoading) { completeLoading(); diff --git a/qendpoint-core/pom.xml b/qendpoint-core/pom.xml index d0522ca2a..46dde9507 100644 --- a/qendpoint-core/pom.xml +++ b/qendpoint-core/pom.xml @@ -48,7 +48,7 @@ 1.5.6 0.9.44 - 4.3.2 + 4.9.0 1.7.30 UTF-8 @@ -75,7 +75,7 @@ org.apache.commons commons-compress - 1.21 + 1.26.0 org.apache.jena diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/AdjacencyList.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/AdjacencyList.java index c60885b40..19bf26e5e 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/AdjacencyList.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/AdjacencyList.java @@ -111,7 +111,7 @@ public long countItemsY(long x) { return last(x) - find(x) + 1; } - public long search(long element, long begin, long end) throws NotFoundException { + public long search(long element, long begin, long end) { if (end - begin > 10) { return binSearch(element, begin, end); } else { @@ -134,7 +134,7 @@ public long binSearch(long element, long begin, long end) { return -1; } - public long linSearch(long element, long begin, long end) throws NotFoundException { + public long linSearch(long element, long begin, long end) { while (begin <= end) { long read = array.get(begin); if (read == element) { @@ -142,7 +142,41 @@ public long linSearch(long element, long begin, long end) throws NotFoundExcepti } begin++; } - throw new NotFoundException(); + return -1; + } + + public long searchLoc(long element, long begin, long end) { + if (end - begin > 10) { + return binSearchLoc(element, begin, end); + } else { + return linSearchLoc(element, begin, end); + } + } + + public long binSearchLoc(long element, long begin, long end) { + while (begin <= end) { + long mid = (begin + end) / 2; + long read = array.get(mid); + if (element > read) { + begin = mid + 1; + } else if (element < read) { + end = mid - 1; + } else { + return mid; + } + } + return -(1 + begin); + } + + public long linSearchLoc(long element, long begin, long end) { + while (begin <= end) { + long read = array.get(begin); + if (read == element) { + return begin; + } + begin++; + } + return -(1 + begin); } public final long get(long pos) { @@ -237,4 +271,11 @@ public void dump() { System.out.println(); } + public Sequence getArray() { + return array; + } + + public Bitmap getBitmap() { + return bitmap; + } } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/Dictionary.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/Dictionary.java index 1d4ce246c..ef183d57f 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/Dictionary.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/Dictionary.java @@ -21,6 +21,7 @@ import com.the_qa_company.qendpoint.core.enums.RDFNodeType; import com.the_qa_company.qendpoint.core.enums.TripleComponentRole; import com.the_qa_company.qendpoint.core.header.Header; +import com.the_qa_company.qendpoint.core.quad.QuadString; import com.the_qa_company.qendpoint.core.triples.TripleID; import com.the_qa_company.qendpoint.core.triples.TripleString; @@ -265,4 +266,16 @@ default TripleID toTripleId(TripleString tsstr) { } return tid; } + + default TripleString toTripleString(TripleID tssid) { + if (tssid.isQuad()) { + return new QuadString(idToString(tssid.getSubject(), TripleComponentRole.SUBJECT), + idToString(tssid.getPredicate(), TripleComponentRole.PREDICATE), + idToString(tssid.getObject(), TripleComponentRole.OBJECT), + idToString(tssid.getGraph(), TripleComponentRole.GRAPH)); + } + return new TripleString(idToString(tssid.getSubject(), TripleComponentRole.SUBJECT), + idToString(tssid.getPredicate(), TripleComponentRole.PREDICATE), + idToString(tssid.getObject(), TripleComponentRole.OBJECT)); + } } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/HDTManagerImpl.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/HDTManagerImpl.java index 492421d91..cdb3a156a 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/HDTManagerImpl.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/HDTManagerImpl.java @@ -1,5 +1,6 @@ package com.the_qa_company.qendpoint.core.hdt; +import com.the_qa_company.qendpoint.core.compact.integer.VByte; import com.the_qa_company.qendpoint.core.enums.CompressionType; import com.the_qa_company.qendpoint.core.enums.RDFNotation; import com.the_qa_company.qendpoint.core.exceptions.NotFoundException; @@ -35,11 +36,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -49,6 +56,7 @@ public class HDTManagerImpl extends HDTManager { private static final Logger logger = LoggerFactory.getLogger(HDTManagerImpl.class); + private static final long HDT_DL_INFO_MAGIC = 0x4f464e4c44544448L; @Override public HDTOptions doReadOptions(String file) throws IOException { @@ -177,6 +185,15 @@ private HDTResult generateChecksumAfter(long checksum, Path checksumPath, HDTOpt public HDTResult doGenerateHDT(String rdfFileName, String baseURI, RDFNotation rdfNotation, HDTOptions spec, ProgressListener listener) throws IOException, ParserException { // choose the importer + long waitTimeStart = spec.getInt(HDTOptionsKeys.LOADER_WAIT_START, 0); + if (waitTimeStart > 0) { + logger.info("Waiting {}ms before start...", waitTimeStart); + try { + Thread.sleep(waitTimeStart); + } catch (InterruptedException ignore) { + } + logger.info("Done waiting"); + } String loaderType = spec.get(HDTOptionsKeys.LOADER_TYPE_KEY); TempHDTImporter loader; boolean isQuad = rdfNotation == RDFNotation.NQUAD; @@ -224,6 +241,9 @@ public HDTResult doGenerateHDT(String rdfFileName, String baseURI, RDFNotation r } else { try { preSize = Files.size(preDownload); + if (preSize == trueSize) { + break; + } } catch (IOException ignore) { preSize = 0; } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/SequentialSearchIteratorTripleID.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/SequentialSearchIteratorTripleID.java index feb76d276..46de10431 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/SequentialSearchIteratorTripleID.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/SequentialSearchIteratorTripleID.java @@ -97,52 +97,6 @@ public TripleID next() { return returnTriple; } - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#hasPrevious() - */ - @Override - public boolean hasPrevious() { - return hasPreviousTriples; - } - - private void doFetchPrevious() { - hasPreviousTriples = false; - - while (iterator.hasPrevious()) { - TripleID previous = iterator.previous(); - - if (previous.match(pattern)) { - hasPreviousTriples = true; - hasMoreTriples = true; - previousTriple.assign(previous); - previousPosition = iterator.getLastTriplePositionSupplier(); - break; - } - } - } - - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#previous() - */ - @Override - public TripleID previous() { - if (goingUp) { - goingUp = false; - if (hasMoreTriples) { - doFetchPrevious(); - } - doFetchPrevious(); - } - returnTriple.assign(previousTriple); - lastPosition = previousPosition; - - doFetchPrevious(); - - return returnTriple; - } - /* * (non-Javadoc) * @see hdt.iterator.IteratorTripleID#goToStart() diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/utils/GraphFilteringTripleId.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/utils/GraphFilteringTripleId.java index 04185bac8..3894787a9 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/utils/GraphFilteringTripleId.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/utils/GraphFilteringTripleId.java @@ -16,16 +16,6 @@ public GraphFilteringTripleId(IteratorTripleID iterator, long[] graphIds) { this.graphIds = graphIds; } - @Override - public boolean hasPrevious() { - throw new NotImplementedException(); - } - - @Override - public TripleID previous() { - throw new NotImplementedException(); - } - @Override public void goToStart() { throw new NotImplementedException(); diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/utils/ListTripleIDIterator.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/utils/ListTripleIDIterator.java index b07b9c972..926b223e6 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/utils/ListTripleIDIterator.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/utils/ListTripleIDIterator.java @@ -42,26 +42,6 @@ public TripleID next() { return triplesList.get(pos++); } - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#hasPrevious() - */ - @Override - public boolean hasPrevious() { - return pos > 0; - } - - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#previous() - */ - @Override - public TripleID previous() { - TripleID tripleID = triplesList.get(--pos); - lastPosition = pos; - return tripleID; - } - /* * (non-Javadoc) * @see hdt.iterator.IteratorTripleID#goToStart() diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/merge/HDTMergeJoinIterator.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/merge/HDTMergeJoinIterator.java new file mode 100644 index 000000000..e5affa3e4 --- /dev/null +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/merge/HDTMergeJoinIterator.java @@ -0,0 +1,154 @@ +package com.the_qa_company.qendpoint.core.merge; + +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; +import com.the_qa_company.qendpoint.core.enums.TripleComponentRole; +import com.the_qa_company.qendpoint.core.iterator.utils.FetcherIterator; +import com.the_qa_company.qendpoint.core.triples.IteratorTripleID; +import com.the_qa_company.qendpoint.core.triples.TripleID; + +import java.util.List; + +public class HDTMergeJoinIterator extends FetcherIterator> { + public static final class MergeIteratorData { + private final IteratorTripleID it; + private final TripleComponentRole role; + private TripleID last; + private boolean loaded; + + public MergeIteratorData(IteratorTripleID it, TripleComponentRole role) { + this.it = it; + this.role = role; + } + + public long getSeekLayer(TripleID id) { + return switch (role) { + case OBJECT -> id.getObject(); + case PREDICATE -> id.getPredicate(); + case SUBJECT -> id.getSubject(); + case GRAPH -> id.getGraph(); + }; + } + + /** + * goto a layer + * + * @param id layer + * @return if we reach the end + */ + public boolean gotoLayer(long id) { + while (hasNext()) { + if (getSeekLayer(last) >= id) { + return false; // good layer or after + } + next(); // force next + } + return true; + } + + public boolean hasNext() { + if (loaded) { + return true; + } + if (!it.hasNext()) { + return false; + } + + last = it.next(); + loaded = true; + return true; + } + + public TripleID peek() { + if (hasNext()) { + return last; + } + return null; + } + + public TripleID next() { + if (hasNext()) { + loaded = false; + return last; + } + return null; + } + } + + private final List iterators; + private boolean loaded; + + public HDTMergeJoinIterator(List iterators) { + this.iterators = iterators; + } + + private void moveNext() { + if (!loaded) { + loaded = true; + return; // start + } + + int minIdx = 0; + if (!iterators.get(minIdx).hasNext()) { + return; + } + TripleID minVal = iterators.get(minIdx).peek(); + TripleComponentOrder minOrder = iterators.get(minIdx).it.getOrder(); + + for (int i = 1; i < iterators.size(); i++) { + MergeIteratorData d = iterators.get(i); + if (!d.hasNext()) { + return; + } + TripleID peek = d.peek(); + + if (peek == null) { + return; // end + } + + TripleComponentOrder ord = d.it.getOrder(); + if (peek.compareTo(minVal, ord, minOrder) < 0) { + minVal = peek; + minOrder = ord; + minIdx = i; + } + } + + // move to next using this iterator + iterators.get(minIdx).next(); + } + + private boolean seekAll() { + MergeIteratorData it1 = iterators.get(0); + if (!it1.hasNext()) { + return false; // no data + } + long seek = it1.getSeekLayer(it1.peek()); + for (int i = 1; i < iterators.size(); i++) { + MergeIteratorData d = iterators.get(i); + + if (d.gotoLayer(seek)) { + return false; // too far + } + + long seekNext = d.getSeekLayer(d.peek()); + + if (seekNext != seek) { + seek = seekNext; + i = -1; // to compensate i++ + } + } + + return true; + } + + @Override + protected List getNext() { + moveNext(); + if (!seekAll()) + return null; + + // all the iterators are peeked with the same layer, we can read + return iterators; + } + +} diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/merge/HDTMergeJoinPreparer.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/merge/HDTMergeJoinPreparer.java new file mode 100644 index 000000000..7fc6352bd --- /dev/null +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/merge/HDTMergeJoinPreparer.java @@ -0,0 +1,177 @@ +package com.the_qa_company.qendpoint.core.merge; + +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; +import com.the_qa_company.qendpoint.core.enums.TripleComponentRole; +import com.the_qa_company.qendpoint.core.exceptions.NotImplementedException; +import com.the_qa_company.qendpoint.core.exceptions.ParserException; +import com.the_qa_company.qendpoint.core.hdt.HDT; +import com.the_qa_company.qendpoint.core.hdt.HDTManager; +import com.the_qa_company.qendpoint.core.listener.ProgressListener; +import com.the_qa_company.qendpoint.core.options.HDTOptions; +import com.the_qa_company.qendpoint.core.quad.QuadString; +import com.the_qa_company.qendpoint.core.triples.TripleID; +import com.the_qa_company.qendpoint.core.triples.TripleString; +import com.the_qa_company.qendpoint.core.util.CommonUtils; +import com.the_qa_company.qendpoint.core.util.StopWatch; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +public class HDTMergeJoinPreparer { + private static final Logger logger = LoggerFactory.getLogger(HDTMergeJoinPreparer.class); + private final HDT hdt; + private final List patterns = new ArrayList<>(); + private int keyIds; + + public HDTMergeJoinPreparer(HDT hdt) { + this.hdt = hdt; + } + + public long createVar() { + return -(++keyIds); + } + + public void addPattern(TripleID tid) { + patterns.add(tid); + } + + public void addPattern(long s, long p, long o) { + addPattern(new TripleID(s, p, o)); + } + + public List buildIteration() { + List lst = new ArrayList<>(); + + if (keyIds == 0) { + // no var + throw new NotImplementedException("No variable");// TODO: + } + int[] occSH = new int[keyIds]; + int[] occP = new int[keyIds]; + + for (TripleID patt : patterns) { + long pp = patt.getPredicate(); + if (pp < 0) { + occP[1 - (int) pp]++; + } + + long ss = patt.getSubject(); + long oo = patt.getObject(); + + if (ss < 0) { + occSH[1 - (int) ss]++; + } + if (oo < 0) { + if (ss != oo) { // avoid double var + occSH[1 - (int) oo]++; + } + } + } + + int maxShIdx = CommonUtils.maxArg(occSH); + int maxPrIdx = CommonUtils.maxArg(occP); + + if (maxShIdx == 0 && maxPrIdx == 0) { + // no var + throw new NotImplementedException("No variable");// TODO: + } + + // fixme: we should also check if all the sub graphs are connected + + if (maxShIdx > maxPrIdx) { + // load shared var + } else { + // load + + } + + return lst; + } + + public static void main(String[] args) throws IOException, ParserException { + if (args.length < 2) { + logger.error("Missing param: Usage [hdt] [desc]"); + return; + } + logger.info("Test preparer"); + String hdtPath = args[0]; + + record TPData(TripleString ts, TripleComponentRole role, TripleComponentOrder order) {} + + List data = new ArrayList<>(); + + try (BufferedReader r = Files.newBufferedReader(Path.of(args[1]))) { + String line; + + while ((line = r.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') + continue; // comment + + QuadString ts = new QuadString(); + ts.read(line, true); + + if (ts.getSubject().equals("var")) + ts.setSubject(null); + if (ts.getPredicate().equals("var")) + ts.setPredicate(null); + if (ts.getObject().equals("var")) + ts.setObject(null); + + logger.info("read {}", ts); + + String orderCfg = ts.getGraph().toString(); + + if (orderCfg.isEmpty()) { + logger.error("Invalid role cfg: empty"); + return; + } + String[] cfg = orderCfg.split(":"); + TripleComponentRole role = TripleComponentRole.valueOf(cfg[0]); + TripleComponentOrder order = TripleComponentOrder.valueOf(cfg[1]); + ts.setGraph(null); + + data.add(new TPData(new TripleString(ts), role, order)); + } + } + + logger.info("Config loaded"); + data.forEach(c -> logger.info("- {}", c)); + logger.info("Loading HDT for query"); + HDTOptions spec = HDTOptions.of("bitmaptriples.index.allowOldOthers", true); + try (HDT hdt = HDTManager.mapIndexedHDT(hdtPath, spec, ProgressListener.sout())) { + List mergeData = new ArrayList<>(); + for (TPData tpd : data) { + TripleID tid = hdt.getDictionary().toTripleId(tpd.ts()); + + if (tid.isNoMatch()) { + logger.error("Triple {} is invalid", tpd.ts()); + return; + } + + mergeData.add(new HDTMergeJoinIterator.MergeIteratorData(hdt.getTriples().search(tid, tpd.order().mask), + tpd.role())); + } + + HDTMergeJoinIterator it = new HDTMergeJoinIterator(mergeData); + + logger.info("results:"); + StopWatch sw = new StopWatch(); + sw.reset(); + long ret = 0; + while (it.hasNext()) { + it.next(); + ret++; + // logger.info("- {}", it.next()); + } + logger.info("Done in {} {}", sw.stopAndShow(), ret); + } + logger.info("Unmapped"); + } + +} diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/options/HDTOptionsKeys.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/options/HDTOptionsKeys.java index 6b61e551b..73a2c799b 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/options/HDTOptionsKeys.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/options/HDTOptionsKeys.java @@ -303,6 +303,12 @@ public class HDTOptionsKeys { */ @Key(type = Key.Type.ENUM, desc = "loading type for HDTCat / HDTDiff") public static final String LOAD_HDT_TYPE_KEY = "loader.hdt.type"; + + /** + * Add time before starting the indexing, in ms, default 0 + */ + @Key(type = Key.Type.NUMBER, desc = "Add time before starting the indexing, in ms") + public static final String LOADER_WAIT_START = "loader.waitStart"; /** * load the HDT file into memory */ diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/quad/impl/BitmapTriplesIteratorGraph.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/quad/impl/BitmapTriplesIteratorGraph.java index aa296be65..8fbcd1efb 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/quad/impl/BitmapTriplesIteratorGraph.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/quad/impl/BitmapTriplesIteratorGraph.java @@ -64,16 +64,6 @@ protected TripleID getNext() { } } - @Override - public boolean hasPrevious() { - throw new NotImplementedException(); - } - - @Override - public TripleID previous() { - throw new NotImplementedException(); - } - @Override public void goToStart() { tidIt.goToStart(); diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/quad/impl/BitmapTriplesIteratorGraphG.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/quad/impl/BitmapTriplesIteratorGraphG.java index b6f73594c..0ef4a97fd 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/quad/impl/BitmapTriplesIteratorGraphG.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/quad/impl/BitmapTriplesIteratorGraphG.java @@ -49,16 +49,6 @@ protected TripleID getNext() { return tripleID; } - @Override - public boolean hasPrevious() { - throw new NotImplementedException(); - } - - @Override - public TripleID previous() { - throw new NotImplementedException(); - } - @Override public void goToStart() { posZ = -1; diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/IteratorTripleID.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/IteratorTripleID.java index 828f58c78..0fe04b126 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/IteratorTripleID.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/IteratorTripleID.java @@ -21,6 +21,7 @@ import com.the_qa_company.qendpoint.core.enums.ResultEstimationType; import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; +import com.the_qa_company.qendpoint.core.exceptions.NotImplementedException; import java.util.Iterator; @@ -29,21 +30,6 @@ */ public interface IteratorTripleID extends Iterator { - /** - * Whether the iterator has previous elements. - * - * @return boolean - */ - boolean hasPrevious(); - - /** - * Get the previous element. Call only if hasPrevious() returns true. It - * moves the cursor of the Iterator to the previous entry. - * - * @return TripleID - */ - TripleID previous(); - /** * Point the cursor to the first element of the data structure. */ @@ -103,4 +89,61 @@ public interface IteratorTripleID extends Iterator { default boolean isLastTriplePositionBoundToOrder() { return false; } + + /** + * goto the next subject >= id + * + * @param id id + * @return true if the next subject == id + * @see #canGoToSubject() if can goto returns false, this function is not + * available + */ + default boolean gotoSubject(long id) { + return false; + } + + /** + * goto the next predicate >= id + * + * @param id id + * @return true if the next predicate == id + * @see #canGoToPredicate() if can goto returns false, this function is not + * available + */ + default boolean gotoPredicate(long id) { + return false; + } + + /** + * goto the next object >= id + * + * @param id id + * @return true if the next object == id + * @see #canGoToObject() if can goto returns false, this function is not + * available + */ + default boolean gotoObject(long id) { + return false; + } + + /** + * @return true if {@link #gotoSubject(long)} can be used, false otherwise + */ + default boolean canGoToSubject() { + return false; + } + + /** + * @return true if {@link #gotoPredicate(long)} can be used, false otherwise + */ + default boolean canGoToPredicate() { + return false; + } + + /** + * @return true if {@link #gotoObject(long)} can be used, false otherwise + */ + default boolean canGoToObject() { + return false; + } } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/TripleID.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/TripleID.java index 4e3f2de68..923697f85 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/TripleID.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/TripleID.java @@ -19,6 +19,8 @@ package com.the_qa_company.qendpoint.core.triples; +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; +import com.the_qa_company.qendpoint.core.enums.TripleComponentRole; import com.the_qa_company.qendpoint.core.util.LongCompare; import java.io.Serial; @@ -238,6 +240,45 @@ public int compareTo(TripleID other) { } } + /** + * get a component value from its role + * + * @param role role + * @return component value + */ + public long get(TripleComponentRole role) { + return switch (role) { + case SUBJECT -> getSubject(); + case PREDICATE -> getPredicate(); + case OBJECT -> getObject(); + case GRAPH -> getGraph(); + }; + } + + /** + * compare this triple id with another triple id using order remap + * + * @param other other triple id + * @param orderThis order of this triple id + * @param orderOther order of the other triple id + * @return compare result + */ + public int compareTo(TripleID other, TripleComponentOrder orderThis, TripleComponentOrder orderOther) { + int result = LongCompare.compare(get(orderThis.getSubjectMapping()), other.get(orderOther.getSubjectMapping())); + + if (result != 0) { + return result; + } + + result = LongCompare.compare(get(orderThis.getPredicateMapping()), other.get(orderOther.getPredicateMapping())); + + if (result != 0) { + return result; + } + + return LongCompare.compare(get(orderThis.getObjectMapping()), other.get(orderOther.getObjectMapping())); + } + /** * Check whether this triple matches a pattern of TripleID. 0 acts as a * wildcard diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java index 16dd2b141..81bed8188 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java @@ -114,7 +114,7 @@ public class BitmapTriples implements TriplesPrivate, BitmapTriplesIndex { protected boolean isClosed; public BitmapTriples() throws IOException { - this(new HDTSpecification()); + this(HDTOptions.empty()); } public BitmapTriples(HDTOptions spec) throws IOException { diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIterator.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIterator.java index 248a09234..d5b465d18 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIterator.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIterator.java @@ -21,6 +21,8 @@ import com.the_qa_company.qendpoint.core.enums.ResultEstimationType; import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; +import com.the_qa_company.qendpoint.core.enums.TripleComponentRole; +import com.the_qa_company.qendpoint.core.exceptions.NotImplementedException; import com.the_qa_company.qendpoint.core.iterator.SuppliableIteratorTripleID; import com.the_qa_company.qendpoint.core.triples.TripleID; import com.the_qa_company.qendpoint.core.compact.bitmap.AdjacencyList; @@ -168,37 +170,6 @@ public TripleID next() { return returnTriple; } - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#hasPrevious() - */ - @Override - public boolean hasPrevious() { - return posZ > minZ; - } - - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#previous() - */ - @Override - public TripleID previous() { - posZ--; - - posY = adjZ.findListIndex(posZ); - - z = adjZ.get(posZ); - y = adjY.get(posY); - x = adjY.findListIndex(posY) + 1; - - nextY = adjY.last(x - 1) + 1; - nextZ = adjZ.last(posY) + 1; - - updateOutput(); - - return returnTriple; - } - /* * (non-Javadoc) * @see hdt.iterator.IteratorTripleID#goToStart() @@ -298,4 +269,171 @@ public long getLastTriplePosition() { public boolean isLastTriplePositionBoundToOrder() { return true; } + + private boolean gotoOrder(long id, TripleComponentRole role) { + switch (role) { + case SUBJECT -> { + if (patX != 0) { + return id == patX; // can't jump or already on the right element + } + + if (x >= id) { + return id == x; + } + + x = id; + posY = adjY.find(x - 1); + posZ = adjZ.find(posY); + y = adjY.get(posY); + nextY = adjY.last(x - 1) + 1; + nextZ = adjZ.find(posY + 1); + + return true; // we know x exists + } + case PREDICATE -> { + if (patY != 0) { + return id == patY; // can't jump or already on the right element + } + + if (posY == nextY) { + return false; // no next element + } + + long curr = this.adjY.get(posY); + + if (curr >= id) { + return curr == id; + } + + boolean res; + if (posY + 1 == nextY) { + // no next element, go next X + x++; + posY = nextY; + nextY = adjY.findNext(posY) + 1; + y = adjY.get(posY); + + res = false; + } else { + long last = this.adjY.get(nextY - 1); + + if (last > id) { + // binary search between curr <-> last id + long loc = this.adjY.searchLoc(id, posY + 1, nextY - 2); + + if (loc > 0) { + res = true; + posY = loc; + y = id; + } else { + res = false; + posY = -loc - 1; + y = adjY.get(posY); + } + } else { + if (last != id) { + // last < id - GOTO end + 1 + posY = nextY; + res = false; + } else { + // last == id - GOTO last + posY = nextY - 1; + y = adjY.get(posY); + res = true; + } + nextY = adjY.findNext(posY) + 1; + } + } + + // down to z/posZ/nextZ? + posZ = adjZ.find(posY); // assert patZ != 0 + nextZ = adjZ.findNext(posZ) + 1; + + return res; + } + case OBJECT -> { + if (patZ != 0) { + return id == patZ; // can't jump or already on the right element + } + + if (posZ == nextZ) { + return false; // no next element + } + + long curr = this.adjZ.get(posZ); + + if (curr >= id) { + return curr == id; + } + if (posZ + 1 == nextZ) { + return false; // no next element + } + + long last = this.adjZ.get(nextZ - 1); + + boolean res; + + if (last > id) { + // binary search between curr <-> last id + long loc = this.adjZ.searchLoc(id, posZ + 1, nextZ - 2); + + if (loc >= 0) { // match + res = true; + posZ = loc; + // z = id; // no need to compute the z, it is only used in + // next() + } else { + res = false; + posZ = -loc - 1; + // z = adjZ.get(posZ); + } + } else if (last != id) { + // last < id - GOTO end + posZ = nextZ; + res = false; + } else { + // last == id - GOTO last + posZ = nextZ - 1; + // z = adjZ.get(posZ); + res = true; + } + + nextZ = adjZ.findNext(posZ) + 1; + + return res; + } + default -> throw new NotImplementedException("goto " + role); + } + } + + @Override + public boolean gotoSubject(long id) { + return gotoOrder(id, idx.getOrder().getSubjectMapping()); + } + + @Override + public boolean gotoPredicate(long id) { + return gotoOrder(id, idx.getOrder().getPredicateMapping()); + } + + @Override + public boolean gotoObject(long id) { + return gotoOrder(id, idx.getOrder().getObjectMapping()); + } + + @Override + public boolean canGoToSubject() { + return true; + } + + @Override + public boolean canGoToPredicate() { + return true; + } + + @Override + public boolean canGoToObject() { + return true; + } + } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorCat.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorCat.java index 28a84759c..0ae1e60b4 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorCat.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorCat.java @@ -55,16 +55,6 @@ public BitmapTriplesIteratorCat(Triples hdt1, Triples hdt2, DictionaryCat dictio } - @Override - public boolean hasPrevious() { - return false; - } - - @Override - public TripleID previous() { - return null; - } - @Override public void goToStart() { diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorMapDiff.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorMapDiff.java index a85ee0610..b7d602804 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorMapDiff.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorMapDiff.java @@ -45,16 +45,6 @@ public BitmapTriplesIteratorMapDiff(HDT hdtOriginal, Bitmap deleteBitmap, Dictio count++; } - @Override - public boolean hasPrevious() { - return false; - } - - @Override - public TripleID previous() { - return null; - } - @Override public void goToStart() { diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorY.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorY.java index 71e7fe5ce..c7ce2bf98 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorY.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorY.java @@ -111,37 +111,6 @@ public TripleID next() { * (non-Javadoc) * @see hdt.iterator.IteratorTripleID#hasPrevious() */ - @Override - public boolean hasPrevious() { - return prevY != -1 || posZ >= prevZ; - } - - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#previous() - */ - @Override - public TripleID previous() { - if (posZ <= prevZ) { - nextY = posY; - posY = prevY; - prevY = adjY.findPreviousAppearance(prevY - 1, patY); - - posZ = prevZ = adjZ.find(posY); - nextZ = adjZ.last(posY); - - x = adjY.findListIndex(posY) + 1; - y = adjY.get(posY); - z = adjZ.get(posZ); - } else { - posZ--; - z = adjZ.get(posZ); - } - - updateOutput(); - - return returnTriple; - } /* * (non-Javadoc) diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorYFOQ.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorYFOQ.java index 726389578..dbbfea499 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorYFOQ.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorYFOQ.java @@ -112,41 +112,6 @@ public TripleID next() { return returnTriple; } - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#hasPrevious() - */ - @Override - public boolean hasPrevious() { - return numOccurrence > 1 || posZ >= prevZ; - } - - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#previous() - */ - @Override - public TripleID previous() { - if (posZ <= prevZ) { - numOccurrence--; - posY = triples.predicateIndex.getOccurrence(predBase, numOccurrence); - - prevZ = adjZ.find(posY); - posZ = nextZ = adjZ.last(posY); - - x = adjY.findListIndex(posY) + 1; - y = adjY.get(posY); - z = adjZ.get(posZ); - } else { - z = adjZ.get(posZ); - posZ--; - } - - updateOutput(); - - return returnTriple; - } - /* * (non-Javadoc) * @see hdt.iterator.IteratorTripleID#goToStart() diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorZ.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorZ.java index 91be651aa..670a59210 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorZ.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorZ.java @@ -94,24 +94,6 @@ public TripleID next() { return returnTriple; } - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#hasPrevious() - */ - @Override - public boolean hasPrevious() { - throw new NotImplementedException(); - } - - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#previous() - */ - @Override - public TripleID previous() { - throw new NotImplementedException(); - } - /* * (non-Javadoc) * @see hdt.iterator.IteratorTripleID#goToStart() diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorZFOQ.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorZFOQ.java index 51d74f537..b1ae4ae10 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorZFOQ.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorZFOQ.java @@ -159,33 +159,6 @@ public TripleID next() { return returnTriple; } - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#hasPrevious() - */ - @Override - public boolean hasPrevious() { - return posIndex > minIndex; - } - - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#previous() - */ - @Override - public TripleID previous() { - posIndex--; - - long posY = adjIndex.get(posIndex); - - z = patZ != 0 ? patZ : adjIndex.findListIndex(posIndex) + 1; - y = patY != 0 ? patY : adjY.get(posY); - x = adjY.findListIndex(posY) + 1; - - updateOutput(); - return returnTriple; - } - /* * (non-Javadoc) * @see hdt.iterator.IteratorTripleID#goToStart() diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/EmptyTriplesIterator.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/EmptyTriplesIterator.java index fc71a1bd5..6eef2b30d 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/EmptyTriplesIterator.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/EmptyTriplesIterator.java @@ -24,16 +24,6 @@ public TripleID next() { throw new NoSuchElementException(); } - @Override - public boolean hasPrevious() { - return false; - } - - @Override - public TripleID previous() { - throw new NoSuchElementException(); - } - @Override public void goToStart() { // Do nothing diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/OneReadTempTriples.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/OneReadTempTriples.java index 96c8f7381..029a65cd5 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/OneReadTempTriples.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/OneReadTempTriples.java @@ -233,16 +233,6 @@ public SimpleIteratorTripleID(Iterator it, TripleComponentOrder order, this.tripleCount = tripleCount; } - @Override - public boolean hasPrevious() { - throw new NotImplementedException(); - } - - @Override - public TripleID previous() { - throw new NotImplementedException(); - } - @Override public void goToStart() { throw new NotImplementedException(); diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/TriplesList.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/TriplesList.java index 9f77fb77f..4b6f522e2 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/TriplesList.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/TriplesList.java @@ -448,25 +448,6 @@ public TripleID next() { return triplesList.arrayOfTriples.get(pos++).asTripleID(); } - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#hasPrevious() - */ - @Override - public boolean hasPrevious() { - return pos > 0; - } - - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#previous() - */ - @Override - public TripleID previous() { - lastPosition = --pos; - return triplesList.arrayOfTriples.get(pos).asTripleID(); - } - /* * (non-Javadoc) * @see hdt.iterator.IteratorTripleID#goToStart() diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/TriplesListLong.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/TriplesListLong.java index 3897a9722..c9157b277 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/TriplesListLong.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/TriplesListLong.java @@ -440,25 +440,6 @@ public TripleID next() { return triplesList.arrayOfTriples.get(pos++); } - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#hasPrevious() - */ - @Override - public boolean hasPrevious() { - return pos > 0; - } - - /* - * (non-Javadoc) - * @see hdt.iterator.IteratorTripleID#previous() - */ - @Override - public TripleID previous() { - lastPosition = --pos; - return triplesList.arrayOfTriples.get(pos); - } - /* * (non-Javadoc) * @see hdt.iterator.IteratorTripleID#goToStart() diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/CommonUtils.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/CommonUtils.java new file mode 100644 index 000000000..5824cbf97 --- /dev/null +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/CommonUtils.java @@ -0,0 +1,39 @@ +package com.the_qa_company.qendpoint.core.util; + +public class CommonUtils { + public static int minArg(int[] array) { + if (array.length < 2) { + return 0; + } + int minIdx = 0; + int minVal = array[0]; + for (int i = 1; i < array.length; i++) { + if (array[i] < minVal) { + minVal = array[i]; + minIdx = i; + } + } + + return minIdx; + } + + public static int maxArg(int[] array) { + if (array.length < 2) { + return 0; + } + int maxIdx = 0; + int maxVal = array[0]; + for (int i = 1; i < array.length; i++) { + if (array[i] > maxVal) { + maxVal = array[i]; + maxIdx = i; + } + } + + return maxIdx; + } + + private CommonUtils() { + throw new RuntimeException(); + }; +} diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/StopWatch.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/StopWatch.java index 46fc58a77..41d0843e1 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/StopWatch.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/StopWatch.java @@ -35,6 +35,10 @@ public long getMeasure() { return end - ini; } + public long getMeasureMillis() { + return (end - ini) / 1_000_000; + } + public long stopAndGet() { stop(); return getMeasure(); diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/io/compress/NoDuplicateTripleIDIterator.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/io/compress/NoDuplicateTripleIDIterator.java index 50c7806f4..b6c75e976 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/io/compress/NoDuplicateTripleIDIterator.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/io/compress/NoDuplicateTripleIDIterator.java @@ -49,16 +49,6 @@ public TripleID next() { return next; } - @Override - public boolean hasPrevious() { - throw new NotImplementedException(); - } - - @Override - public TripleID previous() { - throw new NotImplementedException(); - } - @Override public void goToStart() { throw new NotImplementedException(); diff --git a/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/merge/HDTMergeJoinIteratorTest.java b/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/merge/HDTMergeJoinIteratorTest.java new file mode 100644 index 000000000..e6e0b43da --- /dev/null +++ b/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/merge/HDTMergeJoinIteratorTest.java @@ -0,0 +1,112 @@ +package com.the_qa_company.qendpoint.core.merge; + +import com.the_qa_company.qendpoint.core.dictionary.Dictionary; +import com.the_qa_company.qendpoint.core.enums.RDFNotation; +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; +import com.the_qa_company.qendpoint.core.enums.TripleComponentRole; +import com.the_qa_company.qendpoint.core.exceptions.NotFoundException; +import com.the_qa_company.qendpoint.core.exceptions.ParserException; +import com.the_qa_company.qendpoint.core.hdt.HDT; +import com.the_qa_company.qendpoint.core.hdt.HDTManager; +import com.the_qa_company.qendpoint.core.iterator.SuppliableIteratorTripleID; +import com.the_qa_company.qendpoint.core.listener.ProgressListener; +import com.the_qa_company.qendpoint.core.options.HDTOptions; +import com.the_qa_company.qendpoint.core.options.HDTOptionsKeys; +import com.the_qa_company.qendpoint.core.triples.TripleID; +import com.the_qa_company.qendpoint.core.triples.impl.BitmapTriplesIndexFile; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class HDTMergeJoinIteratorTest { + + @Rule + public TemporaryFolder tempDir = TemporaryFolder.builder().assureDeletion().build(); + + private InputStream getStream(String filename) { + InputStream is = getClass().getResourceAsStream(filename); + Assert.assertNotNull("can't find file " + filename, is); + return is; + } + + @Test + @Ignore("wip") + public void itTest() throws IOException, ParserException, NotFoundException { + Path root = tempDir.newFolder().toPath(); + + Path hdtPath = root.resolve("test.hdt"); + HDTOptions spec = HDTOptions.of(HDTOptionsKeys.LOADER_TYPE_KEY, HDTOptionsKeys.LOADER_TYPE_VALUE_DISK, + HDTOptionsKeys.LOADER_DISK_FUTURE_HDT_LOCATION_KEY, hdtPath, HDTOptionsKeys.LOADER_DISK_LOCATION_KEY, + root.resolve("gd"), HDTOptionsKeys.DICTIONARY_TYPE_KEY, + HDTOptionsKeys.DICTIONARY_TYPE_VALUE_MULTI_OBJECTS_LANG, HDTOptionsKeys.BITMAPTRIPLES_INDEX_METHOD_KEY, + HDTOptionsKeys.BITMAPTRIPLES_INDEX_METHOD_VALUE_DISK, HDTOptionsKeys.BITMAPTRIPLES_INDEX_NO_FOQ, true, + // all indexes + HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, Arrays.stream(TripleComponentOrder.values()) + .map(TripleComponentOrder::name).collect(Collectors.joining(","))); + ProgressListener listener = ProgressListener.ignore(); + String ns = "http://example.org/#"; + try (InputStream is = getStream("/merge_ds.ttl"); + HDT hdt = HDTManager.generateHDT(is, ns, RDFNotation.TURTLE, spec, listener)) { + hdt.saveToHDT(hdtPath); + } + + try (HDT hdt = HDTManager.mapIndexedHDT(hdtPath, spec, listener)) { + // test index creation + assertTrue(Files.exists(BitmapTriplesIndexFile.getIndexPath(hdtPath, TripleComponentOrder.OPS))); + assertTrue(Files.exists(BitmapTriplesIndexFile.getIndexPath(hdtPath, TripleComponentOrder.POS))); + assertTrue(Files.exists(BitmapTriplesIndexFile.getIndexPath(hdtPath, TripleComponentOrder.PSO))); + + /* + * The query is ~that: SELECT * { ?s ex:relative ?o ?o rdfs:name ?n + * ?o ex:id ?id } + */ + + Dictionary dict = hdt.getDictionary(); + long exRelative = dict.stringToId(ns + "relative", TripleComponentRole.PREDICATE); + long rdfsName = dict.stringToId("http://www.w3.org/2000/01/rdf-schema#name", TripleComponentRole.PREDICATE); + long exId = dict.stringToId(ns + "id", TripleComponentRole.PREDICATE); + + TripleID p1 = new TripleID(0, exRelative, 0); + TripleID p2 = new TripleID(0, rdfsName, 0); + TripleID p3 = new TripleID(0, exId, 0); + + assertFalse(p1 + " empty", p1.isEmpty()); + assertFalse(p2 + " empty", p2.isEmpty()); + assertFalse(p3 + " empty", p3.isEmpty()); + + SuppliableIteratorTripleID it1 = hdt.getTriples().search(p1, TripleComponentOrder.POS.mask); + SuppliableIteratorTripleID it2 = hdt.getTriples().search(p2, TripleComponentOrder.PSO.mask); + SuppliableIteratorTripleID it3 = hdt.getTriples().search(p3, TripleComponentOrder.PSO.mask); + + assertSame("invalid order ", TripleComponentOrder.POS, it1.getOrder()); + assertSame("invalid order ", TripleComponentOrder.PSO, it2.getOrder()); + assertSame("invalid order ", TripleComponentOrder.PSO, it3.getOrder()); + + HDTMergeJoinIterator it = new HDTMergeJoinIterator( + List.of(new HDTMergeJoinIterator.MergeIteratorData(it1, TripleComponentRole.OBJECT), + new HDTMergeJoinIterator.MergeIteratorData(it2, TripleComponentRole.SUBJECT), + new HDTMergeJoinIterator.MergeIteratorData(it3, TripleComponentRole.SUBJECT))); + + System.out.println(it.hasNext()); + it.forEachRemaining(lst -> System.out + .println(lst.stream().map(d -> dict.toTripleString(Objects.requireNonNull(d.peek())).toString()) + .collect(Collectors.joining(" - ")))); + } + + } +} diff --git a/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapQuadTriplesTest.java b/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapQuadTriplesTest.java index aeeb40c51..6d18a20a1 100644 --- a/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapQuadTriplesTest.java +++ b/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapQuadTriplesTest.java @@ -34,16 +34,6 @@ private static IteratorTripleID fromList(List lst) { private int current; private int lastLoc; - @Override - public boolean hasPrevious() { - return false; - } - - @Override - public TripleID previous() { - return null; - } - @Override public void goToStart() { current = 0; diff --git a/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorTest.java b/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorTest.java index 706376573..2a13014fd 100644 --- a/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorTest.java +++ b/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIteratorTest.java @@ -1,26 +1,473 @@ package com.the_qa_company.qendpoint.core.triples.impl; -import java.io.IOException; - +import com.the_qa_company.qendpoint.core.enums.RDFNotation; +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; +import com.the_qa_company.qendpoint.core.exceptions.ParserException; +import com.the_qa_company.qendpoint.core.hdt.HDT; +import com.the_qa_company.qendpoint.core.hdt.HDTManager; +import com.the_qa_company.qendpoint.core.listener.ProgressListener; +import com.the_qa_company.qendpoint.core.options.HDTOptions; +import com.the_qa_company.qendpoint.core.options.HDTOptionsKeys; +import com.the_qa_company.qendpoint.core.triples.IteratorTripleID; +import com.the_qa_company.qendpoint.core.triples.TripleID; +import com.the_qa_company.qendpoint.core.util.LargeFakeDataSetStreamSupplier; +import org.apache.commons.io.file.PathUtils; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class BitmapTriplesIteratorTest { + @Rule + public TemporaryFolder tempDir = TemporaryFolder.builder().assureDeletion().build(); + @Before public void setUp() throws Exception { } @Test - public void test() throws IOException { -// HDT hdt = HDTManager.mapHDT("/Users/mck/hdt/swdf.hdt", null); -// -// int t = (int) hdt.getTriples().getNumberOfElements(); -// BitmapTriplesIterator it = new BitmapTriplesIterator((BitmapTriples) hdt.getTriples(), t-10, t); -// -// while(it.hasNext()) { -// System.out.println(it.next()); -// } + public void jumpTest() throws IOException, ParserException { + Path root = tempDir.newFolder().toPath(); + + try { + LargeFakeDataSetStreamSupplier sup = LargeFakeDataSetStreamSupplier.createSupplierWithMaxTriples(1000, 32); + Path hdtPath = root.resolve("test.hdt"); + sup.createAndSaveFakeHDT(HDTOptions.empty(), hdtPath); + + try (HDT hdt = HDTManager.mapIndexedHDT(hdtPath)) { + IteratorTripleID it = hdt.getTriples().searchAll(); + + assertTrue("bad class: " + it.getClass(), it instanceof BitmapTriplesIterator); + + TripleID start = it.next().clone(); + + for (int i = 0; i < 458; i++) { + assertTrue(it.hasNext()); + it.next(); + } + + TripleID lastTest = it.next().clone(); + + assertNotEquals(start, lastTest); + + long posLast = it.getLastTriplePosition(); + + it.goToStart(); + assertEquals(start, it.next()); + + assertEquals(0, it.getLastTriplePosition()); + + it.goTo(posLast); + + assertEquals(lastTest, it.next()); + + assertEquals(posLast, it.getLastTriplePosition()); + } + } finally { + PathUtils.deleteDirectory(root); + } + + } + + private static final String JUMP_XYZ_DATASET = """ + @prefix ex: . + + ex:s1 ex:p1 ex:o0000, ex:o0001, ex:o0002, ex:o0003, ex:o0004, ex:o0005 ; + ex:p2 ex:o0000, ex:o0002, ex:o0003, ex:o0004, ex:o0005 ; + ex:p3 ex:o0000, ex:o0001, ex:o0002, ex:o0003, ex:o0004, ex:o0005 ; + ex:p4 ex:o0000, ex:o0001, ex:o0002, ex:o0004, ex:o0005 ; + ex:p5 ex:o0000, ex:o0001, ex:o0002, ex:o0003, ex:o0004, ex:o0005 . + + + ex:s2 ex:p1 ex:o0006, ex:o0007, ex:o0008, ex:o0009, ex:o0010, ex:o0011 ; + ex:p2 ex:o0008, ex:o0009, ex:o0010, ex:o0011 ; + ex:p3 ex:o0007, ex:o0008, ex:o0009, ex:o0010, ex:o0011 ; + ex:p4 ex:o0006, ex:o0007, ex:o0008, ex:o0009, ex:o0010, ex:o0011 . + + + ex:s3 ex:p1 ex:o0003, ex:o0005, ex:o0007, ex:o0009, ex:o0011, ex:o0015 ; + ex:p2 ex:o0003, ex:o0005, ex:o0007, ex:o0009, ex:o0011, ex:o0015 ; + ex:p3 ex:o0003, ex:o0005, ex:o0007, ex:o0009, ex:o0011, ex:o0015 ; + ex:p4 ex:o0003, ex:o0005, ex:o0007 ; + ex:p5 ex:o0003, ex:o0005, ex:o0007, ex:o0009, ex:o0011, ex:o0015 ; + ex:p6 ex:o0003, ex:o0007, ex:o0009, ex:o0011, ex:o0015 ; + ex:p7 ex:o0003, ex:o0005, ex:o0009, ex:o0011, ex:o0015 . + + + ex:s4 ex:p1 ex:o0003, ex:o0005, ex:o0007, ex:o0009, ex:o0011, ex:o0015 ; + ex:p2 ex:o0005, ex:o0007, ex:o0009, ex:o0011, ex:o0015 ; + ex:p3 ex:o0003, ex:o0005, ex:o0009, ex:o0011, ex:o0015 ; + ex:p4 ex:o0003, ex:o0005, ex:o0007, ex:o0009, ex:o0011, ex:o0015 ; + ex:p5 ex:o0003, ex:o0005, ex:o0007, ex:o0009 . + + """; + private static final long JUMP_XYZ_DATASET_X = 4; + private static final long JUMP_XYZ_DATASET_Y = 4; + + @Test + public void jumpXTest() throws IOException, ParserException { + Path root = tempDir.newFolder().toPath(); + + try { + + Path hdtPath = root.resolve("test.hdt"); + + HDTOptions spec = HDTOptions.of(HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, "spo,sop,pos,pso,ops,osp", + HDTOptionsKeys.BITMAPTRIPLES_INDEX_NO_FOQ, true); + + try (HDT hdt = HDTManager.generateHDT( + new ByteArrayInputStream(JUMP_XYZ_DATASET.getBytes(StandardCharsets.UTF_8)), + LargeFakeDataSetStreamSupplier.BASE_URI, RDFNotation.TURTLE, spec, ProgressListener.ignore())) { + hdt.saveToHDT(hdtPath); + } + + try (HDT hdt = HDTManager.mapIndexedHDT(hdtPath, spec, ProgressListener.ignore())) { + + IteratorTripleID ittt = hdt.getTriples().searchAll(); + assertTrue("bad class: " + ittt.getClass(), ittt instanceof BitmapTriplesIterator); + + for (int sid = 1; sid <= JUMP_XYZ_DATASET_X; sid++) { + IteratorTripleID it = hdt.getTriples().searchAll(); + IteratorTripleID itex = hdt.getTriples().searchAll(); + + assertTrue(it.gotoSubject(sid)); + + assertEquals(sid, it.next().getSubject()); + + long s; + do { + assertTrue(itex.hasNext()); + s = itex.next().getSubject(); + } while (s < sid); + assertEquals(sid, s); + + assertTrue(it.hasNext()); + do { + TripleID ac = it.next(); + assertTrue(itex.hasNext()); + TripleID ex = itex.next(); + assertEquals(itex.getLastTriplePosition(), it.getLastTriplePosition()); + + assertEquals(ex, ac); + } while (it.hasNext()); + + assertFalse(itex.hasNext()); + } + } + } finally { + PathUtils.deleteDirectory(root); + } } + @Test + public void jumpYTest() throws IOException, ParserException { + Path root = tempDir.newFolder().toPath(); + + try { + + Path hdtPath = root.resolve("test.hdt"); + + HDTOptions spec = HDTOptions.of(HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, "spo,sop,pos,pso,ops,osp", + HDTOptionsKeys.BITMAPTRIPLES_INDEX_NO_FOQ, true); + + try (HDT hdt = HDTManager.generateHDT( + new ByteArrayInputStream(JUMP_XYZ_DATASET.getBytes(StandardCharsets.UTF_8)), + LargeFakeDataSetStreamSupplier.BASE_URI, RDFNotation.TURTLE, spec, ProgressListener.ignore())) { + hdt.saveToHDT(hdtPath); + } + + try (HDT hdt = HDTManager.mapIndexedHDT(hdtPath, spec, ProgressListener.ignore())) { + + IteratorTripleID ittt = hdt.getTriples().searchAll(); + assertTrue("bad class: " + ittt.getClass(), ittt instanceof BitmapTriplesIterator); + + String lastPosData; + + for (int sid = 1; sid <= JUMP_XYZ_DATASET_X; sid++) { + for (int pid = 1; pid <= JUMP_XYZ_DATASET_Y; pid++) { + IteratorTripleID it = hdt.getTriples().searchAll(); + IteratorTripleID itex = hdt.getTriples().searchAll(); + + assertTrue(it.gotoSubject(sid)); + assertTrue(it.gotoPredicate(pid)); + + TripleID next = it.next(); + lastPosData = "[sid:" + sid + "/pid:" + pid + "][ac:" + it.getLastTriplePosition() + "/ex" + + itex.getLastTriplePosition() + "]" + next; + assertEquals("invalid pos: " + lastPosData, sid, next.getSubject()); + assertEquals("invalid pos: " + lastPosData, pid, next.getPredicate()); + + long s; + long p; + do { + assertTrue(itex.hasNext()); + TripleID next1 = itex.next(); + s = next1.getSubject(); + p = next1.getPredicate(); + lastPosData = "[sid:" + sid + "/pid:" + pid + "][ac:" + it.getLastTriplePosition() + "/ex" + + itex.getLastTriplePosition() + "]" + next1; + } while (s < sid || p < pid); + assertEquals(lastPosData, sid, s); + assertEquals(lastPosData, pid, p); + + assertTrue(it.hasNext()); + do { + TripleID ac = it.next(); + assertTrue(itex.hasNext()); + TripleID ex = itex.next(); + lastPosData = "[sid:" + sid + "/pid:" + pid + "][ac:" + it.getLastTriplePosition() + "/ex" + + itex.getLastTriplePosition() + "]" + ac + "/" + ex; + assertEquals(lastPosData, itex.getLastTriplePosition(), it.getLastTriplePosition()); + + assertEquals(lastPosData, ex, ac); + } while (it.hasNext()); + + assertFalse(itex.hasNext()); + } + } + } + } finally { + PathUtils.deleteDirectory(root); + } + + } + + @Test + public void jumpXYZTest() throws IOException, ParserException { + Path root = tempDir.newFolder().toPath(); + + try { + Path hdtPath = root.resolve("test.hdt"); + + HDTOptions spec = HDTOptions.of(HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, "spo,sop,pos,pso,ops,osp", + HDTOptionsKeys.BITMAPTRIPLES_INDEX_NO_FOQ, true); + final int count = 100_000; + LargeFakeDataSetStreamSupplier supplier = LargeFakeDataSetStreamSupplier + .createSupplierWithMaxTriples(count, 567890987).withMaxElementSplit(50).withMaxLiteralSize(20); + + supplier.createAndSaveFakeHDT(spec, hdtPath); + + try (HDT hdt = HDTManager.mapIndexedHDT(hdtPath, spec, ProgressListener.ignore())) { + int elements = (int) hdt.getTriples().getNumberOfElements(); + + for (int idx = 0; idx < elements; idx++) { + + IteratorTripleID it = hdt.getTriples().searchAll(); + + assertTrue(it.canGoTo()); + + it.goTo(idx); + + TripleID current = it.next().clone(); + assertEquals(idx, it.getLastTriplePosition()); + + for (int member = 0; member < 3; member++) { + IteratorTripleID itac = hdt.getTriples().searchAll(TripleComponentOrder.SPO.mask); + assertSame("invalid order (" + member + "/" + idx + ")", itac.getOrder(), + TripleComponentOrder.SPO); + + // test subject + assertTrue("Can't jump to subject " + current + " (" + member + "/" + idx + ")", + itac.canGoToSubject() && itac.gotoSubject(current.getSubject())); + + if (member >= 1) { + // test predicate + assertTrue("Can't jump to predicate " + current + " (" + member + "/" + idx + ")", + itac.canGoToPredicate() && itac.gotoPredicate(current.getPredicate())); + + if (member >= 2) { + // test object + assertTrue("Can't jump to object " + current + " (" + member + "/" + idx + ")", + itac.canGoToObject() && itac.gotoObject(current.getObject())); + } + } + + assertTrue("for " + current + " (" + member + "/" + idx + ")", itac.hasNext()); + TripleID next = itac.next(); + String err = "invalid next " + next + " != " + current + " (" + member + "/" + idx + ")"; + switch (member) { + case 2: // object + assertEquals("object err " + err, current.getObject(), next.getObject()); + case 1: // predicate + assertEquals("predicate err " + err, current.getPredicate(), next.getPredicate()); + case 0: // subject only + assertEquals("subject err " + err, current.getSubject(), next.getSubject()); + break; + default: + fail("bad member: " + member); + break; + } + if (member == 2) { + assertEquals("idx err " + err, idx, itac.getLastTriplePosition()); + if (itac.hasNext()) { + TripleID newCurrent = itac.next(); + assertTrue("idx err " + err, idx < itac.getLastTriplePosition()); + + if (current.getSubject() == newCurrent.getSubject()) { + // no jump on X, we should have the sam + assertTrue("Can't jump to subject " + current + " (" + member + "/" + idx + ")", + itac.gotoSubject(current.getSubject())); + + if (current.getPredicate() == newCurrent.getPredicate()) { + // no jump on Y, we should have the same + assertTrue("Can't jump to subject " + current + " (" + member + "/" + idx + ")", + itac.gotoPredicate(current.getPredicate())); + + assertFalse( + "Can't jump to subject " + current + " (" + member + "/" + idx + ")", + itac.gotoObject(current.getObject())); + } else { + assertFalse( + "Can't jump to subject " + current + " (" + member + "/" + idx + ")", + itac.gotoPredicate(current.getPredicate())); + } + + } else { + assertFalse("Can't jump to subject " + current + " (" + member + "/" + idx + ")", + itac.gotoSubject(current.getSubject())); + } + } + + } else { + assertTrue("idx err " + err, idx >= itac.getLastTriplePosition()); + } + } + } + } + } finally { + PathUtils.deleteDirectory(root); + } + } + + @Test + public void jumpXYZNextTest() throws IOException, ParserException { + Path root = tempDir.newFolder().toPath(); + + try { + Path hdtPath = root.resolve("test.hdt"); + + HDTOptions spec = HDTOptions.of(HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, "spo,sop,pos,pso,ops,osp", + HDTOptionsKeys.BITMAPTRIPLES_INDEX_NO_FOQ, true); + final int count = 10_000; + LargeFakeDataSetStreamSupplier supplier = LargeFakeDataSetStreamSupplier + .createSupplierWithMaxTriples(count, 567890987).withMaxElementSplit(50).withMaxLiteralSize(20); + + supplier.createAndSaveFakeHDT(spec, hdtPath); + + try (HDT hdt = HDTManager.mapIndexedHDT(hdtPath, spec, ProgressListener.ignore())) { + int elements = (int) hdt.getTriples().getNumberOfElements(); + for (int idx = 0; idx < elements; idx++) { + + IteratorTripleID it = hdt.getTriples().searchAll(); + + assertTrue(it.canGoTo()); + + it.goTo(idx); + + TripleID current = it.next().clone(); + assertEquals(idx, it.getLastTriplePosition()); + + nextCountLoop: + for (int nextCount = 0; nextCount < 10; nextCount++) { + for (int member = 1; member < 3; member++) { + String memberInfo = " (" + member + "/" + idx + "/" + nextCount + ")"; + IteratorTripleID itac = hdt.getTriples().searchAll(TripleComponentOrder.SPO.mask); + assertSame("invalid order" + memberInfo, itac.getOrder(), TripleComponentOrder.SPO); + + // test subject + assertTrue("Can't jump to subject " + current + memberInfo, + itac.canGoToSubject() && itac.gotoSubject(current.getSubject())); + + for (int j = 0; j < nextCount; j++) { + assertTrue(itac.hasNext()); + TripleID pvid = itac.next(); + + if (itac.getLastTriplePosition() == idx) { + assertEquals(pvid, current); + break nextCountLoop; // we consumed the one + // we were searching + // for, it can't be + // used + } + } + + // test predicate + assertTrue("Can't jump to predicate " + current + memberInfo, + itac.canGoToPredicate() && itac.gotoPredicate(current.getPredicate())); + + if (member >= 2) { + // test object + assertTrue("Can't jump to object " + current + memberInfo, + itac.canGoToObject() && itac.gotoObject(current.getObject())); + } + + assertTrue("for " + current + memberInfo, itac.hasNext()); + TripleID next = itac.next(); + String err = "invalid next " + next + " != " + current + memberInfo; + switch (member) { + case 2: // object + assertEquals("object err " + err, current.getObject(), next.getObject()); + case 1: // predicate + assertEquals("predicate err " + err, current.getPredicate(), next.getPredicate()); + case 0: // subject only + assertEquals("subject err " + err, current.getSubject(), next.getSubject()); + break; + default: + fail("bad member: " + member); + break; + } + if (member == 2) { + assertEquals("idx err " + err, idx, itac.getLastTriplePosition()); + if (itac.hasNext()) { + TripleID newCurrent = itac.next(); + assertTrue("idx err " + err, idx < itac.getLastTriplePosition()); + + if (current.getSubject() == newCurrent.getSubject()) { + // no jump on X, we should have the sam + assertTrue("Can't jump to subject " + current + memberInfo + newCurrent, + itac.gotoSubject(current.getSubject())); + + if (current.getPredicate() == newCurrent.getPredicate()) { + // no jump on Y, we should have the + // same + assertTrue("Can't jump to subject " + current + memberInfo + newCurrent, + itac.gotoPredicate(current.getPredicate())); + + assertFalse("Can't jump to subject " + current + memberInfo + newCurrent, + itac.gotoObject(current.getObject())); + } else { + assertFalse("Can't jump to subject " + current + memberInfo + newCurrent, + itac.gotoPredicate(current.getPredicate())); + } + } else { + assertFalse("Can't jump to subject " + current + memberInfo + newCurrent, + itac.gotoSubject(current.getSubject())); + } + } + + } else { + assertTrue("idx err " + err, idx >= itac.getLastTriplePosition()); + } + } + } + + } + } + } finally { + PathUtils.deleteDirectory(root); + } + } } diff --git a/qendpoint-core/src/test/resources/merge_ds.ttl b/qendpoint-core/src/test/resources/merge_ds.ttl new file mode 100644 index 000000000..3d11f1f92 --- /dev/null +++ b/qendpoint-core/src/test/resources/merge_ds.ttl @@ -0,0 +1,27 @@ + +@prefix ex: . +@prefix rdfs: . + +ex:s1 rdfs:name "test" ; + ex:relative ex:s2, ex:s3 . + + +ex:s2 rdfs:name "test 2" . +ex:s3 rdfs:name "test 3" ; + ex:relative ex:s4 ; + ex:id "id3". + +ex:s4 rdfs:name "test 4" ; + ex:relative ex:s3 ; + ex:id "id42", "id43" . + + +ex:s5 rdfs:name "test 5" ; + ex:relative ex:s5 ; + ex:id "id51", "id52", "id53". + +ex:s6 rdfs:name "test 6" ; + ex:relative ex:s5 ; + ex:relative ex:s6 ; + ex:id "id51". + diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/CompiledSail.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/CompiledSail.java index 215ea43e7..96d889e90 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/CompiledSail.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/CompiledSail.java @@ -6,6 +6,7 @@ import com.the_qa_company.qendpoint.core.hdt.HDTManager; import com.the_qa_company.qendpoint.core.options.HDTOptions; import com.the_qa_company.qendpoint.core.triples.TripleString; +import com.the_qa_company.qendpoint.core.util.StopWatch; import com.the_qa_company.qendpoint.store.EndpointFiles; import com.the_qa_company.qendpoint.store.EndpointStore; import com.the_qa_company.qendpoint.store.exception.EndpointStoreException; @@ -216,6 +217,26 @@ public NotifyingSail getSource() { return source; } + private void reindexSail(LuceneSail sail) { + // bypass filtering system to use the source + NotifyingSail oldSail = sail.getBaseSail(); + try { + sail.setBaseSail(source); + String indexId = sail.getParameter(LuceneSail.INDEX_ID); + if (indexId == null || indexId.isEmpty()) { + indexId = ""; + } + StopWatch sw = new StopWatch(); + sw.reset(); + logger.info("Reindexing sail {}", indexId); + sail.reindex(); + sw.stop(); + logger.info("Sail {} reindexed in {} ({}ms)", indexId, sw, sw.getMeasureMillis()); + } finally { + sail.setBaseSail(oldSail); + } + } + /** * reindex all the compiled lucene sails * @@ -225,19 +246,7 @@ public NotifyingSail getSource() { public void reindexLuceneSails() throws SailException { for (LuceneSail sail : luceneSails) { // bypass filtering system to use the source - NotifyingSail oldSail = sail.getBaseSail(); - try { - sail.setBaseSail(source); - String indexId = sail.getParameter(LuceneSail.INDEX_ID); - if (indexId == null || indexId.isEmpty()) { - indexId = "no id"; - } - logger.info("Reindexing sail: {}", indexId); - sail.reindex(); - } finally { - sail.setBaseSail(oldSail); - } - + reindexSail(sail); } } @@ -254,20 +263,7 @@ public void reindexLuceneSail(String index) throws SailException { if (!index.equals(sail.getParameter(LuceneSail.INDEX_ID))) { continue; // ignore } - // bypass filtering system to use the source - NotifyingSail oldSail = sail.getBaseSail(); - try { - sail.setBaseSail(source); - String indexId = sail.getParameter(LuceneSail.INDEX_ID); - if (indexId == null || indexId.isEmpty()) { - indexId = "no id"; - } - logger.info("Reindexing sail: {}", indexId); - sail.reindex(); - } finally { - sail.setBaseSail(oldSail); - } - + reindexSail(sail); } } diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java index 62a7a488d..14341ee5b 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java @@ -77,6 +77,7 @@ public class EndpointStore extends AbstractNotifyingSail { * disable the optimizer */ public static final String QUERY_CONFIG_NO_OPTIMIZER = "no_optimizer"; + public static final String QUERY_CONFIG_NO_OPTIMIZER_MERGE = "no_optimizer_merge"; /** * get the query plan */ diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java index 62b8fff4b..c9ac865d5 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java @@ -825,6 +825,10 @@ public void run() { } } + public EndpointTripleSource getTripleSource() { + return tripleSource; + } + private class EndpointStoreConnectionListener implements SailConnectionListener { private boolean shouldHandle() { return !endpoint.isMerging() || !endpoint.isNotificationsFreeze(); diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreQueryPreparer.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreQueryPreparer.java index 1ddf27800..74d1e0e1a 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreQueryPreparer.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreQueryPreparer.java @@ -2,6 +2,7 @@ import com.the_qa_company.qendpoint.federation.SPARQLServiceWikibaseLabelResolver; import com.the_qa_company.qendpoint.federation.ServiceClauseOptimizer; +import com.the_qa_company.qendpoint.utils.MergeJoinOptimizer; import com.the_qa_company.qendpoint.utils.VariableToIdSubstitution; import org.eclipse.rdf4j.common.iteration.CloseableIteration; import org.eclipse.rdf4j.query.BindingSet; @@ -132,6 +133,7 @@ protected CloseableIteration evaluate(TupleExpr tupleExpr, new IterativeEvaluationOptimizer().optimize(tupleExpr, dataset, bindings); new FilterOptimizer().optimize(tupleExpr, dataset, bindings); new OrderLimitOptimizer().optimize(tupleExpr, dataset, bindings); + new MergeJoinOptimizer(conn).optimize(tupleExpr, dataset, bindings); } new ServiceClauseOptimizer().optimize(tupleExpr, dataset, bindings); diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointTripleSource.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointTripleSource.java index fb3f40f78..3a8689f77 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointTripleSource.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointTripleSource.java @@ -52,6 +52,10 @@ public EndpointTripleSource(EndpointStoreConnection endpointStoreConnection, End this.enableMergeJoin = endpoint.getHDTSpec().getBoolean(EndpointStore.OPTION_QENDPOINT_MERGE_JOIN, false); } + public boolean hasEnableMergeJoin() { + return enableMergeJoin; + } + private void initHDTIndex() { this.numberOfCurrentTriples = this.endpoint.getHdt().getTriples().getNumberOfElements(); } diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/utils/MergeJoinOptimizer.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/utils/MergeJoinOptimizer.java new file mode 100644 index 000000000..a6eb27e7c --- /dev/null +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/utils/MergeJoinOptimizer.java @@ -0,0 +1,93 @@ +package com.the_qa_company.qendpoint.utils; + +import com.the_qa_company.qendpoint.store.EndpointStore; +import com.the_qa_company.qendpoint.store.EndpointStoreConnection; +import org.eclipse.rdf4j.query.BindingSet; +import org.eclipse.rdf4j.query.Dataset; +import org.eclipse.rdf4j.query.algebra.Join; +import org.eclipse.rdf4j.query.algebra.LeftJoin; +import org.eclipse.rdf4j.query.algebra.StatementPattern; +import org.eclipse.rdf4j.query.algebra.TupleExpr; +import org.eclipse.rdf4j.query.algebra.evaluation.QueryOptimizer; +import org.eclipse.rdf4j.query.algebra.helpers.AbstractQueryModelVisitor; + +import java.util.ArrayList; +import java.util.List; + +public class MergeJoinOptimizer implements QueryOptimizer { + private final EndpointStoreConnection conn; + + public MergeJoinOptimizer(EndpointStoreConnection conn) { + this.conn = conn; + } + + @Override + public void optimize(TupleExpr tupleExpr, Dataset dataset, BindingSet bindingSet) { + if (!conn.getTripleSource().hasEnableMergeJoin() + || conn.hasConfig(EndpointStore.QUERY_CONFIG_NO_OPTIMIZER_MERGE)) { + return; // merge join disabled, ignore + } + + ModelVisitor visitor = new ModelVisitor(); + tupleExpr.visit(visitor); + + } + + protected static class ModelVisitor extends AbstractQueryModelVisitor { + + private boolean getJoinPatterns(Join node, List patterns) { + TupleExpr la = node.getLeftArg(); + if (la instanceof Join laj && getJoinPatterns(laj, patterns)) { + return true; + } + if (!(la instanceof StatementPattern stmt)) { + return true; + } + patterns.add(stmt); + + TupleExpr ra = node.getRightArg(); + if (ra instanceof Join raj && getJoinPatterns(raj, patterns)) { + return true; + } + if (!(ra instanceof StatementPattern stmt2)) { + return true; + } + patterns.add(stmt2); + return false; + } + + private List getJoinPatterns(Join node) { + List patterns = new ArrayList<>(); + if (getJoinPatterns(node, patterns)) { + return List.of(); + } + return patterns; + } + + @Override + public void meet(Join node) { + // stack the triple patterns + TupleExpr la = node.getLeftArg(); + TupleExpr ra = node.getRightArg(); + + List patterns = getJoinPatterns(node); + + if (patterns.isEmpty()) { + super.meet(node); + return; + } + + for (StatementPattern p : patterns) { + // TODO: we can replace the patterns + p.getObjectVar().hasValue(); + } + + } + + @Override + public void meet(LeftJoin node) { + super.meet(node); + } + + } +}