Skip to content

Commit f699dff

Browse files
ate47ate47
authored
Merge pull request #437 from the-qa-company/dev
Dev
2 parents a1dccd4 + 3ec8dc4 commit f699dff

File tree

4 files changed

+240
-0
lines changed

4 files changed

+240
-0
lines changed

‎qendpoint-cli/bin/hdtDiffCat.bat‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
@echo off
2+
3+
call "%~dp0\javaenv.bat"
4+
5+
"%JAVACMD%" %JAVAOPTIONS% -classpath %~dp0\..\lib\* com.the_qa_company.qendpoint.core.tools.HDTDiffCat %*

‎qendpoint-cli/bin/hdtDiffCat.ps1‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
param(
2+
[Parameter()]
3+
[String]
4+
$options,
5+
[Parameter()]
6+
[String]
7+
$config,
8+
[Parameter()]
9+
[String]
10+
$diff,
11+
[Parameter()]
12+
[Switch]
13+
$index,
14+
[Parameter()]
15+
[Switch]
16+
$version,
17+
[Parameter()]
18+
[Switch]
19+
$quiet,
20+
[Parameter()]
21+
[Switch]
22+
$color,
23+
[Parameter(ValueFromRemainingArguments, Position = 0)]
24+
[string[]]
25+
$OtherParams
26+
)
27+
28+
& "$(Get-Item $PSScriptRoot)/javaenv.ps1" "com.the_qa_company.qendpoint.core.tools.HDTDiffCat" -RequiredParameters $PSBoundParameters

‎qendpoint-cli/bin/hdtDiffCat.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
3+
source `dirname $0`/javaenv.sh
4+
5+
#export MAVEN_OPTS="-Xmx6g"
6+
#mvn exec:java -Dexec.mainClass="com.the_qa_company.qendpoint.core.tools.HDTDiffCat" -Dexec.args="$*"
7+
8+
$JAVA $JAVA_OPTIONS -cp $CP:$CLASSPATH com.the_qa_company.qendpoint.core.tools.HDTDiffCat $*
9+
10+
exit $?

‎qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/tools/HDTDiffCat.java‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
package com.the_qa_company.qendpoint.core.tools;
2+
3+
import com.beust.jcommander.JCommander;
4+
import com.beust.jcommander.Parameter;
5+
import com.beust.jcommander.internal.Lists;
6+
import com.the_qa_company.qendpoint.core.compact.bitmap.Bitmap64Big;
7+
import com.the_qa_company.qendpoint.core.enums.RDFNotation;
8+
import com.the_qa_company.qendpoint.core.exceptions.NotFoundException;
9+
import com.the_qa_company.qendpoint.core.exceptions.ParserException;
10+
import com.the_qa_company.qendpoint.core.hdt.HDT;
11+
import com.the_qa_company.qendpoint.core.hdt.HDTManager;
12+
import com.the_qa_company.qendpoint.core.hdt.HDTVersion;
13+
import com.the_qa_company.qendpoint.core.listener.MultiThreadListener;
14+
import com.the_qa_company.qendpoint.core.options.HDTOptions;
15+
import com.the_qa_company.qendpoint.core.options.HDTOptionsKeys;
16+
import com.the_qa_company.qendpoint.core.rdf.RDFParserCallback;
17+
import com.the_qa_company.qendpoint.core.rdf.RDFParserFactory;
18+
import com.the_qa_company.qendpoint.core.triples.IteratorTripleString;
19+
import com.the_qa_company.qendpoint.core.util.StopWatch;
20+
import com.the_qa_company.qendpoint.core.util.io.Closer;
21+
import com.the_qa_company.qendpoint.core.util.listener.ColorTool;
22+
import com.the_qa_company.qendpoint.core.util.listener.MultiThreadListenerConsole;
23+
import org.apache.commons.io.FileUtils;
24+
25+
import java.io.File;
26+
import java.io.IOException;
27+
import java.nio.file.Files;
28+
import java.nio.file.Path;
29+
import java.util.List;
30+
import java.util.stream.Collectors;
31+
32+
public class HDTDiffCat {
33+
34+
private ColorTool colorTool;
35+
36+
@Parameter(description = "<input HDTs>+ <output HDT>")
37+
public List<String> parameters = Lists.newArrayList();
38+
39+
@Parameter(names = "-options", description = "HDT Conversion options (override those of config file)")
40+
public String options;
41+
42+
@Parameter(names = "-config", description = "Conversion config file")
43+
public String configFile;
44+
45+
@Parameter(names = "-diff", description = "File to use to do the diff")
46+
public String diff;
47+
48+
@Parameter(names = "-index", description = "Generate also external indices to solve all queries")
49+
public boolean generateIndex;
50+
51+
@Parameter(names = "-version", description = "Prints the HDT version number")
52+
public static boolean showVersion;
53+
54+
@Parameter(names = "-quiet", description = "Do not show progress of the conversion")
55+
public boolean quiet;
56+
57+
@Parameter(names = "-color", description = "Print using color (if available)")
58+
public boolean color;
59+
60+
private HDT diffcat(String location, HDTOptions spec, MultiThreadListener listener)
61+
throws IOException, ParserException, NotFoundException {
62+
List<String> inputs = parameters.subList(0, parameters.size() - 1);
63+
64+
if (diff == null || diff.isEmpty()) {
65+
return HDTManager.catHDT(inputs, spec, listener);
66+
}
67+
68+
RDFNotation type = RDFNotation.guess(diff);
69+
70+
Bitmap64Big[] bms = new Bitmap64Big[inputs.size()];
71+
HDT[] inputsMap = new HDT[inputs.size()];
72+
73+
try {
74+
for (int i = 0; i < bms.length; i++) {
75+
inputsMap[i] = HDTManager.mapHDT(inputs.get(i));
76+
bms[i] = Bitmap64Big.memory(inputsMap[i].getTriples().getNumberOfElements());
77+
}
78+
79+
RDFParserCallback.RDFCallback callback = ((triple, pos) -> {
80+
for (int i = 0; i < inputsMap.length; i++) {
81+
IteratorTripleString find;
82+
try {
83+
find = inputsMap[i].search(triple);
84+
} catch (NotFoundException e) {
85+
throw new RuntimeException(e);
86+
}
87+
88+
if (find.hasNext()) {
89+
find.next();
90+
bms[i].set(find.getLastTriplePosition(), true); // delete
91+
// it
92+
}
93+
94+
}
95+
});
96+
97+
if (type == RDFNotation.HDT) {
98+
try (HDT diffHDT = HDTManager.mapHDT(diff)) {
99+
IteratorTripleString it = diffHDT.searchAll();
100+
while (it.hasNext()) {
101+
callback.processTriple(it.next(), 0);
102+
}
103+
}
104+
} else {
105+
RDFParserCallback parser = RDFParserFactory.getParserCallback(type, spec);
106+
parser.doParse(diff, "", type, true, callback);
107+
}
108+
} catch (Throwable t) {
109+
try {
110+
Closer.closeSingle(inputsMap);
111+
} catch (Throwable t2) {
112+
t.addSuppressed(t2);
113+
}
114+
throw t;
115+
}
116+
Closer.closeSingle(inputsMap);
117+
118+
return HDTManager.diffBitCatHDT(inputs, List.of(bms), spec, listener);
119+
}
120+
121+
public void execute() throws IOException, ParserException, NotFoundException {
122+
HDTOptions spec;
123+
if (configFile != null) {
124+
spec = HDTOptions.readFromFile(configFile);
125+
} else {
126+
spec = HDTOptions.of();
127+
}
128+
if (options != null) {
129+
spec.setOptions(options);
130+
}
131+
132+
String hdtOutput = parameters.get(parameters.size() - 1);
133+
File file = new File(hdtOutput);
134+
135+
String locationOpt = spec.get(HDTOptionsKeys.HDTCAT_LOCATION);
136+
137+
if (locationOpt == null) {
138+
locationOpt = file.getAbsolutePath() + "_tmp";
139+
spec.set(HDTOptionsKeys.HDTCAT_LOCATION, locationOpt);
140+
}
141+
142+
File theDir = new File(locationOpt);
143+
Files.createDirectories(theDir.toPath());
144+
String location = theDir.getAbsolutePath() + "/";
145+
146+
MultiThreadListener listenerConsole = !quiet ? new MultiThreadListenerConsole(color) : null;
147+
StopWatch startCat = new StopWatch();
148+
try (HDT hdt = diffcat(location, spec, listenerConsole)) {
149+
colorTool.logValue("Files catdiff in ...... ", startCat.stopAndShow(), true);
150+
assert hdt != null;
151+
// Show Basic stats
152+
if (!quiet) {
153+
colorTool.logValue("Total Triples ......... ", String.valueOf(hdt.getTriples().getNumberOfElements()));
154+
colorTool.logValue("Different subjects .... ", String.valueOf(hdt.getDictionary().getNsubjects()));
155+
colorTool.logValue("Different predicates .. ", String.valueOf(hdt.getDictionary().getNpredicates()));
156+
colorTool.logValue("Different objects ..... ", String.valueOf(hdt.getDictionary().getNobjects()));
157+
colorTool.logValue("Common Subject/Object . ", String.valueOf(hdt.getDictionary().getNshared()));
158+
}
159+
160+
// Dump to HDT file
161+
StopWatch sw = new StopWatch();
162+
hdt.saveToHDT(hdtOutput, listenerConsole);
163+
colorTool.logValue("HDT saved to file in .. ", sw.stopAndShow());
164+
Files.deleteIfExists(Path.of(location + "dictionary"));
165+
Files.deleteIfExists(Path.of(location + "triples"));
166+
FileUtils.deleteDirectory(theDir);
167+
168+
// Generate index and dump it to .hdt.index file
169+
sw.reset();
170+
if (generateIndex) {
171+
HDTManager.indexedHDT(hdt, listenerConsole);
172+
colorTool.logValue("Index generated and saved in ", sw.stopAndShow());
173+
}
174+
}
175+
}
176+
177+
public static void main(String[] args) throws Throwable {
178+
HDTDiffCat diffcat = new HDTDiffCat();
179+
JCommander com = new JCommander(diffcat);
180+
com.parse(args);
181+
com.setProgramName("hdtDiffCat");
182+
diffcat.colorTool = new ColorTool(diffcat.color, diffcat.quiet);
183+
184+
if (showVersion) {
185+
diffcat.colorTool.log(HDTVersion.get_version_string("."));
186+
System.exit(0);
187+
} else if (diffcat.parameters.size() < 2) {
188+
com.usage();
189+
System.exit(1);
190+
}
191+
192+
diffcat.colorTool.log("DiffCat "
193+
+ diffcat.parameters.stream().limit(diffcat.parameters.size() - 1).collect(Collectors.joining(", "))
194+
+ " to " + diffcat.parameters.get(diffcat.parameters.size() - 1));
195+
diffcat.execute();
196+
}
197+
}

0 commit comments

Comments
 (0)