feat(elastic): sub index by thing type

importers/src/main/groovy/whelk/reindexer/ElasticReindexer.groovy

@@ -110,7 +110,9 @@ class ElasticReindexer {
 
             threadPool.awaitAllAndShutdown()
             log.info("Done! $counter documents reindexed in ${(System.currentTimeMillis() - startTime) / 1000} seconds.")
-            log.info("New number of mappings/fields in ES: ${whelk.elastic.getFieldCount()}")
+            for (var ix : whelk.elastic.allIndexNames()) {
+                log.info("New number of mappings/fields in ES: ${ix} : ${whelk.elastic.getFieldCount(ix)}")
+            }
             whelk.storage.logStats()
         } catch (Throwable e) {
             log.error("Reindex failed with: ${e}", e)

whelk-core/src/main/groovy/whelk/Document.groovy

@@ -2,6 +2,7 @@ package whelk
 
 import groovy.transform.CompileStatic
 import groovy.util.logging.Log4j2 as Log
+import whelk.exception.WhelkRuntimeException
 import whelk.util.DocumentUtil
 import whelk.util.LegacyIntegrationTools
 import whelk.util.PropertyLoader
@@ -14,6 +15,7 @@ import java.time.ZonedDateTime
 import java.time.format.DateTimeFormatter
 import java.util.function.Predicate
 
+import static whelk.JsonLd.TYPE_KEY
 import static whelk.util.Jackson.mapper
 
 /**
@@ -54,6 +56,7 @@ class Document {
     static final List recordPath = ["@graph", 0]
     static final List recordIdPath = ["@graph", 0, "@id"]
     static final List workIdPath = ["@graph", 1, "instanceOf", "@id"]
+    static final List workTypePath = ["@graph", 1, "instanceOf", "@type"]
     static final List thingMetaPath = ["@graph", 1, "meta", "@id"]
     static final List recordSameAsPath = ["@graph", 0, "sameAs"]
     static final List recordTypedIDsPath = ["@graph", 0, "identifiedBy"]
@@ -929,6 +932,22 @@ class Document {
         return getRecordIdentifiers().first().endsWith("#work-record")
     }
 
+    // FIXME
+    // All these "virtual record" methods are hardcoded for blank Works
+    String singleThingTypeOrVirtualThingType() {
+        var types = isVirtual()
+            ? _get(workTypePath, data)
+            : _get(thingTypePath, data)
+
+        if (types instanceof String) {
+            return (String) types
+        }
+        else if (types instanceof List) {
+            return (String) ((List) types).getFirst()
+        }
+        throw new WhelkRuntimeException("Unexpected ${TYPE_KEY}: ${types}")
+    }
+
     // All these "virtual record" methods are hardcoded for blank Works
     void centerOnVirtualMainEntity() {
         if (!isVirtual()) {

whelk-core/src/main/groovy/whelk/JsonLd.groovy

@@ -743,6 +743,12 @@ class JsonLd {
         return superTermOf
     }
 
+    /**
+     *
+     * @param type
+     * @param baseType
+     * @return true if types is a subclass of baseType OR type == baseType
+     */
     boolean isSubClassOf(String type, String baseType) {
         if (!type) {
             return false

whelk-core/src/main/groovy/whelk/Whelk.groovy

@@ -78,6 +78,12 @@ class Whelk {
     boolean skipIndexDependers = false
     boolean skipSparql = false
 
+    enum EsMode {
+        ELASTIC_ENABLED,
+        ELASTIC_DISABLED
+    }
+    private EsMode esMode
+
     // useCache may be set to true only when doing initial imports (temporary processes with the rest of Libris down).
     // Any other use of this results in a "local" cache, which will not be invalidated when data changes elsewhere,
     // resulting in potential serving of stale data.
@@ -97,15 +103,14 @@ class Whelk {
     }
 
     static Whelk createLoadedSearchWhelk(Properties configuration, boolean useCache = false) {
-        Whelk whelk = new Whelk(configuration, useCache)
+        Whelk whelk = new Whelk(configuration, EsMode.ELASTIC_ENABLED, useCache)
         whelk.configureAndLoad(configuration)
         return whelk
     }
 
-    Whelk(PostgreSQLComponent pg, ElasticSearch es) {
+    Whelk(PostgreSQLComponent pg, EsMode esMode) {
         this(pg)
-        this.elastic = es
-        log.info("Using index: $elastic")
+        this.esMode = esMode
     }
 
     Whelk(PostgreSQLComponent pg) {
@@ -114,8 +119,8 @@ class Whelk {
         log.info("Started with storage: $storage")
     }
 
-    Whelk(Properties conf, useCache = false) {
-        this(useCache ? new CachingPostgreSQLComponent(conf) : new PostgreSQLComponent(conf), new ElasticSearch(conf))
+    Whelk(Properties conf, EsMode esMode, useCache = false) {
+        this(useCache ? new CachingPostgreSQLComponent(conf) : new PostgreSQLComponent(conf), esMode)
     }
 
     private void configureAndLoad(Properties configuration) {
@@ -150,6 +155,11 @@ class Whelk {
 
         loadCoreData(systemContextUri)
 
+        if (this.esMode == EsMode.ELASTIC_ENABLED) {
+            this.elastic = new ElasticSearch(configuration, jsonld)
+            elasticFind = new ElasticFind(new ESQuery(this))
+        }
+
         sparqlUpdater = SparqlUpdater.build(storage, jsonld.context, configuration)
         sparqlQueryClient = new SparqlQueryClient(configuration.getProperty('sparqlEndpoint', null), jsonld);
     }
@@ -241,9 +251,6 @@ class Whelk {
     void setJsonld(JsonLd jsonld) {
         this.jsonld = jsonld
         storage.setJsonld(jsonld)
-        if (elastic) {
-            elasticFind = new ElasticFind(new ESQuery(this))
-        }
         initDocumentNormalizers()
         this.fresnelUtil = new FresnelUtil(jsonld)
     }
@@ -366,7 +373,8 @@ class Whelk {
         removedLinks.each { link ->
             String id = storage.getSystemIdByIri(link.iri)
             if (id) {
-                elastic.decrementReverseLinks(id, link.relation)
+                Document doc = storage.load(id)
+                elastic.decrementReverseLinks(doc, link.relation)
             }
         }
 
@@ -386,7 +394,7 @@ class Whelk {
                     reindexAffectedReverseIntegral(doc)
                 } else {
                     // just update link counter
-                    elastic.incrementReverseLinks(id, link.relation)
+                    elastic.incrementReverseLinks(doc, link.relation)
                 }
             }
         }