From 5a24f67f915b15180f5dc37ff86b444fda9904a8 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Sat, 10 Apr 2021 23:01:06 +0200 Subject: [PATCH 01/11] New procedure spatial.merge.into(a,b) for merging one layer into another This includes tests that pass for various combinations of simple point layers, but should also work for any layers that have property only geometry encoders. There are failing tests for merging into OSM, which should not be supported, and for merging one OSM into another, which we hope to support in the next update. In fact, this commit is preparatory work for OSM-OSM merging capabilities, which are planned, but not implemented yet. --- .../gis/spatial/SpatialTopologyUtils.java | 16 +- .../neo4j/gis/spatial/merge/MergeUtils.java | 93 ++++++++++++ .../org/neo4j/gis/spatial/osm/OSMLayer.java | 12 +- .../spatial/procedures/SpatialProcedures.java | 12 +- .../procedures/SpatialProceduresTest.java | 140 +++++++++++++++++- 5 files changed, 256 insertions(+), 17 deletions(-) create mode 100644 src/main/java/org/neo4j/gis/spatial/merge/MergeUtils.java diff --git a/src/main/java/org/neo4j/gis/spatial/SpatialTopologyUtils.java b/src/main/java/org/neo4j/gis/spatial/SpatialTopologyUtils.java index 436d58f90..5ed3435e7 100644 --- a/src/main/java/org/neo4j/gis/spatial/SpatialTopologyUtils.java +++ b/src/main/java/org/neo4j/gis/spatial/SpatialTopologyUtils.java @@ -136,8 +136,8 @@ public static List findClosestEdges(Transaction tx, Point point, La * @param geometry Geometry to measure * @param measure the distance along the geometry * @return Point at 'measure' distance along the geometry - * @see http://download.oracle.com/docs/cd/B13789_01/appdev.101/b10826/sdo_lrs_ref.htm#i85478 - * @see http://www.vividsolutions.com/jts/javadoc/com/vividsolutions/jts/linearref/LengthIndexedLine.html + * @see SDO_LRS.LOCATE_PT + * @see LengthIndexedLine */ public static Point locatePoint(Layer layer, Geometry geometry, double measure) { return layer.getGeometryFactory().createPoint(locatePoint(geometry, measure)); @@ -153,8 +153,8 @@ public static Point locatePoint(Layer layer, Geometry geometry, double measure) * @param geometry Geometry to measure * @param measure the distance along the geometry * @return Coordinate at 'measure' distance along the geometry - * @see http://download.oracle.com/docs/cd/B13789_01/appdev.101/b10826/sdo_lrs_ref.htm#i85478 - * @see http://www.vividsolutions.com/jts/javadoc/com/vividsolutions/jts/linearref/LengthIndexedLine.html + * @see SDO_LRS.LOCATE_PT + * @see LengthIndexedLine */ public static Coordinate locatePoint(Geometry geometry, double measure) { return new LengthIndexedLine(geometry).extractPoint(measure); @@ -174,8 +174,8 @@ public static Coordinate locatePoint(Geometry geometry, double measure) { * @param measure the distance along the geometry * @param offset the distance offset to the left (or right for negative numbers) * @return Point at 'measure' distance along the geometry, and offset - * @see http://download.oracle.com/docs/cd/B13789_01/appdev.101/b10826/sdo_lrs_ref.htm#i85478 - * @see http://www.vividsolutions.com/jts/javadoc/com/vividsolutions/jts/linearref/LengthIndexedLine.html + * @see SDO_LRS.LOCATE_PT + * @see LengthIndexedLine */ public static Point locatePoint(Layer layer, Geometry geometry, double measure, double offset) { return layer.getGeometryFactory().createPoint(locatePoint(geometry, measure, offset)); @@ -193,8 +193,8 @@ public static Point locatePoint(Layer layer, Geometry geometry, double measure, * @param measure the distance along the geometry * @param offset the distance offset to the left (or right for negative numbers) * @return Point at 'measure' distance along the geometry, and offset - * @see http://download.oracle.com/docs/cd/B13789_01/appdev.101/b10826/sdo_lrs_ref.htm#i85478 - * @see http://www.vividsolutions.com/jts/javadoc/com/vividsolutions/jts/linearref/LengthIndexedLine.html + * @see SDO_LRS.LOCATE_PT + * @see LengthIndexedLine */ public static Coordinate locatePoint(Geometry geometry, double measure, double offset) { return new LengthIndexedLine(geometry).extractPoint(measure, offset); diff --git a/src/main/java/org/neo4j/gis/spatial/merge/MergeUtils.java b/src/main/java/org/neo4j/gis/spatial/merge/MergeUtils.java new file mode 100644 index 000000000..1677d6b6c --- /dev/null +++ b/src/main/java/org/neo4j/gis/spatial/merge/MergeUtils.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2010-2020 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j Spatial. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gis.spatial.merge; + +import org.locationtech.jts.geom.Geometry; +import org.neo4j.gis.spatial.EditableLayer; +import org.neo4j.gis.spatial.GeometryEncoder; +import org.neo4j.gis.spatial.encoders.Configurable; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Transaction; + +import java.util.ArrayList; + +public class MergeUtils { + + public interface Mergeable { + long mergeFrom(Transaction tx, EditableLayer other); + } + + private static boolean encodersIdentical(EditableLayer layer, EditableLayer mergeLayer) { + GeometryEncoder layerEncoder = layer.getGeometryEncoder(); + GeometryEncoder mergeEncoder = mergeLayer.getGeometryEncoder(); + Class layerGeometryClass = layerEncoder.getClass(); + Class mergeGeometryClass = mergeEncoder.getClass(); + if (layerGeometryClass.isAssignableFrom(mergeGeometryClass)) { + if (mergeEncoder instanceof Configurable && layerEncoder instanceof Configurable) { + String mergeConfig = ((Configurable) mergeEncoder).getConfiguration(); + String layerConfig = ((Configurable) layerEncoder).getConfiguration(); + return mergeConfig.equals(layerConfig); + } else { + return false; + } + } + return false; + } + + public static long mergeLayerInto(Transaction tx, EditableLayer layer, EditableLayer mergeLayer) { + long count; + Class layerClass = layer.getClass(); + Class mergeClass = mergeLayer.getClass(); + if (layer instanceof Mergeable) { + count = ((Mergeable) layer).mergeFrom(tx, mergeLayer); + } else if (layerClass.isAssignableFrom(mergeClass)) { + if (encodersIdentical(layer, mergeLayer)) { + // With identical encoders, we can simply add the node as is, but must remove it first + ArrayList toAdd = new ArrayList<>(); + for (Node node : mergeLayer.getIndex().getAllIndexedNodes(tx)) { + toAdd.add(node); + } + for (Node node : toAdd) { + // Remove each from the previous index before adding to the new index, so as not to have multiple incoming RTREE_REFERENCE + mergeLayer.removeFromIndex(tx, node.getId()); + layer.add(tx, node); + } + count = toAdd.size(); + } else { + // With differing encoders, we must decode and re-encode each geometry, so we also create new nodes and remove and delete the actual nodes later + ArrayList toRemove = new ArrayList<>(); + GeometryEncoder fromEncoder = mergeLayer.getGeometryEncoder(); + for (Node node : mergeLayer.getIndex().getAllIndexedNodes(tx)) { + toRemove.add(node); + Geometry geometry = fromEncoder.decodeGeometry(node); + layer.add(tx, geometry); + } + for (Node remove : toRemove) { + mergeLayer.removeFromIndex(tx, remove.getId()); + remove.delete(); + } + count = toRemove.size(); + } + } else { + throw new IllegalArgumentException(String.format("Cannot merge '%s' into '%s': layer classes are not compatible: '%s' cannot be caste as '%s'", mergeLayer.getName(), layer.getName(), mergeClass.getSimpleName(), layerClass.getSimpleName())); + } + return count; + } +} diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java index a0ceb4aba..ff3842fdf 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java @@ -25,6 +25,7 @@ import org.geotools.referencing.crs.DefaultGeographicCRS; import org.json.simple.JSONObject; import org.neo4j.gis.spatial.*; +import org.neo4j.gis.spatial.merge.MergeUtils; import org.neo4j.gis.spatial.rtree.NullListener; import org.neo4j.graphdb.*; import org.opengis.referencing.crs.CoordinateReferenceSystem; @@ -34,7 +35,7 @@ * extends the DynamicLayer class because the OSM dataset can have many layers. * Only one is primary, the layer containing all ways. Other layers are dynamic. */ -public class OSMLayer extends DynamicLayer { +public class OSMLayer extends DynamicLayer implements MergeUtils.Mergeable { private OSMDataset osmDataset; @Override @@ -261,4 +262,13 @@ public File getStyle() { // TODO: Replace with a proper resource lookup, since this will be in the JAR return new File("dev/neo4j/neo4j-spatial/src/main/resources/sld/osm/osm.sld"); } + + @Override + public long mergeFrom(Transaction tx, EditableLayer other) { + if (other instanceof OSMLayer) { + throw new UnsupportedOperationException("Not implemented"); + } else { + throw new IllegalArgumentException("Cannot merge non-OSM layer into OSM layer: '" + other.getName() + "' is not OSM"); + } + } } diff --git a/src/main/java/org/neo4j/gis/spatial/procedures/SpatialProcedures.java b/src/main/java/org/neo4j/gis/spatial/procedures/SpatialProcedures.java index ee6c433f7..f0975d30a 100644 --- a/src/main/java/org/neo4j/gis/spatial/procedures/SpatialProcedures.java +++ b/src/main/java/org/neo4j/gis/spatial/procedures/SpatialProcedures.java @@ -35,6 +35,7 @@ import org.neo4j.gis.spatial.index.LayerGeohashPointIndex; import org.neo4j.gis.spatial.index.LayerHilbertPointIndex; import org.neo4j.gis.spatial.index.LayerZOrderPointIndex; +import org.neo4j.gis.spatial.merge.MergeUtils; import org.neo4j.gis.spatial.osm.OSMGeometryEncoder; import org.neo4j.gis.spatial.osm.OSMImporter; import org.neo4j.gis.spatial.osm.OSMLayer; @@ -63,7 +64,6 @@ import java.nio.charset.Charset; import java.util.*; import java.util.function.BiFunction; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -765,6 +765,16 @@ public void run() { } } + @Procedure(value="spatial.merge.into", mode=WRITE) + @Description("Merges two layers by copying geometries from the second layer into the first and deleting the second layer") + public Stream mergeLayerIntoLayer( + @Name("layerName") String layerName, + @Name("toMerge") String mergeName) { + EditableLayer layer = getEditableLayerOrThrow(tx, spatial(), layerName); + EditableLayer mergeLayer = getEditableLayerOrThrow(tx, spatial(), mergeName); + return Stream.of(new CountResult(MergeUtils.mergeLayerInto(tx, layer, mergeLayer))); + } + @Procedure(value="spatial.bbox", mode=WRITE) @Description("Finds all geometry nodes in the given layer within the lower left and upper right coordinates of a box") public Stream findGeometriesInBBox( diff --git a/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java b/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java index 5aa740111..a68618c1b 100644 --- a/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java +++ b/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java @@ -48,6 +48,7 @@ import java.util.Map; import java.util.function.Consumer; +import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.*; import static org.junit.Assert.*; import static org.neo4j.configuration.GraphDatabaseSettings.DEFAULT_DATABASE_NAME; @@ -346,9 +347,16 @@ private void executeWrite(String call) { } private Node createNode(String call, String column) { + return createNode(call, null, column); + } + + private Node createNode(String call, Map params, String column) { + if (params == null) { + params = emptyMap(); + } Node node; try (Transaction tx = db.beginTx()) { - ResourceIterator nodes = tx.execute(call).columnAs(column); + ResourceIterator nodes = tx.execute(call, params).columnAs(column); node = (Node) nodes.next(); nodes.close(); tx.commit(); @@ -1040,17 +1048,136 @@ private void testCountQuery(String name, String query, long count, String column System.out.println(name + " query took " + (System.currentTimeMillis() - start) + "ms - " + params); } + private Node addPointLayerXYWithNode(String name, double x, double y) { + execute("CALL spatial.addPointLayerXY($name,'lon','lat')", map("name", name)); + Node node = createNode("CREATE (n:Node {lat:$lat,lon:$lon}) WITH n CALL spatial.addNode($name,n) YIELD node RETURN node", map("name", name, "lat", y, "lon", x), "node"); + return node; + } + + private Node addPointLayerWithNode(String name, double x, double y) { + execute("CALL spatial.addPointLayer($name)", map("name", name)); + Node node = createNode("CREATE (n:Node {latitude:$lat,longitude:$lon}) WITH n CALL spatial.addNode($name,n) YIELD node RETURN node", map("name", name, "lat", y, "lon", x), "node"); + return node; + } + + private Node addPointLayerGeohashWithNode(String name, double x, double y) { + execute("CALL spatial.addPointLayerGeohash($name)", map("name", name)); + Node node = createNode("CREATE (n:Node {latitude:$lat,longitude:$lon}) WITH n CALL spatial.addNode($name,n) YIELD node RETURN node", map("name", name, "lat", y, "lon", x), "node"); + return node; + } + + private Node addWKTLayerWithPointNode(String name, double x, double y) { + execute("CALL spatial.addWKTLayer($name,'wkt')", map("name", name)); + Node node = createNode("CALL spatial.addWKT($name,$wkt)", map("name", name, "wkt", String.format("POINT (%f %f)", x, y)), "node"); + return node; + } + + @Test + public void merge_layers_of_identical_type() { + Node nodeA = addPointLayerWithNode("geomA", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", r -> assertEquals(nodeA, r.get("node"))); + Node nodeB = addPointLayerWithNode("geomB", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", r -> assertEquals(nodeB, r.get("node"))); + Node nodeC = addPointLayerWithNode("geomC", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomC',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", r -> assertEquals(nodeC, r.get("node"))); + + testCall(db, "CALL spatial.merge.into($nameB,$nameC)", map("nameB", "geomB", "nameC", "geomC"), r -> assertEquals(1L, r.get("count"))); + testCallCount(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 1); + testCallCount(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 2); + testCallCount(db, "CALL spatial.bbox('geomC',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 0); + + testCall(db, "CALL spatial.merge.into($nameA,$nameB)", map("nameA", "geomA", "nameB", "geomB"), r -> assertEquals(2L, r.get("count"))); + testCallCount(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 3); + testCallCount(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 0); + testCallCount(db, "CALL spatial.bbox('geomC',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 0); + } + + @Test + public void merge_layers_of_similar_type() { + Node nodeA = addPointLayerXYWithNode("geomA", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", r -> assertEquals(nodeA, r.get("node"))); + Node nodeB = addPointLayerWithNode("geomB", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", r -> assertEquals(nodeB, r.get("node"))); + Node nodeC = addPointLayerGeohashWithNode("geomC", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomC',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", r -> assertEquals(nodeC, r.get("node"))); + + testCall(db, "CALL spatial.merge.into($nameB,$nameC)", map("nameB", "geomB", "nameC", "geomC"), r -> assertEquals(1L, r.get("count"))); + testCallCount(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 1); + testCallCount(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 2); + testCallCount(db, "CALL spatial.bbox('geomC',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 0); + + testCall(db, "CALL spatial.merge.into($nameA,$nameB)", map("nameA", "geomA", "nameB", "geomB"), r -> assertEquals(2L, r.get("count"))); + testCallCount(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 3); + testCallCount(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 0); + testCallCount(db, "CALL spatial.bbox('geomC',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 0); + } + + @Test + public void fail_to_merge_layers_of_different_type() { + Node nodeA = addPointLayerXYWithNode("geomA", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", r -> assertEquals(nodeA, r.get("node"))); + Node nodeB = addWKTLayerWithPointNode("geomB", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", r -> assertEquals(nodeB, r.get("node"))); + testCallFails(db, "CALL spatial.merge.into($nameA,$nameB)", map("nameA", "geomA", "nameB", "geomB"), "layer classes are not compatible"); + } + + @Test + public void fail_to_merge_non_OSM_into_OSM() { + execute("CALL spatial.addLayer('osm','OSM','')"); + testCountQuery("importOSMToLayerAndAddGeometry", "CALL spatial.importOSMToLayer($name,'map.osm')", 55, "count", map("name", "osm")); + testCallCount(db, "CALL spatial.layers()", null, 1); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", map("name", "osm", "lon", 15.2, "lat", 60.1), 0); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", map("name", "osm"), 217); + + // Adding a point to the layer + Node node = createNode("CALL spatial.addWKT($name,$wkt)", map("name", "osm", "wkt", String.format("POINT (%f %f)", 15.2, 60.1)), "node"); + testCall(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", map("name", "osm", "lon", 15.2, "lat", 60.1), r -> assertEquals(node, r.get("node"))); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", map("name", "osm", "lon", 15.2, "lat", 60.1), 1); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", map("name", "osm"), 217); + + Node nodeA = addPointLayerXYWithNode("geomA", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", r -> assertEquals(nodeA, r.get("node"))); + Node nodeB = addWKTLayerWithPointNode("geomB", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", r -> assertEquals(nodeB, r.get("node"))); + testCallFails(db, "CALL spatial.merge.into($name,$nameA)", map("name", "osm", "nameA", "geomA"), "Cannot merge non-OSM layer into OSM layer"); + testCallFails(db, "CALL spatial.merge.into($name,$nameB)", map("name", "osm", "nameB", "geomB"), "Cannot merge non-OSM layer into OSM layer"); + } + + @Test + public void should_not_but_still_fail_to_merge_OSM_into_OSM() { + for (String name : new String[]{"osmA", "osmB"}) { + execute("CALL spatial.addLayer($name,'OSM','')", map("name", name)); + testCountQuery("importOSMToLayerAndAddGeometry", "CALL spatial.importOSMToLayer($name,'map.osm')", 55, "count", map("name", name)); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", map("name", name, "lon", 15.2, "lat", 60.1), 0); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", map("name", name), 217); + } + testCallCount(db, "CALL spatial.layers()", null, 2); + + // Adding a point to the second layer + Node node = createNode("CALL spatial.addWKT($name,$wkt)", map("name", "osmB", "wkt", String.format("POINT (%f %f)", 15.2, 60.1)), "node"); + testCall(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", map("name", "osmB", "lon", 15.2, "lat", 60.1), r -> assertEquals(node, r.get("node"))); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", map("name", "osmB", "lon", 15.2, "lat", 60.1), 1); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", map("name", "osmB"), 217); + + // Assert that osmA does not have the extra node + testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", map("name", "osmB", "lon", 15.2, "lat", 60.1), 1); + + // try merge osmB into osmA - should succeed, but does not yet + testCallFails(db, "CALL spatial.merge.into($nameA,$nameB)", map("nameA", "osmA", "nameB", "osmB"), "Not implemented"); + + // Here we should assert that osmA now does have the extra node + //..... + } + @Test public void find_geometries_in_a_bounding_box_short() { - execute("CALL spatial.addPointLayerXY('geom','lon','lat')"); - Node node = createNode("CREATE (n:Node {lat:60.1,lon:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); + Node node = addPointLayerXYWithNode("geom", 15.2, 60.1); testCall(db, "CALL spatial.bbox('geom',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", r -> assertEquals(node, r.get("node"))); } @Test public void find_geometries_in_a_bounding_box() { - execute("CALL spatial.addPointLayer('geom')"); - Node node = createNode("CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); + Node node = addPointLayerWithNode("geom", 15.2, 60.1); testCall(db, "CALL spatial.bbox('geom',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", r -> assertEquals(node, r.get("node"))); } @@ -1064,8 +1191,7 @@ public void find_geometries_in_a_polygon() { @Test public void find_geometries_in_a_bounding_box_geohash() { - execute("CALL spatial.addPointLayerGeohash('geom')"); - Node node = createNode("CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); + Node node = addPointLayerGeohashWithNode("geom", 15.2, 60.1); testCall(db, "CALL spatial.bbox('geom',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", r -> assertEquals(node, r.get("node"))); } From 023d889ebdb782af3505cad0fbcfcfa88845fb81 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 19 Apr 2021 13:35:40 +0200 Subject: [PATCH 02/11] Initial support for OSM layers in spatial.merge.into(a,b) This required some refactoring of the OSM model with changes to OSMImporter, OSMLayer and OSMDataset. In addition we added OSMModel for collecting general contants and functions appropriate to OSM, and we added OSMMerger with the intelligence to merge two OSM datasets together. This is currently only tested in a trivial case, which is two identical models with one trivial change (added a free floating OSM node). One change to the OSM model is that we moved User nodes to a linked list like we do for Ways and Relations. --- .../org/neo4j/gis/spatial/DefaultLayer.java | 26 +- .../org/neo4j/gis/spatial/osm/OSMDataset.java | 75 ++++- .../gis/spatial/osm/OSMGeometryEncoder.java | 150 ++++----- .../neo4j/gis/spatial/osm/OSMImporter.java | 288 ++++++++++-------- .../org/neo4j/gis/spatial/osm/OSMLayer.java | 44 +-- .../org/neo4j/gis/spatial/osm/OSMMerger.java | 258 ++++++++++++++++ .../org/neo4j/gis/spatial/osm/OSMModel.java | 45 +++ .../neo4j/gis/spatial/osm/OSMRelation.java | 2 +- .../neo4j/gis/spatial/OsmAnalysisTest.java | 24 +- .../procedures/SpatialProceduresTest.java | 101 +++++- 10 files changed, 742 insertions(+), 271 deletions(-) create mode 100644 src/main/java/org/neo4j/gis/spatial/osm/OSMMerger.java create mode 100644 src/main/java/org/neo4j/gis/spatial/osm/OSMModel.java diff --git a/src/main/java/org/neo4j/gis/spatial/DefaultLayer.java b/src/main/java/org/neo4j/gis/spatial/DefaultLayer.java index e95c5fb5a..247084c9e 100644 --- a/src/main/java/org/neo4j/gis/spatial/DefaultLayer.java +++ b/src/main/java/org/neo4j/gis/spatial/DefaultLayer.java @@ -32,7 +32,6 @@ import org.neo4j.gis.spatial.rtree.Listener; import org.neo4j.gis.spatial.rtree.filter.SearchFilter; import org.neo4j.gis.spatial.utilities.GeotoolsAdapter; -import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Transaction; import org.opengis.referencing.crs.CoordinateReferenceSystem; @@ -51,8 +50,12 @@ * on the SpatialDatabaseService for correct initialization. */ public class DefaultLayer implements Constants, Layer, SpatialDataset { - - // Public methods + private String name; + protected Long layerNodeId = -1L; + private GeometryEncoder geometryEncoder; + private GeometryFactory geometryFactory; + protected LayerIndexReader indexReader; + protected SpatialIndexWriter indexWriter; public String getName() { return name; @@ -263,23 +266,6 @@ public void delete(Transaction tx, Listener monitor) { layerNodeId = -1L; } - // Private methods - -// protected GraphDatabaseService getDatabase() { -// return spatialDatabase.getDatabase(); -// } - - - // Attributes - - //private SpatialDatabaseService spatialDatabase; - private String name; - protected Long layerNodeId = -1L; - private GeometryEncoder geometryEncoder; - private GeometryFactory geometryFactory; - protected LayerIndexReader indexReader; - protected SpatialIndexWriter indexWriter; - public SpatialDataset getDataset() { return this; } diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMDataset.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMDataset.java index 5015ed864..5c4d450c4 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMDataset.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMDataset.java @@ -23,10 +23,8 @@ import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.neo4j.gis.spatial.*; -import org.neo4j.graphdb.Direction; -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Relationship; -import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.*; +import org.neo4j.graphdb.schema.IndexDefinition; import org.neo4j.graphdb.traversal.Evaluation; import org.neo4j.graphdb.traversal.Evaluators; import org.neo4j.graphdb.traversal.TraversalDescription; @@ -83,19 +81,19 @@ public Iterable getAllUserNodes(Transaction tx) { TraversalDescription td = new MonoDirectionalTraversalDescription() .depthFirst() .relationships(OSMRelation.USERS, Direction.OUTGOING) - .relationships(OSMRelation.OSM_USER, Direction.OUTGOING) - .evaluator(Evaluators.includeWhereLastRelationshipTypeIs(OSMRelation.OSM_USER)); - return td.traverse(tx.getNodeById(datasetNodeId)).nodes(); + .relationships(OSMRelation.NEXT, Direction.OUTGOING) + .evaluator(Evaluators.excludeStartPosition()); + return td.traverse(getDatasetNode(tx)).nodes(); } public Iterable getAllChangesetNodes(Transaction tx) { TraversalDescription td = new MonoDirectionalTraversalDescription() .depthFirst() .relationships(OSMRelation.USERS, Direction.OUTGOING) - .relationships(OSMRelation.OSM_USER, Direction.OUTGOING) + .relationships(OSMRelation.NEXT, Direction.OUTGOING) .relationships(OSMRelation.USER, Direction.INCOMING) .evaluator(Evaluators.includeWhereLastRelationshipTypeIs(OSMRelation.USER)); - return td.traverse(tx.getNodeById(datasetNodeId)).nodes(); + return td.traverse(getDatasetNode(tx)).nodes(); } public Iterable getAllWayNodes(Transaction tx) { @@ -104,7 +102,7 @@ public Iterable getAllWayNodes(Transaction tx) { .relationships(OSMRelation.WAYS, Direction.OUTGOING) .relationships(OSMRelation.NEXT, Direction.OUTGOING) .evaluator(Evaluators.excludeStartPosition()); - return td.traverse(tx.getNodeById(datasetNodeId)).nodes(); + return td.traverse(getDatasetNode(tx)).nodes(); } public Iterable getAllPointNodes(Transaction tx) { @@ -115,7 +113,7 @@ public Iterable getAllPointNodes(Transaction tx) { .relationships(OSMRelation.FIRST_NODE, Direction.OUTGOING) .relationships(OSMRelation.NODE, Direction.OUTGOING) .evaluator(Evaluators.includeWhereLastRelationshipTypeIs(OSMRelation.NODE)); - return td.traverse(tx.getNodeById(datasetNodeId)).nodes(); + return td.traverse(getDatasetNode(tx)).nodes(); } public Iterable getWayNodes(Node way) { @@ -148,6 +146,29 @@ public Node getUser(Node nodeWayOrChangeset) { return results.hasNext() ? results.next() : null; } + public Node getDatasetNode(Transaction tx) { + return tx.getNodeById(datasetNodeId); + } + + public Node getFirstNodeInChain(Transaction tx, RelationshipType relType) { + Relationship chain = getDatasetNode(tx).getSingleRelationship(relType, Direction.OUTGOING); + return chain == null ? null : chain.getEndNode(); + } + + public Node getLastNodeInChain(Transaction tx, RelationshipType relType) { + Relationship chain = getDatasetNode(tx).getSingleRelationship(relType, Direction.OUTGOING); + if (chain == null) { + return null; + } else { + Node last; + do { + last = chain.getEndNode(); + chain = last.getSingleRelationship(OSMRelation.NEXT, Direction.OUTGOING); + } while (chain != null); + return last; + } + } + public Way getWayFromId(Transaction tx, long id) { return getWayFrom(tx.getNodeById(id)); } @@ -354,4 +375,36 @@ public int getChangesetCount(Transaction tx) { public int getUserCount(Transaction tx) { return (Integer) tx.getNodeById(this.datasetNodeId).getProperty("userCount", 0); } + + public String getIndexName(Transaction tx, Label label, String propertyKey) { + Node datasetNode = tx.getNodeById(this.datasetNodeId); + String indexKey = indexKeyFor(label, propertyKey); + return (String) datasetNode.getProperty(indexKey, null); + } + + public IndexDefinition getIndex(Transaction tx, Label label, String propertyKey) { + String indexName = getIndexName(tx, label, propertyKey); + if (indexName == null) { + throw new IllegalArgumentException(String.format("OSM Dataset '%s' does not have an index for label '%s' and property '%s'", this.layer.getName(), label.name(), propertyKey)); + } else { + return tx.schema().getIndexByName(indexName); + } + } + + public static String indexKeyFor(Label label, String propertyKey) { + return String.format("Index:%s:%s", label.name(), propertyKey); + } + + public static String indexNameFor(String layerName, String hashedLabel, String propertyKey) { + return String.format("OSM-%s-%s-%s", layerName, hashedLabel, propertyKey); + } + + public static Label hashedLabelFrom(String indexName) { + String[] fields = indexName.split("-"); + if (fields.length == 4) { + return Label.label(fields[2]); + } else { + throw new IllegalArgumentException(String.format("Index name '%s' is not correctly formatted - cannot extract label hash", indexName)); + } + } } diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java index 622223f8f..b7d8604ec 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java @@ -19,31 +19,22 @@ */ package org.neo4j.gis.spatial.osm; -import static org.neo4j.gis.spatial.utilities.TraverserFactory.createTraverserInBackwardsCompatibleWay; - -import java.text.DateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -import org.neo4j.gis.spatial.rtree.Envelope; +import org.locationtech.jts.algorithm.ConvexHull; +import org.locationtech.jts.geom.*; import org.neo4j.gis.spatial.AbstractGeometryEncoder; import org.neo4j.gis.spatial.SpatialDatabaseException; import org.neo4j.gis.spatial.SpatialDatabaseService; +import org.neo4j.gis.spatial.rtree.Envelope; import org.neo4j.graphdb.*; import org.neo4j.graphdb.traversal.TraversalDescription; - -import org.locationtech.jts.algorithm.ConvexHull; -import org.locationtech.jts.geom.Coordinate; -import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.geom.GeometryFactory; -import org.locationtech.jts.geom.LineString; -import org.locationtech.jts.geom.LinearRing; -import org.locationtech.jts.geom.Polygon; import org.neo4j.kernel.impl.traversal.MonoDirectionalTraversalDescription; +import java.text.DateFormat; +import java.util.*; + +import static org.neo4j.gis.spatial.osm.OSMModel.*; +import static org.neo4j.gis.spatial.utilities.TraverserFactory.createTraverserInBackwardsCompatibleWay; + public class OSMGeometryEncoder extends AbstractGeometryEncoder { private static int decodedCount = 0; @@ -53,7 +44,8 @@ public class OSMGeometryEncoder extends AbstractGeometryEncoder { private static int relationId = 0; private DateFormat dateTimeFormatter; private int vertices; - private int vertexMistmaches = 0; + private int vertexMismatches = 0; + private final HashMap labelHashes = new HashMap<>(); /** * This class allows for OSM to avoid having empty tags nodes when there are @@ -128,7 +120,6 @@ private static Node testIsNode(Entity container) { public Envelope decodeEnvelope(Entity container) { Node geomNode = testIsNode(container); double[] bbox = (double[]) geomNode.getProperty(PROP_BBOX); - // double xmin, double xmax, double ymin, double ymax return new Envelope(bbox[0], bbox[1], bbox[2], bbox[3]); } @@ -136,9 +127,14 @@ public Envelope decodeEnvelope(Entity container) { public void encodeEnvelope(Envelope mbb, Entity container) { container.setProperty(PROP_BBOX, new double[] { mbb.getMinX(), mbb.getMaxX(), mbb.getMinY(), mbb.getMaxY() }); } - + public static Node getOSMNodeFromGeometryNode(Node geomNode) { - return geomNode.getSingleRelationship(OSMRelation.GEOM, Direction.INCOMING).getStartNode(); + Relationship rel = geomNode.getSingleRelationship(OSMRelation.GEOM, Direction.INCOMING); + if (rel != null) { + return rel.getStartNode(); + } else { + throw new IllegalArgumentException("No geom rel"); + } } public static Node getGeometryNodeFromOSMNode(Node osmNode) { @@ -174,12 +170,7 @@ public void remove() { public Iterable getPointNodesFromWayNode(Node wayNode) { final Node firstNode = wayNode.getSingleRelationship(OSMRelation.FIRST_NODE, Direction.OUTGOING).getEndNode(); final NodeProxyIterator iterator = new NodeProxyIterator(firstNode); - return new Iterable() { - - public Iterator iterator() { - return iterator; - } - }; + return () -> iterator; } public Geometry decodeGeometry(Entity container) { @@ -187,10 +178,11 @@ public Geometry decodeGeometry(Entity container) { try { GeometryFactory geomFactory = layer.getGeometryFactory(); Node osmNode = getOSMNodeFromGeometryNode(geomNode); - if (osmNode.hasProperty("node_osm_id")) { - return geomFactory.createPoint(new Coordinate((Double) osmNode.getProperty("lon", 0.0), (Double) osmNode - .getProperty("lat", 0.0))); - } else if (osmNode.hasProperty("way_osm_id")) { + if (osmNode.hasProperty(PROP_NODE_ID)) { + return geomFactory.createPoint(new Coordinate( + (Double) osmNode.getProperty(PROP_NODE_LON, 0.0), + (Double) osmNode.getProperty(PROP_NODE_LAT, 0.0))); + } else if (osmNode.hasProperty(PROP_WAY_ID)) { int vertices = (Integer) geomNode.getProperty("vertices"); int gtype = (Integer) geomNode.getProperty(PROP_TYPE); return decodeGeometryFromWay(osmNode, gtype, vertices, geomFactory); @@ -207,8 +199,7 @@ private Geometry decodeGeometryFromRelation(Node osmNode, int gtype, GeometryFac switch (gtype) { case GTYPE_POLYGON: LinearRing outer = null; - ArrayList inner = new ArrayList(); - // ArrayList rings = new ArrayList(); + ArrayList inner = new ArrayList<>(); for (Relationship rel : osmNode.getRelationships(Direction.OUTGOING, OSMRelation.MEMBER)) { Node wayNode = rel.getEndNode(); String role = (String) rel.getProperty("role", null); @@ -222,7 +213,7 @@ private Geometry decodeGeometryFromRelation(Node osmNode, int gtype, GeometryFac } } if (outer != null) { - return geomFactory.createPolygon(outer, inner.toArray(new LinearRing[inner.size()])); + return geomFactory.createPolygon(outer, inner.toArray(new LinearRing[0])); } else { return null; } @@ -231,19 +222,19 @@ private Geometry decodeGeometryFromRelation(Node osmNode, int gtype, GeometryFac for (Relationship rel : osmNode.getRelationships(Direction.OUTGOING, OSMRelation.MEMBER)) { Node member = rel.getEndNode(); Geometry geometry = null; - if (member.hasProperty("way_osm_id")) { + if (member.hasProperty(PROP_WAY_ID)) { // decode simple polygons from ways geometry = decodeGeometryFromWay(member, GTYPE_POLYGON, -1, geomFactory); - } else if (!member.hasProperty("node_osm_id")) { + } else if (!member.hasProperty(PROP_NODE_ID)) { // decode polygons with holes from relations geometry = decodeGeometryFromRelation(member, GTYPE_POLYGON, geomFactory); } - if (geometry != null && geometry instanceof Polygon) { + if (geometry instanceof Polygon) { polygons.add((Polygon) geometry); } } if (polygons.size() > 0) { - return geomFactory.createMultiPolygon(polygons.toArray(new Polygon[polygons.size()])); + return geomFactory.createMultiPolygon(polygons.toArray(new Polygon[0])); } else { return null; } @@ -276,10 +267,8 @@ private LinearRing getOuterLinearRingFromGeometry(Geometry geometry) { return getConvexHull(ring); } } - } else if (geometry instanceof LinearRing) { - return (LinearRing) geometry; } else if (geometry instanceof Polygon) { - return (LinearRing) ((Polygon) geometry).getExteriorRing(); + return ((Polygon) geometry).getExteriorRing(); } else { return getConvexHull(geometry); } @@ -287,7 +276,7 @@ private LinearRing getOuterLinearRingFromGeometry(Geometry geometry) { /** * Extend the array by copying the first point into the last position - * + * * @param coords original array that is not closed * @return new array one point longer */ @@ -319,18 +308,18 @@ private Geometry decodeGeometryFromWay(Node wayNode, int gtype, int vertices, Ge overrunCount++; break; } - coordinates.add(new Coordinate((Double) node.getProperty("lon"), (Double) node.getProperty("lat"))); + coordinates.add(new Coordinate((Double) node.getProperty(PROP_NODE_LON), (Double) node.getProperty(OSMModel.PROP_NODE_LAT))); } decodedCount++; if (overrun) { System.out.println("Overran expected number of way nodes: " + wayNode + " (" + overrunCount + "/" + decodedCount + ")"); } if (coordinates.size() != vertices) { - if (vertexMistmaches++ < 10) { + if (vertexMismatches++ < 10) { System.err.println("Mismatching vertices size for " + SpatialDatabaseService.convertGeometryTypeToName(gtype) + ":" + wayNode + ": " + coordinates.size() + " != " + vertices); - } else if (vertexMistmaches % 100 == 0) { - System.err.println("Mismatching vertices found " + vertexMistmaches + " times"); + } else if (vertexMismatches % 100 == 0) { + System.err.println("Mismatching vertices found " + vertexMismatches + " times"); } } switch (coordinates.size()) { @@ -346,7 +335,7 @@ private Geometry decodeGeometryFromWay(Node wayNode, int gtype, int vertices, Ge case GTYPE_POLYGON: return geomFactory.createPolygon(geomFactory.createLinearRing(coords), new LinearRing[0]); default: - return geomFactory.createMultiPoint(coords); + return geomFactory.createMultiPointFromCoords(coords); } } } @@ -391,14 +380,44 @@ private Node makeOSMNode(Transaction tx, Geometry geometry, Node geomNode) { return node; } + private void addLabelHash(Transaction tx, OSMDataset dataset, Label label, String propertyKey) { + String indexName = dataset.getIndexName(tx, label, propertyKey); + if (indexName != null) { + labelHashes.put(label, OSMDataset.hashedLabelFrom(indexName)); + } + } + + private void loadLabelHash(Transaction tx) { + if (labelHashes.isEmpty()) { + OSMDataset dataset = OSMDataset.fromLayer(tx, (OSMLayer) layer); + addLabelHash(tx, dataset, LABEL_NODE, PROP_NODE_ID); + addLabelHash(tx, dataset, LABEL_WAY, PROP_WAY_ID); + addLabelHash(tx, dataset, LABEL_RELATION, PROP_RELATION_ID); + addLabelHash(tx, dataset, LABEL_USER, PROP_USER_ID); + addLabelHash(tx, dataset, LABEL_CHANGESET, PROP_CHANGESET); + } + } + + private Label getLabelHash(Transaction tx, Label label) { + loadLabelHash(tx); + return labelHashes.get(label); + } + private Node makeOSMNode(Transaction tx, Coordinate coordinate) { vertices++; nodeId++; - Node node = tx.createNode(); - // TODO: Generate a valid osm id - node.setProperty(OSMId.NODE.toString(), nodeId); - node.setProperty("lat", coordinate.y); - node.setProperty("lon", coordinate.x); + Node node = tx.createNode(OSMModel.LABEL_NODE); + Label hashed = getLabelHash(tx, LABEL_NODE); + if (hashed != null) { + // This allows this node to be found using the same index that the OSMImporter uses + node.addLabel(hashed); + } + // We need a fake osm-id, but cannot know what positive integers are not used + // So we just set it to a negative of the Neo4j nodeId. + // This is only unsafe for nodeId=0, but that is certain to be used for other nodes than these. + node.setProperty(PROP_NODE_ID, -nodeId); + node.setProperty(PROP_NODE_LAT, coordinate.y); + node.setProperty(PROP_NODE_LON, coordinate.x); node.setProperty("timestamp", getTimestamp()); // TODO: Add other common properties, like changeset, uid, user, version return node; @@ -408,7 +427,7 @@ private Node makeOSMWay(Transaction tx, Geometry geometry, Node geomNode, int gt wayId++; Node way = tx.createNode(); // TODO: Generate a valid osm id - way.setProperty(OSMId.WAY.toString(), wayId); + way.setProperty(PROP_WAY_ID, wayId); way.setProperty("timestamp", getTimestamp()); // TODO: Add other common properties, like changeset, uid, user, // version, name @@ -449,7 +468,7 @@ private String getTimestamp() { private class CombinedAttributes { private Node node; private Entity properties; - private HashMap extra = new HashMap<>(); + private final HashMap extra = new HashMap<>(); CombinedAttributes(Node geomNode) { try { @@ -457,10 +476,10 @@ private class CombinedAttributes { properties = node.getSingleRelationship(OSMRelation.TAGS, Direction.OUTGOING).getEndNode(); Node changeset = node.getSingleRelationship(OSMRelation.CHANGESET, Direction.OUTGOING).getEndNode(); if (changeset != null) { - extra.put("changeset", changeset.getProperty("changeset", null)); + extra.put(PROP_CHANGESET, changeset.getProperty(PROP_CHANGESET, null)); Node user = changeset.getSingleRelationship(OSMRelation.USER, Direction.OUTGOING).getEndNode(); if (user != null) { - extra.put("user", user.getProperty("name", null)); + extra.put(PROP_USER_NAME, user.getProperty("name", null)); extra.put("user_id", user.getProperty("uid", null)); } } @@ -498,7 +517,7 @@ private CombinedAttributes getProperties(Node geomNode) { * This means the default way of storing attributes is simply as properties * of the geometry node. This behaviour can be changed by other domain * models with different encodings. - * + * * @param geomNode node to test * @param name attribute to check for existence of * @return true if node has the specified attribute @@ -513,7 +532,7 @@ public boolean hasAttribute(Node geomNode, String name) { * properties of the geometry node. This behaviour can be changed by other * domain models with different encodings. If the property does not exist, * the method returns null. - * + * * @param geomNode node to test * @param name attribute to access * @return attribute value, or null @@ -521,17 +540,4 @@ public boolean hasAttribute(Node geomNode, String name) { public Object getAttribute(Node geomNode, String name) { return getProperties(geomNode).getProperty(name); } - - public enum OSMId { - NODE("node_osm_id"), WAY("way_osm_id"), RELATION("relation_osm_id"); - private String name; - - OSMId(String name) { - this.name = name; - } - - public String toString() { - return name; - } - } } diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java index e5f7a029c..e539d820e 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java @@ -54,30 +54,15 @@ import java.util.concurrent.TimeUnit; import static java.util.Arrays.asList; +import static org.neo4j.gis.spatial.osm.OSMModel.*; public class OSMImporter implements Constants { public static DefaultEllipsoid WGS84 = DefaultEllipsoid.WGS84; - public static Label LABEL_DATASET = Label.label("OSMDataset"); - public static Label LABEL_LAYER = Label.label("OSMLayer"); - public static Label LABEL_BBOX = Label.label("OSMBBox"); - public static Label LABEL_CHANGESET = Label.label("OSMChangeset"); - public static Label LABEL_USER = Label.label("OSMUser"); - public static Label LABEL_TAGS = Label.label("OSMTags"); - public static Label LABEL_NODE = Label.label("OSMNode"); - public static Label LABEL_WAY = Label.label("OSMWay"); - public static Label LABEL_WAY_NODE = Label.label("OSMWayNode"); - public static Label LABEL_RELATION = Label.label("OSMRelation"); - public static String PROP_BBOX = "bbox"; - public static String PROP_CHANGESET = "changeset"; - public static String PROP_USER_NAME = "user"; - public static String PROP_USER_ID = "uid"; - public static String PROP_NODE_ID = "node_osm_id"; - public static String PROP_WAY_ID = "way_osm_id"; - public static String PROP_RELATION_ID = "relation_osm_id"; protected boolean nodesProcessingFinished = false; private final String layerName; - private final StatsManager stats = new StatsManager(); + private final StatsManager tagStats = new StatsManager(); + private final GeomStats geomStats = new GeomStats(); private long osm_dataset = -1; private long missingChangesets = 0; private final Listener monitor; @@ -131,7 +116,6 @@ public String toString() { private static class StatsManager { private final HashMap tagStats = new HashMap<>(); - private final HashMap geomStats = new HashMap<>(); TagStats getTagStats(String type) { if (!tagStats.containsKey(type)) { @@ -160,6 +144,10 @@ void printTagStats() { System.out.println("\t" + key + ": " + stats); } } + } + + public static class GeomStats { + private final HashMap geomStats = new HashMap<>(); void addGeomStats(Node geomNode) { if (geomNode != null) { @@ -180,7 +168,6 @@ void dumpGeomStats() { } geomStats.clear(); } - } public OSMImporter(String layerName) { @@ -233,50 +220,20 @@ public long reIndex(GraphDatabaseService database, int commitInterval, boolean i tx.commit(); } - TraversalDescription traversal = new MonoDirectionalTraversalDescription(); long startTime = System.currentTimeMillis(); - org.neo4j.graphdb.traversal.TraversalDescription findWays = traversal.depthFirst() - .evaluator(Evaluators.excludeStartPosition()) - .relationships(OSMRelation.WAYS, Direction.OUTGOING) - .relationships(OSMRelation.NEXT, Direction.OUTGOING); - org.neo4j.graphdb.traversal.TraversalDescription findNodes = traversal.depthFirst() - .evaluator(Evaluators.excludeStartPosition()) - .relationships(OSMRelation.FIRST_NODE, Direction.OUTGOING) - .relationships(OSMRelation.NEXT, Direction.OUTGOING); Transaction tx = beginTx(database); boolean useWays = missingChangesets > 0; int count = 0; try { - layer.setExtraPropertyNames(stats.getTagStats("all").getTags(), tx); + OSMIndexer indexer = new OSMIndexer(layer, geomStats, includePoints); + layer.setExtraPropertyNames(tagStats.getTagStats("all").getTags(), tx); if (useWays) { beginProgressMonitor(dataset.getWayCount(tx)); - for (Node way : toList(findWays.traverse(tx.getNodeById(osm_dataset)).nodes())) { + for (Node way : indexer.allWays(tx)) { updateProgressMonitor(count); incrLogContext(); - stats.addGeomStats(layer.addWay(tx, way, true)); - if (includePoints) { - long badProxies = 0; - long goodProxies = 0; - for (Node proxy : findNodes.traverse(way).nodes()) { - Relationship nodeRel = proxy.getSingleRelationship(OSMRelation.NODE, Direction.OUTGOING); - if (nodeRel == null) { - badProxies++; - } else { - goodProxies++; - Node node = proxy.getSingleRelationship(OSMRelation.NODE, Direction.OUTGOING).getEndNode(); - stats.addGeomStats(layer.addWay(tx, node, true)); - } - } - if (badProxies > 0) { - System.out.println("Unexpected dangling proxies for way: " + way); - if (way.hasProperty(PROP_WAY_ID)) { - System.out.println("\tWay: " + way.getProperty(PROP_WAY_ID)); - } - System.out.println("\tBad Proxies: " + badProxies); - System.out.println("\tGood Proxies: " + goodProxies); - } - } + indexer.indexByWay(tx, way); if (++count % commitInterval == 0) { tx.commit(); tx.close(); @@ -285,14 +242,12 @@ public long reIndex(GraphDatabaseService database, int commitInterval, boolean i } // TODO ask charset to user? } else { beginProgressMonitor(dataset.getChangesetCount(tx)); - for (Node unsafeNode : toList(dataset.getAllChangesetNodes(tx))) { + for (Node unsafeNode : indexer.allChangesets(tx)) { WrappedNode changeset = new WrappedNode(unsafeNode); changeset.refresh(tx); updateProgressMonitor(count); incrLogContext(); - for (Relationship rel : changeset.getRelationships(Direction.INCOMING, OSMRelation.CHANGESET)) { - stats.addGeomStats(layer.addWay(tx, rel.getStartNode(), true)); - } + indexer.indexByChangeset(tx, changeset.inner); if (++count % commitInterval == 0) { tx.commit(); tx.close(); @@ -309,19 +264,84 @@ public long reIndex(GraphDatabaseService database, int commitInterval, boolean i if (verboseLog) { long stopTime = System.currentTimeMillis(); log("info | Re-indexing elapsed time in seconds: " + (1.0 * (stopTime - startTime) / 1000.0)); - stats.dumpGeomStats(); + geomStats.dumpGeomStats(); } return count; } - private List toList(Iterable iterable) { - ArrayList list = new ArrayList<>(); - if (iterable != null) { - for (Node e : iterable) { - list.add(e); + public static class OSMIndexer { + private static final TraversalDescription traversal = new MonoDirectionalTraversalDescription(); + private static final org.neo4j.graphdb.traversal.TraversalDescription findWays = traversal.depthFirst() + .evaluator(Evaluators.excludeStartPosition()) + .relationships(OSMRelation.WAYS, Direction.OUTGOING) + .relationships(OSMRelation.NEXT, Direction.OUTGOING); + private static final org.neo4j.graphdb.traversal.TraversalDescription findNodes = traversal.depthFirst() + .evaluator(Evaluators.excludeStartPosition()) + .relationships(OSMRelation.FIRST_NODE, Direction.OUTGOING) + .relationships(OSMRelation.NEXT, Direction.OUTGOING); + private final OSMLayer layer; + private final boolean includePoints; + private final GeomStats stats; + + public OSMIndexer(OSMLayer layer, GeomStats stats, boolean includePoints) { + this.layer = layer; + this.stats = stats; + this.includePoints = includePoints; + } + + public void indexByGeometryNode(Transaction tx, Node geomNode) { + if (!layer.getIndex().isNodeIndexed(tx, geomNode.getId())) { + layer.addGeomNode(tx, geomNode, false); } } - return list; + + public void indexByWay(Transaction tx, Node way) { + stats.addGeomStats(layer.addWay(tx, way, true)); + if (includePoints) { + long badProxies = 0; + long goodProxies = 0; + for (Node proxy : findNodes.traverse(way).nodes()) { + Relationship nodeRel = proxy.getSingleRelationship(OSMRelation.NODE, Direction.OUTGOING); + if (nodeRel == null) { + badProxies++; + } else { + goodProxies++; + Node node = proxy.getSingleRelationship(OSMRelation.NODE, Direction.OUTGOING).getEndNode(); + stats.addGeomStats(layer.addWay(tx, node, true)); + } + } + if (badProxies > 0) { + System.out.println("Unexpected dangling proxies for way: " + way); + if (way.hasProperty(PROP_WAY_ID)) { + System.out.println("\tWay: " + way.getProperty(PROP_WAY_ID)); + } + System.out.println("\tBad Proxies: " + badProxies); + System.out.println("\tGood Proxies: " + goodProxies); + } + } + } + public void indexByChangeset(Transaction tx, Node changeset) { + for (Relationship rel : changeset.getRelationships(Direction.INCOMING, OSMRelation.CHANGESET)) { + stats.addGeomStats(layer.addWay(tx, rel.getStartNode(), true)); + } + } + public List allWays(Transaction tx) { + OSMDataset dataset = OSMDataset.fromLayer(tx, layer); + return toList(dataset.getAllWayNodes(tx)); + } + public List allChangesets(Transaction tx) { + OSMDataset dataset = OSMDataset.fromLayer(tx, layer); + return toList(dataset.getAllChangesetNodes(tx)); + } + private List toList(Iterable iterable) { + ArrayList list = new ArrayList<>(); + if (iterable != null) { + for (Node e : iterable) { + list.add(e); + } + } + return list; + } } private static class GeometryMetaData { @@ -384,18 +404,20 @@ private Envelope getBBox() { private static abstract class OSMWriter { private static final int UNKNOWN_CHANGESET = -1; - StatsManager statsManager; + StatsManager tagStats; + GeomStats geomStats; OSMImporter osmImporter; T osm_dataset; long missingChangesets = 0; - private OSMWriter(StatsManager statsManager, OSMImporter osmImporter) { - this.statsManager = statsManager; + private OSMWriter(StatsManager tagStats, GeomStats geomStats, OSMImporter osmImporter) { + this.tagStats = tagStats; + this.geomStats = geomStats; this.osmImporter = osmImporter; } - static OSMWriter fromGraphDatabase(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager stats, OSMImporter osmImporter, int txInterval) throws NoSuchAlgorithmException { - return new OSMGraphWriter(graphDb, securityContext, stats, osmImporter, txInterval); + static OSMWriter fromGraphDatabase(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager tagStats, GeomStats geomStats, OSMImporter osmImporter, int txInterval) throws NoSuchAlgorithmException { + return new OSMGraphWriter(graphDb, securityContext, tagStats, geomStats, osmImporter, txInterval); } protected abstract void startWays(); @@ -540,9 +562,9 @@ private void missingMember(String description) { } } - T currentNode = null; T prev_way = null; T prev_relation = null; + T prev_user = null; int nodeCount = 0; int poiCount = 0; int wayCount = 0; @@ -559,34 +581,25 @@ void addOSMBBox(Map bboxProperties) { } /** - * Create a new OSM node from the specified attributes (including - * location, user, changeset). The node is stored in the currentNode - * field, so that it can be used in the subsequent call to - * addOSMNodeTags after we close the XML tag for OSM nodes. - * - * @param nodeProps HashMap of attributes for the OSM-node + * Create a new OSM node from the specified attributes (including location, user, changeset). */ - void createOSMNode(Map nodeProps) { - T userNode = getUserNode(nodeProps); - T changesetNode = getChangesetNode(nodeProps, userNode); - currentNode = addNode(LABEL_NODE, nodeProps, PROP_NODE_ID); - createRelationship(currentNode, changesetNode, OSMRelation.CHANGESET); + private void createOSMNode(Map nodeProperties, boolean allPoints, LinkedHashMap currentNodeTags) { + T userNode = getUserNode(nodeProperties); + T changesetNode = getChangesetNode(nodeProperties, userNode); + T node = addNode(LABEL_NODE, nodeProperties, PROP_NODE_ID); + createRelationship(node, changesetNode, OSMRelation.CHANGESET); nodeCount++; - } - - private void addOSMNodeTags(boolean allPoints, - LinkedHashMap currentNodeTags) { currentNodeTags.remove("created_by"); // redundant information // Nodes with tags get added to the index as point geometries if (allPoints || currentNodeTags.size() > 0) { - Map nodeProps = getNodeProperties(currentNode); + Map nodeProps = getNodeProperties(node); double[] location = new double[]{ - (Double) nodeProps.get("lon"), - (Double) nodeProps.get("lat")}; - addNodeGeometry(currentNode, GTYPE_POINT, new Envelope(location), 1); + (Double) nodeProps.get(PROP_NODE_LON), + (Double) nodeProps.get(PROP_NODE_LAT)}; + addNodeGeometry(node, GTYPE_POINT, new Envelope(location), 1); poiCount++; } - addNodeTags(currentNode, currentNodeTags, "node"); + addNodeTags(node, currentNodeTags, "node"); } protected void debugNodeWithId(T node, String idName, long[] idValues) { @@ -599,8 +612,7 @@ protected void debugNodeWithId(T node, String idName, long[] idValues) { } } - protected void createOSMWay(Map wayProperties, - ArrayList wayNodes, LinkedHashMap wayTags) { + protected void createOSMWay(Map wayProperties, ArrayList wayNodes, LinkedHashMap wayTags) { RoadDirection direction = getRoadDirection(wayTags); String name = (String) wayTags.get("name"); int geometry = GTYPE_LINESTRING; @@ -654,8 +666,8 @@ protected void createOSMWay(Map wayProperties, createRelationship(proxyNode, pointNode, OSMRelation.NODE, null); Map nodeProps = getNodeProperties(pointNode); double[] location = new double[]{ - (Double) nodeProps.get("lon"), - (Double) nodeProps.get("lat")}; + (Double) nodeProps.get(PROP_NODE_LON), + (Double) nodeProps.get(PROP_NODE_LAT)}; if (bbox == null) { bbox = new Envelope(location); } else { @@ -665,7 +677,7 @@ protected void createOSMWay(Map wayProperties, createRelationship(way, proxyNode, OSMRelation.FIRST_NODE); } else { relProps.clear(); - double[] prevLoc = new double[]{(Double) prevProps.get("lon"), (Double) prevProps.get("lat")}; + double[] prevLoc = new double[]{(Double) prevProps.get(PROP_NODE_LON), (Double) prevProps.get(PROP_NODE_LAT)}; double length = distance(prevLoc[0], prevLoc[1], location[0], location[1]); relProps.put("length", length); /* @@ -742,7 +754,7 @@ private void createOSMRelation(Map relationProperties, } Map nodeProps = getNodeProperties(member); if (memberType.equals("node")) { - double[] location = new double[]{(Double) nodeProps.get("lon"), (Double) nodeProps.get("lat")}; + double[] location = new double[]{(Double) nodeProps.get(PROP_NODE_LON), (Double) nodeProps.get(PROP_NODE_LAT)}; metaGeom.expandToIncludePoint(location); } else if (memberType.equals("nodes")) { System.err.println("Unexpected 'nodes' member type"); @@ -764,8 +776,7 @@ private void createOSMRelation(Map relationProperties, } } if (metaGeom.isValid()) { - addNodeGeometry(relation, metaGeom.getGeometryType(), - metaGeom.getBBox(), metaGeom.getVertices()); + addNodeGeometry(relation, metaGeom.getGeometryType(), metaGeom.getBBox(), metaGeom.getVertices()); } this.relationCount++; } @@ -783,8 +794,7 @@ private void createOSMRelation(Map relationProperties, protected abstract T getOSMNode(long osmId, T changesetNode); - protected abstract void updateGeometryMetaDataFromMember(T member, - GeometryMetaData metaGeom, Map nodeProps); + protected abstract void updateGeometryMetaDataFromMember(T member, GeometryMetaData metaGeom, Map nodeProps); protected abstract void finish(); @@ -872,8 +882,8 @@ private static class OSMGraphWriter extends OSMWriter { private final String layerHash; private final HashMap hashedLabels = new HashMap<>(); - private OSMGraphWriter(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager statsManager, OSMImporter osmImporter, int txInterval) throws NoSuchAlgorithmException { - super(statsManager, osmImporter); + private OSMGraphWriter(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager tagsStats, GeomStats geomStats, OSMImporter osmImporter, int txInterval) throws NoSuchAlgorithmException { + super(tagsStats, geomStats, osmImporter); this.graphDb = graphDb; this.securityContext = securityContext; this.txInterval = txInterval; @@ -919,7 +929,7 @@ private static Transaction beginTx(GraphDatabaseService database, SecurityContex private void beginTx() { tx = beginTx(graphDb); recoverNode(osm_dataset); - recoverNode(currentNode); + recoverNode(prev_user); recoverNode(prev_relation); recoverNode(prev_way); recoverNode(currentChangesetNode); @@ -1003,7 +1013,7 @@ private Node findNodeByLabelProperty(Transaction tx, Label label, String propert private IndexDefinition createIndex(Label label, String propertyKey) { Label hashed = getLabelHashed(label); - String indexName = String.format("OSM-%s-%s-%s", osmImporter.layerName, hashed.name(), propertyKey); + String indexName = OSMDataset.indexNameFor(osmImporter.layerName, hashed.name(), propertyKey); IndexDefinition index = findIndex(tx, indexName, hashed, propertyKey); if (index == null) { successTx(); @@ -1013,10 +1023,23 @@ private IndexDefinition createIndex(Label label, String propertyKey) { } System.out.println("Created index " + index.getName()); beginTx(); + saveIndexName(label, propertyKey, indexName); } return index; } + private void saveIndexName(Label label, String propertyKey, String indexName) { + String indexKey = OSMDataset.indexKeyFor(label, propertyKey); + String previousIndex = (String) osm_dataset.getProperty(indexKey, null); + if (previousIndex == null) { + osm_dataset.setProperty(indexKey, indexName); + } else if (previousIndex.equals(indexName)) { + System.out.println(String.format("OSMLayer '%s' already has matching index definition for '%s': %s", osm_dataset.getProperty("name", ""), indexKey, previousIndex)); + } else { + throw new IllegalStateException(String.format("OSMLayer '%s' already has index definition for '%s': %s")); + } + } + private IndexDefinition createIndexIfNotNull(IndexDefinition index, Label label, String propertyKey) { if (index == null) { index = createIndex(label, propertyKey); @@ -1076,7 +1099,7 @@ private void addProperties(Entity node, Map properties) { protected void addNodeTags(WrappedNode node, LinkedHashMap tags, String type) { logNodeAddition(tags, type); if (node != null && tags.size() > 0) { - statsManager.addToTagStats(type, tags.keySet()); + tagStats.addToTagStats(type, tags.keySet()); WrappedNode tagsNode = createNodeWithLabel(tx, LABEL_TAGS); addProperties(tagsNode.inner, tags); node.createRelationshipTo(tagsNode, OSMRelation.TAGS); @@ -1088,12 +1111,12 @@ protected void addNodeTags(WrappedNode node, LinkedHashMap tags, protected void addNodeGeometry(WrappedNode node, int gtype, Envelope bbox, int vertices) { if (node != null && bbox != null && vertices > 0) { if (gtype == GTYPE_GEOMETRY) gtype = vertices > 1 ? GTYPE_MULTIPOINT : GTYPE_POINT; - Node geomNode = tx.createNode(); + Node geomNode = tx.createNode(LABEL_GEOM); geomNode.setProperty("gtype", gtype); geomNode.setProperty("vertices", vertices); geomNode.setProperty(PROP_BBOX, new double[]{bbox.getMinX(), bbox.getMaxX(), bbox.getMinY(), bbox.getMaxY()}); node.createRelationshipTo(geomNode, OSMRelation.GEOM); - statsManager.addGeomStats(gtype); + geomStats.addGeomStats(gtype); } } @@ -1202,7 +1225,7 @@ protected WrappedNode getChangesetNode(Map nodeProps, WrappedNod } else { LinkedHashMap changesetProps = new LinkedHashMap<>(); changesetProps.put(PROP_CHANGESET, currentChangesetId); - changesetProps.put("timestamp", nodeProps.get("timestamp")); + changesetProps.put(PROP_TIMESTAMP, nodeProps.get(PROP_TIMESTAMP)); currentChangesetNode = addNode(LABEL_CHANGESET, changesetProps, PROP_CHANGESET); changesetCount++; if (userNode != null) { @@ -1232,15 +1255,16 @@ protected WrappedNode getUserNode(Map nodeProps) { } else { LinkedHashMap userProps = new LinkedHashMap<>(); userProps.put(PROP_USER_ID, currentUserId); - userProps.put("name", name); - userProps.put("timestamp", nodeProps.get("timestamp")); + userProps.put(PROP_USER_NAME, name); + userProps.put(PROP_TIMESTAMP, nodeProps.get(PROP_TIMESTAMP)); currentUserNode = addNode(LABEL_USER, userProps, PROP_USER_ID); userCount++; - if (usersNode == null) { - usersNode = createNodeWithLabel(tx, LABEL_USER); - osm_dataset.createRelationshipTo(usersNode, OSMRelation.USERS); + if (prev_user == null) { + createRelationship(osm_dataset, currentUserNode, OSMRelation.USERS); + } else { + createRelationship(prev_user, currentUserNode, OSMRelation.NEXT); } - usersNode.createRelationshipTo(currentUserNode, OSMRelation.OSM_USER); + prev_user = currentUserNode; } } } catch (Exception e) { @@ -1266,7 +1290,7 @@ public void importFile(GraphDatabaseService database, String dataset, int txInte } public void importFile(GraphDatabaseService database, String dataset, boolean allPoints, int txInterval) throws Exception { - importFile(OSMWriter.fromGraphDatabase(database, securityContext, stats, this, txInterval), dataset, allPoints, charset); + importFile(OSMWriter.fromGraphDatabase(database, securityContext, tagStats, geomStats, this, txInterval), dataset, allPoints, charset); } public static class CountedFileReader extends InputStreamReader { @@ -1354,6 +1378,7 @@ public void importFile(OSMWriter osmWriter, String dataset, boolean allPoints try { ArrayList currentXMLTags = new ArrayList<>(); int depth = 0; + Map nodeProperties = null; Map wayProperties = null; ArrayList wayNodes = new ArrayList<>(); Map relationProperties = null; @@ -1376,14 +1401,7 @@ public void importFile(OSMWriter osmWriter, String dataset, boolean allPoints osmWriter.addOSMBBox(extractProperties(PROP_BBOX, parser)); } else if (tagPath.equals("[osm, node]")) { /* */ - boolean includeNode = true; - Map nodeProperties = extractProperties("node", parser); - if (filterEnvelope != null) { - includeNode = filterEnvelope.contains((Double) nodeProperties.get("lon"), (Double) nodeProperties.get("lat")); - } - if (includeNode) { - osmWriter.createOSMNode(nodeProperties); - } + nodeProperties = extractProperties("node", parser); } else if (tagPath.equals("[osm, way]")) { /* */ if (!startedWays) { @@ -1437,7 +1455,9 @@ public void importFile(OSMWriter osmWriter, String dataset, boolean allPoints case javax.xml.stream.XMLStreamConstants.END_ELEMENT: switch (currentXMLTags.toString()) { case "[osm, node]": - osmWriter.addOSMNodeTags(allPoints, currentNodeTags); + if (nodeFilterMatches(nodeProperties)) { + osmWriter.createOSMNode(nodeProperties, allPoints, currentNodeTags); + } break; case "[osm, way]": osmWriter.createOSMWay(wayProperties, wayNodes, currentNodeTags); @@ -1467,8 +1487,18 @@ public void importFile(OSMWriter osmWriter, String dataset, boolean allPoints long stopTime = System.currentTimeMillis(); log("info | Elapsed time in seconds: " + (1.0 * (stopTime - startTime) / 1000.0)); - stats.dumpGeomStats(); - stats.printTagStats(); + geomStats.dumpGeomStats(); + tagStats.printTagStats(); + } + } + + private boolean nodeFilterMatches(Map nodeProperties) { + if (filterEnvelope == null) { + return true; + } else { + Double x = (Double) nodeProperties.get(PROP_NODE_LON); + Double y = (Double) nodeProperties.get(PROP_NODE_LAT); + return x != null && y != null && filterEnvelope.contains(x, y); } } @@ -1500,7 +1530,7 @@ private Map extractProperties(String name, XMLStreamReader parse prop = name + "_osm_id"; name = null; } - if (prop.equals("lat") || prop.equals("lon")) { + if (prop.equals(PROP_NODE_LAT) || prop.equals(PROP_NODE_LON)) { properties.put(prop, Double.parseDouble(value)); } else if (name != null && prop.equals("version")) { properties.put(prop, Integer.parseInt(value)); @@ -1508,7 +1538,7 @@ private Map extractProperties(String name, XMLStreamReader parse if (!value.equals("true") && !value.equals("1")) { properties.put(prop, false); } - } else if (prop.equals("timestamp")) { + } else if (prop.equals(PROP_TIMESTAMP)) { try { Date timestamp = timestampFormat.parse(value); properties.put(prop, timestamp.getTime()); diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java index ff3842fdf..910d6ad55 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java @@ -30,6 +30,8 @@ import org.neo4j.graphdb.*; import org.opengis.referencing.crs.CoordinateReferenceSystem; +import javax.management.relation.Relation; + /** * Instances of this class represent the primary layer of the OSM Dataset. It * extends the DynamicLayer class because the OSM dataset can have many layers. @@ -79,28 +81,35 @@ public Node addWay(Transaction tx, Node way) { public Node addWay(Transaction tx, Node way, boolean verifyGeom) { Relationship geomRel = way.getSingleRelationship(OSMRelation.GEOM, Direction.OUTGOING); if (geomRel != null) { - Node geomNode = geomRel.getEndNode(); - try { - // This is a test of the validity of the geometry, throws exception on error - if (verifyGeom) - getGeometryEncoder().decodeGeometry(geomNode); - indexWriter.add(tx, geomNode); - } catch (Exception e) { - System.err.println("Failed geometry test on node " + geomNode.getProperty("name", geomNode.toString()) + ": " - + e.getMessage()); - for (String key : geomNode.getPropertyKeys()) { - System.err.println("\t" + key + ": " + geomNode.getProperty(key)); - } + return addGeomNode(tx, geomRel.getEndNode(), verifyGeom); + } else { + return null; + } + } + + public Node addGeomNode(Transaction tx, Node geomNode, boolean verifyGeom) { + try { + // This is a test of the validity of the geometry, throws exception on error + if (verifyGeom) + getGeometryEncoder().decodeGeometry(geomNode); + indexWriter.add(tx, geomNode); + } catch (Exception e) { + System.err.printf("Failed geometry test on node '%s': %s%n", geomNode.getProperty("name", geomNode.toString()), e.getMessage()); + for (String key : geomNode.getPropertyKeys()) { + System.err.println("\t" + key + ": " + geomNode.getProperty(key)); + } + Relationship geomRel = geomNode.getSingleRelationship(OSMRelation.GEOM, Direction.INCOMING); + if (geomRel != null) { + Node way = geomRel.getStartNode(); System.err.println("For way node " + way); for (String key : way.getPropertyKeys()) { System.err.println("\t" + key + ": " + way.getProperty(key)); } - // e.printStackTrace(System.err); + } else { + System.err.printf("Geometry node %d has no connected OSM model node%n", geomNode.getId()); } - return geomNode; - } else { - return null; } + return geomNode; } /** @@ -266,7 +275,8 @@ public File getStyle() { @Override public long mergeFrom(Transaction tx, EditableLayer other) { if (other instanceof OSMLayer) { - throw new UnsupportedOperationException("Not implemented"); + OSMMerger merger = new OSMMerger(this); + return merger.merge(tx, (OSMLayer) other); } else { throw new IllegalArgumentException("Cannot merge non-OSM layer into OSM layer: '" + other.getName() + "' is not OSM"); } diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMMerger.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMMerger.java new file mode 100644 index 000000000..f65c779d5 --- /dev/null +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMMerger.java @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2010-2020 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j Spatial. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gis.spatial.osm; + +import org.neo4j.gis.spatial.rtree.Envelope; +import org.neo4j.graphdb.*; +import org.neo4j.graphdb.schema.IndexDefinition; + +import java.util.ArrayList; + +import static org.neo4j.gis.spatial.osm.OSMModel.*; + +public class OSMMerger { + private final OSMLayer layer; + + public OSMMerger(OSMLayer layer) { + this.layer = layer; + } + + private boolean checkNodeMoved(Node thisNode, long node_osm_id, String expectedPropKey, String nodePropKey, String keyDescription, Object value) { + if (expectedPropKey.equals(nodePropKey)) { + double thatValue = (Double) value; + double thisValue = (Double) thisNode.getProperty(nodePropKey); + if (Math.abs(thisValue - thatValue) > 0.00000001) { + System.out.printf("Node '%d' has moved %s from %f to %f%n", node_osm_id, keyDescription, thisValue, thatValue); + return true; + } + } + return false; + } + + private static final long INVALID_OSM_ID = Long.MIN_VALUE; + + private long getOSMId(String propertyKey, Node node) { + Object result = node.getProperty(propertyKey, null); + if (result == null) { + return INVALID_OSM_ID; + } else if (result instanceof Long) { + return (Long) result; + } else if (result instanceof Integer) { + return (Integer) result; + } else { + System.out.printf("Invalid type found for property '%s': %s%n", propertyKey, result); + return INVALID_OSM_ID; + } + } + + public long merge(Transaction tx, OSMLayer otherLayer) { + OSMDataset dataset = OSMDataset.fromLayer(tx, layer); + OSMDataset other = OSMDataset.fromLayer(tx, otherLayer); + otherLayer.clear(tx); + + // Merge all OSM nodes + MergeStats nodesStats = mergeNodes(tx, dataset.getIndex(tx, LABEL_NODE, PROP_NODE_ID), other.getIndex(tx, LABEL_NODE, PROP_NODE_ID)); + + // Merge all OSM Ways + MergeStats waysStats = mergeWays(tx, dataset, dataset.getIndex(tx, LABEL_WAY, PROP_WAY_ID), other.getIndex(tx, LABEL_WAY, PROP_WAY_ID)); + + // Reconnect OSM ways, changesets and users chain to main layer + reconnectChains(tx, dataset, other, OSMRelation.RELATIONS); + reconnectChains(tx, dataset, other, OSMRelation.USERS); + + // TODO: changesets + + // Reindex if necessary + if (nodesStats.needsReindexing() || waysStats.needsReindexing()) { + this.layer.clear(tx); + OSMImporter.GeomStats stats = new OSMImporter.GeomStats(); + OSMImporter.OSMIndexer indexer = new OSMImporter.OSMIndexer(layer, stats, false); + for (Node geomNode : nodesStats.geomNodesToAdd) { + indexer.indexByGeometryNode(tx, geomNode); + } + for (Node way : indexer.allWays(tx)) { + indexer.indexByWay(tx, way); + } + stats.dumpGeomStats(); + } + return nodesStats.changed() + waysStats.changed(); + } + + private void reconnectChains(Transaction tx, OSMDataset dataset, OSMDataset other, RelationshipType relType) { + Node lastNode = dataset.getLastNodeInChain(tx, relType); + Node nextNode = other.getFirstNodeInChain(tx, relType); + nextNode.getSingleRelationship(relType, Direction.INCOMING).delete(); + lastNode.createRelationshipTo(nextNode, OSMRelation.NEXT); + } + + private static class MergeStats { + String name; + long countMoved = 0; + long countReplaced = 0; + long countAdded = 0; + ArrayList geomNodesToAdd = new ArrayList<>(); + + MergeStats(String name) { + this.name = name; + } + + boolean needsReindexing() { + return countMoved > 0 || countAdded > 0; + } + + long changed() { + return countMoved + countAdded; + } + + void printStats() { + if (countReplaced > 0) { + System.out.printf("During merge we found %d existing %s which were replaced%n", countReplaced, name); + } + if (countMoved > 0) { + System.out.printf("During merge %d out of %d existing %s were moved - re-indexing required%n", countMoved, countReplaced, name); + } + if (countAdded > 0) { + System.out.printf("During merge %d %s were added - re-indexing required%n", countAdded, name); + } + if (geomNodesToAdd.size() > 0) { + System.out.printf("During merge %d point geometry nodes were identified%n", geomNodesToAdd.size(), name); + } + } + } + + private MergeStats mergeNodes(Transaction tx, IndexDefinition thisIndex, IndexDefinition thatIndex) { + Label thisLabelHash = OSMDataset.hashedLabelFrom(thisIndex.getName()); + Label thatLabelHash = OSMDataset.hashedLabelFrom(thatIndex.getName()); + ResourceIterator nodes = tx.findNodes(thatLabelHash); + MergeStats stats = new MergeStats("nodes"); + while (nodes.hasNext()) { + Node node = nodes.next(); + long node_osm_id = getOSMId(PROP_NODE_ID, node); + if (node_osm_id != INVALID_OSM_ID) { + Node thisNode = tx.findNode(thisLabelHash, PROP_NODE_ID, node_osm_id); + if (thisNode != null) { + // TODO: Consider comparing 'timestamp' field and always keeping the newer properties instead + stats.countReplaced++; + boolean moved = false; + for (String nodePropKey : node.getPropertyKeys()) { + Object value = node.getProperty(nodePropKey); + moved = moved || checkNodeMoved(thisNode, node_osm_id, PROP_NODE_LON, nodePropKey, "longitude", value); + moved = moved || checkNodeMoved(thisNode, node_osm_id, PROP_NODE_LAT, nodePropKey, "latitude", value); + thisNode.setProperty(nodePropKey, value); + } + if (moved) { + stats.countMoved++; + } + Relationship geomRel = thisNode.getSingleRelationship(OSMRelation.GEOM, Direction.OUTGOING); + if (geomRel != null) { + Node geomNode = geomRel.getEndNode(); + stats.geomNodesToAdd.add(geomNode); + double x = (Double) node.getProperty(PROP_NODE_LON); + double y = (Double) node.getProperty(PROP_NODE_LAT); + geomNode.setProperty(PROP_BBOX, new double[]{x, x, y, y}); + } + } else { + stats.countAdded++; + node.addLabel(thisLabelHash); + node.removeLabel(thatLabelHash); + Relationship geomRel = node.getSingleRelationship(OSMRelation.GEOM, Direction.OUTGOING); + if (geomRel != null) { + Node geomNode = geomRel.getEndNode(); + stats.geomNodesToAdd.add(geomNode); + } + } + } else { + System.out.println("Unexpectedly found OSM node without property: " + PROP_NODE_ID); + } + } + stats.printStats(); + return stats; + } + + private MergeStats mergeWays(Transaction tx, OSMDataset dataset, IndexDefinition thisIndex, IndexDefinition thatIndex) { + Label thisLabelHash = OSMDataset.hashedLabelFrom(thisIndex.getName()); + Label thatLabelHash = OSMDataset.hashedLabelFrom(thatIndex.getName()); + Node lastWayInChain = dataset.getLastNodeInChain(tx, OSMRelation.WAYS); + ResourceIterator nodes = tx.findNodes(thatLabelHash); + MergeStats stats = new MergeStats("ways"); + OSMGeometryEncoder geometryEncoder = (OSMGeometryEncoder) layer.getGeometryEncoder(); + while (nodes.hasNext()) { + Node thatWay = nodes.next(); + long way_osm_id = getOSMId(PROP_WAY_ID, thatWay); + Node thatGeom = thatWay.getSingleRelationship(OSMRelation.GEOM, Direction.OUTGOING).getEndNode(); + Envelope thatEnvelope = geometryEncoder.decodeEnvelope(thatGeom); + if (way_osm_id != INVALID_OSM_ID) { + Node thisWay = tx.findNode(thisLabelHash, PROP_WAY_ID, way_osm_id); + if (thisWay != null) { + // TODO: consider getting 'timestamp' and always saving the newest data + stats.countReplaced++; + for (String nodePropKey : thatWay.getPropertyKeys()) { + Object value = thatWay.getProperty(nodePropKey); + thisWay.setProperty(nodePropKey, value); + } + Node thisGeom = thisWay.getSingleRelationship(OSMRelation.GEOM, Direction.OUTGOING).getEndNode(); + Envelope thisEnvelope = geometryEncoder.decodeEnvelope(thisGeom); + if (!thisEnvelope.equals(thatEnvelope)) { + stats.countMoved++; + // TODO: merge the two ways by finding overlapping nodes + } + } else { + stats.countAdded++; + extractWayFromChain(thatWay); + thatWay.addLabel(thisLabelHash); + thatWay.removeLabel(thatLabelHash); + if (lastWayInChain == null) { + dataset.getDatasetNode(tx).createRelationshipTo(thatWay, OSMRelation.WAYS); + } else { + lastWayInChain.createRelationshipTo(thatWay, OSMRelation.NEXT); + } + lastWayInChain = thatWay; + } + } else { + System.out.println("Unexpectedly found OSM way without property: " + PROP_WAY_ID); + } + } + stats.printStats(); + return stats; + } + + private void extractWayFromChain(Node way) { + if (!extractWayFromChain(way, OSMRelation.WAYS)) { + extractWayFromChain(way, OSMRelation.NEXT); + } + } + + private boolean extractWayFromChain(Node way, RelationshipType incomingType) { + Relationship incomingWays = way.getSingleRelationship(incomingType, Direction.INCOMING); + if (incomingWays != null) { + Node previous = incomingWays.getStartNode(); + incomingWays.delete(); + Relationship outgoingWays = way.getSingleRelationship(OSMRelation.NEXT, Direction.OUTGOING); + if (outgoingWays != null) { + Node next = outgoingWays.getEndNode(); + outgoingWays.delete(); + previous.createRelationshipTo(next, incomingType); + } + return true; + } else { + return false; + } + } +} diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMModel.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMModel.java new file mode 100644 index 000000000..bdf9dc460 --- /dev/null +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMModel.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2010-2020 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j Spatial. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gis.spatial.osm; + +import org.neo4j.graphdb.Label; + +public class OSMModel { + public static Label LABEL_DATASET = Label.label("OSMDataset"); + public static Label LABEL_BBOX = Label.label("OSMBBox"); + public static Label LABEL_CHANGESET = Label.label("OSMChangeset"); + public static Label LABEL_USER = Label.label("OSMUser"); + public static Label LABEL_TAGS = Label.label("OSMTags"); + public static Label LABEL_NODE = Label.label("OSMNode"); + public static Label LABEL_GEOM = Label.label("OSMGeometry"); + public static Label LABEL_WAY = Label.label("OSMWay"); + public static Label LABEL_WAY_NODE = Label.label("OSMWayNode"); + public static Label LABEL_RELATION = Label.label("OSMRelation"); + public static String PROP_BBOX = "bbox"; + public static String PROP_TIMESTAMP = "timestamp"; + public static String PROP_CHANGESET = "changeset"; + public static String PROP_USER_NAME = "user"; + public static String PROP_USER_ID = "uid"; + public static String PROP_NODE_ID = "node_osm_id"; + public static String PROP_NODE_LON = "lon"; + public static String PROP_NODE_LAT = "lat"; + public static String PROP_WAY_ID = "way_osm_id"; + public static String PROP_RELATION_ID = "relation_osm_id"; +} diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMRelation.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMRelation.java index e12e79edf..3c388030c 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMRelation.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMRelation.java @@ -35,5 +35,5 @@ import org.neo4j.graphdb.RelationshipType; public enum OSMRelation implements RelationshipType { - FIRST_NODE, LAST_NODE, OTHER, NEXT, OSM, WAYS, RELATIONS, MEMBERS, MEMBER, TAGS, GEOM, BBOX, NODE, CHANGESET, USER, USERS, OSM_USER; + FIRST_NODE, LAST_NODE, OTHER, NEXT, OSM, WAYS, RELATIONS, MEMBERS, MEMBER, TAGS, GEOM, BBOX, NODE, CHANGESET, USER, USERS; } \ No newline at end of file diff --git a/src/test/java/org/neo4j/gis/spatial/OsmAnalysisTest.java b/src/test/java/org/neo4j/gis/spatial/OsmAnalysisTest.java index cbee1a9c8..d32d6d563 100644 --- a/src/test/java/org/neo4j/gis/spatial/OsmAnalysisTest.java +++ b/src/test/java/org/neo4j/gis/spatial/OsmAnalysisTest.java @@ -19,22 +19,21 @@ */ package org.neo4j.gis.spatial; -import org.locationtech.jts.geom.Coordinate; import org.apache.commons.io.FileUtils; import org.geotools.data.neo4j.StyledImageExporter; import org.geotools.geometry.jts.ReferencedEnvelope; import org.junit.Test; import org.junit.runners.Parameterized; +import org.locationtech.jts.geom.Coordinate; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gis.spatial.filter.SearchRecords; import org.neo4j.gis.spatial.index.IndexManager; import org.neo4j.gis.spatial.osm.OSMDataset; -import org.neo4j.gis.spatial.osm.OSMImporter; import org.neo4j.gis.spatial.osm.OSMLayer; +import org.neo4j.gis.spatial.osm.OSMModel; import org.neo4j.gis.spatial.osm.OSMRelation; import org.neo4j.gis.spatial.rtree.Envelope; import org.neo4j.gis.spatial.rtree.filter.SearchAll; -import org.neo4j.gis.spatial.utilities.ReferenceNodes; import org.neo4j.graphdb.*; import org.neo4j.internal.kernel.api.security.SecurityContext; import org.neo4j.kernel.internal.GraphDatabaseAPI; @@ -47,6 +46,7 @@ import java.util.Map.Entry; import static org.neo4j.configuration.GraphDatabaseSettings.DEFAULT_DATABASE_NAME; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_CHANGESET; public class OsmAnalysisTest extends TestOSMImport { public static final String spatialTestMode = System.getProperty("spatial.test.mode"); @@ -273,10 +273,10 @@ public void testAnalysis(String osm) throws Exception { SortedMap layers; ReferencedEnvelope bbox; try (Transaction tx = graphDb().beginTx()) { - Node osmImport = tx.findNode(OSMImporter.LABEL_DATASET, "name", osm); - Node usersNode = osmImport.getSingleRelationship(OSMRelation.USERS, Direction.OUTGOING).getEndNode(); + Node osmImport = tx.findNode(OSMModel.LABEL_DATASET, "name", osm); + Node firstUser = osmImport.getSingleRelationship(OSMRelation.USERS, Direction.OUTGOING).getEndNode(); - Map userIndex = collectUserChangesetData(usersNode); + Map userIndex = collectUserChangesetData(firstUser); SortedSet topTen = getTopTen(userIndex); SpatialDatabaseService spatial = new SpatialDatabaseService(new IndexManager((GraphDatabaseAPI) db, SecurityContext.AUTH_DISABLED)); @@ -428,23 +428,21 @@ private SortedSet getTopTen(Map userIndex) { return topTen; } - private Map collectUserChangesetData(Node usersNode) { + private Map collectUserChangesetData(Node userNode) { Map userIndex = new HashMap<>(); - for (Relationship r : usersNode.getRelationships(Direction.OUTGOING, OSMRelation.OSM_USER)) { - Node userNode = r.getEndNode(); + while(userNode != null) { String name = (String) userNode.getProperty("name"); - User user = new User(userNode.getId(), name); userIndex.put(name, user); - for (Relationship ur : userNode.getRelationships(Direction.INCOMING, OSMRelation.USER)) { Node node = ur.getStartNode(); - if (node.hasProperty("changeset")) { + if (node.hasProperty(PROP_CHANGESET)) { user.changesets.add(node.getId()); } } + Relationship nextRel = userNode.getSingleRelationship(OSMRelation.NEXT, Direction.OUTGOING); + userNode = nextRel == null ? null : nextRel.getEndNode(); } - return userIndex; } diff --git a/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java b/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java index a68618c1b..0ecc1cb23 100644 --- a/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java +++ b/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java @@ -26,7 +26,9 @@ import org.neo4j.gis.spatial.Layer; import org.neo4j.gis.spatial.SpatialDatabaseService; import org.neo4j.gis.spatial.SpatialRelationshipTypes; +import org.neo4j.gis.spatial.index.Envelope; import org.neo4j.gis.spatial.index.IndexManager; +import org.neo4j.gis.spatial.osm.OSMRelation; import org.neo4j.gis.spatial.utilities.ReferenceNodes; import org.neo4j.graphdb.*; import org.neo4j.graphdb.spatial.Geometry; @@ -42,17 +44,16 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.function.Consumer; import static java.util.Collections.emptyMap; +import static java.util.Collections.lastIndexOfSubList; import static org.hamcrest.Matchers.*; import static org.junit.Assert.*; import static org.neo4j.configuration.GraphDatabaseSettings.DEFAULT_DATABASE_NAME; import static org.neo4j.gis.spatial.Constants.*; +import static org.neo4j.gis.spatial.osm.OSMModel.*; public class SpatialProceduresTest { private DatabaseManagementService databases; @@ -1143,11 +1144,84 @@ public void fail_to_merge_non_OSM_into_OSM() { testCallFails(db, "CALL spatial.merge.into($name,$nameB)", map("name", "osm", "nameB", "geomB"), "Cannot merge non-OSM layer into OSM layer"); } + private static class NodeIdRecorder implements Consumer { + ArrayList nodeIds = new ArrayList<>(); + HashMap osmIds = new HashMap<>(); + HashMap> envelopes = new HashMap<>(); + + @Override + public void accept(Result result) { + while (result.hasNext()) { + Node geom = (Node) result.next().get("node"); + double[] bbox = (double[]) geom.getProperty("bbox"); + Envelope env = new Envelope(bbox[0], bbox[1], bbox[2], bbox[3]); + Relationship geomRel = geom.getSingleRelationship(OSMRelation.GEOM, Direction.INCOMING); + if (geomRel == null) { + System.out.printf("Geometry node %d has no attached model node%n", geom.getId()); + } else { + Node node = geomRel.getStartNode(); + nodeIds.add(node.getId()); + ArrayList envNodes = envelopes.computeIfAbsent(env, k -> new ArrayList<>()); + envNodes.add(node.getId()); + Map properties = node.getAllProperties(); + boolean found = false; + for (String osmIdKey : new String[]{PROP_NODE_ID, PROP_WAY_ID, PROP_RELATION_ID}) { + if (!found) { + if (properties.containsKey(osmIdKey)) { + found = true; + long osmId = (Long) node.getProperty(osmIdKey); + osmIds.put(osmId, node.getId()); + } + } + } + if (!found) { + StringBuilder sb = new StringBuilder(); + for (Label label : node.getLabels()) { + if (sb.length() > 0) { + sb.append(","); + } + sb.append(label.name()); + } + System.out.printf("Found no OSM id in node %d (%s)%n", node.getId(), sb); + } + } + } + } + + public void debug(String name) { + System.out.printf("Results for '%s':%n", name); + System.out.printf("\t%d node ids%n", nodeIds.size()); + System.out.printf("\t%d OSM ids%n", osmIds.size()); + System.out.printf("\t%d envelops%n", envelopes.size()); + for (Envelope env : envelopes.keySet()) { + List ids = envelopes.get(env); + if (ids.size() > 1) { + System.out.printf("\t\t%d entries found in %s%n", ids.size(), env); + } + } + } + + public void shouldContain(NodeIdRecorder expected) { + ArrayList missing = new ArrayList<>(); + ArrayList found = new ArrayList<>(); + for (long bid : expected.osmIds.keySet()) { + if (!this.osmIds.containsKey(bid)) { + //System.out.printf("Failed to find expected node %d in results%n", bid); + missing.add(bid); + } else { + found.add(bid); + } + } + System.out.printf("There were %d/%d found nodes%n", found.size(), this.nodeIds.size()); + System.out.printf("There were %d/%d missing nodes%n", missing.size(), this.nodeIds.size()); + } + } + @Test - public void should_not_but_still_fail_to_merge_OSM_into_OSM() { + public void should_not_fail_to_merge_OSM_into_very_similar_OSM() { for (String name : new String[]{"osmA", "osmB"}) { execute("CALL spatial.addLayer($name,'OSM','')", map("name", name)); - testCountQuery("importOSMToLayerAndAddGeometry", "CALL spatial.importOSMToLayer($name,'map.osm')", 55, "count", map("name", name)); + testCountQuery("should_not_fail_to_merge_OSM_into_very_similar_OSM.importOSMToLayer('" + name + "')", "CALL spatial.importOSMToLayer($name,'map.osm')", 55, "count", map("name", name)); testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", map("name", name, "lon", 15.2, "lat", 60.1), 0); testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", map("name", name), 217); } @@ -1158,15 +1232,26 @@ public void should_not_but_still_fail_to_merge_OSM_into_OSM() { testCall(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", map("name", "osmB", "lon", 15.2, "lat", 60.1), r -> assertEquals(node, r.get("node"))); testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", map("name", "osmB", "lon", 15.2, "lat", 60.1), 1); testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", map("name", "osmB"), 217); + NodeIdRecorder bnodesFound = new NodeIdRecorder(); + testResult(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", map("name", "osmB"), bnodesFound); + bnodesFound.debug("withinDistance('osmB')"); // Assert that osmA does not have the extra node testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", map("name", "osmB", "lon", 15.2, "lat", 60.1), 1); // try merge osmB into osmA - should succeed, but does not yet - testCallFails(db, "CALL spatial.merge.into($nameA,$nameB)", map("nameA", "osmA", "nameB", "osmB"), "Not implemented"); + testCall(db, "CALL spatial.merge.into($nameA,$nameB)", map("nameA", "osmA", "nameB", "osmB"), r -> assertEquals(1L, r.get("count"))); + + // Extra debugging to find differences + NodeIdRecorder anodesFound = new NodeIdRecorder(); + testResult(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", map("name", "osmA"), anodesFound); + anodesFound.debug("withinDistance('osmA'+'osmB')"); + anodesFound.shouldContain(bnodesFound); // Here we should assert that osmA now does have the extra node - //..... + testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", map("name", "osmA"), 217); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", map("name", "osmB"), 0); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", map("name", "osmA", "lon", 15.2, "lat", 60.1), 1); } @Test From 8720eb0d5961bece3898048533b39a1723a897d7 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 19 Apr 2021 13:39:07 +0200 Subject: [PATCH 03/11] Reformatted some OSM files to get conformant with default formatting --- .../gis/spatial/osm/OSMGeometryEncoder.java | 1004 ++++++++--------- .../neo4j/gis/spatial/osm/OSMImporter.java | 4 + 2 files changed, 506 insertions(+), 502 deletions(-) diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java index b7d8604ec..8b768a198 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java @@ -37,507 +37,507 @@ public class OSMGeometryEncoder extends AbstractGeometryEncoder { - private static int decodedCount = 0; - private static int overrunCount = 0; - private static int nodeId = 0; - private static int wayId = 0; - private static int relationId = 0; - private DateFormat dateTimeFormatter; - private int vertices; - private int vertexMismatches = 0; - private final HashMap labelHashes = new HashMap<>(); - - /** - * This class allows for OSM to avoid having empty tags nodes when there are - * no properties on a geometry. - */ - private static final class NullProperties implements Entity { - @Override - public Object getProperty(String key) { - return null; - } - - @Override - public Object getProperty(String key, Object defaultValue) { - return null; - } - - @Override - public Iterable getPropertyKeys() { - return null; - } - - @Override - public Map getProperties(String... strings) { - return null; - } - - @Override - public Map getAllProperties() { - return null; - } - - @Override - public long getId() { - return 0; - } - - @Override - public boolean hasProperty(String key) { - return false; - } - - @Override - public Object removeProperty(String key) { - return null; - } - - @Override - public void setProperty(String key, Object value) { - } - } - - public static class OSMGraphException extends SpatialDatabaseException { - private static final long serialVersionUID = -6892234738075001044L; - - OSMGraphException(String message) { - super(message); - } - - OSMGraphException(String message, Exception cause) { - super(message, cause); - } - } - - private static Node testIsNode(Entity container) { - if (!(container instanceof Node)) { - throw new OSMGraphException("Cannot decode non-node geometry: " + container); - } - return (Node) container; - } - - @Override - public Envelope decodeEnvelope(Entity container) { - Node geomNode = testIsNode(container); - double[] bbox = (double[]) geomNode.getProperty(PROP_BBOX); - return new Envelope(bbox[0], bbox[1], bbox[2], bbox[3]); - } - - @Override - public void encodeEnvelope(Envelope mbb, Entity container) { - container.setProperty(PROP_BBOX, new double[] { mbb.getMinX(), mbb.getMaxX(), mbb.getMinY(), mbb.getMaxY() }); - } - - public static Node getOSMNodeFromGeometryNode(Node geomNode) { - Relationship rel = geomNode.getSingleRelationship(OSMRelation.GEOM, Direction.INCOMING); - if (rel != null) { - return rel.getStartNode(); - } else { - throw new IllegalArgumentException("No geom rel"); - } - } - - public static Node getGeometryNodeFromOSMNode(Node osmNode) { - return osmNode.getSingleRelationship(OSMRelation.GEOM, Direction.OUTGOING).getEndNode(); - } - - /** - * This wrapper class allows the traverser to run simply down the NEXT - * chain, but we wrap this to return the --NODE-->(node) results instead of - * the proxy nodes. - */ - private static class NodeProxyIterator implements Iterator { - Iterator traverser; - - NodeProxyIterator(Node first) { - TraversalDescription traversalDescription = new MonoDirectionalTraversalDescription().relationships(OSMRelation.NEXT, Direction.OUTGOING); - traverser = createTraverserInBackwardsCompatibleWay(traversalDescription, first).iterator(); - } - - public boolean hasNext() { - return traverser.hasNext(); - } - - public Node next() { - return traverser.next().endNode().getSingleRelationship(OSMRelation.NODE, Direction.OUTGOING).getEndNode(); - } - - public void remove() { - } - - } - - public Iterable getPointNodesFromWayNode(Node wayNode) { - final Node firstNode = wayNode.getSingleRelationship(OSMRelation.FIRST_NODE, Direction.OUTGOING).getEndNode(); - final NodeProxyIterator iterator = new NodeProxyIterator(firstNode); - return () -> iterator; - } - - public Geometry decodeGeometry(Entity container) { - Node geomNode = testIsNode(container); - try { - GeometryFactory geomFactory = layer.getGeometryFactory(); - Node osmNode = getOSMNodeFromGeometryNode(geomNode); - if (osmNode.hasProperty(PROP_NODE_ID)) { - return geomFactory.createPoint(new Coordinate( - (Double) osmNode.getProperty(PROP_NODE_LON, 0.0), - (Double) osmNode.getProperty(PROP_NODE_LAT, 0.0))); - } else if (osmNode.hasProperty(PROP_WAY_ID)) { - int vertices = (Integer) geomNode.getProperty("vertices"); - int gtype = (Integer) geomNode.getProperty(PROP_TYPE); - return decodeGeometryFromWay(osmNode, gtype, vertices, geomFactory); - } else { - int gtype = (Integer) geomNode.getProperty(PROP_TYPE); - return decodeGeometryFromRelation(osmNode, gtype, geomFactory); - } - } catch (Exception e) { - throw new OSMGraphException("Failed to decode OSM geometry: " + e.getMessage(), e); - } - } - - private Geometry decodeGeometryFromRelation(Node osmNode, int gtype, GeometryFactory geomFactory) { - switch (gtype) { - case GTYPE_POLYGON: - LinearRing outer = null; - ArrayList inner = new ArrayList<>(); - for (Relationship rel : osmNode.getRelationships(Direction.OUTGOING, OSMRelation.MEMBER)) { - Node wayNode = rel.getEndNode(); - String role = (String) rel.getProperty("role", null); - if (role != null) { - LinearRing ring = getOuterLinearRingFromGeometry(decodeGeometryFromWay(wayNode, GTYPE_POLYGON, -1, geomFactory)); - if (role.equals("outer")) { - outer = ring; - } else if (role.equals("inner")) { - inner.add(ring); - } - } - } - if (outer != null) { - return geomFactory.createPolygon(outer, inner.toArray(new LinearRing[0])); - } else { - return null; - } - case GTYPE_MULTIPOLYGON: - ArrayList polygons = new ArrayList<>(); - for (Relationship rel : osmNode.getRelationships(Direction.OUTGOING, OSMRelation.MEMBER)) { - Node member = rel.getEndNode(); - Geometry geometry = null; - if (member.hasProperty(PROP_WAY_ID)) { - // decode simple polygons from ways - geometry = decodeGeometryFromWay(member, GTYPE_POLYGON, -1, geomFactory); - } else if (!member.hasProperty(PROP_NODE_ID)) { - // decode polygons with holes from relations - geometry = decodeGeometryFromRelation(member, GTYPE_POLYGON, geomFactory); - } - if (geometry instanceof Polygon) { - polygons.add((Polygon) geometry); - } - } - if (polygons.size() > 0) { - return geomFactory.createMultiPolygon(polygons.toArray(new Polygon[0])); - } else { - return null; - } - default: - return null; - } - } - - /** - * Since OSM users can construct any weird combinations of geometries, we - * need general code to make the best guess. This method will find a - * enclosing LinearRing around any geometry except Point and a straight - * LineString, and return that. For sensible types, it returns a more - * sensible result, for example a Polygon will produce its outer LinearRing. - */ - private LinearRing getOuterLinearRingFromGeometry(Geometry geometry) { - if (geometry instanceof LineString) { - LineString line = (LineString) geometry; - if (line.getCoordinates().length < 3) { - return null; - } else { - Coordinate[] coords = line.getCoordinates(); - if (!line.isClosed()) { - coords = closeCoords(coords); - } - LinearRing ring = geometry.getFactory().createLinearRing(coords); - if (ring.isValid()) { - return ring; - } else { - return getConvexHull(ring); - } - } - } else if (geometry instanceof Polygon) { - return ((Polygon) geometry).getExteriorRing(); - } else { - return getConvexHull(geometry); - } - } - - /** - * Extend the array by copying the first point into the last position - * - * @param coords original array that is not closed - * @return new array one point longer - */ - private Coordinate[] closeCoords(Coordinate[] coords) { - Coordinate[] nc = new Coordinate[coords.length + 1]; - System.arraycopy(coords, 0, nc, 0, coords.length); - nc[coords.length] = coords[0]; - coords = nc; - return coords; - } - - /** - * The convex hull is like an elastic band surrounding all points in the - * geometry. - */ - private LinearRing getConvexHull(Geometry geometry) { - return getOuterLinearRingFromGeometry((new ConvexHull(geometry)).getConvexHull()); - } - - private Geometry decodeGeometryFromWay(Node wayNode, int gtype, int vertices, GeometryFactory geomFactory) { - ArrayList coordinates = new ArrayList<>(); - boolean overrun = false; - for (Node node : getPointNodesFromWayNode(wayNode)) { - if (coordinates.size() >= vertices) { - // System.err.println("Exceeding expected number of way nodes: " - // + (index + 1) + - // " > " + vertices); - overrun = true; - overrunCount++; - break; - } - coordinates.add(new Coordinate((Double) node.getProperty(PROP_NODE_LON), (Double) node.getProperty(OSMModel.PROP_NODE_LAT))); - } - decodedCount++; - if (overrun) { - System.out.println("Overran expected number of way nodes: " + wayNode + " (" + overrunCount + "/" + decodedCount + ")"); - } - if (coordinates.size() != vertices) { - if (vertexMismatches++ < 10) { - System.err.println("Mismatching vertices size for " + SpatialDatabaseService.convertGeometryTypeToName(gtype) + ":" - + wayNode + ": " + coordinates.size() + " != " + vertices); - } else if (vertexMismatches % 100 == 0) { - System.err.println("Mismatching vertices found " + vertexMismatches + " times"); - } - } - switch (coordinates.size()) { - case 0: - return null; - case 1: - return geomFactory.createPoint(coordinates.get(0)); - default: - Coordinate[] coords = coordinates.toArray(new Coordinate[0]); - switch (gtype) { - case GTYPE_LINESTRING: - return geomFactory.createLineString(coords); - case GTYPE_POLYGON: - return geomFactory.createPolygon(geomFactory.createLinearRing(coords), new LinearRing[0]); - default: - return geomFactory.createMultiPointFromCoords(coords); - } - } - } - - /** - * For OSM data we can build basic geometry shapes as sub-graphs. This code should produce the same kinds of structures that the utilities in the OSMDataset create. However those structures are created from original OSM data, while here we attempt to create equivalent graphs from JTS Geometries. Note that this code is unable to connect the resulting sub-graph into the OSM data model, since the only node it has is the geometry node. Those connections to the rest of the OSM model need to be done in OSMDataset. - */ - @Override - protected void encodeGeometryShape(Transaction tx, Geometry geometry, Entity container) { - Node geomNode = testIsNode(container); - vertices = 0; - int gtype = SpatialDatabaseService.convertJtsClassToGeometryType(geometry.getClass()); - switch (gtype) { - case GTYPE_POINT: - makeOSMNode(tx, geometry, geomNode); - break; - case GTYPE_LINESTRING: - case GTYPE_MULTIPOINT: - case GTYPE_POLYGON: - makeOSMWay(tx, geometry, geomNode, gtype); - break; - case GTYPE_MULTILINESTRING: - case GTYPE_MULTIPOLYGON: - int gsubtype = gtype == GTYPE_MULTIPOLYGON ? GTYPE_POLYGON : GTYPE_LINESTRING; - Node relationNode = makeOSMRelation(geometry, geomNode); - int num = geometry.getNumGeometries(); - for (int i = 0; i < num; i++) { - Geometry geom = geometry.getGeometryN(i); - Node wayNode = makeOSMWay(tx, geom, tx.createNode(), gsubtype); - relationNode.createRelationshipTo(wayNode, OSMRelation.MEMBER); - } - break; - default: - throw new SpatialDatabaseException("Unsupported geometry: " + geometry.getClass()); - } - geomNode.setProperty("vertices", vertices); - } - - private Node makeOSMNode(Transaction tx, Geometry geometry, Node geomNode) { - Node node = makeOSMNode(tx, geometry.getCoordinate()); - node.createRelationshipTo(geomNode, OSMRelation.GEOM); - return node; - } - - private void addLabelHash(Transaction tx, OSMDataset dataset, Label label, String propertyKey) { - String indexName = dataset.getIndexName(tx, label, propertyKey); - if (indexName != null) { - labelHashes.put(label, OSMDataset.hashedLabelFrom(indexName)); - } - } - - private void loadLabelHash(Transaction tx) { - if (labelHashes.isEmpty()) { - OSMDataset dataset = OSMDataset.fromLayer(tx, (OSMLayer) layer); - addLabelHash(tx, dataset, LABEL_NODE, PROP_NODE_ID); - addLabelHash(tx, dataset, LABEL_WAY, PROP_WAY_ID); - addLabelHash(tx, dataset, LABEL_RELATION, PROP_RELATION_ID); - addLabelHash(tx, dataset, LABEL_USER, PROP_USER_ID); - addLabelHash(tx, dataset, LABEL_CHANGESET, PROP_CHANGESET); - } - } - - private Label getLabelHash(Transaction tx, Label label) { - loadLabelHash(tx); - return labelHashes.get(label); - } - - private Node makeOSMNode(Transaction tx, Coordinate coordinate) { - vertices++; - nodeId++; - Node node = tx.createNode(OSMModel.LABEL_NODE); - Label hashed = getLabelHash(tx, LABEL_NODE); - if (hashed != null) { - // This allows this node to be found using the same index that the OSMImporter uses - node.addLabel(hashed); - } - // We need a fake osm-id, but cannot know what positive integers are not used - // So we just set it to a negative of the Neo4j nodeId. - // This is only unsafe for nodeId=0, but that is certain to be used for other nodes than these. - node.setProperty(PROP_NODE_ID, -nodeId); - node.setProperty(PROP_NODE_LAT, coordinate.y); - node.setProperty(PROP_NODE_LON, coordinate.x); - node.setProperty("timestamp", getTimestamp()); - // TODO: Add other common properties, like changeset, uid, user, version - return node; - } - - private Node makeOSMWay(Transaction tx, Geometry geometry, Node geomNode, int gtype) { - wayId++; - Node way = tx.createNode(); - // TODO: Generate a valid osm id - way.setProperty(PROP_WAY_ID, wayId); - way.setProperty("timestamp", getTimestamp()); - // TODO: Add other common properties, like changeset, uid, user, - // version, name - way.createRelationshipTo(geomNode, OSMRelation.GEOM); - // TODO: if this way is a part of a complex geometry, the sub-geometries - // are not indexed - geomNode.setProperty(PROP_TYPE, gtype); - Node prev = null; - for (Coordinate coord : geometry.getCoordinates()) { - Node node = makeOSMNode(tx, coord); - Node proxyNode = tx.createNode(); - proxyNode.createRelationshipTo(node, OSMRelation.NODE); - if (prev == null) { - way.createRelationshipTo(proxyNode, OSMRelation.FIRST_NODE); - } else { - prev.createRelationshipTo(proxyNode, OSMRelation.NEXT); - } - prev = proxyNode; - } - return way; - } - - private Node makeOSMRelation(Geometry geometry, Node geomNode) { - relationId++; - throw new SpatialDatabaseException("Unimplemented: makeOSMRelation()"); - } - - private String getTimestamp() { - if (dateTimeFormatter == null) - dateTimeFormatter = new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); - return dateTimeFormatter.format(new Date(System.currentTimeMillis())); - } - - private Node lastGeom = null; - private CombinedAttributes lastAttr = null; - private long missingTags = 0; - - private class CombinedAttributes { - private Node node; - private Entity properties; - private final HashMap extra = new HashMap<>(); - - CombinedAttributes(Node geomNode) { - try { - node = geomNode.getSingleRelationship(OSMRelation.GEOM, Direction.INCOMING).getStartNode(); - properties = node.getSingleRelationship(OSMRelation.TAGS, Direction.OUTGOING).getEndNode(); - Node changeset = node.getSingleRelationship(OSMRelation.CHANGESET, Direction.OUTGOING).getEndNode(); - if (changeset != null) { - extra.put(PROP_CHANGESET, changeset.getProperty(PROP_CHANGESET, null)); - Node user = changeset.getSingleRelationship(OSMRelation.USER, Direction.OUTGOING).getEndNode(); - if (user != null) { - extra.put(PROP_USER_NAME, user.getProperty("name", null)); - extra.put("user_id", user.getProperty("uid", null)); - } - } - } catch (NullPointerException e) { - if (missingTags++ < 10) { + private static int decodedCount = 0; + private static int overrunCount = 0; + private static int nodeId = 0; + private static int wayId = 0; + private static int relationId = 0; + private DateFormat dateTimeFormatter; + private int vertices; + private int vertexMismatches = 0; + private final HashMap labelHashes = new HashMap<>(); + + /** + * This class allows for OSM to avoid having empty tags nodes when there are + * no properties on a geometry. + */ + private static final class NullProperties implements Entity { + @Override + public Object getProperty(String key) { + return null; + } + + @Override + public Object getProperty(String key, Object defaultValue) { + return null; + } + + @Override + public Iterable getPropertyKeys() { + return null; + } + + @Override + public Map getProperties(String... strings) { + return null; + } + + @Override + public Map getAllProperties() { + return null; + } + + @Override + public long getId() { + return 0; + } + + @Override + public boolean hasProperty(String key) { + return false; + } + + @Override + public Object removeProperty(String key) { + return null; + } + + @Override + public void setProperty(String key, Object value) { + } + } + + public static class OSMGraphException extends SpatialDatabaseException { + private static final long serialVersionUID = -6892234738075001044L; + + OSMGraphException(String message) { + super(message); + } + + OSMGraphException(String message, Exception cause) { + super(message, cause); + } + } + + private static Node testIsNode(Entity container) { + if (!(container instanceof Node)) { + throw new OSMGraphException("Cannot decode non-node geometry: " + container); + } + return (Node) container; + } + + @Override + public Envelope decodeEnvelope(Entity container) { + Node geomNode = testIsNode(container); + double[] bbox = (double[]) geomNode.getProperty(PROP_BBOX); + return new Envelope(bbox[0], bbox[1], bbox[2], bbox[3]); + } + + @Override + public void encodeEnvelope(Envelope mbb, Entity container) { + container.setProperty(PROP_BBOX, new double[]{mbb.getMinX(), mbb.getMaxX(), mbb.getMinY(), mbb.getMaxY()}); + } + + public static Node getOSMNodeFromGeometryNode(Node geomNode) { + Relationship rel = geomNode.getSingleRelationship(OSMRelation.GEOM, Direction.INCOMING); + if (rel != null) { + return rel.getStartNode(); + } else { + throw new IllegalArgumentException("No geom rel"); + } + } + + public static Node getGeometryNodeFromOSMNode(Node osmNode) { + return osmNode.getSingleRelationship(OSMRelation.GEOM, Direction.OUTGOING).getEndNode(); + } + + /** + * This wrapper class allows the traverser to run simply down the NEXT + * chain, but we wrap this to return the --NODE-->(node) results instead of + * the proxy nodes. + */ + private static class NodeProxyIterator implements Iterator { + Iterator traverser; + + NodeProxyIterator(Node first) { + TraversalDescription traversalDescription = new MonoDirectionalTraversalDescription().relationships(OSMRelation.NEXT, Direction.OUTGOING); + traverser = createTraverserInBackwardsCompatibleWay(traversalDescription, first).iterator(); + } + + public boolean hasNext() { + return traverser.hasNext(); + } + + public Node next() { + return traverser.next().endNode().getSingleRelationship(OSMRelation.NODE, Direction.OUTGOING).getEndNode(); + } + + public void remove() { + } + + } + + public Iterable getPointNodesFromWayNode(Node wayNode) { + final Node firstNode = wayNode.getSingleRelationship(OSMRelation.FIRST_NODE, Direction.OUTGOING).getEndNode(); + final NodeProxyIterator iterator = new NodeProxyIterator(firstNode); + return () -> iterator; + } + + public Geometry decodeGeometry(Entity container) { + Node geomNode = testIsNode(container); + try { + GeometryFactory geomFactory = layer.getGeometryFactory(); + Node osmNode = getOSMNodeFromGeometryNode(geomNode); + if (osmNode.hasProperty(PROP_NODE_ID)) { + return geomFactory.createPoint(new Coordinate( + (Double) osmNode.getProperty(PROP_NODE_LON, 0.0), + (Double) osmNode.getProperty(PROP_NODE_LAT, 0.0))); + } else if (osmNode.hasProperty(PROP_WAY_ID)) { + int vertices = (Integer) geomNode.getProperty("vertices"); + int gtype = (Integer) geomNode.getProperty(PROP_TYPE); + return decodeGeometryFromWay(osmNode, gtype, vertices, geomFactory); + } else { + int gtype = (Integer) geomNode.getProperty(PROP_TYPE); + return decodeGeometryFromRelation(osmNode, gtype, geomFactory); + } + } catch (Exception e) { + throw new OSMGraphException("Failed to decode OSM geometry: " + e.getMessage(), e); + } + } + + private Geometry decodeGeometryFromRelation(Node osmNode, int gtype, GeometryFactory geomFactory) { + switch (gtype) { + case GTYPE_POLYGON: + LinearRing outer = null; + ArrayList inner = new ArrayList<>(); + for (Relationship rel : osmNode.getRelationships(Direction.OUTGOING, OSMRelation.MEMBER)) { + Node wayNode = rel.getEndNode(); + String role = (String) rel.getProperty("role", null); + if (role != null) { + LinearRing ring = getOuterLinearRingFromGeometry(decodeGeometryFromWay(wayNode, GTYPE_POLYGON, -1, geomFactory)); + if (role.equals("outer")) { + outer = ring; + } else if (role.equals("inner")) { + inner.add(ring); + } + } + } + if (outer != null) { + return geomFactory.createPolygon(outer, inner.toArray(new LinearRing[0])); + } else { + return null; + } + case GTYPE_MULTIPOLYGON: + ArrayList polygons = new ArrayList<>(); + for (Relationship rel : osmNode.getRelationships(Direction.OUTGOING, OSMRelation.MEMBER)) { + Node member = rel.getEndNode(); + Geometry geometry = null; + if (member.hasProperty(PROP_WAY_ID)) { + // decode simple polygons from ways + geometry = decodeGeometryFromWay(member, GTYPE_POLYGON, -1, geomFactory); + } else if (!member.hasProperty(PROP_NODE_ID)) { + // decode polygons with holes from relations + geometry = decodeGeometryFromRelation(member, GTYPE_POLYGON, geomFactory); + } + if (geometry instanceof Polygon) { + polygons.add((Polygon) geometry); + } + } + if (polygons.size() > 0) { + return geomFactory.createMultiPolygon(polygons.toArray(new Polygon[0])); + } else { + return null; + } + default: + return null; + } + } + + /** + * Since OSM users can construct any weird combinations of geometries, we + * need general code to make the best guess. This method will find a + * enclosing LinearRing around any geometry except Point and a straight + * LineString, and return that. For sensible types, it returns a more + * sensible result, for example a Polygon will produce its outer LinearRing. + */ + private LinearRing getOuterLinearRingFromGeometry(Geometry geometry) { + if (geometry instanceof LineString) { + LineString line = (LineString) geometry; + if (line.getCoordinates().length < 3) { + return null; + } else { + Coordinate[] coords = line.getCoordinates(); + if (!line.isClosed()) { + coords = closeCoords(coords); + } + LinearRing ring = geometry.getFactory().createLinearRing(coords); + if (ring.isValid()) { + return ring; + } else { + return getConvexHull(ring); + } + } + } else if (geometry instanceof Polygon) { + return ((Polygon) geometry).getExteriorRing(); + } else { + return getConvexHull(geometry); + } + } + + /** + * Extend the array by copying the first point into the last position + * + * @param coords original array that is not closed + * @return new array one point longer + */ + private Coordinate[] closeCoords(Coordinate[] coords) { + Coordinate[] nc = new Coordinate[coords.length + 1]; + System.arraycopy(coords, 0, nc, 0, coords.length); + nc[coords.length] = coords[0]; + coords = nc; + return coords; + } + + /** + * The convex hull is like an elastic band surrounding all points in the + * geometry. + */ + private LinearRing getConvexHull(Geometry geometry) { + return getOuterLinearRingFromGeometry((new ConvexHull(geometry)).getConvexHull()); + } + + private Geometry decodeGeometryFromWay(Node wayNode, int gtype, int vertices, GeometryFactory geomFactory) { + ArrayList coordinates = new ArrayList<>(); + boolean overrun = false; + for (Node node : getPointNodesFromWayNode(wayNode)) { + if (coordinates.size() >= vertices) { + // System.err.println("Exceeding expected number of way nodes: " + // + (index + 1) + + // " > " + vertices); + overrun = true; + overrunCount++; + break; + } + coordinates.add(new Coordinate((Double) node.getProperty(PROP_NODE_LON), (Double) node.getProperty(OSMModel.PROP_NODE_LAT))); + } + decodedCount++; + if (overrun) { + System.out.println("Overran expected number of way nodes: " + wayNode + " (" + overrunCount + "/" + decodedCount + ")"); + } + if (coordinates.size() != vertices) { + if (vertexMismatches++ < 10) { + System.err.println("Mismatching vertices size for " + SpatialDatabaseService.convertGeometryTypeToName(gtype) + ":" + + wayNode + ": " + coordinates.size() + " != " + vertices); + } else if (vertexMismatches % 100 == 0) { + System.err.println("Mismatching vertices found " + vertexMismatches + " times"); + } + } + switch (coordinates.size()) { + case 0: + return null; + case 1: + return geomFactory.createPoint(coordinates.get(0)); + default: + Coordinate[] coords = coordinates.toArray(new Coordinate[0]); + switch (gtype) { + case GTYPE_LINESTRING: + return geomFactory.createLineString(coords); + case GTYPE_POLYGON: + return geomFactory.createPolygon(geomFactory.createLinearRing(coords), new LinearRing[0]); + default: + return geomFactory.createMultiPointFromCoords(coords); + } + } + } + + /** + * For OSM data we can build basic geometry shapes as sub-graphs. This code should produce the same kinds of structures that the utilities in the OSMDataset create. However those structures are created from original OSM data, while here we attempt to create equivalent graphs from JTS Geometries. Note that this code is unable to connect the resulting sub-graph into the OSM data model, since the only node it has is the geometry node. Those connections to the rest of the OSM model need to be done in OSMDataset. + */ + @Override + protected void encodeGeometryShape(Transaction tx, Geometry geometry, Entity container) { + Node geomNode = testIsNode(container); + vertices = 0; + int gtype = SpatialDatabaseService.convertJtsClassToGeometryType(geometry.getClass()); + switch (gtype) { + case GTYPE_POINT: + makeOSMNode(tx, geometry, geomNode); + break; + case GTYPE_LINESTRING: + case GTYPE_MULTIPOINT: + case GTYPE_POLYGON: + makeOSMWay(tx, geometry, geomNode, gtype); + break; + case GTYPE_MULTILINESTRING: + case GTYPE_MULTIPOLYGON: + int gsubtype = gtype == GTYPE_MULTIPOLYGON ? GTYPE_POLYGON : GTYPE_LINESTRING; + Node relationNode = makeOSMRelation(geometry, geomNode); + int num = geometry.getNumGeometries(); + for (int i = 0; i < num; i++) { + Geometry geom = geometry.getGeometryN(i); + Node wayNode = makeOSMWay(tx, geom, tx.createNode(), gsubtype); + relationNode.createRelationshipTo(wayNode, OSMRelation.MEMBER); + } + break; + default: + throw new SpatialDatabaseException("Unsupported geometry: " + geometry.getClass()); + } + geomNode.setProperty("vertices", vertices); + } + + private Node makeOSMNode(Transaction tx, Geometry geometry, Node geomNode) { + Node node = makeOSMNode(tx, geometry.getCoordinate()); + node.createRelationshipTo(geomNode, OSMRelation.GEOM); + return node; + } + + private void addLabelHash(Transaction tx, OSMDataset dataset, Label label, String propertyKey) { + String indexName = dataset.getIndexName(tx, label, propertyKey); + if (indexName != null) { + labelHashes.put(label, OSMDataset.hashedLabelFrom(indexName)); + } + } + + private void loadLabelHash(Transaction tx) { + if (labelHashes.isEmpty()) { + OSMDataset dataset = OSMDataset.fromLayer(tx, (OSMLayer) layer); + addLabelHash(tx, dataset, LABEL_NODE, PROP_NODE_ID); + addLabelHash(tx, dataset, LABEL_WAY, PROP_WAY_ID); + addLabelHash(tx, dataset, LABEL_RELATION, PROP_RELATION_ID); + addLabelHash(tx, dataset, LABEL_USER, PROP_USER_ID); + addLabelHash(tx, dataset, LABEL_CHANGESET, PROP_CHANGESET); + } + } + + private Label getLabelHash(Transaction tx, Label label) { + loadLabelHash(tx); + return labelHashes.get(label); + } + + private Node makeOSMNode(Transaction tx, Coordinate coordinate) { + vertices++; + nodeId++; + Node node = tx.createNode(OSMModel.LABEL_NODE); + Label hashed = getLabelHash(tx, LABEL_NODE); + if (hashed != null) { + // This allows this node to be found using the same index that the OSMImporter uses + node.addLabel(hashed); + } + // We need a fake osm-id, but cannot know what positive integers are not used + // So we just set it to a negative of the Neo4j nodeId. + // This is only unsafe for nodeId=0, but that is certain to be used for other nodes than these. + node.setProperty(PROP_NODE_ID, -nodeId); + node.setProperty(PROP_NODE_LAT, coordinate.y); + node.setProperty(PROP_NODE_LON, coordinate.x); + node.setProperty("timestamp", getTimestamp()); + // TODO: Add other common properties, like changeset, uid, user, version + return node; + } + + private Node makeOSMWay(Transaction tx, Geometry geometry, Node geomNode, int gtype) { + wayId++; + Node way = tx.createNode(); + // TODO: Generate a valid osm id + way.setProperty(PROP_WAY_ID, wayId); + way.setProperty("timestamp", getTimestamp()); + // TODO: Add other common properties, like changeset, uid, user, + // version, name + way.createRelationshipTo(geomNode, OSMRelation.GEOM); + // TODO: if this way is a part of a complex geometry, the sub-geometries + // are not indexed + geomNode.setProperty(PROP_TYPE, gtype); + Node prev = null; + for (Coordinate coord : geometry.getCoordinates()) { + Node node = makeOSMNode(tx, coord); + Node proxyNode = tx.createNode(); + proxyNode.createRelationshipTo(node, OSMRelation.NODE); + if (prev == null) { + way.createRelationshipTo(proxyNode, OSMRelation.FIRST_NODE); + } else { + prev.createRelationshipTo(proxyNode, OSMRelation.NEXT); + } + prev = proxyNode; + } + return way; + } + + private Node makeOSMRelation(Geometry geometry, Node geomNode) { + relationId++; + throw new SpatialDatabaseException("Unimplemented: makeOSMRelation()"); + } + + private String getTimestamp() { + if (dateTimeFormatter == null) + dateTimeFormatter = new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + return dateTimeFormatter.format(new Date(System.currentTimeMillis())); + } + + private Node lastGeom = null; + private CombinedAttributes lastAttr = null; + private long missingTags = 0; + + private class CombinedAttributes { + private Node node; + private Entity properties; + private final HashMap extra = new HashMap<>(); + + CombinedAttributes(Node geomNode) { + try { + node = geomNode.getSingleRelationship(OSMRelation.GEOM, Direction.INCOMING).getStartNode(); + properties = node.getSingleRelationship(OSMRelation.TAGS, Direction.OUTGOING).getEndNode(); + Node changeset = node.getSingleRelationship(OSMRelation.CHANGESET, Direction.OUTGOING).getEndNode(); + if (changeset != null) { + extra.put(PROP_CHANGESET, changeset.getProperty(PROP_CHANGESET, null)); + Node user = changeset.getSingleRelationship(OSMRelation.USER, Direction.OUTGOING).getEndNode(); + if (user != null) { + extra.put(PROP_USER_NAME, user.getProperty("name", null)); + extra.put("user_id", user.getProperty("uid", null)); + } + } + } catch (NullPointerException e) { + if (missingTags++ < 10) { System.err.println("Geometry has no related tags node: " + geomNode); - } else if (missingTags % 100 == 0) { - System.err.println("Geometries without tags found " + missingTags + " times"); - } - properties = new NullProperties(); - } - } - - public boolean hasProperty(String key) { - return extra.containsKey(key) || node.hasProperty(key) || properties.hasProperty(key); - } - - public Object getProperty(String key) { - return extra.containsKey(key) ? extra.get(key) : node.hasProperty(key) ? node.getProperty(key, null) : properties - .getProperty(key, null); - } - - } - - private CombinedAttributes getProperties(Node geomNode) { - if (geomNode != lastGeom) { - lastGeom = geomNode; - lastAttr = new CombinedAttributes(geomNode); - } - return lastAttr; - } - - /** - * This method wraps the hasProperty(String) method on the geometry node. - * This means the default way of storing attributes is simply as properties - * of the geometry node. This behaviour can be changed by other domain - * models with different encodings. - * - * @param geomNode node to test - * @param name attribute to check for existence of - * @return true if node has the specified attribute - */ - public boolean hasAttribute(Node geomNode, String name) { - return getProperties(geomNode).hasProperty(name); - } - - /** - * This method wraps the getProperty(String,null) method on the geometry - * node. This means the default way of storing attributes is simply as - * properties of the geometry node. This behaviour can be changed by other - * domain models with different encodings. If the property does not exist, - * the method returns null. - * - * @param geomNode node to test - * @param name attribute to access - * @return attribute value, or null - */ - public Object getAttribute(Node geomNode, String name) { - return getProperties(geomNode).getProperty(name); - } + } else if (missingTags % 100 == 0) { + System.err.println("Geometries without tags found " + missingTags + " times"); + } + properties = new NullProperties(); + } + } + + public boolean hasProperty(String key) { + return extra.containsKey(key) || node.hasProperty(key) || properties.hasProperty(key); + } + + public Object getProperty(String key) { + return extra.containsKey(key) ? extra.get(key) : node.hasProperty(key) ? node.getProperty(key, null) : properties + .getProperty(key, null); + } + + } + + private CombinedAttributes getProperties(Node geomNode) { + if (geomNode != lastGeom) { + lastGeom = geomNode; + lastAttr = new CombinedAttributes(geomNode); + } + return lastAttr; + } + + /** + * This method wraps the hasProperty(String) method on the geometry node. + * This means the default way of storing attributes is simply as properties + * of the geometry node. This behaviour can be changed by other domain + * models with different encodings. + * + * @param geomNode node to test + * @param name attribute to check for existence of + * @return true if node has the specified attribute + */ + public boolean hasAttribute(Node geomNode, String name) { + return getProperties(geomNode).hasProperty(name); + } + + /** + * This method wraps the getProperty(String,null) method on the geometry + * node. This means the default way of storing attributes is simply as + * properties of the geometry node. This behaviour can be changed by other + * domain models with different encodings. If the property does not exist, + * the method returns null. + * + * @param geomNode node to test + * @param name attribute to access + * @return attribute value, or null + */ + public Object getAttribute(Node geomNode, String name) { + return getProperties(geomNode).getProperty(name); + } } diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java index e539d820e..0aafc63c9 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java @@ -320,19 +320,23 @@ public void indexByWay(Transaction tx, Node way) { } } } + public void indexByChangeset(Transaction tx, Node changeset) { for (Relationship rel : changeset.getRelationships(Direction.INCOMING, OSMRelation.CHANGESET)) { stats.addGeomStats(layer.addWay(tx, rel.getStartNode(), true)); } } + public List allWays(Transaction tx) { OSMDataset dataset = OSMDataset.fromLayer(tx, layer); return toList(dataset.getAllWayNodes(tx)); } + public List allChangesets(Transaction tx) { OSMDataset dataset = OSMDataset.fromLayer(tx, layer); return toList(dataset.getAllChangesetNodes(tx)); } + private List toList(Iterable iterable) { ArrayList list = new ArrayList<>(); if (iterable != null) { From 4479cc21c2b0e635861a005bd4effc35895b854c Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 19 Apr 2021 14:20:53 +0200 Subject: [PATCH 04/11] Reduced number of warnings in OSM code --- .../gis/spatial/osm/OSMGeometryEncoder.java | 7 +- .../neo4j/gis/spatial/osm/OSMImporter.java | 126 +++++++----------- .../org/neo4j/gis/spatial/osm/OSMLayer.java | 27 +--- .../org/neo4j/gis/spatial/osm/OSMMerger.java | 2 +- .../neo4j/gis/spatial/osm/OSMRelation.java | 14 +- .../neo4j/gis/spatial/osm/RoadDirection.java | 2 +- 6 files changed, 57 insertions(+), 121 deletions(-) diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java index 8b768a198..a9818a1cc 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java @@ -301,9 +301,6 @@ private Geometry decodeGeometryFromWay(Node wayNode, int gtype, int vertices, Ge boolean overrun = false; for (Node node : getPointNodesFromWayNode(wayNode)) { if (coordinates.size() >= vertices) { - // System.err.println("Exceeding expected number of way nodes: " - // + (index + 1) + - // " > " + vertices); overrun = true; overrunCount++; break; @@ -374,10 +371,9 @@ protected void encodeGeometryShape(Transaction tx, Geometry geometry, Entity con geomNode.setProperty("vertices", vertices); } - private Node makeOSMNode(Transaction tx, Geometry geometry, Node geomNode) { + private void makeOSMNode(Transaction tx, Geometry geometry, Node geomNode) { Node node = makeOSMNode(tx, geometry.getCoordinate()); node.createRelationshipTo(geomNode, OSMRelation.GEOM); - return node; } private void addLabelHash(Transaction tx, OSMDataset dataset, Label label, String propertyKey) { @@ -450,6 +446,7 @@ private Node makeOSMWay(Transaction tx, Geometry geometry, Node geomNode, int gt return way; } + @SuppressWarnings("unused") private Node makeOSMRelation(Geometry geometry, Node geomNode) { relationId++; throw new SpatialDatabaseException("Unimplemented: makeOSMRelation()"); diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java index 0aafc63c9..68eaaccca 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java @@ -59,7 +59,6 @@ public class OSMImporter implements Constants { public static DefaultEllipsoid WGS84 = DefaultEllipsoid.WGS84; - protected boolean nodesProcessingFinished = false; private final String layerName; private final StatsManager tagStats = new StatsManager(); private final GeomStats geomStats = new GeomStats(); @@ -80,15 +79,13 @@ private static class TagStats { this.name = name; } - int add(String key) { + void add(String key) { count++; if (stats.containsKey(key)) { int num = stats.get(key); stats.put(key, ++num); - return num; } else { stats.put(key, 1); - return 1; } } @@ -124,17 +121,15 @@ TagStats getTagStats(String type) { return tagStats.get(type); } - int addToTagStats(String type, String key) { + void addToTagStats(String type, String key) { getTagStats("all").add(key); - return getTagStats(type).add(key); + getTagStats(type).add(key); } - int addToTagStats(String type, Collection keys) { - int count = 0; + void addToTagStats(String type, Collection keys) { for (String key : keys) { - count += addToTagStats(type, key); + addToTagStats(type, key); } - return count; } void printTagStats() { @@ -271,10 +266,6 @@ public long reIndex(GraphDatabaseService database, int commitInterval, boolean i public static class OSMIndexer { private static final TraversalDescription traversal = new MonoDirectionalTraversalDescription(); - private static final org.neo4j.graphdb.traversal.TraversalDescription findWays = traversal.depthFirst() - .evaluator(Evaluators.excludeStartPosition()) - .relationships(OSMRelation.WAYS, Direction.OUTGOING) - .relationships(OSMRelation.NEXT, Direction.OUTGOING); private static final org.neo4j.graphdb.traversal.TraversalDescription findNodes = traversal.depthFirst() .evaluator(Evaluators.excludeStartPosition()) .relationships(OSMRelation.FIRST_NODE, Direction.OUTGOING) @@ -451,8 +442,6 @@ void createRelationship(T from, T to, OSMRelation relType) { long firstFindTime = 0; long lastFindTime = 0; long firstLogTime = 0; - static int foundNodes = 0; - static int createdNodes = 0; int foundOSMNodes = 0; int missingUserCount = 0; @@ -462,7 +451,7 @@ void logMissingUser(Map nodeProps) { } } - private class LogCounter { + private static class LogCounter { private long count = 0; private long totalTime = 0; } @@ -489,26 +478,20 @@ void logNodesFound(long currentTime) { if (currentTime > 0) { duration = (int) ((currentTime - firstFindTime) / 1000); } - System.out.println(new Date(currentTime) + ": Found " - + foundOSMNodes + " nodes during " - + duration + "s way creation: "); + System.out.printf("%s: Found %d nodes during %ds way creation:%n", new Date(currentTime), foundOSMNodes, duration); for (String type : nodeFindStats.keySet()) { LogCounter found = nodeFindStats.get(type); double rate = 0.0f; if (found.totalTime > 0) { rate = (1000.0 * (float) found.count / (float) found.totalTime); } - System.out.println("\t" + type + ": \t" + found.count - + "/" + (found.totalTime / 1000) - + "s" + " \t(" + rate - + " nodes/second)"); + System.out.printf("\t%s: \t%d/%ds \t%f nodes/second%n", type, found.count, (found.totalTime / 1000), rate); } findTime = currentTime; } } - void logNodeAddition(LinkedHashMap tags, - String type) { + void logNodeAddition(String type) { Integer count = stats.get(type); if (count == null) { count = 1; @@ -522,7 +505,8 @@ void logNodeAddition(LinkedHashMap tags, logTime = currentTime; } if (currentTime - logTime > 1432) { - System.out.println(new Date(currentTime) + ": Saving " + type + " " + count + " \t(" + (1000.0 * (float) count / (float) (currentTime - firstLogTime)) + " " + type + "/second)"); + double rate = (1000.0 * (float) count / (float) (currentTime - firstLogTime)); + System.out.printf("%s: Saving %s %d \t(%f %s/second)%n", new Date(currentTime), type, count, rate, type); logTime = currentTime; } } @@ -543,18 +527,16 @@ void describeLoaded() { private void missingNode(long ndRef) { if (missingNodeCount++ < 10) { - osmImporter.error("Cannot find node for osm-id " + ndRef); + osmImporter.errorf("Cannot find node for osm-id %d%n", ndRef); } } private void describeMissing() { if (missingNodeCount > 0) { - osmImporter.error("When processing the ways, there were " - + missingNodeCount + " missing nodes"); + osmImporter.errorf("When processing the ways, there were %d missing nodes%n", missingNodeCount); } if (missingMemberCount > 0) { - osmImporter.error("When processing the relations, there were " - + missingMemberCount + " missing members"); + osmImporter.errorf("When processing the relations, there were %d missing members%n", missingMemberCount); } } @@ -562,7 +544,7 @@ private void describeMissing() { private void missingMember(String description) { if (missingMemberCount++ < 10) { - osmImporter.error("Cannot find member: " + description); + osmImporter.errorf("Cannot find member: %s%n", description); } } @@ -727,7 +709,7 @@ private void createOSMRelation(Map relationProperties, // We will test for cases that invalidate multilinestring further down GeometryMetaData metaGeom = new GeometryMetaData(GTYPE_MULTILINESTRING); T prevMember = null; - LinkedHashMap relProps = new LinkedHashMap(); + LinkedHashMap relProps = new LinkedHashMap<>(); for (Map memberProps : relationMembers) { String memberType = (String) memberProps.get("type"); long member_ref = Long.parseLong(memberProps.get("ref").toString()); @@ -753,7 +735,7 @@ private void createOSMRelation(Map relationProperties, continue; } if (member == relation) { - osmImporter.error("Cannot add relation to same member: relation[" + relationTags + "] - member[" + memberProps + "]"); + osmImporter.errorf("Cannot add relation to same member: relation[%s] - member[%s]%n", relationTags, memberProps); continue; } Map nodeProps = getNodeProperties(member); @@ -776,7 +758,7 @@ private void createOSMRelation(Map relationProperties, createRelationship(relation, member, OSMRelation.MEMBER, relProps); prevMember = member; } else { - System.err.println("Cannot process invalid relation member: " + memberProps.toString()); + System.err.println("Cannot process invalid relation member: " + memberProps); } } if (metaGeom.isValid()) { @@ -873,7 +855,6 @@ private static class OSMGraphWriter extends OSMWriter { private WrappedNode currentChangesetNode; private long currentUserId = -1; private WrappedNode currentUserNode; - private WrappedNode usersNode; private final HashMap changesetNodes = new HashMap<>(); private Transaction tx; private int checkCount = 0; @@ -902,8 +883,7 @@ private static String md5Hash(String text) throws NoSuchAlgorithmException { MessageDigest md = MessageDigest.getInstance("MD5"); md.update(text.getBytes()); byte[] digest = md.digest(); - String hashed = DatatypeConverter.printHexBinary(digest).toUpperCase(); - return hashed; + return DatatypeConverter.printHexBinary(digest).toUpperCase(); } private void successTx() { @@ -938,7 +918,6 @@ private void beginTx() { recoverNode(prev_way); recoverNode(currentChangesetNode); recoverNode(currentUserNode); - recoverNode(usersNode); changesetNodes.forEach((id, node) -> node.refresh(tx)); } @@ -1038,9 +1017,9 @@ private void saveIndexName(Label label, String propertyKey, String indexName) { if (previousIndex == null) { osm_dataset.setProperty(indexKey, indexName); } else if (previousIndex.equals(indexName)) { - System.out.println(String.format("OSMLayer '%s' already has matching index definition for '%s': %s", osm_dataset.getProperty("name", ""), indexKey, previousIndex)); + System.out.printf("OSMLayer '%s' already has matching index definition for '%s': %s%n", osm_dataset.getProperty("name", ""), indexKey, previousIndex); } else { - throw new IllegalStateException(String.format("OSMLayer '%s' already has index definition for '%s': %s")); + throw new IllegalStateException(String.format("OSMLayer '%s' already has index definition for '%s': %s", osm_dataset.getProperty("name", ""), indexKey, previousIndex)); } } @@ -1101,7 +1080,7 @@ private void addProperties(Entity node, Map properties) { @Override protected void addNodeTags(WrappedNode node, LinkedHashMap tags, String type) { - logNodeAddition(tags, type); + logNodeAddition(type); if (node != null && tags.size() > 0) { tagStats.addToTagStats(type, tags.keySet()); WrappedNode tagsNode = createNodeWithLabel(tx, LABEL_TAGS); @@ -1298,7 +1277,7 @@ public void importFile(GraphDatabaseService database, String dataset, boolean al } public static class CountedFileReader extends InputStreamReader { - private long length = 0; + private final long length; private long charsRead = 0; public CountedFileReader(String path, Charset charset) throws FileNotFoundException { @@ -1306,14 +1285,6 @@ public CountedFileReader(String path, Charset charset) throws FileNotFoundExcept this.length = (new File(path)).length(); } - public long getCharsRead() { - return charsRead; - } - - public long getlength() { - return length; - } - public double getProgress() { return length > 0 ? (double) charsRead / (double) length : 0; } @@ -1322,8 +1293,7 @@ public int getPercentRead() { return (int) (100.0 * getProgress()); } - public int read(char[] cbuf, int offset, int length) - throws IOException { + public int read(char[] cbuf, int offset, int length) throws IOException { int read = super.read(cbuf, offset, length); if (read > 0) charsRead += read; return read; @@ -1440,16 +1410,10 @@ public void importFile(OSMWriter osmWriter, String dataset, boolean allPoints } if (startedRelations) { if (countXMLTags < 10) { - debug("Starting tag at depth " + depth + ": " - + currentXMLTags.get(depth) + " - " - + currentXMLTags.toString()); + debugf("Starting tag at depth %d: %s - %s%n", depth, currentXMLTags.get(depth), currentXMLTags); for (int i = 0; i < parser.getAttributeCount(); i++) { - debug("\t" + currentXMLTags.toString() + ": " - + parser.getAttributeLocalName(i) + "[" - + parser.getAttributeNamespace(i) + "," - + parser.getAttributePrefix(i) + "," - + parser.getAttributeType(i) + "," - + "] = " + parser.getAttributeValue(i)); + debugf("\t%s: %s[%s,%s,%s] = %s%n", currentXMLTags, parser.getAttributeLocalName(i), parser.getAttributeNamespace(i), + parser.getAttributePrefix(i), parser.getAttributeType(i), parser.getAttributeValue(i)); } } countXMLTags++; @@ -1526,7 +1490,7 @@ private Map extractProperties(String name, XMLStreamReader parse */ - LinkedHashMap properties = new LinkedHashMap(); + LinkedHashMap properties = new LinkedHashMap<>(); for (int i = 0; i < parser.getAttributeCount(); i++) { String prop = parser.getAttributeLocalName(i); String value = parser.getAttributeValue(i); @@ -1580,10 +1544,6 @@ public static RoadDirection getRoadDirection(Map wayProperties) /** * Calculate correct distance between 2 points on Earth. * - * @param latA - * @param lonA - * @param latB - * @param lonB * @return distance in meters */ public static double distance(double lonA, double latA, double lonB, double latB) { @@ -1597,24 +1557,31 @@ private void log(PrintStream out, String message) { out.println(message); } + private void logf(PrintStream out, String format, Object... args) { + if (logContext != null) { + format = logContext + "[" + contextLine + "]: " + format; + } + out.printf(format, args); + } + private void log(String message) { if (verboseLog) { log(System.out, message); } } - private void debug(String message) { + private void debugf(String format, Object... args) { if (debugLog) { - log(System.out, message); + logf(System.out, format, args); } } - private void error(String message) { - log(System.err, message); + private void errorf(String format, Object... args) { + logf(System.err, format, args); } private void error(String message, Exception e) { - log(System.err, message); + logf(System.err, message + ": %s", e.getMessage()); e.printStackTrace(System.err); } @@ -1624,8 +1591,7 @@ private void error(String message, Exception e) { private boolean verboseLog = true; // "2008-06-11T12:36:28Z" - private DateFormat timestampFormat = new SimpleDateFormat( - "yyyy-MM-dd'T'HH:mm:ss'Z'"); + private final DateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); public void setDebug(boolean verbose) { this.debugLog = verbose; @@ -1674,7 +1640,7 @@ private static class OSMImportManager { private DatabaseManagementService databases; private GraphDatabaseService graphDb; private File dbPath; - private String databaseName = "neo4j"; // can only be something other than neo4j in enterprise edition + private final String databaseName = "neo4j"; // can only be something other than neo4j in enterprise edition public OSMImportManager(String path) { setDbPath(path); @@ -1692,25 +1658,23 @@ public void setDbPath(String path) { } private void loadTestOsmData(String layerName, int commitInterval) throws Exception { - String osmPath = layerName; - System.out.println("\n=== Loading layer " + layerName + " from " + osmPath + " ===\n"); + System.out.println("\n=== Loading layer " + layerName + " from " + layerName + " ===\n"); long start = System.currentTimeMillis(); OSMImporter importer = new OSMImporter(layerName); prepareDatabase(true); - importer.importFile(graphDb, osmPath, false, commitInterval); + importer.importFile(graphDb, layerName, false, commitInterval); importer.reIndex(graphDb, commitInterval); shutdown(); System.out.println("=== Completed loading " + layerName + " in " + (System.currentTimeMillis() - start) / 1000.0 + " seconds ==="); } - private DatabaseLayout prepareLayout(boolean delete) throws IOException { + private void prepareLayout(boolean delete) throws IOException { Neo4jLayout homeLayout = Neo4jLayout.of(dbPath.toPath()); DatabaseLayout databaseLayout = homeLayout.databaseLayout(databaseName); if (delete) { FileUtils.deleteDirectory(databaseLayout.databaseDirectory()); FileUtils.deleteDirectory(databaseLayout.getTransactionLogsDirectory()); } - return databaseLayout; } private void prepareDatabase(boolean delete) throws IOException { diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java index 910d6ad55..af16a4e6c 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java @@ -19,18 +19,19 @@ */ package org.neo4j.gis.spatial.osm; -import java.io.File; -import java.util.HashMap; - import org.geotools.referencing.crs.DefaultGeographicCRS; import org.json.simple.JSONObject; import org.neo4j.gis.spatial.*; import org.neo4j.gis.spatial.merge.MergeUtils; import org.neo4j.gis.spatial.rtree.NullListener; -import org.neo4j.graphdb.*; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.Transaction; import org.opengis.referencing.crs.CoordinateReferenceSystem; -import javax.management.relation.Relation; +import java.io.File; +import java.util.HashMap; /** * Instances of this class represent the primary layer of the OSM Dataset. It @@ -57,27 +58,15 @@ public Integer getGeometryType() { /** * OSM always uses WGS84 CRS; so we return that. - * - * @param tx */ public CoordinateReferenceSystem getCoordinateReferenceSystem(Transaction tx) { - try { - return DefaultGeographicCRS.WGS84; - } catch (Exception e) { - System.err.println("Failed to decode WGS84 CRS: " + e.getMessage()); - e.printStackTrace(System.err); - return null; - } + return DefaultGeographicCRS.WGS84; } protected void clear(Transaction tx) { indexWriter.clear(tx, new NullListener()); } - public Node addWay(Transaction tx, Node way) { - return addWay(tx, way, false); - } - public Node addWay(Transaction tx, Node way, boolean verifyGeom) { Relationship geomRel = way.getSingleRelationship(OSMRelation.GEOM, Direction.OUTGOING); if (geomRel != null) { @@ -119,7 +108,6 @@ public Node addGeomNode(Transaction tx, Node geomNode, boolean verifyGeom) { * generate a Geometry. There is no restriction on a node belonging to multiple datasets, or * multiple layers within the same dataset. * - * @param tx * @return iterable over geometry nodes in the dataset */ public Iterable getAllGeometryNodes(Transaction tx) { @@ -144,7 +132,6 @@ public boolean removeDynamicLayer(Transaction tx, String name) { * to the way node and then to the tags node to test if the way is a * residential street. */ - @SuppressWarnings("unchecked") public DynamicLayerConfig addDynamicLayerOnWayTags(Transaction tx, String name, int type, HashMap tags) { JSONObject query = new JSONObject(); if (tags != null && !tags.isEmpty()) { diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMMerger.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMMerger.java index f65c779d5..f3d6320f8 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMMerger.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMMerger.java @@ -132,7 +132,7 @@ void printStats() { System.out.printf("During merge %d %s were added - re-indexing required%n", countAdded, name); } if (geomNodesToAdd.size() > 0) { - System.out.printf("During merge %d point geometry nodes were identified%n", geomNodesToAdd.size(), name); + System.out.printf("During merge %d geometry %s were identified for use in re-indexing%n", geomNodesToAdd.size(), name); } } } diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMRelation.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMRelation.java index 3c388030c..87039b59a 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMRelation.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMRelation.java @@ -17,23 +17,11 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -/* AWE - Amanzi Wireless Explorer - * http://awe.amanzi.org - * (C) 2008-2009, AmanziTel AB - * - * This library is provided under the terms of the Eclipse Public License - * as described at http://www.eclipse.org/legal/epl-v10.html. Any use, - * reproduction or distribution of the library constitutes recipient's - * acceptance of this agreement. - * - * This library is distributed WITHOUT ANY WARRANTY; without even the - * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - */ package org.neo4j.gis.spatial.osm; import org.neo4j.graphdb.RelationshipType; public enum OSMRelation implements RelationshipType { - FIRST_NODE, LAST_NODE, OTHER, NEXT, OSM, WAYS, RELATIONS, MEMBERS, MEMBER, TAGS, GEOM, BBOX, NODE, CHANGESET, USER, USERS; + FIRST_NODE, LAST_NODE, OTHER, NEXT, OSM, WAYS, RELATIONS, MEMBERS, MEMBER, TAGS, GEOM, BBOX, NODE, CHANGESET, USER, USERS } \ No newline at end of file diff --git a/src/main/java/org/neo4j/gis/spatial/osm/RoadDirection.java b/src/main/java/org/neo4j/gis/spatial/osm/RoadDirection.java index 2e7759b41..abcc56132 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/RoadDirection.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/RoadDirection.java @@ -33,5 +33,5 @@ package org.neo4j.gis.spatial.osm; public enum RoadDirection { - BOTH, FORWARD, BACKWARD; + BOTH, FORWARD, BACKWARD } \ No newline at end of file From 911aa874c4892824c9a2652736e16da568f2d504 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 10 May 2021 10:50:35 +0200 Subject: [PATCH 05/11] Fixed copyright headers to new text used in Neo4j (removed date range) --- neo.sld.xml | 2 +- src/main/assembly/docs-assembly.xml | 2 +- src/main/assembly/geoserver-plugin.xml | 2 +- src/main/assembly/server-plugin.xml | 2 +- .../data/neo4j/DefaultResourceInfo.java | 2 +- .../data/neo4j/Neo4jFeatureBuilder.java | 2 +- .../data/neo4j/Neo4jSpatialDataStore.java | 2 +- .../neo4j/Neo4jSpatialDataStoreFactory.java | 2 +- .../data/neo4j/Neo4jSpatialFeatureSource.java | 2 +- .../data/neo4j/Neo4jSpatialFeatureStore.java | 2 +- .../data/neo4j/StyledImageExporter.java | 2 +- .../gis/spatial/AbstractGeometryEncoder.java | 2 +- .../neo4j/gis/spatial/ConsoleListener.java | 2 +- .../java/org/neo4j/gis/spatial/Constants.java | 2 +- .../org/neo4j/gis/spatial/DefaultLayer.java | 2 +- .../org/neo4j/gis/spatial/DynamicLayer.java | 2 +- .../neo4j/gis/spatial/DynamicLayerConfig.java | 2 +- .../org/neo4j/gis/spatial/EditableLayer.java | 2 +- .../neo4j/gis/spatial/EditableLayerImpl.java | 2 +- .../neo4j/gis/spatial/GeometryEncoder.java | 2 +- .../java/org/neo4j/gis/spatial/Layer.java | 2 +- .../spatial/LineStringNetworkGenerator.java | 2 +- .../gis/spatial/OrderedEditableLayer.java | 2 +- .../neo4j/gis/spatial/ShapefileExporter.java | 2 +- .../neo4j/gis/spatial/ShapefileImporter.java | 2 +- .../neo4j/gis/spatial/SimplePointLayer.java | 2 +- .../gis/spatial/SpatialDatabaseException.java | 2 +- .../gis/spatial/SpatialDatabaseRecord.java | 2 +- .../gis/spatial/SpatialDatabaseService.java | 2 +- .../org/neo4j/gis/spatial/SpatialDataset.java | 2 +- .../org/neo4j/gis/spatial/SpatialRecord.java | 2 +- .../gis/spatial/SpatialRelationshipTypes.java | 2 +- .../gis/spatial/SpatialTopologyUtils.java | 2 +- .../java/org/neo4j/gis/spatial/Utilities.java | 2 +- .../neo4j/gis/spatial/WKBGeometryEncoder.java | 2 +- .../neo4j/gis/spatial/WKTGeometryEncoder.java | 2 +- .../spatial/attributes/PropertyMapper.java | 2 +- .../attributes/PropertyMappingManager.java | 2 +- .../AbstractSinglePropertyEncoder.java | 19 +++++++++++++++++++ .../gis/spatial/encoders/Configurable.java | 2 +- .../spatial/encoders/NativePointEncoder.java | 2 +- .../spatial/encoders/SimpleGraphEncoder.java | 2 +- .../spatial/encoders/SimplePointEncoder.java | 2 +- .../encoders/SimplePropertyEncoder.java | 2 +- .../gis/spatial/encoders/neo4j/Neo4jCRS.java | 2 +- .../spatial/encoders/neo4j/Neo4jGeometry.java | 2 +- .../spatial/encoders/neo4j/Neo4jPoint.java | 19 +++++++++++++++++++ .../filter/AbstractSearchIntersection.java | 2 +- .../neo4j/gis/spatial/filter/SearchCQL.java | 2 +- .../gis/spatial/filter/SearchIntersect.java | 2 +- .../spatial/filter/SearchIntersectWindow.java | 2 +- .../gis/spatial/filter/SearchRecords.java | 2 +- .../index/ExplicitIndexBackedMonitor.java | 2 +- .../index/ExplicitIndexBackedPointIndex.java | 2 +- .../neo4j/gis/spatial/index/IndexManager.java | 19 +++++++++++++++++++ .../spatial/index/LayerGeohashPointIndex.java | 2 +- .../spatial/index/LayerHilbertPointIndex.java | 2 +- .../gis/spatial/index/LayerIndexReader.java | 2 +- .../gis/spatial/index/LayerRTreeIndex.java | 2 +- .../LayerSpaceFillingCurvePointIndex.java | 2 +- .../spatial/index/LayerTreeIndexReader.java | 2 +- .../spatial/index/LayerZOrderPointIndex.java | 2 +- .../index/PropertyEncodingNodeIndex.java | 19 +++++++++++++++++++ .../gis/spatial/index/SpatialIndexReader.java | 2 +- .../gis/spatial/index/SpatialIndexWriter.java | 2 +- .../spatial/indexfilter/CQLIndexReader.java | 2 +- .../indexfilter/DynamicIndexReader.java | 2 +- .../indexfilter/LayerIndexReaderWrapper.java | 2 +- .../neo4j/gis/spatial/merge/MergeUtils.java | 2 +- .../org/neo4j/gis/spatial/osm/OSMDataset.java | 2 +- .../gis/spatial/osm/OSMGeometryEncoder.java | 2 +- .../neo4j/gis/spatial/osm/OSMImporter.java | 2 +- .../org/neo4j/gis/spatial/osm/OSMLayer.java | 2 +- .../osm/OSMLayerToShapefileExporter.java | 4 ++-- .../org/neo4j/gis/spatial/osm/OSMMerger.java | 2 +- .../org/neo4j/gis/spatial/osm/OSMModel.java | 2 +- .../neo4j/gis/spatial/osm/OSMRelation.java | 2 +- .../neo4j/gis/spatial/osm/RoadDirection.java | 2 +- .../org/neo4j/gis/spatial/package-info.java | 2 +- .../spatial/pipes/AbstractExtractGeoPipe.java | 2 +- .../spatial/pipes/AbstractFilterGeoPipe.java | 2 +- .../gis/spatial/pipes/AbstractGeoPipe.java | 2 +- .../spatial/pipes/AbstractGroupGeoPipe.java | 2 +- .../neo4j/gis/spatial/pipes/GeoPipeFlow.java | 2 +- .../neo4j/gis/spatial/pipes/GeoPipeline.java | 2 +- .../spatial/pipes/filtering/FilterCQL.java | 2 +- .../pipes/filtering/FilterContain.java | 2 +- .../spatial/pipes/filtering/FilterCover.java | 2 +- .../pipes/filtering/FilterCoveredBy.java | 2 +- .../spatial/pipes/filtering/FilterCross.java | 2 +- .../pipes/filtering/FilterDisjoint.java | 2 +- .../spatial/pipes/filtering/FilterEmpty.java | 2 +- .../pipes/filtering/FilterEqualExact.java | 2 +- .../pipes/filtering/FilterEqualNorm.java | 2 +- .../pipes/filtering/FilterEqualTopo.java | 2 +- .../pipes/filtering/FilterInRelation.java | 2 +- .../pipes/filtering/FilterIntersect.java | 2 +- .../filtering/FilterIntersectWindow.java | 2 +- .../pipes/filtering/FilterInvalid.java | 2 +- .../pipes/filtering/FilterOverlap.java | 2 +- .../pipes/filtering/FilterProperty.java | 2 +- .../filtering/FilterPropertyNotNull.java | 2 +- .../pipes/filtering/FilterPropertyNull.java | 2 +- .../spatial/pipes/filtering/FilterTouch.java | 2 +- .../spatial/pipes/filtering/FilterValid.java | 2 +- .../spatial/pipes/filtering/FilterWithin.java | 2 +- .../gis/spatial/pipes/osm/OSMGeoPipeline.java | 2 +- .../osm/filtering/FilterOSMAttributes.java | 2 +- .../osm/processing/ExtractOSMPoints.java | 2 +- .../processing/ApplyAffineTransformation.java | 2 +- .../gis/spatial/pipes/processing/Area.java | 2 +- .../spatial/pipes/processing/Boundary.java | 2 +- .../gis/spatial/pipes/processing/Buffer.java | 2 +- .../spatial/pipes/processing/Centroid.java | 2 +- .../spatial/pipes/processing/ConvexHull.java | 2 +- .../CopyDatabaseRecordProperties.java | 2 +- .../gis/spatial/pipes/processing/Densify.java | 2 +- .../pipes/processing/DensityIslands.java | 2 +- .../spatial/pipes/processing/Difference.java | 2 +- .../spatial/pipes/processing/Dimension.java | 2 +- .../spatial/pipes/processing/Distance.java | 2 +- .../spatial/pipes/processing/EndPoint.java | 2 +- .../spatial/pipes/processing/Envelope.java | 2 +- .../pipes/processing/ExtractGeometries.java | 2 +- .../pipes/processing/ExtractPoints.java | 2 +- .../gis/spatial/pipes/processing/GML.java | 2 +- .../gis/spatial/pipes/processing/GeoJSON.java | 2 +- .../pipes/processing/GeometryType.java | 2 +- .../pipes/processing/InteriorPoint.java | 2 +- .../pipes/processing/IntersectAll.java | 2 +- .../pipes/processing/Intersection.java | 2 +- .../processing/KeyholeMarkupLanguage.java | 2 +- .../gis/spatial/pipes/processing/Length.java | 2 +- .../gis/spatial/pipes/processing/Max.java | 2 +- .../gis/spatial/pipes/processing/Min.java | 2 +- .../pipes/processing/NumGeometries.java | 2 +- .../spatial/pipes/processing/NumPoints.java | 2 +- .../pipes/processing/OrthodromicDistance.java | 2 +- .../pipes/processing/OrthodromicLength.java | 2 +- .../SimplifyPreservingTopology.java | 2 +- .../SimplifyWithDouglasPeucker.java | 2 +- .../gis/spatial/pipes/processing/Sort.java | 2 +- .../spatial/pipes/processing/StartPoint.java | 2 +- .../pipes/processing/SymDifference.java | 2 +- .../gis/spatial/pipes/processing/Union.java | 2 +- .../spatial/pipes/processing/UnionAll.java | 2 +- .../pipes/processing/WellKnownText.java | 2 +- .../spatial/procedures/SpatialProcedures.java | 2 +- .../gis/spatial/process/SpatialProcess.java | 2 +- .../neo4j/gis/spatial/rtree/EmptyMonitor.java | 2 +- .../org/neo4j/gis/spatial/rtree/Envelope.java | 2 +- .../gis/spatial/rtree/EnvelopeDecoder.java | 2 +- .../rtree/EnvelopeDecoderFromDoubleArray.java | 2 +- .../org/neo4j/gis/spatial/rtree/Listener.java | 2 +- .../neo4j/gis/spatial/rtree/NullListener.java | 2 +- .../rtree/ProgressLoggingListener.java | 2 +- .../gis/spatial/rtree/RTreeImageExporter.java | 2 +- .../neo4j/gis/spatial/rtree/RTreeIndex.java | 3 ++- .../neo4j/gis/spatial/rtree/RTreeMonitor.java | 2 +- .../spatial/rtree/RTreeRelationshipTypes.java | 2 +- .../rtree/SpatialIndexRecordCounter.java | 2 +- .../spatial/rtree/SpatialIndexVisitor.java | 2 +- .../neo4j/gis/spatial/rtree/TreeMonitor.java | 2 +- .../AbstractSearchEnvelopeIntersection.java | 2 +- .../gis/spatial/rtree/filter/SearchAll.java | 2 +- .../rtree/filter/SearchCoveredByEnvelope.java | 2 +- .../rtree/filter/SearchEqualEnvelopes.java | 2 +- .../spatial/rtree/filter/SearchFilter.java | 2 +- .../spatial/rtree/filter/SearchResults.java | 2 +- .../gis/spatial/utilities/LayerUtilities.java | 2 +- .../gis/spatial/utilities/ReferenceNodes.java | 2 +- .../spatial/utilities/TraverserFactory.java | 2 +- src/site/site.xml | 2 +- .../neo4j/doc/tools/AsciiDocGenerator.java | 2 +- .../neo4j/doc/tools/DocumentationData.java | 2 +- .../org/neo4j/doc/tools/GraphVizConfig.java | 2 +- .../org/neo4j/doc/tools/JSONPrettifier.java | 2 +- .../doc/tools/JavaTestDocsGenerator.java | 2 +- .../doc/tools/SpatialGraphVizHelper.java | 2 +- .../gis/spatial/AbstractJavaDocTestBase.java | 2 +- .../java/org/neo4j/gis/spatial/FakeIndex.java | 2 +- .../neo4j/gis/spatial/LayerSignatureTest.java | 2 +- .../org/neo4j/gis/spatial/LayersTest.java | 2 +- .../spatial/Neo4jSpatialDataStoreTest.java | 19 +++++++++++++++++++ .../org/neo4j/gis/spatial/Neo4jTestCase.java | 2 +- .../org/neo4j/gis/spatial/Neo4jTestUtils.java | 2 +- .../neo4j/gis/spatial/OsmAnalysisTest.java | 2 +- .../spatial/ProgressLoggingListenerTest.java | 2 +- .../gis/spatial/RTreeBulkInsertTest.java | 19 +++++++++++++++++++ .../org/neo4j/gis/spatial/RTreeTestUtils.java | 19 +++++++++++++++++++ .../spatial/SpatialIndexPerformanceProxy.java | 2 +- .../neo4j/gis/spatial/TestDynamicLayers.java | 2 +- .../spatial/TestIntersectsPathQueries.java | 2 +- .../org/neo4j/gis/spatial/TestOSMImport.java | 2 +- .../org/neo4j/gis/spatial/TestProcess.java | 2 +- .../gis/spatial/TestReadOnlyTransactions.java | 19 +++++++++++++++++++ .../org/neo4j/gis/spatial/TestRemove.java | 2 +- .../gis/spatial/TestSimplePointLayer.java | 2 +- .../org/neo4j/gis/spatial/TestSpatial.java | 2 +- .../neo4j/gis/spatial/TestSpatialQueries.java | 2 +- .../neo4j/gis/spatial/TestSpatialUtils.java | 2 +- .../org/neo4j/gis/spatial/TestsForDocs.java | 2 +- .../gis/spatial/TryWithResourceTest.java | 19 +++++++++++++++++++ .../LayerGeohashNativePointIndexTest.java | 2 +- .../LayerGeohashSimplePointIndexTest.java | 2 +- .../LayerHilbertNativePointIndexTest.java | 2 +- .../LayerHilbertSimplePointIndexTest.java | 2 +- .../gis/spatial/index/LayerIndexTestBase.java | 2 +- .../index/LayerRTreeNativePointIndexTest.java | 2 +- .../index/LayerRTreeSimplePointIndexTest.java | 2 +- .../LayerZOrderNativePointIndexTest.java | 2 +- .../LayerZOrderSimplePointIndexTest.java | 2 +- .../index/NativePointIndexTestBase.java | 2 +- .../index/SimplePointIndexTestBase.java | 2 +- .../gis/spatial/pipes/GeoPipesDocTest.java | 2 +- .../pipes/GeoPipesPerformanceTest.java | 2 +- .../pipes/OrthodromicDistanceTest.java | 2 +- .../procedures/SpatialProceduresTest.java | 2 +- .../gis/spatial/rtree/EnvelopeTests.java | 2 +- .../neo4j/gis/spatial/rtree/RTreeTests.java | 2 +- .../gis/spatial/rtree/TestRTreeIndex.java | 2 +- 221 files changed, 385 insertions(+), 213 deletions(-) diff --git a/neo.sld.xml b/neo.sld.xml index a099c3e9c..084a9ed16 100644 --- a/neo.sld.xml +++ b/neo.sld.xml @@ -1,7 +1,7 @@