Skip to content

Implemented the ability to use BatchInserter #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LudgerPeters wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Implemented the ability to use BatchInserter #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions src/main/java/org/neo4j/neode/Dataset.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,14 @@ <T> T execute( BatchCommand<T> command )

public void end()
{
log.write( String.format( "Store [%s]\n\nEnd [%s] %s\n",
((GraphDatabaseAPI) db).getStoreDir(), description, elapsedTime( runStartTime ) ) );
//this should be updated since it is going to be deprecated
//@deprecated This will be moved to internal packages in the next major release.
if(db instanceof GraphDatabaseAPI)
{
log.write(String.format("Store [%s]\n\nEnd [%s] %s\n",
((GraphDatabaseAPI) db).getStoreDir(), description, elapsedTime(runStartTime)));
}
log.write( String.format( "End [%s] %s\n", description, elapsedTime( runStartTime ) ) );
}

GraphDatabaseService db()
Expand Down
8 changes: 8 additions & 0 deletions src/main/java/org/neo4j/neode/statistics/GraphStatistics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,22 @@
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Transaction;
import org.neo4j.kernel.GraphDatabaseAPI;
import org.neo4j.tooling.GlobalGraphOperations;
import org.neo4j.unsafe.batchinsert.BatchInserterImpl;

public class GraphStatistics
{

public static GraphStatistics create( GraphDatabaseService db, String description,
NodeLabelResolver nodeLabelResolver )
{
//if db is not the normal graph database api
//We need to find another way to do this since the GraphDatabaseAPI is /** @deprecated */
if(!(db instanceof GraphDatabaseAPI))
{
return null;
}
try ( Transaction tx = db.beginTx() )
{
GraphStatistics graphStatistics = new GraphStatistics( description, nodeLabelResolver );
Expand Down
18 changes: 18 additions & 0 deletions src/test/java/org/neo4j/neode/test/Db.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
import org.neo4j.kernel.impl.util.FileUtils;
import org.neo4j.test.TestGraphDatabaseFactory;
import org.neo4j.unsafe.batchinsert.BatchInserters;

public final class Db
{
Expand Down Expand Up @@ -81,4 +83,20 @@ public interface WithSampleDataset
}


public static GraphDatabaseService getBatchInserterDB(String location)
{
File file = new File(location);
if(file.exists() && file.isDirectory())
{
try
{
FileUtils.deleteRecursively(file);
Copy link
Copy Markdown
Author

@LudgerPeters LudgerPeters Oct 20, 2014

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now that i think about this i might want to add some kind of warning here, since this does delete all the files at the target location.

}
catch (IOException e)
{
e.printStackTrace();
}
}
return BatchInserters.batchDatabase(location);
}
}
30 changes: 30 additions & 0 deletions src/test/java/org/neo4j/neode/test/MultipleRelsPerNodeExample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import org.junit.Test;

import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
import org.neo4j.neode.Dataset;
import org.neo4j.neode.DatasetManager;
import org.neo4j.neode.NodeCollection;
Expand Down Expand Up @@ -46,4 +47,33 @@ public void buildMultipleRelsPerNode()
db.shutdown();

}

@Test
public void batchBuildMultipleRelsPerNode()
{
GraphDatabaseService db = Db.getBatchInserterDB("batch.db");
DatasetManager dsm = new DatasetManager( db, SysOutLog.INSTANCE );
Dataset dataset = dsm.newDataset( "Multiple Rels Per Node" );

NodeSpecification root = dsm.nodeSpecification( "root", property("name") );
NodeSpecification leaf = dsm.nodeSpecification( "leaf", property( "price", integerRange( 1, 10 ) ) );

RelationshipSpecification connected_to = dsm.relationshipSpecification( "CONNECTED_TO",
property( "quantity", integerRange( 1, 5 ) ) );

NodeCollection roots = root.create( 1 ).update( dataset );

roots.createRelationshipsTo( TargetNodesStrategy.create( leaf ).numberOfTargetNodes( 2 ).relationship(
connected_to ).relationshipConstraints( Range.exactly( 2 ) ) ).updateNoReturn( dataset );

dataset.end();
db.shutdown();
db = new GraphDatabaseFactory().newEmbeddedDatabase("batch.db");
//since we cant use graph statistics we have to end the dataset shut it down and open it up using graphFactory
GraphStatistics statistics = GraphStatistics.create( db, "Pricing Structure" );
new AsciiDocFormatter( SysOutLog.INSTANCE ).describe( statistics );

db.shutdown();

}
}
82 changes: 82 additions & 0 deletions src/test/java/org/neo4j/neode/test/SocialNetworkExample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import org.neo4j.graphdb.Direction;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Path;
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
import org.neo4j.graphdb.traversal.Evaluation;
import org.neo4j.graphdb.traversal.Evaluator;
import org.neo4j.graphdb.traversal.TraversalDescription;
Expand Down Expand Up @@ -109,4 +110,85 @@ public Evaluation evaluate( Path path )

System.out.println( "Number of topics: " + topics.size() );
}

@Test
public void buildSocialNetworkBatchInserter() throws Exception
{
String SocialNetwork = "socialNetwork.db";
GraphDatabaseService db = Db.getBatchInserterDB(SocialNetwork);
DatasetManager dsm = new DatasetManager( db, SysOutLog.INSTANCE );

TraversalDescription findCompanyProjects = Traversal.description()
.depthFirst()
.relationships( withName( "WORKS_FOR" ), Direction.BOTH )
.relationships( withName( "WORKED_ON" ), Direction.OUTGOING )
.evaluator( new Evaluator()
{
@Override
public Evaluation evaluate( Path path )
{
if ( path.lastRelationship() == null )
{
return Evaluation.EXCLUDE_AND_CONTINUE;
}
if ( path.lastRelationship().isType( withName( "WORKED_ON" ) ) )
{
return Evaluation.INCLUDE_AND_PRUNE;
}
return Evaluation.EXCLUDE_AND_CONTINUE;
}
} );

NodeSpecification user = dsm.nodeSpecification( "user", property( "name" ) );
NodeSpecification topic = dsm.nodeSpecification( "topic", property( "label" ) );
NodeSpecification company = dsm.nodeSpecification( "company", property( "name" ) );
NodeSpecification project = dsm.nodeSpecification( "project", property( "name" ) );

RelationshipSpecification interested_in = dsm.relationshipSpecification( "INTERESTED_IN" );
RelationshipSpecification works_for = dsm.relationshipSpecification( "WORKS_FOR" );
RelationshipSpecification worked_on = dsm.relationshipSpecification( "WORKED_ON" );

Dataset dataset = dsm.newDataset( "Social network example" );

NodeCollection users = user.create( 10 ).update( dataset );

NodeCollection topics = users.createRelationshipsTo(
getOrCreate( topic, 10, normalDistribution() )
.numberOfTargetNodes( minMax( 1, 3 ) )
.relationship( interested_in )
.exactlyOneRelationship() )
.update( dataset );

users.createRelationshipsTo(
getOrCreate( company, 2, flatDistribution() )
.numberOfTargetNodes( 1 )
.relationship( works_for )
.exactlyOneRelationship() )
.updateNoReturn( dataset );

NodeCollection allProjects = users.createRelationshipsTo(
queryBasedGetOrCreate( project, traversal( findCompanyProjects ), 1.2 )
.numberOfTargetNodes( minMax( 1, 3 ) )
.relationship( worked_on )
.exactlyOneRelationship() )
.update( dataset );

users.approxPercentage( 30 ).createRelationshipsTo(
getExisting( allProjects )
.numberOfTargetNodes( minMax( 1, 2 ) )
.relationship( worked_on )
.relationshipConstraints( RelationshipUniqueness.SINGLE_DIRECTION ) )
.updateNoReturn( dataset );

dataset.end();

db.shutdown();
db = new GraphDatabaseFactory().newEmbeddedDatabase(SocialNetwork);
GraphStatistics statistics = GraphStatistics.create( db, "Social Network" );
new AsciiDocFormatter( SysOutLog.INSTANCE ).describe( statistics );

db.shutdown();

System.out.println( "Number of topics: " + topics.size() );
}
}