merged conflicts

Author: mgobec

.travis.yml

@@ -1,5 +1,6 @@
 language: java
-sudo: false
+sudo: true
+
 jdk:
 - oraclejdk8
 install: mvn install -Dgpg.skip=true

README.md

@@ -27,14 +27,24 @@ This means that if you are using the driver version 2.1.9 you can add this depen
 or whatever the latest build version of the migration tool is. Check the version at the [Maven repository](http://mvnrepository.com/artifact/io.smartcat/cassandra-migration-tool).
 
 # Examples
-In tests [MigrationEngineTest](src/test/java/io/smartcat/migration/MigrationEngineTest.java) can serve as an example of one use case which this tool can cover. There is already defined table in cassandra DB with production data in it and requirement is to add column and populate it with data.
+We have two test cases which explain common problems which migration tool can solve. This is only subset of use cases but we think these are most frequent once:
 
-The initial table is simple (can be found in [init db.cql file](src/test/resources/db.cql)) and we use migration classes to do the following:
+**First use case** touches problem of adding new field and populating historic data with value. Cassandra does not have DDL `default` attribute, so you must populate data on application level. [MigrationEngineBooksTest](src/test/java/io/smartcat/migration/MigrationEngineBooksTest.java) can serve as an example of that use case which this tool can cover.
+
+The initial table is simple (can be found in [books.cql file](src/test/resources/books.cql)) and we use migration classes to do the following:
 
 1. Populate data initially with first `data` migration [InsertBooksMigration](src/test/java/io/smartcat/migration/migrations/data/InsertBooksMigration.java)
 2. Add `genre` column with `schema` migration [AddBookGenreFieldMigration](src/test/java/io/smartcat/migration/migrations/schema/AddBookGenreFieldMigration.java)
 3. Populate `genre` column with second `data` migration [AddGenreMigration](src/test/java/io/smartcat/migration/migrations/data/AddGenreMigration.java)
 
+**Second use case** touches problem of query based modeling. Cassandra has good performance because you model your data as you will query it. Often after initial modeling you have request to read it based on different criteria. In Cassandra you do this with another table which is optimized for new requirements. You need to populate this new table with existing data and you can solve this with migration tool. [MigrationEngineItemsTest](src/test/java/io/smartcat/migration/MigrationEngineItemsTest.java) can serve as an example of that use case which this tool can cover.
+
+The initial table is simple (can be found in [items.cql file](src/test/resources/items.cql)) and we use migration classes to do the following:
+
+1. Populate data initially with first `data` migration [InsertInitialItemsMigration](src/test/java/io/smartcat/migration/migrations/data/InsertInitialItemsMigration.java)
+2. Add `items_by_number_external_id` table with `schema` migration [CreateItemByNumberAndExternalIdMigration](src/test/java/io/smartcat/migration/migrations/schema/CreateItemByNumberAndExternalIdMigration.java)
+3. Populate `items_by_number_external_id` table with second `data` migration [PopulateItemByNumberAndExternalIdMigration](src/test/java/io/smartcat/migration/migrations/data/PopulateItemByNumberAndExternalIdMigration.java)
+
 # Schema agreement
 When executing schema migrations it is necessary to wait for cluster to propagate schema on all nodes. Schema agreement is implemented based on this [fix](https://datastax-oss.atlassian.net/browse/JAVA-669) and is exposed through [Migration](src/main/java/io/smartcat/migration/Migration.java) abstract class.
 To execute a statement with schema agreement you can use `executeWithSchemaAgreement` method.

pom.xml

@@ -16,6 +16,7 @@
         <guava.version>16.0.1</guava.version>
         <slf4j.version>1.7.7</slf4j.version>
         <junit.version>4.9</junit.version>
+        <mockito.version>2.0.52-beta</mockito.version>
     </properties>
 
     <dependencies>
@@ -35,6 +36,11 @@
             <version>${guava.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>${mockito.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.cassandraunit</groupId>
             <artifactId>cassandra-unit</artifactId>

src/main/java/io/smartcat/migration/exceptions/MigrationException.java

@@ -2,9 +2,9 @@
 
 public class MigrationException extends Exception {
 
-	private static final long serialVersionUID = 939170349798471411L;
+    private static final long serialVersionUID = 939170349798471411L;
 
-	public MigrationException(final String message) {
+    public MigrationException(final String message) {
         super(message);
     }
 

src/test/java/io/smartcat/migration/BaseTest.java

@@ -0,0 +1,28 @@
+package io.smartcat.migration;
+
+import com.datastax.driver.core.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class BaseTest {
+
+    public void truncateTables(final String keyspace, final Session session) {
+        for (final String table : tables(keyspace, session)) {
+            session.execute(String.format("TRUNCATE %s.%s;", keyspace, table));
+        }
+    }
+
+    private List<String> tables(final String keyspace, final Session session) {
+        final List<String> tables = new ArrayList<>();
+        final Cluster cluster = session.getCluster();
+        final Metadata meta = cluster.getMetadata();
+        final KeyspaceMetadata keyspaceMeta = meta.getKeyspace(keyspace);
+        for (final TableMetadata tableMeta : keyspaceMeta.getTables()) {
+            tables.add(tableMeta.getName());
+        }
+
+        return tables;
+    }
+
+}

src/test/java/io/smartcat/migration/CassandraMetadataAnalyzer.java

@@ -0,0 +1,34 @@
+package io.smartcat.migration;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.Iterables.tryFind;
+
+import com.datastax.driver.core.ColumnMetadata;
+import com.datastax.driver.core.KeyspaceMetadata;
+import com.datastax.driver.core.Metadata;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.TableMetadata;
+import com.google.common.base.Optional;
+
+public class CassandraMetadataAnalyzer {
+
+    private Session session;
+
+    public CassandraMetadataAnalyzer(Session session) {
+        checkNotNull(session, "Session cannot be null");
+        checkNotNull(session.getLoggedKeyspace(), "Session must be logged into a keyspace");
+        this.session = session;
+    }
+
+    public boolean columnExistInTable(String columnName, String tableName) {
+        TableMetadata table = getTableMetadata(this.session, tableName);
+        Optional<ColumnMetadata> column = tryFind(table.getColumns(), new ColumnNameMatcher(columnName));
+        return column.isPresent();
+    }
+
+    private static TableMetadata getTableMetadata(Session session, String tableName) {
+        Metadata metadata = session.getCluster().getMetadata();
+        KeyspaceMetadata keyspaceMetadata = metadata.getKeyspace(session.getLoggedKeyspace());
+        return keyspaceMetadata.getTable(tableName);
+    }
+}

src/test/java/io/smartcat/migration/CassandraVersionerTest.java

@@ -0,0 +1,85 @@
+package io.smartcat.migration;
+
+import static io.smartcat.migration.MigrationType.SCHEMA;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.stubbing.OngoingStubbing;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.Statement;
+
+import io.smartcat.migration.migrations.schema.AddBookGenreFieldMigration;
+
+public class CassandraVersionerTest {
+    private CassandraVersioner versioner;
+    private Session session;
+    private ResultSet versionResultSet;
+
+    @Before
+    public void setUp() throws Exception {
+        session = mock(Session.class);
+        versioner = new CassandraVersioner(session);
+        versionResultSet = mock(ResultSet.class);
+    }
+
+    @Test
+    public void whenSchemaVersionTableIsEmptyThenCurrentVersionShouldBe0() throws Exception {
+        expectRetrieveEmptyCurrentVersion();
+
+        int currentVersion = versioner.getCurrentVersion(SCHEMA);
+
+        assertThat(currentVersion, is(0));
+    }
+
+    @Test
+    public void whenSchemaVersionTableIsNotEmptyThenCurrentVersionShouldBeRetrievedFromTheTable() throws Exception {
+        int expectedVersion = 1;
+
+        expectRetrieveCurrentVersion(expectedVersion);
+
+        int currentVersion = versioner.getCurrentVersion(SCHEMA);
+
+        assertThat(currentVersion, is(expectedVersion));
+    }
+
+    @Test
+    public void updateVersionSucess() throws Exception {
+        versioner.updateVersion(new AddBookGenreFieldMigration(1));
+    }
+
+    private void expectRetrieveEmptyCurrentVersion() {
+        expectRetrieveVersionResultSetWithRow(null);
+    }
+
+    private void expectRetrieveCurrentVersion(int expectedVersion) {
+        Row row = expectRowWithVersion(expectedVersion);
+        expectRetrieveVersionResultSetWithRow(row);
+    }
+
+    private void expectRetrieveVersionResultSetWithRow(Row row) {
+        whenSessionExecuteQuery().thenReturn(versionResultSet);
+        whenRetrieveRowFromVersionResultSet().thenReturn(row);
+    }
+
+    private Row expectRowWithVersion(int version) {
+        Row row = mock(Row.class);
+        when(row.getInt("version")).thenReturn(version);
+        return row;
+    }
+
+    private OngoingStubbing<ResultSet> whenSessionExecuteQuery() {
+        return when(session.execute(Mockito.any(Statement.class)));
+    }
+
+    private OngoingStubbing<Row> whenRetrieveRowFromVersionResultSet() {
+        return when(versionResultSet.one());
+    }
+}

src/test/java/io/smartcat/migration/ColumnNameMatcher.java

@@ -0,0 +1,16 @@
+package io.smartcat.migration;
+
+import com.datastax.driver.core.ColumnMetadata;
+import com.google.common.base.Predicate;
+
+public class ColumnNameMatcher implements Predicate<ColumnMetadata> {
+    private String columnName;
+
+    public ColumnNameMatcher(String columnName) {
+        this.columnName = columnName;
+    }
+
+    public boolean apply(ColumnMetadata column) {
+        return column.getName().equals(columnName);
+    }
+}

src/test/java/io/smartcat/migration/MigrationEngineTest.java renamed to src/test/java/io/smartcat/migration/MigrationEngineBooksTest.java

@@ -2,9 +2,6 @@
 
 import static junit.framework.Assert.assertEquals;
 
-import java.util.HashMap;
-import java.util.Map;
-
 import org.cassandraunit.CQLDataLoader;
 import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
 import org.cassandraunit.utils.EmbeddedCassandraServerHelper;
@@ -25,14 +22,14 @@
 import io.smartcat.migration.migrations.data.InsertBooksMigration;
 import io.smartcat.migration.migrations.schema.AddBookGenreFieldMigration;
 
-public class MigrationEngineTest {
+public class MigrationEngineBooksTest extends BaseTest {
 
-    private static final Logger LOGGER = LoggerFactory.getLogger(MigrationEngineTest.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(MigrationEngineBooksTest.class);
 
     private static final String CONTACT_POINT = "localhost";
     private static final int PORT = 9142;
-    private static final String KEYSPACE = "migration_test";
-    private static final String CQL = "db.cql";
+    private static final String KEYSPACE = "migration_test_books";
+    private static final String CQL = "books.cql";
 
     private static Session session;
     private static Cluster cluster;

src/test/java/io/smartcat/migration/MigrationEngineItemsTest.java

@@ -0,0 +1,91 @@
+package io.smartcat.migration;
+
+import static junit.framework.Assert.assertEquals;
+
+import com.datastax.driver.core.*;
+import com.datastax.driver.core.querybuilder.QueryBuilder;
+import io.smartcat.migration.migrations.data.InsertInitialItemsMigration;
+import io.smartcat.migration.migrations.data.PopulateItemByNumberAndExternalIdMigration;
+import io.smartcat.migration.migrations.schema.CreateItemByNumberAndExternalIdMigration;
+import org.cassandraunit.CQLDataLoader;
+import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
+import org.cassandraunit.utils.EmbeddedCassandraServerHelper;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class MigrationEngineItemsTest extends BaseTest {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(MigrationEngineItemsTest.class);
+
+    private static final String CONTACT_POINT = "localhost";
+    private static final int PORT = 9142;
+    private static final String KEYSPACE = "migration_test_items";
+    private static final String CQL = "items.cql";
+
+    private static Session session;
+    private static Cluster cluster;
+
+    @BeforeClass
+    public static void init() throws Exception {
+        LOGGER.info("Starting embedded cassandra server");
+        EmbeddedCassandraServerHelper.startEmbeddedCassandra("another-cassandra.yaml");
+
+        LOGGER.info("Connect to embedded db");
+        cluster = Cluster.builder().addContactPoints(CONTACT_POINT).withPort(PORT).build();
+        session = cluster.connect();
+
+        LOGGER.info("Initialize keyspace");
+        final CQLDataLoader cqlDataLoader = new CQLDataLoader(session);
+        cqlDataLoader.load(new ClassPathCQLDataSet(CQL, false, true, KEYSPACE));
+    }
+
+    @After
+    public void cleanUp() {
+        truncateTables(KEYSPACE, session);
+    }
+
+    @AfterClass
+    public static void tearDown() {
+        if (cluster != null) {
+            cluster.close();
+            cluster = null;
+        }
+    }
+
+    @Test
+    public void initial_insert_test() {
+        final int count = 100;
+
+        final MigrationResources resources = new MigrationResources();
+        resources.addMigration(new InsertInitialItemsMigration(count, 1));
+        final boolean result = MigrationEngine.withSession(session).migrate(resources);
+
+        assertEquals(true, result);
+
+        final List<Row> rows = session.execute(QueryBuilder.select().from("items_by_id")).all();
+        assertEquals(count, rows.size());
+    }
+
+    @Test
+    public void test_migrations() {
+        final int count = 100;
+
+        final MigrationResources resources = new MigrationResources();
+        resources.addMigration(new InsertInitialItemsMigration(count, 1));
+        resources.addMigration(new CreateItemByNumberAndExternalIdMigration(1));
+        resources.addMigration(new PopulateItemByNumberAndExternalIdMigration(2));
+        final boolean result = MigrationEngine.withSession(session).migrate(resources);
+
+        assertEquals(true, result);
+
+        final List<Row> rows = session.execute(QueryBuilder.select().from("items_by_number_external_id")).all();
+        assertEquals(count, rows.size());
+    }
+
+}