From 0162c1b520cb6b48092536397774f15db9732a94 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 2 Sep 2015 08:41:41 -0700 Subject: [PATCH 1/9] Started on update table stuff --- .../agents/sync/jobs/ProcessorResult.java | 82 +++++++++++++++++ .../jobs/UpdateTableConnectionFactory.java | 88 +++++++++++++++++++ .../agents/sync/jobs/UpdateTablePoller.java | 50 +++++++++++ .../spring/percero-spring-config.xml | 1 + 4 files changed, 221 insertions(+) create mode 100644 src/main/java/com/percero/agents/sync/jobs/ProcessorResult.java create mode 100644 src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java create mode 100644 src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java diff --git a/src/main/java/com/percero/agents/sync/jobs/ProcessorResult.java b/src/main/java/com/percero/agents/sync/jobs/ProcessorResult.java new file mode 100644 index 0000000..7f577c9 --- /dev/null +++ b/src/main/java/com/percero/agents/sync/jobs/ProcessorResult.java @@ -0,0 +1,82 @@ +package com.percero.agents.sync.jobs; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by jonnysamps on 8/31/15. + */ +public class ProcessorResult { + private int total; + private int numFailed; + + private boolean success = true; + private Map details = new HashMap(); + private Map> failures = new HashMap>(); + + public int getTotal() { + return total; + } + + public int getNumFailed(){ + return numFailed; + } + + public boolean isSuccess() { + return success; + } + + public void setSuccess(boolean success) { + this.success = success; + } + + public Map getDetails() { + return details; + } + + public void addResult(String type, boolean wasSuccess, String message){ + total++; + if(!wasSuccess) { + success = false; + numFailed++; + if(!failures.containsKey(type)) failures.put(type, new ArrayList()); + failures.get(type).add(message); + } + else{ + if(!details.containsKey(type)) details.put(type, 0); + details.put(type, details.get(type)+1); + } + } + + /** + * @see ProcessorResult.addResult(String, boolean, String) + * @param type + */ + public void addResult(String type){ + this.addResult(type, true, null); + } + + /** + * Some niceties for logging + * @return + */ + public String toString(){ + StringBuilder sb = new StringBuilder(); + sb.append("Overall Success :"); sb.append(isSuccess()); sb.append("\n"); + sb.append("Total results :"); sb.append(getTotal()); sb.append("\n"); + sb.append("Total failures :"); sb.append(getNumFailed()); sb.append("\n"); + sb.append("Failures by type:"); sb.append("\n"); + for(String key : failures.keySet()){ + sb.append(" "); sb.append(key); sb.append(":"); sb.append("\n"); + int i = 1; + for(String message : failures.get(key)){ + sb.append(" "); sb.append(i); sb.append(": "); sb.append(message); sb.append("\n"); + i++; + } + } + + return sb.toString(); + } +} diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java new file mode 100644 index 0000000..747b09b --- /dev/null +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java @@ -0,0 +1,88 @@ +package com.percero.agents.sync.jobs; + +import com.mchange.v2.c3p0.ComboPooledDataSource; +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.beans.PropertyVetoException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +/** + * Created by jonnysamps on 9/2/15. + */ +@Component +public class UpdateTableConnectionFactory { + + private static Logger logger = Logger.getLogger(UpdateTableConnectionFactory.class); + + @Autowired + @Value("pf{updateTable.driverClassName:com.mysql.jdbc.Driver") + private String driverClassName; + + @Autowired + @Value("pf{updateTable.username") + private String username; + + @Autowired + @Value("pf{updateTable.username") + private String password; + + @Autowired + @Value("pf{updateTable.jdbcUrl:jdbc:mysql://localhost/db") + private String jdbcUrl; + + private ComboPooledDataSource cpds; + @PostConstruct + public void init() throws PropertyVetoException{ + try { + cpds = new ComboPooledDataSource(); + cpds.setDriverClass(driverClassName); //loads the jdbc driver + cpds.setJdbcUrl(jdbcUrl); + cpds.setUser(username); + cpds.setPassword(password); + +// the settings below are optional -- c3p0 can work with defaults + cpds.setMinPoolSize(5); + cpds.setAcquireIncrement(5); + cpds.setMaxPoolSize(20); + + }catch(PropertyVetoException pve){ + logger.error(pve.getMessage(), pve); + throw pve; + } + } + + public Connection getConnection() throws SQLException{ + try{ + return cpds.getConnection(); + }catch(SQLException e){ + logger.error(e.getMessage(), e); + throw e; + } + } + + public static void main(String[] args) throws Exception{ + UpdateTableConnectionFactory cf = new UpdateTableConnectionFactory(); + cf.driverClassName = "com.mysql.jdbc.Driver"; + cf.jdbcUrl = "jdbc:mysql://localhost/test"; + cf.username = "root"; + cf.password = "root"; + cf.init(); + + Connection c = cf.getConnection(); + Statement stmt = c.createStatement(); + + + ResultSet rs = stmt.executeQuery("SELECT * FROM Account"); + while(rs.next()) { + logger.info("ID: " + rs.getInt("ID")); + logger.info("markedForRemoval: "+ rs.getDate("markedForRemoval")); + } + } +} diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java new file mode 100644 index 0000000..65e5e5e --- /dev/null +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java @@ -0,0 +1,50 @@ +package com.percero.agents.sync.jobs; + +import com.mchange.v2.c3p0.ComboPooledDataSource; +import org.apache.log4j.Logger; +import org.hibernate.SessionFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +/** + * Created by jonnysamps on 8/31/15. + */ +@Component +public class UpdateTablePoller { + + private static Logger logger = Logger.getLogger(UpdateTablePoller.class); + + private String[] tableNames = new String[0]; + @Autowired + @Value("$pf{updateTable.tableNames}") + public void setTableNames(String val){ + tableNames = val.split(","); + } + + + @Autowired + UpdateTableConnectionFactory connectionFactory; + + /** + * Run every minute + */ + @Scheduled(fixedDelay=6000, initialDelay=6000) + public void pollUpdateTables(){ + logger.info("Polling Update Tables..."); + for(String tableName : tableNames){ + UpdateTableProcessor processor = new UpdateTableProcessor(tableName, connectionFactory); + ProcessorResult result = processor.process(); + if(result.isSuccess()){ + logger.debug("Update table processor ("+tableName+") finished successfully. Total rows ("+result.getTotal()+")"); + } + else{ + logger.warn("Update table processor ("+tableName+") failed. Details:"); + logger.warn(result); + } + } + } + + +} diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index a7f0cff..6f3b4fb 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -44,6 +44,7 @@ + From 781b3701327a54fab9c54fff2e1a5a96aeff9e81 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Fri, 4 Sep 2015 15:11:57 -0700 Subject: [PATCH 2/9] Added some test infrastructure --- pom.xml | 16 + .../agents/sync/helpers/PostDeleteHelper.java | 33 +- .../jobs/UpdateTableConnectionFactory.java | 42 +-- .../agents/sync/jobs/UpdateTablePoller.java | 10 +- .../sync/jobs/UpdateTableProcessor.java | 257 ++++++++++++++ .../agents/sync/jobs/UpdateTableRow.java | 66 ++++ .../agents/sync/jobs/UpdateTableRowType.java | 8 + .../sync/jobs/UpdateTableProcessorTest.java | 18 + src/test/java/com/percero/example/Block.java | 86 +++++ src/test/java/com/percero/example/Email.java | 190 +++++++++++ .../percero/example/ExampleAccountHelper.java | 21 ++ .../example/ExampleChangeWatcherFactory.java | 10 + .../com/percero/example/ExampleManifest.java | 54 +++ .../percero/example/ExampleProcessHelper.java | 8 + src/test/java/com/percero/example/Person.java | 235 +++++++++++++ .../java/com/percero/example/PersonRole.java | 180 ++++++++++ src/test/resources/google/client_secrets.json | 13 + src/test/resources/google/privatekey.p12 | Bin 0 -> 2572 bytes .../resources/properties/test/env.properties | 68 ++++ .../spring/update_table_processor.xml | 319 ++++++++++++++++++ src/test/sql/example_schema.sql | 0 21 files changed, 1592 insertions(+), 42 deletions(-) create mode 100644 src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java create mode 100644 src/main/java/com/percero/agents/sync/jobs/UpdateTableRow.java create mode 100644 src/main/java/com/percero/agents/sync/jobs/UpdateTableRowType.java create mode 100644 src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java create mode 100644 src/test/java/com/percero/example/Block.java create mode 100644 src/test/java/com/percero/example/Email.java create mode 100644 src/test/java/com/percero/example/ExampleAccountHelper.java create mode 100644 src/test/java/com/percero/example/ExampleChangeWatcherFactory.java create mode 100644 src/test/java/com/percero/example/ExampleManifest.java create mode 100644 src/test/java/com/percero/example/ExampleProcessHelper.java create mode 100644 src/test/java/com/percero/example/Person.java create mode 100644 src/test/java/com/percero/example/PersonRole.java create mode 100644 src/test/resources/google/client_secrets.json create mode 100644 src/test/resources/google/privatekey.p12 create mode 100644 src/test/resources/properties/test/env.properties create mode 100644 src/test/resources/spring/update_table_processor.xml create mode 100644 src/test/sql/example_schema.sql diff --git a/pom.xml b/pom.xml index a67e2fe..f7fd561 100644 --- a/pom.xml +++ b/pom.xml @@ -293,6 +293,20 @@ + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.7 + 1.7 + + + + + com.springsource.repository.bundles.release @@ -315,4 +329,6 @@ http://maven.springframework.org/release + + \ No newline at end of file diff --git a/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java b/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java index abb1644..dfe3855 100644 --- a/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java +++ b/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java @@ -1,14 +1,5 @@ package com.percero.agents.sync.helpers; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import org.apache.log4j.Logger; -import org.codehaus.jackson.map.ObjectMapper; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - import com.percero.agents.sync.access.IAccessManager; import com.percero.agents.sync.services.IPushSyncHelper; import com.percero.agents.sync.services.ISyncAgentService; @@ -16,6 +7,14 @@ import com.percero.agents.sync.vo.PushDeleteResponse; import com.percero.agents.sync.vo.RemovedClassIDPair; import com.percero.framework.vo.IPerceroObject; +import org.apache.log4j.Logger; +import org.codehaus.jackson.map.ObjectMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; @Component public class PostDeleteHelper { @@ -63,25 +62,27 @@ public void postDeleteObject(IPerceroObject theObject, String pusherUserId, Stri log.debug("PostDeleteHelper for " + theObject.toString() + " from clientId " + (pusherClientId == null ? "NULL" : pusherClientId)); ClassIDPair pair = new ClassIDPair(theObject.getID(), theObject.getClass().getCanonicalName()); - + postDeleteObject(pair, pusherUserId, pusherClientId, pushToUser); + } + + public void postDeleteObject(ClassIDPair pair, String pusherUserId, String pusherClientId, boolean pushToUser) throws Exception { // Remove all UpdateJournals for the objects. accessManager.removeUpdateJournalsByObject(pair); // Notify interested users that this object has been deleted. RemovedClassIDPair removedPair = new RemovedClassIDPair(); - removedPair.setClassName(theObject.getClass().getName()); - removedPair.setID(theObject.getID()); + removedPair.setClassName(pair.getClassName()); + removedPair.setID(pair.getID()); + + List clientIds = accessManager.getObjectAccessJournals(pair.getClassName(), pair.getID()); - List clientIds = accessManager.getObjectAccessJournals(theObject - .getClass().getName(), theObject.getID()); - // Now remove the AccessJournals for this Object. accessManager.removeAccessJournalsByObject(pair); // Now remove the ObjectModJournals for this Object. accessManager.removeObjectModJournalsByObject(pair); // Now remove the HistoricalObjects for this Object. accessManager.removeHistoricalObjectsByObject(pair); - + // Now run past the ChangeWatcher. accessManager.checkChangeWatchers(pair, null, null); // Remove ChangeWatchers associated with this object. diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java index 747b09b..8ef0bd9 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java @@ -22,20 +22,32 @@ public class UpdateTableConnectionFactory { private static Logger logger = Logger.getLogger(UpdateTableConnectionFactory.class); @Autowired - @Value("pf{updateTable.driverClassName:com.mysql.jdbc.Driver") + @Value("pf{updateTable.driverClassName:com.mysql.jdbc.Driver}") private String driverClassName; + public void setDriverClassName(String val){ + this.driverClassName = val; + } @Autowired - @Value("pf{updateTable.username") + @Value("pf{updateTable.username}") private String username; + public void setUsername(String val){ + this.username = val; + } @Autowired - @Value("pf{updateTable.username") + @Value("pf{updateTable.username}") private String password; + public void setPassword(String val){ + this.password = val; + } @Autowired - @Value("pf{updateTable.jdbcUrl:jdbc:mysql://localhost/db") + @Value("pf{updateTable.jdbcUrl:jdbc:mysql://localhost/db}") private String jdbcUrl; + public void setJdbcUrl(String val){ + this.jdbcUrl = val; + } private ComboPooledDataSource cpds; @PostConstruct @@ -47,10 +59,9 @@ public void init() throws PropertyVetoException{ cpds.setUser(username); cpds.setPassword(password); -// the settings below are optional -- c3p0 can work with defaults + // the settings below are optional -- c3p0 can work with defaults cpds.setMinPoolSize(5); cpds.setAcquireIncrement(5); - cpds.setMaxPoolSize(20); }catch(PropertyVetoException pve){ logger.error(pve.getMessage(), pve); @@ -66,23 +77,4 @@ public Connection getConnection() throws SQLException{ throw e; } } - - public static void main(String[] args) throws Exception{ - UpdateTableConnectionFactory cf = new UpdateTableConnectionFactory(); - cf.driverClassName = "com.mysql.jdbc.Driver"; - cf.jdbcUrl = "jdbc:mysql://localhost/test"; - cf.username = "root"; - cf.password = "root"; - cf.init(); - - Connection c = cf.getConnection(); - Statement stmt = c.createStatement(); - - - ResultSet rs = stmt.executeQuery("SELECT * FROM Account"); - while(rs.next()) { - logger.info("ID: " + rs.getInt("ID")); - logger.info("markedForRemoval: "+ rs.getDate("markedForRemoval")); - } - } } diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java index 65e5e5e..31ead1b 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java @@ -1,6 +1,8 @@ package com.percero.agents.sync.jobs; import com.mchange.v2.c3p0.ComboPooledDataSource; +import com.percero.agents.sync.helpers.PostDeleteHelper; +import com.percero.framework.bl.IManifest; import org.apache.log4j.Logger; import org.hibernate.SessionFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -27,6 +29,12 @@ public void setTableNames(String val){ @Autowired UpdateTableConnectionFactory connectionFactory; + @Autowired + IManifest manifest; + + @Autowired + PostDeleteHelper postDeleteHelper; + /** * Run every minute */ @@ -34,7 +42,7 @@ public void setTableNames(String val){ public void pollUpdateTables(){ logger.info("Polling Update Tables..."); for(String tableName : tableNames){ - UpdateTableProcessor processor = new UpdateTableProcessor(tableName, connectionFactory); + UpdateTableProcessor processor = new UpdateTableProcessor(tableName, connectionFactory, manifest, postDeleteHelper); ProcessorResult result = processor.process(); if(result.isSuccess()){ logger.debug("Update table processor ("+tableName+") finished successfully. Total rows ("+result.getTotal()+")"); diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java new file mode 100644 index 0000000..e3e428a --- /dev/null +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -0,0 +1,257 @@ +package com.percero.agents.sync.jobs; + +import com.percero.agents.sync.helpers.PostDeleteHelper; +import com.percero.framework.bl.IManifest; +import org.apache.log4j.Logger; +import org.joda.time.DateTime; + +import javax.persistence.Table; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Date; +import java.util.Random; + +/** + * Responsible for querying an update table and processing the rows + * Created by Jonathan Samples on 8/31/15. + */ +public class UpdateTableProcessor { + + private static Logger logger = Logger.getLogger(UpdateTableProcessor.class); + + private static final int EXPIRATION_TIME = 1000*60*30; // 30 minutes + private String tableName; + private UpdateTableConnectionFactory connectionFactory; + private PostDeleteHelper postDeleteHelper; + private IManifest manifest; + + + public UpdateTableProcessor(String tableName, + UpdateTableConnectionFactory connectionFactory, + IManifest manifest, + PostDeleteHelper postDeleteHelper){ + this.tableName = tableName; + this.connectionFactory = connectionFactory; + this.postDeleteHelper = postDeleteHelper; + this.manifest = manifest; + } + + /** + * Update table schema looks like this + * + * `ID` int(11) NOT NULL AUTO_INCREMENT, + * `tableName` varchar(255) DEFAULT NULL, + * `rowID` varchar(255) DEFAULT NULL, + * `type` enum('INSERT','UPDATE','DELETE') NOT NULL DEFAULT 'UPDATE', + * `lockID` int(11) DEFAULT NULL, + * `lockDate` datetime DEFAULT NULL, + * `timestamp` timestamp DEFAULT CURRENT_TIMESTAMP + * + * @return + */ + public ProcessorResult process(){ + ProcessorResult result = new ProcessorResult(); + + while(true) { + UpdateTableRow row = getRow(); + if(row == null) break; + + if(processRow(row)) { + result.addResult(row.getType().toString()); + deleteRow(row); + } + else { + result.addResult(row.getType().toString(), false, ""); + } + + } + + return result; + } + + private boolean processRow(UpdateTableRow row){ + boolean result = true; + + if(row.getRowId() != null) + switch (row.getType()){ + case DELETE: + result = processDeleteSingle(row); + break; + case UPDATE: + result = processUpdateSingle(row); + break; + case INSERT: + result = processInsertSingle(row); + break; + default: // Don't know how to process + result = true; + break; + } + else + switch (row.getType()){ + case DELETE: + result = processDeleteTable(row); + break; + case UPDATE: + result = processUpdateTable(row); + break; + case INSERT: + result = processInsertTable(row); + break; + default: // Don't know how to process + result = true; + break; + } + + return result; + } + + /** + * Process a single record delete + * @param row + * @return + */ + private boolean processDeleteSingle(UpdateTableRow row){ + + return true; + } + + /** + * Process a single record update + * @param row + * @return + */ + private boolean processUpdateSingle(UpdateTableRow row){ + return true; + } + + /** + * process a single record insert + * @param row + * @return + */ + private boolean processInsertSingle(UpdateTableRow row){ + return true; + } + + /** + * process a whole table with deletes + * @param row + * @return + */ + private boolean processDeleteTable(UpdateTableRow row){ + return true; + } + + /** + * Process a whole table with updates + * @param row + * @return + */ + private boolean processUpdateTable(UpdateTableRow row){ + return true; + } + + /** + * Process a whole table with inserts + * @param row + * @return + */ + private boolean processInsertTable(UpdateTableRow row){ + return true; + } + + /** + * Pulls a row off the update table and locks it so that other + * processors don't duplicate the work + * @return + */ + private UpdateTableRow getRow(){ + UpdateTableRow row = null; + + try(Connection conn = connectionFactory.getConnection()){ + + Random rand = new Random(); + + int lockId = rand.nextInt(); + long now = System.currentTimeMillis(); + + DateTime expireThreshold = new DateTime(now - EXPIRATION_TIME); + + /** + * First try to lock a row + */ + String sql = "update :tableName set lockId=:lockId, lockDate=NOW() where lockId is null or lockDate < :expireThreshold limit 1"; + sql.replace(":tableName", tableName); + sql.replace(":lockId", lockId+""); + sql.replace(":expireThreshold", expireThreshold.toString("Y-MM-dd HH:mm:ss")); + + Statement statement; + + statement = conn.createStatement(); + int numUpdated = statement.executeUpdate(sql); + + // Found a row to process + if(numUpdated > 0){ + sql = "select * from :tableName where lockId=:lockId limit 1"; + sql.replace(":tableName", tableName); + sql.replace(":lockId", lockId+""); + statement = conn.createStatement(); + ResultSet rs = statement.executeQuery(sql); + + // If got a row back + if(rs.next()){ + row = UpdateTableRow.fromResultSet(rs); + rs.close(); + } + else{ + logger.warn("Locked a row but couldn't retrieve"); + } + } + + } catch(SQLException e){ + logger.warn(e.getMessage(), e); + } + + return row; + } + + /** + * Deletes the row + * @param row + */ + private void deleteRow(UpdateTableRow row){ + try(Connection conn = connectionFactory.getConnection()){ + String sql = "delete from :tableName where ID=:ID"; + sql.replace(":ID", row.getID()+""); + Statement statement = conn.createStatement(); + int numUpdated = statement.executeUpdate(sql); + if(numUpdated != 1){ + logger.warn("Expected to delete 1, instead "+numUpdated); + } + }catch(SQLException e){ + logger.warn(e.getMessage(), e); + } + } + + public Class getClassForTableName(String tableName){ + Class result = null; + for(Class c : manifest.getClassList()){ + Table table = (Table) c.getAnnotation(Table.class); + if(tableName.equals(table.name())) { + result = c; + break; + } + } + + return result; + } + + public static void main(String[] args){ + DateTime time = new DateTime(); + logger.info(time.toString("Y-MM-dd HH:mm:ss")); + } + +} diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRow.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRow.java new file mode 100644 index 0000000..aa44712 --- /dev/null +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRow.java @@ -0,0 +1,66 @@ +package com.percero.agents.sync.jobs; + +import org.apache.log4j.Logger; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Date; + +/** + * Class to represent a row from the an update table + * Created by jonnysamps on 9/3/15. + */ +public class UpdateTableRow { + + private static Logger logger = Logger.getLogger(UpdateTableRow.class); + + public int ID; + public String tableName; + public String rowId; + public int lockId; + public Date lockDate; + public Date timestamp; + public UpdateTableRowType type; + + public Date getTimestamp() { + return timestamp; + } + + public int getID() { + return ID; + } + + public String getTableName() { + return tableName; + } + + public String getRowId() { + return rowId; + } + + public int getLockId() { + return lockId; + } + + public Date getLockDate() { + return lockDate; + } + + public UpdateTableRowType getType() { + return type; + } + + + public static UpdateTableRow fromResultSet(ResultSet resultSet) throws SQLException{ + UpdateTableRow row = new UpdateTableRow(); + row.ID = resultSet.getInt("ID"); + row.tableName = resultSet.getString("tableName"); + row.rowId = resultSet.getString("rowId"); + row.lockId = resultSet.getInt("lockId"); + row.lockDate = resultSet.getDate("lockDate"); + row.type = UpdateTableRowType.valueOf(resultSet.getString("type")); + row.timestamp = resultSet.getDate("timestamp"); + + return row; + } +} diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRowType.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRowType.java new file mode 100644 index 0000000..e2318a8 --- /dev/null +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRowType.java @@ -0,0 +1,8 @@ +package com.percero.agents.sync.jobs; + +/** + * Created by jonnysamps on 9/3/15. + */ +public enum UpdateTableRowType { + UPDATE, INSERT, DELETE +} diff --git a/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java new file mode 100644 index 0000000..3f25143 --- /dev/null +++ b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java @@ -0,0 +1,18 @@ +package com.percero.agents.sync.jobs; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +/** + * Created by Jonathan Samples on 9/4/15. + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(locations = { "classpath:spring/update_table_processor.xml" }) +public class UpdateTableProcessorTest { + @Test + public void test(){ + System.out.println("A Test ran"); + } +} diff --git a/src/test/java/com/percero/example/Block.java b/src/test/java/com/percero/example/Block.java new file mode 100644 index 0000000..925bcbe --- /dev/null +++ b/src/test/java/com/percero/example/Block.java @@ -0,0 +1,86 @@ +package com.percero.example; + +import com.google.gson.JsonObject; +import com.percero.agents.sync.metadata.annotations.Externalize; +import com.percero.agents.sync.vo.BaseDataObject; +import com.percero.serial.JsonUtils; +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.annotate.JsonProperty; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import java.io.IOException; +import java.io.Serializable; + +@Entity +public class Block extends BaseDataObject implements Serializable{ + + ////////////////////////////////////////////////////// + // Properties + ////////////////////////////////////////////////////// + @Id + @Externalize + @Column(unique=true,name="ID") + private String ID; + @JsonProperty(value="ID") + public String getID() { + return this.ID; + } + @JsonProperty(value="ID") + public void setID(String value) { + this.ID = value; + } + + @Column + @Externalize + private String color; + public String getColor() { + return this.color; + } + public void setColor(String value) + { + this.color = value; + } + + ////////////////////////////////////////////////////// + // JSON + ////////////////////////////////////////////////////// + @Override + public String retrieveJson(ObjectMapper objectMapper) { + String objectJson = super.retrieveJson(objectMapper); + + // Properties + objectJson += ",\"color\":"; + if (getColor() == null) + objectJson += "null"; + else { + if (objectMapper == null) + objectMapper = new ObjectMapper(); + try { + objectJson += objectMapper.writeValueAsString(getColor()); + } catch (JsonGenerationException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (JsonMappingException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (IOException e) { + objectJson += "null"; + e.printStackTrace(); + } + } + + return objectJson; + } + + @Override + protected void fromJson(JsonObject jsonObject) { + super.fromJson(jsonObject); + + // Properties + setColor(JsonUtils.getJsonString(jsonObject, "color")); + } +} diff --git a/src/test/java/com/percero/example/Email.java b/src/test/java/com/percero/example/Email.java new file mode 100644 index 0000000..84ca055 --- /dev/null +++ b/src/test/java/com/percero/example/Email.java @@ -0,0 +1,190 @@ +package com.percero.example; + +import com.google.gson.JsonObject; +import com.percero.agents.auth.vo.IUserIdentifier; +import com.percero.agents.sync.metadata.annotations.*; +import com.percero.agents.sync.vo.BaseDataObject; +import com.percero.serial.BDODeserializer; +import com.percero.serial.BDOSerializer; +import com.percero.serial.JsonUtils; +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.annotate.JsonProperty; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.annotate.JsonDeserialize; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import javax.persistence.*; +import java.io.IOException; +import java.io.Serializable; +import java.util.Date; + + +@EntityInterface(interfaceClass=IUserIdentifier.class) +@Entity +public class Email extends BaseDataObject implements Serializable, IUserIdentifier +{ + ////////////////////////////////////////////////////// + // VERSION + ////////////////////////////////////////////////////// + @Override + public String classVersion() { + return "0.0.0"; + } + + + ////////////////////////////////////////////////////// + // ID + ////////////////////////////////////////////////////// + @Id + @Externalize + @Column(unique=true,name="ID") + private String ID; + @JsonProperty(value="ID") + public String getID() { + return this.ID; + } + @JsonProperty(value="ID") + public void setID(String value) { + this.ID = value; + } + + ////////////////////////////////////////////////////// + // Properties + ////////////////////////////////////////////////////// + @Column + @Externalize + private Date dateCreated; + public Date getDateCreated() { + return this.dateCreated; + } + public void setDateCreated(Date value) + { + this.dateCreated = value; + } + + @Column + @Externalize + private Date dateModified; + public Date getDateModified() { + return this.dateModified; + } + public void setDateModified(Date value) + { + this.dateModified = value; + } + + @Column + @PropertyInterface(entityInterfaceClass=IUserIdentifier.class, propertyName="userIdentifier", + params={@PropertyInterfaceParam(name="paradigm", value="email")} + ) + @Externalize + private String value; + public String getValue() { + return this.value; + } + public void setValue(String value) + { + this.value = value; + } + + + ////////////////////////////////////////////////////// + // Source Relationships + ////////////////////////////////////////////////////// + @RelationshipInterface(entityInterfaceClass=IUserIdentifier.class, sourceVarName="userAnchor") + @Externalize + @JsonSerialize(using=BDOSerializer.class) + @JsonDeserialize(using=BDODeserializer.class) + @JoinColumn(name="user_ID") + @org.hibernate.annotations.ForeignKey(name="FK_User_user_TO_Email") + @ManyToOne(fetch=FetchType.LAZY, optional=false) + private Person person; + public Person getPerson() { + return this.person; + } + public void setUser(Person value) { + this.person = value; + } + + ////////////////////////////////////////////////////// + // Target Relationships + ////////////////////////////////////////////////////// + + + + ////////////////////////////////////////////////////// + // JSON + ////////////////////////////////////////////////////// + @Override + public String retrieveJson(ObjectMapper objectMapper) { + String objectJson = super.retrieveJson(objectMapper); + + // Properties + objectJson += ",\"dateCreated\":"; + if (getDateCreated() == null) + objectJson += "null"; + else { + objectJson += getDateCreated().getTime(); + } + + objectJson += ",\"dateModified\":"; + if (getDateModified() == null) + objectJson += "null"; + else { + objectJson += getDateModified().getTime(); + } + + objectJson += ",\"value\":"; + if (getValue() == null) + objectJson += "null"; + else { + if (objectMapper == null) + objectMapper = new ObjectMapper(); + try { + objectJson += objectMapper.writeValueAsString(getValue()); + } catch (JsonGenerationException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (JsonMappingException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (IOException e) { + objectJson += "null"; + e.printStackTrace(); + } + } + + // Source Relationships + objectJson += ",\"person\":"; + if (getPerson() == null) + objectJson += "null"; + else { + try { + objectJson += ((BaseDataObject) getPerson()).toEmbeddedJson(); + } catch(Exception e) { + objectJson += "null"; + } + } + objectJson += ""; + + // Target Relationships + + return objectJson; + } + + @Override + protected void fromJson(JsonObject jsonObject) { + super.fromJson(jsonObject); + + // Properties + setDateCreated(JsonUtils.getJsonDate(jsonObject, "dateCreated")); + setDateModified(JsonUtils.getJsonDate(jsonObject, "dateModified")); + setValue(JsonUtils.getJsonString(jsonObject, "value")); + + // Source Relationships + this.person = JsonUtils.getJsonPerceroObject(jsonObject, "person"); + + // Target Relationships + } +} diff --git a/src/test/java/com/percero/example/ExampleAccountHelper.java b/src/test/java/com/percero/example/ExampleAccountHelper.java new file mode 100644 index 0000000..c8eb060 --- /dev/null +++ b/src/test/java/com/percero/example/ExampleAccountHelper.java @@ -0,0 +1,21 @@ +package com.percero.example; + +import com.percero.agents.auth.helpers.AccountHelper; +import com.percero.framework.bl.ManifestHelper; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +@Component +public class ExampleAccountHelper extends AccountHelper { + + private static final Logger log = Logger.getLogger(ExampleAccountHelper.class); + + public static final String ROLE_ADMIN = "ADMIN"; + public static final String ROLE_BASIC = "BASIC"; + + public ExampleAccountHelper() { + super(); + manifest = new ExampleManifest(); + ManifestHelper.setManifest(manifest); + } +} diff --git a/src/test/java/com/percero/example/ExampleChangeWatcherFactory.java b/src/test/java/com/percero/example/ExampleChangeWatcherFactory.java new file mode 100644 index 0000000..929d2d9 --- /dev/null +++ b/src/test/java/com/percero/example/ExampleChangeWatcherFactory.java @@ -0,0 +1,10 @@ +package com.percero.example; + +import com.percero.agents.sync.cw.ChangeWatcherHelperFactory; +import org.springframework.stereotype.Component; + + +@Component +public class ExampleChangeWatcherFactory extends ChangeWatcherHelperFactory { + +} \ No newline at end of file diff --git a/src/test/java/com/percero/example/ExampleManifest.java b/src/test/java/com/percero/example/ExampleManifest.java new file mode 100644 index 0000000..a9cf5b9 --- /dev/null +++ b/src/test/java/com/percero/example/ExampleManifest.java @@ -0,0 +1,54 @@ +package com.percero.example; + +import com.percero.framework.bl.IManifest; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by jonnysamps on 9/4/15. + */ +@Component +public class ExampleManifest implements IManifest { + private List classList = null; + public List getClassList() { + if (classList == null) { + classList = new ArrayList(); + // Register Classes here + classList.add(Person.class); + classList.add(PersonRole.class); + classList.add(Email.class); + classList.add(Block.class); + } + return classList; + } + + private List objectList = null; + public List getObjectList() { + if (objectList == null) { + objectList = new ArrayList(); + // Instantiate one of each model object + objectList.add(new Person()); + objectList.add(new PersonRole()); + objectList.add(new Email()); + objectList.add(new Block()); + } + return objectList; + } + + private Map uuidMap = null; + public Map getUuidMap() { + if (uuidMap == null) { + uuidMap = new HashMap(); + // Create UUID map for each model class + uuidMap.put("1", Person.class); + uuidMap.put("2", PersonRole.class); + uuidMap.put("3", Email.class); + uuidMap.put("4", Block.class); + } + return uuidMap; + } +} diff --git a/src/test/java/com/percero/example/ExampleProcessHelper.java b/src/test/java/com/percero/example/ExampleProcessHelper.java new file mode 100644 index 0000000..7318763 --- /dev/null +++ b/src/test/java/com/percero/example/ExampleProcessHelper.java @@ -0,0 +1,8 @@ +package com.percero.example; + +import com.percero.agents.sync.helpers.ProcessHelper; +import org.springframework.stereotype.Component; + +@Component +public class ExampleProcessHelper extends ProcessHelper { +} diff --git a/src/test/java/com/percero/example/Person.java b/src/test/java/com/percero/example/Person.java new file mode 100644 index 0000000..312be9d --- /dev/null +++ b/src/test/java/com/percero/example/Person.java @@ -0,0 +1,235 @@ +package com.percero.example; + +import com.google.gson.JsonObject; +import com.percero.agents.auth.vo.IUserAnchor; +import com.percero.agents.sync.metadata.annotations.EntityInterface; +import com.percero.agents.sync.metadata.annotations.Externalize; +import com.percero.agents.sync.metadata.annotations.PropertyInterface; +import com.percero.agents.sync.vo.BaseDataObject; +import com.percero.serial.JsonUtils; +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.annotate.JsonProperty; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; + +import javax.persistence.*; +import java.io.IOException; +import java.io.Serializable; +import java.util.Date; +import java.util.List; + +@Entity +@EntityInterface(interfaceClass=IUserAnchor.class) +public class Person extends BaseDataObject implements Serializable, IUserAnchor +{ + ////////////////////////////////////////////////////// + // VERSION + ////////////////////////////////////////////////////// + @Override + public String classVersion() { + return "0.0.0"; + } + + ////////////////////////////////////////////////////// + // Properties + ////////////////////////////////////////////////////// + @Id + @Externalize + @Column(unique=true,name="ID") + private String ID; + @JsonProperty(value="ID") + public String getID() { + return this.ID; + } + @JsonProperty(value="ID") + public void setID(String value) { + this.ID = value; + } + + @Column + @Externalize + private Date dateCreated; + public Date getDateCreated() { + return this.dateCreated; + } + public void setDateCreated(Date value) + { + this.dateCreated = value; + } + + @Column + @Externalize + private Date dateModified; + public Date getDateModified() { + return this.dateModified; + } + public void setDateModified(Date value) + { + this.dateModified = value; + } + + @Column + @PropertyInterface(entityInterfaceClass=IUserAnchor.class, propertyName="userId", params={}) + @Externalize + private String userId; + public String getUserId() { + return this.userId; + } + public void setUserId(String value) + { + this.userId = value; + } + + @Column + @PropertyInterface(entityInterfaceClass=IUserAnchor.class, propertyName="firstName", params={}) + @Externalize + private String firstName; + public String getFirstName() { + return this.firstName; + } + public void setFirstName(String value) + { + this.firstName = value; + } + + @Column + @PropertyInterface(entityInterfaceClass=IUserAnchor.class, propertyName="lastName",params={}) + @Externalize + private String lastName; + public String getLastName() { + return this.lastName; + } + public void setLastName(String value) + { + this.lastName = value; + } + + @Externalize + @OneToMany(fetch= FetchType.LAZY, targetEntity=PersonRole.class, mappedBy="person", cascade=javax.persistence.CascadeType.REMOVE) + private List personRoles; + public List getPersonRoles() { + return this.personRoles; + } + public void setPersonRoles(List value) { + this.personRoles = value; + } + + + ////////////////////////////////////////////////////// + // JSON + ////////////////////////////////////////////////////// + @Override + public String retrieveJson(ObjectMapper objectMapper) { + String objectJson = super.retrieveJson(objectMapper); + + // Properties + objectJson += ",\"dateCreated\":"; + if (getDateCreated() == null) + objectJson += "null"; + else { + objectJson += getDateCreated().getTime(); + } + + objectJson += ",\"dateModified\":"; + if (getDateModified() == null) + objectJson += "null"; + else { + objectJson += getDateModified().getTime(); + } + + objectJson += ",\"userId\":"; + if (getUserId() == null) + objectJson += "null"; + else { + if (objectMapper == null) + objectMapper = new ObjectMapper(); + try { + objectJson += objectMapper.writeValueAsString(getUserId()); + } catch (JsonGenerationException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (JsonMappingException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (IOException e) { + objectJson += "null"; + e.printStackTrace(); + } + } + + objectJson += ",\"firstName\":"; + if (getFirstName() == null) + objectJson += "null"; + else { + if (objectMapper == null) + objectMapper = new ObjectMapper(); + try { + objectJson += objectMapper.writeValueAsString(getFirstName()); + } catch (JsonGenerationException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (JsonMappingException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (IOException e) { + objectJson += "null"; + e.printStackTrace(); + } + } + + objectJson += ",\"lastName\":"; + if (getLastName() == null) + objectJson += "null"; + else { + if (objectMapper == null) + objectMapper = new ObjectMapper(); + try { + objectJson += objectMapper.writeValueAsString(getLastName()); + } catch (JsonGenerationException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (JsonMappingException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (IOException e) { + objectJson += "null"; + e.printStackTrace(); + } + } + + objectJson += ",\"personRoles\":["; + if (getPersonRoles() != null) { + int personRolesCounter = 0; + for(PersonRole nextPersonRoles : getPersonRoles()) { + if (personRolesCounter > 0) + objectJson += ","; + try { + objectJson += nextPersonRoles.toEmbeddedJson(); + personRolesCounter++; + } catch(Exception e) { + // Do nothing. + } + } + } + + objectJson += "]"; + + + return objectJson; + } + + @Override + protected void fromJson(JsonObject jsonObject) { + super.fromJson(jsonObject); + + // Properties + setDateCreated(JsonUtils.getJsonDate(jsonObject, "dateCreated")); + setDateModified(JsonUtils.getJsonDate(jsonObject, "dateModified")); + setUserId(JsonUtils.getJsonString(jsonObject, "userId")); + setFirstName(JsonUtils.getJsonString(jsonObject, "firstName")); + setLastName(com.percero.serial.JsonUtils.getJsonString(jsonObject, "lastName")); + + this.personRoles = (List) JsonUtils.getJsonListPerceroObject(jsonObject, "personRoles"); + } + +} diff --git a/src/test/java/com/percero/example/PersonRole.java b/src/test/java/com/percero/example/PersonRole.java new file mode 100644 index 0000000..739b8e2 --- /dev/null +++ b/src/test/java/com/percero/example/PersonRole.java @@ -0,0 +1,180 @@ +package com.percero.example; + +import com.google.gson.JsonObject; +import com.percero.agents.auth.vo.IUserRole; +import com.percero.agents.sync.metadata.annotations.EntityInterface; +import com.percero.agents.sync.metadata.annotations.Externalize; +import com.percero.agents.sync.metadata.annotations.PropertyInterface; +import com.percero.agents.sync.metadata.annotations.RelationshipInterface; +import com.percero.agents.sync.vo.BaseDataObject; +import com.percero.serial.JsonUtils; +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.annotate.JsonProperty; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; + +import javax.persistence.*; +import java.io.IOException; +import java.io.Serializable; +import java.util.Date; + +@EntityInterface(interfaceClass=IUserRole.class) +@Entity +public class PersonRole extends BaseDataObject implements Serializable, IUserRole +{ + ////////////////////////////////////////////////////// + // VERSION + ////////////////////////////////////////////////////// + @Override + public String classVersion() { + return "0.0.0"; + } + + + ////////////////////////////////////////////////////// + // ID + ////////////////////////////////////////////////////// + @Id + @Externalize + @Column(unique=true,name="ID") + private String ID; + @JsonProperty(value="ID") + public String getID() { + return this.ID; + } + @JsonProperty(value="ID") + public void setID(String value) { + this.ID = value; + } + + ////////////////////////////////////////////////////// + // Properties + ////////////////////////////////////////////////////// + @Column + @Externalize + private Date dateCreated; + public Date getDateCreated() { + return this.dateCreated; + } + public void setDateCreated(Date value) + { + this.dateCreated = value; + } + + @Column + @Externalize + private Date dateModified; + public Date getDateModified() { + return this.dateModified; + } + public void setDateModified(Date value) + { + this.dateModified = value; + } + + @Column + @PropertyInterface(entityInterfaceClass=IUserRole.class, propertyName="roleName",params={}) + @Externalize + private String roleName; + public String getRoleName() { + return this.roleName; + } + public void setRoleName(String value) + { + this.roleName = value; + } + + + ////////////////////////////////////////////////////// + // Source Relationships + ////////////////////////////////////////////////////// + @RelationshipInterface(entityInterfaceClass=IUserRole.class, sourceVarName="userAnchor") + @Externalize + @JoinColumn(name="person_ID") + @org.hibernate.annotations.ForeignKey(name="FK_User_user_TO_UserRole") + @ManyToOne(fetch=FetchType.LAZY, optional=false) + private Person person; + public Person getPerson() { + return this.person; + } + public void setPerson(Person value) { + this.person = value; + } + + + ////////////////////////////////////////////////////// + // JSON + ////////////////////////////////////////////////////// + @Override + public String retrieveJson(ObjectMapper objectMapper) { + String objectJson = super.retrieveJson(objectMapper); + + // Properties + objectJson += ",\"dateCreated\":"; + if (getDateCreated() == null) + objectJson += "null"; + else { + objectJson += getDateCreated().getTime(); + } + + objectJson += ",\"dateModified\":"; + if (getDateModified() == null) + objectJson += "null"; + else { + objectJson += getDateModified().getTime(); + } + + objectJson += ",\"roleName\":"; + if (getRoleName() == null) + objectJson += "null"; + else { + if (objectMapper == null) + objectMapper = new ObjectMapper(); + try { + objectJson += objectMapper.writeValueAsString(getRoleName()); + } catch (JsonGenerationException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (JsonMappingException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (IOException e) { + objectJson += "null"; + e.printStackTrace(); + } + } + + // Source Relationships + objectJson += ",\"person\":"; + if (getPerson() == null) + objectJson += "null"; + else { + try { + objectJson += getPerson().toEmbeddedJson(); + } catch(Exception e) { + objectJson += "null"; + } + } + objectJson += ""; + + // Target Relationships + + return objectJson; + } + + @Override + protected void fromJson(JsonObject jsonObject) { + super.fromJson(jsonObject); + + // Properties + setDateCreated(JsonUtils.getJsonDate(jsonObject, "dateCreated")); + setDateModified(JsonUtils.getJsonDate(jsonObject, "dateModified")); + setRoleName(JsonUtils.getJsonString(jsonObject, "roleName")); + + // Source Relationships + this.person = JsonUtils.getJsonPerceroObject(jsonObject, "person"); + + // Target Relationships + } +} + diff --git a/src/test/resources/google/client_secrets.json b/src/test/resources/google/client_secrets.json new file mode 100644 index 0000000..36a97f9 --- /dev/null +++ b/src/test/resources/google/client_secrets.json @@ -0,0 +1,13 @@ +{ + "web": { + "client_id":"426306336879-m3ffk6mqt63pot5pg1kq52b7rmf2lnfo.apps.googleusercontent.com", + "auth_uri":"https://accounts.google.com/o/oauth2/auth", + "token_uri":"https://accounts.google.com/o/oauth2/token", + "auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs", + "client_email":"426306336879-m3ffk6mqt63pot5pg1kq52b7rmf2lnfo@developer.gserviceaccount.com", + "client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/426306336879-m3ffk6mqt63pot5pg1kq52b7rmf2lnfo%40developer.gserviceaccount.com", + "client_secret":"HPH9s_dgj1pLNn4VcB5ZFxre", + "redirect_uris":["https://localhost:8081/oauth2callback.html"], + "javascript_origins":["https://localhost:8081"] + } +} \ No newline at end of file diff --git a/src/test/resources/google/privatekey.p12 b/src/test/resources/google/privatekey.p12 new file mode 100644 index 0000000000000000000000000000000000000000..bbae0079c04df6ff8478e18069f778810e3b36f0 GIT binary patch literal 2572 zcmY+EX*3iH8^;GTW9->78P~Oq_0mj@eb2sT$sQ6jmbhgX2}8KZQkDi~%RbYHs2C~B zh(Y!wO4i9B%NSeqy63#_z32V#oadb1|NlH6e+Vo)gqev2fo1Civ&ko$Chu`Eu`-on z*_wb@HsUF+M_@s>{x1S~0kI(WQ|xj&4q%S|UU9NBF_&V2hX^c?fsh5W{|}!&7X}Kv zfO&q7JO-N!QEZ@i8x&nS5}`^rL%sSyGRv#vn&JHd1{GQDc4TGPD(*on5Bw=>Xw;|Ue z=cZ6RM+%1tHN|!Ls7p|9N1|6pL+*zLJWAD`$)1*;rJBq6ofC4^LXgd6mxB4&T_2;M zx3^jkTY>+jO2S+(L_8tj9Z^f^XbKPM)_!M;MXHd)2%gAVf-b+b@jmT*O4D%OmL&B|!N zhSns{d^qU?ed@YMP}xduUpIi3%aSSaif`ttmoj31*vh`P$P(Os+S_av@X2J_;@gRY zRcNnVbl(E5*lTebx>M_rq2dx3*J&)A+Uw1XzV!x26HFkSg#kaDr0H7$Vh9^}wO{TL zqwvq*{)+9m_StZdTrHsYITm>-CqI4Wv8KD|kKHbNjc<#}ER#Zw!N2R-L&wZ zEz1WH%ifMo{e{v@!{U8)3?+$-mYZ7{+?XGP>j|MGLwneVesAnod`&kXo`3SO?@^QC zv|bwAh#*faMt>4D2uky4z9hM+-Jq4!tp1*Dm#*l8oYhiu)@H3rgpj8Zq3GybAMM|7 zl+?8FaCJ(xXFdaM_0q>$CRw8*2#-XZC?(R{JeCW)z)q~ z;VA`R&yOs$)%dBu(3?qR+LRslntMsdj5^}zbncBlc`f5g&vibXlbZ3AH#&2MyCms4 z4^5AA2POUE3qTu7E$uVd_V|8khn;^hnV*Q^=_N?O_$lK`!!t>e&@NAe;*q#0)Uf_8 z&}C-6a-|qXb<}WQ31d$KnTDe5CQN4gaOci8>e#{A^vYk5GerUyW$^&7DJ*gs-ToFsH>w8 zDAiMkUi_!T3O?1U)F~2UW&)h9bN?nx|0i1T-=bC8P?mp>+yQR~{npT3#X@Fww)6R0 zwDDM$Ip_-CjKrVdrJ_oswRD%1#Bw2zVaWn`l9bt4z7K6^GqfRI?A?z>z~26-GTE_a zDRuz5OmlrOS`Y1N`~aV*h;O@uQp}f_kSp-eZBwE~!}zGn*K~9D{kDABY+fdM&TIvF zqeaA1u4Qjn_|z{w9%+Jqhh%F>+zpDgbWiuz?(;X8Y%Nw7P)3r!%@Uz+f`tLJmyRfB zUD_Rn3#UMK{hzxsM$8@-44xJx({!-5eLz<=rw)OE6`kDo!1^U8grLF!OXQu-2p{|YSvXDi*=Rv zO#p|E=nN;xy`E30wLF_ouYOXY=j|K?geA|p`6(mF6IKl*^09)mbc^s5O4@u)pd4(4oP-ApU@RIFTnAR2!iH^9Z$uY^_0`n!5TstDA5c9g*{#u*wP% zRQPMAiRM0^M#Fer+W^+*#YlbhjkHefle6JN9dHM#ku>X369F!*VHLW((H~1ZI6pA? zd$`EzXsO=|q3RNaKCZSX!lR%LGz7F3-E={hpcvSsqt*ULe5eCv3*0d_#z_aK5K z?^Ygq+_pQE!@JJKzVoN6L=C? z?La^F*!5a(ej*R!*Ala-0mmI}?$tW^Wr(!xoka&iut(!Bw-SxGqY$kzuaMN(MfAQ` zIH^{(8@AuA(D{*xq9XogmSNVSGxRY*FlTosh7i-b74y^WeM|~>sKanr>}9)Z-vw8T zg1Tw`hv?kM*C(l7iWoM1O$l+;l6^kWB@CLPCLOLS_%QYM!sRj2R==)YX%S_-?b*EF zgM;Cx2yKKof(;B(;AUYK1_6KqCz!mdc4{A7@A!!*Fi~iO2kB%26ad3qJxfLh3O%pf RkkV2poHdoO`SrIV{sSPC% + + + + + + + + + + + classpath:properties/test/env.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hibernate.dialect=org.hibernate.dialect.MySQLDialect + hibernate.hbm2ddl.auto=update + + hibernate.show_sql=false + hibernate.format_sql=false + + hibernate.connection.aggressive_release=true + hibernate.jdbc.batch_size=20 + hibernate.connection.autocommit=false + hibernate.enable_lazy_load_no_trans=true + + + + + com.percero.amqp + com.percero.agents.auth.vo + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hibernate.dialect=org.hibernate.dialect.MySQLDialect + hibernate.hbm2ddl.auto=update + + hibernate.show_sql=false + hibernate.format_sql=false + + hibernate.connection.aggressive_release=true + hibernate.jdbc.batch_size=20 + hibernate.connection.autocommit=false + hibernate.connection.autoReconnect=true + hibernate.enable_lazy_load_no_trans=true + + + + + $pf{domain.packageToScan} + + + + + + + + + diff --git a/src/test/sql/example_schema.sql b/src/test/sql/example_schema.sql new file mode 100644 index 0000000..e69de29 From 6f3e57a8531598339e448eb1f1679a83ffb3c739 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Sat, 5 Sep 2015 22:51:30 -0700 Subject: [PATCH 3/9] Update table: individual record updates, inserts, deletes --- .../agents/sync/cache/CacheManager.java | 137 +++++++ .../agents/sync/helpers/PostDeleteHelper.java | 1 - .../jobs/UpdateTableConnectionFactory.java | 8 +- .../agents/sync/jobs/UpdateTablePoller.java | 33 +- .../sync/jobs/UpdateTableProcessor.java | 127 ++++-- .../jobs/UpdateTableProcessorFactory.java | 40 ++ .../agents/sync/services/IDataProvider.java | 1 + .../sync/services/RedisDataProvider.java | 5 + .../sync/services/SyncAgentDataProvider.java | 368 +++++++----------- src/main/resources/log4j.properties | 2 + .../spring/percero-spring-config.xml | 1 + .../sync/jobs/UpdateTableProcessorTest.java | 109 +++++- src/test/java/com/percero/example/Person.java | 1 + .../resources/properties/test/env.properties | 6 +- src/test/sql/example_schema.sql | 15 + 15 files changed, 557 insertions(+), 297 deletions(-) create mode 100644 src/main/java/com/percero/agents/sync/cache/CacheManager.java create mode 100644 src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessorFactory.java diff --git a/src/main/java/com/percero/agents/sync/cache/CacheManager.java b/src/main/java/com/percero/agents/sync/cache/CacheManager.java new file mode 100644 index 0000000..6d45b17 --- /dev/null +++ b/src/main/java/com/percero/agents/sync/cache/CacheManager.java @@ -0,0 +1,137 @@ +package com.percero.agents.sync.cache; + +import com.percero.agents.sync.access.RedisKeyUtils; +import com.percero.agents.sync.datastore.RedisDataStore; +import com.percero.agents.sync.metadata.IMappedClassManager; +import com.percero.agents.sync.metadata.MappedClass; +import com.percero.agents.sync.metadata.MappedClassManagerFactory; +import com.percero.agents.sync.metadata.MappedField; +import com.percero.agents.sync.vo.BaseDataObject; +import com.percero.agents.sync.vo.ClassIDPair; +import com.percero.framework.vo.IPerceroObject; +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import sun.misc.Cache; + +import java.lang.reflect.InvocationTargetException; +import java.util.*; + +/** + * Created by jonnysamps on 9/5/15. + */ +@Component +public class CacheManager { + + private static Logger logger = Logger.getLogger(CacheManager.class); + + @Autowired + RedisDataStore redisDataStore; + + @Autowired + Long cacheTimeout = Long.valueOf(60 * 60 * 24 * 14); // Two weeks + + public void updateCachedObject(IPerceroObject perceroObject, Map> changedFields){ + // TODO: Field-level updates could be REALLY useful here. Would avoid A TON of UNNECESSARY work... + try { + if (cacheTimeout > 0) { + // TODO: Also need to update the caches of anything object that is related to this object. + String key = RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID()); + if (redisDataStore.hasKey(key)) { + redisDataStore.setValue(key, ((BaseDataObject) perceroObject).toJson()); + } + + // Iterate through each changed object and reset the cache for that object. + if (changedFields != null) { +// Iterator>> itrChangedFieldEntrySet = changedFields.entrySet().iterator(); +// Set keysToDelete = new HashSet(); +// while (itrChangedFieldEntrySet.hasNext()) { +// Map.Entry> nextEntry = itrChangedFieldEntrySet.next(); +// ClassIDPair thePair = nextEntry.getKey(); +// if (!thePair.comparePerceroObject(perceroObject)) { +// String nextKey = RedisKeyUtils.classIdPair(thePair.getClassName(), thePair.getID()); +// keysToDelete.add(nextKey); +// } +// } + Iterator itrChangedFieldKeyset = changedFields.keySet().iterator(); + Set keysToDelete = new HashSet(); + while (itrChangedFieldKeyset.hasNext()) { + ClassIDPair thePair = itrChangedFieldKeyset.next(); + if (!thePair.comparePerceroObject(perceroObject)) { + String nextKey = RedisKeyUtils.classIdPair(thePair.getClassName(), thePair.getID()); + keysToDelete.add(nextKey); + } + } + + if (!keysToDelete.isEmpty()) { + redisDataStore.deleteKeys(keysToDelete); + // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? + //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + } + } else { + // No changedFields? We should never get here? + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(perceroObject.getClass().getName()); + Iterator itrToManyFields = mappedClass.toManyFields.iterator(); + while (itrToManyFields.hasNext()) { + MappedField nextMappedField = itrToManyFields.next(); + Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); + if (fieldObject != null) { + if (fieldObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject) fieldObject).getID()); + if (redisDataStore.hasKey(nextKey)) { + redisDataStore.deleteKey(nextKey); + // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? + //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + } + } else if (fieldObject instanceof Collection) { + Iterator itrFieldObject = ((Collection) fieldObject).iterator(); + while (itrFieldObject.hasNext()) { + Object nextListObject = itrFieldObject.next(); + if (nextListObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject) nextListObject).getID()); + if (redisDataStore.hasKey(nextKey)) { + redisDataStore.deleteKey(nextKey); + // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? + //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + } + } + } + } + } + } + Iterator itrToOneFields = mappedClass.toOneFields.iterator(); + while (itrToOneFields.hasNext()) { + MappedField nextMappedField = itrToOneFields.next(); + Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); + if (fieldObject != null) { + if (fieldObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject) fieldObject).getID()); + if (redisDataStore.hasKey(nextKey)) { + redisDataStore.deleteKey(nextKey); + // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? + //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + } + } else if (fieldObject instanceof Collection) { + Iterator itrFieldObject = ((Collection) fieldObject).iterator(); + while (itrFieldObject.hasNext()) { + Object nextListObject = itrFieldObject.next(); + if (nextListObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject) nextListObject).getID()); + if (redisDataStore.hasKey(nextKey)) { + redisDataStore.deleteKey(nextKey); + // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? + //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + } + } + } + } + } + } + } + } + } catch (Exception e){ + logger.warn(e.getMessage(), e); + } + } +} diff --git a/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java b/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java index dfe3855..f089aa4 100644 --- a/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java +++ b/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java @@ -106,7 +106,6 @@ protected void pushObjectDeleteJournals(Collection clientIds, String cla pushDeleteResponse = new PushDeleteResponse(); pushDeleteResponse.setObjectList(new ArrayList()); -// pushDeleteResponse.setClientId(nextClientId); pushDeleteResponse.getObjectList().add(removedPair); pushDeleteResponse.setObjectJson(objectJson); diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java index 8ef0bd9..d5f5cd7 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java @@ -22,28 +22,28 @@ public class UpdateTableConnectionFactory { private static Logger logger = Logger.getLogger(UpdateTableConnectionFactory.class); @Autowired - @Value("pf{updateTable.driverClassName:com.mysql.jdbc.Driver}") + @Value("$pf{updateTable.driverClassName:com.mysql.jdbc.Driver}") private String driverClassName; public void setDriverClassName(String val){ this.driverClassName = val; } @Autowired - @Value("pf{updateTable.username}") + @Value("$pf{updateTable.username}") private String username; public void setUsername(String val){ this.username = val; } @Autowired - @Value("pf{updateTable.username}") + @Value("$pf{updateTable.username}") private String password; public void setPassword(String val){ this.password = val; } @Autowired - @Value("pf{updateTable.jdbcUrl:jdbc:mysql://localhost/db}") + @Value("$pf{updateTable.jdbcUrl:jdbc:mysql://localhost/db}") private String jdbcUrl; public void setJdbcUrl(String val){ this.jdbcUrl = val; diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java index 31ead1b..bdd4ea4 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java @@ -25,33 +25,28 @@ public void setTableNames(String val){ tableNames = val.split(","); } - @Autowired - UpdateTableConnectionFactory connectionFactory; + UpdateTableProcessorFactory updateTableProcessorFactory; - @Autowired - IManifest manifest; - - @Autowired - PostDeleteHelper postDeleteHelper; + public boolean enabled = true; /** * Run every minute */ - @Scheduled(fixedDelay=6000, initialDelay=6000) + @Scheduled(fixedDelay=10000, initialDelay=10000) public void pollUpdateTables(){ - logger.info("Polling Update Tables..."); - for(String tableName : tableNames){ - UpdateTableProcessor processor = new UpdateTableProcessor(tableName, connectionFactory, manifest, postDeleteHelper); - ProcessorResult result = processor.process(); - if(result.isSuccess()){ - logger.debug("Update table processor ("+tableName+") finished successfully. Total rows ("+result.getTotal()+")"); - } - else{ - logger.warn("Update table processor ("+tableName+") failed. Details:"); - logger.warn(result); + if(enabled) + for(String tableName : tableNames){ + UpdateTableProcessor processor = updateTableProcessorFactory.getProcessor(tableName); + ProcessorResult result = processor.process(); + if(result.isSuccess()){ + logger.debug("Update table processor ("+tableName+") finished successfully. Total rows ("+result.getTotal()+")"); + } + else{ + logger.warn("Update table processor ("+tableName+") failed. Details:"); + logger.warn(result); + } } - } } diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index e3e428a..0bfd839 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -1,7 +1,16 @@ package com.percero.agents.sync.jobs; +import com.percero.agents.sync.cache.CacheManager; import com.percero.agents.sync.helpers.PostDeleteHelper; +import com.percero.agents.sync.helpers.PostPutHelper; +import com.percero.agents.sync.metadata.IMappedClassManager; +import com.percero.agents.sync.metadata.MappedClass; +import com.percero.agents.sync.metadata.MappedClassManagerFactory; +import com.percero.agents.sync.services.DataProviderManager; +import com.percero.agents.sync.services.IDataProvider; +import com.percero.agents.sync.vo.ClassIDPair; import com.percero.framework.bl.IManifest; +import com.percero.framework.vo.IPerceroObject; import org.apache.log4j.Logger; import org.joda.time.DateTime; @@ -14,7 +23,7 @@ import java.util.Random; /** - * Responsible for querying an update table and processing the rows + * Responsible for querying an update table and processing the rows. * Created by Jonathan Samples on 8/31/15. */ public class UpdateTableProcessor { @@ -25,17 +34,26 @@ public class UpdateTableProcessor { private String tableName; private UpdateTableConnectionFactory connectionFactory; private PostDeleteHelper postDeleteHelper; + private PostPutHelper postPutHelper; private IManifest manifest; - + private CacheManager cacheManager; + private DataProviderManager dataProviderManager; public UpdateTableProcessor(String tableName, UpdateTableConnectionFactory connectionFactory, IManifest manifest, - PostDeleteHelper postDeleteHelper){ + PostDeleteHelper postDeleteHelper, + PostPutHelper postPutHelper, + CacheManager cacheManager, + DataProviderManager dataProviderManager) + { this.tableName = tableName; this.connectionFactory = connectionFactory; this.postDeleteHelper = postDeleteHelper; + this.postPutHelper = postPutHelper; this.manifest = manifest; + this.cacheManager = cacheManager; + this.dataProviderManager= dataProviderManager; } /** @@ -58,12 +76,15 @@ public ProcessorResult process(){ UpdateTableRow row = getRow(); if(row == null) break; - if(processRow(row)) { - result.addResult(row.getType().toString()); - deleteRow(row); - } - else { - result.addResult(row.getType().toString(), false, ""); + try { + if (processRow(row)) { + result.addResult(row.getType().toString()); + deleteRow(row); + } else { + result.addResult(row.getType().toString(), false, ""); + } + }catch(Exception e){ + result.addResult(row.getType().toString(), false, e.getMessage()); } } @@ -71,7 +92,7 @@ public ProcessorResult process(){ return result; } - private boolean processRow(UpdateTableRow row){ + private boolean processRow(UpdateTableRow row) throws Exception{ boolean result = true; if(row.getRowId() != null) @@ -113,8 +134,9 @@ private boolean processRow(UpdateTableRow row){ * @param row * @return */ - private boolean processDeleteSingle(UpdateTableRow row){ - + private boolean processDeleteSingle(UpdateTableRow row) throws Exception{ + Class clazz = getClassForTableName(row.getTableName()); + postDeleteHelper.postDeleteObject(new ClassIDPair(row.getRowId(), clazz.getCanonicalName()), null, null, true); return true; } @@ -123,7 +145,18 @@ private boolean processDeleteSingle(UpdateTableRow row){ * @param row * @return */ - private boolean processUpdateSingle(UpdateTableRow row){ + private boolean processUpdateSingle(UpdateTableRow row) throws Exception{ + Class clazz = getClassForTableName(row.getTableName()); + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(clazz.getName()); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + ClassIDPair pair = new ClassIDPair(row.getRowId(), clazz.getCanonicalName()); + IPerceroObject object = dataProvider.systemGetById(pair, true); + + if(object != null){ + cacheManager.updateCachedObject(object, null); + postPutHelper.postPutObject(pair, null, null, true, null); + } return true; } @@ -132,7 +165,9 @@ private boolean processUpdateSingle(UpdateTableRow row){ * @param row * @return */ - private boolean processInsertSingle(UpdateTableRow row){ + private boolean processInsertSingle(UpdateTableRow row) throws Exception{ + Class clazz = getClassForTableName(row.getTableName()); + postPutHelper.postPutObject(new ClassIDPair(row.getRowId(), clazz.getCanonicalName()), null, null, true, null); return true; } @@ -168,10 +203,12 @@ private boolean processInsertTable(UpdateTableRow row){ * processors don't duplicate the work * @return */ - private UpdateTableRow getRow(){ + public UpdateTableRow getRow(){ UpdateTableRow row = null; - try(Connection conn = connectionFactory.getConnection()){ + try(Connection conn = connectionFactory.getConnection(); + Statement statement = conn.createStatement()) + { Random rand = new Random(); @@ -183,31 +220,28 @@ private UpdateTableRow getRow(){ /** * First try to lock a row */ - String sql = "update :tableName set lockId=:lockId, lockDate=NOW() where lockId is null or lockDate < :expireThreshold limit 1"; - sql.replace(":tableName", tableName); - sql.replace(":lockId", lockId+""); - sql.replace(":expireThreshold", expireThreshold.toString("Y-MM-dd HH:mm:ss")); - - Statement statement; + String sql = "update `:tableName` set lockID=:lockId, lockDate=NOW() " + + "where lockID is null or " + + "lockDate < ':expireThreshold' " + + "order by timestamp limit 1"; + sql = sql.replace(":tableName", tableName); + sql = sql.replace(":lockId", lockId+""); + sql = sql.replace(":expireThreshold", expireThreshold.toString("Y-MM-dd HH:mm:ss")); - statement = conn.createStatement(); int numUpdated = statement.executeUpdate(sql); // Found a row to process if(numUpdated > 0){ sql = "select * from :tableName where lockId=:lockId limit 1"; - sql.replace(":tableName", tableName); - sql.replace(":lockId", lockId+""); - statement = conn.createStatement(); - ResultSet rs = statement.executeQuery(sql); - - // If got a row back - if(rs.next()){ - row = UpdateTableRow.fromResultSet(rs); - rs.close(); - } - else{ - logger.warn("Locked a row but couldn't retrieve"); + sql = sql.replace(":tableName", tableName); + sql = sql.replace(":lockId", lockId+""); + + try(ResultSet rs = statement.executeQuery(sql)){ + // If got a row back + if(rs.next()) + row = UpdateTableRow.fromResultSet(rs); + else + logger.warn("Locked a row but couldn't retrieve"); } } @@ -225,7 +259,8 @@ private UpdateTableRow getRow(){ private void deleteRow(UpdateTableRow row){ try(Connection conn = connectionFactory.getConnection()){ String sql = "delete from :tableName where ID=:ID"; - sql.replace(":ID", row.getID()+""); + sql = sql.replace(":tableName", tableName); + sql = sql.replace(":ID", row.getID()+""); Statement statement = conn.createStatement(); int numUpdated = statement.executeUpdate(sql); if(numUpdated != 1){ @@ -238,20 +273,26 @@ private void deleteRow(UpdateTableRow row){ public Class getClassForTableName(String tableName){ Class result = null; + + // First look for the @Table annotation for(Class c : manifest.getClassList()){ Table table = (Table) c.getAnnotation(Table.class); - if(tableName.equals(table.name())) { + if(table != null && tableName.equals(table.name())) { result = c; break; } } - return result; - } + // If we didn't find that now look for the simple class name to match + if(result == null){ + for(Class c : manifest.getClassList()){ + if(tableName.equals(c.getSimpleName())) { + result = c; + break; + } + } + } - public static void main(String[] args){ - DateTime time = new DateTime(); - logger.info(time.toString("Y-MM-dd HH:mm:ss")); + return result; } - } diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessorFactory.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessorFactory.java new file mode 100644 index 0000000..0c8e392 --- /dev/null +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessorFactory.java @@ -0,0 +1,40 @@ +package com.percero.agents.sync.jobs; + +import com.percero.agents.sync.cache.CacheManager; +import com.percero.agents.sync.helpers.PostDeleteHelper; +import com.percero.agents.sync.helpers.PostPutHelper; +import com.percero.agents.sync.services.DataProviderManager; +import com.percero.framework.bl.IManifest; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Creates UpdateTableProcessor for the poller + * Created by jonnysamps on 9/4/15. + */ +@Component +public class UpdateTableProcessorFactory { + + @Autowired + UpdateTableConnectionFactory connectionFactory; + + @Autowired + IManifest manifest; + + @Autowired + PostDeleteHelper postDeleteHelper; + + @Autowired + PostPutHelper postPutHelper; + + @Autowired + CacheManager cacheManager; + + @Autowired + DataProviderManager dataProviderManager; + + public UpdateTableProcessor getProcessor(String tableName){ + return new UpdateTableProcessor(tableName, connectionFactory, manifest, + postDeleteHelper, postPutHelper, cacheManager, dataProviderManager); + } +} diff --git a/src/main/java/com/percero/agents/sync/services/IDataProvider.java b/src/main/java/com/percero/agents/sync/services/IDataProvider.java index ec8b82f..59d9ff2 100644 --- a/src/main/java/com/percero/agents/sync/services/IDataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/IDataProvider.java @@ -25,6 +25,7 @@ public interface IDataProvider { public List runQuery(MappedClass mappedClass, String queryName, Object[] queryArguments, String clientId); public IPerceroObject findById(ClassIDPair classIdPair, String userId); public T systemGetById(ClassIDPair classIdPair); + public T systemGetById(ClassIDPair classIdPair, boolean ignoreCache); public List findByIds(ClassIDPairs classIdPairs, String userId); public IPerceroObject findUnique(IPerceroObject theQueryObject, String userId); public List findByExample(IPerceroObject theQueryObject, List excludeProperties, String userId); diff --git a/src/main/java/com/percero/agents/sync/services/RedisDataProvider.java b/src/main/java/com/percero/agents/sync/services/RedisDataProvider.java index de3cbaa..651d0c3 100644 --- a/src/main/java/com/percero/agents/sync/services/RedisDataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/RedisDataProvider.java @@ -153,6 +153,11 @@ public IPerceroObject findById(ClassIDPair classIdPair, String userId) { return null;**/ } + @Override + public T systemGetById(ClassIDPair classIdPair, boolean ignoreCache) { + return null; + } + public IPerceroObject systemGetById(ClassIDPair classIdPair) { return null; } diff --git a/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java b/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java index d933039..297e7d5 100644 --- a/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java @@ -13,6 +13,7 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; +import com.percero.agents.sync.cache.CacheManager; import javassist.ClassPool; import javassist.CtClass; import javassist.CtField; @@ -66,22 +67,25 @@ public class SyncAgentDataProvider implements IDataProvider { // TODO: Better manage Hibernate Sessions (opening and closing). private static final Logger log = Logger.getLogger(SyncAgentDataProvider.class); - + public void initialize() { // Do nothing. } - + public String getName() { return "syncAgent"; } - + @Autowired RedisDataStore redisDataStore; - + @Autowired Long cacheTimeout = Long.valueOf(60 * 60 * 24 * 14); // Two weeks + @Autowired + CacheManager cacheManager; + @Autowired ObjectMapper safeObjectMapper; @@ -90,7 +94,7 @@ public String getName() { public void setAppSessionFactory(SessionFactory value) { appSessionFactory = value; } - + @SuppressWarnings({ "rawtypes", "unchecked" }) // TODO: @Transactional(readOnly=true) @@ -116,33 +120,33 @@ public PerceroList getAllByName(Object aName, Integer pageNumber * readAllQuery optimization * You can now define a readAllQuery on a class to imporove it's initial download time * for briefcase mode. - * + * * Only supports plain SQL for now */ if(mappedClass.getReadAllQuery() != null && mappedClass.getReadAllQuery().getQuery() != null && !mappedClass.getReadAllQuery().getQuery().isEmpty()){ if((mappedClass.getReadAllQuery() instanceof JpqlQuery)){ throw new IllegalArgumentException("Illegal query type on:"+aClassName+". readAllQueries must be plain SQL. JPQL, HQL not supported yet. "); } - + log.debug("Using readAllQuery: "+aClassName); String selectQueryString = mappedClass.getReadAllQuery().getQuery(); - + String countQueryString = "select count(*) from ("+selectQueryString+") as U"; - + // Add the limit clause if (pageSize != null && pageNumber != null && pageSize.intValue() > 0) { int offset = pageSize.intValue() * pageNumber.intValue(); selectQueryString += " limit "+pageSize+" OFFSET "+offset; } - + query = s.createSQLQuery(selectQueryString).addEntity(theClass); countQuery = s.createSQLQuery(countQueryString); - + query.setParameter("userId", userId); countQuery.setParameter("userId", userId); } - else - if (theClass != null) { + else + if (theClass != null) { String countQueryString = ""; if (returnTotal) countQueryString = "SELECT COUNT(getAllResult.ID) FROM " + aClassName + " getAllResult"; @@ -187,8 +191,8 @@ public PerceroList getAllByName(Object aName, Integer pageNumber } if (query != null) { - - log.debug("Get ALL: "+aClassName); + + log.debug("Get ALL: "+aClassName); long t1 = new Date().getTime(); List list = query.list(); long t2 = new Date().getTime(); @@ -196,10 +200,10 @@ public PerceroList getAllByName(Object aName, Integer pageNumber PerceroList result = new PerceroList( (List) SyncHibernateUtils.cleanObject(list, s, userId) ); long t3 = new Date().getTime(); log.debug("Clean Time: "+(t3-t2)); - + result.setPageNumber(pageNumber); result.setPageSize(pageSize); - + if (returnTotal && pageSize != null && pageNumber != null && pageSize.intValue() > 0){ result.setTotalLength(((Number)countQuery.uniqueResult()).intValue()); log.debug("Total Obs: "+result.getTotalLength()); @@ -228,7 +232,7 @@ public Integer countAllByName(String aClassName, String userId) throws Exception try { Query countQuery = null; Class theClass = MappedClass.forName(aClassName); - + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mappedClass = mcm.getMappedClassByClassName(aClassName); boolean isValidReadQuery = false; @@ -237,22 +241,22 @@ public Integer countAllByName(String aClassName, String userId) throws Exception isValidReadQuery = true; } } - + if(mappedClass.getReadAllQuery() != null && mappedClass.getReadAllQuery().getQuery() != null && !mappedClass.getReadAllQuery().getQuery().isEmpty()){ if((mappedClass.getReadAllQuery() instanceof JpqlQuery)){ throw new IllegalArgumentException("Illegal query type on:"+aClassName+". readAllQueries must be plain SQL. JPQL, HQL not supported yet. "); } - + String selectQueryString = mappedClass.getReadAllQuery().getQuery(); - + String countQueryString = "select count(*) from ("+selectQueryString+") as U"; - + countQuery = s.createSQLQuery(countQueryString); countQuery.setParameter("userId", userId); } else if (theClass != null) { String countQueryString = "SELECT COUNT(getAllResult.ID) FROM " + aClassName + " getAllResult"; - + // If the Read Query/Filter uses the ID, then we need to check against each ID here. if (isValidReadQuery) { String queryFilterString = mappedClass.getReadQuery().getQuery(); @@ -265,7 +269,7 @@ else if (theClass != null) { } String countQueryFilterString = queryFilterString; countQueryFilterString = countQueryString + " WHERE (" + countQueryFilterString + ") > 0"; - + countQuery = s.createQuery(countQueryFilterString); mappedClass.getReadQuery().setQueryParameters(countQuery, null, userId); } @@ -274,7 +278,7 @@ else if (theClass != null) { countQuery = s.createQuery(countQueryString); } } - + if (countQuery != null) { // log.debug(countQuery.toString()); Integer result = ((Number)countQuery.uniqueResult()).intValue(); @@ -283,13 +287,13 @@ else if (theClass != null) { else { return null; } - + } catch (Exception e) { log.error("Unable to countAllByName", e); } finally { s.close(); } - + return 0; } @@ -316,33 +320,33 @@ public List runQuery(MappedClass mappedClass, String queryName, Object[] return null; } - + @SuppressWarnings({ "rawtypes", "unchecked" }) protected static List processQueryResults(String resultClassName, Query updateFilter, List updateFilterResult) throws Exception { String[] returnAliases = updateFilter.getReturnAliases(); Type[] returnTypes = updateFilter.getReturnTypes(); - + String[] fieldNames = new String[returnAliases.length]; String[] getMethodNames = new String[returnAliases.length]; String[] setMethodNames = new String[returnAliases.length]; Class[] fieldClasses = new Class[returnAliases.length]; - + Class clazz = null; ClassPool pool = null; CtClass evalClass = null; - + if (returnAliases.length > 1) { try { clazz = MappedClass.forName(resultClassName); } catch(Exception e) { // Class must not yet exist, so let's create it. } - + if (clazz == null) { pool = ClassPool.getDefault(); evalClass = pool.makeClass(resultClassName); } - + // Create a new Class based on the result set. for(int i = 0; i < returnAliases.length; i++) { Type nextType = returnTypes[i]; @@ -355,29 +359,29 @@ protected static List processQueryResults(String resultClassName, Query // Do nothing. Simply means the field name is not a Number. } String nextUpperFieldName = nextFieldName.substring(0, 1).toUpperCase() + nextFieldName.substring(1); - + fieldNames[i] = nextFieldName; getMethodNames[i] = "get" + nextUpperFieldName; setMethodNames[i] = "set" + nextUpperFieldName; fieldClasses[i] = nextType.getReturnedClass(); - + if (evalClass != null) { evalClass.addField(CtField.make("private " + fieldClasses[i].getCanonicalName() + " " + nextFieldName + ";", evalClass)); evalClass.addMethod(CtMethod.make("public void " + setMethodNames[i] + "(" + fieldClasses[i].getCanonicalName() + " value) {this." + nextFieldName + " = value;}", evalClass)); evalClass.addMethod(CtMethod.make("public " + nextTypeCanonicalName +" " + getMethodNames[i] + "() {return this." + nextFieldName + ";}", evalClass)); } } - + if (clazz == null && evalClass != null) { clazz = evalClass.toClass(); } } - + List results = new ArrayList(); // Now populate the newly created objects. for(Object nextResult : (List)updateFilterResult) { - + if (nextResult instanceof Object[]) { Object nextObject = clazz.newInstance(); for(int i = 0; i < returnAliases.length; i++) { @@ -385,15 +389,15 @@ protected static List processQueryResults(String resultClassName, Query Method setMethod = clazz.getDeclaredMethod(setMethodNames[i], formalParams); setMethod.invoke(nextObject, ((Object[])nextResult)[i]); } - + results.add(nextObject); } else results.add(nextResult); } - + return results; } - + //@SuppressWarnings({ "rawtypes", "unchecked" }) public IPerceroObject findById(ClassIDPair classIdPair, String userId) { boolean hasReadQuery = false; @@ -409,7 +413,7 @@ public IPerceroObject findById(ClassIDPair classIdPair, String userId) { try { boolean hasAccess = true; IPerceroObject result = systemGetById(classIdPair); - + if (result != null) { if (hasReadQuery) { if (s == null) @@ -443,21 +447,29 @@ public IPerceroObject findById(ClassIDPair classIdPair, String userId) { } return null; } - + @SuppressWarnings({ "rawtypes", "unchecked" }) public IPerceroObject systemGetById(ClassIDPair classIdPair) { + return systemGetById(classIdPair, false); + } + + public IPerceroObject systemGetById(ClassIDPair classIdPair, boolean ignoreCache) { Session s = null; try { Class theClass = MappedClass.forName(classIdPair.getClassName().toString()); - + if (theClass != null) { - IPerceroObject result = retrieveFromRedisCache(classIdPair); + IPerceroObject result = null; + + if(!ignoreCache) + result = retrieveFromRedisCache(classIdPair); + String key = null; if (result == null) { s = appSessionFactory.openSession(); result = (IPerceroObject) s.get(theClass, classIdPair.getID()); - + // Now put the object in the cache. if (result != null) { key = RedisKeyUtils.classIdPair(result.getClass().getCanonicalName(), result.getID()); @@ -478,7 +490,7 @@ public IPerceroObject systemGetById(ClassIDPair classIdPair) { if (cacheTimeout > 0 && key != null) { redisDataStore.expire(key, cacheTimeout, TimeUnit.SECONDS); } - + return result; } else { return null; @@ -499,7 +511,7 @@ public IPerceroObject systemGetByIdWithClassAndSession(ClassIDPair classIdPair, if (theClass != null) { IPerceroObject result = retrieveFromRedisCache(classIdPair); String key = null; - + if (result == null) { if (s == null || !s.isOpen()) { s = appSessionFactory.openSession(); @@ -508,7 +520,7 @@ public IPerceroObject systemGetByIdWithClassAndSession(ClassIDPair classIdPair, sessionAlreadyOpen = true; } result = (IPerceroObject) s.get(theClass, classIdPair.getID()); - + // Now put the object in the cache. if (result != null) { key = RedisKeyUtils.classIdPair(result.getClass().getCanonicalName(), result.getID()); @@ -523,12 +535,12 @@ public IPerceroObject systemGetByIdWithClassAndSession(ClassIDPair classIdPair, else { key = RedisKeyUtils.classIdPair(result.getClass().getCanonicalName(), result.getID()); } - + // (Re)Set the expiration. if (cacheTimeout > 0 && key != null) { redisDataStore.expire(key, cacheTimeout, TimeUnit.SECONDS); } - + return result; } else { return null; @@ -545,7 +557,7 @@ public IPerceroObject systemGetByIdWithClassAndSession(ClassIDPair classIdPair, } return null; } - + @SuppressWarnings({ "rawtypes", "unchecked" }) private IPerceroObject retrieveFromRedisCache(ClassIDPair classIdPair) throws Exception { IPerceroObject result = null; @@ -553,7 +565,7 @@ private IPerceroObject retrieveFromRedisCache(ClassIDPair classIdPair) throws Ex IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); Class theClass = MappedClass.forName(classIdPair.getClassName()); MappedClass mc = mcm.getMappedClassByClassName(classIdPair.getClassName()); - + String key = RedisKeyUtils.classIdPair(classIdPair.getClassName(), classIdPair.getID()); String jsonObjectString = (String) redisDataStore.getValue(key); if (jsonObjectString != null) { @@ -579,20 +591,20 @@ private IPerceroObject retrieveFromRedisCache(ClassIDPair classIdPair) throws Ex } } } - } - + } + return result; } @SuppressWarnings({ "rawtypes", "unchecked" }) private Map retrieveFromRedisCache(ClassIDPairs classIdPairs, Boolean pleaseSetTimeout) throws Exception { Map result = new HashMap(classIdPairs.getIds().size()); - + if (cacheTimeout > 0) { IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); Class theClass = MappedClass.forName(classIdPairs.getClassName()); MappedClass mc = mcm.getMappedClassByClassName(classIdPairs.getClassName()); - + Set keys = new HashSet(classIdPairs.getIds().size()); Iterator itrIds = classIdPairs.getIds().iterator(); while (itrIds.hasNext()) { @@ -612,7 +624,7 @@ private Map retrieveFromRedisCache(ClassIDPairs classIdP keys.add(nextKey); } } - + // String key = RedisKeyUtils.classIdPair(classIdPair.getClassName(), classIdPair.getID()); List jsonObjectStrings = redisDataStore.getValues(keys); Iterator itrJsonObjectStrings = jsonObjectStrings.iterator(); @@ -628,7 +640,7 @@ private Map retrieveFromRedisCache(ClassIDPairs classIdP IPerceroObject nextPerceroObject = (IPerceroObject) safeObjectMapper.readValue(jsonObjectString, theClass); result.put( nextPerceroObject.getID(), nextPerceroObject); } - + } // else { // // Check MappedClass' child classes. @@ -644,26 +656,26 @@ private Map retrieveFromRedisCache(ClassIDPairs classIdP // } // } } - + if (pleaseSetTimeout) { redisDataStore.expire(keys, cacheTimeout, TimeUnit.SECONDS); } } - + return result; } - + @SuppressWarnings({ "rawtypes" }) public Boolean getReadAccess(ClassIDPair classIdPair, String userId) { Session s = appSessionFactory.openSession(); try { Class theClass = MappedClass.forName(classIdPair.getClassName().toString()); - + if (theClass != null) { IPerceroObject parent = (IPerceroObject) s.get(theClass, classIdPair.getID()); - + boolean hasAccess = (parent != null); - + if (hasAccess) { IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mappedClass = mcm.getMappedClassByClassName(classIdPair.getClassName()); @@ -677,7 +689,7 @@ public Boolean getReadAccess(ClassIDPair classIdPair, String userId) { } } } - + return hasAccess; } } catch (Exception e) { @@ -687,19 +699,19 @@ public Boolean getReadAccess(ClassIDPair classIdPair, String userId) { } return false; } - + @SuppressWarnings({ "rawtypes" }) public Boolean getDeleteAccess(ClassIDPair classIdPair, String userId) { Session s = appSessionFactory.openSession(); try { Class theClass = MappedClass.forName(classIdPair.getClassName().toString()); - + if (theClass != null) { IPerceroObject parent = (IPerceroObject) s.get(theClass, classIdPair.getID()); - + if (parent == null) return true; - + boolean hasAccess = true; IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mappedClass = mcm.getMappedClassByClassName(classIdPair.getClassName()); @@ -712,7 +724,7 @@ public Boolean getDeleteAccess(ClassIDPair classIdPair, String userId) { hasAccess = false; } } - + return hasAccess; } } catch (Exception e) { @@ -722,11 +734,11 @@ public Boolean getDeleteAccess(ClassIDPair classIdPair, String userId) { } return false; } - + @SuppressWarnings({ "rawtypes", "unchecked" }) public List findByIds(ClassIDPairs classIdPairs, String userId) { List result = null; - + boolean hasAccess = true; Class theClass = null; try { @@ -735,7 +747,7 @@ public List findByIds(ClassIDPairs classIdPairs, String userId) log.error("Unable to get Class from class name " + classIdPairs.getClassName(), e2); return result; } - + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mappedClass = mcm.getMappedClassByClassName(classIdPairs.getClassName()); boolean isValidReadQuery = false; @@ -754,13 +766,13 @@ public List findByIds(ClassIDPairs classIdPairs, String userId) try { // Attempt to get as many from the cache as possible... Map cachedObjects = retrieveFromRedisCache(classIdPairs, true); - + if (!cachedObjects.isEmpty()) { result = new ArrayList(cachedObjects.size()); List cachedResults = new ArrayList(cachedObjects.size()); cachedResults.addAll(cachedObjects.values()); idsSet.removeAll(cachedObjects.keySet()); - + // If there is a read query, we need to check each object through that query. if (isValidReadQuery) { Session s = null; @@ -768,7 +780,7 @@ public List findByIds(ClassIDPairs classIdPairs, String userId) // Need to clean each result for the user. // TODO: Need to figure out how to NOT open the Session if we don't have to. The problem lies in the SyncHibernateUtils.cleanObject function s = appSessionFactory.openSession(); - + Iterator itrCachedResults = cachedResults.iterator(); while (itrCachedResults.hasNext()) { IPerceroObject nextCachedResult = itrCachedResults.next(); @@ -779,7 +791,7 @@ public List findByIds(ClassIDPairs classIdPairs, String userId) if (readFilterResult == null || readFilterResult.intValue() <= 0) hasAccess = false; } - + if (hasAccess) { result.add( (IPerceroObject) SyncHibernateUtils.cleanObject(nextCachedResult, s, userId) ); } @@ -801,7 +813,7 @@ public List findByIds(ClassIDPairs classIdPairs, String userId) // TODO: Need to figure out how to NOT open the Session if we don't have to. The problem lies in the SyncHibernateUtils.cleanObject function // TODO: Maybe only clean relationships that are in mappedClass.readAccessRightsFieldReferences? s = appSessionFactory.openSession(); - + Iterator itrCachedResults = cachedResults.iterator(); while (itrCachedResults.hasNext()) { IPerceroObject nextCachedResult = itrCachedResults.next(); @@ -827,7 +839,7 @@ public List findByIds(ClassIDPairs classIdPairs, String userId) // We errored out here, but we can still try and retrieve from the database. log.error("Error retrieving objects from redis cache, attempting to retrieve from database", e1); } - + // Now get the rest from the database. if (!idsSet.isEmpty()) { String queryString = "SELECT findByIdsResult FROM " + classIdPairs.getClassName() + " findByIdsResult WHERE findByIdsResult.ID IN (:idsSet)"; @@ -849,18 +861,18 @@ public List findByIds(ClassIDPairs classIdPairs, String userId) queryFilterString = queryFilterString.replaceAll(":ID", "findByIdsResult.ID"); } queryFilterString = queryString + " AND (" + queryFilterString + ") > 0"; - + query = s.createQuery(queryFilterString); mappedClass.getReadQuery().setQueryParameters(query, null, userId); } else { query = s.createQuery(queryString); } - + query.setParameterList("idsSet", idsSet); List queryResult = query.list(); List cleanedDatabaseObjects = (List) SyncHibernateUtils.cleanObject(queryResult, s, userId); - + // Need to put results into cache. if (cacheTimeout > 0) { Map mapJsonObjectStrings = new HashMap(cleanedDatabaseObjects.size()); @@ -868,16 +880,16 @@ public List findByIds(ClassIDPairs classIdPairs, String userId) while (itrDatabaseObjects.hasNext()) { IPerceroObject nextDatabaseObject = itrDatabaseObjects.next(); String nextCacheKey = RedisKeyUtils.classIdPair(nextDatabaseObject.getClass().getCanonicalName(), nextDatabaseObject.getID()); - + mapJsonObjectStrings.put(nextCacheKey, ((BaseDataObject)nextDatabaseObject).toJson()); } - + // Store the objects in redis. redisDataStore.setValues(mapJsonObjectStrings); // (Re)Set the expiration. redisDataStore.expire(mapJsonObjectStrings.keySet(), cacheTimeout, TimeUnit.SECONDS); } - + if (result == null) { result = cleanedDatabaseObjects; } @@ -905,7 +917,7 @@ public IPerceroObject findUnique(IPerceroObject theQueryObject, String userId) { Session s = appSessionFactory.openSession(); try { Class objectClass = theQueryObject.getClass(); - + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mappedClass = mcm.getMappedClassByClassName(objectClass.getName()); if (mappedClass != null) { @@ -994,7 +1006,7 @@ public List findByExample(IPerceroObject theQueryObject, List systemFindByExample(IPerceroObject theQueryObject, List excludeProperties) { Session s = appSessionFactory.openSession(); @@ -1004,23 +1016,23 @@ public List systemFindByExample(IPerceroObject theQueryObject, L BaseDataObjectPropertySelector propertySelector = new BaseDataObjectPropertySelector(excludeProperties); example.setPropertySelector(propertySelector); criteria.add(example); - + List result = criteria.list(); result = (List) SyncHibernateUtils.cleanObject(result, s); - + return (List) result; } catch (Exception e) { log.error("Unable to findByExample", e); } finally { s.close(); } - + return null; } @SuppressWarnings("unchecked") public List searchByExample(IPerceroObject theQueryObject, - List excludeProperties, String userId) { + List excludeProperties, String userId) { Session s = appSessionFactory.openSession(); try { Criteria criteria = s.createCriteria(theQueryObject.getClass()); @@ -1032,7 +1044,7 @@ public List searchByExample(IPerceroObject theQueryObject, List result = criteria.list(); result = (List) SyncHibernateUtils.cleanObject(result, s, userId); - + return result; } catch (Exception e) { log.error("Unable to searchByExample", e); @@ -1047,7 +1059,7 @@ public List searchByExample(IPerceroObject theQueryObject, public IPerceroObject systemCreateObject(IPerceroObject perceroObject) throws SyncException { Session s = appSessionFactory.openSession(); - + try { // Make sure object has an ID. IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); @@ -1064,14 +1076,14 @@ public IPerceroObject systemCreateObject(IPerceroObject perceroObject) return perceroObject; } } - + Transaction tx = s.beginTransaction(); tx.begin(); s.save(perceroObject); tx.commit(); - + s.refresh(perceroObject); - + perceroObject = (IPerceroObject) SyncHibernateUtils.cleanObject(perceroObject, s); deepCleanObject(perceroObject, mappedClass, s, null); @@ -1128,26 +1140,26 @@ else if (fieldObject instanceof Collection) { } } } - + if (!keysToDelete.isEmpty()) { redisDataStore.deleteKeys(keysToDelete); // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); } } - + return perceroObject; } catch(PropertyValueException pve) { log.error("Error creating object", pve); - + SyncDataException sde = new SyncDataException(SyncDataException.MISSING_REQUIRED_FIELD, SyncDataException.MISSING_REQUIRED_FIELD_CODE, "Missing required field " + pve.getPropertyName()); sde.fieldName = pve.getPropertyName(); throw sde; } catch(Exception e) { log.error("Error creating object", e); - + SyncDataException sde = new SyncDataException(SyncDataException.CREATE_OBJECT_ERROR, SyncDataException.CREATE_OBJECT_ERROR_CODE); throw sde; } @@ -1157,7 +1169,7 @@ else if (fieldObject instanceof Collection) { } } } - + private void deepCleanObject(IPerceroObject perceroObject, MappedClass mappedClass, Session s, String userId) { if (!(perceroObject instanceof IRootObject)) { for(MappedField nextMappedField : mappedClass.externalizableFields) { @@ -1219,7 +1231,7 @@ public IPerceroObject createObject(IPerceroObject perceroObject, String userId) } } - + //////////////////////////////////////////////////// // PUT @@ -1278,113 +1290,15 @@ public IPerceroObject systemPutObject(IPerceroObject perceroObject, Map 0) { - // TODO: Also need to update the caches of anything object that is related to this object. - String key = RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID()); - if (redisDataStore.hasKey(key)) { - redisDataStore.setValue(key, ((BaseDataObject)perceroObject).toJson()); - } - - // Iterate through each changed object and reset the cache for that object. - if (changedFields != null) { -// Iterator>> itrChangedFieldEntrySet = changedFields.entrySet().iterator(); -// Set keysToDelete = new HashSet(); -// while (itrChangedFieldEntrySet.hasNext()) { -// Map.Entry> nextEntry = itrChangedFieldEntrySet.next(); -// ClassIDPair thePair = nextEntry.getKey(); -// if (!thePair.comparePerceroObject(perceroObject)) { -// String nextKey = RedisKeyUtils.classIdPair(thePair.getClassName(), thePair.getID()); -// keysToDelete.add(nextKey); -// } -// } - Iterator itrChangedFieldKeyset = changedFields.keySet().iterator(); - Set keysToDelete = new HashSet(); - while (itrChangedFieldKeyset.hasNext()) { - ClassIDPair thePair = itrChangedFieldKeyset.next(); - if (!thePair.comparePerceroObject(perceroObject)) { - String nextKey = RedisKeyUtils.classIdPair(thePair.getClassName(), thePair.getID()); - keysToDelete.add(nextKey); - } - } - - if (!keysToDelete.isEmpty()) { - redisDataStore.deleteKeys(keysToDelete); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } - } - else { - // No changedFields? We should never get here? - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(perceroObject.getClass().getName()); - Iterator itrToManyFields = mappedClass.toManyFields.iterator(); - while(itrToManyFields.hasNext()) { - MappedField nextMappedField = itrToManyFields.next(); - Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); - if (fieldObject != null) { - if (fieldObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); - if (redisDataStore.hasKey(nextKey)) { - redisDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } - } - else if (fieldObject instanceof Collection) { - Iterator itrFieldObject = ((Collection) fieldObject).iterator(); - while(itrFieldObject.hasNext()) { - Object nextListObject = itrFieldObject.next(); - if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - if (redisDataStore.hasKey(nextKey)) { - redisDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } - } - } - } - } - } - Iterator itrToOneFields = mappedClass.toOneFields.iterator(); - while(itrToOneFields.hasNext()) { - MappedField nextMappedField = itrToOneFields.next(); - Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); - if (fieldObject != null) { - if (fieldObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); - if (redisDataStore.hasKey(nextKey)) { - redisDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } - } - else if (fieldObject instanceof Collection) { - Iterator itrFieldObject = ((Collection) fieldObject).iterator(); - while(itrFieldObject.hasNext()) { - Object nextListObject = itrFieldObject.next(); - if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - if (redisDataStore.hasKey(nextKey)) { - redisDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } - } - } - } - } - } - } - } + cacheManager.updateCachedObject(perceroObject, changedFields); return perceroObject; } catch (Exception e) { log.error("Error putting object", e); - + SyncDataException sde = new SyncDataException(SyncDataException.UPDATE_OBJECT_ERROR, SyncDataException.UPDATE_OBJECT_ERROR_CODE); throw sde; } finally { @@ -1392,7 +1306,7 @@ else if (fieldObject instanceof Collection) { } } - + //////////////////////////////////////////////////// // DELETE @@ -1405,7 +1319,7 @@ public Boolean deleteObject(ClassIDPair theClassIdPair, String userId) throws Sy if (perceroObject == null) { return true; } - + boolean hasAccess = true; IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mappedClass = mcm.getMappedClassByClassName(theClassIdPair.getClassName()); @@ -1422,7 +1336,7 @@ public Boolean deleteObject(ClassIDPair theClassIdPair, String userId) throws Sy if (hasAccess) { if (systemDeleteObject(perceroObject)) { return true; - } + } else { return true; } @@ -1450,7 +1364,7 @@ public Boolean systemDeleteObject(IPerceroObject perceroObject) throws SyncExcep // Make sure all associated objects are loaded. // Clean the object before deletion because otherwise it will fail. perceroObject = (IPerceroObject) SyncHibernateUtils.cleanObject(perceroObjectToDelete, s); - + Transaction tx = s.beginTransaction(); tx.begin(); s.delete(perceroObjectToDelete); @@ -1509,7 +1423,7 @@ else if (fieldObject instanceof Collection) { } } } - + if (!keysToDelete.isEmpty()) { redisDataStore.deleteKeys(keysToDelete); // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? @@ -1520,7 +1434,7 @@ else if (fieldObject instanceof Collection) { result = true; } catch (Exception e) { log.error("Error deleting object", e); - + SyncDataException sde = new SyncDataException(SyncDataException.DELETE_OBJECT_ERROR, SyncDataException.DELETE_OBJECT_ERROR_CODE); throw sde; } finally { @@ -1550,14 +1464,14 @@ public Object cleanObject(Object object, Session s, String userId) { } } } - - + + @SuppressWarnings("rawtypes") public Map> getChangedMappedFields(IPerceroObject newObject) { Map> result = new HashMap>(); Collection baseObjectResult = null; ClassIDPair basePair = new ClassIDPair(newObject.getID(), newObject.getClass().getCanonicalName()); - + String className = newObject.getClass().getCanonicalName(); IPerceroObject oldObject = systemGetById(new ClassIDPair(newObject.getID(), className)); IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); @@ -1573,12 +1487,12 @@ public Map> getChangedMappedFields(IPercero result.put(basePair, baseObjectResult); } baseObjectResult.add(nextMappedField); - + // If this is a relationship field, then need to grab the old and new values. if (nextMappedField.getReverseMappedField() != null) { if (nextMappedField instanceof MappedFieldPerceroObject) { MappedFieldPerceroObject nextMappedFieldPerceroObject = (MappedFieldPerceroObject) nextMappedField; - + IPerceroObject oldReversePerceroObject = (IPerceroObject) nextMappedFieldPerceroObject.getGetter().invoke(oldObject); if (oldReversePerceroObject != null) { ClassIDPair oldReversePair = new ClassIDPair(oldReversePerceroObject.getID(), oldReversePerceroObject.getClass().getCanonicalName()); @@ -1589,7 +1503,7 @@ public Map> getChangedMappedFields(IPercero } oldReverseChangedFields.add(nextMappedField.getReverseMappedField()); } - + IPerceroObject newReversePerceroObject = (IPerceroObject) nextMappedFieldPerceroObject.getGetter().invoke(newObject); if (newReversePerceroObject != null) { ClassIDPair newReversePair = new ClassIDPair(newReversePerceroObject.getID(), newReversePerceroObject.getClass().getCanonicalName()); @@ -1603,22 +1517,22 @@ public Map> getChangedMappedFields(IPercero } else if (nextMappedField instanceof MappedFieldList) { MappedFieldList nextMappedFieldList = (MappedFieldList) nextMappedField; - + List oldReverseList = (List) nextMappedFieldList.getGetter().invoke(oldObject); if (oldReverseList == null) oldReverseList = new ArrayList(); - + List newReverseList = (List) nextMappedFieldList.getGetter().invoke(newObject); if (newReverseList == null) newReverseList = new ArrayList(); - + // Compare each item in the lists. Collection oldChangedList = retrieveObjectsNotInCollection(oldReverseList, newReverseList); Iterator itrOldChangedList = oldChangedList.iterator(); while (itrOldChangedList.hasNext()) { BaseDataObject nextOldChangedObject = (BaseDataObject) itrOldChangedList.next(); ClassIDPair nextOldReversePair = BaseDataObject.toClassIdPair(nextOldChangedObject); - + // Old object is not in new list, so add to list of changed fields. Collection changedFields = result.get(nextOldReversePair); if (changedFields == null) { @@ -1633,7 +1547,7 @@ else if (nextMappedField instanceof MappedFieldList) { while (itrNewChangedList.hasNext()) { BaseDataObject nextNewChangedObject = (BaseDataObject) itrNewChangedList.next(); ClassIDPair nextNewReversePair = BaseDataObject.toClassIdPair(nextNewChangedObject); - + // Old object is not in new list, so add to list of changed fields. Collection changedFields = result.get(nextNewReversePair); if (changedFields == null) { @@ -1650,10 +1564,10 @@ else if (nextMappedField instanceof MappedFieldList) { baseObjectResult.add(nextMappedField); } } - + return result; } - + @SuppressWarnings({ "rawtypes", "unchecked" }) private Collection retrieveObjectsNotInCollection(Collection baseList, Collection compareToList) { Collection result = new HashSet(); @@ -1665,24 +1579,24 @@ private Collection retrieveObjectsNotInCollection(Collection baseList, Collectio BaseDataObject nextBasePerceroObject = (BaseDataObject) itrBaseList.next(); ClassIDPair nextBasePair = BaseDataObject.toClassIdPair(nextBasePerceroObject); nextBasePerceroObject = (BaseDataObject) systemGetById(nextBasePair); - + itrCompareToList = compareToList.iterator(); matchFound = false; while (itrCompareToList.hasNext()) { BaseDataObject nextCompareToPerceroObject = (BaseDataObject) itrCompareToList.next(); nextCompareToPerceroObject = (BaseDataObject) systemGetById(BaseDataObject.toClassIdPair(nextCompareToPerceroObject)); - + if (nextBasePerceroObject.getClass() == nextCompareToPerceroObject.getClass() && nextBasePerceroObject.getID().equalsIgnoreCase(nextCompareToPerceroObject.getID())) { matchFound = true; break; } } - + if (!matchFound) { result.add(nextBasePerceroObject); } } - + return result; } } diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties index e55f929..6a6b6bf 100644 --- a/src/main/resources/log4j.properties +++ b/src/main/resources/log4j.properties @@ -1,5 +1,7 @@ log4j.rootLogger=info, stdout +log4j.category.com.percero.agents.sync.jobs=DEBUG + log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index 6f3b4fb..d3a16c2 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -45,6 +45,7 @@ + diff --git a/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java index 3f25143..7d9a0bf 100644 --- a/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java +++ b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java @@ -1,18 +1,123 @@ package com.percero.agents.sync.jobs; +import com.percero.example.Email; +import com.percero.example.Person; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; + /** * Created by Jonathan Samples on 9/4/15. */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:spring/update_table_processor.xml" }) public class UpdateTableProcessorTest { + + @Autowired + UpdateTableProcessorFactory processorFactory; + @Autowired + UpdateTableConnectionFactory connectionFactory; + @Autowired + UpdateTablePoller poller; + + String tableName = "update_table"; + + @Before + public void before() throws Exception{ + // Disable the poller so it doesn't step on our toes + poller.enabled = false; + try(Connection connection = connectionFactory.getConnection(); + Statement statement = connection.createStatement()) + { + String sql = "delete from " + tableName; + statement.executeUpdate(sql); + + // Add some fixture data + statement.executeUpdate("insert into " + tableName + "(tableName, rowId, type) values " + + "('Email','1','UPDATE')," + + "('Person','1','INSERT')," + + "('Block','1','DELETE')"); + } + } + @Test - public void test(){ - System.out.println("A Test ran"); + public void test() throws Exception{ + try(Connection connection = connectionFactory.getConnection(); + Statement statement = connection.createStatement();) + { + String sql = "select count(*) as 'count' from " + tableName; + ResultSet resultSet = statement.executeQuery(sql); + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(3, resultSet.getInt("count")); + } } + + @Test + public void getClassForTableName_NoTableAnnotation() throws Exception{ + UpdateTableProcessor processor = processorFactory.getProcessor(tableName); + Class clazz = processor.getClassForTableName("Email"); + Assert.assertEquals(Email.class, clazz); + } + + @Test + public void getClassForTableName_TableAnnotation() throws Exception{ + UpdateTableProcessor processor = processorFactory.getProcessor(tableName); + Class clazz = processor.getClassForTableName("Person"); + Assert.assertEquals(Person.class, clazz); + } + + @Test + public void getClassForTableName_NotFound() throws Exception{ + UpdateTableProcessor processor = processorFactory.getProcessor(tableName); + Class clazz = processor.getClassForTableName("NotAnEntity"); + Assert.assertNull(clazz); + } + + @Test + public void getRow() throws Exception { + UpdateTableProcessor processor = processorFactory.getProcessor(tableName); + UpdateTableRow row = processor.getRow(); + + Assert.assertNotNull(row); + Assert.assertNotNull(row.getLockId()); + Assert.assertNotNull(row.getLockDate()); + + String sql = "select * from "+tableName+" where ID="+row.getID(); + + try(Connection connection = connectionFactory.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) + { + Assert.assertTrue(resultSet.next()); + Assert.assertNotNull(resultSet.getInt("lockID")); + Assert.assertNotNull(resultSet.getDate("lockDate")); + } + } + + + @Test + public void process() throws Exception { + UpdateTableProcessor processor = processorFactory.getProcessor(tableName); + ProcessorResult result = processor.process(); + Assert.assertEquals(3, result.getTotal()); + Assert.assertEquals(0, result.getNumFailed()); + Assert.assertTrue(result.isSuccess()); + try(Connection connection = connectionFactory.getConnection(); + Statement statement = connection.createStatement()) + { + String sql = "select count(*) as 'count' from " + tableName; + ResultSet resultSet = statement.executeQuery(sql); + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(0, resultSet.getInt("count")); + } + } + } diff --git a/src/test/java/com/percero/example/Person.java b/src/test/java/com/percero/example/Person.java index 312be9d..fb1b7b0 100644 --- a/src/test/java/com/percero/example/Person.java +++ b/src/test/java/com/percero/example/Person.java @@ -19,6 +19,7 @@ import java.util.List; @Entity +@Table(name = "Person") @EntityInterface(interfaceClass=IUserAnchor.class) public class Person extends BaseDataObject implements Serializable, IUserAnchor { diff --git a/src/test/resources/properties/test/env.properties b/src/test/resources/properties/test/env.properties index 015ab17..2788f4e 100644 --- a/src/test/resources/properties/test/env.properties +++ b/src/test/resources/properties/test/env.properties @@ -65,4 +65,8 @@ pulseHttpAuth.hostPortAndContext=https://localhost:8900/auth pulseHttpAuth.trustAllCerts=true # Update Table -updateTable.tableNames=update_log \ No newline at end of file +updateTable.tableNames=update_log +updateTable.jdbcUrl=jdbc:mysql://localhost/as_example +updateTable.username=root +updateTable.password=root +updateTable.driverClassName=com.mysql.jdbc.Driver \ No newline at end of file diff --git a/src/test/sql/example_schema.sql b/src/test/sql/example_schema.sql index e69de29..c367916 100644 --- a/src/test/sql/example_schema.sql +++ b/src/test/sql/example_schema.sql @@ -0,0 +1,15 @@ +CREATE DATABASE IF NOT EXISTS as_example; + +DROP TABLE IF EXISTS `update_table`; +CREATE TABLE `update_table` ( + `ID` int(11) NOT NULL AUTO_INCREMENT, + `tableName` varchar(255) DEFAULT NULL, + `rowID` varchar(255) DEFAULT NULL, + `type` enum('INSERT','UPDATE','DELETE') NOT NULL DEFAULT 'UPDATE', + `lockID` int(11) DEFAULT NULL, + `lockDate` datetime DEFAULT NULL, + `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`ID`), + KEY `tableName` (`tableName`), + KEY `lockID` (`lockID`) +); \ No newline at end of file From 74e425491c3c5577bf9aa18404d54cc87549dab9 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Tue, 8 Sep 2015 08:22:57 -0700 Subject: [PATCH 4/9] Added redis set for class->objID --- .../percero/agents/sync/access/RedisAccessManager.java | 9 ++++++++- .../com/percero/agents/sync/access/RedisKeyUtils.java | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java index cdcc43b..75ee296 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java @@ -830,7 +830,11 @@ private Long upsertRedisAccessJournal(String userId, String clientId, String cla // Need to add the ObjectID to the Client's AccessJournals set. String clientAccessJournalKey = RedisKeyUtils.clientAccessJournal(clientId); redisDataStore.addSetValue(clientAccessJournalKey, RedisKeyUtils.objectId(className, classId)); - + + // Add to the class's AccessJournals set + String classAccessJournalKey = RedisKeyUtils.classAccessJournal(className); + redisDataStore.addSetValue(classAccessJournalKey, classId); + // Need to add the ClientID to the Object's AccessJournal set. return redisDataStore.addSetValue(key, clientId); } catch(Exception e) { @@ -984,6 +988,9 @@ public void removeAccessJournalsByObject(ClassIDPair classIdPair) { // } // } + String classAccessJournalKey = RedisKeyUtils.classAccessJournal(classIdPair.getClassName()); + redisDataStore.removeSetValue(classAccessJournalKey, classIdPair.getID()); + // Now delete the AccessJournal record. redisDataStore.deleteKey(accessJournalKey); } diff --git a/src/main/java/com/percero/agents/sync/access/RedisKeyUtils.java b/src/main/java/com/percero/agents/sync/access/RedisKeyUtils.java index 4cd1cb4..8dd3df0 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisKeyUtils.java +++ b/src/main/java/com/percero/agents/sync/access/RedisKeyUtils.java @@ -111,6 +111,10 @@ public static String accessJournalPrefix() { public static String clientAccessJournal(String clientId) { return (new StringBuilder(INT_64).append("c:").append(ACCESS_JOURNAL_PREFIX).append(clientId)).toString(); } + + public static String classAccessJournal(String className) { + return (new StringBuilder(INT_64).append("class:").append(ACCESS_JOURNAL_PREFIX).append(className)).toString(); + } public static String historicalObject(String className, String classId) { return (new StringBuilder(INT_128).append("ho:").append(className).append(":").append(classId)).toString(); From d53729ceee997e2b78783c62c37261eeaedb4237 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Tue, 8 Sep 2015 08:24:18 -0700 Subject: [PATCH 5/9] Removed bad comment --- .../java/com/percero/agents/sync/access/RedisAccessManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java index 75ee296..5149ffa 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java @@ -566,7 +566,6 @@ public void destroyClient(String clientId) { * This function checks to see if a User has no more UserDevices. If they do not have * any, then it removes all the User's entries in ALL AccessJournal sets. * - * @param userId * @param clientId */ @SuppressWarnings("unchecked") From 4d1c298758c8a3146769699d1e647557a57cfbb7 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 9 Sep 2015 08:13:28 -0700 Subject: [PATCH 6/9] Finished routines to maintain class access journals --- .../sync/access/RedisAccessManager.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java index 5149ffa..65b6d26 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java @@ -566,7 +566,7 @@ public void destroyClient(String clientId) { * This function checks to see if a User has no more UserDevices. If they do not have * any, then it removes all the User's entries in ALL AccessJournal sets. * - * @param clientId + * @param clientId */ @SuppressWarnings("unchecked") @Transactional @@ -577,6 +577,18 @@ private void deleteClientAccessJournals(String clientId) { // Now remove the Client's AccessJournal key. redisDataStore.deleteKey(clientAccessJournalsKey); + + // Check to see if each Object's access journal set is empty and if so remove it + // from the class access journal set + for(String caj : clientAccessJournals){ + if(redisDataStore.getSetIsEmpty(RedisKeyUtils.ACCESS_JOURNAL_PREFIX+caj)){ + String[] parts = caj.split(":"); + String className = parts[0]; + String ID = parts[1]; + String classAccessJournalKey = RedisKeyUtils.classAccessJournal(className); + redisDataStore.removeSetValue(classAccessJournalKey, ID); + } + } } @SuppressWarnings("unchecked") @@ -831,8 +843,11 @@ private Long upsertRedisAccessJournal(String userId, String clientId, String cla redisDataStore.addSetValue(clientAccessJournalKey, RedisKeyUtils.objectId(className, classId)); // Add to the class's AccessJournals set - String classAccessJournalKey = RedisKeyUtils.classAccessJournal(className); - redisDataStore.addSetValue(classAccessJournalKey, classId); + if(classId != null && !classId.isEmpty() && !classId.equals("0")) { + log.info("Adding to class AccessJournals: "+classId); + String classAccessJournalKey = RedisKeyUtils.classAccessJournal(className); + redisDataStore.addSetValue(classAccessJournalKey, classId); + } // Need to add the ClientID to the Object's AccessJournal set. return redisDataStore.addSetValue(key, clientId); From 8b603e417817d37e4e1a821a9f3434965b98b4e7 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Thu, 10 Sep 2015 08:17:00 -0700 Subject: [PATCH 7/9] Implemented whole table update --- .../agents/sync/access/AccessManager.java | 10 +++ .../agents/sync/access/IAccessManager.java | 2 + .../sync/access/RedisAccessManager.java | 8 +++ .../agents/sync/datastore/RedisDataStore.java | 5 ++ .../sync/jobs/UpdateTableProcessor.java | 66 +++++++++++++++---- .../jobs/UpdateTableProcessorFactory.java | 6 +- 6 files changed, 85 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/access/AccessManager.java b/src/main/java/com/percero/agents/sync/access/AccessManager.java index 33b1fae..6bfcfac 100644 --- a/src/main/java/com/percero/agents/sync/access/AccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/AccessManager.java @@ -870,6 +870,16 @@ public List getObjectAccessJournals(String className, String classId) th return result; } + @Override + public Set getClassAccessJournalIDs(String className) { + return null; + } + + @Override + public long getNumClientsInterestedInWholeClass(String className) { + return 0; + } + @SuppressWarnings("unchecked") public Map> getClientAccessess(Collection classIdPairs) throws Exception { Map> result = new HashMap>(); diff --git a/src/main/java/com/percero/agents/sync/access/IAccessManager.java b/src/main/java/com/percero/agents/sync/access/IAccessManager.java index 7875993..6509912 100644 --- a/src/main/java/com/percero/agents/sync/access/IAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/IAccessManager.java @@ -153,6 +153,8 @@ public Set findClientByUserIdDeviceId(String deviceId, String userId) * @throws Exception */ public List getObjectAccessJournals(String className, String classId) throws Exception; + Set getClassAccessJournalIDs(String className); + long getNumClientsInterestedInWholeClass(String className); public void removeAccessJournalsByObject(ClassIDPair classIdPair); diff --git a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java index 65b6d26..dbea315 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java @@ -899,6 +899,14 @@ public List getObjectAccessJournals(String className, String classId) th return result; } + public Set getClassAccessJournalIDs(String className){ + return (Set) redisDataStore.getSetValue(RedisKeyUtils.classAccessJournal(className)); + } + + public long getNumClientsInterestedInWholeClass(String className){ + return redisDataStore.getSetSize(RedisKeyUtils.accessJournal(className,"0")); + } + public List checkUserListAccessRights(Collection clientIdList, String className, String classId) throws Exception { List result = new ArrayList(); diff --git a/src/main/java/com/percero/agents/sync/datastore/RedisDataStore.java b/src/main/java/com/percero/agents/sync/datastore/RedisDataStore.java index cfbcb4d..fbe8c68 100644 --- a/src/main/java/com/percero/agents/sync/datastore/RedisDataStore.java +++ b/src/main/java/com/percero/agents/sync/datastore/RedisDataStore.java @@ -183,6 +183,11 @@ public Map getHashEntries( final String key ) { public Long getHashSize( final String key ) { return template.opsForHash().size(key); } + + @Transactional + public Long getSetSize( final String key ) { + return template.opsForSet().size(key); + } @Transactional public void deleteHashKey( final String key, final Object hashKey ) { diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 0bfd839..0c81dfc 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -1,5 +1,6 @@ package com.percero.agents.sync.jobs; +import com.percero.agents.sync.access.IAccessManager; import com.percero.agents.sync.cache.CacheManager; import com.percero.agents.sync.helpers.PostDeleteHelper; import com.percero.agents.sync.helpers.PostPutHelper; @@ -19,8 +20,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.Date; -import java.util.Random; +import java.util.*; /** * Responsible for querying an update table and processing the rows. @@ -38,6 +38,7 @@ public class UpdateTableProcessor { private IManifest manifest; private CacheManager cacheManager; private DataProviderManager dataProviderManager; + private IAccessManager accessManager; public UpdateTableProcessor(String tableName, UpdateTableConnectionFactory connectionFactory, @@ -45,7 +46,8 @@ public UpdateTableProcessor(String tableName, PostDeleteHelper postDeleteHelper, PostPutHelper postPutHelper, CacheManager cacheManager, - DataProviderManager dataProviderManager) + DataProviderManager dataProviderManager, + IAccessManager accessManager) { this.tableName = tableName; this.connectionFactory = connectionFactory; @@ -54,6 +56,7 @@ public UpdateTableProcessor(String tableName, this.manifest = manifest; this.cacheManager = cacheManager; this.dataProviderManager= dataProviderManager; + this.accessManager = accessManager; } /** @@ -84,6 +87,7 @@ public ProcessorResult process(){ result.addResult(row.getType().toString(), false, ""); } }catch(Exception e){ + logger.warn("Failed to process update: "+ e.getMessage(), e); result.addResult(row.getType().toString(), false, e.getMessage()); } @@ -147,17 +151,32 @@ private boolean processDeleteSingle(UpdateTableRow row) throws Exception{ */ private boolean processUpdateSingle(UpdateTableRow row) throws Exception{ Class clazz = getClassForTableName(row.getTableName()); + List list = new ArrayList(); + list.add(row.getRowId()); + processUpdates(clazz.getCanonicalName(), list); + return true; + } + + /** + * Takes a class and list of ids that need to be pushed out + * @param className + * @param Ids + * @throws Exception + */ + private void processUpdates(String className, Collection Ids) throws Exception{ IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(clazz.getName()); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - ClassIDPair pair = new ClassIDPair(row.getRowId(), clazz.getCanonicalName()); - IPerceroObject object = dataProvider.systemGetById(pair, true); - if(object != null){ - cacheManager.updateCachedObject(object, null); - postPutHelper.postPutObject(pair, null, null, true, null); + for(String ID : Ids) { + ClassIDPair pair = new ClassIDPair(ID, className); + IPerceroObject object = dataProvider.systemGetById(pair, true); + + if (object != null) { + cacheManager.updateCachedObject(object, null); + postPutHelper.postPutObject(pair, null, null, true, null); + } } - return true; } /** @@ -185,10 +204,35 @@ private boolean processDeleteTable(UpdateTableRow row){ * @param row * @return */ - private boolean processUpdateTable(UpdateTableRow row){ + private boolean processUpdateTable(UpdateTableRow row) throws Exception{ + Class clazz = getClassForTableName(row.getTableName()); + + Set accessedIds = null; + // If there are any clients that have asked for all objects in a class then we have to push everything + if(accessManager.getNumClientsInterestedInWholeClass(clazz.getName()) > 0) + accessedIds = getAllIdsForTable(row.getTableName()); + else + accessedIds = accessManager.getClassAccessJournalIDs(clazz.getName()); + + processUpdates(clazz.getCanonicalName(), accessedIds); return true; } + private Set getAllIdsForTable(String tableName) throws SQLException{ + Set result = new HashSet(); + String query = "select ID from "+tableName; + try(Connection connection = connectionFactory.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(query)){ + + while(resultSet.next()){ + result.add(resultSet.getString("ID")); + } + } + + return result; + } + /** * Process a whole table with inserts * @param row diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessorFactory.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessorFactory.java index 0c8e392..cbf6b89 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessorFactory.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessorFactory.java @@ -1,5 +1,6 @@ package com.percero.agents.sync.jobs; +import com.percero.agents.sync.access.IAccessManager; import com.percero.agents.sync.cache.CacheManager; import com.percero.agents.sync.helpers.PostDeleteHelper; import com.percero.agents.sync.helpers.PostPutHelper; @@ -33,8 +34,11 @@ public class UpdateTableProcessorFactory { @Autowired DataProviderManager dataProviderManager; + @Autowired + IAccessManager accessManager; + public UpdateTableProcessor getProcessor(String tableName){ return new UpdateTableProcessor(tableName, connectionFactory, manifest, - postDeleteHelper, postPutHelper, cacheManager, dataProviderManager); + postDeleteHelper, postPutHelper, cacheManager, dataProviderManager, accessManager); } } From dc246d2af8a61282bbd3ddf11e8c5bde6e577273 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Sat, 12 Sep 2015 07:46:37 -0700 Subject: [PATCH 8/9] Full table update --- .../sync/jobs/UpdateTableProcessor.java | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 0c81dfc..24b9dfc 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -4,11 +4,10 @@ import com.percero.agents.sync.cache.CacheManager; import com.percero.agents.sync.helpers.PostDeleteHelper; import com.percero.agents.sync.helpers.PostPutHelper; -import com.percero.agents.sync.metadata.IMappedClassManager; -import com.percero.agents.sync.metadata.MappedClass; -import com.percero.agents.sync.metadata.MappedClassManagerFactory; +import com.percero.agents.sync.metadata.*; import com.percero.agents.sync.services.DataProviderManager; import com.percero.agents.sync.services.IDataProvider; +import com.percero.agents.sync.vo.BaseDataObject; import com.percero.agents.sync.vo.ClassIDPair; import com.percero.framework.bl.IManifest; import com.percero.framework.vo.IPerceroObject; @@ -195,7 +194,17 @@ private boolean processInsertSingle(UpdateTableRow row) throws Exception{ * @param row * @return */ - private boolean processDeleteTable(UpdateTableRow row){ + private boolean processDeleteTable(UpdateTableRow row) throws Exception{ + Class clazz = getClassForTableName(row.getTableName()); + Set accessedIds = accessManager.getClassAccessJournalIDs(clazz.getName()); + Set allIds = getAllIdsForTable(row.getTableName()); + + // Now we have the set that has been removed that we care about + accessedIds.removeAll(allIds); + for(String id : accessedIds){ + postDeleteHelper.postDeleteObject(new ClassIDPair(id, clazz.getCanonicalName()), null, null, true); + } + return true; } @@ -239,6 +248,32 @@ private Set getAllIdsForTable(String tableName) throws SQLException{ * @return */ private boolean processInsertTable(UpdateTableRow row){ + Class clazz = getClassForTableName(row.getTableName()); + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(clazz.getName()); + +// TODO: uncomment and implement +// for(MappedFieldPerceroObject nextMappedField : mappedClass.externalizablePerceroObjectFields) { +// try { +// if (nextMappedField.getReverseMappedField() != null) { +// IPerceroObject fieldValue = (IPerceroObject) nextMappedField.getValue(perceroObject); +// if (fieldValue != null) { +// ClassIDPair fieldPair = BaseDataObject.toClassIdPair(fieldValue); +// +// Collection classChangedFields = changedObjects.get(fieldPair); +// if (classChangedFields == null) { +// classChangedFields = new HashSet(); +// changedObjects.put(fieldPair, classChangedFields); +// } +// MappedField reverseMappedField = nextMappedField.getReverseMappedField(); +// classChangedFields.add(reverseMappedField); +// } +// } +// } catch(Exception e) { +// log.error("Error in postCreateObject " + mappedClass.className + "." + nextMappedField.getField().getName(), e); +// } +// } + return true; } From f51f3206168dfca11533e0945c560832dcb49b76 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 16 Sep 2015 07:46:46 -0700 Subject: [PATCH 9/9] Started adding the java test client --- .../sync/jobs/UpdateTableProcessor.java | 82 ++++++++----- .../sync/metadata/IMappedClassManager.java | 7 +- .../sync/metadata/MappedClassManager.java | 6 + .../sync/jobs/UpdateTableProcessorTest.java | 55 ++++++--- .../java/com/percero/client/AStackClient.java | 74 ++++++++++++ .../percero/client/AStackClientFactory.java | 18 +++ .../com/percero/client/AStackClientTest.java | 41 +++++++ .../com/percero/client/AStackRPCService.java | 23 ++++ .../java/com/percero/client/Callback.java | 9 ++ src/test/java/com/percero/example/Circle.java | 114 ++++++++++++++++++ src/test/java/com/percero/example/Email.java | 6 +- src/test/java/com/percero/example/Person.java | 53 ++++++++ .../java/com/percero/test/utils/AuthUtil.java | 24 ++++ .../com/percero/test/utils/CleanerUtil.java | 58 +++++++++ .../resources/properties/test/env.properties | 2 +- .../spring/update_table_processor.xml | 6 + src/test/resources/update_table_test_sql.md | 109 +++++++++++++++++ 17 files changed, 634 insertions(+), 53 deletions(-) create mode 100644 src/test/java/com/percero/client/AStackClient.java create mode 100644 src/test/java/com/percero/client/AStackClientFactory.java create mode 100644 src/test/java/com/percero/client/AStackClientTest.java create mode 100644 src/test/java/com/percero/client/AStackRPCService.java create mode 100644 src/test/java/com/percero/client/Callback.java create mode 100644 src/test/java/com/percero/example/Circle.java create mode 100644 src/test/java/com/percero/test/utils/AuthUtil.java create mode 100644 src/test/java/com/percero/test/utils/CleanerUtil.java create mode 100644 src/test/resources/update_table_test_sql.md diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 24b9dfc..a30c9e5 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -7,7 +7,6 @@ import com.percero.agents.sync.metadata.*; import com.percero.agents.sync.services.DataProviderManager; import com.percero.agents.sync.services.IDataProvider; -import com.percero.agents.sync.vo.BaseDataObject; import com.percero.agents.sync.vo.ClassIDPair; import com.percero.framework.bl.IManifest; import com.percero.framework.vo.IPerceroObject; @@ -140,6 +139,7 @@ private boolean processRow(UpdateTableRow row) throws Exception{ private boolean processDeleteSingle(UpdateTableRow row) throws Exception{ Class clazz = getClassForTableName(row.getTableName()); postDeleteHelper.postDeleteObject(new ClassIDPair(row.getRowId(), clazz.getCanonicalName()), null, null, true); + updateReferences(clazz.getName()); return true; } @@ -152,7 +152,8 @@ private boolean processUpdateSingle(UpdateTableRow row) throws Exception{ Class clazz = getClassForTableName(row.getTableName()); List list = new ArrayList(); list.add(row.getRowId()); - processUpdates(clazz.getCanonicalName(), list); + processUpdates(clazz.getName(), list); + updateReferences(clazz.getName()); return true; } @@ -186,6 +187,7 @@ private void processUpdates(String className, Collection Ids) throws Exc private boolean processInsertSingle(UpdateTableRow row) throws Exception{ Class clazz = getClassForTableName(row.getTableName()); postPutHelper.postPutObject(new ClassIDPair(row.getRowId(), clazz.getCanonicalName()), null, null, true, null); + updateReferences(clazz.getName()); return true; } @@ -205,6 +207,8 @@ private boolean processDeleteTable(UpdateTableRow row) throws Exception{ postDeleteHelper.postDeleteObject(new ClassIDPair(id, clazz.getCanonicalName()), null, null, true); } + updateReferences(clazz.getName()); + return true; } @@ -223,7 +227,9 @@ private boolean processUpdateTable(UpdateTableRow row) throws Exception{ else accessedIds = accessManager.getClassAccessJournalIDs(clazz.getName()); - processUpdates(clazz.getCanonicalName(), accessedIds); + processUpdates(clazz.getName(), accessedIds); + updateReferences(clazz.getName()); + return true; } @@ -247,36 +253,56 @@ private Set getAllIdsForTable(String tableName) throws SQLException{ * @param row * @return */ - private boolean processInsertTable(UpdateTableRow row){ - Class clazz = getClassForTableName(row.getTableName()); - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(clazz.getName()); + private boolean processInsertTable(UpdateTableRow row) throws Exception { + + MappedClass mappedClass = getMappedClassForTableName(row.getTableName()); -// TODO: uncomment and implement -// for(MappedFieldPerceroObject nextMappedField : mappedClass.externalizablePerceroObjectFields) { -// try { -// if (nextMappedField.getReverseMappedField() != null) { -// IPerceroObject fieldValue = (IPerceroObject) nextMappedField.getValue(perceroObject); -// if (fieldValue != null) { -// ClassIDPair fieldPair = BaseDataObject.toClassIdPair(fieldValue); -// -// Collection classChangedFields = changedObjects.get(fieldPair); -// if (classChangedFields == null) { -// classChangedFields = new HashSet(); -// changedObjects.put(fieldPair, classChangedFields); -// } -// MappedField reverseMappedField = nextMappedField.getReverseMappedField(); -// classChangedFields.add(reverseMappedField); -// } -// } -// } catch(Exception e) { -// log.error("Error in postCreateObject " + mappedClass.className + "." + nextMappedField.getField().getName(), e); -// } -// } + // if any client needs all of this class then the only choice we have is to push everything + if(accessManager.getNumClientsInterestedInWholeClass(mappedClass.className) > 0){ + Set allIds = getAllIdsForTable(row.getTableName()); + for(String id : allIds) + postPutHelper.postPutObject(new ClassIDPair(id, mappedClass.className), null, null, true, null); + } + + updateReferences(mappedClass.className); return true; } + private MappedClass getMappedClassForTableName(String tableName){ + Class clazz = getClassForTableName(tableName); + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(clazz.getName()); + return mappedClass; + } + + /** + * Finds all back references to this class and pushes updates to all of them. + * @param className + */ + private void updateReferences(String className){ + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + // Go through each mapped field and push all objects of that associated type (just in case any has a reference to a new row + // in the updated table) + // -- + // TODO: is this right? Is it enough to only check the relationships on this class or do we need to look + // through all of the mapped classes? + for(MappedFieldPerceroObject nextMappedField : mappedClass.externalizablePerceroObjectFields) { + try { + // Only care about it if it has a reverse relationship + MappedField mappedField = nextMappedField.getReverseMappedField(); + if (mappedField != null) { + // Find all of this type and push down an update to all + Set ids = accessManager.getClassAccessJournalIDs(mappedField.getMappedClass().className); + processUpdates(mappedField.getMappedClass().className, ids); + } + } catch(Exception e) { + logger.error("Error in postCreateObject " + mappedClass.className + "." + nextMappedField.getField().getName(), e); + } + } + } + /** * Pulls a row off the update table and locks it so that other * processors don't duplicate the work diff --git a/src/main/java/com/percero/agents/sync/metadata/IMappedClassManager.java b/src/main/java/com/percero/agents/sync/metadata/IMappedClassManager.java index 55cfac1..f522792 100644 --- a/src/main/java/com/percero/agents/sync/metadata/IMappedClassManager.java +++ b/src/main/java/com/percero/agents/sync/metadata/IMappedClassManager.java @@ -1,7 +1,10 @@ package com.percero.agents.sync.metadata; +import java.util.Collection; + public interface IMappedClassManager { - public void addMappedClass(MappedClass theMappedClass); - public MappedClass getMappedClassByClassName(String aClassName); + void addMappedClass(MappedClass theMappedClass); + MappedClass getMappedClassByClassName(String aClassName); + Collection getAllMappedClasses(); } diff --git a/src/main/java/com/percero/agents/sync/metadata/MappedClassManager.java b/src/main/java/com/percero/agents/sync/metadata/MappedClassManager.java index 9a46c6d..95598a0 100644 --- a/src/main/java/com/percero/agents/sync/metadata/MappedClassManager.java +++ b/src/main/java/com/percero/agents/sync/metadata/MappedClassManager.java @@ -1,5 +1,6 @@ package com.percero.agents.sync.metadata; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Random; @@ -24,4 +25,9 @@ public MappedClass getMappedClassByClassName(String aClassName) { return mc; } + @Override + public Collection getAllMappedClasses() { + return mappedClassesByName.values(); + } + } diff --git a/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java index 7d9a0bf..c1d0757 100644 --- a/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java +++ b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java @@ -2,6 +2,8 @@ import com.percero.example.Email; import com.percero.example.Person; +import com.percero.test.utils.AuthUtil; +import com.percero.test.utils.CleanerUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -12,6 +14,7 @@ import java.sql.Connection; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; /** @@ -27,6 +30,10 @@ public class UpdateTableProcessorTest { UpdateTableConnectionFactory connectionFactory; @Autowired UpdateTablePoller poller; + @Autowired + CleanerUtil cleanerUtil; + @Autowired + AuthUtil authUtil; String tableName = "update_table"; @@ -34,29 +41,13 @@ public class UpdateTableProcessorTest { public void before() throws Exception{ // Disable the poller so it doesn't step on our toes poller.enabled = false; + cleanerUtil.cleanAll(); try(Connection connection = connectionFactory.getConnection(); Statement statement = connection.createStatement()) { + // Truncate the update table String sql = "delete from " + tableName; statement.executeUpdate(sql); - - // Add some fixture data - statement.executeUpdate("insert into " + tableName + "(tableName, rowId, type) values " + - "('Email','1','UPDATE')," + - "('Person','1','INSERT')," + - "('Block','1','DELETE')"); - } - } - - @Test - public void test() throws Exception{ - try(Connection connection = connectionFactory.getConnection(); - Statement statement = connection.createStatement();) - { - String sql = "select count(*) as 'count' from " + tableName; - ResultSet resultSet = statement.executeQuery(sql); - Assert.assertTrue(resultSet.next()); - Assert.assertEquals(3, resultSet.getInt("count")); } } @@ -81,8 +72,22 @@ public void getClassForTableName_NotFound() throws Exception{ Assert.assertNull(clazz); } + // Shared setup method + public void setupThreeRowsInUpdateTable() throws SQLException{ + try(Connection connection = connectionFactory.getConnection(); + Statement statement = connection.createStatement()) + { + // Add some fixture data + statement.executeUpdate("insert into " + tableName + "(tableName, rowId, type) values " + + "('Email','1','UPDATE')," + + "('Person','1','INSERT')," + + "('Block','1','DELETE')"); + } + } + @Test public void getRow() throws Exception { + setupThreeRowsInUpdateTable(); UpdateTableProcessor processor = processorFactory.getProcessor(tableName); UpdateTableRow row = processor.getRow(); @@ -104,7 +109,9 @@ public void getRow() throws Exception { @Test - public void process() throws Exception { + public void processMultipleRows() throws Exception { + setupThreeRowsInUpdateTable(); + UpdateTableProcessor processor = processorFactory.getProcessor(tableName); ProcessorResult result = processor.process(); Assert.assertEquals(3, result.getTotal()); @@ -120,4 +127,14 @@ public void process() throws Exception { } } + @Test + public void singleRowInsertAllList(){ + + } + + @Test + public void singleRowInsertRefList(){ + + } + } diff --git a/src/test/java/com/percero/client/AStackClient.java b/src/test/java/com/percero/client/AStackClient.java new file mode 100644 index 0000000..f9c39e1 --- /dev/null +++ b/src/test/java/com/percero/client/AStackClient.java @@ -0,0 +1,74 @@ +package com.percero.client; + +import com.percero.agents.auth.vo.AuthenticationRequest; +import com.percero.agents.auth.vo.AuthenticationResponse; +import com.percero.agents.auth.vo.UserToken; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.core.MessageListener; +import org.springframework.amqp.rabbit.connection.ConnectionFactory; +import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer; + +import java.util.UUID; + +/** + * Intended to maintain session with the server and listen for + * push messages. + */ +public class AStackClient { + + /** + * Some service deps + */ + private AStackRPCService rpcService; + private String clientId = UUID.randomUUID().toString(); + private ConnectionFactory amqpConnectionFactory; + public AStackClient(AStackRPCService rpcService, ConnectionFactory amqpConnectionFactory){ + this.rpcService = rpcService; + this.amqpConnectionFactory = amqpConnectionFactory; + } + + /** + * Some internal state + */ + private UserToken userToken; + + public boolean authenticateAnonymously(){ + AuthenticationRequest request = new AuthenticationRequest(); + request.setAuthProvider("anonymous"); + request.setClientId(clientId); + + AuthenticationResponse response = rpcService.authenticate(request); + userToken = response.getResult(); + boolean result = (userToken != null); + if(result){ + setupClientQueueAndStartListening(); + } + + return result; + } + + /** + * Sets up the queue to listen for messages to the client (including responses to RPC) + */ + private SimpleMessageListenerContainer listenerContainer; + private void setupClientQueueAndStartListening(){ + listenerContainer = new SimpleMessageListenerContainer(amqpConnectionFactory); + listenerContainer.setQueueNames(clientId); + listenerContainer.setMessageListener(getMessageListener()); + listenerContainer.start(); + } + + private MessageListener messageListener; + private MessageListener getMessageListener(){ + if(messageListener == null) + messageListener = new MessageListener() { + @Override + public void onMessage(Message message) { + String key = message.getMessageProperties().getReceivedRoutingKey(); + System.out.println("Got Message from server: "+key); + } + }; + + return messageListener; + } +} diff --git a/src/test/java/com/percero/client/AStackClientFactory.java b/src/test/java/com/percero/client/AStackClientFactory.java new file mode 100644 index 0000000..2982c9f --- /dev/null +++ b/src/test/java/com/percero/client/AStackClientFactory.java @@ -0,0 +1,18 @@ +package com.percero.client; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Created by jonnysamps on 9/14/15. + */ +@Component +public class AStackClientFactory { + + @Autowired + AStackRPCService rpcService; + + public AStackClient getClient(){ + return new AStackClient(rpcService); + } +} diff --git a/src/test/java/com/percero/client/AStackClientTest.java b/src/test/java/com/percero/client/AStackClientTest.java new file mode 100644 index 0000000..517a89c --- /dev/null +++ b/src/test/java/com/percero/client/AStackClientTest.java @@ -0,0 +1,41 @@ +package com.percero.client; + +import com.percero.agents.sync.jobs.UpdateTablePoller; +import com.percero.test.utils.CleanerUtil; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import static org.junit.Assert.*; + +/** + * Created by jonnysamps on 9/14/15. + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(locations = { "classpath:spring/update_table_processor.xml" }) +public class AStackClientTest { + + @Autowired + AStackClientFactory clientFactory; + + @Autowired + CleanerUtil cleanerUtil; + + @Autowired + UpdateTablePoller poller; + + @Before + public void before(){ + poller.enabled = false; + cleanerUtil.cleanAll(); + } + + @Test + public void authenticateAnonymously(){ + AStackClient client = clientFactory.getClient(); + boolean result = client.authenticateAnonymously(); + assertTrue(result); + } +} diff --git a/src/test/java/com/percero/client/AStackRPCService.java b/src/test/java/com/percero/client/AStackRPCService.java new file mode 100644 index 0000000..0818695 --- /dev/null +++ b/src/test/java/com/percero/client/AStackRPCService.java @@ -0,0 +1,23 @@ +package com.percero.client; + +import com.percero.agents.auth.vo.AuthenticationRequest; +import com.percero.agents.auth.vo.AuthenticationResponse; +import org.springframework.amqp.core.AmqpTemplate; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Main interface for communicating to the backend through AMQP + * Created by jonnysamps on 9/13/15. + */ +@Component +public class AStackRPCService { + + @Autowired + AmqpTemplate amqpTemplate; + + public AuthenticationResponse authenticate(AuthenticationRequest request){ + return (AuthenticationResponse) amqpTemplate.convertSendAndReceive("authenticate", request); + } + +} diff --git a/src/test/java/com/percero/client/Callback.java b/src/test/java/com/percero/client/Callback.java new file mode 100644 index 0000000..ae6ac4e --- /dev/null +++ b/src/test/java/com/percero/client/Callback.java @@ -0,0 +1,9 @@ +package com.percero.client; + +/** + * Interface to define the way that the client does async processing with callbacks + * Created by jonnysamps on 9/15/15. + */ +public interface Callback { + void execute(T result); +} diff --git a/src/test/java/com/percero/example/Circle.java b/src/test/java/com/percero/example/Circle.java new file mode 100644 index 0000000..d398177 --- /dev/null +++ b/src/test/java/com/percero/example/Circle.java @@ -0,0 +1,114 @@ +package com.percero.example; + +import com.google.gson.JsonObject; +import com.percero.agents.sync.metadata.annotations.Externalize; +import com.percero.agents.sync.vo.BaseDataObject; +import com.percero.serial.JsonUtils; +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.annotate.JsonProperty; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; + +import javax.persistence.*; +import java.io.IOException; +import java.io.Serializable; + +@Entity +public class Circle extends BaseDataObject implements Serializable{ + + ////////////////////////////////////////////////////// + // Properties + ////////////////////////////////////////////////////// + @Id + @Externalize + @Column(unique=true,name="ID") + private String ID; + @JsonProperty(value="ID") + public String getID() { + return this.ID; + } + @JsonProperty(value="ID") + public void setID(String value) { + this.ID = value; + } + + @Column + @Externalize + private String color; + public String getColor() { + return this.color; + } + public void setColor(String value) + { + this.color = value; + } + + ////////////////////////////////////////////////////// + // Source Relationships + ////////////////////////////////////////////////////// + @Externalize + @JoinColumn(name="person_ID") + @org.hibernate.annotations.ForeignKey(name="FK_Person_person_TO_Circle") + @ManyToOne(fetch=FetchType.LAZY, optional=false) + private Person person; + public Person getPerson() { + return this.person; + } + public void setPerson(Person value) { + this.person = value; + } + + ////////////////////////////////////////////////////// + // JSON + ////////////////////////////////////////////////////// + @Override + public String retrieveJson(ObjectMapper objectMapper) { + String objectJson = super.retrieveJson(objectMapper); + + // Properties + objectJson += ",\"color\":"; + if (getColor() == null) + objectJson += "null"; + else { + if (objectMapper == null) + objectMapper = new ObjectMapper(); + try { + objectJson += objectMapper.writeValueAsString(getColor()); + } catch (JsonGenerationException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (JsonMappingException e) { + objectJson += "null"; + e.printStackTrace(); + } catch (IOException e) { + objectJson += "null"; + e.printStackTrace(); + } + } + + objectJson += ",\"person\":"; + if (getPerson() == null) + objectJson += "null"; + else { + try { + objectJson += getPerson().toEmbeddedJson(); + } catch(Exception e) { + objectJson += "null"; + } + } + objectJson += ""; + + return objectJson; + } + + @Override + protected void fromJson(JsonObject jsonObject) { + super.fromJson(jsonObject); + + // Properties + setColor(JsonUtils.getJsonString(jsonObject, "color")); + + // Source Relationships + this.person = JsonUtils.getJsonPerceroObject(jsonObject, "person"); + } +} diff --git a/src/test/java/com/percero/example/Email.java b/src/test/java/com/percero/example/Email.java index 84ca055..9264dac 100644 --- a/src/test/java/com/percero/example/Email.java +++ b/src/test/java/com/percero/example/Email.java @@ -97,13 +97,13 @@ public void setValue(String value) @JsonSerialize(using=BDOSerializer.class) @JsonDeserialize(using=BDODeserializer.class) @JoinColumn(name="user_ID") - @org.hibernate.annotations.ForeignKey(name="FK_User_user_TO_Email") + @org.hibernate.annotations.ForeignKey(name="FK_Person_person_TO_Email") @ManyToOne(fetch=FetchType.LAZY, optional=false) private Person person; public Person getPerson() { return this.person; } - public void setUser(Person value) { + public void setPerson(Person value) { this.person = value; } @@ -161,7 +161,7 @@ public String retrieveJson(ObjectMapper objectMapper) { objectJson += "null"; else { try { - objectJson += ((BaseDataObject) getPerson()).toEmbeddedJson(); + objectJson += getPerson().toEmbeddedJson(); } catch(Exception e) { objectJson += "null"; } diff --git a/src/test/java/com/percero/example/Person.java b/src/test/java/com/percero/example/Person.java index fb1b7b0..1dccf05 100644 --- a/src/test/java/com/percero/example/Person.java +++ b/src/test/java/com/percero/example/Person.java @@ -115,6 +115,25 @@ public void setPersonRoles(List value) { this.personRoles = value; } + @Externalize + @OneToMany(fetch=FetchType.LAZY, targetEntity=Email.class, mappedBy="person", cascade=javax.persistence.CascadeType.REMOVE) + private List emails; + public List getEmails() { + return this.emails; + } + public void setEmails(List value) { + this.emails = value; + } + + @Externalize + @OneToMany(fetch=FetchType.LAZY, targetEntity=Circle.class, mappedBy="person", cascade=javax.persistence.CascadeType.REMOVE) + private List circles; + public List getCircles() { + return this.circles; + } + public void setCircles(List value) { + this.circles = value; + } ////////////////////////////////////////////////////// // JSON @@ -212,10 +231,42 @@ public String retrieveJson(ObjectMapper objectMapper) { } } } + objectJson += "]"; + objectJson += ",\"emails\":["; + if (getEmails() != null) { + int emailsCounter = 0; + for(Email email : getEmails()) { + if (emailsCounter > 0) + objectJson += ","; + try { + objectJson += email.toEmbeddedJson(); + emailsCounter++; + } catch(Exception e) { + // Do nothing. + } + } + } + objectJson += "]"; + + objectJson += ",\"circles\":["; + if (getCircles() != null) { + int circlesCounter = 0; + for(Circle circle : getCircles()) { + if (circlesCounter > 0) + objectJson += ","; + try { + objectJson += circle.toEmbeddedJson(); + circlesCounter++; + } catch(Exception e) { + // Do nothing. + } + } + } objectJson += "]"; + return objectJson; } @@ -231,6 +282,8 @@ protected void fromJson(JsonObject jsonObject) { setLastName(com.percero.serial.JsonUtils.getJsonString(jsonObject, "lastName")); this.personRoles = (List) JsonUtils.getJsonListPerceroObject(jsonObject, "personRoles"); + this.emails = (List) JsonUtils.getJsonListPerceroObject(jsonObject, "emails"); + this.circles = (List) JsonUtils.getJsonListPerceroObject(jsonObject, "circles"); } } diff --git a/src/test/java/com/percero/test/utils/AuthUtil.java b/src/test/java/com/percero/test/utils/AuthUtil.java new file mode 100644 index 0000000..43d6ecb --- /dev/null +++ b/src/test/java/com/percero/test/utils/AuthUtil.java @@ -0,0 +1,24 @@ +package com.percero.test.utils; + +import com.percero.agents.auth.vo.AuthenticationRequest; +import com.percero.agents.auth.vo.AuthenticationResponse; +import com.percero.client.AStackRPCService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Created by jonnysamps on 9/13/15. + */ +@Component +public class AuthUtil { + + @Autowired + AStackRPCService rpcService; + + public AuthenticationResponse authenticateAnonymously(){ + AuthenticationRequest request = new AuthenticationRequest(); + request.setAuthProvider("anonymous"); + AuthenticationResponse response = (AuthenticationResponse) rpcService.authenticate(request); + return response; + } +} diff --git a/src/test/java/com/percero/test/utils/CleanerUtil.java b/src/test/java/com/percero/test/utils/CleanerUtil.java new file mode 100644 index 0000000..dd2f7a7 --- /dev/null +++ b/src/test/java/com/percero/test/utils/CleanerUtil.java @@ -0,0 +1,58 @@ +package com.percero.test.utils; + +import com.percero.agents.sync.metadata.IMappedClassManager; +import com.percero.agents.sync.metadata.MappedClass; +import com.percero.agents.sync.metadata.MappedClassManagerFactory; +import com.percero.agents.sync.services.PerceroRedisTemplate; +import org.hibernate.Session; +import org.hibernate.SessionFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +/** + * Created by jonnysamps on 9/14/15. + */ +@Component +public class CleanerUtil { + + @Autowired + PerceroRedisTemplate redisTemplate; + + @Autowired + SessionFactory appSessionFactory; + + /** + * Wipe the database and redis + */ + public void cleanAll(){ + cleanDB(); + cleanCache(); + } + + @Transactional + public void cleanDB(){ + Session s = null; + try { + s = appSessionFactory.getCurrentSession(); + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + for (MappedClass mappedClass : mcm.getAllMappedClasses()) { + s.createQuery("delete from "+mappedClass.tableName).executeUpdate(); + } + } catch(Exception e){ + System.out.println(e.getMessage()); + } + } + + public void cleanCache(){ + RedisConnection connection = null; + try{ + connection = redisTemplate.getConnectionFactory().getConnection(); + connection.flushAll(); + } finally{ + if(connection != null) + connection.close(); + } + } +} diff --git a/src/test/resources/properties/test/env.properties b/src/test/resources/properties/test/env.properties index 2788f4e..fcc8219 100644 --- a/src/test/resources/properties/test/env.properties +++ b/src/test/resources/properties/test/env.properties @@ -65,7 +65,7 @@ pulseHttpAuth.hostPortAndContext=https://localhost:8900/auth pulseHttpAuth.trustAllCerts=true # Update Table -updateTable.tableNames=update_log +updateTable.tableNames=update_table updateTable.jdbcUrl=jdbc:mysql://localhost/as_example updateTable.username=root updateTable.password=root diff --git a/src/test/resources/spring/update_table_processor.xml b/src/test/resources/spring/update_table_processor.xml index f00f3b1..9d0ba64 100644 --- a/src/test/resources/spring/update_table_processor.xml +++ b/src/test/resources/spring/update_table_processor.xml @@ -43,6 +43,11 @@ + + + + + @@ -103,6 +108,7 @@ class="org.springframework.amqp.support.converter.JsonMessageConverter" /> + diff --git a/src/test/resources/update_table_test_sql.md b/src/test/resources/update_table_test_sql.md new file mode 100644 index 0000000..5c486a7 --- /dev/null +++ b/src/test/resources/update_table_test_sql.md @@ -0,0 +1,109 @@ +# Single Row Insert +## ALL list - PASSING +### First try +```sql +insert into Block set ID='1block', color='#aabbcc'; +insert into update_table set rowID='1block', tableName='Block', type='INSERT'; +``` + +### Second try +```sql +insert into Block set ID='2block', color='#8899aa'; +insert into update_table set rowID='2block', tableName='Block', type='INSERT'; +``` + +## Referenced list - PASSING +### First try +```sql +insert into Circle set ID='1circle', color='#11aa22', person_ID='e3d04ee4-e996-4f4a-9c5e-7e87e5685849'; +insert into update_table set rowID='1circle', tableName='Circle', type='INSERT'; +``` + +### Second try +```sql +insert into Circle set ID='2circle', color='#22bb33', person_ID='e3d04ee4-e996-4f4a-9c5e-7e87e5685849'; +insert into update_table set rowID='2circle', tableName='Circle', type='INSERT'; +``` + +# Whole Table Insert +## ALL list - PASSING +### First try +```sql +insert into Block set ID='3block', color='#a3b3c3'; +insert into Block set ID='4block', color='#a4b4c4'; +insert into update_table set rowID=null, tableName='Block', type='INSERT'; +``` + +### Second try +```sql +insert into Block set ID='5block', color='#b5b5c5'; +insert into Block set ID='6block', color='#b6b6c6'; +insert into update_table set rowID=null, tableName='Block', type='INSERT'; +``` + +## Reference list - PASSING + +```sql +-- First Try +insert into Circle set ID='3circle', color='#ccc333', person_ID='e3d04ee4-e996-4f4a-9c5e-7e87e5685849'; +insert into Circle set ID='4circle', color='#ccc444', person_ID='e3d04ee4-e996-4f4a-9c5e-7e87e5685849'; +insert into update_table set rowID=null, tableName='Circle', type='INSERT'; + +-- Second try +insert into Circle set ID='5circle', color='#ccc555', person_ID='e3d04ee4-e996-4f4a-9c5e-7e87e5685849'; +insert into Circle set ID='6circle', color='#ccc666', person_ID='e3d04ee4-e996-4f4a-9c5e-7e87e5685849'; +insert into update_table set rowID=null, tableName='Circle', type='INSERT'; +``` + +# Single Row Update +## Value update - PASSING +```sql +-- First Try -- +update Block set color='#333333' where ID='1' ; +insert into update_table set rowID='1', tableName='Block', type='UPDATE'; + +-- Second Try -- +update Block set color='#aaaaaa' where ID='1' ; +insert into update_table set rowID='1', tableName='Block', type='UPDATE'; +``` + +## Reference List - PASSING + +Make sure that when updating references that objects are added/removed the appropriate lists + +```sql +-- First Try -- +update Circle set color='green', person_ID='a11df72c-9aa5-4f75-ba41-ce4c4539fdbe' where ID='6circle'; +insert into update_table set rowID='6circle', tableName='Circle', type='UPDATE'; + +-- Second Try -- +update Circle set color='red', person_ID='e3d04ee4-e996-4f4a-9c5e-7e87e5685849' where ID='6circle'; +insert into update_table set rowID='6circle', tableName='Circle', type='UPDATE'; +``` + +# Whole Table update +## Value update - PASSING +```sql +-- First Try -- +update Block set color='coral'; +insert into update_table set rowID=null, tableName='Block', type='UPDATE'; + +-- Second Try -- +update Block set color='lightcoral'; +insert into update_table set rowID=null, tableName='Block', type='UPDATE'; +``` + +## Reference List - +```sql +-- First Try -- +update Circle set person_ID='' where person_ID='a11df72c-9aa5-4f75-ba41-ce4c4539fdbe'; +update Circle set person_ID='a11df72c-9aa5-4f75-ba41-ce4c4539fdbe' where person_ID='e3d04ee4-e996-4f4a-9c5e-7e87e5685849'; +update Circle set person_ID='e3d04ee4-e996-4f4a-9c5e-7e87e5685849' where person_ID=''; +insert into update_table set rowID=null, tableName='Circle', type='UPDATE'; + +-- Second try -- +update Circle set person_ID=null where person_ID='a11df72c-9aa5-4f75-ba41-ce4c4539fdbe'; +update Circle set person_ID='a11df72c-9aa5-4f75-ba41-ce4c4539fdbe' where person_ID='e3d04ee4-e996-4f4a-9c5e-7e87e5685849'; +update Circle set person_ID='e3d04ee4-e996-4f4a-9c5e-7e87e5685849' where person_ID=null; +insert into update_table set rowID=null, tableName='Circle', type='UPDATE'; +``` \ No newline at end of file