From 451f3515e6772ceab4a8752fb79f54772985d145 Mon Sep 17 00:00:00 2001 From: Srinivas Veldurthi Date: Wed, 27 Dec 2017 18:19:24 -0800 Subject: [PATCH 1/5] Fix bug# 67471678 - Support If-Modified-Since. --- .../adaptor/database/DatabaseAdaptor.java | 73 ++++++++-- .../adaptor/database/DatabaseAdaptorTest.java | 130 ++++++++++++++++++ 2 files changed, 189 insertions(+), 14 deletions(-) diff --git a/src/com/google/enterprise/adaptor/database/DatabaseAdaptor.java b/src/com/google/enterprise/adaptor/database/DatabaseAdaptor.java index c0539cf..bae365e 100644 --- a/src/com/google/enterprise/adaptor/database/DatabaseAdaptor.java +++ b/src/com/google/enterprise/adaptor/database/DatabaseAdaptor.java @@ -17,6 +17,8 @@ import static java.util.Locale.US; import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.enterprise.adaptor.AbstractAdaptor; import com.google.enterprise.adaptor.Acl; import com.google.enterprise.adaptor.AdaptorContext; @@ -78,6 +80,9 @@ public class DatabaseAdaptor extends AbstractAdaptor { /** Map from SQL types to SQL type names. */ private static final HashMap sqlTypeNames = new HashMap<>(); + /** Cache for DocId and last modified time stamp. */ + private static Cache docIdLastModifiedMap; + static { for (Field field : Types.class.getFields()) { if (field.getType() == int.class @@ -136,6 +141,9 @@ private static boolean isNullOrEmptyString(String str) { private boolean docIdIsUrl; private String modeOfOperation; + private Calendar updateTimestampTimezone; + private DateFormat formatter; + @Override public void initConfig(Config config) { config.addKey("db.driverClass", null); @@ -284,6 +292,18 @@ public void init(AdaptorContext context) throws Exception { String updateSql = cfg.getValue("db.updateSql"); String tzString = cfg.getValue("db.updateTimestampTimezone"); + + if (isNullOrEmptyString(tzString)) { + updateTimestampTimezone = Calendar.getInstance(); + } else { + updateTimestampTimezone = + Calendar.getInstance(TimeZone.getTimeZone(tzString)); + } + log.config("updateTimestampTimezone: " + + updateTimestampTimezone.getTimeZone().getDisplayName()); + formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z"); + formatter.setTimeZone(updateTimestampTimezone.getTimeZone()); + if (!updateSql.isEmpty()) { context.setPollingIncrementalLister( new DbAdaptorIncrementalLister(updateSql, tzString)); @@ -345,6 +365,9 @@ public void init(AdaptorContext context) throws Exception { uniqueKey = ukBuilder.build(); log.config("primary key: " + uniqueKey); + + docIdLastModifiedMap = + CacheBuilder.newBuilder().maximumSize(1000000).build(); } /** A SQL data type, with a type code and type name. */ @@ -462,6 +485,8 @@ public void getDocIds(DocIdPusher pusher) throws IOException, log.finer("queried for stream"); boolean hasAction = hasColumn(rs.getMetaData(), GsaSpecialColumns.GSA_ACTION); + boolean hasTimestamp = + hasColumn(rs.getMetaData(), GsaSpecialColumns.GSA_TIMESTAMP); log.log(Level.FINEST, "Has GSA_ACTION column: {0}", hasAction); while (rs.next()) { DocId id; @@ -472,6 +497,20 @@ public void getDocIds(DocIdPusher pusher) throws IOException, + e.getMessage()); continue; } + // Cache last modified time stamp for this DocId + if (hasTimestamp) { + Timestamp lastDocIdTimestamp = docIdLastModifiedMap.getIfPresent(id); + Timestamp ts = + rs.getTimestamp(GsaSpecialColumns.GSA_TIMESTAMP.toString(), + updateTimestampTimezone); + if ((ts != null) + && (lastDocIdTimestamp == null || ts.after(lastDocIdTimestamp))) { + lastDocIdTimestamp = ts; + docIdLastModifiedMap.put(id, lastDocIdTimestamp); + log.log(Level.FINE, "docIdLastModifiedMap updated: {0}", + formatter.format(new Date(lastDocIdTimestamp.getTime()))); + } + } DocIdPusher.Record.Builder builder = new DocIdPusher.Record.Builder(id); if (hasAction && isDeleteAction(rs)) { builder.setDeleteFromIndex(true); @@ -673,6 +712,15 @@ public void getDocContent(Request req, Response resp) throws IOException { resp.respondNotFound(); return; } + // Check if modified since last access. + Timestamp lastDocIdTimestamp = docIdLastModifiedMap.getIfPresent(id); + if (lastDocIdTimestamp != null + && !req.hasChangedSinceLastAccess( + new Date(lastDocIdTimestamp.getTime()))) { + log.log(Level.FINE, "Content not modified since last crawl: {0}", id); + resp.respondNotModified(); + return; + } // Generate response metadata first. addMetadataToResponse(resp, rs); // Generate Acl if aclSql is provided. @@ -682,7 +730,8 @@ public void getDocContent(Request req, Response resp) throws IOException { // Generate response body. // In database adaptor's case, we almost never want to follow the URLs. // One record means one document. - resp.setNoFollow(true); + log.log(Level.FINE, "Content modified since last crawl: {0}", id); + resp.setNoFollow(true); respGenerator.generateResponse(rs, resp); } catch (SQLException ex) { throw new IOException("retrieval error", ex); @@ -974,24 +1023,12 @@ private static String getResponseGeneratorMethods() { // next full push to be sent to GSA. private class DbAdaptorIncrementalLister implements PollingIncrementalLister { private final String updateSql; - private final Calendar updateTimestampTimezone; private Timestamp lastUpdateTimestamp; - private final DateFormat formatter; public DbAdaptorIncrementalLister(String updateSql, String tzString) { this.updateSql = updateSql; - if (isNullOrEmptyString(tzString)) { - updateTimestampTimezone = Calendar.getInstance(); - } else { - updateTimestampTimezone = - Calendar.getInstance(TimeZone.getTimeZone(tzString)); - } log.config("update sql: " + this.updateSql); - log.config("updateTimestampTimezone: " - + updateTimestampTimezone.getTimeZone().getDisplayName()); this.lastUpdateTimestamp = new Timestamp(System.currentTimeMillis()); - formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z"); - formatter.setTimeZone(updateTimestampTimezone.getTimeZone()); } @Override @@ -1031,8 +1068,10 @@ public void getModifiedDocIds(DocIdPusher pusher) log.log(Level.FINEST, "doc id: {0}", id); outstream.add(record); - // update latestTimestamp + // update time stamps if (hasTimestamp) { + Timestamp lastDocIdTimestamp = + docIdLastModifiedMap.getIfPresent(id); Timestamp ts = rs.getTimestamp(GsaSpecialColumns.GSA_TIMESTAMP.toString(), updateTimestampTimezone); @@ -1042,6 +1081,12 @@ public void getModifiedDocIds(DocIdPusher pusher) log.log(Level.FINE, "latestTimestamp updated: {0}", formatter.format(new Date(latestTimestamp.getTime()))); } + if (lastDocIdTimestamp == null || ts.after(lastDocIdTimestamp)) { + lastDocIdTimestamp = ts; + docIdLastModifiedMap.put(id, lastDocIdTimestamp); + log.log(Level.FINE, "lastDocIdTimestamp updated: {0}", + formatter.format(new Date(lastDocIdTimestamp.getTime()))); + } } } } diff --git a/test/com/google/enterprise/adaptor/database/DatabaseAdaptorTest.java b/test/com/google/enterprise/adaptor/database/DatabaseAdaptorTest.java index d297e84..8332d50 100644 --- a/test/com/google/enterprise/adaptor/database/DatabaseAdaptorTest.java +++ b/test/com/google/enterprise/adaptor/database/DatabaseAdaptorTest.java @@ -2307,6 +2307,136 @@ public void testGetDocContent_sqlException() throws Exception { adaptor.getDocContent(request, response); } + private Timestamp now() { + return new Timestamp(System.currentTimeMillis()); + } + + private Timestamp nowPlusMinutes(int minutes) { + long millis = System.currentTimeMillis() + MINUTES.toMillis(minutes); + return new Timestamp(millis); + } + + private Map initDataAndConfig(Timestamp modifyTs) + throws SQLException { + executeUpdate("create table data(id integer, content varchar(20), " + + "gsa_timestamp timestamp)"); + executeUpdate("insert into data(id, content, gsa_timestamp) values(1, " + + "'Hello', {ts '" + modifyTs + "'})"); + + Map configEntries = new HashMap(); + configEntries.put("db.uniqueKey", "id:int"); + configEntries.put("db.everyDocIdSql", "select * from data"); + configEntries.put("db.singleDocContentSql", + "select * from data where id = ?"); + configEntries.put("db.modeOfOperation", "contentColumn"); + configEntries.put("db.modeOfOperation.contentColumn.columnName", "content"); + return configEntries; + } + + @Test + public void testIfModifiedSince_null() throws Exception { + Map configEntries = initDataAndConfig(now()); + + DatabaseAdaptor adaptor = getObjectUnderTest(configEntries); + MockRequest request = new MockRequest(new DocId("1"), null); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + RecordingResponse response = new RecordingResponse(baos); + adaptor.getDocContent(request, response); + assertEquals("Hello", baos.toString(UTF_8.toString())); + } + + private Response testIfModifiedSince_getDocIds(Timestamp modifyTs, + Timestamp lastAccessTs) throws Exception { + Map configEntries = initDataAndConfig(modifyTs); + + DatabaseAdaptor adaptor = getObjectUnderTest(configEntries); + RecordingDocIdPusher pusher = new RecordingDocIdPusher(); + adaptor.getDocIds(pusher); + MockRequest request = + new MockRequest(new DocId("1"), (lastAccessTs == null ? null + : new Date(lastAccessTs.getTime()))); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + RecordingResponse response = new RecordingResponse(baos); + adaptor.getDocContent(request, response); + return response; + } + + @Test + public void testIfModifiedSince_getDocIds_lastAccess_null() throws Exception { + Response response = testIfModifiedSince_getDocIds(now(), null); + ByteArrayOutputStream baos = + (ByteArrayOutputStream) response.getOutputStream(); + assertEquals("Hello", baos.toString(UTF_8.toString())); + } + + @Test + public void testIfModifiedSince_getDocIds_lastAccess_later() + throws Exception { + RecordingResponse response = + (RecordingResponse) testIfModifiedSince_getDocIds(now(), nowPlusMinutes(1)); + assertEquals(RecordingResponse.State.NOT_MODIFIED, response.getState()); + } + + @Test + public void testIfModifiedSince_getDocIds_lastAccess_earlier() + throws Exception { + Response response = testIfModifiedSince_getDocIds(nowPlusMinutes(1), now()); + ByteArrayOutputStream baos2 = + (ByteArrayOutputStream) response.getOutputStream(); + assertEquals("Hello", baos2.toString(UTF_8.toString())); + } + + private Response testIfModifiedSince_getModifiedDocIds(Timestamp modifyTs, + Date lastAccessTs) throws Exception { + Map configEntries = initDataAndConfig(modifyTs); + configEntries.put("db.updateSql", + "select * from data where gsa_timestamp >= ? order by id"); + + Holder contextHolder = new Holder<>(); + DatabaseAdaptor adaptor = getObjectUnderTest(configEntries, contextHolder); + PollingIncrementalLister lister = + contextHolder.get().getPollingIncrementalLister(); + RecordingDocIdPusher pusher = new RecordingDocIdPusher(); + lister.getModifiedDocIds(pusher); + + MockRequest request = + new MockRequest(new DocId("1"), (lastAccessTs == null ? null + : new Date(lastAccessTs.getTime()))); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + RecordingResponse response = new RecordingResponse(baos); + adaptor.getDocContent(request, response); + return response; + } + + @Test + public void testIfModifiedSince_getModifiedDocIds_lastAccess_null() + throws Exception { + Response response = + testIfModifiedSince_getModifiedDocIds(nowPlusMinutes(1), null); + ByteArrayOutputStream baos = + (ByteArrayOutputStream) response.getOutputStream(); + assertEquals("Hello", baos.toString(UTF_8.toString())); + } + + @Test + public void testIfModifiedSince_getModifiedDocIds_lastAccess_later() + throws Exception { + RecordingResponse response = + (RecordingResponse) testIfModifiedSince_getModifiedDocIds( + nowPlusMinutes(1), nowPlusMinutes(2)); + assertEquals(RecordingResponse.State.NOT_MODIFIED, response.getState()); + } + + @Test + public void testIfModifiedSince_getModifiedDocIds_lastAccess_earlier() + throws Exception { + Response response = + testIfModifiedSince_getModifiedDocIds(nowPlusMinutes(1), now()); + ByteArrayOutputStream baos2 = + (ByteArrayOutputStream) response.getOutputStream(); + assertEquals("Hello", baos2.toString(UTF_8.toString())); + } + /** @see testMetadataColumns(int, String, Object, String) */ private void testMetadataColumns(int sqlType, Object input, String output) throws Exception { From 2a0bd9621b5914acc89486002f9152e148d41c1e Mon Sep 17 00:00:00 2001 From: Srinivas Veldurthi Date: Thu, 4 Jan 2018 15:47:26 -0800 Subject: [PATCH 2/5] Changes as per review. --- .../adaptor/database/DatabaseAdaptor.java | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/src/com/google/enterprise/adaptor/database/DatabaseAdaptor.java b/src/com/google/enterprise/adaptor/database/DatabaseAdaptor.java index bae365e..bd9204a 100644 --- a/src/com/google/enterprise/adaptor/database/DatabaseAdaptor.java +++ b/src/com/google/enterprise/adaptor/database/DatabaseAdaptor.java @@ -80,8 +80,8 @@ public class DatabaseAdaptor extends AbstractAdaptor { /** Map from SQL types to SQL type names. */ private static final HashMap sqlTypeNames = new HashMap<>(); - /** Cache for DocId and last modified time stamp. */ - private static Cache docIdLastModifiedMap; + /** Cache for DocId and last modified date. */ + private static Cache lastModifiedCache; static { for (Field field : Types.class.getFields()) { @@ -142,7 +142,6 @@ private static boolean isNullOrEmptyString(String str) { private String modeOfOperation; private Calendar updateTimestampTimezone; - private DateFormat formatter; @Override public void initConfig(Config config) { @@ -301,8 +300,6 @@ public void init(AdaptorContext context) throws Exception { } log.config("updateTimestampTimezone: " + updateTimestampTimezone.getTimeZone().getDisplayName()); - formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z"); - formatter.setTimeZone(updateTimestampTimezone.getTimeZone()); if (!updateSql.isEmpty()) { context.setPollingIncrementalLister( @@ -366,8 +363,9 @@ public void init(AdaptorContext context) throws Exception { uniqueKey = ukBuilder.build(); log.config("primary key: " + uniqueKey); - docIdLastModifiedMap = - CacheBuilder.newBuilder().maximumSize(1000000).build(); + lastModifiedCache = + CacheBuilder.newBuilder().initialCapacity(10000).maximumSize(1000000L) + .build(); } /** A SQL data type, with a type code and type name. */ @@ -478,16 +476,19 @@ static Map verifyColumnNames(Connection conn, @Override public void getDocIds(DocIdPusher pusher) throws IOException, InterruptedException { + DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z"); + formatter.setTimeZone(updateTimestampTimezone.getTimeZone()); BufferedPusher outstream = new BufferedPusher(pusher); + lastModifiedCache.invalidateAll(); try (Connection conn = makeNewConnection(); PreparedStatement stmt = getStreamFromDb(conn, everyDocIdSql); ResultSet rs = stmt.executeQuery()) { log.finer("queried for stream"); - boolean hasAction - = hasColumn(rs.getMetaData(), GsaSpecialColumns.GSA_ACTION); - boolean hasTimestamp = - hasColumn(rs.getMetaData(), GsaSpecialColumns.GSA_TIMESTAMP); + ResultSetMetaData rsmd = rs.getMetaData(); + boolean hasAction = hasColumn(rsmd, GsaSpecialColumns.GSA_ACTION); log.log(Level.FINEST, "Has GSA_ACTION column: {0}", hasAction); + boolean hasTimestamp = hasColumn(rsmd, GsaSpecialColumns.GSA_TIMESTAMP); + log.log(Level.FINEST, "Has GSA_TIMESTAMP column: {0}", hasTimestamp); while (rs.next()) { DocId id; try { @@ -499,16 +500,15 @@ public void getDocIds(DocIdPusher pusher) throws IOException, } // Cache last modified time stamp for this DocId if (hasTimestamp) { - Timestamp lastDocIdTimestamp = docIdLastModifiedMap.getIfPresent(id); + Date cachedLastModified = lastModifiedCache.getIfPresent(id); Timestamp ts = rs.getTimestamp(GsaSpecialColumns.GSA_TIMESTAMP.toString(), updateTimestampTimezone); - if ((ts != null) - && (lastDocIdTimestamp == null || ts.after(lastDocIdTimestamp))) { - lastDocIdTimestamp = ts; - docIdLastModifiedMap.put(id, lastDocIdTimestamp); - log.log(Level.FINE, "docIdLastModifiedMap updated: {0}", - formatter.format(new Date(lastDocIdTimestamp.getTime()))); + if ((ts != null)) { + cachedLastModified = new Date(ts.getTime()); + lastModifiedCache.put(id, cachedLastModified); + log.log(Level.FINE, "cachedLastModified updated: {0}", + formatter.format(cachedLastModified)); } } DocIdPusher.Record.Builder builder = new DocIdPusher.Record.Builder(id); @@ -702,6 +702,14 @@ public void getDocContent(Request req, Response resp) throws IOException { return; } DocId id = req.getDocId(); + // Check if modified since last access. + Date cachedLastModified = lastModifiedCache.getIfPresent(id); + if (cachedLastModified != null + && !req.hasChangedSinceLastAccess(cachedLastModified)) { + log.log(Level.FINE, "Content not modified since last crawl: {0}", id); + resp.respondNotModified(); + return; + } try (Connection conn = makeNewConnection(); PreparedStatement stmt = getDocFromDb(conn, id.getUniqueId()); ResultSet rs = stmt.executeQuery()) { @@ -712,15 +720,7 @@ public void getDocContent(Request req, Response resp) throws IOException { resp.respondNotFound(); return; } - // Check if modified since last access. - Timestamp lastDocIdTimestamp = docIdLastModifiedMap.getIfPresent(id); - if (lastDocIdTimestamp != null - && !req.hasChangedSinceLastAccess( - new Date(lastDocIdTimestamp.getTime()))) { - log.log(Level.FINE, "Content not modified since last crawl: {0}", id); - resp.respondNotModified(); - return; - } + log.log(Level.FINE, "Content modified since last crawl: {0}", id); // Generate response metadata first. addMetadataToResponse(resp, rs); // Generate Acl if aclSql is provided. @@ -730,8 +730,7 @@ public void getDocContent(Request req, Response resp) throws IOException { // Generate response body. // In database adaptor's case, we almost never want to follow the URLs. // One record means one document. - log.log(Level.FINE, "Content modified since last crawl: {0}", id); - resp.setNoFollow(true); + resp.setNoFollow(true); respGenerator.generateResponse(rs, resp); } catch (SQLException ex) { throw new IOException("retrieval error", ex); @@ -1034,6 +1033,8 @@ public DbAdaptorIncrementalLister(String updateSql, String tzString) { @Override public void getModifiedDocIds(DocIdPusher pusher) throws IOException, InterruptedException { + DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z"); + formatter.setTimeZone(updateTimestampTimezone.getTimeZone()); BufferedPusher outstream = new BufferedPusher(pusher); // latestTimestamp will be used to update lastUpdateTimestamp // if GSA_TIMESTAMP column is present in the ResultSet and there is at @@ -1070,8 +1071,7 @@ public void getModifiedDocIds(DocIdPusher pusher) // update time stamps if (hasTimestamp) { - Timestamp lastDocIdTimestamp = - docIdLastModifiedMap.getIfPresent(id); + Date cachedLastModified = lastModifiedCache.getIfPresent(id); Timestamp ts = rs.getTimestamp(GsaSpecialColumns.GSA_TIMESTAMP.toString(), updateTimestampTimezone); @@ -1081,11 +1081,11 @@ public void getModifiedDocIds(DocIdPusher pusher) log.log(Level.FINE, "latestTimestamp updated: {0}", formatter.format(new Date(latestTimestamp.getTime()))); } - if (lastDocIdTimestamp == null || ts.after(lastDocIdTimestamp)) { - lastDocIdTimestamp = ts; - docIdLastModifiedMap.put(id, lastDocIdTimestamp); - log.log(Level.FINE, "lastDocIdTimestamp updated: {0}", - formatter.format(new Date(lastDocIdTimestamp.getTime()))); + if (cachedLastModified == null) { + cachedLastModified = new Date(ts.getTime()); + lastModifiedCache.put(id, cachedLastModified); + log.log(Level.FINE, "cachedLastModified updated: {0}", + formatter.format(cachedLastModified)); } } } From e239eef5039ac60abd12dc2910ac4a0241c5f86d Mon Sep 17 00:00:00 2001 From: Srinivas Veldurthi Date: Mon, 8 Jan 2018 15:52:18 -0800 Subject: [PATCH 3/5] Changes as suggested. --- .../adaptor/database/DatabaseAdaptor.java | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/com/google/enterprise/adaptor/database/DatabaseAdaptor.java b/src/com/google/enterprise/adaptor/database/DatabaseAdaptor.java index bd9204a..df6c8be 100644 --- a/src/com/google/enterprise/adaptor/database/DatabaseAdaptor.java +++ b/src/com/google/enterprise/adaptor/database/DatabaseAdaptor.java @@ -476,10 +476,10 @@ static Map verifyColumnNames(Connection conn, @Override public void getDocIds(DocIdPusher pusher) throws IOException, InterruptedException { + lastModifiedCache.invalidateAll(); DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z"); formatter.setTimeZone(updateTimestampTimezone.getTimeZone()); BufferedPusher outstream = new BufferedPusher(pusher); - lastModifiedCache.invalidateAll(); try (Connection conn = makeNewConnection(); PreparedStatement stmt = getStreamFromDb(conn, everyDocIdSql); ResultSet rs = stmt.executeQuery()) { @@ -500,15 +500,14 @@ public void getDocIds(DocIdPusher pusher) throws IOException, } // Cache last modified time stamp for this DocId if (hasTimestamp) { - Date cachedLastModified = lastModifiedCache.getIfPresent(id); Timestamp ts = rs.getTimestamp(GsaSpecialColumns.GSA_TIMESTAMP.toString(), updateTimestampTimezone); - if ((ts != null)) { - cachedLastModified = new Date(ts.getTime()); - lastModifiedCache.put(id, cachedLastModified); - log.log(Level.FINE, "cachedLastModified updated: {0}", - formatter.format(cachedLastModified)); + if (ts != null) { + Date lastModified = new Date(ts.getTime()); + lastModifiedCache.put(id, lastModified); + log.log(Level.FINE, "lastModifiedCache updated: {0}", + formatter.format(lastModified)); } } DocIdPusher.Record.Builder builder = new DocIdPusher.Record.Builder(id); @@ -1071,22 +1070,19 @@ public void getModifiedDocIds(DocIdPusher pusher) // update time stamps if (hasTimestamp) { - Date cachedLastModified = lastModifiedCache.getIfPresent(id); Timestamp ts = rs.getTimestamp(GsaSpecialColumns.GSA_TIMESTAMP.toString(), updateTimestampTimezone); if (ts != null) { + Date lastModified = new Date(ts.getTime()); if (latestTimestamp == null || ts.after(latestTimestamp)) { latestTimestamp = ts; log.log(Level.FINE, "latestTimestamp updated: {0}", - formatter.format(new Date(latestTimestamp.getTime()))); - } - if (cachedLastModified == null) { - cachedLastModified = new Date(ts.getTime()); - lastModifiedCache.put(id, cachedLastModified); - log.log(Level.FINE, "cachedLastModified updated: {0}", - formatter.format(cachedLastModified)); + formatter.format(lastModified)); } + lastModifiedCache.put(id, lastModified); + log.log(Level.FINE, "lastModifiedCache updated: {0}", + formatter.format(lastModified)); } } } From 836b9c917d32bf950e40b37a0d3a583fade1b7ab Mon Sep 17 00:00:00 2001 From: Srinivas Veldurthi Date: Wed, 10 Jan 2018 14:54:36 -0800 Subject: [PATCH 4/5] Added additional tests. Passed 'content' as argument to helper methods in tests. --- .../adaptor/database/DatabaseAdaptorTest.java | 100 +++++++++++++----- 1 file changed, 75 insertions(+), 25 deletions(-) diff --git a/test/com/google/enterprise/adaptor/database/DatabaseAdaptorTest.java b/test/com/google/enterprise/adaptor/database/DatabaseAdaptorTest.java index 8332d50..6d94464 100644 --- a/test/com/google/enterprise/adaptor/database/DatabaseAdaptorTest.java +++ b/test/com/google/enterprise/adaptor/database/DatabaseAdaptorTest.java @@ -2316,12 +2316,12 @@ private Timestamp nowPlusMinutes(int minutes) { return new Timestamp(millis); } - private Map initDataAndConfig(Timestamp modifyTs) - throws SQLException { + private Map initDataAndConfig(String content, + Timestamp modifyTs) throws SQLException { executeUpdate("create table data(id integer, content varchar(20), " + "gsa_timestamp timestamp)"); - executeUpdate("insert into data(id, content, gsa_timestamp) values(1, " - + "'Hello', {ts '" + modifyTs + "'})"); + executeUpdate("insert into data(id, content, gsa_timestamp) values(1, '" + + content + "', {ts '" + modifyTs + "'})"); Map configEntries = new HashMap(); configEntries.put("db.uniqueKey", "id:int"); @@ -2335,19 +2335,20 @@ private Map initDataAndConfig(Timestamp modifyTs) @Test public void testIfModifiedSince_null() throws Exception { - Map configEntries = initDataAndConfig(now()); + String content = "Hello"; + Map configEntries = initDataAndConfig(content, now()); DatabaseAdaptor adaptor = getObjectUnderTest(configEntries); MockRequest request = new MockRequest(new DocId("1"), null); ByteArrayOutputStream baos = new ByteArrayOutputStream(); RecordingResponse response = new RecordingResponse(baos); adaptor.getDocContent(request, response); - assertEquals("Hello", baos.toString(UTF_8.toString())); + assertEquals(content, baos.toString(UTF_8.toString())); } - private Response testIfModifiedSince_getDocIds(Timestamp modifyTs, - Timestamp lastAccessTs) throws Exception { - Map configEntries = initDataAndConfig(modifyTs); + private Response testIfModifiedSince_getDocIds(String content, + Timestamp modifyTs, Timestamp lastAccessTs) throws Exception { + Map configEntries = initDataAndConfig(content, modifyTs); DatabaseAdaptor adaptor = getObjectUnderTest(configEntries); RecordingDocIdPusher pusher = new RecordingDocIdPusher(); @@ -2363,32 +2364,36 @@ private Response testIfModifiedSince_getDocIds(Timestamp modifyTs, @Test public void testIfModifiedSince_getDocIds_lastAccess_null() throws Exception { - Response response = testIfModifiedSince_getDocIds(now(), null); + String content = "Hello"; + Response response = testIfModifiedSince_getDocIds(content, now(), null); ByteArrayOutputStream baos = (ByteArrayOutputStream) response.getOutputStream(); - assertEquals("Hello", baos.toString(UTF_8.toString())); + assertEquals(content, baos.toString(UTF_8.toString())); } @Test public void testIfModifiedSince_getDocIds_lastAccess_later() throws Exception { RecordingResponse response = - (RecordingResponse) testIfModifiedSince_getDocIds(now(), nowPlusMinutes(1)); + (RecordingResponse) testIfModifiedSince_getDocIds("Hello", now(), + nowPlusMinutes(1)); assertEquals(RecordingResponse.State.NOT_MODIFIED, response.getState()); } @Test public void testIfModifiedSince_getDocIds_lastAccess_earlier() throws Exception { - Response response = testIfModifiedSince_getDocIds(nowPlusMinutes(1), now()); - ByteArrayOutputStream baos2 = + String content = "Hello"; + Response response = + testIfModifiedSince_getDocIds(content, nowPlusMinutes(1), now()); + ByteArrayOutputStream baos = (ByteArrayOutputStream) response.getOutputStream(); - assertEquals("Hello", baos2.toString(UTF_8.toString())); + assertEquals(content, baos.toString(UTF_8.toString())); } - private Response testIfModifiedSince_getModifiedDocIds(Timestamp modifyTs, - Date lastAccessTs) throws Exception { - Map configEntries = initDataAndConfig(modifyTs); + private Response testIfModifiedSince_getModifiedDocIds(String content, + Timestamp modifyTs, Date lastAccessTs) throws Exception { + Map configEntries = initDataAndConfig(content, modifyTs); configEntries.put("db.updateSql", "select * from data where gsa_timestamp >= ? order by id"); @@ -2411,18 +2416,19 @@ private Response testIfModifiedSince_getModifiedDocIds(Timestamp modifyTs, @Test public void testIfModifiedSince_getModifiedDocIds_lastAccess_null() throws Exception { + String content = "Hello"; Response response = - testIfModifiedSince_getModifiedDocIds(nowPlusMinutes(1), null); + testIfModifiedSince_getModifiedDocIds(content, nowPlusMinutes(1), null); ByteArrayOutputStream baos = (ByteArrayOutputStream) response.getOutputStream(); - assertEquals("Hello", baos.toString(UTF_8.toString())); + assertEquals(content, baos.toString(UTF_8.toString())); } @Test public void testIfModifiedSince_getModifiedDocIds_lastAccess_later() throws Exception { RecordingResponse response = - (RecordingResponse) testIfModifiedSince_getModifiedDocIds( + (RecordingResponse) testIfModifiedSince_getModifiedDocIds("Hello", nowPlusMinutes(1), nowPlusMinutes(2)); assertEquals(RecordingResponse.State.NOT_MODIFIED, response.getState()); } @@ -2430,11 +2436,55 @@ public void testIfModifiedSince_getModifiedDocIds_lastAccess_later() @Test public void testIfModifiedSince_getModifiedDocIds_lastAccess_earlier() throws Exception { - Response response = - testIfModifiedSince_getModifiedDocIds(nowPlusMinutes(1), now()); - ByteArrayOutputStream baos2 = + String content = "Hello"; + Response response = testIfModifiedSince_getModifiedDocIds(content, + nowPlusMinutes(1), now()); + ByteArrayOutputStream baos = (ByteArrayOutputStream) response.getOutputStream(); - assertEquals("Hello", baos2.toString(UTF_8.toString())); + assertEquals(content, baos.toString(UTF_8.toString())); + } + + @Test + public void testIfModifiedSince_getDocIdsDelDoc() throws Exception { + String content = "Hello"; + Timestamp modifyTs = now(); + Timestamp lastAccessTs = now(); + Map configEntries = initDataAndConfig(content, modifyTs); + + DatabaseAdaptor adaptor = getObjectUnderTest(configEntries); + RecordingDocIdPusher pusher = new RecordingDocIdPusher(); + adaptor.getDocIds(pusher); + + executeUpdate("delete from data where content = '" + content + "'"); + + MockRequest request = + new MockRequest(new DocId("1"), new Date(lastAccessTs.getTime())); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + RecordingResponse response = new RecordingResponse(baos); + adaptor.getDocContent(request, response); + assertEquals(RecordingResponse.State.NOT_MODIFIED, response.getState()); + } + + @Test + public void testIfModifiedSince_getDocIdsDelDocGetDocIds() throws Exception { + String content = "Hello"; + Timestamp modifyTs = now(); + Timestamp lastAccessTs = now(); + Map configEntries = initDataAndConfig(content, modifyTs); + + DatabaseAdaptor adaptor = getObjectUnderTest(configEntries); + RecordingDocIdPusher pusher = new RecordingDocIdPusher(); + adaptor.getDocIds(pusher); + + executeUpdate("delete from data where content = '" + content + "'"); + adaptor.getDocIds(pusher); + + MockRequest request = + new MockRequest(new DocId("1"), new Date(lastAccessTs.getTime())); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + RecordingResponse response = new RecordingResponse(baos); + adaptor.getDocContent(request, response); + assertEquals(RecordingResponse.State.NOT_FOUND, response.getState()); } /** @see testMetadataColumns(int, String, Object, String) */ From ed8ffe7e17429c76db61e2beda8b35e3c8ce66ba Mon Sep 17 00:00:00 2001 From: John Lacey Date: Wed, 10 Jan 2018 17:53:49 -0800 Subject: [PATCH 5/5] Replace MockRequest with DocRequest. --- .../adaptor/database/DatabaseAdaptorTest.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/com/google/enterprise/adaptor/database/DatabaseAdaptorTest.java b/test/com/google/enterprise/adaptor/database/DatabaseAdaptorTest.java index 244e2ea..d8e3d6c 100644 --- a/test/com/google/enterprise/adaptor/database/DatabaseAdaptorTest.java +++ b/test/com/google/enterprise/adaptor/database/DatabaseAdaptorTest.java @@ -2402,7 +2402,7 @@ public void testIfModifiedSince_null() throws Exception { Map configEntries = initDataAndConfig(content, now()); DatabaseAdaptor adaptor = getObjectUnderTest(configEntries); - MockRequest request = new MockRequest(new DocId("1"), null); + DocRequest request = new DocRequest(new DocId("1"), null); ByteArrayOutputStream baos = new ByteArrayOutputStream(); RecordingResponse response = new RecordingResponse(baos); adaptor.getDocContent(request, response); @@ -2416,8 +2416,8 @@ private Response testIfModifiedSince_getDocIds(String content, DatabaseAdaptor adaptor = getObjectUnderTest(configEntries); RecordingDocIdPusher pusher = new RecordingDocIdPusher(); adaptor.getDocIds(pusher); - MockRequest request = - new MockRequest(new DocId("1"), (lastAccessTs == null ? null + DocRequest request = + new DocRequest(new DocId("1"), (lastAccessTs == null ? null : new Date(lastAccessTs.getTime()))); ByteArrayOutputStream baos = new ByteArrayOutputStream(); RecordingResponse response = new RecordingResponse(baos); @@ -2467,8 +2467,8 @@ private Response testIfModifiedSince_getModifiedDocIds(String content, RecordingDocIdPusher pusher = new RecordingDocIdPusher(); lister.getModifiedDocIds(pusher); - MockRequest request = - new MockRequest(new DocId("1"), (lastAccessTs == null ? null + DocRequest request = + new DocRequest(new DocId("1"), (lastAccessTs == null ? null : new Date(lastAccessTs.getTime()))); ByteArrayOutputStream baos = new ByteArrayOutputStream(); RecordingResponse response = new RecordingResponse(baos); @@ -2520,8 +2520,8 @@ public void testIfModifiedSince_getDocIdsDelDoc() throws Exception { executeUpdate("delete from data where content = '" + content + "'"); - MockRequest request = - new MockRequest(new DocId("1"), new Date(lastAccessTs.getTime())); + DocRequest request = + new DocRequest(new DocId("1"), new Date(lastAccessTs.getTime())); ByteArrayOutputStream baos = new ByteArrayOutputStream(); RecordingResponse response = new RecordingResponse(baos); adaptor.getDocContent(request, response); @@ -2542,8 +2542,8 @@ public void testIfModifiedSince_getDocIdsDelDocGetDocIds() throws Exception { executeUpdate("delete from data where content = '" + content + "'"); adaptor.getDocIds(pusher); - MockRequest request = - new MockRequest(new DocId("1"), new Date(lastAccessTs.getTime())); + DocRequest request = + new DocRequest(new DocId("1"), new Date(lastAccessTs.getTime())); ByteArrayOutputStream baos = new ByteArrayOutputStream(); RecordingResponse response = new RecordingResponse(baos); adaptor.getDocContent(request, response);