diff --git a/pom.xml b/pom.xml index 0353cee..ed336da 100644 --- a/pom.xml +++ b/pom.xml @@ -1,5 +1,5 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 com.epimorphics armlib @@ -7,12 +7,20 @@ jar armlib Asynchronous Request Management library - + scm:git:ssh://git@github.com:epimorphics/armlib.git HEAD + + UTF-8 + 21 + 21 + 4.0.0-SNAPSHOT + 3.1.11 + + apache-repo-releases @@ -20,7 +28,6 @@ true - @@ -33,31 +40,31 @@ true - - - epi-public-s3-snapshot - Epimorphics S3 snapshot repository - https://s3-eu-west-1.amazonaws.com/epi-repository/snapshot - - false - - - true - - - - - epi-public-s3-release - Epimorphics S3 release repository - https://s3-eu-west-1.amazonaws.com/epi-repository/release - - true - - - false - - - + + + epi-public-s3-snapshot + Epimorphics S3 snapshot repository + https://s3-eu-west-1.amazonaws.com/epi-repository/snapshot + + false + + + true + + + + + epi-public-s3-release + Epimorphics S3 release repository + https://s3-eu-west-1.amazonaws.com/epi-repository/release + + true + + + false + + + true @@ -69,20 +76,19 @@ Epimorphics Public Repository https://repository.epimorphics.com - - - - - software.amazon.awssdk - bom - 2.33.4 - pom - import - - - + + + + software.amazon.awssdk + bom + 2.33.4 + pom + import + + + @@ -94,58 +100,60 @@ - software.amazon.awssdk - s3 + software.amazon.awssdk + s3 2.33.13 - software.amazon.awssdk - dynamodb-enhanced + software.amazon.awssdk + dynamodb-enhanced 2.33.13 com.epimorphics appbase - 3.1.14 + ${appbase.version} - + - javax.servlet - javax.servlet-api - 3.1.0 + jakarta.servlet + jakarta.servlet-api + 6.0.0 - org.glassfish.jersey.containers - - jersey-container-servlet - 2.47 + org.glassfish.jersey.containers + + jersey-container-servlet + ${jersey.version} + + ch.qos.logback + logback-classic + 1.5.20 + test + - - UTF-8 - - org.apache.maven.plugins maven-compiler-plugin - 2.3.2 + 3.3 - 1.8 - 1.8 + ${maven.compiler.source} + ${maven.compiler.target} - + org.apache.maven.plugins maven-scm-plugin - 1.8 + 1.9.5 developerConnection @@ -154,45 +162,49 @@ org.apache.maven.plugins maven-release-plugin - 2.5 + 2.5.3 - org.apache.maven.wagon wagon-ftp - 2.6 + 2.12 - + org.springframework.build aws-maven 5.0.0.RELEASE - - - - epi-public-s3-release - Epimorphics S3 release repository - s3://epi-repository/release - - - - epi-public-s3-snapshot - Epimorphics S3 snapshot repository - s3://epi-repository/snapshot - - + + epi-public-s3-release + Epimorphics S3 release repository + s3://epi-repository/release + + true + + + false + + + + + epi-public-s3-snapshot + Epimorphics S3 snapshot repository + s3://epi-repository/snapshot + + false + + + true + + diff --git a/src/main/java/com/epimorphics/armlib/BatchRequest.java b/src/main/java/com/epimorphics/armlib/BatchRequest.java index a18aa22..8691b56 100644 --- a/src/main/java/com/epimorphics/armlib/BatchRequest.java +++ b/src/main/java/com/epimorphics/armlib/BatchRequest.java @@ -2,32 +2,31 @@ * File: BatchRequest.java * Created by: Dave Reynolds * Created on: 11 Nov 2015 - * + * * (c) Copyright 2015, Epimorphics Limited * *****************************************************************/ package com.epimorphics.armlib; +import com.epimorphics.util.EpiException; +import jakarta.ws.rs.core.MultivaluedMap; +import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap; + +import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; -import javax.ws.rs.core.MultivaluedMap; - -import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap; - -import com.epimorphics.util.EpiException; - /** * Represents a request, originally submitted via some REST front end, which * is to be queued as a batch job. Comprises a request URI and a set of parameter * values. *

Requests can also include a flag ("sticky") to indicate that the result * should be persistently cached.

- * + * * @author Dave Reynolds */ public class BatchRequest { @@ -38,21 +37,21 @@ public class BatchRequest { protected String key; protected long estimatedTime = 60000; protected boolean sticky; - + public BatchRequest(String requestURI, MultivaluedMap parameters) { this(requestURI, parameters, false); } - + public BatchRequest(String requestURI, MultivaluedMap parameters, boolean sticky) { this.requestURI = requestURI; this.parameters = parameters; this.sticky = sticky; } - + public BatchRequest(String requestURI, String parameterString) { this(requestURI, parameterString, false); } - + public BatchRequest(String requestURI, String parameterString, boolean sticky) { this.requestURI = requestURI; this.parameters = decodeParameterString(parameterString); @@ -67,7 +66,7 @@ public String getRequestURI() { } /** - * All relevant parameters for the request. Will typically include at least query parameters + * All relevant parameters for the request. Will typically include at least query parameters * but may also include path parameters. Process parameters not relevant to generating * the result should be stripped. */ @@ -79,20 +78,20 @@ public MultivaluedMap getParameters() { * Return the parameters for the request as a single query string */ public String getParameterString() { - StringBuffer buff = new StringBuffer(); + StringBuilder buff = new StringBuilder(); boolean started = false; for (String param : parameters.keySet()) { for (String value : parameters.get(param)) { if (started) buff.append("&"); - buff.append( param ) ; + buff.append(param); buff.append("="); - buff.append( value ); + buff.append(value); started = true; } } return buff.toString(); } - + public static MultivaluedMap decodeParameterString(String paramString) { MultivaluedMap parameters = new MultivaluedStringMap(); for (String binding : paramString.split("&")) { @@ -106,7 +105,7 @@ public static MultivaluedMap decodeParameterString(String paramS } return parameters; } - + /** * If true this indicates the result should be cached in a separate persistent cache * which is not subject to whatever timeout/garbage collection policy applies to @@ -115,17 +114,18 @@ public static MultivaluedMap decodeParameterString(String paramS public boolean isSticky() { return sticky; } - + public void setSticky(boolean sticky) { this.sticky = sticky; } - + /** * Assign a key which identifies the request, useful if the key name should * be readable. * If one is not assigned one will be generated. - * @param key The key to use must be shorter than MAX_KEY_LENGTH and must - * not contain "/" path separators. + * + * @param key The key to use must be shorter than MAX_KEY_LENGTH and must + * not contain "/" path separators. */ public void setKey(String key) { if (key.length() > MAX_KEY_LENGTH || key.contains("/")) { @@ -133,42 +133,43 @@ public void setKey(String key) { } this.key = key; } - + /** * Returns a short unique key identifying the request. This is guaranteed to fit * within the limitations of S3 key lengths. */ public String getKey() { if (key == null) { - StringBuffer kb = new StringBuffer(); + StringBuilder kb = new StringBuilder(); kb.append(requestURI); kb.append("_"); for (String p : sorted(parameters.keySet())) { kb.append(p); kb.append("_"); for (String value : sorted(parameters.get(p))) { - kb.append(value); kb.append("_"); + kb.append(value); + kb.append("_"); } } - + key = kb.toString(); key = key.replace("/", "%2F"); - key = key.substring(0, key.length()-1); // Strip last "_" + key = key.substring(0, key.length() - 1); // Strip last "_" if (key.length() > MAX_KEY_LENGTH) { // Explicit coding too big, so use digest try { MessageDigest md = MessageDigest.getInstance("MD5"); - md.update( requestURI.getBytes("UTF-8") ); + md.update(requestURI.getBytes(StandardCharsets.UTF_8)); for (String p : sorted(parameters.keySet())) { for (String value : sorted(parameters.get(p))) { String k = p + "=" + value; - md.update( k.getBytes("UTF-8") ); + md.update(k.getBytes(StandardCharsets.UTF_8)); } } byte[] array = md.digest(); - StringBuffer sb = new StringBuffer(); - for (int i = 0; i < array.length; ++i) { - sb.append(Integer.toHexString((array[i] & 0xFF) | 0x100).substring(1,3)); + StringBuilder sb = new StringBuilder(); + for (byte b : array) { + sb.append(Integer.toHexString((b & 0xFF) | 0x100), 1, 3); } key = sb.toString(); } catch (Exception e) { @@ -178,7 +179,7 @@ public String getKey() { } return key; } - + /** * Get the estimated time to process this request (in ms) */ diff --git a/src/test/java/com/epimorphics/armlib/TestBatchRequest.java b/src/test/java/com/epimorphics/armlib/TestBatchRequest.java index 7af0f05..aaf1f72 100644 --- a/src/test/java/com/epimorphics/armlib/TestBatchRequest.java +++ b/src/test/java/com/epimorphics/armlib/TestBatchRequest.java @@ -2,23 +2,19 @@ * File: TestBatchRequest.java * Created by: Dave Reynolds * Created on: 11 Nov 2015 - * + * * (c) Copyright 2015, Epimorphics Limited * *****************************************************************/ package com.epimorphics.armlib; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; - -import javax.ws.rs.core.MultivaluedMap; - +import jakarta.ws.rs.core.MultivaluedMap; import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap; import org.junit.Test; +import static org.junit.Assert.*; + public class TestBatchRequest { @Test @@ -28,33 +24,33 @@ public void testBasics() { parameters.add("key1", "value2"); parameters.add("key2", "value3"); BatchRequest request = new BatchRequest("http://localhost/service", parameters); - + String paramString = request.getParameterString(); - assertTrue( paramString.contains("key1=value1") ); - assertTrue( paramString.contains("key1=value2") ); - assertTrue( paramString.contains("key2=value3") ); - + assertTrue(paramString.contains("key1=value1")); + assertTrue(paramString.contains("key1=value2")); + assertTrue(paramString.contains("key2=value3")); + assertEquals("http://localhost/service", request.getRequestURI()); assertEquals("value3", request.getParameters().getFirst("key2")); String key = request.getKey(); assertNotNull(key); - assertTrue( key.length() <= 800); - + assertTrue(key.length() <= 800); + request.setKey("NewKey"); assertEquals("NewKey", request.getKey()); - + parameters.add("key2", "value4"); String keyOther = new BatchRequest("http://localhost/service", parameters).getKey(); assertNotSame(key, keyOther); - + parameters.addFirst("key3", null); // Just parameter no value - assertTrue( parameters.containsKey("key3") ); - + assertTrue(parameters.containsKey("key3")); + request = new BatchRequest("http://localhost/service", parameters); String enc = request.getParameterString(); - MultivaluedMap recovered = BatchRequest.decodeParameterString( enc ); - + MultivaluedMap recovered = BatchRequest.decodeParameterString(enc); + assertEquals(parameters, recovered); - + } } diff --git a/src/test/java/com/epimorphics/armlib/TestRequestManager.java b/src/test/java/com/epimorphics/armlib/TestRequestManager.java index 8b4c648..bf148fd 100644 --- a/src/test/java/com/epimorphics/armlib/TestRequestManager.java +++ b/src/test/java/com/epimorphics/armlib/TestRequestManager.java @@ -2,19 +2,24 @@ * File: TestRequestManager.java * Created by: Dave Reynolds * Created on: 17 Nov 2015 - * + * * (c) Copyright 2015, Epimorphics Limited * *****************************************************************/ package com.epimorphics.armlib; -import static com.epimorphics.armlib.impl.DynQueueManager.COMPLETED_TIME_INDEX; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import com.epimorphics.armlib.BatchStatus.StatusFlag; +import com.epimorphics.armlib.impl.*; +import com.epimorphics.util.FileUtil; +import jakarta.ws.rs.core.MultivaluedMap; +import org.apache.jena.util.FileManager; +import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap; +import org.junit.Ignore; +import org.junit.Test; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.ScanRequest; +import software.amazon.awssdk.services.dynamodb.model.ScanResponse; import java.io.IOException; import java.io.InputStream; @@ -24,29 +29,14 @@ import java.util.Map; import java.util.zip.GZIPInputStream; -import javax.ws.rs.core.MultivaluedMap; - -import org.apache.jena.util.FileManager; -import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap; -import org.junit.Ignore; -import org.junit.Test; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.ScanRequest; -import software.amazon.awssdk.services.dynamodb.model.ScanResponse; - -import com.epimorphics.armlib.BatchStatus.StatusFlag; -import com.epimorphics.armlib.impl.DynQueueManager; -import com.epimorphics.armlib.impl.FileCacheManager; -import com.epimorphics.armlib.impl.MemQueueManager; -import com.epimorphics.armlib.impl.S3CacheManager; -import com.epimorphics.armlib.impl.StandardRequestManager; -import com.epimorphics.util.FileUtil; +import static com.epimorphics.armlib.impl.DynQueueManager.COMPLETED_TIME_INDEX; +import static org.junit.Assert.*; /** * Generic test for request managers. Can be configured to run * AWS based versions as well but default release testing uses * test implementations. - * + * * @author Dave Reynolds */ public class TestRequestManager { @@ -56,42 +46,42 @@ public void testGenericRequestManager() throws IOException, InterruptedException doLocalTest(false); doLocalTest(true); } - + protected void doLocalTest(boolean compress) throws IOException, InterruptedException { FileCacheManager cache = new FileCacheManager(); String testDir = Files.createTempDirectory("testmonitor").toFile().getPath(); - cache.setCacheDir( testDir ); + cache.setCacheDir(testDir); cache.setCompressed(compress); MemQueueManager queue = new MemQueueManager(); queue.setCheckInterval(5); - + doTestStandardRequestManager(queue, cache); FileUtil.deleteDirectory(testDir); } - + // Test requires local instance of DynamoDB running on port 8000 @Ignore @Test public void testWithDyn() throws IOException, InterruptedException { FileCacheManager cache = new FileCacheManager(); String testDir = Files.createTempDirectory("testmonitor").toFile().getPath(); - cache.setCacheDir( testDir ); + cache.setCacheDir(testDir); DynQueueManager queue = new DynQueueManager(); queue.setCheckInterval(100); queue.setLocalTestEndpoint("http://localhost:8000"); queue.setTablePrefix("Test-"); queue.startup(null); - + doTestStandardRequestManager(queue, cache); addCompletedRequest(queue, 1); assertEquals(2, countCompleted(queue)); long cutoff = System.currentTimeMillis(); - + addCompletedRequest(queue, 3); assertEquals(3, countCompleted(queue)); - + queue.removeOldCompletedRequests(cutoff); assertEquals(1, countCompleted(queue)); } @@ -101,21 +91,21 @@ private void addCompletedRequest(DynQueueManager queue, int i) { queue.submit(request); queue.finishRequest(request.getKey()); } - + private int countCompleted(DynQueueManager queue) { ScanResponse result = queue.getDynamoClient().scan(ScanRequest.builder() .tableName(queue.getCompletedTableName()) .indexName(COMPLETED_TIME_INDEX) .build()); int count = 0; - for (Map item : result.items()) { + for (Map item : result.items()) { if (item.get("Status").s().equals(StatusFlag.Completed.name())) { count++; } } return count; } - + // Test requires credentials and default profile for access to aws-expt @Ignore @Test @@ -126,35 +116,35 @@ public void testWithS3() throws IOException, InterruptedException { MemQueueManager queue = new MemQueueManager(); queue.setCheckInterval(5); - + doTestStandardRequestManager(queue, cm); cm.clear(); } - - protected static void doTestStandardRequestManager(QueueManager qm, CacheManager cm) throws InterruptedException, IOException { + + protected static void doTestStandardRequestManager(QueueManager qm, CacheManager cm) throws InterruptedException, IOException { StandardRequestManager rm = new StandardRequestManager(); rm.setCacheManager(cm); rm.setQueueManager(qm); - + // Empty queue - assertNull( qm.nextRequest(12) ); - + assertNull(qm.nextRequest(12)); + // Request not present in empty queue BatchRequest req1 = request("/test1", false, "p", "foo", "q", "bar"); req1.setEstimatedTime(100); - assertNull( rm.findRequest(req1.getKey()) ); - BatchStatus s1 = rm.getStatus( req1.getKey() ); - assertNotNull( s1 ); + assertNull(rm.findRequest(req1.getKey())); + BatchStatus s1 = rm.getStatus(req1.getKey()); + assertNotNull(s1); assertEquals(StatusFlag.Unknown, s1.getStatus()); - + // Submit request and then its visible s1 = rm.submit(req1); assertEquals(StatusFlag.Pending, s1.getStatus()); - s1 = rm.getStatus( req1.getKey() ); + s1 = rm.getStatus(req1.getKey()); assertEquals(StatusFlag.Pending, s1.getStatus()); - + // Request parameter order can vary - BatchRequest req1b = rm.findRequest( request("/test1", true, "q", "bar", "p", "foo").getKey() ); + BatchRequest req1b = rm.findRequest(request("/test1", true, "q", "bar", "p", "foo").getKey()); assertEquals(req1.getRequestURI(), req1b.getRequestURI()); assertEquals(req1.getParameters(), req1b.getParameters()); @@ -164,86 +154,86 @@ protected static void doTestStandardRequestManager(QueueManager qm, CacheManager rm.submit(req2); BatchStatus s2 = rm.getFullStatus(req2.getKey()); assertEquals(StatusFlag.Pending, s2.getStatus()); - assertEquals(150, (long)s2.getEta().get()); - assertEquals(2, (int)s2.getPositionInQueue().get()); + assertEquals(150, (long) s2.getEta().get()); + assertEquals(2, (int) s2.getPositionInQueue().get()); // Queue summary looks right checkQueue(rm, req1.getKey(), StatusFlag.Pending, req2.getKey(), StatusFlag.Pending); - + // Nothing in the cache yet for either - assertFalse( cm.isReady( req1.getKey() ) ); - assertFalse( cm.isReady( req2.getKey() ) ); - + assertFalse(cm.isReady(req1.getKey())); + assertFalse(cm.isReady(req2.getKey())); + // Start one request and that changes its state long start1 = System.currentTimeMillis(); BatchRequest next = qm.nextRequest(12); long start2 = System.currentTimeMillis(); - assertNotNull( next ); + assertNotNull(next); assertEquals(req1.getRequestURI(), next.getRequestURI()); checkQueue(rm, req1.getKey(), StatusFlag.InProgress, req2.getKey(), StatusFlag.Pending); - s1 = rm.getStatus( req1.getKey() ); - assertTrue( s1.getStarted().isPresent() ); - assertTrue( s1.getStarted().get() >= start1 ); - assertTrue( s1.getStarted().get() <= start2 ); + s1 = rm.getStatus(req1.getKey()); + assertTrue(s1.getStarted().isPresent()); + assertTrue(s1.getStarted().get() >= start1); + assertTrue(s1.getStarted().get() <= start2); // Put it back and try again - qm.abortRequest( next.getKey() ); + qm.abortRequest(next.getKey()); checkQueue(rm, req1.getKey(), StatusFlag.Pending, req2.getKey(), StatusFlag.Pending); next = qm.nextRequest(12); assertEquals(req1.getRequestURI(), next.getRequestURI()); checkQueue(rm, req1.getKey(), StatusFlag.InProgress, req2.getKey(), StatusFlag.Pending); - + // Generate a dummy result for the request and finish it Pipe pipe = cm.upload(next); OutputStream out = pipe.getSource(); - out.write( "Test1 result".getBytes() ); + out.write("Test1 result".getBytes()); out.close(); pipe.waitForCompletion(); qm.finishRequest(next.getKey()); checkQueue(rm, req2.getKey(), StatusFlag.Pending); s2 = rm.getFullStatus(req2.getKey()); - assertEquals(50, (long)s2.getEta().get()); - assertEquals(1, (int)s2.getPositionInQueue().get()); - assertEquals(StatusFlag.Completed, rm.getStatus( req1.getKey() ).getStatus()); - + assertEquals(50, (long) s2.getEta().get()); + assertEquals(1, (int) s2.getPositionInQueue().get()); + assertEquals(StatusFlag.Completed, rm.getStatus(req1.getKey()).getStatus()); + // Cache sees it OK - assertTrue( cm.isReady( req1.getKey() ) ); - InputStream in = cm.readResult( req1.getKey() ); + assertTrue(cm.isReady(req1.getKey())); + InputStream in = cm.readResult(req1.getKey()); if (cm.isCompressed()) { in = new GZIPInputStream(in); } - String value = FileManager.get().readWholeFileAsUTF8( in ); - assertEquals( "Test1 result", value); - + String value = FileManager.get().readWholeFileAsUTF8(in); + assertEquals("Test1 result", value); + // Get next request but fail it next = qm.nextRequest(12); s2 = rm.getFullStatus(req2.getKey()); - assertEquals(0, (int)s2.getPositionInQueue().get()); + assertEquals(0, (int) s2.getPositionInQueue().get()); assertEquals(StatusFlag.InProgress, s2.getStatus()); assertEquals(req2.getKey(), next.getKey()); - qm.failRequest( next.getKey() ); - assertTrue ( rm.getQueue().isEmpty() ); + qm.failRequest(next.getKey()); + assertTrue(rm.getQueue().isEmpty()); s2 = rm.getFullStatus(req2.getKey()); assertEquals(StatusFlag.Failed, s2.getStatus()); } - - private static BatchRequest request(String url, boolean sticky, String...args) { + + private static BatchRequest request(String url, boolean sticky, String... args) { MultivaluedMap parameters = new MultivaluedStringMap(); - for (int i = 0; i < args.length;) { + for (int i = 0; i < args.length; ) { String key = args[i++]; String value = args[i++]; parameters.add(key, value); } return new BatchRequest(url, parameters, sticky); } - - private static void checkQueue(RequestManager rm, Object...args) { + + private static void checkQueue(RequestManager rm, Object... args) { List queue = rm.getQueue(); - assertEquals(args.length/2, queue.size()); - for (int i = 0; i < args.length;) { - BatchStatus status = queue.get( i/2 ); - String key = (String)args[i++]; - StatusFlag s = (StatusFlag)args[i++]; + assertEquals(args.length / 2, queue.size()); + for (int i = 0; i < args.length; ) { + BatchStatus status = queue.get(i / 2); + String key = (String) args[i++]; + StatusFlag s = (StatusFlag) args[i++]; assertEquals(key, status.getKey()); assertEquals(s, status.getStatus()); }