From d43c4bb7f1c84803f383175a3943904e7ae57b89 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Wed, 16 Oct 2024 12:30:25 -0300 Subject: [PATCH 01/26] Embedding support in Java Generator --- .../java/com/genexus/db/ForEachCursor.java | 1 + .../db/driver/GXCallableStatement.java | 5 ++ .../db/driver/GXPreparedStatement.java | 4 +- .../com/genexus/db/driver/GXResultSet.java | 5 ++ .../com/genexus/db/BufferIFieldGetter.java | 4 + .../com/genexus/db/CachedIFieldGetter.java | 5 ++ .../java/com/genexus/db/IFieldGetter.java | 1 + .../java/com/genexus/db/IFieldSetter.java | 1 + gxembedding/pom.xml | 27 +++++++ .../main/java/com/genexus/db/GXEmbedding.java | 71 +++++++++++++++++ .../genexus/embedding/EmbeddingService.java | 77 +++++++++++++++++++ java/client.cfg | 2 + java/pom.xml | 5 ++ java/src/main/java/com/genexus/GXutil.java | 5 ++ .../java/com/genexus/db/ForEachCursor.java | 1 + .../db/driver/GXCallableStatement.java | 8 ++ .../com/genexus/db/driver/GXConnection.java | 3 +- .../db/driver/GXPreparedStatement.java | 21 +++++ .../com/genexus/db/driver/GXResultSet.java | 19 +++++ .../genexus/embedding/GXEmbeddingTest.java | 27 +++++++ pom.xml | 1 + 21 files changed, 290 insertions(+), 3 deletions(-) create mode 100644 gxembedding/pom.xml create mode 100644 gxembedding/src/main/java/com/genexus/db/GXEmbedding.java create mode 100644 gxembedding/src/main/java/com/genexus/embedding/EmbeddingService.java create mode 100644 java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java diff --git a/android/src/main/java/com/genexus/db/ForEachCursor.java b/android/src/main/java/com/genexus/db/ForEachCursor.java index 9f6f230b6..9f65bd7da 100644 --- a/android/src/main/java/com/genexus/db/ForEachCursor.java +++ b/android/src/main/java/com/genexus/db/ForEachCursor.java @@ -217,6 +217,7 @@ public void setTimestamp(int index, java.sql.Timestamp value) throws SQLExceptio public void setBLOBFile(int index, String fileName) throws SQLException {} public void setBLOBFile(int index, String fileName, boolean isMultiMedia) throws SQLException {} public void setBLOBFile(int index, String fileName, boolean isMultiMedia, boolean downloadContent) throws SQLException {} + public void setEmbedding(int index, Float[] value) throws SQLException {} public void setVarchar(int index, String value, int length, boolean allowsNull) throws SQLException {} public void setLongVarchar(int index, String value, boolean allowsNull) throws SQLException {} diff --git a/android/src/main/java/com/genexus/db/driver/GXCallableStatement.java b/android/src/main/java/com/genexus/db/driver/GXCallableStatement.java index a068a36ea..ea677b6eb 100644 --- a/android/src/main/java/com/genexus/db/driver/GXCallableStatement.java +++ b/android/src/main/java/com/genexus/db/driver/GXCallableStatement.java @@ -651,6 +651,11 @@ public byte[] getBytes(int columnIndex) throws SQLException return stmt.getBytes(columnIndex); } + public Float[] getGxembedding (int columnIndex) throws SQLException + { + return null; + } + public java.util.UUID getGUID(int columnIndex) throws SQLException { if (DEBUG) diff --git a/android/src/main/java/com/genexus/db/driver/GXPreparedStatement.java b/android/src/main/java/com/genexus/db/driver/GXPreparedStatement.java index 38a78e4e6..44df48d60 100644 --- a/android/src/main/java/com/genexus/db/driver/GXPreparedStatement.java +++ b/android/src/main/java/com/genexus/db/driver/GXPreparedStatement.java @@ -1307,7 +1307,9 @@ else if (file.exists() && !fileNameNew.startsWith(blobBasePath)) } } } - + + public void setEmbedding(int index, Float[] value) throws SQLException{ + } public void setBinaryStream(int index, java.io.InputStream value, int length) throws SQLException { if (DEBUG) diff --git a/android/src/main/java/com/genexus/db/driver/GXResultSet.java b/android/src/main/java/com/genexus/db/driver/GXResultSet.java index 931b9c3a2..04b706525 100644 --- a/android/src/main/java/com/genexus/db/driver/GXResultSet.java +++ b/android/src/main/java/com/genexus/db/driver/GXResultSet.java @@ -770,6 +770,11 @@ public byte[] getBytes(int columnIndex) throws SQLException return result.getBytes(columnIndex); } + public Float[] getGxembedding (int columnIndex) throws SQLException + { + return null; + } + public java.util.UUID getGUID(int columnIndex) throws SQLException { java.util.UUID value; diff --git a/common/src/main/java/com/genexus/db/BufferIFieldGetter.java b/common/src/main/java/com/genexus/db/BufferIFieldGetter.java index dc1b6b00f..d310cd713 100644 --- a/common/src/main/java/com/genexus/db/BufferIFieldGetter.java +++ b/common/src/main/java/com/genexus/db/BufferIFieldGetter.java @@ -85,6 +85,10 @@ public byte[] getBytes(int columnIndex) throws SQLException { return ((byte[]) value[columnIndex - 1]); } + public Float[] getGxembedding(int columnIndex) throws SQLException { + return ((Float[]) value[columnIndex - 1]); + } + public java.sql.Date getDate(int columnIndex) throws SQLException { return ((java.sql.Date) value[columnIndex - 1]); } diff --git a/common/src/main/java/com/genexus/db/CachedIFieldGetter.java b/common/src/main/java/com/genexus/db/CachedIFieldGetter.java index 80cf0e7f7..e112653bc 100644 --- a/common/src/main/java/com/genexus/db/CachedIFieldGetter.java +++ b/common/src/main/java/com/genexus/db/CachedIFieldGetter.java @@ -220,6 +220,11 @@ public byte[] getBytes(int columnIndex) throws SQLException else return ((byte[][])value[index])[0]; } + + public Float[] getGxembedding(int columnIndex) throws SQLException + { + return ((Float[][])value[getColumnIndex(columnIndex)])[0]; + } public java.sql.Date getDate(int columnIndex) throws SQLException { diff --git a/common/src/main/java/com/genexus/db/IFieldGetter.java b/common/src/main/java/com/genexus/db/IFieldGetter.java index e0a842811..810d5dec1 100644 --- a/common/src/main/java/com/genexus/db/IFieldGetter.java +++ b/common/src/main/java/com/genexus/db/IFieldGetter.java @@ -33,4 +33,5 @@ public interface IFieldGetter String getMultimediaUri(int columnIndex) throws SQLException; String getMultimediaUri(int columnIndex, boolean absPath) throws SQLException; java.util.UUID getGUID(int columnIndex) throws SQLException; + Float[] getGxembedding(int columnIndex) throws SQLException; } diff --git a/common/src/main/java/com/genexus/db/IFieldSetter.java b/common/src/main/java/com/genexus/db/IFieldSetter.java index 0eafd787e..05ddd5557 100644 --- a/common/src/main/java/com/genexus/db/IFieldSetter.java +++ b/common/src/main/java/com/genexus/db/IFieldSetter.java @@ -35,6 +35,7 @@ public interface IFieldSetter public void setBLOBFile(int index, String fileName) throws SQLException; public void setBLOBFile(int index, String fileName, boolean isMultiMedia) throws SQLException; public void setBLOBFile(int index, String fileName, boolean isMultiMedia, boolean downloadContent) throws SQLException; + public void setEmbedding(int index, Float[] value) throws SQLException; public void setVarchar(int index, String value, int length, boolean allowsNull) throws SQLException; public void setLongVarchar(int index, String value, boolean allowsNull) throws SQLException; diff --git a/gxembedding/pom.xml b/gxembedding/pom.xml new file mode 100644 index 000000000..1c481084e --- /dev/null +++ b/gxembedding/pom.xml @@ -0,0 +1,27 @@ + + + 4.0.0 + + + com.genexus + parent + ${revision}${changelist} + + + gxembedding + GeneXus Embedding + + + + ${project.groupId} + gxcommon + ${project.version} + + + + + gxembedding + + \ No newline at end of file diff --git a/gxembedding/src/main/java/com/genexus/db/GXEmbedding.java b/gxembedding/src/main/java/com/genexus/db/GXEmbedding.java new file mode 100644 index 000000000..f44ca61b1 --- /dev/null +++ b/gxembedding/src/main/java/com/genexus/db/GXEmbedding.java @@ -0,0 +1,71 @@ +package com.genexus.db; + +import com.genexus.CommonUtil; +import com.genexus.GXBaseCollection; +import com.genexus.SdtMessages_Message; +import com.genexus.embedding.EmbeddingService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class GXEmbedding { + + private String model; + private int dimensions; + private List embedding; + + public GXEmbedding() { + } + + public GXEmbedding(String model, int dimensions) { + this.model = model; + this.dimensions = dimensions; + embedding = new ArrayList<>(dimensions); + } + + public GXEmbedding(Float[] embedding, String model, int dimensions) { + this.model = model; + this.dimensions = dimensions; + this.embedding = Arrays.asList(embedding); + } + + public GXEmbedding(List embedding) { + this.embedding = embedding; + } + + public String getModel() { + return model; + } + + public int getDimensions() { + return dimensions; + } + + public void setEmbedding(List embedding) { + this.embedding = embedding; + } + + public Float[] getFloatArray() { + return embedding.toArray(new Float[0]); + } + + public static GXEmbedding generateEmbedding(GXEmbedding embeddingInfo, String text, GXBaseCollection Messages) { + try { + List embedding = EmbeddingService.getInstance().getEmbedding(embeddingInfo.getModel(), embeddingInfo.getDimensions(), text); + embeddingInfo.setEmbedding(embedding); + } + catch (Exception ex) { + CommonUtil.ErrorToMessages("GenerateEmbedding Error", ex.getMessage(), Messages); + } + return embeddingInfo; + } + + public String toString() + { + return embedding.stream() + .map(String::valueOf) + .collect(Collectors.joining(",")); + } +} diff --git a/gxembedding/src/main/java/com/genexus/embedding/EmbeddingService.java b/gxembedding/src/main/java/com/genexus/embedding/EmbeddingService.java new file mode 100644 index 000000000..fa15b4ef1 --- /dev/null +++ b/gxembedding/src/main/java/com/genexus/embedding/EmbeddingService.java @@ -0,0 +1,77 @@ +package com.genexus.embedding; + +import com.genexus.common.interfaces.SpecificImplementation; +import com.genexus.diagnostics.core.ILogger; +import com.genexus.diagnostics.core.LogManager; +import com.genexus.internet.HttpClient; +import json.org.json.JSONArray; +import json.org.json.JSONObject; + +import java.util.ArrayList; +import java.util.List; + +public class EmbeddingService +{ + private static final ILogger logger = LogManager.getLogger(EmbeddingService.class); + + private static EmbeddingService instance; + private final HttpClient client; + private String apiKey; + private String aiProvider; + + public EmbeddingService() { + client = new HttpClient(); + aiProvider = (String) SpecificImplementation.Application.getProperty("AI_PROVIDER", ""); + apiKey = (String) SpecificImplementation.Application.getProperty("AI_PROVIDER_API_KEY", ""); + } + public static EmbeddingService getInstance() { + if (instance == null) + instance = new EmbeddingService(); + return instance; + } + + public List getEmbedding(String model, int dimensions, String input) { + List inputList = new ArrayList<>(); + inputList.add(input); + return getEmbedding(model, dimensions, inputList); + } + + public List getEmbedding(String model, int dimensions, List inputList) { + List embeddingsList = new ArrayList<>(); + try { + String requestBody = String.format( + "{\n" + + " \"model\": \"%s\",\n" + + " \"input\": %s,\n" + + " \"dimensions\": %d\n" + + "}", + model, new JSONArray(inputList), dimensions + ); + client.setSecure(1); + client.addHeader("Content-Type", "application/json"); + client.addHeader("Authorization", "Bearer " + apiKey); + client.addHeader("X-Saia-Source", "Embedding"); + client.addString(requestBody); + client.execute("POST", aiProvider); + if (client.getStatusCode() == 200) { + JSONObject jsonResponse = new JSONObject(client.getString()); + JSONArray embeddingsArray = jsonResponse.getJSONArray("data"); + + for (int i = 0; i < embeddingsArray.length(); i++) { + JSONArray embedding = ((JSONObject)embeddingsArray.get(0)).getJSONArray("embedding");; + for (int j = 0; j < embedding.length(); j++) { + embeddingsList.add((float) embedding.getDouble(j)); + } + } + } + else { + logger.error(String.format("Error calling embedding API, StatusCode: %d, ReasonLine: %s", + client.getStatusCode(), + client.getReasonLine())); + } + } catch (Exception e) { + logger.error("GenerateEmbedding Error", e); + } + return embeddingsList; + } +} diff --git a/java/client.cfg b/java/client.cfg index 4866b57ed..3b23c517a 100644 --- a/java/client.cfg +++ b/java/client.cfg @@ -47,6 +47,8 @@ ORQ_CLIENT_URL= ORQ_SERVER_DIR= TMPMEDIA_DIR=PrivateTempStorage PRINT_LAYOUT_METADATA_DIR=LayoutMetadata +AI_PROVIDER=https://api.saia.ai/chat +AI_PROVIDER_API_KEY=xxx HTTP_PROTOCOL=Unsecure SAMESITE_COOKIE=Lax StorageTimeZone= 1 diff --git a/java/pom.xml b/java/pom.xml index 50b250f30..489c8f2db 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -34,6 +34,11 @@ gxmail ${project.version} + + ${project.groupId} + gxembedding + ${project.version} + org.apache.commons commons-collections4 diff --git a/java/src/main/java/com/genexus/GXutil.java b/java/src/main/java/com/genexus/GXutil.java index d141ccde4..aaf783793 100644 --- a/java/src/main/java/com/genexus/GXutil.java +++ b/java/src/main/java/com/genexus/GXutil.java @@ -9,6 +9,7 @@ import com.genexus.common.interfaces.SpecificImplementation; import com.genexus.db.DataStoreProvider; +import com.genexus.db.GXEmbedding; import com.genexus.internet.HttpContext; import com.genexus.internet.StringCollection; import com.genexus.platform.INativeFunctions; @@ -1777,4 +1778,8 @@ public static String buildURLFromHttpClient(com.genexus.internet.HttpClient GXSo return url.toString(); } + public static String embeddingToStr(GXEmbedding embedding) { + return embedding.toString(); + } + } diff --git a/java/src/main/java/com/genexus/db/ForEachCursor.java b/java/src/main/java/com/genexus/db/ForEachCursor.java index 328b14b87..63cf7465c 100644 --- a/java/src/main/java/com/genexus/db/ForEachCursor.java +++ b/java/src/main/java/com/genexus/db/ForEachCursor.java @@ -327,6 +327,7 @@ public void setTimestamp(int index, java.sql.Timestamp value) throws SQLExceptio public void setBLOBFile(int index, String fileName) throws SQLException {} public void setBLOBFile(int index, String fileName, boolean isMultiMedia) throws SQLException {} public void setBLOBFile(int index, String fileName, boolean isMultiMedia, boolean downloadContet) throws SQLException {} + public void setEmbedding(int index, Float[] value) throws SQLException {} public void setVarchar(int index, String value, int length, boolean allowsNull) throws SQLException {} public void setLongVarchar(int index, String value, boolean allowsNull) throws SQLException {} diff --git a/java/src/main/java/com/genexus/db/driver/GXCallableStatement.java b/java/src/main/java/com/genexus/db/driver/GXCallableStatement.java index c580d639f..77d8c515c 100644 --- a/java/src/main/java/com/genexus/db/driver/GXCallableStatement.java +++ b/java/src/main/java/com/genexus/db/driver/GXCallableStatement.java @@ -704,6 +704,14 @@ public byte[] getBytes(int columnIndex) throws SQLException return stmt.getBytes(columnIndex); } + public Float[] getGxembedding (int columnIndex) throws SQLException + { + if (DEBUG ) + log(GXDBDebug.LOG_MAX, "Warning: getEmbedding"); + + return(Float[]) stmt.getArray(columnIndex).getArray(); + } + public java.util.UUID getGUID(int columnIndex) throws SQLException { if (DEBUG) diff --git a/java/src/main/java/com/genexus/db/driver/GXConnection.java b/java/src/main/java/com/genexus/db/driver/GXConnection.java index 85da9eb00..4c88f2991 100644 --- a/java/src/main/java/com/genexus/db/driver/GXConnection.java +++ b/java/src/main/java/com/genexus/db/driver/GXConnection.java @@ -1744,8 +1744,7 @@ public void setClientInfo(String arg0, String arg1) public Array createArrayOf(String typeName, Object[] elements) throws SQLException { - // TODO Auto-generated method stub - return null; + return getJDBCConnection().createArrayOf(typeName, elements); } public void abort(Executor executor) diff --git a/java/src/main/java/com/genexus/db/driver/GXPreparedStatement.java b/java/src/main/java/com/genexus/db/driver/GXPreparedStatement.java index 4bd757dba..7d848fb33 100644 --- a/java/src/main/java/com/genexus/db/driver/GXPreparedStatement.java +++ b/java/src/main/java/com/genexus/db/driver/GXPreparedStatement.java @@ -1506,6 +1506,27 @@ else if(blobFiles.length < index) } } + public void setEmbedding(int index, Float[] value) throws SQLException{ + Array sqlArray = con.createArrayOf("float4", value); + if (DEBUG) + { + log(GXDBDebug.LOG_MAX, "setEmbedding - index : " + index); + try + { + stmt.setArray(index, sqlArray); + } + catch (SQLException sqlException) + { + if (con.isLogEnabled()) con.logSQLException(con.getHandle(), sqlException); + throw sqlException; + } + } + else + { + stmt.setArray(index, sqlArray); + } + } + public void setBinaryStream(int index, java.io.InputStream value, int length) throws SQLException { if (DEBUG) diff --git a/java/src/main/java/com/genexus/db/driver/GXResultSet.java b/java/src/main/java/com/genexus/db/driver/GXResultSet.java index b59803f84..7379d2ab1 100644 --- a/java/src/main/java/com/genexus/db/driver/GXResultSet.java +++ b/java/src/main/java/com/genexus/db/driver/GXResultSet.java @@ -831,6 +831,25 @@ public byte[] getBytes(int columnIndex) throws SQLException return result.getBytes(columnIndex); } + public Float[] getGxembedding (int columnIndex) throws SQLException + { + if (DEBUG ) + log(GXDBDebug.LOG_MAX, "Warning: getEmbedding"); + + return(Float[]) convertVectorStringToFloatArray(result.getArray(columnIndex).toString()); + } + + private static Float[] convertVectorStringToFloatArray(String vectorString) { + vectorString = vectorString.replace("[", "").replace("]", "").trim(); + String[] stringValues = vectorString.split(","); + Float[] floatArray = new Float[stringValues.length]; + + for (int i = 0; i < stringValues.length; i++) { + floatArray[i] = Float.parseFloat(stringValues[i].trim()); + } + return floatArray; + } + public java.util.UUID getGUID(int columnIndex) throws SQLException { java.util.UUID value; diff --git a/java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java b/java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java new file mode 100644 index 000000000..e599cf8b0 --- /dev/null +++ b/java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java @@ -0,0 +1,27 @@ +package com.genexus.embedding; + +import com.genexus.Application; +import com.genexus.sampleapp.GXcfg; +import com.genexus.specific.java.Connect; +import com.genexus.specific.java.LogManager; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; +import java.util.stream.Collectors; + +public class GXEmbeddingTest { + @Test + public void EmbeddingTest(){ + Connect.init(); + LogManager.initialize("."); + Application.init(GXcfg.class); + EmbeddingService service = EmbeddingService.getInstance(); + List embedding = service.getEmbedding("openai/text-embedding-3-small", 512, "Hello World"); + String result = embedding.stream() + .map(String::valueOf) + .collect(Collectors.joining(",")); + System.out.println("Embedding: " + result); + Assert.assertNotNull(embedding); + } +} diff --git a/pom.xml b/pom.xml index d14528e41..9a2b020d1 100644 --- a/pom.xml +++ b/pom.xml @@ -86,6 +86,7 @@ wrapperjakarta wrappercommon java + gxembedding gxcryptocommon gxdynamiccall gxmail From 22a47c7108da708812c281e6a00f65edf02a252c Mon Sep 17 00:00:00 2001 From: iroqueta Date: Tue, 5 Nov 2024 15:26:01 -0300 Subject: [PATCH 02/26] - Add Assitant Object support in Java Generator - Change Embedding implementation to use common util classes to call saia. --- .../java/com/genexus/util/GXProperties.java | 14 +- gxembedding/pom.xml | 27 --- .../genexus/embedding/EmbeddingService.java | 77 -------- .../main/java/com/genexus/GXProcedure.java | 16 +- .../main/java/com/genexus/db/GXEmbedding.java | 26 ++- .../com/genexus/util/saia/OpenAIRequest.java | 85 +++++++++ .../com/genexus/util/saia/OpenAIResponse.java | 171 ++++++++++++++++++ .../com/genexus/util/saia/SaiaService.java | 46 +++++ .../java/com/genexus/assistant/Assistant.java | 55 ++++++ .../com/genexus/assistant/TestAssistant.java | 25 +++ .../genexus/embedding/GXEmbeddingTest.java | 4 +- pom.xml | 1 - 12 files changed, 433 insertions(+), 114 deletions(-) delete mode 100644 gxembedding/pom.xml delete mode 100644 gxembedding/src/main/java/com/genexus/embedding/EmbeddingService.java rename {gxembedding => java}/src/main/java/com/genexus/db/GXEmbedding.java (61%) create mode 100644 java/src/main/java/com/genexus/util/saia/OpenAIRequest.java create mode 100644 java/src/main/java/com/genexus/util/saia/OpenAIResponse.java create mode 100644 java/src/main/java/com/genexus/util/saia/SaiaService.java create mode 100644 java/src/test/java/com/genexus/assistant/Assistant.java create mode 100644 java/src/test/java/com/genexus/assistant/TestAssistant.java diff --git a/common/src/main/java/com/genexus/util/GXProperties.java b/common/src/main/java/com/genexus/util/GXProperties.java index 9f645da8e..7d4aef6e5 100644 --- a/common/src/main/java/com/genexus/util/GXProperties.java +++ b/common/src/main/java/com/genexus/util/GXProperties.java @@ -1,6 +1,6 @@ package com.genexus.util; -import java.util.LinkedHashMap; +import java.util.*; import com.genexus.internet.IGxJSONSerializable; @@ -9,8 +9,6 @@ import com.genexus.CommonUtil; import com.genexus.SdtMessages_Message; import com.genexus.GXBaseCollection; -import java.util.Iterator; -import java.util.Map; public class GXProperties implements IGxJSONSerializable { private LinkedHashMap < String, GXProperty > properties = new LinkedHashMap < > (); @@ -117,6 +115,16 @@ public String toJSonString() { return jObj.toString(); } + public List getList() { + List list = new ArrayList<>(); + int i = 0; + while (count() > i) { + list.add(item(i)); + i++; + } + return list; + } + public boolean fromJSonString(String s) { return fromJSonString(s, null); } diff --git a/gxembedding/pom.xml b/gxembedding/pom.xml deleted file mode 100644 index 1c481084e..000000000 --- a/gxembedding/pom.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - 4.0.0 - - - com.genexus - parent - ${revision}${changelist} - - - gxembedding - GeneXus Embedding - - - - ${project.groupId} - gxcommon - ${project.version} - - - - - gxembedding - - \ No newline at end of file diff --git a/gxembedding/src/main/java/com/genexus/embedding/EmbeddingService.java b/gxembedding/src/main/java/com/genexus/embedding/EmbeddingService.java deleted file mode 100644 index fa15b4ef1..000000000 --- a/gxembedding/src/main/java/com/genexus/embedding/EmbeddingService.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.genexus.embedding; - -import com.genexus.common.interfaces.SpecificImplementation; -import com.genexus.diagnostics.core.ILogger; -import com.genexus.diagnostics.core.LogManager; -import com.genexus.internet.HttpClient; -import json.org.json.JSONArray; -import json.org.json.JSONObject; - -import java.util.ArrayList; -import java.util.List; - -public class EmbeddingService -{ - private static final ILogger logger = LogManager.getLogger(EmbeddingService.class); - - private static EmbeddingService instance; - private final HttpClient client; - private String apiKey; - private String aiProvider; - - public EmbeddingService() { - client = new HttpClient(); - aiProvider = (String) SpecificImplementation.Application.getProperty("AI_PROVIDER", ""); - apiKey = (String) SpecificImplementation.Application.getProperty("AI_PROVIDER_API_KEY", ""); - } - public static EmbeddingService getInstance() { - if (instance == null) - instance = new EmbeddingService(); - return instance; - } - - public List getEmbedding(String model, int dimensions, String input) { - List inputList = new ArrayList<>(); - inputList.add(input); - return getEmbedding(model, dimensions, inputList); - } - - public List getEmbedding(String model, int dimensions, List inputList) { - List embeddingsList = new ArrayList<>(); - try { - String requestBody = String.format( - "{\n" + - " \"model\": \"%s\",\n" + - " \"input\": %s,\n" + - " \"dimensions\": %d\n" + - "}", - model, new JSONArray(inputList), dimensions - ); - client.setSecure(1); - client.addHeader("Content-Type", "application/json"); - client.addHeader("Authorization", "Bearer " + apiKey); - client.addHeader("X-Saia-Source", "Embedding"); - client.addString(requestBody); - client.execute("POST", aiProvider); - if (client.getStatusCode() == 200) { - JSONObject jsonResponse = new JSONObject(client.getString()); - JSONArray embeddingsArray = jsonResponse.getJSONArray("data"); - - for (int i = 0; i < embeddingsArray.length(); i++) { - JSONArray embedding = ((JSONObject)embeddingsArray.get(0)).getJSONArray("embedding");; - for (int j = 0; j < embedding.length(); j++) { - embeddingsList.add((float) embedding.getDouble(j)); - } - } - } - else { - logger.error(String.format("Error calling embedding API, StatusCode: %d, ReasonLine: %s", - client.getStatusCode(), - client.getReasonLine())); - } - } catch (Exception e) { - logger.error("GenerateEmbedding Error", e); - } - return embeddingsList; - } -} diff --git a/java/src/main/java/com/genexus/GXProcedure.java b/java/src/main/java/com/genexus/GXProcedure.java index 2185b0840..5a8ffe760 100644 --- a/java/src/main/java/com/genexus/GXProcedure.java +++ b/java/src/main/java/com/genexus/GXProcedure.java @@ -11,8 +11,10 @@ import com.genexus.mock.GXMockProvider; import com.genexus.performance.ProcedureInfo; import com.genexus.performance.ProceduresInfo; -import com.genexus.util.ReorgSubmitThreadPool; -import com.genexus.util.SubmitThreadPool; +import com.genexus.util.*; +import com.genexus.util.saia.OpenAIRequest; +import com.genexus.util.saia.OpenAIResponse; +import com.genexus.util.saia.SaiaService; public abstract class GXProcedure implements IErrorHandler, ISubmitteable { public abstract void initialize(); @@ -257,4 +259,14 @@ protected void mockExecute() { } privateExecute( ); } + + protected String callAssistant(String assistant, GXProperties properties, Object response) { + OpenAIRequest aiRequest = new OpenAIRequest(); + aiRequest.setModel(String.format("saia:agent:%s", assistant)); + aiRequest.setVariables(properties.getList()); + OpenAIResponse aiResponse = SaiaService.call(aiRequest); + if (aiResponse != null) + return aiResponse.getChoices().get(0).getMessage().getContent(); + return ""; + } } diff --git a/gxembedding/src/main/java/com/genexus/db/GXEmbedding.java b/java/src/main/java/com/genexus/db/GXEmbedding.java similarity index 61% rename from gxembedding/src/main/java/com/genexus/db/GXEmbedding.java rename to java/src/main/java/com/genexus/db/GXEmbedding.java index f44ca61b1..e2e3d4f86 100644 --- a/gxembedding/src/main/java/com/genexus/db/GXEmbedding.java +++ b/java/src/main/java/com/genexus/db/GXEmbedding.java @@ -3,7 +3,9 @@ import com.genexus.CommonUtil; import com.genexus.GXBaseCollection; import com.genexus.SdtMessages_Message; -import com.genexus.embedding.EmbeddingService; +import com.genexus.util.saia.OpenAIRequest; +import com.genexus.util.saia.OpenAIResponse; +import com.genexus.util.saia.SaiaService; import java.util.ArrayList; import java.util.Arrays; @@ -53,7 +55,7 @@ public Float[] getFloatArray() { public static GXEmbedding generateEmbedding(GXEmbedding embeddingInfo, String text, GXBaseCollection Messages) { try { - List embedding = EmbeddingService.getInstance().getEmbedding(embeddingInfo.getModel(), embeddingInfo.getDimensions(), text); + List embedding = getEmbedding(embeddingInfo.getModel(), embeddingInfo.getDimensions(), text); embeddingInfo.setEmbedding(embedding); } catch (Exception ex) { @@ -62,6 +64,26 @@ public static GXEmbedding generateEmbedding(GXEmbedding embeddingInfo, String te return embeddingInfo; } + public static List getEmbedding(String model, int dimensions, String input) { + List inputList = new ArrayList<>(); + inputList.add(input); + return getEmbedding(model, dimensions, inputList); + } + + public static List getEmbedding(String model, int dimensions, List inputList) { + OpenAIRequest aiRequest = new OpenAIRequest(); + aiRequest.setModel(model); + aiRequest.setInput(inputList); + aiRequest.setDimension(dimensions); + OpenAIResponse aiResponse = SaiaService.call(aiRequest, true); + if (aiResponse != null) + return aiResponse.getData().get(0).getEmbedding().stream() + .map(Double::floatValue) + .collect(Collectors.toList()); + + return new ArrayList<>(); + } + public String toString() { return embedding.stream() diff --git a/java/src/main/java/com/genexus/util/saia/OpenAIRequest.java b/java/src/main/java/com/genexus/util/saia/OpenAIRequest.java new file mode 100644 index 000000000..d51da8567 --- /dev/null +++ b/java/src/main/java/com/genexus/util/saia/OpenAIRequest.java @@ -0,0 +1,85 @@ +package com.genexus.util.saia; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.genexus.util.GXProperty; + +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class OpenAIRequest { + + @JsonProperty("model") + private String model; + + @JsonProperty("prompt") + private String prompt; + + @JsonProperty("input") + private List input; + + @JsonProperty("max_tokens") + private Integer maxTokens; + + @JsonProperty("temperature") + private Double temperature; + + @JsonProperty("stream") + private Boolean stream; + + @JsonProperty("stop") + private List stop; + + @JsonProperty("presence_penalty") + private Double presencePenalty; + + @JsonProperty("frequency_penalty") + private Double frequencyPenalty; + + @JsonProperty("user") + private String user; + + @JsonProperty("variables") + private List variables; + + @JsonProperty("dimensions") + private int dimension; + + public String getModel() { return model; } + public void setModel(String model) { this.model = model; } + + public String getPrompt() { return prompt; } + public void setPrompt(String prompt) { this.prompt = prompt; } + + public List getInput() { return input; } + public void setInput(List input) { this.input = input; } + + public Integer getMaxTokens() { return maxTokens; } + public void setMaxTokens(Integer maxTokens) { this.maxTokens = maxTokens; } + + public Double getTemperature() { return temperature; } + public void setTemperature(Double temperature) { this.temperature = temperature; } + + public Boolean getStream() { return stream; } + public void setStream(Boolean stream) { this.stream = stream; } + + public List getStop() { return stop; } + public void setStop(List stop) { this.stop = stop; } + + public Double getPresencePenalty() { return presencePenalty; } + public void setPresencePenalty(Double presencePenalty) { this.presencePenalty = presencePenalty; } + + public Double getFrequencyPenalty() { return frequencyPenalty; } + public void setFrequencyPenalty(Double frequencyPenalty) { this.frequencyPenalty = frequencyPenalty; } + + public String getUser() { return user; } + public void setUser(String user) { this.user = user; } + + public List getVariables() { return variables; } + public void setVariables(List variables) { this.variables = variables; } + + public int getDimension() { return dimension; } + public void setDimension(int dimension) { this.dimension = dimension; } +} \ No newline at end of file diff --git a/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java new file mode 100644 index 000000000..d10c028b7 --- /dev/null +++ b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java @@ -0,0 +1,171 @@ +package com.genexus.util.saia; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class OpenAIResponse { + + @JsonProperty("id") + private String id; + + @JsonProperty("object") + private String object; + + @JsonProperty("created") + private long created; + + @JsonProperty("choices") + private List choices; + + @JsonProperty("usage") + private Usage usage; + + @JsonProperty("data") + private List data; + + public String getId() { return id; } + public void setId(String id) { this.id = id; } + + public String getObject() { return object; } + public void setObject(String object) { this.object = object; } + + public long getCreated() { return created; } + public void setCreated(long created) { this.created = created; } + + public List getChoices() { return choices; } + public void setChoices(List choices) { this.choices = choices; } + + public Usage getUsage() { return usage; } + public void setUsage(Usage usage) { this.usage = usage; } + + public List getData() { return data; } + public void setData(List data) { this.data = data; } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Choice { + + @JsonProperty("index") + private int index; + + @JsonProperty("message") + private Message message; + + @JsonProperty("finish_reason") + private String finishReason; + + public int getIndex() { return index; } + public void setIndex(int index) { this.index = index; } + + public Message getMessage() { return message; } + public void setMessage(Message message) { this.message = message; } + + public String getFinishReason() { return finishReason; } + public void setFinishReason(String finishReason) { this.finishReason = finishReason; } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Message { + + @JsonProperty("role") + private String role; + + @JsonProperty("content") + private String content; + + @JsonProperty("tool_calls") + private List toolCalls; + + public String getRole() { return role; } + public void setRole(String role) { this.role = role; } + + public String getContent() { return content; } + public void setContent(String content) { this.content = content; } + + public List getToolCalls() { return toolCalls; } + public void setToolCalls(List toolCalls) { this.toolCalls = toolCalls; } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class ToolCall { + + @JsonProperty("tool_name") + private String toolName; + + @JsonProperty("tool_input") + private ToolInput toolInput; + + @JsonProperty("tool_output") + private String toolOutput; + + public String getToolName() { return toolName; } + public void setToolName(String toolName) { this.toolName = toolName; } + + public ToolInput getToolInput() { return toolInput; } + public void setToolInput(ToolInput toolInput) { this.toolInput = toolInput; } + + public String getToolOutput() { return toolOutput; } + public void setToolOutput(String toolOutput) { this.toolOutput = toolOutput; } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class ToolInput { + + @JsonProperty("prompt") + private String prompt; + + @JsonProperty("size") + private String size; + + public String getPrompt() { return prompt; } + public void setPrompt(String prompt) { this.prompt = prompt; } + + public String getSize() { return size; } + public void setSize(String size) { this.size = size; } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Usage { + + @JsonProperty("prompt_tokens") + private int promptTokens; + + @JsonProperty("completion_tokens") + private int completionTokens; + + @JsonProperty("total_tokens") + private int totalTokens; + + public int getPromptTokens() { return promptTokens; } + public void setPromptTokens(int promptTokens) { this.promptTokens = promptTokens; } + + public int getCompletionTokens() { return completionTokens; } + public void setCompletionTokens(int completionTokens) { this.completionTokens = completionTokens; } + + public int getTotalTokens() { return totalTokens; } + public void setTotalTokens(int totalTokens) { this.totalTokens = totalTokens; } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class DataItem { + + @JsonProperty("id") + private String id; + + @JsonProperty("object") + private String object; + + @JsonProperty("embedding") + private List embedding; + + public String getId() { return id; } + public void setId(String id) { this.id = id; } + + public String getObject() { return object; } + public void setObject(String object) { this.object = object; } + + public List getEmbedding() { return embedding; } + public void setEmbedding(List embedding) { this.embedding = embedding; } + } +} diff --git a/java/src/main/java/com/genexus/util/saia/SaiaService.java b/java/src/main/java/com/genexus/util/saia/SaiaService.java new file mode 100644 index 000000000..70b870198 --- /dev/null +++ b/java/src/main/java/com/genexus/util/saia/SaiaService.java @@ -0,0 +1,46 @@ +package com.genexus.util.saia; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.genexus.common.interfaces.SpecificImplementation; +import com.genexus.diagnostics.core.ILogger; +import com.genexus.diagnostics.core.LogManager; +import com.genexus.internet.HttpClient; +import json.org.json.JSONObject; + +public class SaiaService { + private static final ILogger logger = LogManager.getLogger(SaiaService.class); + private static final String apiKey = (String) SpecificImplementation.Application.getProperty("AI_PROVIDER_API_KEY", "");; + private static final String aiProvider = (String) SpecificImplementation.Application.getProperty("AI_PROVIDER", ""); + + public static OpenAIResponse call(OpenAIRequest request) { + return call(request, false); + } + + public static OpenAIResponse call(OpenAIRequest request, boolean isEmbedding) { + try { + String jsonRequest = new ObjectMapper().writeValueAsString(request); + + HttpClient client = new HttpClient(); + client.setSecure(1); + client.addHeader("Content-Type", "application/json"); + client.addHeader("Authorization", "Bearer " + apiKey); + if (isEmbedding) + client.addHeader("X-Saia-Source", "Embedding"); + client.addString(jsonRequest); + client.execute("POST", aiProvider); + if (client.getStatusCode() == 200) { + JSONObject jsonResponse = new JSONObject(client.getString()); + return new ObjectMapper().readValue(jsonResponse.toString(), OpenAIResponse.class); + } + else { + logger.error(String.format("Error calling Enterprise AI API, StatusCode: %d, ReasonLine: %s", + client.getStatusCode(), + client.getReasonLine())); + } + } + catch (Exception e) { + logger.error("Calling Enterprise AI API Error", e); + } + return null; + } +} diff --git a/java/src/test/java/com/genexus/assistant/Assistant.java b/java/src/test/java/com/genexus/assistant/Assistant.java new file mode 100644 index 000000000..32d196ea2 --- /dev/null +++ b/java/src/test/java/com/genexus/assistant/Assistant.java @@ -0,0 +1,55 @@ +package com.genexus.assistant; + +import com.genexus.*; + +public final class Assistant extends GXProcedure +{ + public Assistant( int remoteHandle ) + { + super( remoteHandle , new ModelContext( Assistant.class ), "" ); + } + + public void execute( String aP0 , + String aP1 , + String[] aP2 ) + { + execute_int(aP0, aP1, aP2); + } + + private void execute_int( String aP0 , + String aP1 , + String[] aP2 ) + { + AV3Parameter1 = aP0; + AV4Parameter2 = aP1; + this.aP2 = aP2; + privateExecute(); + } + + protected void privateExecute( ) + { + Gxproperties = new com.genexus.util.GXProperties(); + Gxproperties.set("&Parameter1", AV3Parameter1); + Gxproperties.set("&Parameter2", AV4Parameter2); + Gxproperties.set("$context", "Los Angeles"); + AV5OutputVariable = callAssistant( "The weatherman", Gxproperties, null) ; + cleanup(); + } + + protected void cleanup( ) + { + this.aP2[0] = AV5OutputVariable; + } + + public void initialize( ) + { + } + + String AV3Parameter1 ; + String AV4Parameter2 ; + String AV5OutputVariable ; + com.genexus.util.GXProperties Gxproperties ; + String[] aP2 ; +} + + diff --git a/java/src/test/java/com/genexus/assistant/TestAssistant.java b/java/src/test/java/com/genexus/assistant/TestAssistant.java new file mode 100644 index 000000000..9f55942c0 --- /dev/null +++ b/java/src/test/java/com/genexus/assistant/TestAssistant.java @@ -0,0 +1,25 @@ +package com.genexus.assistant; + +import com.genexus.Application; +import com.genexus.sampleapp.GXcfg; +import com.genexus.specific.java.Connect; +import com.genexus.specific.java.LogManager; +import org.junit.Before; +import org.junit.Test; + +public class TestAssistant { + + @Before + public void setUpStreams() { + Connect.init(); + LogManager.initialize("."); + Application.init(GXcfg.class); + } + + @Test + public void testAPICallAssistant() { + String[] GXv_char4 = new String[1] ; + new com.genexus.assistant.Assistant(-1).execute( "Today", "Tomorrow", GXv_char4) ; + System.out.println(GXv_char4[0]); + } +} diff --git a/java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java b/java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java index e599cf8b0..d9a37fe94 100644 --- a/java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java +++ b/java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java @@ -1,6 +1,7 @@ package com.genexus.embedding; import com.genexus.Application; +import com.genexus.db.GXEmbedding; import com.genexus.sampleapp.GXcfg; import com.genexus.specific.java.Connect; import com.genexus.specific.java.LogManager; @@ -16,8 +17,7 @@ public void EmbeddingTest(){ Connect.init(); LogManager.initialize("."); Application.init(GXcfg.class); - EmbeddingService service = EmbeddingService.getInstance(); - List embedding = service.getEmbedding("openai/text-embedding-3-small", 512, "Hello World"); + List embedding = GXEmbedding.getEmbedding("openai/text-embedding-3-small", 512, "Hello World"); String result = embedding.stream() .map(String::valueOf) .collect(Collectors.joining(",")); diff --git a/pom.xml b/pom.xml index 9a2b020d1..d14528e41 100644 --- a/pom.xml +++ b/pom.xml @@ -86,7 +86,6 @@ wrapperjakarta wrappercommon java - gxembedding gxcryptocommon gxdynamiccall gxmail From 7ce5e6a2cf0e034957b883e7fcac692fe05d8e4d Mon Sep 17 00:00:00 2001 From: iroqueta Date: Tue, 5 Nov 2024 15:32:45 -0300 Subject: [PATCH 03/26] GXEmbedding module do not exists any more --- java/pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 489c8f2db..50b250f30 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -34,11 +34,6 @@ gxmail ${project.version} - - ${project.groupId} - gxembedding - ${project.version} - org.apache.commons commons-collections4 From c9d3a4a533336d283b27ef38a794843fd962967b Mon Sep 17 00:00:00 2001 From: iroqueta Date: Wed, 13 Nov 2024 19:03:19 -0300 Subject: [PATCH 04/26] Implement CallResult External Object to use in Agent Object --- .../java/com/genexus/util/CallResult.java | 30 +++++++++++++++++++ .../main/java/com/genexus/GXProcedure.java | 4 +-- .../main/java/com/genexus/db/GXEmbedding.java | 3 +- .../com/genexus/util/saia/SaiaService.java | 26 ++++++++++++---- .../java/com/genexus/assistant/Assistant.java | 3 +- 5 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 common/src/main/java/com/genexus/util/CallResult.java diff --git a/common/src/main/java/com/genexus/util/CallResult.java b/common/src/main/java/com/genexus/util/CallResult.java new file mode 100644 index 000000000..c7e4f98ab --- /dev/null +++ b/common/src/main/java/com/genexus/util/CallResult.java @@ -0,0 +1,30 @@ +package com.genexus.util; + +import com.genexus.GXBaseCollection; +import com.genexus.SdtMessages_Message; + +public class CallResult { + private boolean success = true; + private boolean fail; + private final GXBaseCollection messages = new GXBaseCollection<>(); + + public boolean success() { + return success; + } + + public void setFail() { + fail = true; + success =false; + } + + public boolean fail() { + return fail; + } + + public void addMessage(SdtMessages_Message message) { + messages.add(message); + } + public GXBaseCollection getmessages() { + return messages; + } +} diff --git a/java/src/main/java/com/genexus/GXProcedure.java b/java/src/main/java/com/genexus/GXProcedure.java index 5a8ffe760..34c37cc1f 100644 --- a/java/src/main/java/com/genexus/GXProcedure.java +++ b/java/src/main/java/com/genexus/GXProcedure.java @@ -260,11 +260,11 @@ protected void mockExecute() { privateExecute( ); } - protected String callAssistant(String assistant, GXProperties properties, Object response) { + protected String callAssistant(String assistant, GXProperties properties, CallResult result) { OpenAIRequest aiRequest = new OpenAIRequest(); aiRequest.setModel(String.format("saia:agent:%s", assistant)); aiRequest.setVariables(properties.getList()); - OpenAIResponse aiResponse = SaiaService.call(aiRequest); + OpenAIResponse aiResponse = SaiaService.call(aiRequest, result); if (aiResponse != null) return aiResponse.getChoices().get(0).getMessage().getContent(); return ""; diff --git a/java/src/main/java/com/genexus/db/GXEmbedding.java b/java/src/main/java/com/genexus/db/GXEmbedding.java index e2e3d4f86..35581c10e 100644 --- a/java/src/main/java/com/genexus/db/GXEmbedding.java +++ b/java/src/main/java/com/genexus/db/GXEmbedding.java @@ -3,6 +3,7 @@ import com.genexus.CommonUtil; import com.genexus.GXBaseCollection; import com.genexus.SdtMessages_Message; +import com.genexus.util.CallResult; import com.genexus.util.saia.OpenAIRequest; import com.genexus.util.saia.OpenAIResponse; import com.genexus.util.saia.SaiaService; @@ -75,7 +76,7 @@ public static List getEmbedding(String model, int dimensions, List Date: Thu, 14 Nov 2024 14:24:06 -0300 Subject: [PATCH 05/26] Change Saia URL to consider Embedding and Chat --- java/src/main/java/com/genexus/util/saia/SaiaService.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/java/src/main/java/com/genexus/util/saia/SaiaService.java b/java/src/main/java/com/genexus/util/saia/SaiaService.java index 07d67f0cd..11a1af976 100644 --- a/java/src/main/java/com/genexus/util/saia/SaiaService.java +++ b/java/src/main/java/com/genexus/util/saia/SaiaService.java @@ -21,15 +21,19 @@ public static OpenAIResponse call(OpenAIRequest request, CallResult result) { public static OpenAIResponse call(OpenAIRequest request, boolean isEmbedding, CallResult result) { try { String jsonRequest = new ObjectMapper().writeValueAsString(request); + String providerURL = aiProvider + "/chat";; HttpClient client = new HttpClient(); client.setSecure(1); client.addHeader("Content-Type", "application/json"); client.addHeader("Authorization", "Bearer " + apiKey); - if (isEmbedding) + if (isEmbedding) { client.addHeader("X-Saia-Source", "Embedding"); + providerURL = providerURL + "/embedding"; + } + client.addString(jsonRequest); - client.execute("POST", aiProvider); + client.execute("POST", providerURL); if (client.getStatusCode() == 200) { JSONObject jsonResponse = new JSONObject(client.getString()); return new ObjectMapper().readValue(jsonResponse.toString(), OpenAIResponse.class); From e4e1da516f302c92a2bc2145864de046875d1eda Mon Sep 17 00:00:00 2001 From: iroqueta Date: Fri, 15 Nov 2024 15:47:15 -0300 Subject: [PATCH 06/26] Add support for chat and tool call messages --- .../main/java/com/genexus/GXProcedure.java | 16 +++++- .../com/genexus/util/saia/OpenAIRequest.java | 6 +++ .../com/genexus/util/saia/OpenAIResponse.java | 54 +++++++++++-------- .../java/com/genexus/assistant/Assistant.java | 30 +++++++++-- .../com/genexus/assistant/TestAssistant.java | 4 ++ 5 files changed, 83 insertions(+), 27 deletions(-) diff --git a/java/src/main/java/com/genexus/GXProcedure.java b/java/src/main/java/com/genexus/GXProcedure.java index 34c37cc1f..0c85b2978 100644 --- a/java/src/main/java/com/genexus/GXProcedure.java +++ b/java/src/main/java/com/genexus/GXProcedure.java @@ -3,6 +3,8 @@ import java.sql.SQLException; import java.util.Date; +import java.util.List; + import com.genexus.db.Namespace; import com.genexus.db.UserInformation; import com.genexus.diagnostics.GXDebugInfo; @@ -261,12 +263,22 @@ protected void mockExecute() { } protected String callAssistant(String assistant, GXProperties properties, CallResult result) { + return callAssistant(assistant, properties, null, result); + } + + protected String callAssistant(String assistant, GXProperties properties, List messages, CallResult result) { OpenAIRequest aiRequest = new OpenAIRequest(); aiRequest.setModel(String.format("saia:agent:%s", assistant)); + if (messages != null) + aiRequest.setMessages(messages); aiRequest.setVariables(properties.getList()); OpenAIResponse aiResponse = SaiaService.call(aiRequest, result); - if (aiResponse != null) - return aiResponse.getChoices().get(0).getMessage().getContent(); + if (aiResponse != null) { + for (OpenAIResponse.Choice element : aiResponse.getChoices()) { + if (element.getFinishReason().equals("stop")) + return element.getMessage().getContent(); + } + } return ""; } } diff --git a/java/src/main/java/com/genexus/util/saia/OpenAIRequest.java b/java/src/main/java/com/genexus/util/saia/OpenAIRequest.java index d51da8567..9335811ae 100644 --- a/java/src/main/java/com/genexus/util/saia/OpenAIRequest.java +++ b/java/src/main/java/com/genexus/util/saia/OpenAIRequest.java @@ -14,6 +14,9 @@ public class OpenAIRequest { @JsonProperty("model") private String model; + @JsonProperty("messages") + private List messages; + @JsonProperty("prompt") private String prompt; @@ -50,6 +53,9 @@ public class OpenAIRequest { public String getModel() { return model; } public void setModel(String model) { this.model = model; } + public List getMessages() { return messages; } + public void setMessages(List messages) { this.messages = messages; } + public String getPrompt() { return prompt; } public void setPrompt(String prompt) { this.prompt = prompt; } diff --git a/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java index d10c028b7..2d5c16291 100644 --- a/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java +++ b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java @@ -22,6 +22,9 @@ public class OpenAIResponse { @JsonProperty("usage") private Usage usage; + @JsonProperty("tool_calls") + private List tool_calls; + @JsonProperty("data") private List data; @@ -40,6 +43,9 @@ public class OpenAIResponse { public Usage getUsage() { return usage; } public void setUsage(Usage usage) { this.usage = usage; } + public List getToolCalls() { return tool_calls; } + public void setToolCalls(List tool_calls) { this.tool_calls = tool_calls; } + public List getData() { return data; } public void setData(List data) { this.data = data; } @@ -77,6 +83,9 @@ public static class Message { @JsonProperty("tool_calls") private List toolCalls; + @JsonProperty("tool_call_id") + private String toolCallId; + public String getRole() { return role; } public void setRole(String role) { this.role = role; } @@ -85,44 +94,47 @@ public static class Message { public List getToolCalls() { return toolCalls; } public void setToolCalls(List toolCalls) { this.toolCalls = toolCalls; } + + public String getToolCallId() { return toolCallId; } + public void setToolCallId(String toolCallId) { this.toolCallId = toolCallId; } } @JsonIgnoreProperties(ignoreUnknown = true) public static class ToolCall { - @JsonProperty("tool_name") - private String toolName; + @JsonProperty("id") + private String id; - @JsonProperty("tool_input") - private ToolInput toolInput; + @JsonProperty("type") + private String type; - @JsonProperty("tool_output") - private String toolOutput; + @JsonProperty("function") + private Function function; - public String getToolName() { return toolName; } - public void setToolName(String toolName) { this.toolName = toolName; } + public String getId() { return id; } + public void setId(String id) { this.id = id; } - public ToolInput getToolInput() { return toolInput; } - public void setToolInput(ToolInput toolInput) { this.toolInput = toolInput; } + public String getType() { return type; } + public void setType(String type) { this.type = type; } - public String getToolOutput() { return toolOutput; } - public void setToolOutput(String toolOutput) { this.toolOutput = toolOutput; } + public Function getFunction() { return function; } + public void setFunction(Function function) { this.function = function; } } @JsonIgnoreProperties(ignoreUnknown = true) - public static class ToolInput { + public static class Function { - @JsonProperty("prompt") - private String prompt; + @JsonProperty("name") + private String name; - @JsonProperty("size") - private String size; + @JsonProperty("arguments") + private String arguments; - public String getPrompt() { return prompt; } - public void setPrompt(String prompt) { this.prompt = prompt; } + public String getName() { return name; } + public void setName(String name) { this.name = name; } - public String getSize() { return size; } - public void setSize(String size) { this.size = size; } + public String getArguments() { return arguments; } + public void setArguments(String arguments) { this.arguments = arguments; } } @JsonIgnoreProperties(ignoreUnknown = true) diff --git a/java/src/test/java/com/genexus/assistant/Assistant.java b/java/src/test/java/com/genexus/assistant/Assistant.java index ad99c83fe..229dcaac0 100644 --- a/java/src/test/java/com/genexus/assistant/Assistant.java +++ b/java/src/test/java/com/genexus/assistant/Assistant.java @@ -2,6 +2,10 @@ import com.genexus.*; import com.genexus.util.CallResult; +import com.genexus.util.saia.OpenAIResponse; + +import java.util.ArrayList; +import java.util.List; public final class Assistant extends GXProcedure { @@ -30,10 +34,28 @@ private void execute_int( String aP0 , protected void privateExecute( ) { Gxproperties = new com.genexus.util.GXProperties(); - Gxproperties.set("&Parameter1", AV3Parameter1); - Gxproperties.set("&Parameter2", AV4Parameter2); - Gxproperties.set("$context", "Los Angeles"); - AV5OutputVariable = callAssistant( "The weatherman", Gxproperties, new CallResult()) ; + List messages = null; + if (AV3Parameter1.equals("chat")) { + messages = new ArrayList<>(); + OpenAIResponse.Message message = new OpenAIResponse.Message(); + message.setRole("user"); + message.setContent("Dime el clima en Lima - Peru"); + messages.add(message); + message = new OpenAIResponse.Message(); + message.setRole("assistant"); + message.setContent("El clima actual en Lima, Perú, es soleado con una temperatura de 20.9°C (69.6°F). La dirección del viento es del suroeste (SSW) a 15.1 km/h (9.4 mph), y la humedad relativa es del 68%. La presión atmosférica es de 1013 mb. La visibilidad es de 10 km y el índice UV es de 12.5."); + messages.add(message); + message = new OpenAIResponse.Message(); + message.setRole("user"); + message.setContent("Que me puedes contar de la ciudad que te pedi el clima previamente?"); + messages.add(message); + } + else { + Gxproperties.set("&Parameter1", AV3Parameter1); + Gxproperties.set("&Parameter2", AV4Parameter2); + Gxproperties.set("$context", "Los Angeles"); + } + AV5OutputVariable = callAssistant( "The weatherman", Gxproperties, messages, new CallResult()) ; cleanup(); } diff --git a/java/src/test/java/com/genexus/assistant/TestAssistant.java b/java/src/test/java/com/genexus/assistant/TestAssistant.java index 9f55942c0..ed1419f97 100644 --- a/java/src/test/java/com/genexus/assistant/TestAssistant.java +++ b/java/src/test/java/com/genexus/assistant/TestAssistant.java @@ -21,5 +21,9 @@ public void testAPICallAssistant() { String[] GXv_char4 = new String[1] ; new com.genexus.assistant.Assistant(-1).execute( "Today", "Tomorrow", GXv_char4) ; System.out.println(GXv_char4[0]); + + String[] GXv_char5 = new String[1] ; + new com.genexus.assistant.Assistant(-1).execute( "chat", "", GXv_char5) ; + System.out.println(GXv_char5[0]); } } From 50847330f4d299571b90641281dda4df1f0f79c2 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Tue, 19 Nov 2024 11:50:31 -0300 Subject: [PATCH 07/26] Message was not added to CallResult --- java/src/main/java/com/genexus/util/saia/SaiaService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/src/main/java/com/genexus/util/saia/SaiaService.java b/java/src/main/java/com/genexus/util/saia/SaiaService.java index 11a1af976..1ead76829 100644 --- a/java/src/main/java/com/genexus/util/saia/SaiaService.java +++ b/java/src/main/java/com/genexus/util/saia/SaiaService.java @@ -61,6 +61,6 @@ private static void addResultMessage(String id, byte type, String description, C msg.setgxTv_SdtMessages_Message_Id(id); msg.setgxTv_SdtMessages_Message_Type(type); msg.setgxTv_SdtMessages_Message_Description(description); - + result.addMessage(msg); } } From 0afbf8eb89b1840aa74a308ce2c0911f5d34acae Mon Sep 17 00:00:00 2001 From: iroqueta Date: Tue, 19 Nov 2024 11:55:08 -0300 Subject: [PATCH 08/26] Json API was updated in master --- java/src/main/java/com/genexus/util/saia/SaiaService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/src/main/java/com/genexus/util/saia/SaiaService.java b/java/src/main/java/com/genexus/util/saia/SaiaService.java index 1ead76829..9801c5017 100644 --- a/java/src/main/java/com/genexus/util/saia/SaiaService.java +++ b/java/src/main/java/com/genexus/util/saia/SaiaService.java @@ -7,7 +7,7 @@ import com.genexus.diagnostics.core.LogManager; import com.genexus.internet.HttpClient; import com.genexus.util.CallResult; -import json.org.json.JSONObject; +import org.json.JSONObject; public class SaiaService { private static final ILogger logger = LogManager.getLogger(SaiaService.class); From a14331b8b20721f5ca4cce7a1ff63e239b9de122 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Wed, 20 Nov 2024 14:28:51 -0300 Subject: [PATCH 09/26] Change getmessages method in CallResult to getMessages --- common/src/main/java/com/genexus/util/CallResult.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/com/genexus/util/CallResult.java b/common/src/main/java/com/genexus/util/CallResult.java index c7e4f98ab..95894760c 100644 --- a/common/src/main/java/com/genexus/util/CallResult.java +++ b/common/src/main/java/com/genexus/util/CallResult.java @@ -24,7 +24,7 @@ public boolean fail() { public void addMessage(SdtMessages_Message message) { messages.add(message); } - public GXBaseCollection getmessages() { + public GXBaseCollection getMessages() { return messages; } } From e1e519044673891f7d6eceb4f0327560dffc4410 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Fri, 22 Nov 2024 12:49:36 -0300 Subject: [PATCH 10/26] Change callAssistant to callAgent Add getExternalInstance method in EO Collections. Support ChatMessage EO --- .../com/genexus/GXExternalCollection.java | 18 ++++- .../java/com/genexus/util/GXProperties.java | 4 +- .../main/java/com/genexus/GXProcedure.java | 11 +-- .../main/java/com/genexus/db/GXEmbedding.java | 4 +- .../com/genexus/util/saia/OpenAIRequest.java | 26 +++---- .../com/genexus/util/saia/OpenAIResponse.java | 33 ++++---- .../test/java/com/genexus/agent/Agent.java | 76 +++++++++++++++++++ .../java/com/genexus/agent/TestAgent.java | 29 +++++++ 8 files changed, 162 insertions(+), 39 deletions(-) create mode 100644 java/src/test/java/com/genexus/agent/Agent.java create mode 100644 java/src/test/java/com/genexus/agent/TestAgent.java diff --git a/common/src/main/java/com/genexus/GXExternalCollection.java b/common/src/main/java/com/genexus/GXExternalCollection.java index de7795165..baf60e363 100644 --- a/common/src/main/java/com/genexus/GXExternalCollection.java +++ b/common/src/main/java/com/genexus/GXExternalCollection.java @@ -98,6 +98,22 @@ public Vector getStruct() } return struct; } - + + public ArrayList getExternalInstance() { + ArrayList list = new ArrayList(); + for (T Item : this) + { + try + { + list.add(Item.getClass().getMethod("getExternalInstance", new Class[]{}).invoke(Item)); + } + catch (Exception e) + { + e.printStackTrace(); + } + } + return list; + } + } diff --git a/common/src/main/java/com/genexus/util/GXProperties.java b/common/src/main/java/com/genexus/util/GXProperties.java index 8c0de3a23..b5cc8d3c5 100644 --- a/common/src/main/java/com/genexus/util/GXProperties.java +++ b/common/src/main/java/com/genexus/util/GXProperties.java @@ -116,8 +116,8 @@ public String toJSonString() { return jObj.toString(); } - public List getList() { - List list = new ArrayList<>(); + public ArrayList getList() { + ArrayList list = new ArrayList<>(); int i = 0; while (count() > i) { list.add(item(i)); diff --git a/java/src/main/java/com/genexus/GXProcedure.java b/java/src/main/java/com/genexus/GXProcedure.java index 0c85b2978..1e421ea9e 100644 --- a/java/src/main/java/com/genexus/GXProcedure.java +++ b/java/src/main/java/com/genexus/GXProcedure.java @@ -2,6 +2,7 @@ package com.genexus; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -262,14 +263,14 @@ protected void mockExecute() { privateExecute( ); } - protected String callAssistant(String assistant, GXProperties properties, CallResult result) { - return callAssistant(assistant, properties, null, result); + protected String callAssistant(String agent, GXProperties properties, ArrayList messages, CallResult result) { + return callAgent(agent, properties, messages, result); } - protected String callAssistant(String assistant, GXProperties properties, List messages, CallResult result) { + protected String callAgent(String agent, GXProperties properties, ArrayList messages, CallResult result) { OpenAIRequest aiRequest = new OpenAIRequest(); - aiRequest.setModel(String.format("saia:agent:%s", assistant)); - if (messages != null) + aiRequest.setModel(String.format("saia:agent:%s", agent)); + if (!messages.isEmpty()) aiRequest.setMessages(messages); aiRequest.setVariables(properties.getList()); OpenAIResponse aiResponse = SaiaService.call(aiRequest, result); diff --git a/java/src/main/java/com/genexus/db/GXEmbedding.java b/java/src/main/java/com/genexus/db/GXEmbedding.java index 35581c10e..17bd06ce4 100644 --- a/java/src/main/java/com/genexus/db/GXEmbedding.java +++ b/java/src/main/java/com/genexus/db/GXEmbedding.java @@ -66,12 +66,12 @@ public static GXEmbedding generateEmbedding(GXEmbedding embeddingInfo, String te } public static List getEmbedding(String model, int dimensions, String input) { - List inputList = new ArrayList<>(); + ArrayList inputList = new ArrayList<>(); inputList.add(input); return getEmbedding(model, dimensions, inputList); } - public static List getEmbedding(String model, int dimensions, List inputList) { + public static List getEmbedding(String model, int dimensions, ArrayList inputList) { OpenAIRequest aiRequest = new OpenAIRequest(); aiRequest.setModel(model); aiRequest.setInput(inputList); diff --git a/java/src/main/java/com/genexus/util/saia/OpenAIRequest.java b/java/src/main/java/com/genexus/util/saia/OpenAIRequest.java index 9335811ae..59a7bc866 100644 --- a/java/src/main/java/com/genexus/util/saia/OpenAIRequest.java +++ b/java/src/main/java/com/genexus/util/saia/OpenAIRequest.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.genexus.util.GXProperty; -import java.util.List; +import java.util.ArrayList; @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) @@ -15,13 +15,13 @@ public class OpenAIRequest { private String model; @JsonProperty("messages") - private List messages; + private ArrayList messages; @JsonProperty("prompt") private String prompt; @JsonProperty("input") - private List input; + private ArrayList input; @JsonProperty("max_tokens") private Integer maxTokens; @@ -33,7 +33,7 @@ public class OpenAIRequest { private Boolean stream; @JsonProperty("stop") - private List stop; + private ArrayList stop; @JsonProperty("presence_penalty") private Double presencePenalty; @@ -45,7 +45,7 @@ public class OpenAIRequest { private String user; @JsonProperty("variables") - private List variables; + private ArrayList variables; @JsonProperty("dimensions") private int dimension; @@ -53,14 +53,14 @@ public class OpenAIRequest { public String getModel() { return model; } public void setModel(String model) { this.model = model; } - public List getMessages() { return messages; } - public void setMessages(List messages) { this.messages = messages; } + public ArrayList getMessages() { return messages; } + public void setMessages(ArrayList messages) { this.messages = messages; } public String getPrompt() { return prompt; } public void setPrompt(String prompt) { this.prompt = prompt; } - public List getInput() { return input; } - public void setInput(List input) { this.input = input; } + public ArrayList getInput() { return input; } + public void setInput(ArrayList input) { this.input = input; } public Integer getMaxTokens() { return maxTokens; } public void setMaxTokens(Integer maxTokens) { this.maxTokens = maxTokens; } @@ -71,8 +71,8 @@ public class OpenAIRequest { public Boolean getStream() { return stream; } public void setStream(Boolean stream) { this.stream = stream; } - public List getStop() { return stop; } - public void setStop(List stop) { this.stop = stop; } + public ArrayList getStop() { return stop; } + public void setStop(ArrayList stop) { this.stop = stop; } public Double getPresencePenalty() { return presencePenalty; } public void setPresencePenalty(Double presencePenalty) { this.presencePenalty = presencePenalty; } @@ -83,8 +83,8 @@ public class OpenAIRequest { public String getUser() { return user; } public void setUser(String user) { this.user = user; } - public List getVariables() { return variables; } - public void setVariables(List variables) { this.variables = variables; } + public ArrayList getVariables() { return variables; } + public void setVariables(ArrayList variables) { this.variables = variables; } public int getDimension() { return dimension; } public void setDimension(int dimension) { this.dimension = dimension; } diff --git a/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java index 2d5c16291..cf30e124a 100644 --- a/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java +++ b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java @@ -2,7 +2,8 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; + +import java.util.ArrayList; @JsonIgnoreProperties(ignoreUnknown = true) public class OpenAIResponse { @@ -17,16 +18,16 @@ public class OpenAIResponse { private long created; @JsonProperty("choices") - private List choices; + private ArrayList choices; @JsonProperty("usage") private Usage usage; @JsonProperty("tool_calls") - private List tool_calls; + private ArrayList tool_calls; @JsonProperty("data") - private List data; + private ArrayList data; public String getId() { return id; } public void setId(String id) { this.id = id; } @@ -37,17 +38,17 @@ public class OpenAIResponse { public long getCreated() { return created; } public void setCreated(long created) { this.created = created; } - public List getChoices() { return choices; } - public void setChoices(List choices) { this.choices = choices; } + public ArrayList getChoices() { return choices; } + public void setChoices(ArrayList choices) { this.choices = choices; } public Usage getUsage() { return usage; } public void setUsage(Usage usage) { this.usage = usage; } - public List getToolCalls() { return tool_calls; } - public void setToolCalls(List tool_calls) { this.tool_calls = tool_calls; } + public ArrayList getToolCalls() { return tool_calls; } + public void setToolCalls(ArrayList tool_calls) { this.tool_calls = tool_calls; } - public List getData() { return data; } - public void setData(List data) { this.data = data; } + public ArrayList getData() { return data; } + public void setData(ArrayList data) { this.data = data; } @JsonIgnoreProperties(ignoreUnknown = true) public static class Choice { @@ -81,7 +82,7 @@ public static class Message { private String content; @JsonProperty("tool_calls") - private List toolCalls; + private ArrayList toolCalls; @JsonProperty("tool_call_id") private String toolCallId; @@ -92,8 +93,8 @@ public static class Message { public String getContent() { return content; } public void setContent(String content) { this.content = content; } - public List getToolCalls() { return toolCalls; } - public void setToolCalls(List toolCalls) { this.toolCalls = toolCalls; } + public ArrayList getToolCalls() { return toolCalls; } + public void setToolCalls(ArrayList toolCalls) { this.toolCalls = toolCalls; } public String getToolCallId() { return toolCallId; } public void setToolCallId(String toolCallId) { this.toolCallId = toolCallId; } @@ -169,7 +170,7 @@ public static class DataItem { private String object; @JsonProperty("embedding") - private List embedding; + private ArrayList embedding; public String getId() { return id; } public void setId(String id) { this.id = id; } @@ -177,7 +178,7 @@ public static class DataItem { public String getObject() { return object; } public void setObject(String object) { this.object = object; } - public List getEmbedding() { return embedding; } - public void setEmbedding(List embedding) { this.embedding = embedding; } + public ArrayList getEmbedding() { return embedding; } + public void setEmbedding(ArrayList embedding) { this.embedding = embedding; } } } diff --git a/java/src/test/java/com/genexus/agent/Agent.java b/java/src/test/java/com/genexus/agent/Agent.java new file mode 100644 index 000000000..2a98df0c2 --- /dev/null +++ b/java/src/test/java/com/genexus/agent/Agent.java @@ -0,0 +1,76 @@ +package com.genexus.agent; + +import com.genexus.*; +import com.genexus.util.CallResult; +import com.genexus.util.saia.OpenAIResponse; + +import java.util.ArrayList; + +public final class Agent extends GXProcedure +{ + public Agent(int remoteHandle ) + { + super( remoteHandle , new ModelContext( Agent.class ), "" ); + } + + public void execute( String aP0 , + String aP1 , + String[] aP2 ) + { + execute_int(aP0, aP1, aP2); + } + + private void execute_int( String aP0 , + String aP1 , + String[] aP2 ) + { + AV3Parameter1 = aP0; + AV4Parameter2 = aP1; + this.aP2 = aP2; + privateExecute(); + } + + protected void privateExecute( ) + { + Gxproperties = new com.genexus.util.GXProperties(); + ArrayList messages = new ArrayList();; + if (AV3Parameter1.equals("chat")) { + OpenAIResponse.Message message = new OpenAIResponse.Message(); + message.setRole("user"); + message.setContent("Dime el clima en Lima - Peru"); + messages.add(message); + message = new OpenAIResponse.Message(); + message.setRole("assistant"); + message.setContent("El clima actual en Lima, Perú, es soleado con una temperatura de 20.9°C (69.6°F). La dirección del viento es del suroeste (SSW) a 15.1 km/h (9.4 mph), y la humedad relativa es del 68%. La presión atmosférica es de 1013 mb. La visibilidad es de 10 km y el índice UV es de 12.5."); + messages.add(message); + message = new OpenAIResponse.Message(); + message.setRole("user"); + message.setContent("Que me puedes contar de la ciudad que te pedi el clima previamente?"); + messages.add(message); + } + else { + Gxproperties.set("&Parameter1", AV3Parameter1); + Gxproperties.set("&Parameter2", AV4Parameter2); + Gxproperties.set("$context", "Los Angeles"); + } + AV5OutputVariable = callAgent( "The weatherman", Gxproperties, messages, new CallResult()) ; + cleanup(); + } + + protected void cleanup( ) + { + this.aP2[0] = AV5OutputVariable; + } + + public void initialize( ) + { + } + + String AV3Parameter1 ; + String AV4Parameter2 ; + String AV5OutputVariable ; + com.genexus.util.GXProperties Gxproperties ; + String[] aP2 ; +} + + diff --git a/java/src/test/java/com/genexus/agent/TestAgent.java b/java/src/test/java/com/genexus/agent/TestAgent.java new file mode 100644 index 000000000..ffd149e6a --- /dev/null +++ b/java/src/test/java/com/genexus/agent/TestAgent.java @@ -0,0 +1,29 @@ +package com.genexus.agent; + +import com.genexus.Application; +import com.genexus.sampleapp.GXcfg; +import com.genexus.specific.java.Connect; +import com.genexus.specific.java.LogManager; +import org.junit.Before; +import org.junit.Test; + +public class TestAgent { + + @Before + public void setUpStreams() { + Connect.init(); + LogManager.initialize("."); + Application.init(GXcfg.class); + } + + @Test + public void testAPICallAgent() { + String[] GXv_char4 = new String[1] ; + new Agent(-1).execute( "Today", "Tomorrow", GXv_char4) ; + System.out.println(GXv_char4[0]); + + String[] GXv_char5 = new String[1] ; + new Agent(-1).execute( "chat", "", GXv_char5) ; + System.out.println(GXv_char5[0]); + } +} From 88f77fadda294dbc708b514c5ab5478db7b8e733 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Fri, 22 Nov 2024 12:59:37 -0300 Subject: [PATCH 11/26] Fix Build error in Github --- .../java/com/genexus/agent/Assistant.java | 77 +++++++++++++++++++ .../com/genexus/assistant/TestAssistant.java | 29 ------- 2 files changed, 77 insertions(+), 29 deletions(-) create mode 100644 java/src/test/java/com/genexus/agent/Assistant.java delete mode 100644 java/src/test/java/com/genexus/assistant/TestAssistant.java diff --git a/java/src/test/java/com/genexus/agent/Assistant.java b/java/src/test/java/com/genexus/agent/Assistant.java new file mode 100644 index 000000000..63b3de087 --- /dev/null +++ b/java/src/test/java/com/genexus/agent/Assistant.java @@ -0,0 +1,77 @@ +package com.genexus.agent; + +import com.genexus.GXProcedure; +import com.genexus.ModelContext; +import com.genexus.util.CallResult; +import com.genexus.util.saia.OpenAIResponse; + +import java.util.ArrayList; + +public final class Assistant extends GXProcedure +{ + public Assistant(int remoteHandle ) + { + super( remoteHandle , new ModelContext( Assistant.class ), "" ); + } + + public void execute( String aP0 , + String aP1 , + String[] aP2 ) + { + execute_int(aP0, aP1, aP2); + } + + private void execute_int( String aP0 , + String aP1 , + String[] aP2 ) + { + AV3Parameter1 = aP0; + AV4Parameter2 = aP1; + this.aP2 = aP2; + privateExecute(); + } + + protected void privateExecute( ) + { + Gxproperties = new com.genexus.util.GXProperties(); + ArrayList messages = new ArrayList();; + if (AV3Parameter1.equals("chat")) { + OpenAIResponse.Message message = new OpenAIResponse.Message(); + message.setRole("user"); + message.setContent("Dime el clima en Lima - Peru"); + messages.add(message); + message = new OpenAIResponse.Message(); + message.setRole("assistant"); + message.setContent("El clima actual en Lima, Perú, es soleado con una temperatura de 20.9°C (69.6°F). La dirección del viento es del suroeste (SSW) a 15.1 km/h (9.4 mph), y la humedad relativa es del 68%. La presión atmosférica es de 1013 mb. La visibilidad es de 10 km y el índice UV es de 12.5."); + messages.add(message); + message = new OpenAIResponse.Message(); + message.setRole("user"); + message.setContent("Que me puedes contar de la ciudad que te pedi el clima previamente?"); + messages.add(message); + } + else { + Gxproperties.set("&Parameter1", AV3Parameter1); + Gxproperties.set("&Parameter2", AV4Parameter2); + Gxproperties.set("$context", "Los Angeles"); + } + AV5OutputVariable = callAgent( "The weatherman", Gxproperties, messages, new CallResult()) ; + cleanup(); + } + + protected void cleanup( ) + { + this.aP2[0] = AV5OutputVariable; + } + + public void initialize( ) + { + } + + String AV3Parameter1 ; + String AV4Parameter2 ; + String AV5OutputVariable ; + com.genexus.util.GXProperties Gxproperties ; + String[] aP2 ; +} + + diff --git a/java/src/test/java/com/genexus/assistant/TestAssistant.java b/java/src/test/java/com/genexus/assistant/TestAssistant.java deleted file mode 100644 index ed1419f97..000000000 --- a/java/src/test/java/com/genexus/assistant/TestAssistant.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.genexus.assistant; - -import com.genexus.Application; -import com.genexus.sampleapp.GXcfg; -import com.genexus.specific.java.Connect; -import com.genexus.specific.java.LogManager; -import org.junit.Before; -import org.junit.Test; - -public class TestAssistant { - - @Before - public void setUpStreams() { - Connect.init(); - LogManager.initialize("."); - Application.init(GXcfg.class); - } - - @Test - public void testAPICallAssistant() { - String[] GXv_char4 = new String[1] ; - new com.genexus.assistant.Assistant(-1).execute( "Today", "Tomorrow", GXv_char4) ; - System.out.println(GXv_char4[0]); - - String[] GXv_char5 = new String[1] ; - new com.genexus.assistant.Assistant(-1).execute( "chat", "", GXv_char5) ; - System.out.println(GXv_char5[0]); - } -} From fd8af2ee8ae633d94c2a9fafc4ae9f9bb57e0c40 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Fri, 22 Nov 2024 13:05:58 -0300 Subject: [PATCH 12/26] Fix Build error in Github --- .../java/com/genexus/agent/Assistant.java | 77 ------------------- .../java/com/genexus/assistant/Assistant.java | 11 ++- 2 files changed, 5 insertions(+), 83 deletions(-) delete mode 100644 java/src/test/java/com/genexus/agent/Assistant.java diff --git a/java/src/test/java/com/genexus/agent/Assistant.java b/java/src/test/java/com/genexus/agent/Assistant.java deleted file mode 100644 index 63b3de087..000000000 --- a/java/src/test/java/com/genexus/agent/Assistant.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.genexus.agent; - -import com.genexus.GXProcedure; -import com.genexus.ModelContext; -import com.genexus.util.CallResult; -import com.genexus.util.saia.OpenAIResponse; - -import java.util.ArrayList; - -public final class Assistant extends GXProcedure -{ - public Assistant(int remoteHandle ) - { - super( remoteHandle , new ModelContext( Assistant.class ), "" ); - } - - public void execute( String aP0 , - String aP1 , - String[] aP2 ) - { - execute_int(aP0, aP1, aP2); - } - - private void execute_int( String aP0 , - String aP1 , - String[] aP2 ) - { - AV3Parameter1 = aP0; - AV4Parameter2 = aP1; - this.aP2 = aP2; - privateExecute(); - } - - protected void privateExecute( ) - { - Gxproperties = new com.genexus.util.GXProperties(); - ArrayList messages = new ArrayList();; - if (AV3Parameter1.equals("chat")) { - OpenAIResponse.Message message = new OpenAIResponse.Message(); - message.setRole("user"); - message.setContent("Dime el clima en Lima - Peru"); - messages.add(message); - message = new OpenAIResponse.Message(); - message.setRole("assistant"); - message.setContent("El clima actual en Lima, Perú, es soleado con una temperatura de 20.9°C (69.6°F). La dirección del viento es del suroeste (SSW) a 15.1 km/h (9.4 mph), y la humedad relativa es del 68%. La presión atmosférica es de 1013 mb. La visibilidad es de 10 km y el índice UV es de 12.5."); - messages.add(message); - message = new OpenAIResponse.Message(); - message.setRole("user"); - message.setContent("Que me puedes contar de la ciudad que te pedi el clima previamente?"); - messages.add(message); - } - else { - Gxproperties.set("&Parameter1", AV3Parameter1); - Gxproperties.set("&Parameter2", AV4Parameter2); - Gxproperties.set("$context", "Los Angeles"); - } - AV5OutputVariable = callAgent( "The weatherman", Gxproperties, messages, new CallResult()) ; - cleanup(); - } - - protected void cleanup( ) - { - this.aP2[0] = AV5OutputVariable; - } - - public void initialize( ) - { - } - - String AV3Parameter1 ; - String AV4Parameter2 ; - String AV5OutputVariable ; - com.genexus.util.GXProperties Gxproperties ; - String[] aP2 ; -} - - diff --git a/java/src/test/java/com/genexus/assistant/Assistant.java b/java/src/test/java/com/genexus/assistant/Assistant.java index 229dcaac0..c979d1f52 100644 --- a/java/src/test/java/com/genexus/assistant/Assistant.java +++ b/java/src/test/java/com/genexus/assistant/Assistant.java @@ -1,15 +1,15 @@ package com.genexus.assistant; -import com.genexus.*; +import com.genexus.GXProcedure; +import com.genexus.ModelContext; import com.genexus.util.CallResult; import com.genexus.util.saia.OpenAIResponse; import java.util.ArrayList; -import java.util.List; public final class Assistant extends GXProcedure { - public Assistant( int remoteHandle ) + public Assistant(int remoteHandle ) { super( remoteHandle , new ModelContext( Assistant.class ), "" ); } @@ -34,9 +34,8 @@ private void execute_int( String aP0 , protected void privateExecute( ) { Gxproperties = new com.genexus.util.GXProperties(); - List messages = null; + ArrayList messages = new ArrayList();; if (AV3Parameter1.equals("chat")) { - messages = new ArrayList<>(); OpenAIResponse.Message message = new OpenAIResponse.Message(); message.setRole("user"); message.setContent("Dime el clima en Lima - Peru"); @@ -55,7 +54,7 @@ protected void privateExecute( ) Gxproperties.set("&Parameter2", AV4Parameter2); Gxproperties.set("$context", "Los Angeles"); } - AV5OutputVariable = callAssistant( "The weatherman", Gxproperties, messages, new CallResult()) ; + AV5OutputVariable = callAgent( "The weatherman", Gxproperties, messages, new CallResult()) ; cleanup(); } From 898abb198ea62471efacdf0ebe1138aee66a6404 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Fri, 22 Nov 2024 13:18:02 -0300 Subject: [PATCH 13/26] Remove Assistant Test --- .../java/com/genexus/assistant/Assistant.java | 77 ------------------- 1 file changed, 77 deletions(-) delete mode 100644 java/src/test/java/com/genexus/assistant/Assistant.java diff --git a/java/src/test/java/com/genexus/assistant/Assistant.java b/java/src/test/java/com/genexus/assistant/Assistant.java deleted file mode 100644 index c979d1f52..000000000 --- a/java/src/test/java/com/genexus/assistant/Assistant.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.genexus.assistant; - -import com.genexus.GXProcedure; -import com.genexus.ModelContext; -import com.genexus.util.CallResult; -import com.genexus.util.saia.OpenAIResponse; - -import java.util.ArrayList; - -public final class Assistant extends GXProcedure -{ - public Assistant(int remoteHandle ) - { - super( remoteHandle , new ModelContext( Assistant.class ), "" ); - } - - public void execute( String aP0 , - String aP1 , - String[] aP2 ) - { - execute_int(aP0, aP1, aP2); - } - - private void execute_int( String aP0 , - String aP1 , - String[] aP2 ) - { - AV3Parameter1 = aP0; - AV4Parameter2 = aP1; - this.aP2 = aP2; - privateExecute(); - } - - protected void privateExecute( ) - { - Gxproperties = new com.genexus.util.GXProperties(); - ArrayList messages = new ArrayList();; - if (AV3Parameter1.equals("chat")) { - OpenAIResponse.Message message = new OpenAIResponse.Message(); - message.setRole("user"); - message.setContent("Dime el clima en Lima - Peru"); - messages.add(message); - message = new OpenAIResponse.Message(); - message.setRole("assistant"); - message.setContent("El clima actual en Lima, Perú, es soleado con una temperatura de 20.9°C (69.6°F). La dirección del viento es del suroeste (SSW) a 15.1 km/h (9.4 mph), y la humedad relativa es del 68%. La presión atmosférica es de 1013 mb. La visibilidad es de 10 km y el índice UV es de 12.5."); - messages.add(message); - message = new OpenAIResponse.Message(); - message.setRole("user"); - message.setContent("Que me puedes contar de la ciudad que te pedi el clima previamente?"); - messages.add(message); - } - else { - Gxproperties.set("&Parameter1", AV3Parameter1); - Gxproperties.set("&Parameter2", AV4Parameter2); - Gxproperties.set("$context", "Los Angeles"); - } - AV5OutputVariable = callAgent( "The weatherman", Gxproperties, messages, new CallResult()) ; - cleanup(); - } - - protected void cleanup( ) - { - this.aP2[0] = AV5OutputVariable; - } - - public void initialize( ) - { - } - - String AV3Parameter1 ; - String AV4Parameter2 ; - String AV5OutputVariable ; - com.genexus.util.GXProperties Gxproperties ; - String[] aP2 ; -} - - From 6a429656263c7fe17be38b8a658cd06bbc3f1b83 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Tue, 26 Nov 2024 16:15:26 -0300 Subject: [PATCH 14/26] Add Tool Calls support --- .../main/java/com/genexus/GXProcedure.java | 31 +++++++++++++++++-- .../test/java/com/genexus/agent/Agent.java | 20 ++++++++++++ .../java/com/genexus/agent/TestAgent.java | 4 +++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/java/src/main/java/com/genexus/GXProcedure.java b/java/src/main/java/com/genexus/GXProcedure.java index 1e421ea9e..0cdb379b7 100644 --- a/java/src/main/java/com/genexus/GXProcedure.java +++ b/java/src/main/java/com/genexus/GXProcedure.java @@ -4,7 +4,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Date; -import java.util.List; import com.genexus.db.Namespace; import com.genexus.db.UserInformation; @@ -263,6 +262,10 @@ protected void mockExecute() { privateExecute( ); } + protected String callTool(String name, String arguments) throws Exception { + return ""; + } + protected String callAssistant(String agent, GXProperties properties, ArrayList messages, CallResult result) { return callAgent(agent, properties, messages, result); } @@ -276,10 +279,34 @@ protected String callAgent(String agent, GXProperties properties, ArrayList messages) { + String result; + String functionName = toolCall.getFunction().getName(); + try { + result = callTool(functionName, toolCall.getFunction().getArguments()); + } + catch (Exception e) { + result = String.format("Error calling tool %s", functionName); + } + OpenAIResponse.Message toolCallMessage = new OpenAIResponse.Message(); + toolCallMessage.setRole("tool"); + toolCallMessage.setContent(result); + toolCallMessage.setToolCallId(toolCall.getId()); + messages.add(toolCallMessage); + } } diff --git a/java/src/test/java/com/genexus/agent/Agent.java b/java/src/test/java/com/genexus/agent/Agent.java index 2a98df0c2..2c595d3f3 100644 --- a/java/src/test/java/com/genexus/agent/Agent.java +++ b/java/src/test/java/com/genexus/agent/Agent.java @@ -48,6 +48,15 @@ protected void privateExecute( ) message.setContent("Que me puedes contar de la ciudad que te pedi el clima previamente?"); messages.add(message); } + else if (AV3Parameter1.equals("toolcall")) { + OpenAIResponse.Message message = new OpenAIResponse.Message(); + message.setRole("user"); + message.setContent("Necesito nombre y descripcion del producto 1779"); + messages.add(message); + AV5OutputVariable = callAgent( "ProductInfo", Gxproperties, messages, new CallResult()) ; + cleanup(); + return; + } else { Gxproperties.set("&Parameter1", AV3Parameter1); Gxproperties.set("&Parameter2", AV4Parameter2); @@ -62,6 +71,17 @@ protected void cleanup( ) this.aP2[0] = AV5OutputVariable; } + protected String callTool(String name, String arguments) throws Exception { + switch (name) { + case "GetProductName": + return "Televisor Panavox 80 pulgadas"; + case "GetProductDescription": + return "Flor de Televisor el Panavox de 80 pulgadas"; + default: + return String.format("Unknown function %s", name); + } + } + public void initialize( ) { } diff --git a/java/src/test/java/com/genexus/agent/TestAgent.java b/java/src/test/java/com/genexus/agent/TestAgent.java index ffd149e6a..7e719ee9f 100644 --- a/java/src/test/java/com/genexus/agent/TestAgent.java +++ b/java/src/test/java/com/genexus/agent/TestAgent.java @@ -25,5 +25,9 @@ public void testAPICallAgent() { String[] GXv_char5 = new String[1] ; new Agent(-1).execute( "chat", "", GXv_char5) ; System.out.println(GXv_char5[0]); + + String[] GXv_char6 = new String[1] ; + new Agent(-1).execute( "toolcall", "", GXv_char6) ; + System.out.println(GXv_char6[0]); } } From 8d80ee527706790af55a7c838cec7e293c5f7bed Mon Sep 17 00:00:00 2001 From: iroqueta Date: Mon, 2 Dec 2024 11:55:04 -0300 Subject: [PATCH 15/26] TRN with Embedded fails when the PK is validated in Java issue 202652 --- java/src/main/java/com/genexus/db/GXEmbedding.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/src/main/java/com/genexus/db/GXEmbedding.java b/java/src/main/java/com/genexus/db/GXEmbedding.java index 17bd06ce4..7424a8061 100644 --- a/java/src/main/java/com/genexus/db/GXEmbedding.java +++ b/java/src/main/java/com/genexus/db/GXEmbedding.java @@ -20,6 +20,7 @@ public class GXEmbedding { private List embedding; public GXEmbedding() { + embedding = new ArrayList<>(); } public GXEmbedding(String model, int dimensions) { From a7fcc6de91a40ec0b8de1a6334c49eef9ec0e31e Mon Sep 17 00:00:00 2001 From: iroqueta Date: Tue, 3 Dec 2024 12:04:57 -0300 Subject: [PATCH 16/26] Embedding must be initialized with empty values. --- java/src/main/java/com/genexus/db/GXEmbedding.java | 8 ++++++-- .../test/java/com/genexus/embedding/GXEmbeddingTest.java | 8 ++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/java/src/main/java/com/genexus/db/GXEmbedding.java b/java/src/main/java/com/genexus/db/GXEmbedding.java index 7424a8061..d935dffac 100644 --- a/java/src/main/java/com/genexus/db/GXEmbedding.java +++ b/java/src/main/java/com/genexus/db/GXEmbedding.java @@ -10,6 +10,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -26,7 +27,7 @@ public GXEmbedding() { public GXEmbedding(String model, int dimensions) { this.model = model; this.dimensions = dimensions; - embedding = new ArrayList<>(dimensions); + embedding = new ArrayList<>(Collections.nCopies(dimensions, 0.0f)); } public GXEmbedding(Float[] embedding, String model, int dimensions) { @@ -48,7 +49,8 @@ public int getDimensions() { } public void setEmbedding(List embedding) { - this.embedding = embedding; + if (!embedding.isEmpty()) + this.embedding = embedding; } public Float[] getFloatArray() { @@ -67,6 +69,8 @@ public static GXEmbedding generateEmbedding(GXEmbedding embeddingInfo, String te } public static List getEmbedding(String model, int dimensions, String input) { + if (input.isEmpty()) + return new ArrayList<>(); ArrayList inputList = new ArrayList<>(); inputList.add(input); return getEmbedding(model, dimensions, inputList); diff --git a/java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java b/java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java index d9a37fe94..d4af124e3 100644 --- a/java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java +++ b/java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java @@ -1,6 +1,8 @@ package com.genexus.embedding; import com.genexus.Application; +import com.genexus.GXBaseCollection; +import com.genexus.SdtMessages_Message; import com.genexus.db.GXEmbedding; import com.genexus.sampleapp.GXcfg; import com.genexus.specific.java.Connect; @@ -23,5 +25,11 @@ public void EmbeddingTest(){ .collect(Collectors.joining(",")); System.out.println("Embedding: " + result); Assert.assertNotNull(embedding); + + GXBaseCollection AV8Messages = new GXBaseCollection<>(); + GXEmbedding A7ProductEmbedding = new GXEmbedding("openai/text-embedding-3-small",512) ; + GXEmbedding AV9ProductEmbedding = GXEmbedding.generateEmbedding(A7ProductEmbedding, "", AV8Messages) ; + result = AV9ProductEmbedding.toString(); + System.out.println("Empty Embedding: " + result); } } From d84e9ad4606d25ce95c8bfde0fdce24c1f801724 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Tue, 3 Dec 2024 14:58:40 -0300 Subject: [PATCH 17/26] Add ToolCall y Stream support to Agent Object Add logging calling saia. --- .../main/java/com/genexus/GXProcedure.java | 61 +++++++++++++++++-- .../com/genexus/util/saia/OpenAIResponse.java | 6 ++ .../com/genexus/util/saia/SaiaService.java | 27 ++++++-- .../test/java/com/genexus/agent/Agent.java | 47 ++++++++++++-- .../java/com/genexus/agent/TestAgent.java | 7 +++ 5 files changed, 132 insertions(+), 16 deletions(-) diff --git a/java/src/main/java/com/genexus/GXProcedure.java b/java/src/main/java/com/genexus/GXProcedure.java index 0cdb379b7..814dab171 100644 --- a/java/src/main/java/com/genexus/GXProcedure.java +++ b/java/src/main/java/com/genexus/GXProcedure.java @@ -5,10 +5,12 @@ import java.util.ArrayList; import java.util.Date; +import com.fasterxml.jackson.databind.ObjectMapper; import com.genexus.db.Namespace; import com.genexus.db.UserInformation; import com.genexus.diagnostics.GXDebugInfo; import com.genexus.diagnostics.GXDebugManager; +import com.genexus.internet.HttpClient; import com.genexus.internet.HttpContext; import com.genexus.mock.GXMockProvider; import com.genexus.performance.ProcedureInfo; @@ -17,6 +19,7 @@ import com.genexus.util.saia.OpenAIRequest; import com.genexus.util.saia.OpenAIResponse; import com.genexus.util.saia.SaiaService; +import org.json.JSONObject; public abstract class GXProcedure implements IErrorHandler, ISubmitteable { public abstract void initialize(); @@ -32,7 +35,8 @@ public abstract class GXProcedure implements IErrorHandler, ISubmitteable { UserInformation ui=null; private Date beginExecute; - + private HttpClient client; + public static final int IN_NEW_UTL = -2; public GXProcedure(int remoteHandle, ModelContext context, String location) { @@ -271,12 +275,19 @@ protected String callAssistant(String agent, GXProperties properties, ArrayList< } protected String callAgent(String agent, GXProperties properties, ArrayList messages, CallResult result) { + return callAgent(agent, false, properties, messages, result); + } + + protected String callAgent(String agent, boolean stream, GXProperties properties, ArrayList messages, CallResult result) { OpenAIRequest aiRequest = new OpenAIRequest(); aiRequest.setModel(String.format("saia:agent:%s", agent)); if (!messages.isEmpty()) aiRequest.setMessages(messages); aiRequest.setVariables(properties.getList()); - OpenAIResponse aiResponse = SaiaService.call(aiRequest, result); + if (stream) + aiRequest.setStream(true); + client = new HttpClient(); + OpenAIResponse aiResponse = SaiaService.call(aiRequest, client, result); if (aiResponse != null) { for (OpenAIResponse.Choice element : aiResponse.getChoices()) { String finishReason = element.getFinishReason(); @@ -284,16 +295,22 @@ protected String callAgent(String agent, GXProperties properties, ArrayList messages, CallResult result, ArrayList toolCalls) { + for (OpenAIResponse.ToolCall tollCall : toolCalls) { + processToolCall(tollCall, messages); + } + return callAgent(agent, stream, properties, messages, result); + } + private void processToolCall(OpenAIResponse.ToolCall toolCall, ArrayList messages) { String result; String functionName = toolCall.getFunction().getName(); @@ -309,4 +326,36 @@ private void processToolCall(OpenAIResponse.ToolCall toolCall, ArrayList messages, CallResult result) { + String data = client.readChunk(); + if (data.isEmpty()) + return ""; + int index = data.indexOf("data:") + "data:".length(); + String chunkJson = data.substring(index).trim(); + try { + JSONObject jsonResponse = new JSONObject(chunkJson); + OpenAIResponse chunkResponse = new ObjectMapper().readValue(jsonResponse.toString(), OpenAIResponse.class); + OpenAIResponse.Choice choise = chunkResponse.getChoices().get(0); + if (choise.getFinishReason() != null && choise.getFinishReason().equals("tool_calls") && agent != null) { + messages.add(choise.getMessage()); + return processNotChunkedResponse(agent, true, properties, messages, result, choise.getMessage().getToolCalls()); + } + String chunkString = choise.getDelta().getContent(); + if (chunkString == null) + return ""; + return chunkString; + } + catch (Exception e) { + return ""; + } + } + + protected boolean isStreamEOF() { + return client.getEof(); + } } diff --git a/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java index cf30e124a..06cfd7f44 100644 --- a/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java +++ b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java @@ -59,6 +59,9 @@ public static class Choice { @JsonProperty("message") private Message message; + @JsonProperty("delta") + private Message delta; + @JsonProperty("finish_reason") private String finishReason; @@ -68,6 +71,9 @@ public static class Choice { public Message getMessage() { return message; } public void setMessage(Message message) { this.message = message; } + public Message getDelta() { return message; } + public void setDelta(Message message) { this.message = message; } + public String getFinishReason() { return finishReason; } public void setFinishReason(String finishReason) { this.finishReason = finishReason; } } diff --git a/java/src/main/java/com/genexus/util/saia/SaiaService.java b/java/src/main/java/com/genexus/util/saia/SaiaService.java index 9801c5017..30230b281 100644 --- a/java/src/main/java/com/genexus/util/saia/SaiaService.java +++ b/java/src/main/java/com/genexus/util/saia/SaiaService.java @@ -6,24 +6,31 @@ import com.genexus.diagnostics.core.ILogger; import com.genexus.diagnostics.core.LogManager; import com.genexus.internet.HttpClient; -import com.genexus.util.CallResult; import org.json.JSONObject; +import com.genexus.util.CallResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SaiaService { private static final ILogger logger = LogManager.getLogger(SaiaService.class); private static final String apiKey = (String) SpecificImplementation.Application.getProperty("AI_PROVIDER_API_KEY", "");; private static final String aiProvider = (String) SpecificImplementation.Application.getProperty("AI_PROVIDER", ""); + private static final Logger log = LoggerFactory.getLogger(SaiaService.class); - public static OpenAIResponse call(OpenAIRequest request, CallResult result) { - return call(request, false, result); + public static OpenAIResponse call(OpenAIRequest request, HttpClient client, CallResult result) { + return call(request, false, client, result); } public static OpenAIResponse call(OpenAIRequest request, boolean isEmbedding, CallResult result) { + return call(request, isEmbedding, new HttpClient(), result); + } + + public static OpenAIResponse call(OpenAIRequest request, boolean isEmbedding, HttpClient client, CallResult result) { try { String jsonRequest = new ObjectMapper().writeValueAsString(request); + logger.debug("Agent payload: " + jsonRequest); String providerURL = aiProvider + "/chat";; - HttpClient client = new HttpClient(); client.setSecure(1); client.addHeader("Content-Type", "application/json"); client.addHeader("Authorization", "Bearer " + apiKey); @@ -35,8 +42,15 @@ public static OpenAIResponse call(OpenAIRequest request, boolean isEmbedding, Ca client.addString(jsonRequest); client.execute("POST", providerURL); if (client.getStatusCode() == 200) { - JSONObject jsonResponse = new JSONObject(client.getString()); - return new ObjectMapper().readValue(jsonResponse.toString(), OpenAIResponse.class); + if (client.getHeader("Content-Type").contains("text/event-stream")){ + return null; + } + else { + String saiaResponse = client.getString(); + logger.debug("Agent response: " + saiaResponse); + JSONObject jsonResponse = new JSONObject(saiaResponse); + return new ObjectMapper().readValue(jsonResponse.toString(), OpenAIResponse.class); + } } else { String errorDescription = String.format("Error calling Enterprise AI API, StatusCode: %d, ReasonLine: %s", @@ -44,6 +58,7 @@ public static OpenAIResponse call(OpenAIRequest request, boolean isEmbedding, Ca client.getReasonLine()); addResultMessage("SAIA_ERROR_CALL", (byte)1, errorDescription, result); logger.error(errorDescription); + logger.debug("Agent error response: " + client.getString()); } } catch (Exception e) { diff --git a/java/src/test/java/com/genexus/agent/Agent.java b/java/src/test/java/com/genexus/agent/Agent.java index 2c595d3f3..906838388 100644 --- a/java/src/test/java/com/genexus/agent/Agent.java +++ b/java/src/test/java/com/genexus/agent/Agent.java @@ -3,7 +3,6 @@ import com.genexus.*; import com.genexus.util.CallResult; import com.genexus.util.saia.OpenAIResponse; - import java.util.ArrayList; public final class Agent extends GXProcedure @@ -47,6 +46,26 @@ protected void privateExecute( ) message.setRole("user"); message.setContent("Que me puedes contar de la ciudad que te pedi el clima previamente?"); messages.add(message); + AV5OutputVariable = callAgent( "The weatherman", Gxproperties, messages, new CallResult()) ; + } + else if (AV3Parameter1.equals("chat_stream")) { + OpenAIResponse.Message message = new OpenAIResponse.Message(); + message.setRole("user"); + message.setContent("Dime el clima en Lima - Peru"); + messages.add(message); + message = new OpenAIResponse.Message(); + message.setRole("assistant"); + message.setContent("El clima actual en Lima, Perú, es soleado con una temperatura de 20.9°C (69.6°F). La dirección del viento es del suroeste (SSW) a 15.1 km/h (9.4 mph), y la humedad relativa es del 68%. La presión atmosférica es de 1013 mb. La visibilidad es de 10 km y el índice UV es de 12.5."); + messages.add(message); + message = new OpenAIResponse.Message(); + message.setRole("user"); + message.setContent("Que me puedes contar de la ciudad que te pedi el clima previamente?"); + messages.add(message); + AV5OutputVariable = callAgent( "The weatherman", true, Gxproperties, messages, new CallResult()) ; + System.out.print(AV5OutputVariable); + while (!isStreamEOF()) { + System.out.print(readChunk()); + } } else if (AV3Parameter1.equals("toolcall")) { OpenAIResponse.Message message = new OpenAIResponse.Message(); @@ -54,15 +73,33 @@ else if (AV3Parameter1.equals("toolcall")) { message.setContent("Necesito nombre y descripcion del producto 1779"); messages.add(message); AV5OutputVariable = callAgent( "ProductInfo", Gxproperties, messages, new CallResult()) ; - cleanup(); - return; + message = new OpenAIResponse.Message(); + message.setRole("assistant"); + message.setContent(AV5OutputVariable); + messages.add(message); + message = new OpenAIResponse.Message(); + message.setRole("user"); + message.setContent("Quiero que traduzcas la descripcion del producto que me habias enviado previamente"); + messages.add(message); + AV5OutputVariable = callAgent( "ProductInfo", Gxproperties, messages, new CallResult()) ; + } + else if (AV3Parameter1.equals("toolcall_stream")) { + OpenAIResponse.Message message = new OpenAIResponse.Message(); + message.setRole("user"); + message.setContent("Necesito nombre y descripcion del producto 1779"); + messages.add(message); + AV5OutputVariable = callAgent( "ProductInfo", true, Gxproperties, messages, new CallResult()) ; + System.out.print(AV5OutputVariable); + while (!isStreamEOF()) { + System.out.print(readChunk()); + } } else { Gxproperties.set("&Parameter1", AV3Parameter1); Gxproperties.set("&Parameter2", AV4Parameter2); Gxproperties.set("$context", "Los Angeles"); + AV5OutputVariable = callAgent( "The weatherman", Gxproperties, messages, new CallResult()) ; } - AV5OutputVariable = callAgent( "The weatherman", Gxproperties, messages, new CallResult()) ; cleanup(); } @@ -73,6 +110,8 @@ protected void cleanup( ) protected String callTool(String name, String arguments) throws Exception { switch (name) { + case "TranslateDescription": + return "The Panavox Television 80 inches are wonderful"; case "GetProductName": return "Televisor Panavox 80 pulgadas"; case "GetProductDescription": diff --git a/java/src/test/java/com/genexus/agent/TestAgent.java b/java/src/test/java/com/genexus/agent/TestAgent.java index 7e719ee9f..2a8352f58 100644 --- a/java/src/test/java/com/genexus/agent/TestAgent.java +++ b/java/src/test/java/com/genexus/agent/TestAgent.java @@ -26,8 +26,15 @@ public void testAPICallAgent() { new Agent(-1).execute( "chat", "", GXv_char5) ; System.out.println(GXv_char5[0]); + String[] GXv_char8 = new String[1] ; + new Agent(-1).execute( "chat_stream", "", GXv_char8) ; + String[] GXv_char6 = new String[1] ; new Agent(-1).execute( "toolcall", "", GXv_char6) ; + System.out.println(); System.out.println(GXv_char6[0]); + + String[] GXv_char7 = new String[1] ; + new Agent(-1).execute( "toolcall_stream", "", GXv_char7) ; } } From 07bdd8f75c8e13b89574dd6e303554f604b75c72 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Thu, 12 Dec 2024 18:11:02 -0300 Subject: [PATCH 18/26] Changes to better CallTools support. --- java/src/main/java/com/genexus/GXProcedure.java | 4 ++-- java/src/test/java/com/genexus/agent/Agent.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/java/src/main/java/com/genexus/GXProcedure.java b/java/src/main/java/com/genexus/GXProcedure.java index 814dab171..2bd6b6e79 100644 --- a/java/src/main/java/com/genexus/GXProcedure.java +++ b/java/src/main/java/com/genexus/GXProcedure.java @@ -266,7 +266,7 @@ protected void mockExecute() { privateExecute( ); } - protected String callTool(String name, String arguments) throws Exception { + protected String callTool(String name, String arguments) { return ""; } @@ -317,7 +317,7 @@ private void processToolCall(OpenAIResponse.ToolCall toolCall, ArrayList Date: Thu, 27 Feb 2025 15:46:18 -0300 Subject: [PATCH 19/26] Avoid Exception procesing Agent response when Choices is not in the json response. --- java/src/main/java/com/genexus/GXProcedure.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/src/main/java/com/genexus/GXProcedure.java b/java/src/main/java/com/genexus/GXProcedure.java index 2bd6b6e79..86ff64171 100644 --- a/java/src/main/java/com/genexus/GXProcedure.java +++ b/java/src/main/java/com/genexus/GXProcedure.java @@ -288,7 +288,7 @@ protected String callAgent(String agent, boolean stream, GXProperties properties aiRequest.setStream(true); client = new HttpClient(); OpenAIResponse aiResponse = SaiaService.call(aiRequest, client, result); - if (aiResponse != null) { + if (aiResponse != null && aiResponse.getChoices() != null) { for (OpenAIResponse.Choice element : aiResponse.getChoices()) { String finishReason = element.getFinishReason(); if (finishReason.equals("stop")) From ca9cdceb6de31c1264fbce8407f423b456f4d990 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Fri, 14 Mar 2025 16:32:12 -0300 Subject: [PATCH 20/26] Implements Embedding in MySQL --- .../java/com/genexus/db/ForEachCursor.java | 2 ++ .../com/genexus/db/driver/DataSource.java | 21 +++++++++++-- .../com/genexus/db/driver/GXConnection.java | 1 + .../db/driver/GXPreparedStatement.java | 31 +++++++++++++++++-- .../com/genexus/db/driver/GXResultSet.java | 16 +++++++++- 5 files changed, 65 insertions(+), 6 deletions(-) diff --git a/java/src/main/java/com/genexus/db/ForEachCursor.java b/java/src/main/java/com/genexus/db/ForEachCursor.java index 63cf7465c..274aa9be6 100644 --- a/java/src/main/java/com/genexus/db/ForEachCursor.java +++ b/java/src/main/java/com/genexus/db/ForEachCursor.java @@ -340,6 +340,8 @@ public void setParameterRT(String name, String value) boolean isLike = false; if(value.equals("like")) isLike = true; + else if (value.equals("Distance")) + value = ds.getDistanceFunction(); else if(!value.equals("=") && !value.equals(">") && !value.equals(">=") && !value.equals("<=") && !value.equals("<") && !value.equals("<>")) { diff --git a/java/src/main/java/com/genexus/db/driver/DataSource.java b/java/src/main/java/com/genexus/db/driver/DataSource.java index d45f9ea86..b8548928d 100644 --- a/java/src/main/java/com/genexus/db/driver/DataSource.java +++ b/java/src/main/java/com/genexus/db/driver/DataSource.java @@ -71,6 +71,7 @@ public class DataSource extends AbstractDataSource public String jdbcDataSource; private String namespace; + private String dbmsVersion; public DataSource( String name, @@ -581,7 +582,15 @@ public DataSource copy() copyDataSource.setConnectionPools(this.getConnectionPools()); return copyDataSource; } - + + public String getDistanceFunction() { + String distanceFunction = "DISTANCE"; + if (dbms.getId() == GXDBMS.DBMS_MYSQL && getDbmsVersion() != null && getDbmsVersion().contains("MariaDB")) { + distanceFunction = "vec_distance_cosine"; + } + return distanceFunction; + } + public String[] concatOp() { switch(dbms.getId()) @@ -616,7 +625,15 @@ public void RWPoolRecycle() { if (getConnectionPool().getRWConnectionPool(defaultUser) != null) getConnectionPool().getRWConnectionPool(defaultUser).PoolRecycle(); - } + } + + public void setDbmsVersion(String dbmsVersion) { + this.dbmsVersion = dbmsVersion; + } + + public String getDbmsVersion() { + return dbmsVersion; + } } diff --git a/java/src/main/java/com/genexus/db/driver/GXConnection.java b/java/src/main/java/com/genexus/db/driver/GXConnection.java index 4c88f2991..ce3c7ddc0 100644 --- a/java/src/main/java/com/genexus/db/driver/GXConnection.java +++ b/java/src/main/java/com/genexus/db/driver/GXConnection.java @@ -190,6 +190,7 @@ public GXConnection( ModelContext context, int handle, String user, String passw try { version = dma.getDatabaseProductVersion(); + dataSource.setDbmsVersion(version); } catch (SQLException e) { diff --git a/java/src/main/java/com/genexus/db/driver/GXPreparedStatement.java b/java/src/main/java/com/genexus/db/driver/GXPreparedStatement.java index 7d848fb33..01ba0cb81 100644 --- a/java/src/main/java/com/genexus/db/driver/GXPreparedStatement.java +++ b/java/src/main/java/com/genexus/db/driver/GXPreparedStatement.java @@ -13,6 +13,7 @@ import java.math.BigDecimal; import java.net.MalformedURLException; import java.net.URL; +import java.nio.ByteBuffer; import java.sql.Array; import java.sql.Blob; import java.sql.Clob; @@ -1506,14 +1507,35 @@ else if(blobFiles.length < index) } } + private static byte[] floatArrayToByteArray(Float[] floats) { + ByteBuffer buffer = ByteBuffer.allocate(floats.length * Float.BYTES); + + for (Float f : floats) { + if (f != null) { + buffer.putFloat(f); + } else { + buffer.putFloat(0.0f); + } + } + return buffer.array(); + } + public void setEmbedding(int index, Float[] value) throws SQLException{ - Array sqlArray = con.createArrayOf("float4", value); + byte[] bytes = null; + Array sqlArray = null; + if (con.getDBMS().getId() == GXDBMS.DBMS_POSTGRESQL) + sqlArray = con.createArrayOf("float4", value); + else + bytes = floatArrayToByteArray(value); if (DEBUG) { log(GXDBDebug.LOG_MAX, "setEmbedding - index : " + index); try { - stmt.setArray(index, sqlArray); + if (con.getDBMS().getId() == GXDBMS.DBMS_POSTGRESQL) + stmt.setArray(index, sqlArray); + else + stmt.setBytes(index, bytes); } catch (SQLException sqlException) { @@ -1523,7 +1545,10 @@ public void setEmbedding(int index, Float[] value) throws SQLException{ } else { - stmt.setArray(index, sqlArray); + if (con.getDBMS().getId() == GXDBMS.DBMS_POSTGRESQL) + stmt.setArray(index, sqlArray); + else + stmt.setBytes(index, bytes); } } diff --git a/java/src/main/java/com/genexus/db/driver/GXResultSet.java b/java/src/main/java/com/genexus/db/driver/GXResultSet.java index 7379d2ab1..166703275 100644 --- a/java/src/main/java/com/genexus/db/driver/GXResultSet.java +++ b/java/src/main/java/com/genexus/db/driver/GXResultSet.java @@ -9,6 +9,7 @@ import java.io.OutputStream; import java.io.Reader; import java.math.BigDecimal; +import java.nio.ByteBuffer; import java.sql.Array; import java.sql.Blob; import java.sql.Clob; @@ -836,7 +837,20 @@ public Float[] getGxembedding (int columnIndex) throws SQLException if (DEBUG ) log(GXDBDebug.LOG_MAX, "Warning: getEmbedding"); - return(Float[]) convertVectorStringToFloatArray(result.getArray(columnIndex).toString()); + if (con.getDBMS().getId() == GXDBMS.DBMS_POSTGRESQL) + return convertVectorStringToFloatArray(result.getArray(columnIndex).toString()); + else + return byteArrayToFloatObjectArray(result.getBytes(columnIndex)); + } + + private static Float[] byteArrayToFloatObjectArray(byte[] bytes) { + Float[] floats = new Float[bytes.length / Float.BYTES]; + + ByteBuffer buffer = ByteBuffer.wrap(bytes); + for (int i = 0; i < floats.length; i++) { + floats[i] = buffer.getFloat(); + } + return floats; } private static Float[] convertVectorStringToFloatArray(String vectorString) { From e33217a7ba98c0757d5e508059c9440d28a21e0e Mon Sep 17 00:00:00 2001 From: iroqueta Date: Fri, 4 Apr 2025 12:03:36 -0300 Subject: [PATCH 21/26] Add stream in chat Agent calls. --- .../main/java/com/genexus/GXProcedure.java | 41 +++---------- .../java/com/genexus/util/ChatResult.java | 58 +++++++++++++++++++ .../test/java/com/genexus/agent/Agent.java | 15 +++-- 3 files changed, 72 insertions(+), 42 deletions(-) create mode 100644 java/src/main/java/com/genexus/util/ChatResult.java diff --git a/java/src/main/java/com/genexus/GXProcedure.java b/java/src/main/java/com/genexus/GXProcedure.java index 86ff64171..cece64bbc 100644 --- a/java/src/main/java/com/genexus/GXProcedure.java +++ b/java/src/main/java/com/genexus/GXProcedure.java @@ -274,6 +274,11 @@ protected String callAssistant(String agent, GXProperties properties, ArrayList< return callAgent(agent, properties, messages, result); } + protected ChatResult chatAgent(String agent, GXProperties properties, ArrayList messages, CallResult result) { + callAgent(agent, true, properties, messages, result); + return new ChatResult(this, agent, properties, messages, result, client); + } + protected String callAgent(String agent, GXProperties properties, ArrayList messages, CallResult result) { return callAgent(agent, false, properties, messages, result); } @@ -299,12 +304,12 @@ protected String callAgent(String agent, boolean stream, GXProperties properties } } } else if (client.getStatusCode() == 200) { - return readChunk(agent, properties, messages, result); + return ""; } return ""; } - private String processNotChunkedResponse(String agent, boolean stream, GXProperties properties, ArrayList messages, CallResult result, ArrayList toolCalls) { + public String processNotChunkedResponse(String agent, boolean stream, GXProperties properties, ArrayList messages, CallResult result, ArrayList toolCalls) { for (OpenAIResponse.ToolCall tollCall : toolCalls) { processToolCall(tollCall, messages); } @@ -326,36 +331,4 @@ private void processToolCall(OpenAIResponse.ToolCall toolCall, ArrayList messages, CallResult result) { - String data = client.readChunk(); - if (data.isEmpty()) - return ""; - int index = data.indexOf("data:") + "data:".length(); - String chunkJson = data.substring(index).trim(); - try { - JSONObject jsonResponse = new JSONObject(chunkJson); - OpenAIResponse chunkResponse = new ObjectMapper().readValue(jsonResponse.toString(), OpenAIResponse.class); - OpenAIResponse.Choice choise = chunkResponse.getChoices().get(0); - if (choise.getFinishReason() != null && choise.getFinishReason().equals("tool_calls") && agent != null) { - messages.add(choise.getMessage()); - return processNotChunkedResponse(agent, true, properties, messages, result, choise.getMessage().getToolCalls()); - } - String chunkString = choise.getDelta().getContent(); - if (chunkString == null) - return ""; - return chunkString; - } - catch (Exception e) { - return ""; - } - } - - protected boolean isStreamEOF() { - return client.getEof(); - } } diff --git a/java/src/main/java/com/genexus/util/ChatResult.java b/java/src/main/java/com/genexus/util/ChatResult.java new file mode 100644 index 000000000..9989dc68c --- /dev/null +++ b/java/src/main/java/com/genexus/util/ChatResult.java @@ -0,0 +1,58 @@ +package com.genexus.util; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.genexus.GXProcedure; +import com.genexus.internet.HttpClient; +import com.genexus.util.saia.OpenAIResponse; +import org.json.JSONObject; + +import java.util.ArrayList; + +public class ChatResult { + private HttpClient client = null; + private String agent = null; + private GXProperties properties = null; + private ArrayList messages = null; + private CallResult result = null; + private GXProcedure agentProcedure = null; + + public ChatResult() { + } + + public ChatResult(GXProcedure agentProcedure, String agent, GXProperties properties, ArrayList messages, CallResult result, HttpClient client) { + this.agentProcedure = agentProcedure; + this.agent = agent; + this.properties = properties; + this.messages = messages; + this.result = result; + this.client = client; + } + + public boolean hasMoreData() { + return !client.getEof(); + } + + public String getMoreData() { + String data = client.readChunk(); + if (data.isEmpty()) + return ""; + int index = data.indexOf("data:") + "data:".length(); + String chunkJson = data.substring(index).trim(); + try { + JSONObject jsonResponse = new JSONObject(chunkJson); + OpenAIResponse chunkResponse = new ObjectMapper().readValue(jsonResponse.toString(), OpenAIResponse.class); + OpenAIResponse.Choice choise = chunkResponse.getChoices().get(0); + if (choise.getFinishReason() != null && choise.getFinishReason().equals("tool_calls") && agent != null) { + messages.add(choise.getMessage()); + return agentProcedure.processNotChunkedResponse(agent, true, properties, messages, result, choise.getMessage().getToolCalls()); + } + String chunkString = choise.getDelta().getContent(); + if (chunkString == null) + return ""; + return chunkString; + } + catch (Exception e) { + return ""; + } + } +} diff --git a/java/src/test/java/com/genexus/agent/Agent.java b/java/src/test/java/com/genexus/agent/Agent.java index 44368cdc5..d9e216a0f 100644 --- a/java/src/test/java/com/genexus/agent/Agent.java +++ b/java/src/test/java/com/genexus/agent/Agent.java @@ -2,6 +2,7 @@ import com.genexus.*; import com.genexus.util.CallResult; +import com.genexus.util.ChatResult; import com.genexus.util.saia.OpenAIResponse; import java.util.ArrayList; @@ -61,10 +62,9 @@ else if (AV3Parameter1.equals("chat_stream")) { message.setRole("user"); message.setContent("Que me puedes contar de la ciudad que te pedi el clima previamente?"); messages.add(message); - AV5OutputVariable = callAgent( "The weatherman", true, Gxproperties, messages, new CallResult()) ; - System.out.print(AV5OutputVariable); - while (!isStreamEOF()) { - System.out.print(readChunk()); + ChatResult chatResult = chatAgent( "The weatherman", Gxproperties, messages, new CallResult()) ; + while (chatResult.hasMoreData()) { + System.out.print(chatResult.hasMoreData()); } } else if (AV3Parameter1.equals("toolcall")) { @@ -88,10 +88,9 @@ else if (AV3Parameter1.equals("toolcall_stream")) { message.setRole("user"); message.setContent("Necesito nombre y descripcion del producto 1779"); messages.add(message); - AV5OutputVariable = callAgent( "ProductInfo", true, Gxproperties, messages, new CallResult()) ; - System.out.print(AV5OutputVariable); - while (!isStreamEOF()) { - System.out.print(readChunk()); + ChatResult chatResult = chatAgent( "ProductInfo", Gxproperties, messages, new CallResult()) ; + while (chatResult.hasMoreData()) { + System.out.print(chatResult.hasMoreData()); } } else { From 174cba2171533d4cb529a3bb36e6e23286ba7db6 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Wed, 7 May 2025 10:54:32 -0300 Subject: [PATCH 22/26] Implements support for multimodal content in Agent messages. --- .../main/java/com/genexus/GXProcedure.java | 4 +- .../java/com/genexus/util/ChatResult.java | 2 +- .../com/genexus/util/saia/OpenAIResponse.java | 103 +++++++++++++++++- .../test/java/com/genexus/agent/Agent.java | 20 ++-- 4 files changed, 113 insertions(+), 16 deletions(-) diff --git a/java/src/main/java/com/genexus/GXProcedure.java b/java/src/main/java/com/genexus/GXProcedure.java index cece64bbc..d2b65dfd2 100644 --- a/java/src/main/java/com/genexus/GXProcedure.java +++ b/java/src/main/java/com/genexus/GXProcedure.java @@ -297,7 +297,7 @@ protected String callAgent(String agent, boolean stream, GXProperties properties for (OpenAIResponse.Choice element : aiResponse.getChoices()) { String finishReason = element.getFinishReason(); if (finishReason.equals("stop")) - return element.getMessage().getContent(); + return element.getMessage().getStringContent(); if (finishReason.equals("tool_calls")) { messages.add(element.getMessage()); return processNotChunkedResponse(agent, stream, properties, messages, result, element.getMessage().getToolCalls()); @@ -327,7 +327,7 @@ private void processToolCall(OpenAIResponse.ToolCall toolCall, ArrayList toolCalls; @@ -96,8 +102,14 @@ public static class Message { public String getRole() { return role; } public void setRole(String role) { this.role = role; } - public String getContent() { return content; } - public void setContent(String content) { this.content = content; } + @JsonIgnore + public String getStringContent() { return ((StringContent) content).getValue(); } + public Content getContent() { return content; } + @JsonIgnore + public void setStringContent(String content) { this.content = new StringContent(content); } + @JsonIgnore + public void setStructuredContent(StructuredContent content) { this.content = content; } + public void setContent(Content content) { this.content = content; } public ArrayList getToolCalls() { return toolCalls; } public void setToolCalls(ArrayList toolCalls) { this.toolCalls = toolCalls; } @@ -106,6 +118,91 @@ public static class Message { public void setToolCallId(String toolCallId) { this.toolCallId = toolCallId; } } + public interface Content { } + + public static class StringContent implements Content { + public StringContent() {} + + private String value; + + public StringContent(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + } + + public static class StructuredContent implements Content { + public StructuredContent() {} + + private ArrayList items; + + public StructuredContent(ArrayList items) { + this.items = items; + } + + public ArrayList getItems() { + return items; + } + + public void setItems(ArrayList items) { + this.items = items; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class StructuredContentItem { + public StructuredContentItem() {} + + private String type; + private String text; + private ImageUrl image_url; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public ImageUrl getImage_url() { + return image_url; + } + + public void setImage_url(ImageUrl image_url) { + this.image_url = image_url; + } + + public static class ImageUrl { + public ImageUrl() {} + + private String url; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + } + } + @JsonIgnoreProperties(ignoreUnknown = true) public static class ToolCall { diff --git a/java/src/test/java/com/genexus/agent/Agent.java b/java/src/test/java/com/genexus/agent/Agent.java index d9e216a0f..33a283c09 100644 --- a/java/src/test/java/com/genexus/agent/Agent.java +++ b/java/src/test/java/com/genexus/agent/Agent.java @@ -37,30 +37,30 @@ protected void privateExecute( ) if (AV3Parameter1.equals("chat")) { OpenAIResponse.Message message = new OpenAIResponse.Message(); message.setRole("user"); - message.setContent("Dime el clima en Lima - Peru"); + message.setStringContent("Dime el clima en Lima - Peru"); messages.add(message); message = new OpenAIResponse.Message(); message.setRole("assistant"); - message.setContent("El clima actual en Lima, Perú, es soleado con una temperatura de 20.9°C (69.6°F). La dirección del viento es del suroeste (SSW) a 15.1 km/h (9.4 mph), y la humedad relativa es del 68%. La presión atmosférica es de 1013 mb. La visibilidad es de 10 km y el índice UV es de 12.5."); + message.setStringContent("El clima actual en Lima, Perú, es soleado con una temperatura de 20.9°C (69.6°F). La dirección del viento es del suroeste (SSW) a 15.1 km/h (9.4 mph), y la humedad relativa es del 68%. La presión atmosférica es de 1013 mb. La visibilidad es de 10 km y el índice UV es de 12.5."); messages.add(message); message = new OpenAIResponse.Message(); message.setRole("user"); - message.setContent("Que me puedes contar de la ciudad que te pedi el clima previamente?"); + message.setStringContent("Que me puedes contar de la ciudad que te pedi el clima previamente?"); messages.add(message); AV5OutputVariable = callAgent( "The weatherman", Gxproperties, messages, new CallResult()) ; } else if (AV3Parameter1.equals("chat_stream")) { OpenAIResponse.Message message = new OpenAIResponse.Message(); message.setRole("user"); - message.setContent("Dime el clima en Lima - Peru"); + message.setStringContent("Dime el clima en Lima - Peru"); messages.add(message); message = new OpenAIResponse.Message(); message.setRole("assistant"); - message.setContent("El clima actual en Lima, Perú, es soleado con una temperatura de 20.9°C (69.6°F). La dirección del viento es del suroeste (SSW) a 15.1 km/h (9.4 mph), y la humedad relativa es del 68%. La presión atmosférica es de 1013 mb. La visibilidad es de 10 km y el índice UV es de 12.5."); + message.setStringContent("El clima actual en Lima, Perú, es soleado con una temperatura de 20.9°C (69.6°F). La dirección del viento es del suroeste (SSW) a 15.1 km/h (9.4 mph), y la humedad relativa es del 68%. La presión atmosférica es de 1013 mb. La visibilidad es de 10 km y el índice UV es de 12.5."); messages.add(message); message = new OpenAIResponse.Message(); message.setRole("user"); - message.setContent("Que me puedes contar de la ciudad que te pedi el clima previamente?"); + message.setStringContent("Que me puedes contar de la ciudad que te pedi el clima previamente?"); messages.add(message); ChatResult chatResult = chatAgent( "The weatherman", Gxproperties, messages, new CallResult()) ; while (chatResult.hasMoreData()) { @@ -70,23 +70,23 @@ else if (AV3Parameter1.equals("chat_stream")) { else if (AV3Parameter1.equals("toolcall")) { OpenAIResponse.Message message = new OpenAIResponse.Message(); message.setRole("user"); - message.setContent("Necesito nombre y descripcion del producto 1779"); + message.setStringContent("Necesito nombre y descripcion del producto 1779"); messages.add(message); AV5OutputVariable = callAgent( "ProductInfo", Gxproperties, messages, new CallResult()) ; message = new OpenAIResponse.Message(); message.setRole("assistant"); - message.setContent(AV5OutputVariable); + message.setStringContent(AV5OutputVariable); messages.add(message); message = new OpenAIResponse.Message(); message.setRole("user"); - message.setContent("Quiero que traduzcas la descripcion del producto que me habias enviado previamente"); + message.setStringContent("Quiero que traduzcas la descripcion del producto que me habias enviado previamente"); messages.add(message); AV5OutputVariable = callAgent( "ProductInfo", Gxproperties, messages, new CallResult()) ; } else if (AV3Parameter1.equals("toolcall_stream")) { OpenAIResponse.Message message = new OpenAIResponse.Message(); message.setRole("user"); - message.setContent("Necesito nombre y descripcion del producto 1779"); + message.setStringContent("Necesito nombre y descripcion del producto 1779"); messages.add(message); ChatResult chatResult = chatAgent( "ProductInfo", Gxproperties, messages, new CallResult()) ; while (chatResult.hasMoreData()) { From 78bbb254b152b176391b3a97b975aeb457746663 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Wed, 7 May 2025 11:02:36 -0300 Subject: [PATCH 23/26] Implements support for multimodal content in Agent messages. --- .../util/saia/ContentDeserializer.java | 33 +++++++++++++++++++ .../genexus/util/saia/ContentSerializer.java | 30 +++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 java/src/main/java/com/genexus/util/saia/ContentDeserializer.java create mode 100644 java/src/main/java/com/genexus/util/saia/ContentSerializer.java diff --git a/java/src/main/java/com/genexus/util/saia/ContentDeserializer.java b/java/src/main/java/com/genexus/util/saia/ContentDeserializer.java new file mode 100644 index 000000000..d32fbc47c --- /dev/null +++ b/java/src/main/java/com/genexus/util/saia/ContentDeserializer.java @@ -0,0 +1,33 @@ +package com.genexus.util.saia; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.ObjectCodec; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; + +import java.io.IOException; +import java.util.ArrayList; + +class ContentDeserializer extends JsonDeserializer { + public ContentDeserializer() {} + + @Override + public OpenAIResponse.Content deserialize(JsonParser p, DeserializationContext ct) throws IOException { + ObjectCodec codec = p.getCodec(); + JsonNode node = codec.readTree(p); + + if (node.isTextual()) { + return new OpenAIResponse.StringContent(node.asText()); + } else if (node.isArray()) { + ArrayList items = new ArrayList<>(); + for (JsonNode itemNode : node) { + OpenAIResponse.StructuredContentItem item = codec.treeToValue(itemNode, OpenAIResponse.StructuredContentItem.class); + items.add(item); + } + return new OpenAIResponse.StructuredContent(items); + } + + throw new IOException("Invalid content format"); + } +} diff --git a/java/src/main/java/com/genexus/util/saia/ContentSerializer.java b/java/src/main/java/com/genexus/util/saia/ContentSerializer.java new file mode 100644 index 000000000..665a9c380 --- /dev/null +++ b/java/src/main/java/com/genexus/util/saia/ContentSerializer.java @@ -0,0 +1,30 @@ +package com.genexus.util.saia; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; + +class ContentSerializer extends JsonSerializer { + public ContentSerializer() { + } + + @Override + public void serialize(OpenAIResponse.Content value, JsonGenerator gen, SerializerProvider serializers) throws IOException { + ObjectMapper mapper = (ObjectMapper) gen.getCodec(); + + if (value instanceof OpenAIResponse.StringContent) { + gen.writeString(((OpenAIResponse.StringContent) value).getValue()); + } else if (value instanceof OpenAIResponse.StructuredContent) { + gen.writeStartArray(); + for (OpenAIResponse.StructuredContentItem item : ((OpenAIResponse.StructuredContent) value).getItems()) { + mapper.writeValue(gen, item); + } + gen.writeEndArray(); + } else { + gen.writeNull(); + } + } +} From 0fdb8df47dc3efb36076e2e43a89a04b9c7463d5 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Wed, 7 May 2025 13:54:18 -0300 Subject: [PATCH 24/26] Implements support for multimodal content in Agent messages. --- .../com/genexus/util/saia/OpenAIResponse.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java index 7a2ad67ad..d5117f732 100644 --- a/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java +++ b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java @@ -104,6 +104,8 @@ public static class Message { @JsonIgnore public String getStringContent() { return ((StringContent) content).getValue(); } + @JsonIgnore + public StructuredContent getStructuredContent() {return (StructuredContent)content;} public Content getContent() { return content; } @JsonIgnore public void setStringContent(String content) { this.content = new StringContent(content); } @@ -162,7 +164,7 @@ public StructuredContentItem() {} private String type; private String text; - private ImageUrl image_url; + private ImageUrl imageURL; public String getType() { return type; @@ -180,18 +182,20 @@ public void setText(String text) { this.text = text; } - public ImageUrl getImage_url() { - return image_url; + public ImageUrl getImageUrl() { + return imageURL; } - public void setImage_url(ImageUrl image_url) { - this.image_url = image_url; + public void setImageUrl(ImageUrl imageURL) { + this.imageURL = imageURL; } + @JsonInclude(JsonInclude.Include.NON_NULL) public static class ImageUrl { public ImageUrl() {} private String url; + private String detail; public String getUrl() { return url; @@ -200,6 +204,14 @@ public String getUrl() { public void setUrl(String url) { this.url = url; } + + public String getDetail() { + return detail; + } + + public void setDetail(String detail) { + this.detail = detail; + } } } From 2232b63153d6a000b4482e13b61bb98bfffac45b Mon Sep 17 00:00:00 2001 From: iroqueta Date: Thu, 8 May 2025 12:14:50 -0300 Subject: [PATCH 25/26] Implements support for multimodal content in Agent messages. --- .../com/genexus/util/saia/OpenAIResponse.java | 42 ++++++++----------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java index d5117f732..f9c124bf4 100644 --- a/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java +++ b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java @@ -164,7 +164,9 @@ public StructuredContentItem() {} private String type; private String text; - private ImageUrl imageURL; + private String image_url; + private String filename; + private String file_data; public String getType() { return type; @@ -182,36 +184,28 @@ public void setText(String text) { this.text = text; } - public ImageUrl getImageUrl() { - return imageURL; + public String getImage_url() { + return image_url; } - public void setImageUrl(ImageUrl imageURL) { - this.imageURL = imageURL; + public void setImage_url(String imageURL) { + this.image_url = imageURL; } - @JsonInclude(JsonInclude.Include.NON_NULL) - public static class ImageUrl { - public ImageUrl() {} - - private String url; - private String detail; + public String getFilename() { + return filename; + } - public String getUrl() { - return url; - } + public void setFilename(String filename) { + this.filename = filename; + } - public void setUrl(String url) { - this.url = url; - } + public String getFile_data() { + return file_data; + } - public String getDetail() { - return detail; - } - - public void setDetail(String detail) { - this.detail = detail; - } + public void setFile_data(String file_data) { + this.file_data = file_data; } } From 175cc4717a29bc545c37a74ce827569ffc157d90 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Mon, 12 May 2025 16:22:04 -0300 Subject: [PATCH 26/26] Implements support for multimodal content in Agent messages. --- .../com/genexus/util/saia/OpenAIResponse.java | 38 +++++++++++-------- .../test/java/com/genexus/agent/Agent.java | 20 ++++++++++ .../java/com/genexus/agent/TestAgent.java | 4 ++ 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java index f9c124bf4..27a7a0899 100644 --- a/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java +++ b/java/src/main/java/com/genexus/util/saia/OpenAIResponse.java @@ -164,9 +164,7 @@ public StructuredContentItem() {} private String type; private String text; - private String image_url; - private String filename; - private String file_data; + private ImageUrl image_url; public String getType() { return type; @@ -184,28 +182,36 @@ public void setText(String text) { this.text = text; } - public String getImage_url() { + public ImageUrl getImage_url() { return image_url; } - public void setImage_url(String imageURL) { + public void setImage_url(ImageUrl imageURL) { this.image_url = imageURL; } - public String getFilename() { - return filename; - } + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class ImageUrl { + public ImageUrl() {} - public void setFilename(String filename) { - this.filename = filename; - } + private String url; + private String detail; - public String getFile_data() { - return file_data; - } + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } - public void setFile_data(String file_data) { - this.file_data = file_data; + public String getDetail() { + return detail; + } + + public void setDetail(String detail) { + this.detail = detail; + } } } diff --git a/java/src/test/java/com/genexus/agent/Agent.java b/java/src/test/java/com/genexus/agent/Agent.java index 33a283c09..2098be5fc 100644 --- a/java/src/test/java/com/genexus/agent/Agent.java +++ b/java/src/test/java/com/genexus/agent/Agent.java @@ -49,6 +49,26 @@ protected void privateExecute( ) messages.add(message); AV5OutputVariable = callAgent( "The weatherman", Gxproperties, messages, new CallResult()) ; } + else if (AV3Parameter1.equals("eval_image")) { + OpenAIResponse.StructuredContent content = new OpenAIResponse.StructuredContent(); + ArrayList items = new ArrayList<>(); + OpenAIResponse.StructuredContentItem contentItem = new OpenAIResponse.StructuredContentItem(); + contentItem.setType("text"); + contentItem.setText("De que se trata esta imagen"); + items.add(contentItem); + OpenAIResponse.StructuredContentItem contentItem1 = new OpenAIResponse.StructuredContentItem(); + contentItem1.setType("image_url"); + OpenAIResponse.StructuredContentItem.ImageUrl imageURL = new OpenAIResponse.StructuredContentItem.ImageUrl(); + imageURL.setUrl("https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"); + contentItem1.setImage_url(imageURL); + items.add(contentItem1); + content.setItems(items); + OpenAIResponse.Message message = new OpenAIResponse.Message(); + message.setRole("user"); + message.setStructuredContent(content); + messages.add(message); + AV5OutputVariable = callAgent( "The weatherman", Gxproperties, messages, new CallResult()) ; + } else if (AV3Parameter1.equals("chat_stream")) { OpenAIResponse.Message message = new OpenAIResponse.Message(); message.setRole("user"); diff --git a/java/src/test/java/com/genexus/agent/TestAgent.java b/java/src/test/java/com/genexus/agent/TestAgent.java index 2a8352f58..043c61b1e 100644 --- a/java/src/test/java/com/genexus/agent/TestAgent.java +++ b/java/src/test/java/com/genexus/agent/TestAgent.java @@ -26,6 +26,10 @@ public void testAPICallAgent() { new Agent(-1).execute( "chat", "", GXv_char5) ; System.out.println(GXv_char5[0]); + String[] GXv_char9 = new String[1] ; + new Agent(-1).execute( "eval_image", "", GXv_char9) ; + System.out.println(GXv_char9[0]); + String[] GXv_char8 = new String[1] ; new Agent(-1).execute( "chat_stream", "", GXv_char8) ;