Embedding support in Java Generator

iroqueta · iroqueta · commit d43c4bb7f1c8 · 2024-10-16T12:30:25.000-03:00
diff --git a/android/src/main/java/com/genexus/db/ForEachCursor.java b/android/src/main/java/com/genexus/db/ForEachCursor.java
@@ -217,6 +217,7 @@ public void setTimestamp(int index, java.sql.Timestamp value) throws SQLExceptio
 	    public void setBLOBFile(int index, String fileName) throws SQLException {}
 	    public void setBLOBFile(int index, String fileName, boolean isMultiMedia) throws SQLException {}
 		public void setBLOBFile(int index, String fileName, boolean isMultiMedia, boolean downloadContent) throws SQLException {}
+		public void setEmbedding(int index, Float[] value) throws SQLException {}
 
 		public void setVarchar(int index, String value, int length, boolean allowsNull) throws SQLException {}
 	    public void setLongVarchar(int index, String value, boolean allowsNull) throws SQLException {}
diff --git a/android/src/main/java/com/genexus/db/driver/GXCallableStatement.java b/android/src/main/java/com/genexus/db/driver/GXCallableStatement.java
@@ -651,6 +651,11 @@ public byte[] getBytes(int columnIndex) throws SQLException
 		return stmt.getBytes(columnIndex);
 	}
 
+	public Float[] getGxembedding (int columnIndex) throws SQLException
+	{
+		return null;
+	}
+
 	public java.util.UUID getGUID(int columnIndex) throws SQLException
 	{
 		if	(DEBUG)
diff --git a/android/src/main/java/com/genexus/db/driver/GXPreparedStatement.java b/android/src/main/java/com/genexus/db/driver/GXPreparedStatement.java
@@ -1307,7 +1307,9 @@ else if (file.exists() && !fileNameNew.startsWith(blobBasePath))
 			}
 		}
 	}
-	
+
+	public void setEmbedding(int index, Float[] value) throws SQLException{
+	}
     public void setBinaryStream(int index, java.io.InputStream value, int length) throws SQLException
 	{
 		if	(DEBUG)
diff --git a/android/src/main/java/com/genexus/db/driver/GXResultSet.java b/android/src/main/java/com/genexus/db/driver/GXResultSet.java
@@ -770,6 +770,11 @@ public byte[] getBytes(int columnIndex) throws SQLException
 		return result.getBytes(columnIndex);
 	}
 
+	public Float[] getGxembedding (int columnIndex) throws SQLException
+	{
+		return null;
+	}
+
 	public java.util.UUID getGUID(int columnIndex) throws SQLException
 	{
 		java.util.UUID value;
diff --git a/common/src/main/java/com/genexus/db/BufferIFieldGetter.java b/common/src/main/java/com/genexus/db/BufferIFieldGetter.java
@@ -85,6 +85,10 @@ public byte[] getBytes(int columnIndex) throws SQLException {
         return ((byte[]) value[columnIndex - 1]);
     }
 
+	public Float[] getGxembedding(int columnIndex) throws SQLException {
+		return ((Float[]) value[columnIndex - 1]);
+	}
+
     public java.sql.Date getDate(int columnIndex) throws SQLException {
         return ((java.sql.Date) value[columnIndex - 1]);
     }
diff --git a/common/src/main/java/com/genexus/db/CachedIFieldGetter.java b/common/src/main/java/com/genexus/db/CachedIFieldGetter.java
@@ -220,6 +220,11 @@ public byte[] getBytes(int columnIndex) throws SQLException
 		else 
 			return ((byte[][])value[index])[0];
 	}
+
+	public Float[] getGxembedding(int columnIndex) throws SQLException
+	{
+		return ((Float[][])value[getColumnIndex(columnIndex)])[0];
+	}
 		
 	public java.sql.Date getDate(int columnIndex) throws SQLException
 	{
diff --git a/common/src/main/java/com/genexus/db/IFieldGetter.java b/common/src/main/java/com/genexus/db/IFieldGetter.java
@@ -33,4 +33,5 @@ public interface IFieldGetter
 	String getMultimediaUri(int columnIndex) throws SQLException;
 	String getMultimediaUri(int columnIndex, boolean absPath) throws SQLException;
 	java.util.UUID getGUID(int columnIndex) throws SQLException;
+	Float[] getGxembedding(int columnIndex) throws SQLException;
 }
diff --git a/common/src/main/java/com/genexus/db/IFieldSetter.java b/common/src/main/java/com/genexus/db/IFieldSetter.java
@@ -35,6 +35,7 @@ public interface IFieldSetter
     public void setBLOBFile(int index, String fileName) throws SQLException;
     public void setBLOBFile(int index, String fileName, boolean isMultiMedia) throws SQLException;
 	public void setBLOBFile(int index, String fileName, boolean isMultiMedia, boolean downloadContent) throws SQLException;
+	public void setEmbedding(int index, Float[] value) throws SQLException;
 
     public void setVarchar(int index, String value, int length, boolean allowsNull) throws SQLException;
     public void setLongVarchar(int index, String value, boolean allowsNull) throws SQLException;
diff --git a/gxembedding/pom.xml b/gxembedding/pom.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+	<parent>
+		<groupId>com.genexus</groupId>
+		<artifactId>parent</artifactId>
+		<version>${revision}${changelist}</version>
+	</parent>
+		
+    <artifactId>gxembedding</artifactId>
+    <name>GeneXus Embedding</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>gxcommon</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+
+	<build>
+		<finalName>gxembedding</finalName>
+    </build>
+</project>
diff --git a/gxembedding/src/main/java/com/genexus/db/GXEmbedding.java b/gxembedding/src/main/java/com/genexus/db/GXEmbedding.java
@@ -0,0 +1,71 @@
+package com.genexus.db;
+
+import com.genexus.CommonUtil;
+import com.genexus.GXBaseCollection;
+import com.genexus.SdtMessages_Message;
+import com.genexus.embedding.EmbeddingService;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class GXEmbedding {
+
+	private String model;
+	private int dimensions;
+	private List<Float> embedding;
+
+	public GXEmbedding() {
+	}
+
+	public GXEmbedding(String model, int dimensions) {
+		this.model = model;
+		this.dimensions = dimensions;
+		embedding = new ArrayList<>(dimensions);
+	}
+
+	public GXEmbedding(Float[] embedding, String model, int dimensions) {
+		this.model = model;
+		this.dimensions = dimensions;
+		this.embedding = Arrays.asList(embedding);
+	}
+
+	public GXEmbedding(List<Float> embedding) {
+		this.embedding = embedding;
+	}
+
+	public String getModel() {
+		return model;
+	}
+
+	public int getDimensions() {
+		return dimensions;
+	}
+
+	public void setEmbedding(List<Float> embedding) {
+		this.embedding = embedding;
+	}
+
+	public Float[] getFloatArray() {
+		return embedding.toArray(new Float[0]);
+	}
+
+	public static GXEmbedding generateEmbedding(GXEmbedding embeddingInfo, String text, GXBaseCollection<SdtMessages_Message> Messages) {
+		try {
+			List<Float> embedding = EmbeddingService.getInstance().getEmbedding(embeddingInfo.getModel(), embeddingInfo.getDimensions(), text);
+			embeddingInfo.setEmbedding(embedding);
+		}
+		catch (Exception ex) {
+			CommonUtil.ErrorToMessages("GenerateEmbedding Error", ex.getMessage(), Messages);
+		}
+		return embeddingInfo;
+	}
+
+	public String toString()
+	{
+		return embedding.stream()
+			.map(String::valueOf)
+			.collect(Collectors.joining(","));
+	}
+}
diff --git a/gxembedding/src/main/java/com/genexus/embedding/EmbeddingService.java b/gxembedding/src/main/java/com/genexus/embedding/EmbeddingService.java
@@ -0,0 +1,77 @@
+package com.genexus.embedding;
+
+import com.genexus.common.interfaces.SpecificImplementation;
+import com.genexus.diagnostics.core.ILogger;
+import com.genexus.diagnostics.core.LogManager;
+import com.genexus.internet.HttpClient;
+import json.org.json.JSONArray;
+import json.org.json.JSONObject;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class EmbeddingService
+{
+	private static final ILogger logger = LogManager.getLogger(EmbeddingService.class);
+
+	private static EmbeddingService instance;
+	private final HttpClient client;
+	private String apiKey;
+	private String aiProvider;
+
+	public EmbeddingService() {
+		client = new HttpClient();
+		aiProvider = (String) SpecificImplementation.Application.getProperty("AI_PROVIDER", "");
+		apiKey = (String) SpecificImplementation.Application.getProperty("AI_PROVIDER_API_KEY", "");
+	}
+	public static EmbeddingService getInstance() {
+		if (instance == null)
+			instance = new EmbeddingService();
+		return instance;
+	}
+
+	public List<Float> getEmbedding(String model, int dimensions, String input) {
+		List<String> inputList = new ArrayList<>();
+		inputList.add(input);
+		return getEmbedding(model, dimensions, inputList);
+	}
+
+	public List<Float> getEmbedding(String model, int dimensions, List<String> inputList) {
+		List<Float> embeddingsList = new ArrayList<>();
+		try {
+			String requestBody = String.format(
+				"{\n" +
+					"  \"model\": \"%s\",\n" +
+					"  \"input\": %s,\n" +
+					"  \"dimensions\": %d\n" +
+					"}",
+				model, new JSONArray(inputList), dimensions
+			);
+			client.setSecure(1);
+			client.addHeader("Content-Type", "application/json");
+			client.addHeader("Authorization", "Bearer " + apiKey);
+			client.addHeader("X-Saia-Source", "Embedding");
+			client.addString(requestBody);
+			client.execute("POST", aiProvider);
+			if (client.getStatusCode() == 200) {
+				JSONObject jsonResponse = new JSONObject(client.getString());
+				JSONArray embeddingsArray = jsonResponse.getJSONArray("data");
+
+				for (int i = 0; i < embeddingsArray.length(); i++) {
+					JSONArray embedding = ((JSONObject)embeddingsArray.get(0)).getJSONArray("embedding");;
+					for (int j = 0; j < embedding.length(); j++) {
+						embeddingsList.add((float) embedding.getDouble(j));
+					}
+				}
+			}
+			else {
+				logger.error(String.format("Error calling embedding API, StatusCode: %d, ReasonLine: %s",
+					client.getStatusCode(),
+					client.getReasonLine()));
+			}
+		} catch (Exception e) {
+			logger.error("GenerateEmbedding Error", e);
+		}
+		return embeddingsList;
+	}
+}
diff --git a/java/client.cfg b/java/client.cfg
@@ -47,6 +47,8 @@ ORQ_CLIENT_URL=
 ORQ_SERVER_DIR=
 TMPMEDIA_DIR=PrivateTempStorage
 PRINT_LAYOUT_METADATA_DIR=LayoutMetadata
+AI_PROVIDER=https://api.saia.ai/chat
+AI_PROVIDER_API_KEY=xxx
 HTTP_PROTOCOL=Unsecure
 SAMESITE_COOKIE=Lax
 StorageTimeZone= 1
diff --git a/java/pom.xml b/java/pom.xml
@@ -34,6 +34,11 @@
 			<artifactId>gxmail</artifactId>
 			<version>${project.version}</version>
 		</dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>gxembedding</artifactId>
+            <version>${project.version}</version>
+        </dependency>
 		<dependency>
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-collections4</artifactId>
diff --git a/java/src/main/java/com/genexus/GXutil.java b/java/src/main/java/com/genexus/GXutil.java
@@ -9,6 +9,7 @@
 
 import com.genexus.common.interfaces.SpecificImplementation;
 import com.genexus.db.DataStoreProvider;
+import com.genexus.db.GXEmbedding;
 import com.genexus.internet.HttpContext;
 import com.genexus.internet.StringCollection;
 import com.genexus.platform.INativeFunctions;
@@ -1777,4 +1778,8 @@ public static String buildURLFromHttpClient(com.genexus.internet.HttpClient GXSo
 		return url.toString();
 	}
 
+	public static String embeddingToStr(GXEmbedding embedding) {
+		return embedding.toString();
+	}
+
 }
diff --git a/java/src/main/java/com/genexus/db/ForEachCursor.java b/java/src/main/java/com/genexus/db/ForEachCursor.java
@@ -327,6 +327,7 @@ public void setTimestamp(int index, java.sql.Timestamp value) throws SQLExceptio
 	    public void setBLOBFile(int index, String fileName) throws SQLException {}
 	    public void setBLOBFile(int index, String fileName, boolean isMultiMedia) throws SQLException {}
 		public void setBLOBFile(int index, String fileName, boolean isMultiMedia, boolean downloadContet) throws SQLException {}
+		public void setEmbedding(int index, Float[] value) throws SQLException {}
 
 		public void setVarchar(int index, String value, int length, boolean allowsNull) throws SQLException {}
 	    public void setLongVarchar(int index, String value, boolean allowsNull) throws SQLException {}
diff --git a/java/src/main/java/com/genexus/db/driver/GXCallableStatement.java b/java/src/main/java/com/genexus/db/driver/GXCallableStatement.java
@@ -704,6 +704,14 @@ public byte[] getBytes(int columnIndex) throws SQLException
 		return stmt.getBytes(columnIndex);
 	}
 
+	public Float[] getGxembedding (int columnIndex) throws SQLException
+	{
+		if	(DEBUG )
+			log(GXDBDebug.LOG_MAX, "Warning: getEmbedding");
+
+		return(Float[]) stmt.getArray(columnIndex).getArray();
+	}
+
 	public java.util.UUID getGUID(int columnIndex) throws SQLException
 	{
 		if	(DEBUG)
diff --git a/java/src/main/java/com/genexus/db/driver/GXConnection.java b/java/src/main/java/com/genexus/db/driver/GXConnection.java
@@ -1744,8 +1744,7 @@ public void setClientInfo(String arg0, String arg1)
 
 	public Array createArrayOf(String typeName, Object[] elements)
 			throws SQLException {
-		// TODO Auto-generated method stub
-		return null;
+		return getJDBCConnection().createArrayOf(typeName, elements);
 	}
 
 	public void abort(Executor executor)
diff --git a/java/src/main/java/com/genexus/db/driver/GXPreparedStatement.java b/java/src/main/java/com/genexus/db/driver/GXPreparedStatement.java
@@ -1506,6 +1506,27 @@ else if(blobFiles.length < index)
 		}
 	}
 
+	public void setEmbedding(int index, Float[] value) throws SQLException{
+		Array sqlArray = con.createArrayOf("float4", value);
+		if	(DEBUG)
+		{
+			log(GXDBDebug.LOG_MAX, "setEmbedding - index : " + index);
+			try
+			{
+				stmt.setArray(index, sqlArray);
+			}
+			catch (SQLException sqlException)
+			{
+				if	(con.isLogEnabled()) con.logSQLException(con.getHandle(), sqlException);
+				throw sqlException;
+			}
+		}
+		else
+		{
+			stmt.setArray(index, sqlArray);
+		}
+	}
+
     public void setBinaryStream(int index, java.io.InputStream value, int length) throws SQLException
 	{
 		if	(DEBUG)
diff --git a/java/src/main/java/com/genexus/db/driver/GXResultSet.java b/java/src/main/java/com/genexus/db/driver/GXResultSet.java
@@ -831,6 +831,25 @@ public byte[] getBytes(int columnIndex) throws SQLException
 		return result.getBytes(columnIndex);
 	}
 
+	public Float[] getGxembedding (int columnIndex) throws SQLException
+	{
+		if	(DEBUG )
+			log(GXDBDebug.LOG_MAX, "Warning: getEmbedding");
+
+		return(Float[]) convertVectorStringToFloatArray(result.getArray(columnIndex).toString());
+	}
+
+	private static Float[] convertVectorStringToFloatArray(String vectorString) {
+		vectorString = vectorString.replace("[", "").replace("]", "").trim();
+		String[] stringValues = vectorString.split(",");
+		Float[] floatArray = new Float[stringValues.length];
+
+		for (int i = 0; i < stringValues.length; i++) {
+			floatArray[i] = Float.parseFloat(stringValues[i].trim());
+		}
+		return floatArray;
+	}
+
 	public java.util.UUID getGUID(int columnIndex) throws SQLException
 	{
 		java.util.UUID value;
diff --git a/java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java b/java/src/test/java/com/genexus/embedding/GXEmbeddingTest.java
diff --git a/pom.xml b/pom.xml

Original file line number	Diff line number	Diff line change
`@@ -651,6 +651,11 @@ public byte[] getBytes(int columnIndex) throws SQLException`
`651`	`651`	`return stmt.getBytes(columnIndex);`
`652`	`652`	`}`
`653`	`653`
	`654`	`+ public Float[] getGxembedding (int columnIndex) throws SQLException`
	`655`	`+ {`
	`656`	`+ return null;`
	`657`	`+ }`
	`658`	`+`
`654`	`659`	`public java.util.UUID getGUID(int columnIndex) throws SQLException`
`655`	`660`	`{`
`656`	`661`	`if (DEBUG)`
Original file line number	Diff line number	Diff line change
`@@ -1307,7 +1307,9 @@ else if (file.exists() && !fileNameNew.startsWith(blobBasePath))`
`1307`	`1307`	`}`
`1308`	`1308`	`}`
`1309`	`1309`	`}`
`1310`		`-`
	`1310`	`+`
	`1311`	`+ public void setEmbedding(int index, Float[] value) throws SQLException{`
	`1312`	`+ }`
`1311`	`1313`	`public void setBinaryStream(int index, java.io.InputStream value, int length) throws SQLException`
`1312`	`1314`	`{`
`1313`	`1315`	`if (DEBUG)`
Original file line number	Diff line number	Diff line change
`@@ -770,6 +770,11 @@ public byte[] getBytes(int columnIndex) throws SQLException`
`770`	`770`	`return result.getBytes(columnIndex);`
`771`	`771`	`}`
`772`	`772`
	`773`	`+ public Float[] getGxembedding (int columnIndex) throws SQLException`
	`774`	`+ {`
	`775`	`+ return null;`
	`776`	`+ }`
	`777`	`+`
`773`	`778`	`public java.util.UUID getGUID(int columnIndex) throws SQLException`
`774`	`779`	`{`
`775`	`780`	`java.util.UUID value;`
Original file line number	Diff line number	Diff line change
`@@ -85,6 +85,10 @@ public byte[] getBytes(int columnIndex) throws SQLException {`
`85`	`85`	`return ((byte[]) value[columnIndex - 1]);`
`86`	`86`	`}`
`87`	`87`
	`88`	`+ public Float[] getGxembedding(int columnIndex) throws SQLException {`
	`89`	`+ return ((Float[]) value[columnIndex - 1]);`
	`90`	`+ }`
	`91`	`+`
`88`	`92`	`public java.sql.Date getDate(int columnIndex) throws SQLException {`
`89`	`93`	`return ((java.sql.Date) value[columnIndex - 1]);`
`90`	`94`	`}`
Original file line number	Diff line number	Diff line change
`@@ -220,6 +220,11 @@ public byte[] getBytes(int columnIndex) throws SQLException`
`220`	`220`	`else`
`221`	`221`	`return ((byte[][])value[index])[0];`
`222`	`222`	`}`
	`223`	`+`
	`224`	`+ public Float[] getGxembedding(int columnIndex) throws SQLException`
	`225`	`+ {`
	`226`	`+ return ((Float[][])value[getColumnIndex(columnIndex)])[0];`
	`227`	`+ }`
`223`	`228`
`224`	`229`	`public java.sql.Date getDate(int columnIndex) throws SQLException`
`225`	`230`	`{`
Original file line number	Diff line number	Diff line change
`@@ -33,4 +33,5 @@ public interface IFieldGetter`
`33`	`33`	`String getMultimediaUri(int columnIndex) throws SQLException;`
`34`	`34`	`String getMultimediaUri(int columnIndex, boolean absPath) throws SQLException;`
`35`	`35`	`java.util.UUID getGUID(int columnIndex) throws SQLException;`
	`36`	`+ Float[] getGxembedding(int columnIndex) throws SQLException;`
`36`	`37`	`}`