Update Duplicate Detection

.gitignore

@@ -65,3 +65,4 @@ perf.data.old
 .idea/
 .vscode/
 centrallix-os/tmp/*
+centrallix-os/datasets/

centrallix-doc/Widgets/onepage.html

@@ -5976,7 +5976,7 @@ <h3 class="childproperties">Child Properties:</h3>
 <tr>
 <td class="name">type</td>
 <td class="type">string</td>
-<td class="description">The type of the column: "text", "check", or "image".  "text" is a normal column, and displays the textual value of the data element.  "check" displays a checkmark if the data is non-zero (integers) or for strings if the value is non-empty and not "N" or "No".  "image" displays the image referred to by the pathname contained in the data value.</td>
+<td class="description">The type of the column: "text", "check", "image", or "progress".  "text" is a normal column, and displays the textual value of the data element.  "check" displays a checkmark if the data is non-zero (integers) or for strings if the value is non-empty and not "N" or "No".  "image" displays the image referred to by the pathname contained in the data value. "progress" displays a progress bar, with additional fields such as bar_color, bar_textcollor, and bar_padding.</td>
 </tr>
 <tr>
 <td class="name">width</td>

centrallix-doc/Widgets/table.html

@@ -329,7 +329,7 @@ <h3 class="childproperties">Child Properties:</h3>
 <tr>
 <td class="name">type</td>
 <td class="type">string</td>
-<td class="description">The type of the column: "text", "check", or "image".  "text" is a normal column, and displays the textual value of the data element.  "check" displays a checkmark if the data is non-zero (integers) or for strings if the value is non-empty and not "N" or "No".  "image" displays the image referred to by the pathname contained in the data value.</td>
+<td class="description">The type of the column: "text", "check", "image", or "progress".  "text" is a normal column, and displays the textual value of the data element.  "check" displays a checkmark if the data is non-zero (integers) or for strings if the value is non-empty and not "N" or "No".  "image" displays the image referred to by the pathname contained in the data value. "progress" displays a progress bar, with additional fields such as bar_color, bar_textcollor, and bar_padding.</td>
 </tr>
 <tr>
 <td class="name">width</td>

centrallix-doc/Widgets/widgets.xml

@@ -3731,7 +3731,7 @@ myTabControl "widget/tab"
 
 				<childproperty name="title" type="string">The title of the column to be displayed in the header row.</childproperty>
 
-				<childproperty name="type" type="string">The type of the column: "text", "check", or "image".  "text" is a normal column, and displays the textual value of the data element.  "check" displays a checkmark if the data is non-zero (integers) or for strings if the value is non-empty and not "N" or "No".  "image" displays the image referred to by the pathname contained in the data value.</childproperty>
+				<childproperty name="type" type="string">The type of the column: "text", "check", "image", or "progress".  "text" is a normal column, and displays the textual value of the data element.  "check" displays a checkmark if the data is non-zero (integers) or for strings if the value is non-empty and not "N" or "No".  "image" displays the image referred to by the pathname contained in the data value. "progress" displays a progress bar, with additional fields such as bar_color, bar_textcollor, and bar_padding.</childproperty>
 
 				<childproperty name="width" type="integer">width of the column.</childproperty>
 

centrallix-lib/Makefile.in

@@ -59,14 +59,14 @@ LIBS = @LIBS@
 ##
 PROFILE=@PROFILE@
 COVERAGE=@COVERAGE@
-CFLAGS=@CFLAGS@ @DEFS@ -Iinclude -DCXLIB_INTERNAL -DNM_USE_SYSMALLOC -Wall $(PROFILE) $(COVERAGE) -g
+CFLAGS=@CFLAGS@ @DEFS@ -Iinclude -DCXLIB_INTERNAL -DNM_USE_SYSMALLOC -Wall $(PROFILE) $(COVERAGE) -g -lm
 MTCFLAGS=@CFLAGS@ @DEFS@ -Iinclude -DCXLIB_INTERNAL -DNM_USE_SYSMALLOC -Wall $(PROFILE) $(COVERAGE) -g -O0
 TCFLAGS=$(patsubst -DNDEBUG,,$(CFLAGS))
 
-XSTATICFILES=mtask.o mtlexer.o memstr.o xarray.o xhash.o xstring.o mtsession.o newmalloc.o xhashqueue.o bdqs_transport.o xhandle.o xringqueue.o cxsec.o smmalloc.o qprintf.o strtcpy.o util.o
+XSTATICFILES=mtask.o mtlexer.o memstr.o xarray.o xhash.o xstring.o mtsession.o newmalloc.o xhashqueue.o bdqs_transport.o xhandle.o xringqueue.o cxsec.o smmalloc.o clusters.o qprintf.o strtcpy.o util.o
 STATICFILES=$(patsubst %,src/%,$(XSTATICFILES))
 
-XDYNAMICFILES=mtask.lo mtlexer.lo memstr.lo xarray.lo xhash.lo xstring.lo mtsession.lo newmalloc.lo xhashqueue.lo bdqs_transport.lo xhandle.lo xringqueue.lo cxsec.lo smmalloc.lo qprintf.lo strtcpy.lo util.lo
+XDYNAMICFILES=mtask.lo mtlexer.lo memstr.lo xarray.lo xhash.lo xstring.lo mtsession.lo newmalloc.lo xhashqueue.lo bdqs_transport.lo xhandle.lo xringqueue.lo cxsec.lo smmalloc.lo clusters.lo qprintf.lo strtcpy.lo util.lo
 DYNAMICFILES=$(patsubst %,src/%,$(XDYNAMICFILES))
 
 INCLUDEFILES:=$(wildcard include/*.h)

centrallix-lib/aclocal.m4

@@ -47,7 +47,7 @@ AC_DEFUN(CHECK_BUILTIN_EXPECT,
 		AC_DEFINE([HAVE_BUILTIN_EXPECT], [1], [Define if __builtin_expect is available])
 		AC_MSG_RESULT([yes])
 	    ],
-	    [AC_MSG_RESULT([no])],
+	    [AC_MSG_RESULT([no])]
 	)
     ]
 )

centrallix-lib/configure

@@ -4104,6 +4104,7 @@ $as_echo "yes" >&6; }
 else
   { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
 $as_echo "no" >&6; }
+
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
 

centrallix-lib/include/clusters.h

@@ -0,0 +1,161 @@
+#ifndef CLUSTERS_H
+#define	CLUSTERS_H
+
+/************************************************************************/
+/* Centrallix Application Server System					*/
+/* Centrallix Core							*/
+/* 									*/
+/* Copyright (C) 1998-2026 LightSys Technology Services, Inc.		*/
+/* 									*/
+/* This program is free software; you can redistribute it and/or modify	*/
+/* it under the terms of the GNU General Public License as published by	*/
+/* the Free Software Foundation; either version 2 of the License, or	*/
+/* (at your option) any later version.					*/
+/* 									*/
+/* This program is distributed in the hope that it will be useful,	*/
+/* but WITHOUT ANY WARRANTY; without even the implied warranty of	*/
+/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the	*/
+/* GNU General Public License for more details.				*/
+/* 									*/
+/* You should have received a copy of the GNU General Public License	*/
+/* along with this program; if not, write to the Free Software		*/
+/* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA		*/
+/* 02111-1307  USA							*/
+/* 									*/
+/* A copy of the GNU General Public License has been included in this	*/
+/* distribution in the file "COPYING".					*/
+/* 									*/
+/* Module:	lib_cluster.c, lib_cluster.h				*/
+/* Author:	Israel Fuller						*/
+/* Creation:	September 29, 2025					*/
+/* Description	Clustering library used to cluster and search data with	*/
+/*		cosine or Levenshtein (aka. edit distance) similarity 	*/
+/*		measures. Used by the "clustering driver".		*/
+/*		For more information on how to use this library, see	*/
+/*		string-similarity.md in the centrallix-sysdoc folder.	*/
+/************************************************************************/
+
+#include <stdlib.h>
+#include <stdbool.h>
+
+#ifdef CXLIB_INTERNAL
+#include "xarray.h"
+#else
+#include "cxlib/xarray.h"
+#endif
+
+/** This file has additional documentation in string_similarity.md. **/
+
+
+/*** This value defines the number of dimensions used for a sparse
+ *** vector.  The higher the number, the fewer collisions will be
+ *** encountered when using these vectors for cosine comparisons.
+ *** This is also called the vector table size, if viewing the
+ *** vector as a hash table of character pairs.
+ *** 
+ *** 2147483647 is the signed int max, and is also a prime number.
+ *** Using this value ensures that the longest run of 0s will not
+ *** cause an int underflow with the current encoding scheme.
+ *** 
+ *** Unfortunately, we can't use a number this large yet because
+ *** kmeans algorithm creates densely allocated centroids with
+ *** `CA_NUM_DIMS` dimensions, so a large number causes it to fail.
+ *** This, we use 251 as the largest prime number less than 256,
+ *** giving us a decent balance between collision reduction and
+ *** kmeans centroid performance/memory overhead.
+ ***/
+#define CA_NUM_DIMS 251
+
+/*** The character used to create a pair with the first and last characters
+ *** of a string.  Currently set to 96, the character just before 'a' (97)
+ *** in the ASCII table.
+ ***/
+#define CA_BOUNDARY_CHAR ((unsigned char)('a' - 1))
+
+/** Types. **/
+typedef int* pVector;      /* Sparse vector. */
+typedef double* pCentroid; /* Dense centroid. */
+#define CENTROID_SIZE (CA_NUM_DIMS * sizeof(double))
+
+/*** Information about detected matching pairs.
+ ***
+ *** @param i The index into the provided data for the first element of the pair.
+ *** @param j The index into the provided data for the second element of the pair.
+ *** @param similarity A number from 0 to 1, from a similarity function, showing
+ *** 	how similar the pairs are.
+ ***/
+typedef struct
+    {
+    unsigned int i, j;
+    double similarity;
+    }
+    Pair, *pPair;
+
+
+/** Edit distance function. **/
+int ca_edit_dist(const char* str1, const char* str2, const size_t str1_length, const size_t str2_length);
+
+/** Vector functions. **/
+pVector ca_build_vector(const char* str);
+unsigned int ca_sparse_len(const pVector vector);
+void ca_print_vector(const pVector vector);
+void ca_free_vector(pVector sparse_vector);
+
+/** k-means function. **/
+int ca_kmeans(
+    pVector* vectors,
+    const unsigned int num_vectors,
+    const unsigned int num_clusters,
+    const unsigned int max_iter,
+    const double min_improvement,
+    unsigned int* labels,
+    double* vector_sims,
+    bool auto_seed);
+
+/** Vector helper macros. **/
+#define ca_is_empty(vector) (vector[0] == -CA_NUM_DIMS)
+/*** Note: Given that CA_NUM_DIMS == 251, ca_build_vector("") will give the
+ *** vector we check for in the ca_has_no_pairs() macro, [-172, 11, -78],
+ *** which has a single pair of boundary characters.
+ *** If CA_NUM_DIMS is modified, this macro will need to be updated, hence the
+ *** compiler directive causing it to be undefined in this case, likely leading
+ *** to a lot of compiler or linker issues to remind the developer about this.
+ ***/
+#if CA_NUM_DIMS == 251
+#define ca_has_no_pairs(vector) \
+    ({ \
+	__typeof__ (vector) _v = (vector); \
+	_v[0] == -172 && _v[1] == 11 && _v[2] == -78; \
+    })
+#endif
+
+/** Comparison functions (see ca_search()). **/
+double ca_cos_compare(void* v1, void* v2);
+double ca_lev_compare(void* str1, void* str2);
+bool ca_eql(pVector v1, pVector v2);
+
+/** Similarity search functions. **/
+void* ca_most_similar(
+    void* target,
+    void** data,
+    const unsigned int num_data,
+    const double (*similarity)(void*, void*),
+    const double threshold);
+pXArray ca_sliding_search(
+    void** data,
+    const unsigned int num_data,
+    const unsigned int window_size,
+    const double (*similarity)(void*, void*),
+    const double threshold,
+    pXArray maybe_pairs);
+pXArray ca_complete_search(
+    void** data,
+    const unsigned int num_data,
+    const double (*similarity)(void*, void*),
+    const double threshold,
+    pXArray maybe_pairs);
+
+/** Module management functions. **/
+void ca_init(void);
+
+#endif /* End of .h file. */

centrallix-lib/include/cxlibconfig-internal.h.in

@@ -1,5 +1,10 @@
 /* include/cxlibconfig-internal.h.in.  Generated from configure.ac by autoheader.  */
 
+/*** cxlibconfig-internal.h.in lists defines that are used by centrallix-lib
+ *** internally and will not be needed in other modules.  For external
+ *** usecases, see cxlibconfig.h.in.
+ ***/
+
 /* Define to the address where bug reports for this package should be sent. */
 #undef PACKAGE_BUGREPORT
 
@@ -34,4 +39,3 @@
 
 /* defined to 1 if SIOCOUTQ is available */
 #undef HAVE_SIOCOUTQ
-

centrallix-lib/include/cxlibconfig.h.in

@@ -1,5 +1,11 @@
 /* include/cxlibconfig.h.in.  Generated from configure.ac by autoheader.  */
 
+/*** cxlibconfig-h.in lists defines that should be availabe to anyone
+ *** using centrallix-lib. For example, a define used in an external .h 
+ *** file should be written here so that the .h file can include it even
+ *** if it is used in another project.
+ ***/
+
 /* Define to 1 if you have the `endservent' function. */
 #undef HAVE_ENDSERVENT
 
@@ -33,6 +39,9 @@
 /* Define to 1 if you have the <stdlib.h> header file. */
 #undef HAVE_STDLIB_H
 
+/* Define to 1 if you have the `__builtin_expect' function. */
+#undef HAVE_BUILTIN_EXPECT
+
 /* Define to 1 if you have the `strdup' function. */
 #undef HAVE_STRDUP
 

centrallix-lib/include/expect.h

@@ -11,7 +11,7 @@
 /* GNU Lesser General Public License, Version 2.1, contained in the	*/
 /* included file "COPYING".						*/
 /* 									*/
-/* Module:	Expect Branch Optimization Module (mtask.c, mtask.h)	*/
+/* Module:	Expect Branch Optimization Module (expect.h)		*/
 /* Author:	Israel Fuller						*/
 /* Date:	February 27th, 2026					*/
 /*									*/
@@ -23,19 +23,26 @@
 /* 		true, UNLIKELY() indicates it is likely to be false.	*/
 /************************************************************************/
 
+#ifdef HAVE_CONFIG_H
+ #ifdef CXLIB_INTERNAL
+  #include "cxlibconfig.h"
+ #else
+  #include "cxlib/cxlibconfig.h"
+ #endif
+#endif
 
 /** Define macros for signalling the more likely branch to the compiler. **/
 #ifdef HAVE_BUILTIN_EXPECT
-/** Use the GCC __builtin_expect() function for optimization. **/
-/** Note: We use !!(x) which normalizes to 0 or 1 to help the compiler. **/
-#define LIKELY(x)   (__builtin_expect(!!(x), 1))
-#define UNLIKELY(x) (__builtin_expect(!!(x), 0))
+ /** Use the GCC __builtin_expect() function for optimization. **/
+ /** Note: We use !!(x) which normalizes to 0 or 1 to help the compiler. **/
+ #define LIKELY(x)   (__builtin_expect(!!(x), 1))
+ #define UNLIKELY(x) (__builtin_expect(!!(x), 0))
 #else
-/*** Fallback: Define pass through functions to support compilers that don't
- *** have this feature.
- ***/
-#define LIKELY(x)   (x)
-#define UNLIKELY(x) (x)
+ /*** Fallback: Define pass through functions to support compilers that don't
+  *** have this feature.
+  ***/
+ #define LIKELY(x)   (x)
+ #define UNLIKELY(x) (x)
 #endif
 
 #endif

centrallix-lib/include/glyph.h

@@ -0,0 +1,79 @@
+#ifndef GLYPH_H
+#define	GLYPH_H
+
+/************************************************************************/
+/* Centrallix Application Server System                                 */
+/* Centrallix Core                                                      */
+/*                                                                      */
+/* Copyright (C) 1998-2026 LightSys Technology Services, Inc.           */
+/*                                                                      */
+/* This program is free software; you can redistribute it and/or modify */
+/* it under the terms of the GNU General Public License as published by */
+/* the Free Software Foundation; either version 2 of the License, or    */
+/* (at your option) any later version.                                  */
+/*                                                                      */
+/* This program is distributed in the hope that it will be useful,      */
+/* but WITHOUT ANY WARRANTY; without even the implied warranty of       */
+/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the        */
+/* GNU General Public License for more details.                         */
+/*                                                                      */
+/* You should have received a copy of the GNU General Public License    */
+/* along with this program; if not, write to the Free Software          */
+/* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA             */
+/* 02111-1307  USA                                                      */
+/*                                                                      */
+/* A copy of the GNU General Public License has been included in this   */
+/* distribution in the file "COPYING".                                  */
+/*                                                                      */
+/* Module:      glyph.h                                                 */
+/* Author:      Israel Fuller                                           */
+/* Creation:    October 27, 2025                                        */
+/* Description: A simple debug visualizer to make pretty patterns in    */
+/*              developer's terminal which can be surprisingly useful   */
+/*              for debugging algorithms.                               */
+/************************************************************************/
+
+#include <stdlib.h>
+
+/** Uncomment to activate glyphs. **/
+/** Should not be enabled in production code on the master branch. */
+// #define ENABLE_GLYPHS
+
+#ifdef ENABLE_GLYPHS
+#define glyph_print(s) printf("%s", s);
+
+/*** Initialize a simple debug visualizer to make pretty patterns in the
+ *** developer's terminal. Great for when you need to run a long task and
+ *** want a super simple way to make sure it's still working.
+ *** 
+ *** @attention - Relies on storing data in variables in scope, so calling
+ ***    glyph() requires a call to glyph_init() previously in the same scope.
+ *** 
+ *** @param name The symbol name of the visualizer.
+ *** @param str The string printed for the visualization.
+ *** @param interval The number of invocations of glyph() required to print.
+ *** @param flush Whether to flush on output.
+ ***/
+#define glyph_init(name, str, interval, flush) \
+	const char* vis_##name##_str = str; \
+	const unsigned int vis_##name##_interval = interval; \
+	const bool vis_##name##_flush = flush; \
+	unsigned int vis_##name##_i = 0u;
+
+/*** Invoke a visualizer.
+ *** 
+ *** @param name The name of the visualizer to invoke.
+ ***/
+#define glyph(name) \
+	if (++vis_##name##_i % vis_##name##_interval == 0) \
+	    { \
+	    glyph_print(vis_##name##_str); \
+	    if (vis_##name##_flush) fflush(stdout); \
+	    }
+#else
+#define glyph_print(str)
+#define glyph_init(name, str, interval, flush)
+#define glyph(name)
+#endif
+
+#endif /* End of .h file. */