License query improvement: search also using latest component version

inc/component.h

@@ -59,6 +59,7 @@ typedef struct component_data_t
 	int health_stats[3]; /* health stats: forks, watchers, contributors */
 	int rank; /* purl ranking - optional*/
 	int path_depth; /* depth of the matched file path*/
+	int third_party_rank; /* Saves third party ranking*/
 } component_data_t;
 
 component_data_t * component_init(void);

inc/debug.h

@@ -32,6 +32,7 @@ extern bool debug_on; //= false; //set debug mode from main.
 extern bool quiet;
 
 
+bool scanlog_init(void);
 void scanlog(const char *fmt, ...);
 void map_dump(scan_data_t *scan);
 long microseconds_now(void);

inc/match.h

@@ -16,7 +16,7 @@ typedef struct match_data_t
 	int lines_matched; /*number of matched lines*/
 	char * line_ranges; /*input snippet line ranges */
 	char * oss_ranges; /* kb snippet line ranges */
-	char * matched_percent; /* matched percent */
+	int matched_percent; /* matched percent */
 	int  path_ln; /*file path lenght*/ //TODO check if this is needed.
 	uint8_t file_md5[MD5_LEN]; /* file md5 */
 	char source_md5[MD5_LEN * 2 + 1]; /*matched file md5 in hex format */

inc/scanoss.h

@@ -31,15 +31,15 @@
 #include "limits.h"
 
 #define MAX_FILE_PATH 1024
-#define FETCH_MAX_FILES 20000
+#define FETCH_MAX_FILES 12000
 #define MIN_FILE_SIZE 256 // files below this size will be ignored
 #define CRC_LIST_LEN 1024 // list of crc checksums to avoid metadata duplicates
 #define SNIPPET_LINE_TOLERANCE 10
 
 #define WFP_LN 4
 #define WFP_REC_LN 18
 
-#define SCANOSS_VERSION "5.4.18"
+#define SCANOSS_VERSION "5.4.19"
 
 /* Log files */
 #define SCAN_LOG "/tmp/scanoss_scan.log"

inc/util.h

@@ -63,7 +63,7 @@ char * str_cat_realloc(char **a, char * b);
 
 void free_and_null(void ** pr);
 
-int path_is_third_party(const char* path);
+int path_is_third_party(component_data_t *comp);
 
 /* Counts the number of '/' characters in a path string */
 int path_depth(char* path);

src/component.c

@@ -209,8 +209,9 @@ static char * look_for_version(char *in)
 void fill_component_path(component_data_t *component, char *file_path)
 {
 	component->file = strdup(look_for_version(file_path));
-	component->path_ln = strlen(file_path);
+	component->path_ln = strlen(component->file);
 	flip_slashes(component->file);
+	component->path_depth = path_depth(component->file);
 }
 
 /**
@@ -298,9 +299,6 @@ bool fill_component(component_data_t *component, uint8_t *url_key, char *file_pa
 	}
 	else
 		component->rank = COMPONENT_DEFAULT_RANK;
-
-	component->path_depth = path_depth(component->file);
-
 	return true;
 }
 

src/license.c

@@ -51,7 +51,7 @@
 	 4 = Scancode detection 
 	 5 = Scancode detection at mining time
 	 6 = osslot */
-const char *license_sources[] = {"component_declared", "file_spdx_tag", "file_header", "license_file", "scancode", "scancode-file", "osselot"};
+const char *license_sources[] = {"component_declared", "file_spdx_tag", "file_header", "license_file", "scancode-file", "scancode", "osselot"};
 bool full_license_report = false;
 
 
@@ -220,7 +220,7 @@ static char *json_from_license(uint32_t *crclist, char *buffer, char *license, i
 	string_clean(license);
 	int len = 0;
 
-	if (strlen(license) < 2)
+	if (!*license || strlen(license) < 2)
 		return buffer;
 	/* Calculate CRC to avoid duplicates */
 	uint32_t CRC = string_crc32c(license);
@@ -272,7 +272,7 @@ static char *split_in_json_array(uint32_t *crclist, char *buffer, char *license,
 
 	} while (next_lic);
 
-	return buffer;
+	return r;  // Return the updated buffer pointer, not the original
 }
 
 char *  license_to_json(uint32_t *crclist, char *buffer, char *license, int src, bool *first_record)
@@ -336,10 +336,9 @@ bool print_licenses_item(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *
 	free(CSV);
 
 	int src = atoi(source);
+	scanlog("Fetched License %s - source ID %d\n", license, src);
 
-	scanlog("Fetched license %s\n", license);
-
-	if (strlen(license) > 2 && (src < (sizeof(license_sources) / sizeof(license_sources[0]))))
+	if (src < (sizeof(license_sources) / sizeof(license_sources[0])))
 		license_add_to_list(&licenses[src], license);
 
 	free(source);
@@ -401,8 +400,30 @@ void print_licenses(component_data_t *comp)
 		scanlog("License for %s@%s license returns %d hits\n", comp->purls[i], comp->version, records);
 
 		if (records)
-			break;
+		{
+			//Look if someone of the prefered liceses ids already has a match
+			for (int i = 0; i < 4; i++)
+			{
+				if (licenses_by_type[i].count > 0)
+				{
+					scanlog("Stop searching for licenses\n");
+					break;
+				}
+			}
+		}
 
+		if (strcmp(comp->version, comp->latest_version) != 0)
+		{
+			/* Calculate purl@latest_version md5 */
+			purl_version_md5(purlversion_md5, comp->purls[i], comp->latest_version);
+
+			records = ldb_fetch_recordset(NULL, oss_license, purlversion_md5, false, print_licenses_item, &licenses_by_type);
+			scanlog("License for %s@%s license returns %d hits\n", comp->purls[i], comp->latest_version, records);
+
+			if (records)
+				break;
+		}
+		/* Unversioned purl license */
 		records = ldb_fetch_recordset(NULL, oss_license, comp->purls_md5[i], false, print_licenses_item, &licenses_by_type);
 		scanlog("License for %s license returns %d hits\n", comp->purls[i], records);
 
@@ -411,7 +432,7 @@ void print_licenses(component_data_t *comp)
 	}
 
 	/* Open licenses structure */
-	char * result = calloc(MAX_FIELD_LN * 100, 1);
+	char * result = calloc(MAX_FIELD_LN * 1024, 1);
 	char * buffer = result;
 	int len = 0;
 

src/limits.c

@@ -9,17 +9,8 @@
   * @see https://github.com/scanoss/engine/blob/master/src/limits.c
   */
 
-
-int consecutive_score = 4000;   /** Maximumm sUsed for snippet selection */
-
-/* During snippet scanning, when a wfp (with more than consecutive_threshold wfps) produces a score higher 
-   than consecutive_score by consecutive_hits in a row, the scan will skip consecutive_jump snippets */
-int consecutive_hits = 4;       
-int consecutive_jump = 5;      
-int consecutive_threshold = 50; 
-
 int range_tolerance = 5;  /** A maximum number of non-matched lines tolerated inside a matching range */
 int min_match_lines = 10; /** Minimum number of lines matched for a match range to be acepted */
 int min_match_hits  = 4;  /** Minimum number of snippet ID hits to produce a snippet match*/
 
-const int max_vulnerabilities = 50; /** Show only the first N vulnerabilities */
+const int max_vulnerabilities = 50; /** Show only the first N vulnerabilities */