fix: related-purl ignore + normalize_license overflow + snippet fallback (#120) [main port]

inc/component.h

@@ -108,4 +108,6 @@ int asset_declared(component_data_t * comp);
 void component_item_free(component_item * comp_item);
 void fill_component_path(component_data_t *component, char *file_path);
 char * look_for_version(char *in);
+bool ignored_purl(const char *purl);
+bool component_ignored_by_purl(component_data_t *component);
 #endif

inc/match_list.h

@@ -144,6 +144,7 @@ void component_list_init(component_list_t *comp_list, int max_items);
 bool component_list_add(component_list_t * list, component_data_t * new_comp, bool (* val) (component_data_t * a, component_data_t * b), bool remove_a);
 void component_list_print(component_list_t * list, bool (*printer) (component_data_t * fpa), char * separator);
 void component_list_destroy(component_list_t *list);
+int component_list_remove_ignored(component_list_t *list);
 bool component_list_add_binary(component_list_t *list, component_data_t *new_comp, bool (*val)(component_data_t *a, component_data_t *b), bool remove_a);
 bool match_list_eval(match_list_t *list, match_data_t * in,  bool (*eval)(match_data_t *fpa, match_data_t *fpb));
 void match_list_tolerance_set(float in);

inc/scanoss.h

@@ -33,7 +33,7 @@
 #define WFP_LN 4
 #define WFP_REC_LN 18
 
-#define SCANOSS_VERSION "5.4.27"
+#define SCANOSS_VERSION "5.4.28"
 
 /* Log files */
 #define SCAN_LOG "/tmp/scanoss_scan.log"

src/component.c

@@ -34,6 +34,7 @@
 #include "util.h"
 #include "debug.h"
 #include "query.h"
+#include "url.h"
 
 /**
  * @brief Free component object
@@ -117,6 +118,33 @@ component_data_t *component_data_copy(component_data_t *in)
     return out;
 }
 
+/**
+ * @brief Return true if the given purl is listed in ignore_components (-b parameter)
+ * @param purl purl string to test
+ */
+bool ignored_purl(const char *purl)
+{
+	if (!ignore_components || !purl)
+		return false;
+
+	/* Travel ignore_components looking for a purl rule matching this purl */
+	for (int i = 0; i < MAX_SBOM_ITEMS; i++)
+	{
+		char *dvendor = ignore_components[i].vendor;
+		char *dcomponent = ignore_components[i].component;
+		char *dpurl = ignore_components[i].purl;
+
+		/* Exit if reached the end */
+		if (!dcomponent && !dvendor && !dpurl)
+			break;
+
+		if (dpurl && !strcmp(purl, dpurl))
+			return true;
+	}
+
+	return false;
+}
+
 /**
  * @brief Return true if asset is found in ignore_components (-b parameter)
  * @param url_record pointer to url record
@@ -180,6 +208,41 @@ bool ignored_asset_match(uint8_t *url_record)
 	return found;
 }
 
+/**
+ * @brief Return true if a component relates to an ignored purl (-b parameter).
+ *
+ * ignored_asset_match() only compares the single primary purl stored in the url
+ * record (field 6). A project may be published under several purls (e.g. a
+ * github repo mirrored as a golang module), and those sibling identities are
+ * only materialised later by fetch_related_purls(). This helper resolves the
+ * related purls and checks every purl of the component against the ignore list,
+ * so ignoring any one of a project's purls suppresses it regardless of which
+ * identity ended up as the primary one.
+ *
+ * @param component component to test
+ */
+bool component_ignored_by_purl(component_data_t *component)
+{
+	if (!ignore_components || !component)
+		return false;
+
+	/* Resolve sibling purls so we match the project under any of its identities */
+	fetch_related_purls(component);
+
+	for (int i = 0; i < MAX_PURLS; i++)
+	{
+		if (!component->purls[i])
+			break;
+		if (ignored_purl(component->purls[i]))
+		{
+			scanlog("Component ignored by purl %s\n", component->purls[i]);
+			return true;
+		}
+	}
+
+	return false;
+}
+
 char * look_for_version(char *in)
 {
 	if (!in)

src/license.c

@@ -46,6 +46,13 @@
 
 bool full_license_report = false;
 
+/* normalize_license() may replace a license string in place with its canonical
+   SPDX id (e.g. "Apache" -> "Apache-2.0"), which can be LONGER than the original
+   text. License buffers must therefore be sized to hold the longest canonical id
+   in license_normalization[] (currently "LGPL-3.0-or-later", 17 chars) even when
+   the matched text is shorter. 32 leaves headroom for future table entries. */
+#define MAX_NORMALIZED_LICENSE_LN 32
+
 struct license_type
 {
 	char * text;
@@ -108,9 +115,13 @@ bool license_add_to_list(struct license_list * ptr, char * license, int license_
 	ptr->licenses[ptr->count].text = NULL;
 	ptr->licenses[ptr->count].id = 0;
 
-	/* Allocate with extra padding for CRC32C hardware reads (8-byte blocks) */
+	/* Allocate with extra padding for CRC32C hardware reads (8-byte blocks).
+	   Reserve at least MAX_NORMALIZED_LICENSE_LN so normalize_license() can
+	   replace the text in place with a longer canonical SPDX id without
+	   overflowing the buffer. */
 	size_t len = strlen(license);
-	size_t padded_len = ((len + 8) / 8) * 8;  /* Round up to next 8-byte boundary */
+	size_t capacity = len < MAX_NORMALIZED_LICENSE_LN ? MAX_NORMALIZED_LICENSE_LN : len;
+	size_t padded_len = ((capacity + 8) / 8) * 8;  /* Round up to next 8-byte boundary */
 	ptr->licenses[ptr->count].text = calloc(1, padded_len);
 	if (!ptr->licenses[ptr->count].text)
 		return false;
@@ -199,7 +210,10 @@ void normalize_license(char *license)
 		{
 			if (stricmp(license, token))
 			{
-				strcpy(license, spdx);
+				/* Defensive: never write a canonical id that would not fit the
+				   buffers reserved by license_add_to_list(). */
+				if (spdx && strlen(spdx) < MAX_NORMALIZED_LICENSE_LN)
+					strcpy(license, spdx);
 				return;
 			}
 			token = strtok(NULL, ",");

src/match.c

@@ -784,6 +784,13 @@ bool load_matches(match_data_t *match)
 	records = ldb_fetch_recordset(NULL, oss_file, match->file_md5, false, component_from_file,(void *)&match->component_list);
 	scanlog("Found %d file entries\n", records);
 
+	/* Drop components whose primary OR related purl is in the ignore list (-b).
+	   handle_url_record()/get_oldest_url() only compare the url record's primary
+	   purl, so a project indexed under several purls would otherwise survive under
+	   its other identity and the ignored purl would resurface as a related purl. */
+	if (ignore_components)
+		component_list_remove_ignored(&match->component_list);
+
 	/* Final optimization based on the available information for a component */
 	/* If two components have the date date, select the one with more available information */
 

src/match_list.c

@@ -24,6 +24,38 @@ void component_list_destroy(component_list_t *list)
     }
 }
 
+int component_list_remove_ignored(component_list_t *list)
+{
+    if (!list || !ignore_components)
+        return 0;
+
+    int removed = 0;
+    struct comp_entry *item = list->headp.lh_first;
+    while (item != NULL)
+    {
+        struct comp_entry *next = item->entries.le_next;
+
+        if (component_ignored_by_purl(item->component))
+        {
+            component_data_free(item->component);
+            LIST_REMOVE(item, entries);
+            free(item);
+            list->items--;
+            removed++;
+        }
+        item = next;
+    }
+
+    /* Structural changes invalidate the cached tail pointers kept for inserts */
+    list->last_element = NULL;
+    list->last_element_aux = NULL;
+
+    if (removed)
+        scanlog("Removed %d component(s) matching the ignore list\n", removed);
+
+    return removed;
+}
+
 void component_list_init(component_list_t *comp_list, int max_items)
 {
     LIST_INIT(&comp_list->headp); /* Initialize the list. */

src/scan.c

@@ -98,6 +98,49 @@ void scan_data_free(scan_data_t * scan)
 	scan = NULL;
 }
 
+/** @brief Discards the scan RESULTS (matches, matchmap, best match) while
+    keeping the scan identity, configuration and the reusable hash/line buffers,
+    so the same scan can be re-run over the same file (e.g. a snippet second
+    pass). Mirrors the result-freeing half of scan_data_free().
+    @param scan Scan data */
+static void scan_data_reset(scan_data_t *scan)
+{
+	/* Clear the slots too: biggest_snippet() only creates a list when the slot is NULL */
+	for (int i = 0; i < scan->matches_list_array_index; i++)
+	{
+		match_list_destroy(scan->matches_list_array[i]);
+		scan->matches_list_array[i] = NULL;
+	}
+	scan->matches_list_array_index = 0;
+	memset(scan->matches_list_array_indirection, 0, sizeof(scan->matches_list_array_indirection));
+
+	for (int i = 0; i < scan->matchmap_size; i++)
+		free(scan->matchmap[i].range);
+	free(scan->matchmap);
+	scan->matchmap = NULL;
+	scan->matchmap_size = 0;
+	memset(scan->matchmap_rank_by_sector, 0, sizeof(scan->matchmap_rank_by_sector));
+
+	scan->match_ptr = NULL;
+	scan->best_match = NULL;
+	scan->match_type = MATCH_NONE;
+	scan->printed_succed = false;
+}
+
+/** @brief True if the scan produced at least one match that still has
+    components to report (i.e. it would not be rendered as "none").
+    @param scan Scan data */
+static bool scan_has_reportable_match(scan_data_t *scan)
+{
+	for (int i = 0; i < scan->matches_list_array_index; i++)
+	{
+		match_list_t *list = scan->matches_list_array[i];
+		if (list && list->best_match && list->best_match->component_list.items > 0)
+			return true;
+	}
+	return false;
+}
+
 /** @brief Returns true if md5 is the md5sum for NULL
     @param md5 File ID (md5)
     @result Empty file result
@@ -486,53 +529,83 @@ void ldb_scan(scan_data_t *scan)
 	strcpy(scan->source_md5, tmp_md5_hex);
 	free(tmp_md5_hex);
 
-	/* Look for full file match or url match in ldb */
-	scan->match_type = ldb_scan_file(scan);
+	/* A full-file match whose candidates are all removed by the ignore list
+	   collapses to "none". When that happens we discard the result and retry
+	   once as a snippet scan over the same file, honoring the same ignore/
+	   include lists (force_snippets drives that second pass). */
+	bool force_snippets = false;
+	bool retried = false;
 
-	/* If no match, scan snippets */
-	if (scan->match_type == MATCH_NONE || force_snippet_scan)
+	while (true)
 	{
-		/* Load snippets into scan data */
-		if (!scan->preload)
-		{
-			/* Read file into memory */
-			char *src = calloc(MAX_FILE_SIZE, 1);
-			if (file_size < MAX_FILE_SIZE)
-				read_file(src, scan->file_path, 0);
+		/* Look for full file match or url match in ldb (skipped when forcing snippets) */
+		scan->match_type = force_snippets ? MATCH_NONE : ldb_scan_file(scan);
 
-			/* If HPSM is enable calculate the crc8 line hash calling the shared lib */
-			if (hpsm_enabled)
+		/* If no match (or snippets are forced), scan snippets */
+		if (scan->match_type == MATCH_NONE || force_snippet_scan || force_snippets)
+		{
+			/* Load snippets into scan data */
+			if (!scan->preload)
 			{
-				char *aux = hpsm_hash_file_contents(src);
-				if (aux)
+				/* Read file into memory */
+				char *src = calloc(MAX_FILE_SIZE, 1);
+				if (file_size < MAX_FILE_SIZE)
+					read_file(src, scan->file_path, 0);
+
+				/* If HPSM is enable calculate the crc8 line hash calling the shared lib */
+				if (hpsm_enabled)
 				{
-					hpsm_crc_lines = strdup(&aux[5]);
-					scanlog("HPSM lines CRC: %s\n", hpsm_crc_lines);
-					free(aux);
+					char *aux = hpsm_hash_file_contents(src);
+					if (aux)
+					{
+						hpsm_crc_lines = strdup(&aux[5]);
+						scanlog("HPSM lines CRC: %s\n", hpsm_crc_lines);
+						free(aux);
+					}
 				}
+				/* Determine if file is to skip snippet search */
+				if (!skip_snippets(src, file_size))
+				{ /* Load wfps into scan structure */
+					scan->hash_count = winnowing(src, scan->hashes, scan->lines, MAX_FILE_SIZE);
+					if (scan->hash_count)
+						scan->total_lines = scan->lines[scan->hash_count - 1];
+				}
+				free(src);
 			}
-			/* Determine if file is to skip snippet search */
-			if (!skip_snippets(src, file_size))
-			{ /* Load wfps into scan structure */
-				scan->hash_count = winnowing(src, scan->hashes, scan->lines, MAX_FILE_SIZE);
-				if (scan->hash_count)
-					scan->total_lines = scan->lines[scan->hash_count - 1];
-			}
-			free(src);
+			else if (scan->hash_count)
+				scan->total_lines = scan->lines[scan->hash_count - 1];
+
+			/* Perform snippet scan */
+			if (scan->total_lines)
+				scan->match_type = ldb_scan_snippets(scan);
+
+			else
+				scanlog("File skipped\n");
 		}
-		else if (scan->hash_count)
-			scan->total_lines = scan->lines[scan->hash_count - 1];
 
-		/* Perform snippet scan */
-		if (scan->total_lines)
-			scan->match_type = ldb_scan_snippets(scan);
+		/* Type found by this pass, captured before compile_matches, which resets
+		   match_type to MATCH_NONE when the selected match has no components. */
+		match_t scanned_type = scan->match_type;
 
-		else
-			scanlog("File skipped\n");
-	}
+		/* Compile matches */
+		compile_matches(scan);
+
+		/* Retry once as a snippet scan if a full-file match was entirely
+		   filtered out by the ignore list (otherwise it would print "none"). */
+		if (!retried && !force_snippets
+			&& scanned_type == MATCH_FILE
+			&& ignore_components
+			&& !scan_has_reportable_match(scan))
+		{
+			scanlog("Full-file match filtered out by ignore list; retrying as snippet scan\n");
+			scan_data_reset(scan);
+			force_snippets = true;
+			retried = true;
+			continue;
+		}
 
-	/* Compile matches */
-	compile_matches(scan);
+		break;
+	}
 
 	if (!scan->best_match)
 		scanlog("No best match\n");