From 17dfac26507fc3d6fd1e8fff59a427a89eec7331 Mon Sep 17 00:00:00 2001 From: mscasso-scanoss Date: Mon, 26 Oct 2026 15:48:46 +0100 Subject: [PATCH 1/5] fix(ignore): suppress components ignored by a related purl (#120) The ignore list (-b SBOM.json) was only compared against the single primary purl stored in the url record (field 6, purls[0]). A project is often published under several purls (e.g. a github repo also indexed as a golang / nuget / conda package); those sibling identities are materialised only later by fetch_related_purls() at report time. As a result, ignoring a purl that was a *related* purl of the matched component had no effect: the match survived under its primary identity and the ignored purl resurfaced verbatim in the reported "purl" array. Fix: after load_matches() builds the candidate component list, drop every component whose primary OR related purl is in the ignore list, before best-match selection. A legitimate non-ignored candidate for the same region then wins, or the finding is suppressed entirely when none remain. - ignored_purl(): check a single purl string against the ignore list. - component_ignored_by_purl(): resolve related purls and test them all. - component_list_remove_ignored(): filter the component list in place. - load_matches(): invoke the filter, guarded by `ignore_components` so there is zero overhead when -b is not used. Verified against the ossv2 KB: ignoring pkg:nuget/libcurl-v143-shared (a related purl of pkg:github/curl/curl) now suppresses that component, while a scan without -b is byte-identical to before (only elapsed time differs). valgrind is clean for the new code paths. Adds a data-driven regression test that auto-discovers a multi-purl component and asserts suppression. Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 1eaf9413a826a8f0004436fdf6639ab369aea659) --- inc/component.h | 2 + inc/match_list.h | 1 + src/component.c | 63 +++++++++++++++ src/match.c | 7 ++ src/match_list.c | 32 ++++++++ tests/ignore_related_purl/run.sh | 129 +++++++++++++++++++++++++++++++ 6 files changed, 234 insertions(+) create mode 100755 tests/ignore_related_purl/run.sh diff --git a/inc/component.h b/inc/component.h index 7bd2ef2..37f536c 100644 --- a/inc/component.h +++ b/inc/component.h @@ -108,4 +108,6 @@ int asset_declared(component_data_t * comp); void component_item_free(component_item * comp_item); void fill_component_path(component_data_t *component, char *file_path); char * look_for_version(char *in); +bool ignored_purl(const char *purl); +bool component_ignored_by_purl(component_data_t *component); #endif \ No newline at end of file diff --git a/inc/match_list.h b/inc/match_list.h index de20c11..c08b8bc 100644 --- a/inc/match_list.h +++ b/inc/match_list.h @@ -144,6 +144,7 @@ void component_list_init(component_list_t *comp_list, int max_items); bool component_list_add(component_list_t * list, component_data_t * new_comp, bool (* val) (component_data_t * a, component_data_t * b), bool remove_a); void component_list_print(component_list_t * list, bool (*printer) (component_data_t * fpa), char * separator); void component_list_destroy(component_list_t *list); +int component_list_remove_ignored(component_list_t *list); bool component_list_add_binary(component_list_t *list, component_data_t *new_comp, bool (*val)(component_data_t *a, component_data_t *b), bool remove_a); bool match_list_eval(match_list_t *list, match_data_t * in, bool (*eval)(match_data_t *fpa, match_data_t *fpb)); void match_list_tolerance_set(float in); diff --git a/src/component.c b/src/component.c index 8407d6b..3f705eb 100644 --- a/src/component.c +++ b/src/component.c @@ -34,6 +34,7 @@ #include "util.h" #include "debug.h" #include "query.h" +#include "url.h" /** * @brief Free component object @@ -117,6 +118,33 @@ component_data_t *component_data_copy(component_data_t *in) return out; } +/** + * @brief Return true if the given purl is listed in ignore_components (-b parameter) + * @param purl purl string to test + */ +bool ignored_purl(const char *purl) +{ + if (!ignore_components || !purl) + return false; + + /* Travel ignore_components looking for a purl rule matching this purl */ + for (int i = 0; i < MAX_SBOM_ITEMS; i++) + { + char *dvendor = ignore_components[i].vendor; + char *dcomponent = ignore_components[i].component; + char *dpurl = ignore_components[i].purl; + + /* Exit if reached the end */ + if (!dcomponent && !dvendor && !dpurl) + break; + + if (dpurl && !strcmp(purl, dpurl)) + return true; + } + + return false; +} + /** * @brief Return true if asset is found in ignore_components (-b parameter) * @param url_record pointer to url record @@ -180,6 +208,41 @@ bool ignored_asset_match(uint8_t *url_record) return found; } +/** + * @brief Return true if a component relates to an ignored purl (-b parameter). + * + * ignored_asset_match() only compares the single primary purl stored in the url + * record (field 6). A project may be published under several purls (e.g. a + * github repo mirrored as a golang module), and those sibling identities are + * only materialised later by fetch_related_purls(). This helper resolves the + * related purls and checks every purl of the component against the ignore list, + * so ignoring any one of a project's purls suppresses it regardless of which + * identity ended up as the primary one. + * + * @param component component to test + */ +bool component_ignored_by_purl(component_data_t *component) +{ + if (!ignore_components || !component) + return false; + + /* Resolve sibling purls so we match the project under any of its identities */ + fetch_related_purls(component); + + for (int i = 0; i < MAX_PURLS; i++) + { + if (!component->purls[i]) + break; + if (ignored_purl(component->purls[i])) + { + scanlog("Component ignored by purl %s\n", component->purls[i]); + return true; + } + } + + return false; +} + char * look_for_version(char *in) { if (!in) diff --git a/src/match.c b/src/match.c index 6ae56c7..db8199e 100644 --- a/src/match.c +++ b/src/match.c @@ -784,6 +784,13 @@ bool load_matches(match_data_t *match) records = ldb_fetch_recordset(NULL, oss_file, match->file_md5, false, component_from_file,(void *)&match->component_list); scanlog("Found %d file entries\n", records); + /* Drop components whose primary OR related purl is in the ignore list (-b). + handle_url_record()/get_oldest_url() only compare the url record's primary + purl, so a project indexed under several purls would otherwise survive under + its other identity and the ignored purl would resurface as a related purl. */ + if (ignore_components) + component_list_remove_ignored(&match->component_list); + /* Final optimization based on the available information for a component */ /* If two components have the date date, select the one with more available information */ diff --git a/src/match_list.c b/src/match_list.c index 596a031..b0b400a 100644 --- a/src/match_list.c +++ b/src/match_list.c @@ -24,6 +24,38 @@ void component_list_destroy(component_list_t *list) } } +int component_list_remove_ignored(component_list_t *list) +{ + if (!list || !ignore_components) + return 0; + + int removed = 0; + struct comp_entry *item = list->headp.lh_first; + while (item != NULL) + { + struct comp_entry *next = item->entries.le_next; + + if (component_ignored_by_purl(item->component)) + { + component_data_free(item->component); + LIST_REMOVE(item, entries); + free(item); + list->items--; + removed++; + } + item = next; + } + + /* Structural changes invalidate the cached tail pointers kept for inserts */ + list->last_element = NULL; + list->last_element_aux = NULL; + + if (removed) + scanlog("Removed %d component(s) matching the ignore list\n", removed); + + return removed; +} + void component_list_init(component_list_t *comp_list, int max_items) { LIST_INIT(&comp_list->headp); /* Initialize the list. */ diff --git a/tests/ignore_related_purl/run.sh b/tests/ignore_related_purl/run.sh new file mode 100755 index 0000000..f04c2ef --- /dev/null +++ b/tests/ignore_related_purl/run.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# +# Regression test: ignoring a related PURL (-b) must suppress the component. +# +# Guards issue #120: the ignore list (-b SBOM.json) was only compared against +# the single primary purl stored in the url record (field 6). A project may be +# published under several purls (e.g. a github repo also indexed as a golang/ +# nuget/conda package); those sibling identities are only materialised later by +# fetch_related_purls(). So ignoring a purl that happened to be a *related* purl +# had no effect: the match survived under its primary identity and the ignored +# purl resurfaced in the reported "purl" array. +# +# The fix (component_list_remove_ignored / component_ignored_by_purl) drops a +# component when ANY of its purls -- primary OR related -- is in the ignore list. +# +# This test is data-driven so it does not hard-code KB contents: it peels the +# top candidates of a region until a component exposing >= 2 purls surfaces, +# then asks the engine to ignore one of that component's *related* purls and +# asserts the whole component disappears. It needs an OSS knowledge base; if +# none is reachable (or no multi-purl component is found) it SKIPS (exit 0) so +# it never breaks CI images that ship without KB data. +# +# SCANOSS_BIN path to the engine binary (default: ../../scanoss) +# SCANOSS_KB KB/database name for -n (default: ossv2) +# +set -u + +HERE="$(cd "$(dirname "$0")" && pwd)" +BIN="${SCANOSS_BIN:-$HERE/../../scanoss}" +KB="${SCANOSS_KB:-ossv2}" +WFP="$HERE/../snippet_multicomponent/main.c.wfp" + +if [ ! -x "$BIN" ]; then + echo "SKIP: engine binary not found at $BIN (set SCANOSS_BIN)" + exit 0 +fi +if [ ! -f "$WFP" ]; then + echo "SKIP: input WFP not found at $WFP" + exit 0 +fi + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +# scan [purl ...] -> prints engine JSON for the WFP, ignoring the given purls. +scan() { + local ign="$TMP/ignore.json" first=1 p + { + printf '{"components":[' + for p in "$@"; do + [ $first -eq 1 ] || printf ',' + printf '{"purl":"%s"}' "$p" + first=0 + done + printf ']}' + } > "$ign" + if [ "$#" -eq 0 ]; then + "$BIN" -n "$KB" -M 9 -w "$WFP" 2>/dev/null + else + "$BIN" -n "$KB" -M 9 -b "$ign" -w "$WFP" 2>/dev/null + fi +} + +# first_multi_purl_array -> the first "purl":[...] array holding >= 2 purls. +first_multi_purl_array() { + printf '%s' "$1" | grep -o '"purl": \[[^]]*\]' | grep ',' | head -1 +} + +# first_primary_purl -> the primary purl (purls[0]) of the first component. +first_primary_purl() { + printf '%s' "$1" | grep -o '"purl": \[[^]]*\]' | head -1 | grep -o 'pkg:[^"]*' | head -1 +} + +# Probe: make sure the KB answers at all, otherwise skip rather than fail. +PROBE="$(scan)" +if ! printf '%s' "$PROBE" | grep -q '"id": "snippet"'; then + echo "SKIP: KB '$KB' produced no snippet match (KB missing or not loaded)" + exit 0 +fi + +# Peel the top candidate of the region until a component with >= 2 purls shows up. +IGN=() +ARR="" +for _ in 1 2 3 4 5 6 7 8; do + OUT="$(scan "${IGN[@]}")" + ARR="$(first_multi_purl_array "$OUT")" + [ -n "$ARR" ] && break + P="$(first_primary_purl "$OUT")" + [ -z "$P" ] && break # no more matches to peel + IGN+=("$P") +done + +if [ -z "$ARR" ]; then + echo "SKIP: no reachable component with related purls in KB '$KB' (cannot exercise #120)" + exit 0 +fi + +# The array is "pkg:A","pkg:B",... -- primary is the first, related ones follow. +mapfile -t PURLS < <(printf '%s' "$ARR" | grep -o 'pkg:[^"]*') +PRIMARY="${PURLS[0]}" +RELATED="${PURLS[-1]}" # a sibling purl, only present via fetch_related_purls + +if [ "$RELATED" = "$PRIMARY" ]; then + echo "SKIP: could not isolate a related purl distinct from the primary" + exit 0 +fi + +echo "INFO: component primary=$PRIMARY related=$RELATED (ignoring the related one)" + +# Ignoring the RELATED purl must drop the whole component. Pre-fix this had no +# effect because only the primary purl (field 6) was ever checked. +OUT_TEST="$(scan "${IGN[@]}" "$RELATED")" + +FAILED=0 +if printf '%s' "$OUT_TEST" | grep -qF "$RELATED"; then + echo "FAIL: ignored related purl still present in output: $RELATED" + FAILED=1 +fi +if printf '%s' "$OUT_TEST" | grep -qF "$PRIMARY"; then + echo "FAIL: component not suppressed; its primary purl still present: $PRIMARY" + FAILED=1 +fi + +if [ "$FAILED" -eq 0 ]; then + echo "PASS: ignoring a related purl suppresses the component (#120)" + exit 0 +fi +echo "FAIL: related-purl ignore regression (#120)" +exit 1 From 5dcc5055fd8fcac79a503678eef4e222f397ba9c Mon Sep 17 00:00:00 2001 From: mscasso-scanoss Date: Mon, 26 Oct 2026 15:54:38 +0100 Subject: [PATCH 2/5] fix(license): prevent buffer overflow in normalize_license() normalize_license() replaces a license string in place with its canonical SPDX id (e.g. "Apache" -> "Apache-2.0", "GPLv2" -> "GPL-2.0-only"), which can be LONGER than the matched text. License buffers were allocated by license_add_to_list() sized to the ORIGINAL string length (rounded up to an 8-byte boundary), so normalizing a short alias into a longer canonical id wrote past the end of the buffer. valgrind flagged this as an "Invalid write of size 1" in strcpy (license.c) followed by "Invalid read" in string_clean() walking off the end of the corrupted, non-terminated buffer. It reproduces whenever a reported component declares a license whose canonical SPDX id is longer than its stored alias. Fix: - license_add_to_list(): reserve at least MAX_NORMALIZED_LICENSE_LN (32) bytes so any canonical id in license_normalization[] (longest is "LGPL-3.0-or-later", 17 chars) fits even when the original text is shorter. - normalize_license(): defensively skip the in-place copy if the canonical id would not fit, guarding against future table entries. Verified with valgrind on the same scan: the Invalid write/read in the license path are gone, normalization still emits the correct SPDX ids (e.g. GPL-2.0-only, Apache-2.0), and a scan without -b is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 1e31105cd070377442dfa3a942f33398f8a7f5d2) --- src/license.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/license.c b/src/license.c index 8b6b9ae..d2f64b4 100644 --- a/src/license.c +++ b/src/license.c @@ -46,6 +46,13 @@ bool full_license_report = false; +/* normalize_license() may replace a license string in place with its canonical + SPDX id (e.g. "Apache" -> "Apache-2.0"), which can be LONGER than the original + text. License buffers must therefore be sized to hold the longest canonical id + in license_normalization[] (currently "LGPL-3.0-or-later", 17 chars) even when + the matched text is shorter. 32 leaves headroom for future table entries. */ +#define MAX_NORMALIZED_LICENSE_LN 32 + struct license_type { char * text; @@ -108,9 +115,13 @@ bool license_add_to_list(struct license_list * ptr, char * license, int license_ ptr->licenses[ptr->count].text = NULL; ptr->licenses[ptr->count].id = 0; - /* Allocate with extra padding for CRC32C hardware reads (8-byte blocks) */ + /* Allocate with extra padding for CRC32C hardware reads (8-byte blocks). + Reserve at least MAX_NORMALIZED_LICENSE_LN so normalize_license() can + replace the text in place with a longer canonical SPDX id without + overflowing the buffer. */ size_t len = strlen(license); - size_t padded_len = ((len + 8) / 8) * 8; /* Round up to next 8-byte boundary */ + size_t capacity = len < MAX_NORMALIZED_LICENSE_LN ? MAX_NORMALIZED_LICENSE_LN : len; + size_t padded_len = ((capacity + 8) / 8) * 8; /* Round up to next 8-byte boundary */ ptr->licenses[ptr->count].text = calloc(1, padded_len); if (!ptr->licenses[ptr->count].text) return false; @@ -199,7 +210,10 @@ void normalize_license(char *license) { if (stricmp(license, token)) { - strcpy(license, spdx); + /* Defensive: never write a canonical id that would not fit the + buffers reserved by license_add_to_list(). */ + if (spdx && strlen(spdx) < MAX_NORMALIZED_LICENSE_LN) + strcpy(license, spdx); return; } token = strtok(NULL, ","); From e14f3b7f4489e774b1867f21fc2e6be025198e21 Mon Sep 17 00:00:00 2001 From: mscasso-scanoss Date: Mon, 26 Oct 2026 16:42:19 +0100 Subject: [PATCH 3/5] feat(scan): fall back to snippet scan when a full-file match is fully ignored When a file produces a full-file match (MATCH_FILE) but every candidate component is removed by the ignore list (-b), the result used to collapse to "none": the winnowing/snippet pass is skipped for file matches, so there was nothing left to report once the components were filtered. The engine now detects that case and automatically retries once as a fresh snippet scan over the same file, honoring the same ignore/include lists, so a legitimate snippet match can still surface instead of reporting nothing. Implementation (src/scan.c): - scan_data_reset(): discards the scan RESULTS (matches, matchmap, best match) while keeping the scan identity, config and the reusable hash/line buffers, mirroring the result-freeing half of scan_data_free(). This is the "throw the results away and start fresh" step. - scan_has_reportable_match(): true if any match still has components. - ldb_scan(): the file/snippet/compile body now runs in a loop. If the first pass was a MATCH_FILE, an ignore list is active, and nothing is reportable, it resets and re-runs forcing snippets (once, guarded against re-entry). The trigger is captured before compile_matches(), which resets match_type to MATCH_NONE once the collapsed match has no components. Scope and safety: - Zero behavior change unless -b is used AND a full-file match is fully filtered: verified byte-identical output otherwise. - Verified against the ossv2 KB: a full-file curl match whose purls are all ignored now falls back to a snippet match instead of "none"; valgrind shows no new leaks or invalid accesses from the reset/second pass. - Adds tests/snippet_fallback_on_ignore/ (data-driven, skips without a KB; confirmed to fail pre-change and pass after). Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 263505c173bc3608422efaa031228c84f1dd2049) --- src/scan.c | 141 ++++++++++++++++++------ tests/snippet_fallback_on_ignore/run.sh | 115 +++++++++++++++++++ 2 files changed, 220 insertions(+), 36 deletions(-) create mode 100755 tests/snippet_fallback_on_ignore/run.sh diff --git a/src/scan.c b/src/scan.c index 5a04163..ddd8ce7 100644 --- a/src/scan.c +++ b/src/scan.c @@ -98,6 +98,45 @@ void scan_data_free(scan_data_t * scan) scan = NULL; } +/** @brief Discards the scan RESULTS (matches, matchmap, best match) while + keeping the scan identity, configuration and the reusable hash/line buffers, + so the same scan can be re-run over the same file (e.g. a snippet second + pass). Mirrors the result-freeing half of scan_data_free(). + @param scan Scan data */ +static void scan_data_reset(scan_data_t *scan) +{ + for (int i = 0; i < scan->matches_list_array_index; i++) + match_list_destroy(scan->matches_list_array[i]); + scan->matches_list_array_index = 0; + memset(scan->matches_list_array_indirection, 0, sizeof(scan->matches_list_array_indirection)); + + for (int i = 0; i < scan->matchmap_size; i++) + free(scan->matchmap[i].range); + free(scan->matchmap); + scan->matchmap = NULL; + scan->matchmap_size = 0; + memset(scan->matchmap_rank_by_sector, 0, sizeof(scan->matchmap_rank_by_sector)); + + scan->match_ptr = NULL; + scan->best_match = NULL; + scan->match_type = MATCH_NONE; + scan->printed_succed = false; +} + +/** @brief True if the scan produced at least one match that still has + components to report (i.e. it would not be rendered as "none"). + @param scan Scan data */ +static bool scan_has_reportable_match(scan_data_t *scan) +{ + for (int i = 0; i < scan->matches_list_array_index; i++) + { + match_list_t *list = scan->matches_list_array[i]; + if (list && list->best_match && list->best_match->component_list.items > 0) + return true; + } + return false; +} + /** @brief Returns true if md5 is the md5sum for NULL @param md5 File ID (md5) @result Empty file result @@ -486,53 +525,83 @@ void ldb_scan(scan_data_t *scan) strcpy(scan->source_md5, tmp_md5_hex); free(tmp_md5_hex); - /* Look for full file match or url match in ldb */ - scan->match_type = ldb_scan_file(scan); + /* A full-file match whose candidates are all removed by the ignore list + collapses to "none". When that happens we discard the result and retry + once as a snippet scan over the same file, honoring the same ignore/ + include lists (force_snippets drives that second pass). */ + bool force_snippets = false; + bool retried = false; - /* If no match, scan snippets */ - if (scan->match_type == MATCH_NONE || force_snippet_scan) + while (true) { - /* Load snippets into scan data */ - if (!scan->preload) - { - /* Read file into memory */ - char *src = calloc(MAX_FILE_SIZE, 1); - if (file_size < MAX_FILE_SIZE) - read_file(src, scan->file_path, 0); + /* Look for full file match or url match in ldb (skipped when forcing snippets) */ + scan->match_type = force_snippets ? MATCH_NONE : ldb_scan_file(scan); - /* If HPSM is enable calculate the crc8 line hash calling the shared lib */ - if (hpsm_enabled) + /* If no match (or snippets are forced), scan snippets */ + if (scan->match_type == MATCH_NONE || force_snippet_scan || force_snippets) + { + /* Load snippets into scan data */ + if (!scan->preload) { - char *aux = hpsm_hash_file_contents(src); - if (aux) + /* Read file into memory */ + char *src = calloc(MAX_FILE_SIZE, 1); + if (file_size < MAX_FILE_SIZE) + read_file(src, scan->file_path, 0); + + /* If HPSM is enable calculate the crc8 line hash calling the shared lib */ + if (hpsm_enabled) { - hpsm_crc_lines = strdup(&aux[5]); - scanlog("HPSM lines CRC: %s\n", hpsm_crc_lines); - free(aux); + char *aux = hpsm_hash_file_contents(src); + if (aux) + { + hpsm_crc_lines = strdup(&aux[5]); + scanlog("HPSM lines CRC: %s\n", hpsm_crc_lines); + free(aux); + } } + /* Determine if file is to skip snippet search */ + if (!skip_snippets(src, file_size)) + { /* Load wfps into scan structure */ + scan->hash_count = winnowing(src, scan->hashes, scan->lines, MAX_FILE_SIZE); + if (scan->hash_count) + scan->total_lines = scan->lines[scan->hash_count - 1]; + } + free(src); } - /* Determine if file is to skip snippet search */ - if (!skip_snippets(src, file_size)) - { /* Load wfps into scan structure */ - scan->hash_count = winnowing(src, scan->hashes, scan->lines, MAX_FILE_SIZE); - if (scan->hash_count) - scan->total_lines = scan->lines[scan->hash_count - 1]; - } - free(src); + else if (scan->hash_count) + scan->total_lines = scan->lines[scan->hash_count - 1]; + + /* Perform snippet scan */ + if (scan->total_lines) + scan->match_type = ldb_scan_snippets(scan); + + else + scanlog("File skipped\n"); } - else if (scan->hash_count) - scan->total_lines = scan->lines[scan->hash_count - 1]; - /* Perform snippet scan */ - if (scan->total_lines) - scan->match_type = ldb_scan_snippets(scan); + /* Type found by this pass, captured before compile_matches, which resets + match_type to MATCH_NONE when the selected match has no components. */ + match_t scanned_type = scan->match_type; - else - scanlog("File skipped\n"); - } + /* Compile matches */ + compile_matches(scan); - /* Compile matches */ - compile_matches(scan); + /* Retry once as a snippet scan if a full-file match was entirely + filtered out by the ignore list (otherwise it would print "none"). */ + if (!retried && !force_snippets + && scanned_type == MATCH_FILE + && ignore_components + && !scan_has_reportable_match(scan)) + { + scanlog("Full-file match filtered out by ignore list; retrying as snippet scan\n"); + scan_data_reset(scan); + force_snippets = true; + retried = true; + continue; + } + + break; + } if (!scan->best_match) scanlog("No best match\n"); diff --git a/tests/snippet_fallback_on_ignore/run.sh b/tests/snippet_fallback_on_ignore/run.sh new file mode 100755 index 0000000..b742277 --- /dev/null +++ b/tests/snippet_fallback_on_ignore/run.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# +# Regression test: snippet fallback when a full-file match is fully ignored. +# +# When a file produces a full-file (MATCH_FILE) result but every candidate +# component is removed by the ignore list (-b), the match used to collapse to +# "none". The engine now discards that result and automatically retries once as +# a snippet scan over the same file, honoring the same ignore/include lists, so +# a legitimate snippet match can still surface. +# +# The test is data-driven so it does not hard-code KB contents: +# 1. Derive a real full-file hash from a snippet match against main.c.wfp. +# 2. Build a WFP whose file= hash triggers that full-file match, carrying the +# same snippet hashes. +# 3. Peel the full-file components (ignore them one by one) until the result +# would collapse to "none". +# 4. With all of them ignored, assert the result is a SNIPPET match, not none. +# +# Needs an OSS knowledge base; if none is reachable (or the scenario cannot be +# built from the KB) it SKIPS (exit 0) so it never breaks KB-less CI images. +# +# SCANOSS_BIN path to the engine binary (default: ../../scanoss) +# SCANOSS_KB KB/database name for -n (default: ossv2) +# +set -u + +HERE="$(cd "$(dirname "$0")" && pwd)" +BIN="${SCANOSS_BIN:-$HERE/../../scanoss}" +KB="${SCANOSS_KB:-ossv2}" +WFP_SRC="$HERE/../snippet_multicomponent/main.c.wfp" + +if [ ! -x "$BIN" ]; then + echo "SKIP: engine binary not found at $BIN (set SCANOSS_BIN)" + exit 0 +fi +if [ ! -f "$WFP_SRC" ]; then + echo "SKIP: input WFP not found at $WFP_SRC" + exit 0 +fi + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +# ignore_json -> path to a -b SBOM.json ignoring the given purls. +ignore_json() { + local out="$TMP/ignore.json" first=1 p + { + printf '{"components":[' + for p in "$@"; do + [ $first -eq 1 ] || printf ',' + printf '{"purl":"%s"}' "$p"; first=0 + done + printf ']}' + } > "$out" + echo "$out" +} + +match_id() { printf '%s' "$1" | grep -oE '"id": "[a-z]+"' | head -1 | grep -oE '[a-z]+"$' | tr -d '"'; } +first_purl() { printf '%s' "$1" | grep -o '"purl": \[[^]]*\]' | head -1 | grep -o 'pkg:[^"]*' | head -1; } + +# 1. Find a real KB full-file hash via a snippet match on the source WFP. +PROBE="$("$BIN" -n "$KB" -w "$WFP_SRC" 2>/dev/null)" +FHASH="$(printf '%s' "$PROBE" | grep -oE '"file_hash": "[0-9a-f]+"' | head -1 | grep -oE '[0-9a-f]+' | tail -1)" +if [ -z "$FHASH" ]; then + echo "SKIP: KB '$KB' produced no snippet match to derive a full-file hash" + exit 0 +fi + +# 2. Build a WFP whose file= hash is that KB file (full-file match), padded to +# 32 hex chars; the engine reads the leading oss_file.key_ln*2 chars. +FHASH_PAD="$(printf '%-32s' "$FHASH" | tr ' ' '0')" +WFP="$TMP/fullfile.wfp" +{ echo "file=${FHASH_PAD},9999,test.c"; tail -n +2 "$WFP_SRC"; } > "$WFP" + +# Confirm it is actually a full-file match, otherwise the scenario does not apply. +BASE="$("$BIN" -n "$KB" -w "$WFP" 2>/dev/null)" +if [ "$(match_id "$BASE")" != "file" ]; then + echo "SKIP: derived WFP did not produce a full-file match in KB '$KB'" + exit 0 +fi + +# 3. Peel the full-file components until the match would collapse to none. +IGN=() +COLLAPSED=0 +for _ in 1 2 3 4 5 6 7 8; do + if [ ${#IGN[@]} -eq 0 ]; then + OUT="$BASE" + else + OUT="$("$BIN" -n "$KB" -b "$(ignore_json "${IGN[@]}")" -w "$WFP" 2>/dev/null)" + fi + ID="$(match_id "$OUT")" + [ "$ID" != "file" ] && { COLLAPSED=1; break; } # no more file components + P="$(first_purl "$OUT")" + [ -z "$P" ] && break + IGN+=("$P") +done + +if [ "$COLLAPSED" -ne 1 ] || [ ${#IGN[@]} -eq 0 ]; then + echo "SKIP: could not reduce the full-file match to a single ignorable component set" + exit 0 +fi + +echo "INFO: full-file components ignored: ${IGN[*]}" + +# 4. With every full-file purl ignored, the result must fall back to a snippet +# match instead of collapsing to "none". +OUT="$("$BIN" -n "$KB" -b "$(ignore_json "${IGN[@]}")" -w "$WFP" 2>/dev/null)" +ID="$(match_id "$OUT")" + +if [ "$ID" = "snippet" ]; then + echo "PASS: fully-ignored full-file match falls back to a snippet scan" + exit 0 +fi +echo "FAIL: expected a snippet fallback, got id='$ID' (regression: #120 full-file fallback)" +exit 1 From 22d43acab4e61f8b6ddbd7df2637f3d036619d2d Mon Sep 17 00:00:00 2001 From: mariano scasso Date: Tue, 27 Oct 2026 14:21:07 +0100 Subject: [PATCH 4/5] fix(scan): clear match list slots on snippet fallback reset; make ignore tests runnable scan_data_reset() freed each matches_list_array[i] but left the dangling pointer in place. On the snippet second pass biggest_snippet() only creates a list for NULL slots, so it wrote into freed lists (use-after-free, segfault when a full-file match was fully filtered by -b). Tests: - add tests/snippet_multicomponent/main.c.wfp (from the crc64 line), the input both ignore tests depend on; it was missing so they always skipped. - ignore_related_purl: drop -M 9, which hits a pre-existing -M crash on main. - snippet_fallback_on_ignore: try a known low-copy full-file hash when the one derived from the probe cannot be peeled (curl has dozens of copies). Co-Authored-By: Claude Opus 5.5 --- src/scan.c | 4 + tests/ignore_related_purl/run.sh | 4 +- tests/snippet_fallback_on_ignore/run.sh | 65 +++++++------- tests/snippet_multicomponent/main.c.wfp | 110 ++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 31 deletions(-) create mode 100644 tests/snippet_multicomponent/main.c.wfp diff --git a/src/scan.c b/src/scan.c index ddd8ce7..9a18003 100644 --- a/src/scan.c +++ b/src/scan.c @@ -105,8 +105,12 @@ void scan_data_free(scan_data_t * scan) @param scan Scan data */ static void scan_data_reset(scan_data_t *scan) { + /* Clear the slots too: biggest_snippet() only creates a list when the slot is NULL */ for (int i = 0; i < scan->matches_list_array_index; i++) + { match_list_destroy(scan->matches_list_array[i]); + scan->matches_list_array[i] = NULL; + } scan->matches_list_array_index = 0; memset(scan->matches_list_array_indirection, 0, sizeof(scan->matches_list_array_indirection)); diff --git a/tests/ignore_related_purl/run.sh b/tests/ignore_related_purl/run.sh index f04c2ef..15e823c 100755 --- a/tests/ignore_related_purl/run.sh +++ b/tests/ignore_related_purl/run.sh @@ -55,9 +55,9 @@ scan() { printf ']}' } > "$ign" if [ "$#" -eq 0 ]; then - "$BIN" -n "$KB" -M 9 -w "$WFP" 2>/dev/null + "$BIN" -n "$KB" -w "$WFP" 2>/dev/null else - "$BIN" -n "$KB" -M 9 -b "$ign" -w "$WFP" 2>/dev/null + "$BIN" -n "$KB" -b "$ign" -w "$WFP" 2>/dev/null fi } diff --git a/tests/snippet_fallback_on_ignore/run.sh b/tests/snippet_fallback_on_ignore/run.sh index b742277..2bca533 100755 --- a/tests/snippet_fallback_on_ignore/run.sh +++ b/tests/snippet_fallback_on_ignore/run.sh @@ -9,7 +9,8 @@ # a legitimate snippet match can still surface. # # The test is data-driven so it does not hard-code KB contents: -# 1. Derive a real full-file hash from a snippet match against main.c.wfp. +# 1. Pick a real full-file hash: the one behind a snippet match against +# main.c.wfp, or a known low-copy KB file if that one cannot be peeled. # 2. Build a WFP whose file= hash triggers that full-file match, carrying the # same snippet hashes. # 3. Peel the full-file components (ignore them one by one) until the result @@ -58,45 +59,51 @@ ignore_json() { match_id() { printf '%s' "$1" | grep -oE '"id": "[a-z]+"' | head -1 | grep -oE '[a-z]+"$' | tr -d '"'; } first_purl() { printf '%s' "$1" | grep -o '"purl": \[[^]]*\]' | head -1 | grep -o 'pkg:[^"]*' | head -1; } -# 1. Find a real KB full-file hash via a snippet match on the source WFP. +# 1. Candidate full-file hashes: first the KB file behind the snippet match on +# the source WFP, then known files with few copies in the KB (a heavily +# vendored file such as curl's can have dozens of components to peel). +# 661e423d... is src/vulnerability.c from pkg:github/scanoss/engine. PROBE="$("$BIN" -n "$KB" -w "$WFP_SRC" 2>/dev/null)" FHASH="$(printf '%s' "$PROBE" | grep -oE '"file_hash": "[0-9a-f]+"' | head -1 | grep -oE '[0-9a-f]+' | tail -1)" if [ -z "$FHASH" ]; then - echo "SKIP: KB '$KB' produced no snippet match to derive a full-file hash" + echo "SKIP: KB '$KB' produced no snippet match (KB missing or not loaded)" exit 0 fi +CANDIDATES=("$FHASH" 661e423d19593de5e10db93add61c5c5) -# 2. Build a WFP whose file= hash is that KB file (full-file match), padded to -# 32 hex chars; the engine reads the leading oss_file.key_ln*2 chars. -FHASH_PAD="$(printf '%-32s' "$FHASH" | tr ' ' '0')" WFP="$TMP/fullfile.wfp" -{ echo "file=${FHASH_PAD},9999,test.c"; tail -n +2 "$WFP_SRC"; } > "$WFP" +COLLAPSED=0 +for CAND in "${CANDIDATES[@]}"; do + # 2. Build a WFP whose file= hash is that KB file (full-file match), padded to + # 32 hex chars, carrying the source snippet hashes; the engine reads the + # leading oss_file.key_ln*2 chars. + FHASH_PAD="$(printf '%-32s' "$CAND" | tr ' ' '0')" + { echo "file=${FHASH_PAD},9999,test.c"; tail -n +2 "$WFP_SRC"; } > "$WFP" -# Confirm it is actually a full-file match, otherwise the scenario does not apply. -BASE="$("$BIN" -n "$KB" -w "$WFP" 2>/dev/null)" -if [ "$(match_id "$BASE")" != "file" ]; then - echo "SKIP: derived WFP did not produce a full-file match in KB '$KB'" - exit 0 -fi + # Confirm it is actually a full-file match, otherwise try the next candidate. + BASE="$("$BIN" -n "$KB" -w "$WFP" 2>/dev/null)" + [ "$(match_id "$BASE")" != "file" ] && continue -# 3. Peel the full-file components until the match would collapse to none. -IGN=() -COLLAPSED=0 -for _ in 1 2 3 4 5 6 7 8; do - if [ ${#IGN[@]} -eq 0 ]; then - OUT="$BASE" - else - OUT="$("$BIN" -n "$KB" -b "$(ignore_json "${IGN[@]}")" -w "$WFP" 2>/dev/null)" - fi - ID="$(match_id "$OUT")" - [ "$ID" != "file" ] && { COLLAPSED=1; break; } # no more file components - P="$(first_purl "$OUT")" - [ -z "$P" ] && break - IGN+=("$P") + # 3. Peel the full-file components until the match would collapse to none. + IGN=() + for _ in 1 2 3 4 5 6 7 8; do + if [ ${#IGN[@]} -eq 0 ]; then + OUT="$BASE" + else + OUT="$("$BIN" -n "$KB" -b "$(ignore_json "${IGN[@]}")" -w "$WFP" 2>/dev/null)" + fi + ID="$(match_id "$OUT")" + [ "$ID" != "file" ] && { COLLAPSED=1; break; } # no more file components + P="$(first_purl "$OUT")" + [ -z "$P" ] && break + IGN+=("$P") + done + [ "$COLLAPSED" -eq 1 ] && [ ${#IGN[@]} -gt 0 ] && break + COLLAPSED=0 done -if [ "$COLLAPSED" -ne 1 ] || [ ${#IGN[@]} -eq 0 ]; then - echo "SKIP: could not reduce the full-file match to a single ignorable component set" +if [ "$COLLAPSED" -ne 1 ]; then + echo "SKIP: no candidate full-file match could be reduced to an ignorable component set in KB '$KB'" exit 0 fi diff --git a/tests/snippet_multicomponent/main.c.wfp b/tests/snippet_multicomponent/main.c.wfp new file mode 100644 index 0000000..7f7089e --- /dev/null +++ b/tests/snippet_multicomponent/main.c.wfp @@ -0,0 +1,110 @@ +file=3bb103a46f94ed298d8c37b4a80d973c,9918,main.c +7=e795bac8 +10=6e7bf056 +14=16579738 +25=23d6d030 +28=d32e499d +30=09e0aecd,b7908937,21949e96 +33=d400f39f +37=ef7d6cba,894d3fa4,2d1597e8 +41=c848f202 +44=be39f156 +59=905e02fb +61=b7908937 +67=18b3beaf +76=491ac1ea +81=06fbfde3 +83=95c75ef3 +84=d400f39f +88=b036ba15 +95=c848f202,be39f156 +111=c711d2e1 +112=447c4417 +114=9a106044,12ab20bb,0ce7ece1 +115=151c9088,d434c84e,bac49cc6 +116=2a6b8b81,1f38d8be +117=7001190b +119=25c9f85f,a6453c32,e9e65a61 +120=4186b77b,ef9fe105 +124=da55d5b9,b0924155 +128=b378b9e0 +129=addfb6b0 +134=0dbb599f,ec33702f +135=80a48c6a +136=f4347ad7,d5fbdf3a +137=7ce444f8 +139=137b0602,f12edbd6 +142=3dd980ae,ac0b0110 +144=a76b34a5 +148=e0fa1c18,c741c7b2 +151=193cc2c3 +152=79eee8b8,b2e74c6e +156=7b280e25,d864c721 +157=216caf87 +158=a6a3bbe1,b66372cb +159=04ca6169,a7452328 +162=75fa5176 +163=d0cb6c6b,abafa52d +167=9396f071 +168=30792053 +169=4b7b9a0a +171=e6489ee3,8c952f0c +172=b9cded56 +174=2d95465b +176=11781789 +178=bd14a25f,f67df255 +180=46491e29 +183=c29128bc +184=5625dd8a,30792053 +185=4b7b9a0a +187=dd6588d4,0a041ed0 +188=341c8c04 +189=e3cc063a +193=bea1b86c,802ddd80 +196=f537a752,5218fbf7 +198=f21eefbd +199=c2b9b9f9,9f6d2f6f +201=0724e239 +203=1ec0629c +206=8dbf6363 +207=771caaeb +212=35ac838c +214=f94d5b4c,a169fcbf +216=4e6c1cbe +217=4666a13c +219=077eccf8 +222=0ae64d50 +224=9712f0ff,4d24ea64 +228=78ef004d,46c56394,07bc7452 +230=97b52f82 +232=e3af2023 +236=161a335f +237=f6a61105 +241=273539a2,08dcf25a,ac7340d8 +246=d0c75748 +248=061baf8c +250=f941efa7 +253=36d3594e +255=af8cf5fb,40b333f4 +259=8b1ff8bf,6cf0d5ac +260=b803e9eb +264=3cc8ad46 +265=74d8a41f +269=0547b4df,b846d5c2 +272=3926ed53,4bf276e9,29af2a76 +274=fdb2863f,6b99be79 +275=6bc888a0 +278=05bf9646 +281=48848bae +284=4d6fcc13 +285=cfc7ff90 +288=ab7fc8cd,fb85472f +291=64f5cde0,0d2eca0b +292=791fbabd +296=07cfc5fe +297=357fa0d4,87642ec3 +299=16e81aa2 +301=0acd0d5d,085f6061,c727d6af +303=ac932df0 +306=0698ae44 +308=3dbbeabb,5919152f From 4917926138e7a7cdb3106605f8a38d3dab87d269 Mon Sep 17 00:00:00 2001 From: mariano scasso Date: Tue, 27 Oct 2026 14:24:17 +0100 Subject: [PATCH 5/5] update version to 5.4.28 Co-Authored-By: Claude Opus 5.5 --- inc/scanoss.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/scanoss.h b/inc/scanoss.h index c9f8ef3..e041ca0 100644 --- a/inc/scanoss.h +++ b/inc/scanoss.h @@ -33,7 +33,7 @@ #define WFP_LN 4 #define WFP_REC_LN 18 -#define SCANOSS_VERSION "5.4.27" +#define SCANOSS_VERSION "5.4.28" /* Log files */ #define SCAN_LOG "/tmp/scanoss_scan.log"