diff --git a/inc/component.h b/inc/component.h index 7bd2ef2..37f536c 100644 --- a/inc/component.h +++ b/inc/component.h @@ -108,4 +108,6 @@ int asset_declared(component_data_t * comp); void component_item_free(component_item * comp_item); void fill_component_path(component_data_t *component, char *file_path); char * look_for_version(char *in); +bool ignored_purl(const char *purl); +bool component_ignored_by_purl(component_data_t *component); #endif \ No newline at end of file diff --git a/inc/match_list.h b/inc/match_list.h index c45f6f6..5ee77a2 100644 --- a/inc/match_list.h +++ b/inc/match_list.h @@ -158,6 +158,7 @@ void component_list_init(component_list_t *comp_list, int max_items); bool component_list_add(component_list_t * list, component_data_t * new_comp, bool (* val) (component_data_t * a, component_data_t * b), bool remove_a); void component_list_print(component_list_t * list, bool (*printer) (component_data_t * fpa), char * separator); void component_list_destroy(component_list_t *list); +int component_list_remove_ignored(component_list_t *list); bool component_list_add_binary(component_list_t *list, component_data_t *new_comp, bool (*val)(component_data_t *a, component_data_t *b), bool remove_a); void component_list_sort(struct comp_entry *np, bool (*val)(component_data_t *a, component_data_t *b)); list_update_t component_list_update(component_list_t *list, component_data_t *in, list_update_t (*eval)(component_data_t *fpa, component_data_t *fpb)); diff --git a/inc/scanoss.h b/inc/scanoss.h index b626f9e..f4ab4b5 100644 --- a/inc/scanoss.h +++ b/inc/scanoss.h @@ -33,7 +33,7 @@ #define WFP_LN 4 #define WFP_REC_LN 18 -#define SCANOSS_VERSION "6.0.2-crc64" +#define SCANOSS_VERSION "6.1.0-crc64" /* Log files */ #define SCAN_LOG "/tmp/scanoss_scan.log" diff --git a/src/component.c b/src/component.c index 6e28cf5..06900c1 100644 --- a/src/component.c +++ b/src/component.c @@ -34,6 +34,7 @@ #include "util.h" #include "debug.h" #include "query.h" +#include "url.h" /** * @brief Free component object @@ -117,6 +118,33 @@ component_data_t *component_data_copy(component_data_t *in) return out; } +/** + * @brief Return true if the given purl is listed in ignore_components (-b parameter) + * @param purl purl string to test + */ +bool ignored_purl(const char *purl) +{ + if (!ignore_components || !purl) + return false; + + /* Travel ignore_components looking for a purl rule matching this purl */ + for (int i = 0; i < MAX_SBOM_ITEMS; i++) + { + char *dvendor = ignore_components[i].vendor; + char *dcomponent = ignore_components[i].component; + char *dpurl = ignore_components[i].purl; + + /* Exit if reached the end */ + if (!dcomponent && !dvendor && !dpurl) + break; + + if (dpurl && !strcmp(purl, dpurl)) + return true; + } + + return false; +} + /** * @brief Return true if asset is found in ignore_components (-b parameter) * @param url_record pointer to url record @@ -180,6 +208,41 @@ bool ignored_asset_match(uint8_t *url_record) return found; } +/** + * @brief Return true if a component relates to an ignored purl (-b parameter). + * + * ignored_asset_match() only compares the single primary purl stored in the url + * record (field 6). A project may be published under several purls (e.g. a + * github repo mirrored as a golang module), and those sibling identities are + * only materialised later by fetch_related_purls(). This helper resolves the + * related purls and checks every purl of the component against the ignore list, + * so ignoring any one of a project's purls suppresses it regardless of which + * identity ended up as the primary one. + * + * @param component component to test + */ +bool component_ignored_by_purl(component_data_t *component) +{ + if (!ignore_components || !component) + return false; + + /* Resolve sibling purls so we match the project under any of its identities */ + fetch_related_purls(component); + + for (int i = 0; i < MAX_PURLS; i++) + { + if (!component->purls[i]) + break; + if (ignored_purl(component->purls[i])) + { + scanlog("Component ignored by purl %s\n", component->purls[i]); + return true; + } + } + + return false; +} + char * look_for_version(char *in) { if (!in) diff --git a/src/license.c b/src/license.c index 798b90f..808381e 100644 --- a/src/license.c +++ b/src/license.c @@ -46,6 +46,13 @@ bool full_license_report = false; +/* normalize_license() may replace a license string in place with its canonical + SPDX id (e.g. "Apache" -> "Apache-2.0"), which can be LONGER than the original + text. License buffers must therefore be sized to hold the longest canonical id + in license_normalization[] (currently "LGPL-3.0-or-later", 17 chars) even when + the matched text is shorter. 32 leaves headroom for future table entries. */ +#define MAX_NORMALIZED_LICENSE_LN 32 + struct license_type { char * text; @@ -108,9 +115,13 @@ bool license_add_to_list(struct license_list * ptr, char * license, int license_ ptr->licenses[ptr->count].text = NULL; ptr->licenses[ptr->count].id = 0; - /* Allocate with extra padding for CRC32C hardware reads (8-byte blocks) */ + /* Allocate with extra padding for CRC32C hardware reads (8-byte blocks). + Reserve at least MAX_NORMALIZED_LICENSE_LN so normalize_license() can + replace the text in place with a longer canonical SPDX id without + overflowing the buffer. */ size_t len = strlen(license); - size_t padded_len = ((len + 8) / 8) * 8; /* Round up to next 8-byte boundary */ + size_t capacity = len < MAX_NORMALIZED_LICENSE_LN ? MAX_NORMALIZED_LICENSE_LN : len; + size_t padded_len = ((capacity + 8) / 8) * 8; /* Round up to next 8-byte boundary */ ptr->licenses[ptr->count].text = calloc(1, padded_len); if (!ptr->licenses[ptr->count].text) return false; @@ -199,7 +210,10 @@ void normalize_license(char *license) { if (stricmp(license, token)) { - strcpy(license, spdx); + /* Defensive: never write a canonical id that would not fit the + buffers reserved by license_add_to_list(). */ + if (spdx && strlen(spdx) < MAX_NORMALIZED_LICENSE_LN) + strcpy(license, spdx); return; } token = strtok(NULL, ","); diff --git a/src/match.c b/src/match.c index 823b94b..641f783 100644 --- a/src/match.c +++ b/src/match.c @@ -836,6 +836,13 @@ bool load_matches(match_data_t *match) records = ldb_fetch_recordset(NULL, oss_file, match->file_md5, false, component_from_file,(void *)&match->component_list); scanlog("Found %d file entries\n", records); + /* Drop components whose primary OR related purl is in the ignore list (-b). + handle_url_record()/get_oldest_url() only compare the url record's primary + purl, so a project indexed under several purls would otherwise survive under + its other identity and the ignored purl would resurface as a related purl. */ + if (ignore_components) + component_list_remove_ignored(&match->component_list); + /* Final optimization based on the available information for a component */ /* If two components have the date date, select the one with more available information */ diff --git a/src/match_list.c b/src/match_list.c index be5ec8b..b240b1c 100644 --- a/src/match_list.c +++ b/src/match_list.c @@ -24,6 +24,38 @@ void component_list_destroy(component_list_t *list) } } +int component_list_remove_ignored(component_list_t *list) +{ + if (!list || !ignore_components) + return 0; + + int removed = 0; + struct comp_entry *item = list->headp.lh_first; + while (item != NULL) + { + struct comp_entry *next = item->entries.le_next; + + if (component_ignored_by_purl(item->component)) + { + component_data_free(item->component); + LIST_REMOVE(item, entries); + free(item); + list->items--; + removed++; + } + item = next; + } + + /* Structural changes invalidate the cached tail pointers kept for inserts */ + list->last_element = NULL; + list->last_element_aux = NULL; + + if (removed) + scanlog("Removed %d component(s) matching the ignore list\n", removed); + + return removed; +} + void component_list_init(component_list_t *comp_list, int max_items) { LIST_INIT(&comp_list->headp); /* Initialize the list. */ diff --git a/src/scan.c b/src/scan.c index f9880ea..54b366f 100644 --- a/src/scan.c +++ b/src/scan.c @@ -98,6 +98,45 @@ void scan_data_free(scan_data_t * scan) scan = NULL; } +/** @brief Discards the scan RESULTS (matches, matchmap, best match) while + keeping the scan identity, configuration and the reusable hash/line buffers, + so the same scan can be re-run over the same file (e.g. a snippet second + pass). Mirrors the result-freeing half of scan_data_free(). + @param scan Scan data */ +static void scan_data_reset(scan_data_t *scan) +{ + for (int i = 0; i < scan->matches_list_array_index; i++) + match_list_destroy(scan->matches_list_array[i]); + scan->matches_list_array_index = 0; + memset(scan->matches_list_array_indirection, 0, sizeof(scan->matches_list_array_indirection)); + + for (int i = 0; i < scan->matchmap_size; i++) + free(scan->matchmap[i].range); + free(scan->matchmap); + scan->matchmap = NULL; + scan->matchmap_size = 0; + memset(scan->matchmap_rank_by_sector, 0, sizeof(scan->matchmap_rank_by_sector)); + + scan->match_ptr = NULL; + scan->best_match = NULL; + scan->match_type = MATCH_NONE; + scan->printed_succed = false; +} + +/** @brief True if the scan produced at least one match that still has + components to report (i.e. it would not be rendered as "none"). + @param scan Scan data */ +static bool scan_has_reportable_match(scan_data_t *scan) +{ + for (int i = 0; i < scan->matches_list_array_index; i++) + { + match_list_t *list = scan->matches_list_array[i]; + if (list && list->best_match && list->best_match->component_list.items > 0) + return true; + } + return false; +} + /** @brief Returns true if md5 is the md5sum for NULL @param md5 File ID (md5) @result Empty file result @@ -482,53 +521,83 @@ void ldb_scan(scan_data_t *scan) /* Scan full file */ ldb_bin_to_hex(scan->md5, oss_file.key_ln, scan->source_md5); - /* Look for full file match or url match in ldb */ - scan->match_type = ldb_scan_file(scan); + /* A full-file match whose candidates are all removed by the ignore list + collapses to "none". When that happens we discard the result and retry + once as a snippet scan over the same file, honoring the same ignore/ + include lists (force_snippets drives that second pass). */ + bool force_snippets = false; + bool retried = false; - /* If no match, scan snippets */ - if (scan->match_type == MATCH_NONE || force_snippet_scan) + while (true) { - /* Load snippets into scan data */ - if (!scan->preload) - { - /* Read file into memory */ - char *src = calloc(MAX_FILE_SIZE, 1); - if (file_size < MAX_FILE_SIZE) - read_file(src, scan->file_path, 0); + /* Look for full file match or url match in ldb (skipped when forcing snippets) */ + scan->match_type = force_snippets ? MATCH_NONE : ldb_scan_file(scan); - /* If HPSM is enable calculate the crc8 line hash calling the shared lib */ - if (hpsm_enabled) + /* If no match (or snippets are forced), scan snippets */ + if (scan->match_type == MATCH_NONE || force_snippet_scan || force_snippets) + { + /* Load snippets into scan data */ + if (!scan->preload) { - char *aux = hpsm_hash_file_contents(src); - if (aux) + /* Read file into memory */ + char *src = calloc(MAX_FILE_SIZE, 1); + if (file_size < MAX_FILE_SIZE) + read_file(src, scan->file_path, 0); + + /* If HPSM is enable calculate the crc8 line hash calling the shared lib */ + if (hpsm_enabled) { - hpsm_crc_lines = strdup(&aux[5]); - scanlog("HPSM lines CRC: %s\n", hpsm_crc_lines); - free(aux); + char *aux = hpsm_hash_file_contents(src); + if (aux) + { + hpsm_crc_lines = strdup(&aux[5]); + scanlog("HPSM lines CRC: %s\n", hpsm_crc_lines); + free(aux); + } } + /* Determine if file is to skip snippet search */ + if (!skip_snippets(src, file_size)) + { /* Load wfps into scan structure */ + scan->hash_count = winnowing(src, scan->hashes, scan->lines, MAX_FILE_SIZE); + if (scan->hash_count) + scan->total_lines = scan->lines[scan->hash_count - 1]; + } + free(src); } - /* Determine if file is to skip snippet search */ - if (!skip_snippets(src, file_size)) - { /* Load wfps into scan structure */ - scan->hash_count = winnowing(src, scan->hashes, scan->lines, MAX_FILE_SIZE); - if (scan->hash_count) - scan->total_lines = scan->lines[scan->hash_count - 1]; - } - free(src); + else if (scan->hash_count) + scan->total_lines = scan->lines[scan->hash_count - 1]; + + /* Perform snippet scan */ + if (scan->total_lines) + scan->match_type = ldb_scan_snippets(scan); + + else + scanlog("File skipped\n"); } - else if (scan->hash_count) - scan->total_lines = scan->lines[scan->hash_count - 1]; - /* Perform snippet scan */ - if (scan->total_lines) - scan->match_type = ldb_scan_snippets(scan); + /* Type found by this pass, captured before compile_matches, which resets + match_type to MATCH_NONE when the selected match has no components. */ + match_t scanned_type = scan->match_type; - else - scanlog("File skipped\n"); - } + /* Compile matches */ + compile_matches(scan); - /* Compile matches */ - compile_matches(scan); + /* Retry once as a snippet scan if a full-file match was entirely + filtered out by the ignore list (otherwise it would print "none"). */ + if (!retried && !force_snippets + && scanned_type == MATCH_FILE + && ignore_components + && !scan_has_reportable_match(scan)) + { + scanlog("Full-file match filtered out by ignore list; retrying as snippet scan\n"); + scan_data_reset(scan); + force_snippets = true; + retried = true; + continue; + } + + break; + } if (!scan->best_match) scanlog("No best match\n"); diff --git a/tests/ignore_related_purl/run.sh b/tests/ignore_related_purl/run.sh new file mode 100755 index 0000000..f04c2ef --- /dev/null +++ b/tests/ignore_related_purl/run.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# +# Regression test: ignoring a related PURL (-b) must suppress the component. +# +# Guards issue #120: the ignore list (-b SBOM.json) was only compared against +# the single primary purl stored in the url record (field 6). A project may be +# published under several purls (e.g. a github repo also indexed as a golang/ +# nuget/conda package); those sibling identities are only materialised later by +# fetch_related_purls(). So ignoring a purl that happened to be a *related* purl +# had no effect: the match survived under its primary identity and the ignored +# purl resurfaced in the reported "purl" array. +# +# The fix (component_list_remove_ignored / component_ignored_by_purl) drops a +# component when ANY of its purls -- primary OR related -- is in the ignore list. +# +# This test is data-driven so it does not hard-code KB contents: it peels the +# top candidates of a region until a component exposing >= 2 purls surfaces, +# then asks the engine to ignore one of that component's *related* purls and +# asserts the whole component disappears. It needs an OSS knowledge base; if +# none is reachable (or no multi-purl component is found) it SKIPS (exit 0) so +# it never breaks CI images that ship without KB data. +# +# SCANOSS_BIN path to the engine binary (default: ../../scanoss) +# SCANOSS_KB KB/database name for -n (default: ossv2) +# +set -u + +HERE="$(cd "$(dirname "$0")" && pwd)" +BIN="${SCANOSS_BIN:-$HERE/../../scanoss}" +KB="${SCANOSS_KB:-ossv2}" +WFP="$HERE/../snippet_multicomponent/main.c.wfp" + +if [ ! -x "$BIN" ]; then + echo "SKIP: engine binary not found at $BIN (set SCANOSS_BIN)" + exit 0 +fi +if [ ! -f "$WFP" ]; then + echo "SKIP: input WFP not found at $WFP" + exit 0 +fi + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +# scan [purl ...] -> prints engine JSON for the WFP, ignoring the given purls. +scan() { + local ign="$TMP/ignore.json" first=1 p + { + printf '{"components":[' + for p in "$@"; do + [ $first -eq 1 ] || printf ',' + printf '{"purl":"%s"}' "$p" + first=0 + done + printf ']}' + } > "$ign" + if [ "$#" -eq 0 ]; then + "$BIN" -n "$KB" -M 9 -w "$WFP" 2>/dev/null + else + "$BIN" -n "$KB" -M 9 -b "$ign" -w "$WFP" 2>/dev/null + fi +} + +# first_multi_purl_array -> the first "purl":[...] array holding >= 2 purls. +first_multi_purl_array() { + printf '%s' "$1" | grep -o '"purl": \[[^]]*\]' | grep ',' | head -1 +} + +# first_primary_purl -> the primary purl (purls[0]) of the first component. +first_primary_purl() { + printf '%s' "$1" | grep -o '"purl": \[[^]]*\]' | head -1 | grep -o 'pkg:[^"]*' | head -1 +} + +# Probe: make sure the KB answers at all, otherwise skip rather than fail. +PROBE="$(scan)" +if ! printf '%s' "$PROBE" | grep -q '"id": "snippet"'; then + echo "SKIP: KB '$KB' produced no snippet match (KB missing or not loaded)" + exit 0 +fi + +# Peel the top candidate of the region until a component with >= 2 purls shows up. +IGN=() +ARR="" +for _ in 1 2 3 4 5 6 7 8; do + OUT="$(scan "${IGN[@]}")" + ARR="$(first_multi_purl_array "$OUT")" + [ -n "$ARR" ] && break + P="$(first_primary_purl "$OUT")" + [ -z "$P" ] && break # no more matches to peel + IGN+=("$P") +done + +if [ -z "$ARR" ]; then + echo "SKIP: no reachable component with related purls in KB '$KB' (cannot exercise #120)" + exit 0 +fi + +# The array is "pkg:A","pkg:B",... -- primary is the first, related ones follow. +mapfile -t PURLS < <(printf '%s' "$ARR" | grep -o 'pkg:[^"]*') +PRIMARY="${PURLS[0]}" +RELATED="${PURLS[-1]}" # a sibling purl, only present via fetch_related_purls + +if [ "$RELATED" = "$PRIMARY" ]; then + echo "SKIP: could not isolate a related purl distinct from the primary" + exit 0 +fi + +echo "INFO: component primary=$PRIMARY related=$RELATED (ignoring the related one)" + +# Ignoring the RELATED purl must drop the whole component. Pre-fix this had no +# effect because only the primary purl (field 6) was ever checked. +OUT_TEST="$(scan "${IGN[@]}" "$RELATED")" + +FAILED=0 +if printf '%s' "$OUT_TEST" | grep -qF "$RELATED"; then + echo "FAIL: ignored related purl still present in output: $RELATED" + FAILED=1 +fi +if printf '%s' "$OUT_TEST" | grep -qF "$PRIMARY"; then + echo "FAIL: component not suppressed; its primary purl still present: $PRIMARY" + FAILED=1 +fi + +if [ "$FAILED" -eq 0 ]; then + echo "PASS: ignoring a related purl suppresses the component (#120)" + exit 0 +fi +echo "FAIL: related-purl ignore regression (#120)" +exit 1 diff --git a/tests/snippet_fallback_on_ignore/run.sh b/tests/snippet_fallback_on_ignore/run.sh new file mode 100755 index 0000000..b742277 --- /dev/null +++ b/tests/snippet_fallback_on_ignore/run.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# +# Regression test: snippet fallback when a full-file match is fully ignored. +# +# When a file produces a full-file (MATCH_FILE) result but every candidate +# component is removed by the ignore list (-b), the match used to collapse to +# "none". The engine now discards that result and automatically retries once as +# a snippet scan over the same file, honoring the same ignore/include lists, so +# a legitimate snippet match can still surface. +# +# The test is data-driven so it does not hard-code KB contents: +# 1. Derive a real full-file hash from a snippet match against main.c.wfp. +# 2. Build a WFP whose file= hash triggers that full-file match, carrying the +# same snippet hashes. +# 3. Peel the full-file components (ignore them one by one) until the result +# would collapse to "none". +# 4. With all of them ignored, assert the result is a SNIPPET match, not none. +# +# Needs an OSS knowledge base; if none is reachable (or the scenario cannot be +# built from the KB) it SKIPS (exit 0) so it never breaks KB-less CI images. +# +# SCANOSS_BIN path to the engine binary (default: ../../scanoss) +# SCANOSS_KB KB/database name for -n (default: ossv2) +# +set -u + +HERE="$(cd "$(dirname "$0")" && pwd)" +BIN="${SCANOSS_BIN:-$HERE/../../scanoss}" +KB="${SCANOSS_KB:-ossv2}" +WFP_SRC="$HERE/../snippet_multicomponent/main.c.wfp" + +if [ ! -x "$BIN" ]; then + echo "SKIP: engine binary not found at $BIN (set SCANOSS_BIN)" + exit 0 +fi +if [ ! -f "$WFP_SRC" ]; then + echo "SKIP: input WFP not found at $WFP_SRC" + exit 0 +fi + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +# ignore_json -> path to a -b SBOM.json ignoring the given purls. +ignore_json() { + local out="$TMP/ignore.json" first=1 p + { + printf '{"components":[' + for p in "$@"; do + [ $first -eq 1 ] || printf ',' + printf '{"purl":"%s"}' "$p"; first=0 + done + printf ']}' + } > "$out" + echo "$out" +} + +match_id() { printf '%s' "$1" | grep -oE '"id": "[a-z]+"' | head -1 | grep -oE '[a-z]+"$' | tr -d '"'; } +first_purl() { printf '%s' "$1" | grep -o '"purl": \[[^]]*\]' | head -1 | grep -o 'pkg:[^"]*' | head -1; } + +# 1. Find a real KB full-file hash via a snippet match on the source WFP. +PROBE="$("$BIN" -n "$KB" -w "$WFP_SRC" 2>/dev/null)" +FHASH="$(printf '%s' "$PROBE" | grep -oE '"file_hash": "[0-9a-f]+"' | head -1 | grep -oE '[0-9a-f]+' | tail -1)" +if [ -z "$FHASH" ]; then + echo "SKIP: KB '$KB' produced no snippet match to derive a full-file hash" + exit 0 +fi + +# 2. Build a WFP whose file= hash is that KB file (full-file match), padded to +# 32 hex chars; the engine reads the leading oss_file.key_ln*2 chars. +FHASH_PAD="$(printf '%-32s' "$FHASH" | tr ' ' '0')" +WFP="$TMP/fullfile.wfp" +{ echo "file=${FHASH_PAD},9999,test.c"; tail -n +2 "$WFP_SRC"; } > "$WFP" + +# Confirm it is actually a full-file match, otherwise the scenario does not apply. +BASE="$("$BIN" -n "$KB" -w "$WFP" 2>/dev/null)" +if [ "$(match_id "$BASE")" != "file" ]; then + echo "SKIP: derived WFP did not produce a full-file match in KB '$KB'" + exit 0 +fi + +# 3. Peel the full-file components until the match would collapse to none. +IGN=() +COLLAPSED=0 +for _ in 1 2 3 4 5 6 7 8; do + if [ ${#IGN[@]} -eq 0 ]; then + OUT="$BASE" + else + OUT="$("$BIN" -n "$KB" -b "$(ignore_json "${IGN[@]}")" -w "$WFP" 2>/dev/null)" + fi + ID="$(match_id "$OUT")" + [ "$ID" != "file" ] && { COLLAPSED=1; break; } # no more file components + P="$(first_purl "$OUT")" + [ -z "$P" ] && break + IGN+=("$P") +done + +if [ "$COLLAPSED" -ne 1 ] || [ ${#IGN[@]} -eq 0 ]; then + echo "SKIP: could not reduce the full-file match to a single ignorable component set" + exit 0 +fi + +echo "INFO: full-file components ignored: ${IGN[*]}" + +# 4. With every full-file purl ignored, the result must fall back to a snippet +# match instead of collapsing to "none". +OUT="$("$BIN" -n "$KB" -b "$(ignore_json "${IGN[@]}")" -w "$WFP" 2>/dev/null)" +ID="$(match_id "$OUT")" + +if [ "$ID" = "snippet" ]; then + echo "PASS: fully-ignored full-file match falls back to a snippet scan" + exit 0 +fi +echo "FAIL: expected a snippet fallback, got id='$ID' (regression: #120 full-file fallback)" +exit 1