diff --git a/src/fosslight_source/_exclude.py b/src/fosslight_source/_exclude.py index 830382f..cb05fcf 100644 --- a/src/fosslight_source/_exclude.py +++ b/src/fosslight_source/_exclude.py @@ -1,10 +1,14 @@ # Copyright (c) 2026 LG Electronics Inc. # SPDX-License-Identifier: Apache-2.0 -"""Source-scanner filename excludes (not applied to dependency discovery).""" +"""Source-scanner exclusion helpers (not applied to dependency discovery).""" + +import os from fosslight_util.exclude import is_excluded_filename +DEFAULT_SKIP_SMALL_FILE_SIZE = 150 + # Build/package config noise for license scans. EXCLUDE_FILENAME_SOURCE = frozenset({ "changelog", "config.guess", "config.sub", "changes", "ltmain.sh", @@ -19,3 +23,11 @@ def is_excluded_source_filename(file_path: str) -> bool: return is_excluded_filename(file_path, EXCLUDE_FILENAME_SOURCE) + + +def is_small_file(file_path: str) -> bool: + """Return whether a file is at or below the KB/SCANOSS size threshold.""" + try: + return os.path.getsize(file_path) <= DEFAULT_SKIP_SMALL_FILE_SIZE + except OSError: + return False diff --git a/src/fosslight_source/_help.py b/src/fosslight_source/_help.py index cd8844b..0051bf7 100644 --- a/src/fosslight_source/_help.py +++ b/src/fosslight_source/_help.py @@ -47,7 +47,6 @@ --kb_token KB bearer token (priority: parameter > KB_TOKEN env) HTTPS uses the OS certificate store. Set KB_SSL_VERIFY=false only if certificate verification must be skipped. - 💡 Examples ──────────────────────────────────────────────────────────────────── # Scan current directory diff --git a/src/fosslight_source/_scan_item.py b/src/fosslight_source/_scan_item.py index c0dce53..33b42f7 100644 --- a/src/fosslight_source/_scan_item.py +++ b/src/fosslight_source/_scan_item.py @@ -63,7 +63,7 @@ def __init__(self, value: str) -> None: self.checksum = get_checksum_sha1(value) self.kb_origin_url = "" # URL from OSS KB self.kb_evidence = "" # Evidence from KB API (exact_match or code snippet) - self._cached_kb_md5 = "" # MD5 precomputed for KB lookup (set by _collect_kb_file_hashes) + self._cached_kb_md5: str | None = None # None: not checked; empty: checked but no KB hash def __del__(self) -> None: pass @@ -192,8 +192,9 @@ def set_oss_item( item = OssItem(self.oss_name, self.oss_version, self.licenses) if kb_origin_urls and not self.is_license_text: md5_hash = self._cached_kb_md5 - if not md5_hash: + if md5_hash is None: md5_hash, _wfp = self._get_hash(path_to_scan) + self._cached_kb_md5 = md5_hash if md5_hash: origin_url = kb_origin_urls.get(md5_hash, "") if origin_url: diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index 3d1d778..00b64a6 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -22,7 +22,7 @@ from fosslight_util.parsing_yaml import SUPPORT_OSS_INFO_FILES from .run_scancode import run_scan from fosslight_util.exclude import get_excluded_paths -from ._exclude import EXCLUDE_FILENAME_SOURCE, is_excluded_source_filename +from ._exclude import EXCLUDE_FILENAME_SOURCE, is_excluded_source_filename, is_small_file from .run_scanoss import run_scanoss_py from .run_scanoss import get_scanoss_extra_info import yaml @@ -93,7 +93,6 @@ def main() -> None: parser.add_argument('--ui', action='store_true', required=False) args = parser.parse_args() - if args.help: print_help_msg_source_scanner() if args.version: @@ -362,9 +361,15 @@ def _collect_kb_file_hashes( extra_candidates: list[tuple[SourceItem, str]] = [] for item in scancode_result: - if item.is_license_text or is_notice_file(item.source_name_or_path): + if item.is_license_text or item.download_location or is_notice_file(item.source_name_or_path): + item._cached_kb_md5 = "" continue - if item.download_location: + item_path = item.source_name_or_path + if not os.path.isabs(item_path): + item_path = os.path.join(path_to_scan, item_path) + if is_small_file(item_path): + # An empty cached hash means this item is checked and not KB-eligible. + item._cached_kb_md5 = "" continue md5_hash, _wfp = item._get_hash(path_to_scan) if md5_hash: @@ -384,6 +389,8 @@ def _collect_kb_file_hashes( if (rel_path in scancode_paths or rel_path in excluded_files or is_excluded_source_filename(rel_path) or is_notice_file(file_path)): continue + if is_small_file(file_path): + continue extra_item = SourceItem(rel_path) md5_hash, _wfp = extra_item._get_hash(path_to_scan) if md5_hash: diff --git a/src/fosslight_source/run_scanoss.py b/src/fosslight_source/run_scanoss.py index 51b0b99..4217286 100755 --- a/src/fosslight_source/run_scanoss.py +++ b/src/fosslight_source/run_scanoss.py @@ -10,6 +10,7 @@ import json from typing import Tuple import fosslight_util.constant as constant +from ._exclude import DEFAULT_SKIP_SMALL_FILE_SIZE from ._parsing_scanoss_file import parsing_scan_result # scanoss from ._parsing_scanoss_file import parsing_extra_info # scanoss from scanoss.scanner import Scanner, ScanType @@ -66,6 +67,7 @@ def run_scanoss_py(path_to_scan: str, output_path: str = "", format: list = [], scanner = Scanner( ignore_cert_errors=True, skip_folders=list(path_to_exclude) if path_to_exclude else [], + skip_size=DEFAULT_SKIP_SMALL_FILE_SIZE + 1, scan_output=output_json_file, scan_options=ScanType.SCAN_SNIPPETS.value, nb_threads=num_threads if num_threads > 0 else 10,