diff --git a/pyproject.toml b/pyproject.toml index d63e851..b2978de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "coloredlogs", "beautifulsoup4", "jsonmerge", - "spdx-tools==0.8.2; sys_platform == 'linux'", + "spdx-tools==0.8.2", "setuptools>=65.5.1", "numpy", "requests", diff --git a/requirements-dev.txt b/requirements-dev.txt index 3e433a3..896ba78 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,4 +3,5 @@ pytest pytest-cov flake8 fosslight-source +spdx-tools==0.8.2 pyopenssl>=26.0.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/src/fosslight_util/oss_item.py b/src/fosslight_util/oss_item.py index 24e58a0..28a2685 100644 --- a/src/fosslight_util/oss_item.py +++ b/src/fosslight_util/oss_item.py @@ -23,6 +23,7 @@ def __init__(self, name="", version="", license="", dl_url=""): self.version = version self._license = [] self.license = license + self.license_texts: Dict[str, str] = {} self.download_location = dl_url self.exclude = False self.comment = "" diff --git a/src/fosslight_util/output_format.py b/src/fosslight_util/output_format.py index bea363d..7e02108 100644 --- a/src/fosslight_util/output_format.py +++ b/src/fosslight_util/output_format.py @@ -3,7 +3,6 @@ # Copyright (c) 2021 LG Electronics Inc. # SPDX-License-Identifier: Apache-2.0 import os -import platform from fosslight_util.write_excel import write_result_to_excel, write_result_to_csv from fosslight_util.write_opossum import write_opossum from fosslight_util.write_yaml import write_yaml @@ -189,11 +188,8 @@ def write_output_file(output_file_without_ext: str, file_extension: str, scan_it elif format == 'yaml': success, msg, _ = write_yaml(result_file, scan_item, False) elif format.startswith('spdx'): - if platform.system() == 'Windows' or platform.system() == 'Darwin': - success = False - msg = f'{platform.system()} not support spdx format.' - else: - success, msg, _ = write_spdx(output_file_without_ext, file_extension, scan_item, spdx_version) + success, msg, _ = write_spdx(output_file_without_ext, file_extension, scan_item, spdx_version, + scanner_covers=scanner_covers) elif format.startswith('cyclonedx'): success, msg, result_file = write_cyclonedx( output_file_without_ext, diff --git a/src/fosslight_util/write_spdx.py b/src/fosslight_util/write_spdx.py index faf9683..62f8a41 100644 --- a/src/fosslight_util/write_spdx.py +++ b/src/fosslight_util/write_spdx.py @@ -3,22 +3,30 @@ # Copyright (c) 2022 LG Electronics Inc. # SPDX-License-Identifier: Apache-2.0 -import os -import uuid +import hashlib +import json import logging +import ntpath +import os import re +import uuid +from collections import defaultdict +from importlib.metadata import PackageNotFoundError, version as package_version from pathlib import Path -from datetime import datetime +from datetime import datetime, timezone from fosslight_util.spdx_licenses import get_spdx_licenses_json, get_license_from_nick from fosslight_util.constant import (LOGGER_NAME, FOSSLIGHT_DEPENDENCY, FOSSLIGHT_SCANNER, - FOSSLIGHT_BINARY, FOSSLIGHT_SOURCE) -from fosslight_util.oss_item import CHECKSUM_NULL, get_checksum_sha1 -import traceback + FOSSLIGHT_BINARY, FOSSLIGHT_SOURCE, FOSSLIGHT_OCI, FOSSLIGHT_OCI_BINARY) +from fosslight_util.oss_item import get_checksum_sha1 logger = logging.getLogger(LOGGER_NAME) +_spdx_import_error = None try: + from packageurl import PackageURL from spdx_tools.common.spdx_licensing import spdx_licensing + from spdx_tools.spdx.jsonschema.document_converter import DocumentConverter + from spdx_tools.spdx.jsonschema.external_package_ref_converter import ExternalPackageRefConverter from spdx_tools.spdx.model import ( Actor, ActorType, @@ -26,7 +34,11 @@ ChecksumAlgorithm, CreationInfo, Document, + ExternalPackageRef, + ExternalPackageRefCategory, + ExtractedLicensingInfo, File, + FileType, Package, Relationship, RelationshipType, @@ -34,8 +46,13 @@ SpdxNone ) from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document + from spdx_tools.spdx.validation.uri_validators import validate_download_location, validate_url + from spdx_tools.spdx.writer.json import json_writer from spdx_tools.spdx.writer.write_anything import write_file -except Exception: + from spdx_tools.spdx.writer.xml import xml_writer + from spdx_tools.spdx.writer.yaml import yaml_writer +except Exception as error: + _spdx_import_error = error logger.info('No import spdx-tools') @@ -49,181 +66,337 @@ def get_license_list_version(): return version -def write_spdx(output_file_without_ext, output_extension, scan_item, spdx_version='2.3'): - success = True - error_msg = '' - - if scan_item: - try: - cover_name = scan_item.cover.get_print_json()["Tool information"].split('(').pop(0).strip() - match = re.search(r"(.+) v([0-9.]+)", cover_name) - if match: - scanner_name = match.group(1) - else: - scanner_name = FOSSLIGHT_SCANNER - except Exception: - cover_name = FOSSLIGHT_SCANNER - scanner_name = FOSSLIGHT_SCANNER - creation_info = CreationInfo(spdx_version=f'SPDX-{spdx_version}', - spdx_id='SPDXRef-DOCUMENT', - name=f'SPDX Document by {scanner_name.upper()}', - data_license='CC0-1.0', - document_namespace=f'http://spdx.org/spdxdocs/{scanner_name.lower()}-{uuid.uuid4()}', - creators=[Actor(name=cover_name, actor_type=ActorType.TOOL)], - created=datetime.now()) - doc = Document(creation_info=creation_info) - - relation_tree = {} - spdx_id_packages = [] - - output_dir = os.path.dirname(output_file_without_ext) - Path(output_dir).mkdir(parents=True, exist_ok=True) - try: - file_id = 0 - package_id = 0 - root_package = False - for scanner_name, file_items in scan_item.file_items.items(): - for file_item in file_items: - file = '' # file의 license, copyright은 oss item에서 append - if scanner_name in [FOSSLIGHT_BINARY, FOSSLIGHT_SOURCE]: - if file_item.exclude: - continue - if file_item.checksum == CHECKSUM_NULL: - if os.path.exists(file_item.source_name_or_path): - file_item.checksum = get_checksum_sha1(file_item.source_name_or_path) - if file_item.checksum == CHECKSUM_NULL: - logger.info(f'Failed to get checksum, Skip: {file_item.source_name_or_path}') - continue - file_id += 1 - file = File(name=file_item.source_name_or_path, - spdx_id=f'SPDXRef-File{file_id}', - checksums=[Checksum(ChecksumAlgorithm.SHA1, file_item.checksum)]) - file_license = [] - file_copyright = [] - file_comment = [] - for oss_item in file_item.oss_items: - oss_licenses = [] - declared_oss_licenses = [] - lic_comment = [] - for oi in oss_item.license: - oi = check_input_license_format(oi) - try: - oi_spdx = spdx_licensing.parse(oi, validate=True) - oss_licenses.append(oi_spdx) - declared_oss_licenses.append(oi) - except Exception: - logger.debug(f'No spdx license name: {oi}') - lic_comment.append(oi) - file_comment.append(oi) - if oss_licenses: - file_license.extend(oss_licenses) - if oss_item.copyright != '': - file_copyright.append(oss_item.copyright) - - if oss_item.download_location == '': - if scanner_name == FOSSLIGHT_DEPENDENCY: - download_location = SpdxNone() - else: - continue - else: - download_location = oss_item.download_location - if scanner_name != FOSSLIGHT_DEPENDENCY and oss_item.name == '': +def write_spdx(output_file_without_ext, output_extension, scan_item, spdx_version='2.3', scanner_covers=None): + if _spdx_import_error is not None: + return False, f'Failed to import spdx-tools: {_spdx_import_error}', '' + if not scan_item: + return False, 'No item to write in output file.', '' + result_file = '' + try: + document = Document(creation_info=_creation_info(scan_item, scanner_covers)) + root = Package( + name=document.creation_info.name, spdx_id=_ROOT_PACKAGE_ID, + download_location=SpdxNoAssertion(), files_analyzed=False + ) + document.packages.append(root) + seen_relationships = set() + _add_relationship(document, seen_relationships, 'SPDXRef-DOCUMENT', RelationshipType.DESCRIBES, root.spdx_id) + package_ids_by_purl = defaultdict(list) + dependency_rows = [] + root_purls = set() + checksums = {} + registry = _LicenseRegistry() + for scanner_name, file_items in scan_item.file_items.items(): + scanner_type = { + FOSSLIGHT_OCI: FOSSLIGHT_DEPENDENCY, + FOSSLIGHT_OCI_BINARY: FOSSLIGHT_BINARY, + }.get(scanner_name, scanner_name) + if scanner_type not in (FOSSLIGHT_DEPENDENCY, FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY): + continue + is_dependency = scanner_type == FOSSLIGHT_DEPENDENCY + for file_item in file_items: + if file_item.exclude: + continue + oss_items = [oss_item for oss_item in file_item.oss_items if not oss_item.exclude] + if file_item.oss_items and not oss_items: + continue + file_info = None if is_dependency else _file_info( + file_item, scan_item.cover.input_path, scanner_type, f'SPDXRef-File{len(document.files) + 1}', checksums + ) + if file_info: + for oss_item in oss_items: + file_info.license_info_in_file.extend( + registry.licenses(oss_item.license, getattr(oss_item, 'license_texts', None)) + ) + file_info.license_info_in_file = _unique_licenses(file_info.license_info_in_file) + copyright_lines = { + line for oss_item in oss_items if oss_item.copyright != '-' + for line in oss_item.copyright.splitlines() if line + } + file_info.copyright_text = '\n'.join(sorted(copyright_lines)) or None + document.files.append(file_info) + _add_relationship(document, seen_relationships, 'SPDXRef-DOCUMENT', RelationshipType.DESCRIBES, + file_info.spdx_id) + for oss_item in oss_items: + classification = oss_item.comment + purl = _normalized_purl(getattr(file_item, 'purl', '')) if is_dependency else '' + if is_dependency and re.search(r'\broot package\b', classification, re.IGNORECASE): + package = root + else: + name = '' if oss_item.name == '-' else oss_item.name + raw_download_location = '' if oss_item.download_location in ('-', None) else oss_item.download_location + version = '' if oss_item.version == '-' else oss_item.version + copyright_text = '' if oss_item.copyright == '-' else oss_item.copyright + if not is_dependency and not name and not raw_download_location: continue - package_id += 1 - package = Package(name=oss_item.name, - spdx_id=f'SPDXRef-Package{package_id}', - download_location=download_location) - - if oss_item.version != '': - package.version = oss_item.version - - if scanner_name == FOSSLIGHT_DEPENDENCY: - package.files_analyzed = False # If omitted, the default value of true is assumed. - else: - package.files_analyzed = True - if oss_item.copyright != '': - package.cr_text = oss_item.copyright - if oss_item.homepage != '': - package.homepage = oss_item.homepage - - if declared_oss_licenses: - package.license_declared = spdx_licensing.parse(' AND '.join(declared_oss_licenses)) - if lic_comment: - package.license_comment = ' '.join(lic_comment) - - doc.packages.append(package) - - if scanner_name == FOSSLIGHT_DEPENDENCY: - purl = file_item.purl - spdx_id_packages.append([purl, package.spdx_id]) - relation_tree[purl] = {} - relation_tree[purl]['id'] = package.spdx_id - relation_tree[purl]['dep'] = [] - if 'root package' in oss_item.comment: - root_package = True - relationship = Relationship(doc.creation_info.spdx_id, - RelationshipType.DESCRIBES, - package.spdx_id) - doc.relationships.append(relationship) - relation_tree[purl]['dep'].extend(file_item.depends_on) - - if scanner_name in [FOSSLIGHT_BINARY, FOSSLIGHT_SOURCE]: - if file_license: - file.license_info_in_file = file_license - if file_copyright: - file.copyright_text = '\n'.join(file_copyright) - if file_comment: - file.license_comment = ' '.join(file_comment) - doc.files.append(file) - - if len(doc.packages) > 0: - for pkg in relation_tree: - if len(relation_tree[pkg]['dep']) > 0: - pkg_spdx_id = relation_tree[pkg]['id'] - if len(relation_tree[pkg]['dep']) > 0: - for pname in relation_tree[pkg]['dep']: - ans = next(filter(lambda x: x[0] == pname, spdx_id_packages), None) - if ans is None: - continue - rel_pkg_spdx_id = ans[1] - relationship = Relationship(pkg_spdx_id, RelationshipType.DEPENDS_ON, rel_pkg_spdx_id) - doc.relationships.append(relationship) - if not root_package: - root_package = Package(name='root package', - spdx_id='SPDXRef-ROOT-PACKAGE', - download_location=SpdxNoAssertion()) - root_package.files_analyzed = False - root_package.license_declared = SpdxNoAssertion() - doc.packages.append(root_package) - relationship = Relationship(doc.creation_info.spdx_id, RelationshipType.DESCRIBES, root_package.spdx_id) - doc.relationships.append(relationship) - - except Exception as e: - success = False - error_msg = f'Failed to create spdx document object:{e}, {traceback.format_exc()}' - else: - success = False - error_msg = 'No item to write in output file.' + package = Package( + name=name or 'NOASSERTION', spdx_id=f'SPDXRef-Package{len(document.packages)}', + download_location=_spdx_download_location(oss_item.download_location), + version=version or None, homepage=_spdx_homepage(oss_item.homepage), + copyright_text=copyright_text or None, + files_analyzed=not is_dependency + ) + if is_dependency: + licenses = registry.licenses(oss_item.license, getattr(oss_item, 'license_texts', None)) + licenses = [ + value for value in licenses if not isinstance(value, (SpdxNone, SpdxNoAssertion)) + ] or licenses + package.license_declared = ( + spdx_licensing.parse(' AND '.join(f'({value})' for value in licenses)) if len(licenses) > 1 + else licenses[0] if licenses else SpdxNone() + ) + document.packages.append(package) + _add_relationship(document, seen_relationships, 'SPDXRef-DOCUMENT', RelationshipType.DESCRIBES, + package.spdx_id) + if file_info: + _add_relationship(document, seen_relationships, package.spdx_id, RelationshipType.CONTAINS, + file_info.spdx_id) + if is_dependency: + if purl: + if package.spdx_id not in package_ids_by_purl[purl]: + package_ids_by_purl[purl].append(package.spdx_id) + if package is not root or purl not in root_purls: + package.external_references.append(ExternalPackageRef( + ExternalPackageRefCategory.PACKAGE_MANAGER, 'purl', purl + )) + if package is root: + root_purls.add(purl) + depends_on = getattr(file_item, 'depends_on', []) or [] + dependency_rows.append((package.spdx_id, [depends_on] if isinstance(depends_on, str) else depends_on)) + if package is not root and re.search(r'\bdirect\b', classification, re.IGNORECASE): + _add_relationship(document, seen_relationships, root.spdx_id, RelationshipType.DEPENDS_ON, + package.spdx_id) + for source, dependencies in dependency_rows: + for dependency in dependencies: + for target in package_ids_by_purl.get(_normalized_purl(dependency), []): + _add_relationship(document, seen_relationships, source, RelationshipType.DEPENDS_ON, target) + registry.finalize(document) + _populate_package_licenses(document) + messages = validate_full_spdx_document(document) + if messages: + return False, 'Failed to validate SPDX document: ' + '; '.join( + message.validation_message for message in messages + ), '' + result_file = str(output_file_without_ext) + output_extension + Path(result_file).parent.mkdir(parents=True, exist_ok=True) + if output_extension == '.tag': + write_file(document, result_file, validate=False) + else: + class _ExternalRefConverter(ExternalPackageRefConverter): + def convert(self, external_ref, document=None): + converted = super().convert(external_ref, document) + converted['referenceCategory'] = converted['referenceCategory'].replace('_', '-') + return converted + + converter = DocumentConverter() + converter.package_converter.external_package_ref_converter = _ExternalRefConverter() + writer = {'.json': json_writer, '.yaml': yaml_writer, '.xml': xml_writer}[output_extension] + writer.write_document_to_file(document, result_file, validate=False, converter=converter, drop_duplicates=False) + return True, '', result_file + except Exception as error: + logger.debug('Failed to create or write SPDX document', exc_info=True) + if result_file and os.path.isfile(result_file): + os.remove(result_file) + return False, f'Failed to create or write SPDX document: {error}', result_file + + +_SCANNER_TOOL_NAMES = { + FOSSLIGHT_SCANNER: 'FOSSLIGHT_SCANNER', + FOSSLIGHT_DEPENDENCY: 'FOSSLIGHT_Dependency', + FOSSLIGHT_SOURCE: 'FOSSLIGHT_Source', + FOSSLIGHT_BINARY: 'FOSSLIGHT_Binary', + FOSSLIGHT_OCI: 'FOSSLIGHT_OCI', + FOSSLIGHT_OCI_BINARY: 'FOSSLIGHT_OCI_Binary', +} +_TOOL_PATTERN = re.compile(r'^([A-Za-z0-9_-]+)\s+v([^\s(]+)') +_SHA1_PATTERN = re.compile(r'^[0-9a-fA-F]{40}$') +_TLSH_PATTERN = re.compile(r'^(?:T1)?[0-9a-f]{70}$', re.IGNORECASE) +_ROOT_PACKAGE_ID = 'SPDXRef-ROOT-PACKAGE' + + +def _tool(cover): + tool_name = getattr(cover, 'tool_name', '') + if not tool_name and hasattr(cover, 'get_print_json'): + tool_name = cover.get_print_json().get('Tool information', '') + return _TOOL_PATTERN.match(tool_name.strip()) + + +def _unique_licenses(licenses): + return list({str(license_value): license_value for license_value in sorted(licenses, key=str)}.values()) + + +class _LicenseRegistry: + def __init__(self): + self.aliases = get_license_from_nick() or {} + self.parsed = {} + self.references = {} + + def _reference(self, name, texts): + text = texts.get(name, '') or '' + identity = (name, text) + if identity not in self.references: + self.references[identity] = spdx_licensing.parse(f'LicenseRef-FOSSLight-Internal{len(self.references) + 1}') + return self.references[identity] + + def licenses(self, values, texts=None): + texts = texts or {} + result = [] + for value in [values] if isinstance(values, str) else values: + if not value or value == '-': + continue + if value in ('NONE', 'NOASSERTION'): + result.append(SpdxNone() if value == 'NONE' else SpdxNoAssertion()) + continue + canonical = self.aliases.get(value.casefold(), value) + try: + if canonical not in self.parsed: + self.parsed[canonical] = spdx_licensing.parse(canonical) + expression = self.parsed[canonical] + except Exception: + result.append(self._reference(value, texts)) + continue + replacements = {} + for symbol in spdx_licensing.license_symbols(expression, unique=True, decompose=True): + if symbol.key not in spdx_licensing.known_symbols: + alias = self.aliases.get(symbol.key.casefold()) + replacements[symbol] = spdx_licensing.parse(alias) if alias else self._reference(symbol.key, texts) + result.append(expression.subs(replacements) if replacements else expression) + return _unique_licenses(result) + + def finalize(self, document): + groups = defaultdict(list) + for identity, expression in self.references.items(): + name = re.sub(r'[\s_(]+', '-', identity[0].removeprefix('LicenseRef-')) + identifier = re.sub(r'[^A-Za-z0-9.\-]', '', convert_to_spdx_style(name)) + groups[identifier].append((identity, expression)) + used_ids = {base for base, entries in groups.items() if base != 'LicenseRef-' and len(entries) == 1} + replacements = {} + definitions = [] + for base, entries in sorted(groups.items()): + for (name, text), expression in sorted(entries, key=lambda entry: entry[0]): + identifier = base + if base == 'LicenseRef-' or len(entries) > 1: + digest = hashlib.sha256(json.dumps([name, text], ensure_ascii=False).encode('utf-8')).hexdigest() + for length in range(8, 65, 4): + identifier = f"{base}{'' if base == 'LicenseRef-' else '-'}{digest[:length]}" + if identifier not in used_ids: + break + else: + raise ValueError('Could not assign a unique LicenseRef identifier') + used_ids.add(identifier) + replacements[expression] = spdx_licensing.parse(identifier) + definitions.append(ExtractedLicensingInfo( + license_id=identifier, extracted_text=text or name, license_name=name + )) + for file_info in document.files: + file_info.license_info_in_file = [ + value.subs(replacements) if hasattr(value, 'subs') else value + for value in file_info.license_info_in_file + ] + for package in document.packages: + if hasattr(package.license_declared, 'subs'): + package.license_declared = package.license_declared.subs(replacements) + document.extracted_licensing_info = sorted(definitions, key=lambda definition: definition.license_id) + + +def _creation_info(scan_item, scanner_covers): + input_path = scan_item.cover.input_path + path_module = ntpath if ntpath.splitdrive(input_path)[0] else os.path + document_name = path_module.basename(path_module.normpath(path_module.abspath(input_path))) or 'Root Component' + first_tool = _tool(scan_item.cover) + scanner_name = first_tool.group(1).lower() if first_tool else FOSSLIGHT_SCANNER + creators = [] + seen = set() + for cover in [scan_item.cover] if scanner_covers is None else scanner_covers: + match = _tool(cover) + if not match: + continue + name = _SCANNER_TOOL_NAMES.get(match.group(1).lower()) + identity = (name, match.group(2)) + if name and identity not in seen: + creators.append(Actor(ActorType.TOOL, f'{name} - {match.group(2)}')) + seen.add(identity) + try: + spdx_tools_version = package_version('spdx-tools') + except PackageNotFoundError: + spdx_tools_version = 'unknown' + creators.append(Actor(ActorType.TOOL, f'spdx-tools - {spdx_tools_version}')) + return CreationInfo( + spdx_version='SPDX-2.3', spdx_id='SPDXRef-DOCUMENT', name=document_name, data_license='CC0-1.0', + document_namespace=f'http://opensource.lge.com/spdxdocs/{scanner_name}-{uuid.uuid4()}', + creators=creators, created=datetime.now(timezone.utc) + ) - validation_messages = validate_full_spdx_document(doc) - for message in validation_messages: - logger.warning(message.validation_message) - logger.warning(message.context) - # assert validation_messages == [] +def _file_info(file_item, input_path, scanner_name, identifier, checksums): + name = file_item.source_name_or_path + if not name or name.endswith(('/', '\\')): + return None + path_module = ntpath if ntpath.splitdrive(input_path)[0] else os.path + name = name.replace('\\', '/') + full_path = path_module.normpath(name if path_module.isabs(name) else path_module.join(input_path, name)) + try: + relative_path = path_module.relpath(full_path, input_path).replace('\\', '/') + except ValueError: + return None + if relative_path == '.' or os.path.isdir(full_path): + return None + checksum = file_item.checksum + if not _SHA1_PATTERN.fullmatch(checksum): + if full_path not in checksums: + checksums[full_path] = get_checksum_sha1(full_path) + checksum = checksums[full_path] + if not _SHA1_PATTERN.fullmatch(checksum): + logger.info(f'Failed to get SHA-1, skip File: {name}') + return None + tlsh = str(getattr(file_item, 'tlsh', '') or '').strip() + comment = f'fosslight:tlsh={tlsh}' if scanner_name == FOSSLIGHT_BINARY and _TLSH_PATTERN.fullmatch(tlsh) else None + return File( + name=f'./{relative_path}', spdx_id=identifier, + checksums=[Checksum(ChecksumAlgorithm.SHA1, checksum.lower())], + file_types=[FileType.BINARY if scanner_name == FOSSLIGHT_BINARY else FileType.SOURCE], comment=comment + ) - result_file = '' - if success: - result_file = output_file_without_ext + output_extension - try: - write_file(doc, result_file) - except Exception as e: - success = False - error_msg = f'Failed to write spdx document: {e}' - if os.path.exists(result_file): - os.remove(result_file) - - return success, error_msg, result_file + +def _spdx_download_location(value): + if value == '-' or value is None: + value = '' + if value in ('NONE', 'NOASSERTION', ''): + return SpdxNone() if value == 'NONE' else SpdxNoAssertion() + if validate_download_location(value): + return SpdxNoAssertion() + return value + + +def _spdx_homepage(value): + if value == '-' or not value or validate_url(value): + return None + return value + + +def _normalized_purl(value): + try: + return str(PackageURL.from_string(value)) + except (TypeError, ValueError): + return '' + + +def _add_relationship(document, seen, source, relation_type, target): + identity = (source, relation_type, target) + if source != target and identity not in seen: + document.relationships.append(Relationship(source, relation_type, target)) + seen.add(identity) + + +def _populate_package_licenses(document): + files = {file_info.spdx_id: file_info for file_info in document.files} + licenses = defaultdict(list) + for relationship in document.relationships: + if relationship.relationship_type == RelationshipType.CONTAINS and relationship.related_spdx_element_id in files: + licenses[relationship.spdx_element_id].extend(files[relationship.related_spdx_element_id].license_info_in_file) + for package in document.packages: + if package.files_analyzed: + package.license_info_from_files = _unique_licenses(licenses[package.spdx_id]) or [SpdxNone()] def convert_to_spdx_style(input_string): diff --git a/tests/test_cyclonedx.py b/tests/test_cyclonedx.py index 16bff0c..4c74677 100644 --- a/tests/test_cyclonedx.py +++ b/tests/test_cyclonedx.py @@ -358,7 +358,7 @@ def test_cyclonedx_serialization_failure_is_reported(monkeypatch, scan_item): @pytest.mark.parametrize("system", ["Windows", "Darwin"]) def test_cyclonedx_is_supported_on_all_platforms(monkeypatch, scan_item, system): - monkeypatch.setattr("fosslight_util.output_format.platform.system", lambda: system) + monkeypatch.setattr("platform.system", lambda: system) output_file_without_ext = os.path.join(constants.TEST_RESULT_DIR, system, "cyclonedx") success, err_msg, result_file = write_output_file( @@ -369,15 +369,44 @@ def test_cyclonedx_is_supported_on_all_platforms(monkeypatch, scan_item, system) @pytest.mark.parametrize("system", ["Windows", "Darwin"]) -def test_spdx_platform_restriction_is_unchanged(monkeypatch, scan_item, system): - monkeypatch.setattr("fosslight_util.output_format.platform.system", lambda: system) +@pytest.mark.parametrize("format_name, extension", [ + ("spdx-json", ".json"), + ("spdx-xml", ".xml"), + ("spdx-tag", ".tag"), +]) +def test_spdx_formats_are_supported_on_all_platforms(tmp_path, monkeypatch, scan_item, system, format_name, extension): + monkeypatch.setattr("platform.system", lambda: system) + success, err_msg, result_file = write_output_file( + os.path.join(str(tmp_path), format_name), + extension, + scan_item, + format=format_name, + ) + + assert success is True, err_msg + assert os.path.isfile(result_file) + if extension == ".json": + with open(result_file, encoding="utf-8") as spdx_file: + assert json.load(spdx_file)["spdxVersion"] == "SPDX-2.3" + elif extension == ".xml": + assert ElementTree.parse(result_file).getroot().tag == "Document" + else: + with open(result_file, encoding="utf-8") as spdx_file: + assert "SPDXVersion: SPDX-2.3" in spdx_file.read() + + +def test_spdx_import_failure_is_reported(monkeypatch, tmp_path, scan_item): + monkeypatch.setattr( + "fosslight_util.write_spdx._spdx_import_error", + ImportError("spdx-tools is unavailable"), + ) success, err_msg, _ = write_output_file( - os.path.join(constants.TEST_RESULT_DIR, system, "spdx"), + os.path.join(str(tmp_path), "spdx"), ".json", scan_item, format="spdx-json", ) assert success is False - assert err_msg == f"{system} not support spdx format." + assert "spdx-tools is unavailable" in err_msg diff --git a/tests/test_spdx.py b/tests/test_spdx.py new file mode 100644 index 0000000..f83489e --- /dev/null +++ b/tests/test_spdx.py @@ -0,0 +1,798 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2026 LG Electronics Inc. +# SPDX-License-Identifier: Apache-2.0 +import hashlib +import json +import re +import xml.etree.ElementTree as ElementTree +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from types import SimpleNamespace +from uuid import UUID + +import pytest +import yaml +from spdx_tools.common.spdx_licensing import spdx_licensing +from spdx_tools.spdx.model import Checksum, ChecksumAlgorithm, File, Package, Relationship, RelationshipType, SpdxNoAssertion +from spdx_tools.spdx.parser.parse_anything import parse_file +from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document + +from fosslight_util import write_spdx as spdx_writer +from fosslight_util.constant import ( + FOSSLIGHT_BINARY, FOSSLIGHT_DEPENDENCY, FOSSLIGHT_OCI, FOSSLIGHT_OCI_BINARY, FOSSLIGHT_SCANNER, FOSSLIGHT_SOURCE, +) +from fosslight_util.oss_item import FileItem, OssItem, ScannerItem +from fosslight_util.output_format import write_output_file + + +def _scan_item(tmp_path, scanner_name=FOSSLIGHT_SOURCE): + scan_item = ScannerItem(scanner_name) + scan_item.cover.tool_name = f"{scanner_name} v1.2.3" + scan_item.set_cover_pathinfo(str(tmp_path / "project"), []) + return scan_item + + +def _write_json(tmp_path, scan_item, scanner_covers=None): + success, error, result_file = write_output_file( + str(tmp_path / "report"), ".json", scan_item, format="spdx-json", scanner_covers=scanner_covers + ) + assert success, error + return json.loads(Path(result_file).read_text(encoding="utf-8")) + + +def _file(name, oss_items=(), checksum="a" * 40): + file_item = FileItem("") + file_item.source_name_or_path = name + file_item.checksum = checksum + file_item.oss_items.extend(oss_items) + return file_item + + +def _dependency(name, purl="", depends_on=(), classification="", licenses=""): + file_item = _file("manifest.json", [OssItem(name, "", licenses)]) + file_item.purl = purl + file_item.depends_on = list(depends_on) + file_item.oss_items[0].comment = classification + return file_item + + +def _relationships(document, relationship_type): + return { + (relationship["spdxElementId"], relationship["relatedSpdxElement"]) + for relationship in document["relationships"] + if relationship["relationshipType"] == relationship_type + } + + +def test_empty_scan_has_named_document_and_root(tmp_path): + success, error, result_file = write_output_file( + str(tmp_path / "report"), ".json", _scan_item(tmp_path), format="spdx-json" + ) + + assert success, error + document = json.loads(Path(result_file).read_text(encoding="utf-8")) + assert document["name"] == "project" + assert document["documentNamespace"].startswith("http://opensource.lge.com/spdxdocs/fosslight_source-") + assert document["packages"] == [{ + "name": "project", + "SPDXID": "SPDXRef-ROOT-PACKAGE", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": False, + }] + assert document["relationships"] == [{ + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-ROOT-PACKAGE", + }] + + +def test_creators_use_executed_covers_and_deduplicate_by_name_and_version(tmp_path): + scan_item = _scan_item(tmp_path, FOSSLIGHT_SCANNER) + scan_item.cover.tool_name += " (fosslight_binary v9.9.9)" + covers = [ + scan_item.cover, + SimpleNamespace(tool_name="fosslight_source v1.2.3"), + SimpleNamespace(tool_name="fosslight_source v1.2.3"), + SimpleNamespace(tool_name="fosslight_source v2.0.0"), + ] + document = _write_json(tmp_path, scan_item, covers) + + assert set(document["creationInfo"]["creators"]) == { + "Tool: FOSSLIGHT_SCANNER - 1.2.3", + "Tool: FOSSLIGHT_Source - 1.2.3", + "Tool: FOSSLIGHT_Source - 2.0.0", + f"Tool: spdx-tools - {version('spdx-tools')}", + } + assert re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", document["creationInfo"]["created"]) + assert "licenseListVersion" not in document["creationInfo"] + + +def test_missing_spdx_tools_metadata_does_not_prevent_output(tmp_path, monkeypatch): + def missing_version(distribution): + raise PackageNotFoundError(distribution) + + monkeypatch.setattr(spdx_writer, "package_version", missing_version) + document = _write_json(tmp_path, _scan_item(tmp_path)) + + assert set(document["creationInfo"]["creators"]) == { + "Tool: FOSSLIGHT_Source - 1.2.3", + "Tool: spdx-tools - unknown", + } + + +@pytest.mark.parametrize("scanner_name, tool_name", [ + (FOSSLIGHT_OCI, "FOSSLIGHT_OCI"), + (FOSSLIGHT_OCI_BINARY, "FOSSLIGHT_OCI_Binary"), +]) +def test_empty_oci_scan_preserves_executed_creator_without_duplicates(tmp_path, scanner_name, tool_name): + scan_item = _scan_item(tmp_path, scanner_name) + + document = _write_json(tmp_path, scan_item, [scan_item.cover, scan_item.cover]) + + assert document["creationInfo"]["creators"] == [ + f"Tool: {tool_name} - 1.2.3", + f"Tool: spdx-tools - {version('spdx-tools')}", + ] + assert len(document["packages"]) == 1 + assert not document.get("files") + + +def test_file_is_shared_by_packages_and_supplies_their_full_license_set(tmp_path): + scan_item = _scan_item(tmp_path) + named_oss = OssItem("first", "-", "MIT") + url_oss = OssItem("", "2.0", "Apache-2.0", "https://example.com/archive.tar.gz") + excluded_oss = OssItem("excluded", "", "GPL-3.0-only") + excluded_oss.exclude = True + excluded_oss.copyright = "Excluded copyright" + scan_item.append_file_items([_file("src/example.c", [named_oss, url_oss, excluded_oss], "A" * 40)]) + document = _write_json(tmp_path, scan_item) + + assert len(document["files"]) == 1 + file_info = document["files"][0] + assert file_info["fileName"] == "./src/example.c" + assert file_info["fileTypes"] == ["SOURCE"] + assert file_info["checksums"] == [{"algorithm": "SHA1", "checksumValue": "a" * 40}] + assert set(file_info["licenseInfoInFiles"]) == {"MIT", "Apache-2.0"} + assert "copyrightText" not in file_info + packages = document["packages"][1:] + assert [package["name"] for package in packages] == ["first", "NOASSERTION"] + assert "versionInfo" not in packages[0] + assert packages[0]["downloadLocation"] == "NOASSERTION" + for package in packages: + assert package["filesAnalyzed"] is True + assert "licenseDeclared" not in package + assert "licenseConcluded" not in package + assert set(package["licenseInfoFromFiles"]) == {"MIT", "Apache-2.0"} + assert _relationships(document, "CONTAINS") == { + (package["SPDXID"], file_info["SPDXID"]) for package in packages + } + assert len(_relationships(document, "DESCRIBES")) == 4 + + +@pytest.mark.parametrize("extension", [".json", ".yaml", ".xml", ".tag"]) +@pytest.mark.parametrize("homepage, expected", [ + ("-", None), + ("", None), + (None, None), + ("Unknown", None), + ("https://example.com/project", "https://example.com/project"), +]) +def test_package_homepage_preserves_urls_and_omits_missing_values(tmp_path, extension, homepage, expected): + scan_item = _scan_item(tmp_path) + oss_item = OssItem("package", "", "MIT") + oss_item.homepage = homepage + scan_item.append_file_items([_file("example.c", [oss_item])]) + + success, error, result_file = spdx_writer.write_spdx(str(tmp_path / "homepage"), extension, scan_item) + + assert success, error + document = parse_file(result_file) + assert validate_full_spdx_document(document) == [] + assert document.packages[1].homepage == expected + assert oss_item.homepage == homepage + + +def test_non_url_package_locations_do_not_drop_the_document(tmp_path): + scan_item = _scan_item(tmp_path, FOSSLIGHT_DEPENDENCY) + unknown = _dependency("unknown-pkg", classification="direct", licenses="MIT") + unknown.oss_items[0].download_location = "Unknown" + unknown.oss_items[0].homepage = "not a url" + valid = _dependency("valid-pkg", "pkg:npm/valid@1", classification="direct", licenses="Apache-2.0") + valid.oss_items[0].download_location = "https://example.com/valid" + valid.oss_items[0].homepage = "https://example.com/valid" + scan_item.append_file_items([unknown, valid]) + + document = _write_json(tmp_path, scan_item) + + packages = {package["name"]: package for package in document["packages"]} + assert packages["unknown-pkg"]["downloadLocation"] == "NOASSERTION" + assert "homepage" not in packages["unknown-pkg"] + assert packages["valid-pkg"]["downloadLocation"] == "https://example.com/valid" + assert packages["valid-pkg"]["homepage"] == "https://example.com/valid" + assert unknown.oss_items[0].download_location == "Unknown" + assert unknown.oss_items[0].homepage == "not a url" + + +def test_dependency_roots_and_duplicate_purls_keep_all_targets(tmp_path): + scan_item = _scan_item(tmp_path, FOSSLIGHT_DEPENDENCY) + target_purl = "pkg:npm/target@1.0" + other_purl = "pkg:npm/other@2.0" + scan_item.append_file_items([ + _dependency("root-a", "pkg:npm/root-a@1", [target_purl], "root package"), + _dependency("root-b", "pkg:npm/root-b@1", [other_purl], "root package"), + _dependency("first", target_purl, [other_purl, target_purl, "invalid"], "direct", ["MIT", "Apache-2.0"]), + _dependency("second", target_purl, [], "transitive", "MIT"), + _dependency("other", other_purl, [], "transitive"), + ]) + document = _write_json(tmp_path, scan_item) + + assert len(document["packages"]) == 4 + assert not document.get("files") + root, first, second, other = document["packages"] + assert root["name"] == "project" + assert root["SPDXID"] == "SPDXRef-ROOT-PACKAGE" + assert root["filesAnalyzed"] is False + assert "licenseDeclared" not in root + assert "licenseInfoFromFiles" not in first + assert first["licenseDeclared"] in {"MIT AND Apache-2.0", "Apache-2.0 AND MIT"} + assert other["licenseDeclared"] == "NONE" + assert first["externalRefs"] == [{ + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": target_purl, + }] + assert _relationships(document, "DEPENDS_ON") == { + (root["SPDXID"], first["SPDXID"]), + (root["SPDXID"], second["SPDXID"]), + (root["SPDXID"], other["SPDXID"]), + (first["SPDXID"], second["SPDXID"]), + (first["SPDXID"], other["SPDXID"]), + } + + +@pytest.mark.parametrize("scanner_name", [FOSSLIGHT_DEPENDENCY, FOSSLIGHT_OCI]) +def test_direct_dependency_does_not_require_purl_or_name(tmp_path, scanner_name): + scan_item = _scan_item(tmp_path, scanner_name) + scan_item.append_file_items([ + _dependency("-", "", ["invalid"], "direct"), + _dependency("unclassified"), + _dependency("indirect", "pkg:npm/indirect@1", [], "transitive"), + ]) + document = _write_json(tmp_path, scan_item) + + direct = document["packages"][1] + assert direct["name"] == "NOASSERTION" + assert direct["downloadLocation"] == "NOASSERTION" + assert direct["licenseDeclared"] == "NONE" + assert "externalRefs" not in direct + assert _relationships(document, "DEPENDS_ON") == {("SPDXRef-ROOT-PACKAGE", direct["SPDXID"])} + + +@pytest.mark.parametrize("extension", [".json", ".yaml", ".xml", ".tag"]) +@pytest.mark.parametrize("scanner_name", [FOSSLIGHT_OCI, FOSSLIGHT_OCI_BINARY]) +def test_oci_results_preserve_package_and_file_semantics(tmp_path, extension, scanner_name): + scan_item = _scan_item(tmp_path, FOSSLIGHT_OCI) + if scanner_name == FOSSLIGHT_OCI: + items = [ + _dependency("root", "pkg:generic/image@1", ["pkg:generic/parent@1"], "root package"), + _dependency("parent", "pkg:generic/parent@1", ["pkg:generic/child@1"], "direct", "MIT"), + _dependency("child", "pkg:generic/child@1", [], "transitive", "Apache-2.0"), + ] + else: + binary = _file("usr/bin/example", [OssItem("binary", "1", "MIT")]) + binary.tlsh = "T1" + "A" * 70 + items = [binary] + scan_item.append_file_items(items, scanner_name) + + success, error, result_file = spdx_writer.write_spdx(str(tmp_path / "oci"), extension, scan_item) + + assert success, error + document = parse_file(result_file) + assert validate_full_spdx_document(document) == [] + assert {creator.name for creator in document.creation_info.creators} == { + "FOSSLIGHT_OCI - 1.2.3", f"spdx-tools - {version('spdx-tools')}", + } + root = document.packages[0] + assert root.spdx_id == "SPDXRef-ROOT-PACKAGE" + relationships = { + (relationship.spdx_element_id, relationship.relationship_type, relationship.related_spdx_element_id) + for relationship in document.relationships + } + if scanner_name == FOSSLIGHT_OCI: + assert len(document.packages) == 3 + assert not document.files + parent, child = document.packages[1:] + assert [package.name for package in document.packages] == ["project", "parent", "child"] + for package, license_value, purl in ( + (parent, "MIT", "pkg:generic/parent@1"), + (child, "Apache-2.0", "pkg:generic/child@1"), + ): + assert package.files_analyzed is False + assert str(package.license_declared) == license_value + assert not package.license_info_from_files + assert [reference.locator for reference in package.external_references] == [purl] + assert { + relation for relation in relationships if relation[1] == RelationshipType.DEPENDS_ON + } == { + (root.spdx_id, RelationshipType.DEPENDS_ON, parent.spdx_id), + (parent.spdx_id, RelationshipType.DEPENDS_ON, child.spdx_id), + } + else: + assert len(document.packages) == 2 + assert len(document.files) == 1 + package = document.packages[1] + file_info = document.files[0] + assert package.name == "binary" + assert package.files_analyzed is True + assert package.license_declared is None + assert {str(value) for value in package.license_info_from_files} == {"MIT"} + assert file_info.name == "./usr/bin/example" + assert [file_type.name for file_type in file_info.file_types] == ["BINARY"] + assert file_info.checksums[0].value == "a" * 40 + assert file_info.comment == "fosslight:tlsh=" + "T1" + "A" * 70 + assert (package.spdx_id, RelationshipType.CONTAINS, file_info.spdx_id) in relationships + assert scan_item.file_items[scanner_name] == items + assert scan_item.cover.tool_name == f"{FOSSLIGHT_OCI} v1.2.3" + + +def test_invalid_checksum_omits_file_but_preserves_package(tmp_path): + scan_item = _scan_item(tmp_path) + scan_item.append_file_items([_file("missing.c", [OssItem("package", "", "MIT")], "invalid")]) + document = _write_json(tmp_path, scan_item) + + assert not document.get("files") + assert len(document["packages"]) == 2 + assert document["packages"][1]["licenseInfoFromFiles"] == ["NONE"] + assert not _relationships(document, "CONTAINS") + + +@pytest.mark.parametrize("scanner_name", [FOSSLIGHT_SOURCE, FOSSLIGHT_DEPENDENCY]) +def test_legacy_oss_without_license_texts_remains_supported(tmp_path, scanner_name): + scan_item = _scan_item(tmp_path, scanner_name) + oss_item = OssItem("legacy", "1.0", "MIT") + del oss_item.license_texts + scan_item.append_file_items([_file("legacy.c", [oss_item])]) + + document = _write_json(tmp_path, scan_item) + + package = document["packages"][1] + assert package["name"] == "legacy" + if scanner_name == FOSSLIGHT_DEPENDENCY: + assert package["licenseDeclared"] == "MIT" + else: + assert package["licenseInfoFromFiles"] == ["MIT"] + + +def test_license_only_and_unmatched_files_are_independent_of_package_creation(tmp_path): + scan_item = _scan_item(tmp_path) + excluded_oss = OssItem("excluded", "", "GPL-3.0-only") + excluded_oss.exclude = True + excluded_file = _file("excluded.c", [OssItem("excluded-file", "", "ISC")]) + excluded_file.exclude = True + scan_item.append_file_items([ + _file("LICENSE", [OssItem("", "", "MIT")]), + _file("unknown.c"), + _file("all-excluded.c", [excluded_oss]), + excluded_file, + ]) + document = _write_json(tmp_path, scan_item) + + assert len(document["packages"]) == 1 + assert [file_info["fileName"] for file_info in document["files"]] == ["./LICENSE", "./unknown.c"] + assert document["files"][0]["licenseInfoInFiles"] == ["MIT"] + assert "licenseInfoInFiles" not in document["files"][1] + + +def test_package_licenses_follow_all_contained_files_without_leaking_between_packages(tmp_path): + success, error, result_file = spdx_writer.write_spdx(str(tmp_path / "report"), ".json", _scan_item(tmp_path)) + assert success, error + document = parse_file(result_file) + file_licenses = [("MIT",), ("MIT", "Apache-2.0"), ("BSD-3-Clause",), ("ISC",), ("MPL-2.0",)] + for index, licenses in enumerate(file_licenses, start=1): + file_info = File( + name=f"./file{index}.c", spdx_id=f"SPDXRef-File{index}", + checksums=[Checksum(ChecksumAlgorithm.SHA1, "a" * 40)], + license_info_in_file=[spdx_licensing.parse(license_value) for license_value in licenses], + ) + document.files.append(file_info) + document.relationships.append(Relationship("SPDXRef-DOCUMENT", RelationshipType.DESCRIBES, file_info.spdx_id)) + for name, indexes in (("A", (1, 2)), ("B", (3, 4))): + package = Package(name=name, spdx_id=f"SPDXRef-Package{name}", download_location=SpdxNoAssertion()) + document.packages.append(package) + document.relationships.append(Relationship("SPDXRef-DOCUMENT", RelationshipType.DESCRIBES, package.spdx_id)) + document.relationships.extend( + Relationship(package.spdx_id, RelationshipType.CONTAINS, f"SPDXRef-File{index}") for index in indexes + ) + + spdx_writer._populate_package_licenses(document) + + assert {str(value) for value in document.packages[1].license_info_from_files} == {"MIT", "Apache-2.0"} + assert {str(value) for value in document.packages[2].license_info_from_files} == {"BSD-3-Clause", "ISC"} + assert validate_full_spdx_document(document) == [] + + +def test_unknown_license_keeps_original_name_and_detected_text(tmp_path): + scan_item = _scan_item(tmp_path) + license_name = "BSD-like License (example)" + text = "This file is licensed under BSD-like License (example)." + oss_item = OssItem("custom", "", [license_name, "MIT"]) + oss_item.license_texts[license_name] = text + scan_item.append_file_items([_file("custom.c", [oss_item])]) + document = _write_json(tmp_path, scan_item) + + assert document["hasExtractedLicensingInfos"] == [{ + "licenseId": "LicenseRef-BSD-like-License-example", + "extractedText": text, + "name": license_name, + }] + assert set(document["files"][0]["licenseInfoInFiles"]) == {"MIT", "LicenseRef-BSD-like-License-example"} + assert set(document["packages"][1]["licenseInfoFromFiles"]) == {"MIT", "LicenseRef-BSD-like-License-example"} + assert "licenseComments" not in document["files"][0] + + +def test_license_ref_collisions_share_only_identical_name_and_text(tmp_path): + scan_item = _scan_item(tmp_path) + oss_items = [] + for name, text in (("Custom License", "first"), ("Custom_License", "first"), + ("Custom License", "second"), ("Custom License", "first")): + oss_item = OssItem("package", "", name) + oss_item.license_texts[name] = text + oss_items.append(oss_item) + excluded = OssItem("excluded", "", "Excluded License") + excluded.exclude = True + scan_item.append_file_items([_file("custom.c", [*oss_items, excluded])]) + document = _write_json(tmp_path, scan_item) + + definitions = document["hasExtractedLicensingInfos"] + assert len(definitions) == 3 + assert {(definition["name"], definition["extractedText"]) for definition in definitions} == { + ("Custom License", "first"), ("Custom_License", "first"), ("Custom License", "second") + } + assert all(re.fullmatch(r"LicenseRef-Custom-License-[0-9a-f]{8}", definition["licenseId"]) for definition in definitions) + assert set(document["files"][0]["licenseInfoInFiles"]) == {definition["licenseId"] for definition in definitions} + + +def test_license_ref_extends_hash_suffix_when_prefix_collides(tmp_path, monkeypatch): + scan_item = _scan_item(tmp_path) + scan_item.append_file_items([_file("custom.c", [ + OssItem("first", "", "Custom License"), OssItem("second", "", "Custom_License"), + ])]) + + def digest(value): + name, _text = json.loads(value) + suffix = "a" if name == "Custom License" else "b" + return SimpleNamespace(hexdigest=lambda: "12345678" + suffix * 56) + + monkeypatch.setattr(spdx_writer.hashlib, "sha256", digest) + document = _write_json(tmp_path, scan_item) + + identifiers = {definition["licenseId"] for definition in document["hasExtractedLicensingInfos"]} + assert len(identifiers) == 2 + assert all(re.fullmatch(r"LicenseRef-Custom-License-12345678[ab]{0,56}", identifier) for identifier in identifiers) + + +def test_empty_normalized_license_name_uses_hash_and_name_as_text(tmp_path): + scan_item = _scan_item(tmp_path) + license_name = "\ub77c\uc774\uc120\uc2a4" + scan_item.append_file_items([_file("custom.c", [OssItem("custom", "", license_name)])]) + document = _write_json(tmp_path, scan_item) + + definition = document["hasExtractedLicensingInfos"][0] + assert re.fullmatch(r"LicenseRef-[0-9a-f]{8}", definition["licenseId"]) + assert definition["name"] == license_name + assert definition["extractedText"] == license_name + + +@pytest.mark.parametrize("name, expected", [ + ("Custom_License", "LicenseRef-Custom-License"), + ("Custom(License)", "LicenseRef-CustomLicense"), + ("LicenseRef-Custom", "LicenseRef-LicenseRef-Custom"), + ("\ub77c\uc774\uc120\uc2a4", "LicenseRef-\ub77c\uc774\uc120\uc2a4"), +]) +def test_public_license_helper_keeps_existing_behavior(name, expected): + from fosslight_util.write_spdx import convert_to_spdx_style + + assert convert_to_spdx_style(name) == expected + + +def test_dependency_license_expression_preserves_or_within_and(tmp_path): + scan_item = _scan_item(tmp_path, FOSSLIGHT_DEPENDENCY) + scan_item.append_file_items([ + _dependency("compound", licenses=["MIT OR Apache-2.0", "BSD-3-Clause"]), + _dependency("exception", licenses="GPL-2.0-only WITH Classpath-exception-2.0"), + ]) + document = _write_json(tmp_path, scan_item) + + assert spdx_licensing.is_equivalent( + document["packages"][1]["licenseDeclared"], "(MIT OR Apache-2.0) AND BSD-3-Clause" + ) + assert document["packages"][2]["licenseDeclared"] == "GPL-2.0-only WITH Classpath-exception-2.0" + assert "hasExtractedLicensingInfos" not in document + + +@pytest.mark.parametrize("licenses, expected", [ + (["MIT", "NONE"], "MIT"), + (["MIT", "NOASSERTION"], "MIT"), + (["MIT", "NONE", "NOASSERTION"], "MIT"), + (["MIT OR Apache-2.0", "BSD-3-Clause", "NONE", "NOASSERTION"], "(MIT OR Apache-2.0) AND BSD-3-Clause"), + (["GPL-2.0-only WITH Classpath-exception-2.0", "NONE"], "GPL-2.0-only WITH Classpath-exception-2.0"), + (["Custom License", "NOASSERTION"], "LicenseRef-Custom-License"), + (["NONE"], "NONE"), + (["NOASSERTION"], "NOASSERTION"), +]) +def test_dependency_license_declaration_preserves_real_expressions_and_standalone_special_values(tmp_path, licenses, expected): + scan_item = _scan_item(tmp_path, FOSSLIGHT_DEPENDENCY) + scan_item.append_file_items([_dependency("dependency", licenses=licenses)]) + + document = _write_json(tmp_path, scan_item) + + assert spdx_licensing.is_equivalent(document["packages"][1]["licenseDeclared"], expected) + + +@pytest.mark.parametrize("license_value", ["NONE", "NOASSERTION"]) +def test_explicit_file_license_sentinels_are_preserved(tmp_path, license_value): + scan_item = _scan_item(tmp_path) + scan_item.append_file_items([_file("example.c", [OssItem("package", "", license_value)])]) + document = _write_json(tmp_path, scan_item) + + assert document["files"][0]["licenseInfoInFiles"] == [license_value] + assert document["packages"][1]["licenseInfoFromFiles"] == [license_value] + assert "hasExtractedLicensingInfos" not in document + + +@pytest.mark.parametrize("format_name, extension", [ + ("spdx-json", ".json"), ("spdx-yaml", ".yaml"), ("spdx-xml", ".xml"), ("spdx-tag", ".tag"), +]) +@pytest.mark.parametrize("include_oci", [False, True]) +def test_all_formats_round_trip_the_same_mixed_scan(tmp_path, format_name, extension, include_oci): + scan_item = _scan_item(tmp_path, FOSSLIGHT_SCANNER) + custom_oss = OssItem("custom", "", "Custom License") + custom_oss.license_texts["Custom License"] = "Custom detected text." + scan_item.append_file_items([_file("source.c", [custom_oss])], FOSSLIGHT_SOURCE) + binary = _file("app.bin", [OssItem("binary", "", "ISC")]) + binary.tlsh = "T1" + "A" * 70 + scan_item.append_file_items([binary], FOSSLIGHT_BINARY) + purl = "pkg:npm/dependency@1.0" + scan_item.append_file_items([_dependency("dependency", purl, [], "direct", "MIT")], FOSSLIGHT_DEPENDENCY) + covers = [scan_item.cover, *[ + SimpleNamespace(tool_name=f"{scanner_name} v1.2.3") + for scanner_name in (FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY, FOSSLIGHT_DEPENDENCY) + ]] + if include_oci: + scan_item.append_file_items([_dependency("oci-dependency", purl, [], "direct", "MIT")], FOSSLIGHT_OCI) + oci_binary = _file("oci/app.bin", [OssItem("oci-binary", "", "ISC")]) + oci_binary.tlsh = binary.tlsh + scan_item.append_file_items([oci_binary], FOSSLIGHT_OCI_BINARY) + covers.append(SimpleNamespace(tool_name=f"{FOSSLIGHT_OCI} v1.2.3")) + + success, error, result_file = write_output_file( + str(tmp_path / "mixed"), extension, scan_item, format=format_name, scanner_covers=covers + ) + + assert success, error + parsed = parse_file(result_file) + assert validate_full_spdx_document(parsed) == [] + assert parsed.creation_info.spdx_version == "SPDX-2.3" + assert parsed.creation_info.name == "project" + assert len(parsed.creation_info.creators) == (6 if include_oci else 5) + assert len(parsed.packages) == (6 if include_oci else 4) + assert len(parsed.files) == (3 if include_oci else 2) + assert len(parsed.extracted_licensing_info) == 1 + assert parsed.files[1].comment == f"fosslight:tlsh={binary.tlsh}" + assert {str(value) for value in parsed.packages[1].license_info_from_files} == {"LicenseRef-Custom-License"} + if include_oci: + assert {creator.name for creator in parsed.creation_info.creators} == { + "FOSSLIGHT_SCANNER - 1.2.3", "FOSSLIGHT_Source - 1.2.3", "FOSSLIGHT_Binary - 1.2.3", + "FOSSLIGHT_Dependency - 1.2.3", "FOSSLIGHT_OCI - 1.2.3", f"spdx-tools - {version('spdx-tools')}", + } + assert parsed.packages[4].files_analyzed is False + assert str(parsed.packages[4].license_declared) == "MIT" + assert parsed.files[2].file_types == parsed.files[1].file_types + assert parsed.files[2].comment == parsed.files[1].comment + assert set(scan_item.file_items) == { + FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY, FOSSLIGHT_DEPENDENCY, FOSSLIGHT_OCI, FOSSLIGHT_OCI_BINARY, + } + if extension in (".json", ".yaml"): + text = Path(result_file).read_text(encoding="utf-8") + data = json.loads(text) if extension == ".json" else yaml.safe_load(text) + assert data["packages"][3]["externalRefs"][0]["referenceCategory"] == "PACKAGE-MANAGER" + assert "licenseConcluded" not in data["files"][0] + assert "licenseConcluded" not in data["packages"][1] + assert "licenseDeclared" not in data["packages"][1] + assert "licenseListVersion" not in data["creationInfo"] + elif extension == ".xml": + root = ElementTree.parse(result_file).getroot() + assert [element.text for element in root.iter("referenceCategory")] == ["PACKAGE-MANAGER"] * (2 if include_oci else 1) + assert not list(root.iter("licenseConcluded")) + else: + text = Path(result_file).read_text(encoding="utf-8") + assert f"ExternalRef: PACKAGE-MANAGER purl {purl}" in text + assert "LicenseConcluded:" not in text + assert "LicenseListVersion:" not in text + + +@pytest.mark.parametrize("input_path, expected_name", [ + ("/", "Root Component"), ("C:\\", "Root Component"), + ("C:\\work\\project\\", "project"), ("\\\\server\\share\\", "Root Component"), +]) +def test_document_name_normalizes_filesystem_roots_and_windows_paths(tmp_path, input_path, expected_name): + scan_item = _scan_item(tmp_path) + scan_item.cover.input_path = input_path + document = _write_json(tmp_path, scan_item) + + assert document["name"] == expected_name + assert document["packages"][0]["name"] == expected_name + + +def test_namespace_changes_on_repeated_output_and_falls_back_without_valid_tool(tmp_path): + scan_item = _scan_item(tmp_path) + scan_item.cover.tool_name = "not a tool version" + first = _write_json(tmp_path, scan_item) + second = _write_json(tmp_path, scan_item) + + assert first["documentNamespace"] != second["documentNamespace"] + for document in (first, second): + namespace = document["documentNamespace"] + prefix = "http://opensource.lge.com/spdxdocs/fosslight_scanner-" + assert namespace.startswith(prefix) + assert "#" not in namespace + assert UUID(namespace[len(prefix):]).version == 4 + + +def test_checksum_lookup_uses_analysis_path_and_does_not_mutate_input(tmp_path, monkeypatch): + scan_item = _scan_item(tmp_path) + source_path = Path(scan_item.cover.input_path) / "src" / "example.c" + source_path.parent.mkdir(parents=True) + contents = b"source contents\n" + source_path.write_bytes(contents) + file_item = _file("src\\example.c", [OssItem("package", "", "MIT")], "0") + scan_item.append_file_items([file_item]) + monkeypatch.chdir(tmp_path) + document = _write_json(tmp_path, scan_item) + + assert document["files"][0]["fileName"] == "./src/example.c" + assert document["files"][0]["checksums"] == [{ + "algorithm": "SHA1", "checksumValue": hashlib.sha1(contents).hexdigest(), + }] + assert file_item.checksum == "0" + assert file_item.source_name_or_path == "src\\example.c" + + +@pytest.mark.parametrize("name", ["", ".", "folder/"]) +def test_unidentifiable_file_paths_do_not_remove_packages(tmp_path, name): + scan_item = _scan_item(tmp_path) + scan_item.append_file_items([_file(name, [OssItem("package", "", "MIT")])]) + document = _write_json(tmp_path, scan_item) + + assert not document.get("files") + assert len(document["packages"]) == 2 + assert document["packages"][1]["licenseInfoFromFiles"] == ["NONE"] + + +@pytest.mark.parametrize("tlsh, expected_tlsh", [ + ("T1" + "A" * 70, "T1" + "A" * 70), + ("A" * 70, "A" * 70), + (" \tT1" + "A" * 70 + "\n", "T1" + "A" * 70), + (" " + "A" * 70 + " ", "A" * 70), + ("", None), + (" \t\n", None), + (None, None), + ("0", None), + ("TNULL", None), + ("T1INVALID", None), +]) +def test_binary_tlsh_is_separate_from_sha1_and_only_valid_values_are_written(tmp_path, tlsh, expected_tlsh): + scan_item = _scan_item(tmp_path, FOSSLIGHT_BINARY) + binary = _file("app.bin", [OssItem("binary", "", "MIT")]) + binary.tlsh = tlsh + scan_item.append_file_items([binary]) + document = _write_json(tmp_path, scan_item) + + file_info = document["files"][0] + assert file_info["fileTypes"] == ["BINARY"] + assert file_info["checksums"] == [{"algorithm": "SHA1", "checksumValue": "a" * 40}] + if expected_tlsh is not None: + assert file_info["comment"] == f"fosslight:tlsh={expected_tlsh}" + else: + assert "comment" not in file_info + assert binary.tlsh == tlsh + + +@pytest.mark.parametrize("name", ["-", " padded.c "]) +def test_file_name_is_not_normalized_as_package_metadata(tmp_path, name): + scan_item = _scan_item(tmp_path) + scan_item.append_file_items([_file(name, [OssItem("package", "", "MIT")])]) + document = _write_json(tmp_path, scan_item) + + assert document["files"][0]["fileName"] == f"./{name}" + + +def test_same_name_and_version_in_different_files_are_not_merged(tmp_path): + scan_item = _scan_item(tmp_path) + scan_item.append_file_items([ + _file("first.c", [OssItem("package", "1.0", "MIT")]), + _file("second.c", [OssItem("package", "1.0", "Apache-2.0")]), + ]) + document = _write_json(tmp_path, scan_item) + + assert len(document["packages"]) == 3 + assert _relationships(document, "CONTAINS") == { + ("SPDXRef-Package1", "SPDXRef-File1"), ("SPDXRef-Package2", "SPDXRef-File2"), + } + assert document["packages"][1]["licenseInfoFromFiles"] == ["MIT"] + assert document["packages"][2]["licenseInfoFromFiles"] == ["Apache-2.0"] + + +def test_bare_output_filename_is_supported(tmp_path, monkeypatch): + scan_item = _scan_item(tmp_path) + monkeypatch.chdir(tmp_path) + + success, error, result_file = spdx_writer.write_spdx("bare", ".json", scan_item) + + assert success, error + assert result_file == "bare.json" + assert (tmp_path / result_file).is_file() + + +@pytest.mark.parametrize("call_style", ["three_positional", "four_positional", "keyword"]) +def test_write_spdx_legacy_call_forms(tmp_path, call_style): + output_path = str(tmp_path / "legacy") + scan_item = _scan_item(tmp_path) + if call_style == "three_positional": + result = spdx_writer.write_spdx(output_path, ".json", scan_item) + elif call_style == "four_positional": + result = spdx_writer.write_spdx(output_path, ".json", scan_item, "2.3") + else: + result = spdx_writer.write_spdx( + output_file_without_ext=output_path, output_extension=".json", scan_item=scan_item, spdx_version="2.3" + ) + + success, error, result_file = result + assert success, error + assert result_file == output_path + ".json" + assert json.loads(Path(result_file).read_text(encoding="utf-8"))["spdxVersion"] == "SPDX-2.3" + + +def test_validation_failure_is_reported_without_creating_output(tmp_path, monkeypatch): + monkeypatch.setattr(spdx_writer, "validate_full_spdx_document", lambda document: [ + SimpleNamespace(validation_message="invalid document") + ]) + + success, error, result_file = write_output_file( + str(tmp_path / "invalid"), ".json", _scan_item(tmp_path), format="spdx-json" + ) + + assert success is False + assert "invalid document" in error + assert result_file == str(tmp_path / "invalid.json") + assert not (tmp_path / "invalid.json").exists() + + +def test_failed_serialization_returns_output_path_and_removes_partial_file(tmp_path, monkeypatch): + result_path = tmp_path / "report.json" + + def failed_write(document, file_name, **kwargs): + Path(file_name).write_text("partial", encoding="utf-8") + raise OSError("write failed") + + monkeypatch.setattr(spdx_writer.json_writer, "write_document_to_file", failed_write) + success, error, result_file = write_output_file( + str(tmp_path / "report"), ".json", _scan_item(tmp_path), format="spdx-json" + ) + + assert success is False + assert "write failed" in error + assert result_file == str(result_path) + assert not result_path.exists() + + +def test_missing_input_returns_failure(tmp_path): + success, error, result_file = spdx_writer.write_spdx( + str(tmp_path / "invalid"), ".json", None + ) + + assert success is False + assert error + assert result_file == "" + assert not (tmp_path / "invalid.json").exists()