diff --git a/scanpipe/models.py b/scanpipe/models.py index 47eba38eba..fa4e09b809 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -70,6 +70,7 @@ import saneyaml from aboutcode.api_auth import AbstractAPIToken from commoncode.fileutils import parent_directory +from commoncode.fileutils import splitext_name from cyclonedx import model as cyclonedx_model from cyclonedx.model import component as cyclonedx_component from cyclonedx.model import contact as cyclonedx_contact @@ -79,6 +80,7 @@ from licensedcode.cache import get_licensing from matchcode_toolkit.fingerprinting import IGNORED_DIRECTORY_FINGERPRINTS from packagedcode.models import build_package_uid +from packagedcode.utils import combine_expressions from packagedcode.utils import get_base_purl from packageurl import PackageURL from packageurl import normalize_qualifiers @@ -2987,6 +2989,11 @@ def is_symlink(self): """Return True, if the resource is a symlink.""" return self.type == self.Type.SYMLINK + @property + def base_name(self): + base_name, _extension = splitext_name(self.name, is_file=self.is_file) + return base_name + def get_path_segments_with_subpath(self): """ Return a list of path segment name along its subpath for this resource. @@ -3257,6 +3264,29 @@ def as_spdx(self): types=self.get_spdx_types(), ) + def update_license_expressions(self): + """ + Update the license expressions for a discovered package from it's license + detections. + """ + if not self.license_detections: + return + + license_expressions = [ + detection.get("license_expression") for detection in self.license_detections + ] + self.detected_license_expression = combine_expressions( + expressions=license_expressions, + relation="AND", + unique=True, + ) + self.detected_license_expression_spdx = str( + build_spdx_license_expression( + license_expression=self.detected_license_expression + ) + ) + self.save() + class CodebaseRelation( UUIDPKModel, @@ -3893,9 +3923,32 @@ def get_declared_license_expression_spdx(self): if self.declared_license_expression_spdx: return self.declared_license_expression_spdx elif self.declared_license_expression: - return build_spdx_license_expression(self.declared_license_expression) + return str(build_spdx_license_expression(self.declared_license_expression)) return "" + def update_license_expressions(self): + """ + Update the license expressions for a discovered package from it's license + detections. + """ + if not self.license_detections: + return + + license_expressions = [ + detection.get("license_expression") for detection in self.license_detections + ] + self.declared_license_expression = combine_expressions( + expressions=license_expressions, + relation="AND", + unique=True, + ) + self.declared_license_expression_spdx = str( + build_spdx_license_expression( + license_expression=self.declared_license_expression + ) + ) + self.save() + def as_spdx(self): """Return this DiscoveredPackage as an SPDX Package entry.""" from scanpipe.pipes import spdx diff --git a/scanpipe/pipelines/analyze_root_filesystem.py b/scanpipe/pipelines/analyze_root_filesystem.py index 69f2a21d8e..4a26d25b67 100644 --- a/scanpipe/pipelines/analyze_root_filesystem.py +++ b/scanpipe/pipelines/analyze_root_filesystem.py @@ -129,9 +129,15 @@ def scan_for_files(self): def collect_and_create_license_detections(self): """ Collect and create unique license detections from resources and - package data. + package data. Also look for license issues and try to resolve them. """ scancode.collect_and_create_license_detections(project=self.project) + scancode.follow_and_resolve_referenced_licenses_in_resources( + project=self.project + ) + scancode.follow_and_resolve_referenced_licenses_in_packages( + project=self.project + ) def analyze_scanned_files(self): """Analyze single file scan results for completeness.""" diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index 3c6313c497..f326e05ad7 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -534,9 +534,15 @@ def scan_mapped_from_for_files(self): def collect_and_create_license_detections(self): """ Collect and create unique license detections from resources and - package data. + package data. Also look for license issues and try to resolve them. """ scancode.collect_and_create_license_detections(project=self.project) + scancode.follow_and_resolve_referenced_licenses_in_resources( + project=self.project + ) + scancode.follow_and_resolve_referenced_licenses_in_packages( + project=self.project + ) def create_local_files_packages(self): """Create local-files packages for codebase resources not part of a package.""" diff --git a/scanpipe/pipelines/scan_codebase.py b/scanpipe/pipelines/scan_codebase.py index fd6580e456..e1a7bc071e 100644 --- a/scanpipe/pipelines/scan_codebase.py +++ b/scanpipe/pipelines/scan_codebase.py @@ -43,8 +43,8 @@ def steps(cls): cls.collect_and_create_codebase_resources, cls.flag_empty_files, cls.flag_ignored_resources, - cls.scan_for_application_packages, cls.scan_for_files, + cls.scan_for_application_packages, cls.collect_and_create_license_detections, ) @@ -59,17 +59,23 @@ def collect_and_create_codebase_resources(self): """Collect and create codebase resources.""" pipes.collect_and_create_codebase_resources(self.project) - def scan_for_application_packages(self): - """Scan unknown resources for packages information.""" - scancode.scan_for_application_packages(self.project, progress_logger=self.log) - def scan_for_files(self): """Scan unknown resources for copyrights, licenses, emails, and urls.""" scancode.scan_for_files(self.project, progress_logger=self.log) + def scan_for_application_packages(self): + """Scan unknown resources for packages information.""" + scancode.scan_for_application_packages(self.project, progress_logger=self.log) + def collect_and_create_license_detections(self): """ Collect and create unique license detections from resources and - package data. + package data. Also look for license issues and try to resolve them. """ scancode.collect_and_create_license_detections(project=self.project) + scancode.follow_and_resolve_referenced_licenses_in_resources( + project=self.project + ) + scancode.follow_and_resolve_referenced_licenses_in_packages( + project=self.project + ) diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 0951bd0000..03bcd651b6 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -355,7 +355,10 @@ def update_or_create_license_detection( license_detection = project.discoveredlicenses.get_or_none( identifier=detection_identifier, ) - detection_data = _clean_license_detection_data(detection_data) + detection_data = _clean_license_detection_data( + detection_data=detection_data, + path=resource_path, + ) if license_detection: license_detection.update_from_data(detection_data) @@ -390,8 +393,11 @@ def update_or_create_license_detection( return license_detection -def _clean_license_detection_data(detection_data): +def _clean_license_detection_data(detection_data, path): detection_data = detection_data.copy() + if "from_package" in detection_data: + detection_data.pop("from_package") + if "reference_matches" in detection_data: matches = detection_data.pop("reference_matches") detection_data["matches"] = matches @@ -401,6 +407,8 @@ def _clean_license_detection_data(detection_data): from_file_path = match_data["from_file"] if from_file_path: match_data["from_file"] = from_file_path.removeprefix("codebase/") + elif path: + match_data["from_file"] = path updated_matches.append(match_data) diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 9a3f557876..2042f60799 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -24,6 +24,7 @@ import logging import multiprocessing import os +import posixpath import shlex import warnings from collections import defaultdict @@ -34,9 +35,11 @@ from django.apps import apps from django.db.models import ObjectDoesNotExist from django.db.models import Q +from django.forms.models import model_to_dict from commoncode import fileutils from commoncode.resource import VirtualCodebase +from commoncode.resource import clean_path from extractcode import api as extractcode_api from licensedcode.detection import DetectionCategory from licensedcode.detection import FileRegion @@ -44,8 +47,12 @@ from licensedcode.detection import LicenseMatchFromResult from licensedcode.detection import UniqueDetection from licensedcode.detection import get_ambiguous_license_detections_by_type +from licensedcode.detection import get_referenced_filenames +from licensedcode.detection import has_resolved_referenced_file +from licensedcode.detection import update_detection_from_referenced_files from packagedcode import get_package_handler from packagedcode import models as packagedcode_models +from packagedcode.licensing import get_license_detections_from_sibling_file from scancode import Scanner from scancode import api as scancode_api from scancode import cli as scancode_cli @@ -55,6 +62,7 @@ from aboutcode.pipeline import LoopProgress from scanpipe import pipes +from scanpipe.models import AbstractLicenseDetection from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage @@ -435,7 +443,10 @@ def scan_for_application_packages( controlled through the SCANCODEIO_PROCESSES setting. """ if not resource_qs: - resource_qs = project.codebaseresources.no_status() + resource_qs = ( + project.codebaseresources.no_status() + | project.codebaseresources.filter(status=flag.SCANNED) + ) scan_func_kwargs = { "package_only": package_only, @@ -1057,6 +1068,11 @@ def create_discovered_licenses(project, scanned_codebase): def load_todo_issues(project, scanned_codebase): + """ + While loading a codebase from a full scancode result file, + load detection issues from the codebase, and mark them as + needs review. + """ if hasattr(scanned_codebase.attributes, "todo"): for todo_issue in scanned_codebase.attributes.todo: pipes.update_license_detection_with_issue(project, todo_issue) @@ -1071,6 +1087,10 @@ def load_todo_issues(project, scanned_codebase): def check_license_detection_for_issues(discovered_license): + """ + If a DiscoveredLicense instance has detection issues in it, + mark them as needs review. + """ file_regions = [ FileRegion( path=file_region.get("path"), @@ -1104,6 +1124,203 @@ def check_license_detection_for_issues(discovered_license): ) +def follow_and_resolve_referenced_licenses_in_packages(project): + + for discovered_package in project.discoveredpackages.has_no_license_detections(): + for datafile_path in discovered_package.datafile_paths: + datafile_resource = project.codebaseresources.get_or_none( + path=datafile_path + ) + if datafile_resource and datafile_resource.license_detections: + discovered_package.license_detections.extend( + datafile_resource.license_detections + ) + + discovered_package.update_license_expressions() + if discovered_package.declared_license_expression: + continue + + sibling_detections_by_id = {} + for datafile_path in discovered_package.datafile_paths: + datafile_resource = project.codebaseresources.get_or_none( + path=datafile_path + ) + if not datafile_resource: + continue + + detections, _le = get_license_detections_from_sibling_file( + resource=datafile_resource, + codebase=project, + ) + for detection in detections: + sibling_detections_by_id[detection.get("identifier")] = detection + + discovered_package.license_detections = list(sibling_detections_by_id.values()) + discovered_package.update_license_expressions() + + +def follow_and_resolve_referenced_licenses_in_resources(project): + """ + In case of a referenced license from a file/package like `See license in LICENSE.md` + try to find and resolve from the referenced resource the license detection. + """ + for discovered_license in project.discoveredlicenses.all(): + example_path = discovered_license.file_regions[0].get("path") + old_license_id = discovered_license.identifier + license_detection_mapping = model_to_dict(discovered_license) + license_detection = LicenseDetectionFromResult.from_license_detection_mapping( + license_detection_mapping=license_detection_mapping, + file_path=example_path, + ) + referenced_filenames = get_referenced_filenames(license_detection.matches) + if not referenced_filenames or has_resolved_referenced_file( + license_detection.matches + ): + continue + + resource = project.codebaseresources.get_or_none(path=example_path) + if not resource: + continue + + is_modified = update_detection_from_referenced_files( + referenced_filenames=referenced_filenames, + license_detection_mapping=license_detection_mapping, + resource=resource, + codebase=project, + analysis=DetectionCategory.UNKNOWN_FILE_REFERENCE_LOCAL.value, + find_referenced_resource_func=find_referenced_resource_in_codebase, + ) + + if is_modified: + old_license = discovered_license.license_expression + discovered_license.update_from_data( + data=license_detection_mapping, + override=True, + ) + if old_license != discovered_license.license_expression: + mark_discovered_license_as_correct_detection(discovered_license) + update_file_license_detections( + project=project, + discovered_license=discovered_license, + old_license_id=old_license_id, + ) + update_package_license_detections( + project=project, + discovered_license=discovered_license, + old_license_id=old_license_id, + ) + + +detection_fields = [field.name for field in AbstractLicenseDetection._meta.fields] + + +def mark_discovered_license_as_correct_detection(discovered_license): + """ + Mark a DiscoveredLicense instance as it doesn't need review. + Can be called after a detection issue is resolved, like by + following references. + """ + discovered_license.needs_review = False + discovered_license.review_comments = [] + discovered_license.save() + + +def update_file_license_detections(project, discovered_license, old_license_id): + """ + Post updating license detections, also update license detections and expressions + for related CodebaseResource objects. + """ + for file_region in discovered_license.file_regions: + resource = project.codebaseresources.get(path=file_region.get("path")) + update_instance_with_new_detection( + instance=resource, + discovered_license=discovered_license, + old_license_id=old_license_id, + ) + + +def update_package_license_detections(project, discovered_license, old_license_id): + """ + Post updating license detections, also update license detections and expressions + for related DiscoveredPackage objects. + """ + for file_region in discovered_license.file_regions: + resource = project.codebaseresources.get(path=file_region.get("path")) + for discovered_package in resource.discovered_packages.all(): + update_instance_with_new_detection( + instance=discovered_package, + discovered_license=discovered_license, + old_license_id=old_license_id, + ) + + +def update_instance_with_new_detection(instance, discovered_license, old_license_id): + """ + Update and save the license fields for a DiscoveredPackage or CodebaseResource + instances from a new and resolved DiscoveredLicense instance, replacing an + old license detection with `old_license_id`. + """ + if not instance.license_detections: + return + + license_ids = [ + detection.get("identifier") for detection in instance.license_detections + ] + if old_license_id not in license_ids: + return + + new_detections = [] + for detection in instance.license_detections: + if detection.get("identifier") == old_license_id: + new_detections.append( + model_to_dict(instance=discovered_license, fields=detection_fields) + ) + else: + new_detections.append(detection) + + if not new_detections: + return + + instance.license_detections = new_detections + instance.update_license_expressions() + + +def find_referenced_resource_in_codebase( + referenced_filename, resource, codebase, **kwargs +): + """ + Try to find and return a referenced resource using database calls instead of a + codebase walk first and fall back only if a resource is not found. + """ + if not (resource and resource.parent_path): + return + + # this can be a path or a plain name + referenced_filename = clean_path(referenced_filename) + referenced_path = posixpath.join(resource.parent_path, referenced_filename) + referenced_resource = codebase.get_resource(path=referenced_path) + if referenced_resource: + return referenced_resource + + for package in resource.discovered_packages.all(): + datafile_roots = [] + for datafile_path in package.datafile_paths: + datafile_resource = codebase.get_resource(path=datafile_path) + if not (datafile_resource and datafile_resource.has_parent()): + continue + + if datafile_resource.parent_path in datafile_roots: + continue + + datafile_roots.append(datafile_resource.parent_path) + referenced_path = posixpath.join( + datafile_resource.parent_path, referenced_filename + ) + referenced_resource = codebase.get_resource(path=referenced_path) + if referenced_resource: + return referenced_resource + + def set_codebase_resource_for_package(codebase_resource, discovered_package): """ Assign the `discovered_package` to the `codebase_resource` and set its diff --git a/scanpipe/tests/data/scancode/activemq-camel-5.16.8.jar b/scanpipe/tests/data/scancode/activemq-camel-5.16.8.jar new file mode 100644 index 0000000000..5fde26cc41 Binary files /dev/null and b/scanpipe/tests/data/scancode/activemq-camel-5.16.8.jar differ diff --git a/scanpipe/tests/data/scancode/built_collection-5.1.1.tar.xz b/scanpipe/tests/data/scancode/built_collection-5.1.1.tar.xz new file mode 100644 index 0000000000..83650f72e7 Binary files /dev/null and b/scanpipe/tests/data/scancode/built_collection-5.1.1.tar.xz differ diff --git a/scanpipe/tests/data/scancode/daglib-0.6.0-py3-none-any.whl_scan_codebase.json b/scanpipe/tests/data/scancode/daglib-0.6.0-py3-none-any.whl_scan_codebase.json index ac90cd5896..cc9b20dd5f 100644 --- a/scanpipe/tests/data/scancode/daglib-0.6.0-py3-none-any.whl_scan_codebase.json +++ b/scanpipe/tests/data/scancode/daglib-0.6.0-py3-none-any.whl_scan_codebase.json @@ -819,14 +819,65 @@ "tag": "", "extension": "", "programming_language": "Objective-C", - "detected_license_expression": "", - "detected_license_expression_spdx": "", - "license_detections": [], + "detected_license_expression": "mit", + "detected_license_expression_spdx": "MIT", + "license_detections": [ + { + "matches": [ + { + "score": 100.0, + "matcher": "2-aho", + "end_line": 6, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/mit_30.RULE", + "from_file": null, + "start_line": 6, + "matched_text": "License: MIT", + "match_coverage": 100.0, + "matched_length": 2, + "rule_relevance": 100, + "rule_identifier": "mit_30.RULE", + "license_expression": "mit", + "license_expression_spdx": "MIT" + } + ], + "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee", + "license_expression": "mit", + "license_expression_spdx": "MIT" + }, + { + "matches": [ + { + "score": 100.0, + "matcher": "2-aho", + "end_line": 11, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pypi_mit_license.RULE", + "from_file": null, + "start_line": 11, + "matched_text": "Classifier: License :: OSI Approved :: MIT License", + "match_coverage": 100.0, + "matched_length": 5, + "rule_relevance": 100, + "rule_identifier": "pypi_mit_license.RULE", + "license_expression": "mit", + "license_expression_spdx": "MIT" + } + ], + "identifier": "mit-24a5293c-14d7-5403-efac-1a8b7532c0e8", + "license_expression": "mit", + "license_expression_spdx": "MIT" + } + ], "license_clues": [], - "percentage_of_license_text": null, + "percentage_of_license_text": 1.13, "copyrights": [], "holders": [], - "authors": [], + "authors": [ + { + "author": "Michael Harris Author-email mharris@luabase.com", + "end_line": 8, + "start_line": 7 + } + ], "package_data": [ { "md5": null, @@ -1032,8 +1083,80 @@ "declared_license_expression_spdx": "MIT" } ], - "emails": [], - "urls": [], + "emails": [ + { + "email": "mharris@luabase.com", + "end_line": 8, + "start_line": 8 + } + ], + "urls": [ + { + "url": "https://github.com/mharrisb1/daglib", + "end_line": 5, + "start_line": 5 + }, + { + "url": "https://mharrisb1.github.io/daglib/", + "end_line": 20, + "start_line": 20 + }, + { + "url": "https://badge.fury.io/py/daglib.svg", + "end_line": 26, + "start_line": 26 + }, + { + "url": "https://badge.fury.io/py/daglib", + "end_line": 26, + "start_line": 26 + }, + { + "url": "https://img.shields.io/pypi/dm/daglib", + "end_line": 27, + "start_line": 27 + }, + { + "url": "https://pypi.org/project/daglib", + "end_line": 27, + "start_line": 27 + }, + { + "url": "https://img.shields.io/pypi/pyversions/daglib.svg", + "end_line": 28, + "start_line": 28 + }, + { + "url": "https://img.shields.io/badge/code%20style-black-000000.svg", + "end_line": 29, + "start_line": 29 + }, + { + "url": "https://github.com/ambv/black", + "end_line": 29, + "start_line": 29 + }, + { + "url": "https://img.shields.io/badge/mypy-checked-blue.svg", + "end_line": 30, + "start_line": 30 + }, + { + "url": "https://mypy.readthedocs.io/en/stable", + "end_line": 30, + "start_line": 30 + }, + { + "url": "https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white", + "end_line": 31, + "start_line": 31 + }, + { + "url": "https://github.com/pre-commit/pre-commit", + "end_line": 31, + "start_line": 31 + } + ], "md5": "b6d025c596a6b970c121819e0719b6be", "sha1": "e683bb3db97ada2163e5115f636a8615f6db71d0", "sha256": "f01ba45128006ed4748cef8bd1e7f66bd4b5f86f1a657e3f3e4fee3a4d328cf2", diff --git a/scanpipe/tests/data/scancode/flutter_playtabs_bridge.tar.xz b/scanpipe/tests/data/scancode/flutter_playtabs_bridge.tar.xz new file mode 100644 index 0000000000..4eaabe589e Binary files /dev/null and b/scanpipe/tests/data/scancode/flutter_playtabs_bridge.tar.xz differ diff --git a/scanpipe/tests/data/scancode/is-npm-1.0.0_scan_codebase.json b/scanpipe/tests/data/scancode/is-npm-1.0.0_scan_codebase.json index 7f0adab5f3..cb3e8b4fac 100644 --- a/scanpipe/tests/data/scancode/is-npm-1.0.0_scan_codebase.json +++ b/scanpipe/tests/data/scancode/is-npm-1.0.0_scan_codebase.json @@ -303,14 +303,43 @@ "tag": "", "extension": ".json", "programming_language": "", - "detected_license_expression": "", - "detected_license_expression_spdx": "", - "license_detections": [], + "detected_license_expression": "mit", + "detected_license_expression_spdx": "MIT", + "license_detections": [ + { + "matches": [ + { + "score": 100.0, + "matcher": "2-aho", + "end_line": 5, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/mit_30.RULE", + "from_file": null, + "start_line": 5, + "matched_text": " \"license\": \"MIT\",", + "match_coverage": 100.0, + "matched_length": 2, + "rule_relevance": 100, + "rule_identifier": "mit_30.RULE", + "license_expression": "mit", + "license_expression_spdx": "MIT" + } + ], + "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee", + "license_expression": "mit", + "license_expression_spdx": "MIT" + } + ], "license_clues": [], - "percentage_of_license_text": null, + "percentage_of_license_text": 3.33, "copyrights": [], "holders": [], - "authors": [], + "authors": [ + { + "author": "Sindre Sorhus", + "end_line": 8, + "start_line": 7 + } + ], "package_data": [ { "md5": null, @@ -409,8 +438,20 @@ "declared_license_expression_spdx": "MIT" } ], - "emails": [], - "urls": [], + "emails": [ + { + "email": "sindresorhus@gmail.com", + "end_line": 9, + "start_line": 9 + } + ], + "urls": [ + { + "url": "http://sindresorhus.com/", + "end_line": 10, + "start_line": 10 + } + ], "md5": "c843e88ecb274d5d573c71be330bff8b", "sha1": "943efb8abda9b9cdaa4ea73f5709563b948af50b", "sha256": "522879426298e078881de533b9b3a82fa3bece1336a84bddacf4620008b5d0a3", diff --git a/scanpipe/tests/data/scancode/unknown-ref-to-key-file-root.tar.xz b/scanpipe/tests/data/scancode/unknown-ref-to-key-file-root.tar.xz new file mode 100644 index 0000000000..7b5f9cd713 Binary files /dev/null and b/scanpipe/tests/data/scancode/unknown-ref-to-key-file-root.tar.xz differ diff --git a/scanpipe/tests/pipes/test_scancode.py b/scanpipe/tests/pipes/test_scancode.py index 05cc1033d0..ce9756631f 100644 --- a/scanpipe/tests/pipes/test_scancode.py +++ b/scanpipe/tests/pipes/test_scancode.py @@ -746,3 +746,75 @@ def test_scanpipe_pipes_scancode_scan_single_package_correct_parent_path(self): file1 = project1.codebaseresources.get(path="package/index.js") self.assertEqual("package", file1.parent_path) + + def test_scanpipe_scancode_follow_and_resolve_referenced_licenses_resources(self): + project1 = Project.objects.create(name="Utility: PurlDB") + archive_location = ( + self.data / "scancode" / "unknown-ref-to-key-file-root.tar.xz" + ) + copy_input(archive_location, project1.codebase_path) + scancode.extract_archives(project1.codebase_path) + collect_and_create_codebase_resources(project1) + scancode.scan_for_files(project1) + scancode.scan_for_application_packages(project1, assemble=True) + scancode.collect_and_create_license_detections(project1) + scancode.follow_and_resolve_referenced_licenses_in_resources(project1) + scancode.follow_and_resolve_referenced_licenses_in_packages(project1) + + file1 = project1.codebaseresources.get( + path="unknown-ref-to-key-file-root.tar.xz-extract/unknown-ref-to-key-file-root/esprima-compare.js" + ) + self.assertEqual(file1.detected_license_expression, "mit") + self.assertIn( + "unknown-reference-to-local-file", + file1.license_detections[0].get("detection_log"), + ) + + def test_scanpipe_scancode_follow_and_resolve_referenced_licenses_packages(self): + project1 = Project.objects.create(name="Analysis") + archive_location = self.data / "scancode" / "flutter_playtabs_bridge.tar.xz" + copy_input(archive_location, project1.codebase_path) + scancode.extract_archives(project1.codebase_path) + collect_and_create_codebase_resources(project1) + scancode.scan_for_files(project1) + scancode.scan_for_application_packages(project1, assemble=True) + scancode.collect_and_create_license_detections(project1) + scancode.follow_and_resolve_referenced_licenses_in_resources(project1) + scancode.follow_and_resolve_referenced_licenses_in_packages(project1) + + package1 = project1.discoveredpackages.get(name="flutter_paytabs_bridge") + self.assertEqual(package1.declared_license_expression, "mit") + self.assertIn( + "unknown-reference-to-local-file", + package1.license_detections[0].get("detection_log"), + ) + + def test_scanpipe_scancode_package_license_from_manifest_file(self): + project1 = Project.objects.create(name="Analysis") + archive_location = self.data / "scancode" / "activemq-camel-5.16.8.jar" + copy_input(archive_location, project1.codebase_path) + scancode.extract_archives(project1.codebase_path) + collect_and_create_codebase_resources(project1) + scancode.scan_for_files(project1) + scancode.scan_for_application_packages(project1) + scancode.collect_and_create_license_detections(project1) + scancode.follow_and_resolve_referenced_licenses_in_resources(project1) + scancode.follow_and_resolve_referenced_licenses_in_packages(project1) + + package1 = project1.discoveredpackages.get(name="activemq-camel") + self.assertEqual(package1.declared_license_expression, "apache-2.0") + + def test_scanpipe_scancode_package_license_from_beside_manifest_file(self): + project1 = Project.objects.create(name="Analysis") + archive_location = self.data / "scancode" / "built_collection-5.1.1.tar.xz" + copy_input(archive_location, project1.codebase_path) + scancode.extract_archives(project1.codebase_path) + collect_and_create_codebase_resources(project1) + scancode.scan_for_files(project1) + scancode.scan_for_application_packages(project1) + scancode.collect_and_create_license_detections(project1) + scancode.follow_and_resolve_referenced_licenses_in_resources(project1) + scancode.follow_and_resolve_referenced_licenses_in_packages(project1) + + package1 = project1.discoveredpackages.get(name="built_collection") + self.assertEqual(package1.declared_license_expression, "bsd-new")