Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 54 additions & 1 deletion scanpipe/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import saneyaml
from aboutcode.api_auth import AbstractAPIToken
from commoncode.fileutils import parent_directory
from commoncode.fileutils import splitext_name
from cyclonedx import model as cyclonedx_model
from cyclonedx.model import component as cyclonedx_component
from cyclonedx.model import contact as cyclonedx_contact
Expand All @@ -79,6 +80,7 @@
from licensedcode.cache import get_licensing
from matchcode_toolkit.fingerprinting import IGNORED_DIRECTORY_FINGERPRINTS
from packagedcode.models import build_package_uid
from packagedcode.utils import combine_expressions
from packagedcode.utils import get_base_purl
from packageurl import PackageURL
from packageurl import normalize_qualifiers
Expand Down Expand Up @@ -2987,6 +2989,11 @@ def is_symlink(self):
"""Return True, if the resource is a symlink."""
return self.type == self.Type.SYMLINK

@property
def base_name(self):
base_name, _extension = splitext_name(self.name, is_file=self.is_file)
return base_name
Comment on lines +2992 to +2995

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nothing in ScanCode.io calls this, I guess it's required by the toolkit is_legal_or_readme. A short comment saying so would keep it from being removed later.


def get_path_segments_with_subpath(self):
"""
Return a list of path segment name along its subpath for this resource.
Expand Down Expand Up @@ -3257,6 +3264,29 @@ def as_spdx(self):
types=self.get_spdx_types(),
)

def update_license_expressions(self):
"""
Update the license expressions for a discovered package from it's license
detections.
Comment on lines +3269 to +3270

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy/paste: this is a resource, not a package. Also "its", not "it's".

Suggested change
Update the license expressions for a discovered package from it's license
detections.
Update the license expressions for this resource from its license
detections.

"""
if not self.license_detections:
return

license_expressions = [
detection.get("license_expression") for detection in self.license_detections
]
self.detected_license_expression = combine_expressions(
expressions=license_expressions,
relation="AND",
unique=True,
)
self.detected_license_expression_spdx = str(
build_spdx_license_expression(
license_expression=self.detected_license_expression
)
)
self.save()
Comment on lines +3267 to +3288

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This duplicates DiscoveredPackage.update_license_expressions, only the field names change. It also relies on the caller setting license_detections on the instance beforehand, then does a full save(). What about a shared function for the expressions, and a method that takes the detections and uses update()

The str() calls are not needed: combine_expressions and build_spdx_license_expression already return a str.



class CodebaseRelation(
UUIDPKModel,
Expand Down Expand Up @@ -3893,9 +3923,32 @@ def get_declared_license_expression_spdx(self):
if self.declared_license_expression_spdx:
return self.declared_license_expression_spdx
elif self.declared_license_expression:
return build_spdx_license_expression(self.declared_license_expression)
return str(build_spdx_license_expression(self.declared_license_expression))
return ""

def update_license_expressions(self):
"""
Update the license expressions for a discovered package from it's license
detections.
"""
if not self.license_detections:
return

license_expressions = [
detection.get("license_expression") for detection in self.license_detections
]
self.declared_license_expression = combine_expressions(
expressions=license_expressions,
relation="AND",
unique=True,
)
self.declared_license_expression_spdx = str(
build_spdx_license_expression(
license_expression=self.declared_license_expression
)
)
self.save()
Comment on lines +3929 to +3950

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as above. Duplicated.


def as_spdx(self):
"""Return this DiscoveredPackage as an SPDX Package entry."""
from scanpipe.pipes import spdx
Expand Down
8 changes: 7 additions & 1 deletion scanpipe/pipelines/analyze_root_filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,15 @@ def scan_for_files(self):
def collect_and_create_license_detections(self):
"""
Collect and create unique license detections from resources and
package data.
package data. Also look for license issues and try to resolve them.
"""
scancode.collect_and_create_license_detections(project=self.project)
scancode.follow_and_resolve_referenced_licenses_in_resources(
project=self.project
)
scancode.follow_and_resolve_referenced_licenses_in_packages(
project=self.project
)

def analyze_scanned_files(self):
"""Analyze single file scan results for completeness."""
Expand Down
8 changes: 7 additions & 1 deletion scanpipe/pipelines/deploy_to_develop.py
Original file line number Diff line number Diff line change
Expand Up @@ -534,9 +534,15 @@ def scan_mapped_from_for_files(self):
def collect_and_create_license_detections(self):
"""
Collect and create unique license detections from resources and
package data.
package data. Also look for license issues and try to resolve them.
"""
scancode.collect_and_create_license_detections(project=self.project)
scancode.follow_and_resolve_referenced_licenses_in_resources(
project=self.project
)
scancode.follow_and_resolve_referenced_licenses_in_packages(
project=self.project
)

def create_local_files_packages(self):
"""Create local-files packages for codebase resources not part of a package."""
Expand Down
18 changes: 12 additions & 6 deletions scanpipe/pipelines/scan_codebase.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ def steps(cls):
cls.collect_and_create_codebase_resources,
cls.flag_empty_files,
cls.flag_ignored_resources,
cls.scan_for_application_packages,
cls.scan_for_files,
cls.scan_for_application_packages,
Comment on lines 46 to +47

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why this change? Manifests are now file scanned too, which is where the new data in the expected *_scan_codebase.json files comes from. Resources flagged by scan_for_files as scanned-with-error (timeout) or ignored-by-max-file-size are also no longer scanned for packages. A package-lock.json over SCAN_MAX_FILE_SIZE would lose all its dependencies. The class docstring ("does not further scan the files contained in a package") is out of sync as well.

cls.collect_and_create_license_detections,
)

Expand All @@ -59,17 +59,23 @@ def collect_and_create_codebase_resources(self):
"""Collect and create codebase resources."""
pipes.collect_and_create_codebase_resources(self.project)

def scan_for_application_packages(self):
"""Scan unknown resources for packages information."""
scancode.scan_for_application_packages(self.project, progress_logger=self.log)

def scan_for_files(self):
"""Scan unknown resources for copyrights, licenses, emails, and urls."""
scancode.scan_for_files(self.project, progress_logger=self.log)

def scan_for_application_packages(self):
"""Scan unknown resources for packages information."""
scancode.scan_for_application_packages(self.project, progress_logger=self.log)

def collect_and_create_license_detections(self):
"""
Collect and create unique license detections from resources and
package data.
package data. Also look for license issues and try to resolve them.
"""
scancode.collect_and_create_license_detections(project=self.project)
scancode.follow_and_resolve_referenced_licenses_in_resources(
project=self.project
)
scancode.follow_and_resolve_referenced_licenses_in_packages(
project=self.project
)
12 changes: 10 additions & 2 deletions scanpipe/pipes/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,10 @@ def update_or_create_license_detection(
license_detection = project.discoveredlicenses.get_or_none(
identifier=detection_identifier,
)
detection_data = _clean_license_detection_data(detection_data)
detection_data = _clean_license_detection_data(
detection_data=detection_data,
path=resource_path,
)

if license_detection:
license_detection.update_from_data(detection_data)
Expand Down Expand Up @@ -390,8 +393,11 @@ def update_or_create_license_detection(
return license_detection


def _clean_license_detection_data(detection_data):
def _clean_license_detection_data(detection_data, path):
detection_data = detection_data.copy()
if "from_package" in detection_data:
detection_data.pop("from_package")
Comment on lines +398 to +399

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where does from_package come from in the detection data?

Suggested change
if "from_package" in detection_data:
detection_data.pop("from_package")
detection_data.pop("from_package", None)


if "reference_matches" in detection_data:
matches = detection_data.pop("reference_matches")
detection_data["matches"] = matches
Expand All @@ -401,6 +407,8 @@ def _clean_license_detection_data(detection_data):
from_file_path = match_data["from_file"]
if from_file_path:
match_data["from_file"] = from_file_path.removeprefix("codebase/")
elif path:
match_data["from_file"] = path

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this needed? A DiscoveredLicense is shared by all files with the same detection, so the stored from_file is only the path of the first file processed. This changes the stored matches for every pipeline.


updated_matches.append(match_data)

Expand Down
Loading
Loading