Repository navigation
Follow and resolve license references #2237
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -70,6 +70,7 @@ | |||||||||
| import saneyaml | ||||||||||
| from aboutcode.api_auth import AbstractAPIToken | ||||||||||
| from commoncode.fileutils import parent_directory | ||||||||||
| from commoncode.fileutils import splitext_name | ||||||||||
| from cyclonedx import model as cyclonedx_model | ||||||||||
| from cyclonedx.model import component as cyclonedx_component | ||||||||||
| from cyclonedx.model import contact as cyclonedx_contact | ||||||||||
|
|
@@ -79,6 +80,7 @@ | |||||||||
| from licensedcode.cache import get_licensing | ||||||||||
| from matchcode_toolkit.fingerprinting import IGNORED_DIRECTORY_FINGERPRINTS | ||||||||||
| from packagedcode.models import build_package_uid | ||||||||||
| from packagedcode.utils import combine_expressions | ||||||||||
| from packagedcode.utils import get_base_purl | ||||||||||
| from packageurl import PackageURL | ||||||||||
| from packageurl import normalize_qualifiers | ||||||||||
|
|
@@ -2987,6 +2989,11 @@ def is_symlink(self): | |||||||||
| """Return True, if the resource is a symlink.""" | ||||||||||
| return self.type == self.Type.SYMLINK | ||||||||||
|
|
||||||||||
| @property | ||||||||||
| def base_name(self): | ||||||||||
| base_name, _extension = splitext_name(self.name, is_file=self.is_file) | ||||||||||
| return base_name | ||||||||||
|
|
||||||||||
| def get_path_segments_with_subpath(self): | ||||||||||
| """ | ||||||||||
| Return a list of path segment name along its subpath for this resource. | ||||||||||
|
|
@@ -3257,6 +3264,29 @@ def as_spdx(self): | |||||||||
| types=self.get_spdx_types(), | ||||||||||
| ) | ||||||||||
|
|
||||||||||
| def update_license_expressions(self): | ||||||||||
| """ | ||||||||||
| Update the license expressions for a discovered package from it's license | ||||||||||
| detections. | ||||||||||
|
Comment on lines
+3269
to
+3270
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Copy/paste: this is a resource, not a package. Also "its", not "it's".
Suggested change
|
||||||||||
| """ | ||||||||||
| if not self.license_detections: | ||||||||||
| return | ||||||||||
|
|
||||||||||
| license_expressions = [ | ||||||||||
| detection.get("license_expression") for detection in self.license_detections | ||||||||||
| ] | ||||||||||
| self.detected_license_expression = combine_expressions( | ||||||||||
| expressions=license_expressions, | ||||||||||
| relation="AND", | ||||||||||
| unique=True, | ||||||||||
| ) | ||||||||||
| self.detected_license_expression_spdx = str( | ||||||||||
| build_spdx_license_expression( | ||||||||||
| license_expression=self.detected_license_expression | ||||||||||
| ) | ||||||||||
| ) | ||||||||||
| self.save() | ||||||||||
|
Comment on lines
+3267
to
+3288
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This duplicates The |
||||||||||
|
|
||||||||||
|
|
||||||||||
| class CodebaseRelation( | ||||||||||
| UUIDPKModel, | ||||||||||
|
|
@@ -3893,9 +3923,32 @@ def get_declared_license_expression_spdx(self): | |||||||||
| if self.declared_license_expression_spdx: | ||||||||||
| return self.declared_license_expression_spdx | ||||||||||
| elif self.declared_license_expression: | ||||||||||
| return build_spdx_license_expression(self.declared_license_expression) | ||||||||||
| return str(build_spdx_license_expression(self.declared_license_expression)) | ||||||||||
| return "" | ||||||||||
|
|
||||||||||
| def update_license_expressions(self): | ||||||||||
| """ | ||||||||||
| Update the license expressions for a discovered package from it's license | ||||||||||
| detections. | ||||||||||
| """ | ||||||||||
| if not self.license_detections: | ||||||||||
| return | ||||||||||
|
|
||||||||||
| license_expressions = [ | ||||||||||
| detection.get("license_expression") for detection in self.license_detections | ||||||||||
| ] | ||||||||||
| self.declared_license_expression = combine_expressions( | ||||||||||
| expressions=license_expressions, | ||||||||||
| relation="AND", | ||||||||||
| unique=True, | ||||||||||
| ) | ||||||||||
| self.declared_license_expression_spdx = str( | ||||||||||
| build_spdx_license_expression( | ||||||||||
| license_expression=self.declared_license_expression | ||||||||||
| ) | ||||||||||
| ) | ||||||||||
| self.save() | ||||||||||
|
Comment on lines
+3929
to
+3950
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same as above. Duplicated. |
||||||||||
|
|
||||||||||
| def as_spdx(self): | ||||||||||
| """Return this DiscoveredPackage as an SPDX Package entry.""" | ||||||||||
| from scanpipe.pipes import spdx | ||||||||||
|
|
||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -43,8 +43,8 @@ def steps(cls): | |
| cls.collect_and_create_codebase_resources, | ||
| cls.flag_empty_files, | ||
| cls.flag_ignored_resources, | ||
| cls.scan_for_application_packages, | ||
| cls.scan_for_files, | ||
| cls.scan_for_application_packages, | ||
|
Comment on lines
46
to
+47
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why this change? Manifests are now file scanned too, which is where the new data in the expected |
||
| cls.collect_and_create_license_detections, | ||
| ) | ||
|
|
||
|
|
@@ -59,17 +59,23 @@ def collect_and_create_codebase_resources(self): | |
| """Collect and create codebase resources.""" | ||
| pipes.collect_and_create_codebase_resources(self.project) | ||
|
|
||
| def scan_for_application_packages(self): | ||
| """Scan unknown resources for packages information.""" | ||
| scancode.scan_for_application_packages(self.project, progress_logger=self.log) | ||
|
|
||
| def scan_for_files(self): | ||
| """Scan unknown resources for copyrights, licenses, emails, and urls.""" | ||
| scancode.scan_for_files(self.project, progress_logger=self.log) | ||
|
|
||
| def scan_for_application_packages(self): | ||
| """Scan unknown resources for packages information.""" | ||
| scancode.scan_for_application_packages(self.project, progress_logger=self.log) | ||
|
|
||
| def collect_and_create_license_detections(self): | ||
| """ | ||
| Collect and create unique license detections from resources and | ||
| package data. | ||
| package data. Also look for license issues and try to resolve them. | ||
| """ | ||
| scancode.collect_and_create_license_detections(project=self.project) | ||
| scancode.follow_and_resolve_referenced_licenses_in_resources( | ||
| project=self.project | ||
| ) | ||
| scancode.follow_and_resolve_referenced_licenses_in_packages( | ||
| project=self.project | ||
| ) | ||
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -355,7 +355,10 @@ def update_or_create_license_detection( | |||||||
| license_detection = project.discoveredlicenses.get_or_none( | ||||||||
| identifier=detection_identifier, | ||||||||
| ) | ||||||||
| detection_data = _clean_license_detection_data(detection_data) | ||||||||
| detection_data = _clean_license_detection_data( | ||||||||
| detection_data=detection_data, | ||||||||
| path=resource_path, | ||||||||
| ) | ||||||||
|
|
||||||||
| if license_detection: | ||||||||
| license_detection.update_from_data(detection_data) | ||||||||
|
|
@@ -390,8 +393,11 @@ def update_or_create_license_detection( | |||||||
| return license_detection | ||||||||
|
|
||||||||
|
|
||||||||
| def _clean_license_detection_data(detection_data): | ||||||||
| def _clean_license_detection_data(detection_data, path): | ||||||||
| detection_data = detection_data.copy() | ||||||||
| if "from_package" in detection_data: | ||||||||
| detection_data.pop("from_package") | ||||||||
|
Comment on lines
+398
to
+399
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Where does
Suggested change
|
||||||||
|
|
||||||||
| if "reference_matches" in detection_data: | ||||||||
| matches = detection_data.pop("reference_matches") | ||||||||
| detection_data["matches"] = matches | ||||||||
|
|
@@ -401,6 +407,8 @@ def _clean_license_detection_data(detection_data): | |||||||
| from_file_path = match_data["from_file"] | ||||||||
| if from_file_path: | ||||||||
| match_data["from_file"] = from_file_path.removeprefix("codebase/") | ||||||||
| elif path: | ||||||||
| match_data["from_file"] = path | ||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why is this needed? A |
||||||||
|
|
||||||||
| updated_matches.append(match_data) | ||||||||
|
|
||||||||
|
|
||||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nothing in ScanCode.io calls this, I guess it's required by the toolkit
is_legal_or_readme. A short comment saying so would keep it from being removed later.