From 15b8b3841711be9ebd2ade07d1c8f4156d59fbb1 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Mon, 14 Nov 2022 15:03:29 +0100 Subject: [PATCH 1/8] [issue-181] add files at document-level leads to failing test concerning rdf-files Signed-off-by: Meret Behrens --- spdx/document.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/spdx/document.py b/spdx/document.py index c41d04666..f2d76e239 100644 --- a/spdx/document.py +++ b/spdx/document.py @@ -8,12 +8,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import List, Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from spdx.file import File from spdx.license import ExtractedLicense from spdx.parsers.loggers import ErrorMessages import warnings from functools import total_ordering +from spdx.relationship import Relationship @total_ordering @@ -48,7 +53,7 @@ def __lt__(self, other): other.check_sum, ) - def validate(self, messages): + def validate(self, messages: ErrorMessages) -> ErrorMessages: """ Check that all the fields are valid. Appends any error messages to messages parameter shall be a ErrorMessages. @@ -56,6 +61,7 @@ def validate(self, messages): self.validate_ext_doc_id(messages) self.validate_spdx_doc_uri(messages) self.validate_checksum(messages) + return messages def validate_ext_doc_id(self, messages): if not self.external_document_id: @@ -118,13 +124,14 @@ def __init__( self.comment = comment self.namespace = namespace self.creation_info = CreationInfo() + self.files: Optional[List['File']] = [] self.packages = [] if package is not None: self.packages.append(package) self.extracted_licenses = [] self.reviews = [] self.annotations = [] - self.relationships = [] + self.relationships: List[Relationship] = [] self.snippet = [] def add_review(self, review): @@ -148,6 +155,9 @@ def add_snippet(self, snip): def add_package(self, package): self.packages.append(package) + def add_file(self, file: 'File') -> None: + self.files.append(file) + # For backwards compatibility with older versions, we support a # mode where the first package in a document may be referred to as # the document's "package", and the files it contains may be @@ -173,17 +183,6 @@ def package(self, value): else: self.packages[0] = value - @property - def files(self): - if self.packages: - return self.package.files - else: - return [] - - @files.setter - def files(self, value): - self.package.files = value - @property def has_comment(self): return self.comment is not None @@ -206,6 +205,7 @@ def validate(self, messages=None): self.validate_namespace(messages) self.validate_ext_document_references(messages) self.validate_creation_info(messages) + self.validate_files(messages) self.validate_packages(messages) self.validate_extracted_licenses(messages) self.validate_reviews(messages) @@ -251,10 +251,15 @@ def validate_ext_document_references(self, messages): "External document references must be of the type " "spdx.document.ExternalDocumentRef and not " + str(type(doc)) ] + def validate_reviews(self, messages): for review in self.reviews: messages = review.validate(messages) + def validate_files(self, messages: ErrorMessages) -> None: + for file in self.files: + messages = file.validate(messages) + def validate_annotations(self, messages): for annotation in self.annotations: messages = annotation.validate(messages) From 50e318f95dbbfac6eb2881f3e86ea47afe9d7dc0 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Tue, 15 Nov 2022 10:57:28 +0100 Subject: [PATCH 2/8] [issue-181] add support for files at document-level in jsonyamlxml writer/parser leads to failing tests for rdf and tv files Signed-off-by: Meret Behrens --- spdx/document.py | 2 + spdx/parsers/jsonyamlxml.py | 43 +- spdx/parsers/tagvaluebuilders.py | 17 +- spdx/utils.py | 8 + spdx/writers/jsonyamlxml.py | 112 +++-- spdx/writers/tagvalue.py | 8 + tests/data/doc_parse/SBOMexpected.json | 293 ++++++------ tests/data/doc_parse/expected.json | 611 +++++++++++++------------ tests/test_document.py | 11 +- tests/test_jsonyamlxml_writer.py | 57 ++- tests/test_rdf_writer.py | 6 +- tests/test_tag_value_parser.py | 6 +- tests/utils_test.py | 19 +- 13 files changed, 661 insertions(+), 532 deletions(-) diff --git a/spdx/document.py b/spdx/document.py index f2d76e239..afddf4e97 100644 --- a/spdx/document.py +++ b/spdx/document.py @@ -20,6 +20,8 @@ from functools import total_ordering from spdx.relationship import Relationship +from spdx.relationship import Relationship + @total_ordering class ExternalDocumentRef(object): diff --git a/spdx/parsers/jsonyamlxml.py b/spdx/parsers/jsonyamlxml.py index 1850d1227..61fa04751 100644 --- a/spdx/parsers/jsonyamlxml.py +++ b/spdx/parsers/jsonyamlxml.py @@ -10,12 +10,12 @@ # limitations under the License. from datetime import datetime from enum import Enum, auto -from typing import List, Dict, Tuple +from typing import List, Dict, Tuple, Callable from spdx import document from spdx import utils from spdx.license import LicenseConjunction, LicenseDisjunction -from spdx.package import ExternalPackageRef, PackagePurpose +from spdx.package import ExternalPackageRef, PackagePurpose, Package from spdx.parsers import rdf from spdx.parsers.builderexceptions import SPDXValueError, CardinalityError, OrderError from spdx.parsers.loggers import ErrorMessages @@ -1110,6 +1110,16 @@ def parse_file_chksum(self, file_chksum): else: self.value_error("FILE_CHECKSUM", file_chksum) + def parse_files(self, files: List[Dict]) -> None: + if files is None: + return + if isinstance(files, list): + for file in files: + self.parse_file(file) + else: + self.value_error("FILES", files) + + class PackageParser(BaseParser): def __init__(self, builder, logger): @@ -1120,7 +1130,7 @@ def package(self): # current package being parsed is the last one return self.document.packages[-1] - def parse_package(self, package): + def parse_package(self, package: Package, method_to_parse_relationship: Callable): """ Parse Package Information fields - package: Python dict with Package Information fields in it @@ -1149,7 +1159,7 @@ def parse_package(self, package): self.parse_pkg_description(package.get("description")) self.parse_annotations(package.get("annotations"), spdx_id=package.get("SPDXID")) self.parse_pkg_attribution_text(package.get("attributionTexts")) - self.parse_pkg_files(package.get("files")) + self.parse_pkg_files(package.get("hasFiles"), method_to_parse_relationship) self.parse_pkg_chksum(package.get("sha1")) self.parse_package_external_refs(package.get("externalRefs")) self.parse_primary_package_purpose(package.get("primaryPackagePurpose")) @@ -1561,24 +1571,24 @@ def parse_pkg_description(self, pkg_description): elif pkg_description is not None: self.value_error("PKG_DESCRIPTION", pkg_description) - def parse_pkg_files(self, pkg_files): + def parse_pkg_files(self, pkg_has_files: List[str], method_to_parse_relationship: Callable) -> None: """ Parse Package files - - pkg_files: Python list of dicts as in FileParser.parse_file + - pkg_has_files: Python list of spdx_ids """ if not self.package.are_files_analyzed: - if pkg_files is not None: - self.value_error("PKG_FILES", pkg_files) + if pkg_has_files is not None: + self.value_error("PKG_FILES", pkg_has_files) return - if isinstance(pkg_files, list): - for pkg_file in pkg_files: - if isinstance(pkg_file, dict): - self.parse_file(pkg_file.get("File")) + if isinstance(pkg_has_files, list): + for pkg_file_spdx_id in pkg_has_files: + if isinstance(pkg_file_spdx_id, str): + method_to_parse_relationship(self.package.spdx_id, "CONTAINS", pkg_file_spdx_id) else: - self.value_error("PKG_FILE", pkg_file) - elif pkg_files is not None: - self.value_error("PKG_FILES", pkg_files) + self.value_error("PKG_FILE", pkg_file_spdx_id) + elif pkg_has_files is not None: + self.value_error("PKG_HAS_FILES", pkg_has_files) def parse_pkg_chksum(self, pkg_chksum): """ @@ -1730,6 +1740,7 @@ def parse(self): self.parse_snippets(self.document_object.get("snippets")) self.parse_packages(self.document_object.get("packages")) + self.parse_files(self.document_object.get("files")) if self.document_object.get("documentDescribes"): self.parse_doc_described_objects(self.document_object.get("documentDescribes")) @@ -1853,7 +1864,7 @@ def parse_packages(self, packages): return if isinstance(packages, list): for package in packages: - self.parse_package(package) + self.parse_package(package, self.parse_relationship) return True else: self.value_error("PACKAGES", packages) diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index a7718de69..9bdd5f595 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -1114,13 +1114,7 @@ def __init__(self): self.reset_file_stat() def set_file_name(self, doc, name): - """ - Raise OrderError if no package defined. - """ - if not self.has_package(doc): - raise OrderError("File::Name") - - doc.packages[-1].files.append(file.File(name)) + doc.files.append(file.File(name)) # A file name marks the start of a new file instance. # The builder must be reset # FIXME: this state does not make sense @@ -1342,16 +1336,15 @@ def set_file_atrificat_of_project(self, doc, symbol, value): def file(self, doc): """ - Return the last file in the document's package's file list. + Return the last file in the document's file list. """ - return doc.packages[-1].files[-1] + return doc.files[-1] def has_file(self, doc): """ - Return true if the document's package has at least one file. - Does not test if the document has a package. + Return true if the document has at least one file. """ - return len(doc.packages[-1].files) != 0 + return len(doc.files) != 0 def has_package(self, doc): """ diff --git a/spdx/utils.py b/spdx/utils.py index d691edb06..98c0b88c1 100644 --- a/spdx/utils.py +++ b/spdx/utils.py @@ -11,6 +11,7 @@ import datetime import re +from typing import Dict from ply import lex from ply import yacc @@ -203,3 +204,10 @@ def parse(self, data): return self.yacc.parse(data, lexer=self.lex) except: return None + + +def update_dict_item_with_new_item(current_state: Dict, key: str, item_to_add: str) -> None: + if key not in current_state: + current_state[key] = [item_to_add] + elif item_to_add not in current_state[key]: + current_state[key].append(item_to_add) diff --git a/spdx/writers/jsonyamlxml.py b/spdx/writers/jsonyamlxml.py index a33f5833c..28d57f8d9 100644 --- a/spdx/writers/jsonyamlxml.py +++ b/spdx/writers/jsonyamlxml.py @@ -8,12 +8,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict +from typing import Dict, List from rdflib import Literal from spdx import license, utils from spdx.package import ExternalPackageRef +from spdx.relationship import Relationship +from spdx.utils import update_dict_item_with_new_item class BaseWriter(object): @@ -200,16 +202,9 @@ def create_package_info(self, package, annotations_by_spdx_id): package_object["externalRefs"] = [self.external_reference_as_dict(external_ref) for external_ref in package.pkg_ext_refs] if package.spdx_id in annotations_by_spdx_id: - package["annotations"] = annotations_by_spdx_id[package.spdx_id] + package_object["annotations"] = annotations_by_spdx_id[package.spdx_id] - files_in_package = [] - if package.has_optional_field("files"): - package_object["hasFiles"] = [] - for file in package.files: - package_object["hasFiles"].append(file.spdx_id) - files_in_package.append(self.create_file_info(file, annotations_by_spdx_id)) - - return package_object, files_in_package + return package_object class FileWriter(BaseWriter): @@ -364,26 +359,17 @@ class RelationshipInfoWriter(BaseWriter): def __init__(self, document): super(RelationshipInfoWriter, self).__init__(document) - def create_relationship_info(self): - relationship_objects = [] - - for relationship_term in self.document.relationships: - if relationship_term.relationship_type == "DESCRIBES": - continue - if relationship_term.relationship_type == "CONTAINS": - continue - relationship_object = dict() - relationship_object["spdxElementId"] = relationship_term.spdx_element_id - relationship_object[ - "relatedSpdxElement" - ] = relationship_term.related_spdx_element - relationship_object["relationshipType"] = relationship_term.relationship_type - if relationship_term.has_comment: - relationship_object["comment"] = relationship_term.relationship_comment - - relationship_objects.append(relationship_object) + def create_relationship_info(self, relationship: Relationship): + relationship_object = dict() + relationship_object["spdxElementId"] = relationship.spdx_element_id + relationship_object[ + "relatedSpdxElement" + ] = relationship.related_spdx_element + relationship_object["relationshipType"] = relationship.relationship_type + if relationship.has_comment: + relationship_object["comment"] = relationship.relationship_comment - return relationship_objects + return relationship_object class SnippetWriter(BaseWriter): @@ -444,7 +430,6 @@ def create_snippet_info(self, annotations_by_spdx_id): return snippet_info_objects - class ExtractedLicenseWriter(BaseWriter): """ Represent spdx.document.ExtractedLicense as json-serializable objects @@ -549,20 +534,44 @@ def create_ext_document_references(self): return ext_document_reference_objects - def create_document_describes(self): - """ - Create a list of SPDXIDs that the document describes according to DESCRIBES-relationship. - """ - document_describes = [] - remove_rel = [] + def create_relationships(self) -> List[Dict]: + packages_spdx_ids = [package.spdx_id for package in self.document.packages] + files_spdx_ids = [file.spdx_id for file in self.document.files] + # we take the package_objects from document_object because we will modify them to add jsonyamlxml-specific fields + packages_by_spdx_id = {package["SPDXID"]: package for package in self.document_object["packages"]} + + relationship_objects = [] for relationship in self.document.relationships: - if relationship.relationship_type == "DESCRIBES": - document_describes.append(relationship.related_spdx_element) - if not relationship.has_comment: - remove_rel.append(relationship) - for relationship in remove_rel: - self.document.relationships.remove(relationship) - return document_describes + if relationship.relationship_type == "CONTAINS" and relationship.spdx_element_id in packages_spdx_ids \ + and relationship.related_spdx_element in files_spdx_ids: + update_dict_item_with_new_item(packages_by_spdx_id[relationship.spdx_element_id], "hasFiles", + relationship.related_spdx_element) + if relationship.has_comment: + relationship_objects.append(self.create_relationship_info(relationship)) + + elif relationship.relationship_type == "CONTAINED_BY" and relationship.spdx_element_id in files_spdx_ids \ + and relationship.related_spdx_element in packages_spdx_ids: + update_dict_item_with_new_item(packages_by_spdx_id[relationship.related_spdx_element], + "hasFiles", relationship.spdx_element_id) + if relationship.has_comment: + relationship_objects.append(self.create_relationship_info(relationship)) + + elif relationship.relationship_type == "DESCRIBES" and relationship.spdx_element_id == self.document.spdx_id: + update_dict_item_with_new_item(self.document_object, "documentDescribes", + relationship.related_spdx_element) + if relationship.has_comment: + relationship_objects.append(self.create_relationship_info(relationship)) + + elif relationship.relationship_type == "DESCRIBED_BY" and relationship.related_spdx_element == self.document.spdx_id: + update_dict_item_with_new_item(self.document_object, "documentDescribes", + relationship.spdx_element_id) + if relationship.has_comment: + relationship_objects.append(self.create_relationship_info(relationship)) + + else: + relationship_objects.append(self.create_relationship_info(relationship)) + + return relationship_objects def create_document(self): self.document_object = dict() @@ -573,9 +582,7 @@ def create_document(self): self.document_object["dataLicense"] = self.license(self.document.data_license) self.document_object["SPDXID"] = self.doc_spdx_id self.document_object["name"] = self.document.name - annotations_by_spdx_id= self.create_annotations_by_spdx_id() - document_describes = self.create_document_describes() - self.document_object["documentDescribes"] = document_describes + annotations_by_spdx_id = self.create_annotations_by_spdx_id() unique_doc_packages = {} for doc_package in self.document.packages: @@ -583,12 +590,15 @@ def create_document(self): unique_doc_packages[doc_package.spdx_id] = doc_package package_objects = [] - file_objects = [] for package in unique_doc_packages.values(): - package_info_object, files_in_package = self.create_package_info(package, annotations_by_spdx_id) + package_info_object = self.create_package_info(package, annotations_by_spdx_id) package_objects.append(package_info_object) - file_objects.extend(file for file in files_in_package if file not in file_objects) self.document_object["packages"] = package_objects + + file_objects = [] + for file in self.document.files: + file_object = self.create_file_info(file, annotations_by_spdx_id) + file_objects.append(file_object) self.document_object["files"] = file_objects if self.document.has_comment: @@ -614,6 +624,8 @@ def create_document(self): self.document_object["annotations"] = annotations_by_spdx_id[self.doc_spdx_id] if self.document.relationships: - self.document_object["relationships"] = self.create_relationship_info() + relationship_objects = self.create_relationships() + if relationship_objects: + self.document_object["relationships"] = relationship_objects return self.document_object diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index 0566c477a..c4c30a843 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -390,6 +390,14 @@ def write_document(document, out, validate=True): write_relationship(relationship, out) write_separators(out) + # Write file info + contains_relationships = [relationship.related_spdx_element for relationship in document.relationships if + (relationship.relationship_type == "CONTAINS")] + for file in document.files: + if file.spdx_id not in contains_relationships: + write_file(file, out) + write_separators(out) + # Write out package info for package in document.packages: write_package(package, out) diff --git a/tests/data/doc_parse/SBOMexpected.json b/tests/data/doc_parse/SBOMexpected.json index 848d19ab0..6fdfbe7d0 100644 --- a/tests/data/doc_parse/SBOMexpected.json +++ b/tests/data/doc_parse/SBOMexpected.json @@ -1,144 +1,159 @@ { - "id": "SPDXRef-DOCUMENT", - "specVersion": { - "major": 2, - "minor": 2 - }, - "documentNamespace": "http://spdx.org/spdxdocs/spdx-document-xyz", - "name": "xyz-0.1.0", - "comment": null, - "dataLicense": { - "type": "Single", - "identifier": "CC0-1.0", - "name": "Creative Commons Zero v1.0 Universal" - }, - "licenseListVersion": { - "major": 3, - "minor": 9 - }, - "creators": [ - { - "name": "Example Inc.", - "email": null, - "type": "Organization" + "id": "SPDXRef-DOCUMENT", + "specVersion": { + "major": 2, + "minor": 2 }, - { - "name": "Thomas Steenbergen", - "email": null, - "type": "Person" - } - ], - "created": "2020-07-23T18:30:22Z", - "creatorComment": null, - "packages": [ - { - "id": "SPDXRef-Package-xyz", - "name": "xyz", - "packageFileName": null, - "summary": "Awesome product created by Example Inc.", - "description": null, - "versionInfo": "0.1.0", - "sourceInfo": null, - "downloadLocation": "git+ssh://gitlab.example.com:3389/products/xyz.git@b2c358080011af6a366d2512a25a379fbe7b1f78", - "homepage": "https://example.com/products/xyz", - "originator": null, - "supplier": null, - "licenseConcluded": { + "documentNamespace": "http://spdx.org/spdxdocs/spdx-document-xyz", + "name": "xyz-0.1.0", + "comment": null, + "dataLicense": { "type": "Single", - "identifier": "NOASSERTION", - "name": "NOASSERTION" - }, - "licenseDeclared": { - "type": "Conjunction", - "identifier": [ - "Apache-2.0", - "LicenseRef-Proprietary-ExampleInc", - "curl" - ], - "name": [ - "Apache License 2.0", - "LicenseRef-Proprietary-ExampleInc", - "curl License" - ] - }, - "copyrightText": "copyright 2004-2020 Example Inc. All Rights Reserved.", - "licenseComment": null, - "checksum": null, - "files": [], - "licenseInfoFromFiles": [], - "verificationCode": { - "value": null, - "excludedFilesNames": [] - } + "identifier": "CC0-1.0", + "name": "Creative Commons Zero v1.0 Universal" }, - { - "id": "SPDXRef-Package-curl", - "name": "curl", - "packageFileName": "./libs/curl", - "summary": null, - "description": "A command line tool and library for transferring data with URL syntax, supporting HTTP, HTTPS, FTP, FTPS, GOPHER, TFTP, SCP, SFTP, SMB, TELNET, DICT, LDAP, LDAPS, MQTT, FILE, IMAP, SMTP, POP3, RTSP and RTMP. libcurl offers a myriad of powerful features.", - "versionInfo": "7.70.0", - "sourceInfo": null, - "downloadLocation": "https://github.com/curl/curl/releases/download/curl-7_70_0/curl-7.70.0.tar.gz", - "homepage": "https://curl.haxx.se/", - "originator": null, - "supplier": null, - "licenseConcluded": { - "type": "Single", - "identifier": "NOASSERTION", - "name": "NOASSERTION" - }, - "licenseDeclared": { - "type": "Single", - "identifier": "curl", - "name": "curl License" - }, - "copyrightText": "Copyright (c) 1996 - 2020, Daniel Stenberg, , and many contributors, see the THANKS file.", - "licenseComment": null, - "checksum": null, - "files": [], - "licenseInfoFromFiles": [], - "verificationCode": { - "value": null, - "excludedFilesNames": [] - } + "licenseListVersion": { + "major": 3, + "minor": 9 }, - { - "id": "SPDXRef-Package-openssl", - "name": "openssl", - "packageFileName": "./libs/openssl", - "summary": null, - "description": "OpenSSL is a robust, commercial-grade, full-featured Open Source Toolkit for the Transport Layer Security (TLS) protocol formerly known as the Secure Sockets Layer (SSL) protocol. The protocol implementation is based on a full-strength general purpose cryptographic library, which can also be used stand-alone.", - "versionInfo": "1.1.1g", - "sourceInfo": null, - "downloadLocation": "git+ssh://github.com/openssl/openssl.git@e2e09d9fba1187f8d6aafaa34d4172f56f1ffb72", - "homepage": "https://www.openssl.org/", - "originator": null, - "supplier": null, - "licenseConcluded": { - "type": "Single", - "identifier": "NOASSERTION", - "name": "NOASSERTION" - }, - "licenseDeclared": { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" - }, - "copyrightText": "copyright 2004-2020 The OpenSSL Project Authors. All Rights Reserved.", - "licenseComment": null, - "checksum": null, - "files": [], - "licenseInfoFromFiles": [], - "verificationCode": { - "value": null, - "excludedFilesNames": [] - } - } - ], - "externalDocumentRefs": [], - "extractedLicenses": [], - "annotations": [], - "reviews": [], - "snippets": [] -} \ No newline at end of file + "creators": [ + { + "name": "Example Inc.", + "email": null, + "type": "Organization" + }, + { + "name": "Thomas Steenbergen", + "email": null, + "type": "Person" + } + ], + "created": "2020-07-23T18:30:22Z", + "creatorComment": null, + "files": [], + "packages": [ + { + "id": "SPDXRef-Package-xyz", + "name": "xyz", + "packageFileName": null, + "summary": "Awesome product created by Example Inc.", + "description": null, + "versionInfo": "0.1.0", + "sourceInfo": null, + "downloadLocation": "git+ssh://gitlab.example.com:3389/products/xyz.git@b2c358080011af6a366d2512a25a379fbe7b1f78", + "homepage": "https://example.com/products/xyz", + "originator": null, + "supplier": null, + "licenseConcluded": { + "type": "Single", + "identifier": "NOASSERTION", + "name": "NOASSERTION" + }, + "licenseDeclared": { + "type": "Conjunction", + "identifier": [ + "Apache-2.0", + "LicenseRef-Proprietary-ExampleInc", + "curl" + ], + "name": [ + "Apache License 2.0", + "LicenseRef-Proprietary-ExampleInc", + "curl License" + ] + }, + "copyrightText": "copyright 2004-2020 Example Inc. All Rights Reserved.", + "licenseComment": null, + "checksum": null, + "licenseInfoFromFiles": [], + "verificationCode": { + "value": null, + "excludedFilesNames": [] + } + }, + { + "id": "SPDXRef-Package-curl", + "name": "curl", + "packageFileName": "./libs/curl", + "summary": null, + "description": "A command line tool and library for transferring data with URL syntax, supporting HTTP, HTTPS, FTP, FTPS, GOPHER, TFTP, SCP, SFTP, SMB, TELNET, DICT, LDAP, LDAPS, MQTT, FILE, IMAP, SMTP, POP3, RTSP and RTMP. libcurl offers a myriad of powerful features.", + "versionInfo": "7.70.0", + "sourceInfo": null, + "downloadLocation": "https://github.com/curl/curl/releases/download/curl-7_70_0/curl-7.70.0.tar.gz", + "homepage": "https://curl.haxx.se/", + "originator": null, + "supplier": null, + "licenseConcluded": { + "type": "Single", + "identifier": "NOASSERTION", + "name": "NOASSERTION" + }, + "licenseDeclared": { + "type": "Single", + "identifier": "curl", + "name": "curl License" + }, + "copyrightText": "Copyright (c) 1996 - 2020, Daniel Stenberg, , and many contributors, see the THANKS file.", + "licenseComment": null, + "checksum": null, + "licenseInfoFromFiles": [], + "verificationCode": { + "value": null, + "excludedFilesNames": [] + } + }, + { + "id": "SPDXRef-Package-openssl", + "name": "openssl", + "packageFileName": "./libs/openssl", + "summary": null, + "description": "OpenSSL is a robust, commercial-grade, full-featured Open Source Toolkit for the Transport Layer Security (TLS) protocol formerly known as the Secure Sockets Layer (SSL) protocol. The protocol implementation is based on a full-strength general purpose cryptographic library, which can also be used stand-alone.", + "versionInfo": "1.1.1g", + "sourceInfo": null, + "downloadLocation": "git+ssh://github.com/openssl/openssl.git@e2e09d9fba1187f8d6aafaa34d4172f56f1ffb72", + "homepage": "https://www.openssl.org/", + "originator": null, + "supplier": null, + "licenseConcluded": { + "type": "Single", + "identifier": "NOASSERTION", + "name": "NOASSERTION" + }, + "licenseDeclared": { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + "copyrightText": "copyright 2004-2020 The OpenSSL Project Authors. All Rights Reserved.", + "licenseComment": null, + "checksum": null, + "licenseInfoFromFiles": [], + "verificationCode": { + "value": null, + "excludedFilesNames": [] + } + } + ], + "externalDocumentRefs": [], + "extractedLicenses": [], + "annotations": [], + "reviews": [], + "snippets": [], + "relationships": [ + { + "spdx_element_id": "SPDXRef-Package-xyz", + "relationship_type": "CONTAINS", + "related_spdx_element": "SPDXRef-Package-curl" + }, + { + "spdx_element_id": "SPDXRef-Package-xyz", + "relationship_type": "CONTAINS", + "related_spdx_element": "SPDXRef-Package-openssl" + }, + { + "spdx_element_id": "SPDXRef-DOCUMENT", + "relationship_type": "DESCRIBES", + "related_spdx_element": "SPDXRef-Package-xyz" + } + ] +} diff --git a/tests/data/doc_parse/expected.json b/tests/data/doc_parse/expected.json index 085b73f04..a81ff4528 100644 --- a/tests/data/doc_parse/expected.json +++ b/tests/data/doc_parse/expected.json @@ -1,325 +1,340 @@ { - "id": "SPDXRef-DOCUMENT", - "specVersion": { - "major": 2, - "minor": 1 - }, - "documentNamespace": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", - "name": "Sample_Document-V2.1", - "comment": "This is a sample spreadsheet", - "dataLicense": { - "type": "Single", - "identifier": "CC0-1.0", - "name": "Creative Commons Zero v1.0 Universal" - }, - "licenseListVersion": { - "major": 3, - "minor": 6 - }, - "creators": [ - { - "name": "Gary O'Neall", - "email": null, - "type": "Person" + "id": "SPDXRef-DOCUMENT", + "specVersion": { + "major": 2, + "minor": 1 }, - { - "name": "Source Auditor Inc.", - "email": null, - "type": "Organization" + "documentNamespace": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", + "name": "Sample_Document-V2.1", + "comment": "This is a sample spreadsheet", + "dataLicense": { + "type": "Single", + "identifier": "CC0-1.0", + "name": "Creative Commons Zero v1.0 Universal" + }, + "licenseListVersion": { + "major": 3, + "minor": 6 }, - { - "name": "SourceAuditor-V1.2", - "type": "Tool" - } - ], - "created": "2010-02-03T00:00:00Z", - "creatorComment": "This is an example of an SPDX spreadsheet format", - "packages": [ - { - "id": "SPDXRef-Package", - "name": "SPDX Translator", - "packageFileName": "spdxtranslator-1.0.zip", - "summary": "SPDX Translator utility", - "description": "This utility translates and SPDX RDF XML document to a spreadsheet, translates a spreadsheet to an SPDX RDF XML document and translates an SPDX RDFa document to an SPDX RDF XML document.", - "versionInfo": "Version 0.9.2", - "sourceInfo": "Version 1.0 of the SPDX Translator application", - "downloadLocation": "http://www.spdx.org/tools", - "homepage": null, - "originator": { - "name": "SPDX", - "email": null, - "type": "Organization" - }, - "supplier": { - "name": "Linux Foundation", - "email": null, - "type": "Organization" - }, - "licenseConcluded": { - "type": "Conjunction", - "identifier": [ - "Apache-1.0", - "Apache-2.0", - "LicenseRef-1", - "LicenseRef-2", - "LicenseRef-3", - "LicenseRef-4", - "MPL-1.1" - ], - "name": [ - "Apache License 1.0", - "Apache License 2.0", - "CyberNeko License", - "LicenseRef-1", - "LicenseRef-2", - "LicenseRef-4", - "Mozilla Public License 1.1" - ] - }, - "licenseDeclared": { - "type": "Conjunction", - "identifier": [ - "Apache-2.0", - "LicenseRef-1", - "LicenseRef-2", - "LicenseRef-3", - "LicenseRef-4", - "MPL-1.1" - ], - "name": [ - "Apache License 2.0", - "CyberNeko License", - "LicenseRef-1", - "LicenseRef-2", - "LicenseRef-4", - "Mozilla Public License 1.1" - ] - }, - "copyrightText": " Copyright 2010, 2011 Source Auditor Inc.", - "licenseComment": "The declared license information can be found in the NOTICE file at the root of the archive file", - "checksum": { - "identifier": "SHA1", - "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" - }, - "files": [ + "creators": [ { - "id": "SPDXRef-File1", - "fileName": "Jenna-2.6.3/jena-2.6.3-sources.jar", - "type": 3, - "comment": "This file belongs to Jena", - "licenseConcluded": { - "type": "Single", - "identifier": "LicenseRef-1", - "name": "LicenseRef-1" - }, - "copyrightText": "(c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP", - "licenseComment": "This license is used by Jena", - "notice": null, - "checksum": { - "identifier": "SHA1", - "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125" - }, - "licenseInfoInFiles": [ - { - "type": "Single", - "identifier": "LicenseRef-1", - "name": "LicenseRef-1" - } - ], - "contributors": [], - "dependencies": [], - "artifactOfProjectName": [ - "Jena" - ], - "artifactOfProjectHome": [ - "http://www.openjena.org/" - ], - "artifactOfProjectURI": [ - "http://subversion.apache.org/doap.rdf" - ] + "name": "Gary O'Neall", + "email": null, + "type": "Person" }, { - "id": "SPDXRef-File2", - "fileName": "src/org/spdx/parser/DOAPProject.java", - "type": 1, - "comment": null, - "licenseConcluded": { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" - }, - "copyrightText": "Copyright 2010, 2011 Source Auditor Inc.", - "licenseComment": null, - "notice": null, - "checksum": { - "identifier": "SHA1", - "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" - }, - "licenseInfoInFiles": [ - { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" - } - ], - "contributors": [], - "dependencies": [], - "artifactOfProjectName": [], - "artifactOfProjectHome": [], - "artifactOfProjectURI": [] + "name": "Source Auditor Inc.", + "email": null, + "type": "Organization" + }, + { + "name": "SourceAuditor-V1.2", + "type": "Tool" } - ], - "licenseInfoFromFiles": [ + ], + "created": "2010-02-03T00:00:00Z", + "creatorComment": "This is an example of an SPDX spreadsheet format", + "files": [ { - "type": "Single", - "identifier": "Apache-1.0", - "name": "Apache License 1.0" + "id": "SPDXRef-File1", + "fileName": "Jenna-2.6.3/jena-2.6.3-sources.jar", + "type": 3, + "comment": "This file belongs to Jena", + "licenseConcluded": { + "type": "Single", + "identifier": "LicenseRef-1", + "name": "LicenseRef-1" + }, + "copyrightText": "(c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP", + "licenseComment": "This license is used by Jena", + "notice": null, + "checksum": { + "identifier": "SHA1", + "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125" + }, + "licenseInfoInFiles": [ + { + "type": "Single", + "identifier": "LicenseRef-1", + "name": "LicenseRef-1" + } + ], + "contributors": [], + "dependencies": [], + "artifactOfProjectName": [ + "Jena" + ], + "artifactOfProjectHome": [ + "http://www.openjena.org/" + ], + "artifactOfProjectURI": [ + "http://subversion.apache.org/doap.rdf" + ] }, { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" - }, + "id": "SPDXRef-File2", + "fileName": "src/org/spdx/parser/DOAPProject.java", + "type": 1, + "comment": null, + "licenseConcluded": { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + "copyrightText": "Copyright 2010, 2011 Source Auditor Inc.", + "licenseComment": null, + "notice": null, + "checksum": { + "identifier": "SHA1", + "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + }, + "licenseInfoInFiles": [ + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + } + ], + "contributors": [], + "dependencies": [], + "artifactOfProjectName": [], + "artifactOfProjectHome": [], + "artifactOfProjectURI": [] + } + ], + "packages": [ { - "type": "Single", - "identifier": "LicenseRef-1", - "name": "LicenseRef-1" - }, + "id": "SPDXRef-Package", + "name": "SPDX Translator", + "packageFileName": "spdxtranslator-1.0.zip", + "summary": "SPDX Translator utility", + "description": "This utility translates and SPDX RDF XML document to a spreadsheet, translates a spreadsheet to an SPDX RDF XML document and translates an SPDX RDFa document to an SPDX RDF XML document.", + "versionInfo": "Version 0.9.2", + "sourceInfo": "Version 1.0 of the SPDX Translator application", + "downloadLocation": "http://www.spdx.org/tools", + "homepage": null, + "originator": { + "name": "SPDX", + "email": null, + "type": "Organization" + }, + "supplier": { + "name": "Linux Foundation", + "email": null, + "type": "Organization" + }, + "licenseConcluded": { + "type": "Conjunction", + "identifier": [ + "Apache-1.0", + "Apache-2.0", + "LicenseRef-1", + "LicenseRef-2", + "LicenseRef-3", + "LicenseRef-4", + "MPL-1.1" + ], + "name": [ + "Apache License 1.0", + "Apache License 2.0", + "CyberNeko License", + "LicenseRef-1", + "LicenseRef-2", + "LicenseRef-4", + "Mozilla Public License 1.1" + ] + }, + "licenseDeclared": { + "type": "Conjunction", + "identifier": [ + "Apache-2.0", + "LicenseRef-1", + "LicenseRef-2", + "LicenseRef-3", + "LicenseRef-4", + "MPL-1.1" + ], + "name": [ + "Apache License 2.0", + "CyberNeko License", + "LicenseRef-1", + "LicenseRef-2", + "LicenseRef-4", + "Mozilla Public License 1.1" + ] + }, + "copyrightText": " Copyright 2010, 2011 Source Auditor Inc.", + "licenseComment": "The declared license information can be found in the NOTICE file at the root of the archive file", + "checksum": { + "identifier": "SHA1", + "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + }, + "licenseInfoFromFiles": [ + { + "type": "Single", + "identifier": "Apache-1.0", + "name": "Apache License 1.0" + }, + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + { + "type": "Single", + "identifier": "LicenseRef-1", + "name": "LicenseRef-1" + }, + { + "type": "Single", + "identifier": "LicenseRef-2", + "name": "LicenseRef-2" + }, + { + "type": "Single", + "identifier": "LicenseRef-3", + "name": "CyberNeko License" + }, + { + "type": "Single", + "identifier": "LicenseRef-4", + "name": "LicenseRef-4" + }, + { + "type": "Single", + "identifier": "MPL-1.1", + "name": "Mozilla Public License 1.1" + } + ], + "verificationCode": { + "value": "4e3211c67a2d28fced849ee1bb76e7391b93feba", + "excludedFilesNames": [ + "SpdxTranslatorSpdx.rdf", + "SpdxTranslatorSpdx.txt" + ] + }, + "builtDate": "2020-01-01T12:00:00Z", + "releaseDate": "2021-01-01T12:00:00Z", + "validUntilDate": "2022-01-01T12:00:00Z", + "primaryPackagePurpose": "OPERATING-SYSTEM"} + ], + "externalDocumentRefs": [ { - "type": "Single", - "identifier": "LicenseRef-2", - "name": "LicenseRef-2" + "externalDocumentId": "DocumentRef-spdx-tool-2.1", + "spdxDocument": "https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301", + "checksum": { + "identifier": "SHA1", + "value": "d6a770ba38583ed4bb4525bd96e50461655d2759" + } + } + ], + "extractedLicenses": [ + { + "name": "LicenseRef-1", + "identifier": "LicenseRef-1", + "text": "/*\n * (c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */", + "comment": null, + "cross_refs": [] }, { - "type": "Single", - "identifier": "LicenseRef-3", - "name": "CyberNeko License" + "name": "LicenseRef-2", + "identifier": "LicenseRef-2", + "text": "This package includes the GRDDL parser developed by Hewlett Packard under the following license:\n\u00a9 Copyright 2007 Hewlett-Packard Development Company, LP\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:\n\nRedistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.\nRedistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.\nThe name of the author may not be used to endorse or promote products derived from this software without specific prior written permission.\nTHIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ", + "comment": null, + "cross_refs": [] }, { - "type": "Single", - "identifier": "LicenseRef-4", - "name": "LicenseRef-4" + "name": "CyberNeko License", + "identifier": "LicenseRef-3", + "text": "The CyberNeko Software License, Version 1.0\n\n\n(C) Copyright 2002-2005, Andy Clark. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions\nare met:\n\n1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in\n the documentation and/or other materials provided with the\n distribution.\n\n3. The end-user documentation included with the redistribution,\n if any, must include the following acknowledgment:\n \"This product includes software developed by Andy Clark.\"\n Alternately, this acknowledgment may appear in the software itself,\n if and wherever such third-party acknowledgments normally appear.\n\n4. The names \"CyberNeko\" and \"NekoHTML\" must not be used to endorse\n or promote products derived from this software without prior\n written permission. For written permission, please contact\n andyc@cyberneko.net.\n\n5. Products derived from this software may not be called \"CyberNeko\",\n nor may \"CyberNeko\" appear in their name, without prior written\n permission of the author.\n\nTHIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED\nWARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\nOF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR OTHER CONTRIBUTORS\nBE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,\nOR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT\nOF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR\nBUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,\nWHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE\nOR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,\nEVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", + "comment": "This is tye CyperNeko License", + "cross_refs": [ + "http://justasample.url.com", + "http://people.apache.org/~andyc/neko/LICENSE" + ] }, { - "type": "Single", - "identifier": "MPL-1.1", - "name": "Mozilla Public License 1.1" + "name": "LicenseRef-4", + "identifier": "LicenseRef-4", + "text": "/*\n * (c) Copyright 2009 University of Bristol\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */ ", + "comment": null, + "cross_refs": [] } - ], - "verificationCode": { - "value": "4e3211c67a2d28fced849ee1bb76e7391b93feba", - "excludedFilesNames": [ - "SpdxTranslatorSpdx.rdf", - "SpdxTranslatorSpdx.txt" - ] - }, - "builtDate": "2020-01-01T12:00:00Z", - "releaseDate": "2021-01-01T12:00:00Z", - "validUntilDate": "2022-01-01T12:00:00Z", - "primaryPackagePurpose": "OPERATING-SYSTEM" - } - ], - "externalDocumentRefs": [ - { - "externalDocumentId": "DocumentRef-spdx-tool-2.1", - "spdxDocument": "https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301", - "checksum": { - "identifier": "SHA1", - "value": "d6a770ba38583ed4bb4525bd96e50461655d2759" - } - } - ], - "extractedLicenses": [ - { - "name": "LicenseRef-1", - "identifier": "LicenseRef-1", - "text": "/*\n * (c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */", - "comment": null, - "cross_refs": [] - }, - { - "name": "LicenseRef-2", - "identifier": "LicenseRef-2", - "text": "This package includes the GRDDL parser developed by Hewlett Packard under the following license:\n\u00a9 Copyright 2007 Hewlett-Packard Development Company, LP\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:\n\nRedistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.\nRedistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.\nThe name of the author may not be used to endorse or promote products derived from this software without specific prior written permission.\nTHIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ", - "comment": null, - "cross_refs": [] - }, - { - "name": "CyberNeko License", - "identifier": "LicenseRef-3", - "text": "The CyberNeko Software License, Version 1.0\n\n\n(C) Copyright 2002-2005, Andy Clark. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions\nare met:\n\n1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in\n the documentation and/or other materials provided with the\n distribution.\n\n3. The end-user documentation included with the redistribution,\n if any, must include the following acknowledgment:\n \"This product includes software developed by Andy Clark.\"\n Alternately, this acknowledgment may appear in the software itself,\n if and wherever such third-party acknowledgments normally appear.\n\n4. The names \"CyberNeko\" and \"NekoHTML\" must not be used to endorse\n or promote products derived from this software without prior\n written permission. For written permission, please contact\n andyc@cyberneko.net.\n\n5. Products derived from this software may not be called \"CyberNeko\",\n nor may \"CyberNeko\" appear in their name, without prior written\n permission of the author.\n\nTHIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED\nWARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\nOF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR OTHER CONTRIBUTORS\nBE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,\nOR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT\nOF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR\nBUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,\nWHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE\nOR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,\nEVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", - "comment": "This is tye CyperNeko License", - "cross_refs": [ - "http://justasample.url.com", - "http://people.apache.org/~andyc/neko/LICENSE" - ] - }, - { - "name": "LicenseRef-4", - "identifier": "LicenseRef-4", - "text": "/*\n * (c) Copyright 2009 University of Bristol\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */ ", - "comment": null, - "cross_refs": [] - } - ], - "annotations": [ - { - "id": "SPDXRef-DOCUMENT", - "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", - "type": "REVIEW", - "annotator": { - "name": "Jim Reviewer", - "email": null, - "type": "Person" - }, - "date": "2012-06-13T00:00:00Z" - } - ], - "reviews": [ - { - "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", - "reviewer": { - "name": "Joe Reviewer", - "email": null, - "type": "Person" - }, - "date": "2010-02-10T00:00:00Z" - }, - { - "comment": "Another example reviewer.", - "reviewer": { - "name": "Suzanne Reviewer", - "email": null, - "type": "Person" - }, - "date": "2011-03-13T00:00:00Z" - } - ], - "snippets": [ - { - "id": "SPDXRef-Snippet", - "name": "from linux kernel", - "comment": "This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0-or-later.", - "copyrightText": "Copyright 2008-2010 John Smith", - "licenseComments": "The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz.", - "snippetFromFile": "SPDXRef-DoapSource", - "licenseConcluded": { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" - }, - "licenseInfoInSnippets": [ + ], + "annotations": [ { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" + "id": "SPDXRef-DOCUMENT", + "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", + "type": "REVIEW", + "annotator": { + "name": "Jim Reviewer", + "email": null, + "type": "Person" + }, + "date": "2012-06-13T00:00:00Z" + } + ], + "reviews": [ + { + "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", + "reviewer": { + "name": "Joe Reviewer", + "email": null, + "type": "Person" + }, + "date": "2010-02-10T00:00:00Z" }, + { + "comment": "Another example reviewer.", + "reviewer": { + "name": "Suzanne Reviewer", + "email": null, + "type": "Person" + }, + "date": "2011-03-13T00:00:00Z" + } + ], + "snippets": [ + { + "id": "SPDXRef-Snippet", + "name": "from linux kernel", + "comment": "This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0-or-later.", + "copyrightText": "Copyright 2008-2010 John Smith", + "licenseComments": "The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz.", + "snippetFromFile": "SPDXRef-DoapSource", + "licenseConcluded": { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + "licenseInfoInSnippets": [ + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0"}, { "type": "Single", "identifier": "GPL-2.0-only", "name": "GNU General Public License v2.0 only" + } + ] + } + ], + "relationships": [ + { + "spdx_element_id": "SPDXRef-Package", + "relationship_type": "CONTAINS", + "related_spdx_element": "SPDXRef-File1" + }, + { + "spdx_element_id": "SPDXRef-Package", + "relationship_type": "CONTAINS", + "related_spdx_element": "SPDXRef-File2" + }, + { + "spdx_element_id": "SPDXRef-DOCUMENT", + "relationship_type": "DESCRIBES", + "related_spdx_element": "SPDXRef-Package" } - ] - } - ] + ] } diff --git a/tests/test_document.py b/tests/test_document.py index e03fc608d..527f83c9c 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -196,9 +196,11 @@ def _get_lgpl_doc(self, or_later=False): lic1 = License.from_identifier('LGPL-2.1-or-later') file1.add_lics(lic1) + doc.add_file(file1) package.add_lics_from_file(lic1) - package.add_file(file1) + relationship = package.spdx_id + " " + "CONTAINS" + " " + file1.spdx_id + doc.add_relationships(Relationship(relationship)) relationship = doc.spdx_id + " " + "DESCRIBES" + " " + package.spdx_id doc.add_relationships(Relationship(relationship)) return doc @@ -249,12 +251,11 @@ def _get_lgpl_multi_package_doc(self, or_later=False): file1.add_lics(lic1) package2.add_lics_from_file(lic1) - package2.add_file(file1) package3.add_lics_from_file(lic1) - package3.add_file(file1) doc.add_package(package2) doc.add_package(package3) + doc.add_file(file1) relationship = doc.spdx_id + " " + "DESCRIBES" + " " + package1.spdx_id doc.add_relationships(Relationship(relationship)) @@ -262,6 +263,10 @@ def _get_lgpl_multi_package_doc(self, or_later=False): doc.add_relationships(Relationship(relationship)) relationship = doc.spdx_id + " " + "DESCRIBES" + " " + package3.spdx_id doc.add_relationships(Relationship(relationship)) + relationship = package2.spdx_id + " " + "CONTAINS" + " " + file1.spdx_id + doc.add_relationships(Relationship(relationship)) + relationship = package3.spdx_id + " " + "CONTAINS" + " " + file1.spdx_id + doc.add_relationships(Relationship(relationship)) return doc diff --git a/tests/test_jsonyamlxml_writer.py b/tests/test_jsonyamlxml_writer.py index bf2d4cf74..291ed1386 100644 --- a/tests/test_jsonyamlxml_writer.py +++ b/tests/test_jsonyamlxml_writer.py @@ -5,12 +5,17 @@ import pytest +from spdx.checksum import Algorithm from spdx.document import Document +from spdx.file import File +from spdx.license import License from spdx.package import Package, ExternalPackageRef, PackagePurpose from spdx.parsers.parse_anything import parse_file +from spdx.relationship import Relationship from spdx.snippet import Snippet -from spdx.writers import json, write_anything -from tests.test_rdf_writer import minimal_document +from spdx.utils import update_dict_item_with_new_item +from spdx.writers import write_anything +from tests.test_rdf_writer import minimal_document_with_package tested_formats: List[str] = ['yaml', 'xml', 'json'] @@ -26,7 +31,7 @@ def temporary_file_path() -> str: @pytest.mark.parametrize("out_format", tested_formats) def test_external_package_references(temporary_file_path: str, out_format: str) -> None: - document: Document = minimal_document() + document: Document = minimal_document_with_package() package: Package = document.packages[0] first_ref = ExternalPackageRef(category="PACKAGE-MANAGER") second_ref = ExternalPackageRef(category="SECURITY") @@ -47,7 +52,7 @@ def test_external_package_references(temporary_file_path: str, out_format: str) @pytest.mark.parametrize("out_format", tested_formats) def test_primary_package_purpose(temporary_file_path: str, out_format: str): - document: Document = minimal_document() + document: Document = minimal_document_with_package() package: Package = document.packages[0] package.primary_package_purpose = PackagePurpose.OPERATING_SYSTEM @@ -62,7 +67,7 @@ def test_primary_package_purpose(temporary_file_path: str, out_format: str): @pytest.mark.parametrize("out_format", tested_formats) def test_release_built_valid_until_date(temporary_file_path: str, out_format: str): - document: Document = minimal_document() + document: Document = minimal_document_with_package() package: Package = document.packages[0] package.release_date = datetime(2021, 1, 1, 12, 0, 0) package.built_date = datetime(2021, 1, 1, 12, 0, 0) @@ -107,7 +112,7 @@ def minimal_snippet(): def write_and_parse_snippet(out_format, snippet, temporary_file_path): - document: Document = minimal_document() + document: Document = minimal_document_with_package() document.add_snippet(snippet) file_path_with_ending = temporary_file_path + "." + out_format write_anything.write_file(document, file_path_with_ending, validate=False) @@ -116,3 +121,43 @@ def write_and_parse_snippet(out_format, snippet, temporary_file_path): return parsed_snippet + +@pytest.mark.parametrize("out_format", ['yaml', 'xml', 'json']) +def test_files_without_package(temporary_file_path, out_format): + document: Document = minimal_document() + document.spdx_id = "SPDXRef-DOCUMENT" + file: File = minimal_file() + document.add_file(file) + describes_relationship: Relationship = Relationship("SPDXRef-DOCUMENT DESCRIBES SPDXRef-File") + document.add_relationships(relationship=describes_relationship) + + file_path_with_ending = temporary_file_path + "." + out_format + write_anything.write_file(document, file_path_with_ending, validate=False) + + parsed_document: Document = parse_file(file_path_with_ending)[0] + parsed_file: File = parsed_document.files[0] + + assert parsed_file.name == "Example File" + + +def minimal_document(): + document = Document(data_license=License.from_identifier('CC0-1.0')) + document.creation_info.set_created_now() + return document + + +def minimal_file(): + file = File(name="Example File", spdx_id="SPDXRef-File", chksum=Algorithm('SHA1', 'SOME-SHA1')) + return file + + +@pytest.mark.parametrize("key,value,expected_length", + [("existing_key", "new_value", 2), ("existing_key", "existing_value", 1), + ("new_key", "new_value", 1)]) +def test_update_dict_item_with_new_item(key, value, expected_length): + current_state = {"existing_key": ["existing_value"]} + update_dict_item_with_new_item(current_state, key, value) + + assert key in current_state + assert value in current_state[key] + assert len(current_state[key]) == expected_length diff --git a/tests/test_rdf_writer.py b/tests/test_rdf_writer.py index 267ecd269..9153dbac6 100644 --- a/tests/test_rdf_writer.py +++ b/tests/test_rdf_writer.py @@ -25,7 +25,7 @@ def temporary_file_path() -> str: # that part is refactored. def test_accept_provided_doc_node(temporary_file_path) -> None: doc_node = URIRef("http://www.spdx.org/tools#SPDXRef-DOCUMENT") - document: Document = minimal_document() + document: Document = minimal_document_with_package() with open(temporary_file_path, "wb") as out: writer = Writer(document, out) @@ -41,7 +41,7 @@ def test_accept_provided_doc_node(temporary_file_path) -> None: def test_external_package_references(temporary_file_path) -> None: - document: Document = minimal_document() + document: Document = minimal_document_with_package() package: Package = document.packages[0] first_ref = ExternalPackageRef(category="PACKAGE-MANAGER") second_ref = ExternalPackageRef(category="SECURITY") @@ -62,7 +62,7 @@ def test_external_package_references(temporary_file_path) -> None: assert second_ref.category in parsed_reference_categories -def minimal_document() -> Document: +def minimal_document_with_package() -> Document: document = Document(data_license=License.from_identifier('CC0-1.0')) document.creation_info.set_created_now() package: Package = minimal_package() diff --git a/tests/test_tag_value_parser.py b/tests/test_tag_value_parser.py index e7bddd561..fef36439d 100644 --- a/tests/test_tag_value_parser.py +++ b/tests/test_tag_value_parser.py @@ -339,7 +339,7 @@ def test_package(self): assert document.package.version == 'Version 0.9.2' assert len(document.package.licenses_from_files) == 2 assert (document.package.conc_lics.identifier == 'LicenseRef-2.0 AND Apache-2.0') - assert document.package.files_analyzed == True + assert document.package.files_analyzed is True assert document.package.comment == 'Comment on the package.' assert document.package.pkg_ext_refs[-1].category == 'SECURITY' assert document.package.pkg_ext_refs[-1].pkg_ext_ref_type == 'cpe23Type' @@ -354,8 +354,8 @@ def test_file(self): document, error = self.p.parse(self.complete_str) assert document is not None assert not error - assert len(document.package.files) == 1 - spdx_file = document.package.files[0] + assert len(document.files) == 1 + spdx_file = document.files[0] assert spdx_file.name == 'testfile.java' assert spdx_file.spdx_id == 'SPDXRef-File' assert spdx_file.type == spdx.file.FileType.SOURCE diff --git a/tests/utils_test.py b/tests/utils_test.py index ce608529f..1a39e334b 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -18,13 +18,14 @@ import os import posixpath import re +from typing import List import xmltodict import yaml import spdx from spdx import utils - +from spdx.relationship import Relationship test_data_dir = os.path.join(os.path.dirname(__file__), 'data') @@ -361,7 +362,6 @@ def package_to_dict(cls, package): ('copyrightText', package.cr_text), ('licenseComment', package.license_comment), ('checksum', cls.checksum_to_dict(package.checksum)), - ('files', cls.files_to_list(sorted(package.files))), ('licenseInfoFromFiles', [cls.license_to_dict(lic) for lic in lics_from_files]), ('verificationCode', OrderedDict([ ('value', package.verif_code), @@ -511,6 +511,19 @@ def snippets_to_list(cls, snippets): return snippets_list + @classmethod + def relationships_to_dict_list(cls, relationships: List[Relationship]) -> List[OrderedDict]: + relationships_list = [] + for relationship in relationships: + relationship_dict = OrderedDict([ + ('spdx_element_id', relationship.spdx_element_id), + ('relationship_type', relationship.relationship_type), + ('related_spdx_element', relationship.related_spdx_element) + ]) + relationships_list.append(relationship_dict) + + return relationships_list + @classmethod def to_dict(cls, doc): """ @@ -528,10 +541,12 @@ def to_dict(cls, doc): ('creators', [cls.entity_to_dict(creator) for creator in creators]), ('created', utils.datetime_iso_format(doc.creation_info.created)), ('creatorComment', doc.creation_info.comment), + ('files', cls.files_to_list(sorted(doc.files))), ('packages', [cls.package_to_dict(p) for p in doc.packages]), ('externalDocumentRefs', cls.ext_document_references_to_list(sorted(doc.ext_document_references))), ('extractedLicenses', cls.extracted_licenses_to_list(sorted(doc.extracted_licenses))), ('annotations', cls.annotations_to_list(sorted(doc.annotations))), ('reviews', cls.reviews_to_list(sorted(doc.reviews))), ('snippets', cls.snippets_to_list(sorted(doc.snippet))), + ('relationships', cls.relationships_to_dict_list(doc.relationships)) ]) From 4de77daf86defab37ecc76a7551732a9f935be3d Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Tue, 15 Nov 2022 16:45:59 +0100 Subject: [PATCH 3/8] [issue-181] add support for files at document-level in tv-parser/writer Signed-off-by: Meret Behrens --- spdx/parsers/tagvalue.py | 13 +++++- spdx/parsers/tagvaluebuilders.py | 47 +++++++++++++++------ spdx/writers/tagvalue.py | 71 ++++++++++++++++++++++++++------ tests/test_write_anything.py | 7 ++-- 4 files changed, 109 insertions(+), 29 deletions(-) diff --git a/spdx/parsers/tagvalue.py b/spdx/parsers/tagvalue.py index 39095775b..a5772549f 100644 --- a/spdx/parsers/tagvalue.py +++ b/spdx/parsers/tagvalue.py @@ -575,6 +575,9 @@ def p_file_name_1(self, p): try: value = p[2] self.builder.set_file_name(self.document, value) + self.builder.set_current_name("file", value) + self.builder.set_current_id("file", None) + except OrderError: self.order_error("FileName", "PackageName", p.lineno(1)) @@ -589,10 +592,14 @@ def p_spdx_id(self, p): value = p[2] if not self.builder.doc_spdx_id_set: self.builder.set_doc_spdx_id(self.document, value) - elif not self.builder.package_spdx_id_set: + elif self.builder.current_package_has_name() and not self.builder.current_package_has_id(): self.builder.set_pkg_spdx_id(self.document, value) + self.builder.set_current_id("package", value) else: self.builder.set_file_spdx_id(self.document, value) + self.builder.set_current_id("file", value) + if self.builder.current_package_has_name: + self.builder.add_relationship(self.document, self.builder.current_package["spdx_id"] + " CONTAINS " + value) def p_file_comment_1(self, p): """file_comment : FILE_COMMENT text_or_line""" @@ -1068,6 +1075,8 @@ def p_pkg_file_name(self, p): try: value = p[2] self.builder.set_pkg_file_name(self.document, value) + self.builder.set_current_name("file", value) + self.builder.set_current_id("file", None) except OrderError: self.order_error("PackageFileName", "PackageName", p.lineno(1)) except CardinalityError: @@ -1100,6 +1109,8 @@ def p_package_name(self, p): try: value = p[2] self.builder.create_package(self.document, value) + self.builder.set_current_name("package", value) + self.builder.set_current_id("package", None) except CardinalityError: self.more_than_one_error("PackageName", p.lineno(1)) diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index 9bdd5f595..1c603bf8d 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -10,6 +10,7 @@ # limitations under the License. import re +from typing import Dict from spdx import annotation from spdx import checksum @@ -1128,7 +1129,7 @@ def set_file_spdx_id(self, doc, spdx_id): Raise SPDXValueError if malformed value. Raise CardinalityError if more than one spdx_id set. """ - if not self.has_package(doc) or not self.has_file(doc): + if not self.has_file(doc): raise OrderError("File::SPDXID") if self.file_spdx_id_set: @@ -1147,7 +1148,7 @@ def set_file_comment(self, doc, text): Raise CardinalityError if more than one comment set. Raise SPDXValueError if text is not free form text or single line of text. """ - if not self.has_package(doc) or not self.has_file(doc): + if not self.has_file(doc): raise OrderError("File::Comment") if self.file_comment_set: @@ -1166,7 +1167,7 @@ def set_file_attribution_text(self, doc, text): Raise OrderError if no package or no file defined. Raise SPDXValueError if text is not free form text or single line of text. """ - if not self.has_package(doc) or not self.has_file(doc): + if not self.has_file(doc): raise OrderError("File::AttributionText") if not validations.validate_file_attribution_text(text): @@ -1187,7 +1188,7 @@ def set_file_type(self, doc, type_value): "ARCHIVE": file.FileType.ARCHIVE, "OTHER": file.FileType.OTHER, } - if not self.has_package(doc) or not self.has_file(doc): + if not self.has_file(doc): raise OrderError("File::Type") if self.file_type_set: @@ -1205,7 +1206,7 @@ def set_file_chksum(self, doc, chksum): Raise OrderError if no package or file defined. Raise CardinalityError if more than one chksum set. """ - if not self.has_package(doc) or not self.has_file(doc): + if not self.has_file(doc): raise OrderError("File::CheckSum") if self.file_chksum_set: @@ -1221,7 +1222,7 @@ def set_concluded_license(self, doc, lic): Raise CardinalityError if already set. Raise SPDXValueError if malformed. """ - if not self.has_package(doc) or not self.has_file(doc): + if not self.has_file(doc): raise OrderError("File::ConcludedLicense") if self.file_conc_lics_set: @@ -1239,7 +1240,7 @@ def set_file_license_in_file(self, doc, lic): Raise OrderError if no package or file defined. Raise SPDXValueError if malformed value. """ - if not self.has_package(doc) or not self.has_file(doc): + if not self.has_file(doc): raise OrderError("File::LicenseInFile") if not validations.validate_file_lics_in_file(lic): @@ -1254,7 +1255,7 @@ def set_file_license_comment(self, doc, text): Raise SPDXValueError if text is not free form text or single line of text. Raise CardinalityError if more than one per file. """ - if not self.has_package(doc) or not self.has_file(doc): + if not self.has_file(doc): raise OrderError("File::LicenseComment") if self.file_license_comment_set: @@ -1272,7 +1273,7 @@ def set_file_copyright(self, doc, text): Raise SPDXValueError if not free form text or NONE or NO_ASSERT or single line of text. Raise CardinalityError if more than one. """ - if not self.has_package(doc) or not self.has_file(doc): + if not self.has_file(doc): raise OrderError("File::CopyRight") if self.file_copytext_set: @@ -1294,7 +1295,7 @@ def set_file_notice(self, doc, text): Raise SPDXValueError if not free form text or single line of text. Raise CardinalityError if more than one. """ - if not self.has_package(doc) or not self.has_file(doc): + if not self.has_file(doc): raise OrderError("File::Notice") if self.file_notice_set: @@ -1310,7 +1311,7 @@ def add_file_contribution(self, doc, value): """ Raise OrderError if no package or file defined. """ - if not self.has_package(doc) or not self.has_file(doc): + if not self.has_file(doc): raise OrderError("File::Contributor") self.file(doc).add_contrib(value) @@ -1319,7 +1320,7 @@ def add_file_dep(self, doc, value): """ Raise OrderError if no package or file defined. """ - if not self.has_package(doc) or not self.has_file(doc): + if not self.has_file(doc): raise OrderError("File::Dependency") self.file(doc).add_depend(value) @@ -1329,7 +1330,7 @@ def set_file_atrificat_of_project(self, doc, symbol, value): Set a file name, uri or home artifact. Raise OrderError if no package or file defined. """ - if not self.has_package(doc) or not self.has_file(doc): + if not self.has_file(doc): raise OrderError("File::Artifact") self.file(doc).add_artifact(symbol, value) @@ -1683,6 +1684,26 @@ def __init__(self): super(Builder, self).__init__() # FIXME: this state does not make sense self.reset() + self.current_package: Dict = dict() + self.current_file: Dict = dict() + + def set_current_name(self, indicator: str, name: str) -> None: + if indicator == "package": + self.current_package["name"] = name + elif indicator == "file": + self.current_file["name"] = name + + def set_current_id(self, indicator: str, spdx_id: str) -> None: + if indicator == "package": + self.current_package["spdx_id"] = spdx_id + elif indicator == "file": + self.current_file["spdx_id"] = spdx_id + + def current_package_has_name(self): + return bool("name" in self.current_package) and (self.current_package["name"]) + + def current_package_has_id(self): + return bool("spdx_id" in self.current_package) and (self.current_package["spdx_id"]) def reset(self): """ diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index c4c30a843..2619dbae8 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -10,10 +10,15 @@ # limitations under the License. from itertools import zip_longest +from typing import List, TextIO, Tuple, Dict from spdx import license, utils from spdx import file as spdx_file +from spdx.file import File +from spdx.package import Package from spdx.parsers.loggers import ErrorMessages +from spdx.relationship import Relationship +from spdx.snippet import Snippet class InvalidDocumentError(Exception): @@ -23,8 +28,11 @@ class InvalidDocumentError(Exception): pass + def write_separator(out): out.write("\n") + + def write_separators(out): out.write("\n" * 2) @@ -39,6 +47,7 @@ def format_verif_code(package): def write_value(tag, value, out): out.write("{0}: {1}\n".format(tag, value)) + def write_range(tag, value, out): out.write("{0}: {1}:{2}\n".format(tag, value[0], value[1])) @@ -303,12 +312,6 @@ def write_package(package, out): write_value("ValidUntilDate", utils.datetime_iso_format(package.valid_until_date), out) - # Write sorted files. - for spdx_file in sorted(package.files): - write_separators(out) - write_file(spdx_file, out) - - def write_extracted_licenses(lics, out): """ Write extracted licenses fields to out. @@ -383,25 +386,37 @@ def write_document(document, out, validate=True): write_separator(out) write_separator(out) - # Write relationships - if document.relationships: + relationships_to_write, packages_containing_files = scan_relationships(document.relationships, document.packages, + document.files) + files_containing_snippets = determine_files_containing_snippets(document.snippet) + packaged_files = [file_spdx_id for file_spdx_id_list in packages_containing_files.values() + for file_spdx_id in file_spdx_id_list] + + # Write Relationships + if relationships_to_write: out.write("# Relationships\n\n") - for relationship in document.relationships: + for relationship in relationships_to_write: write_relationship(relationship, out) write_separators(out) # Write file info - contains_relationships = [relationship.related_spdx_element for relationship in document.relationships if - (relationship.relationship_type == "CONTAINS")] for file in document.files: - if file.spdx_id not in contains_relationships: + if file.spdx_id not in packaged_files: write_file(file, out) write_separators(out) + write_snippets_contained_in_file(file, files_containing_snippets, out) # Write out package info for package in document.packages: write_package(package, out) write_separators(out) + if package.spdx_id in packages_containing_files: + for file_spdx_id in packages_containing_files[package.spdx_id]: + for file in document.files: + if file.spdx_id == file_spdx_id: + write_file(file, out) + write_snippets_contained_in_file(file, files_containing_snippets, out) + break # Write out snippet info for snippet in document.snippet: @@ -414,3 +429,35 @@ def write_document(document, out, validate=True): write_extracted_licenses(lic, out) write_separator(out) write_separator(out) + + +def write_snippets_contained_in_file(file, files_contain_snippets: List, out: TextIO) -> None: + if file.spdx_id in files_contain_snippets: + for snippet in files_contain_snippets[file.spdx_id]: + write_snippet(snippet, out) + + +def scan_relationships(relationships: List[Relationship], packages: List[Package], files: List[File]) -> Tuple[ + List, Dict]: + packages_containing_files = dict() + relationships_to_write = [] + for relationship in relationships: + if relationship.relationship_type == "CONTAINS" and \ + relationship.spdx_element_id in [package.spdx_id for package in packages] and \ + relationship.related_spdx_element in [file.spdx_id for file in files]: + packages_containing_files.setdefault(relationship.spdx_element_id, []).append( + relationship.related_spdx_element) + if relationship.has_comment: + relationships_to_write.append(relationship) + else: + relationships_to_write.append(relationship) + + return relationships_to_write, packages_containing_files + + +def determine_files_containing_snippets(snippets: List[Snippet]) -> Dict: + files_containing_snippets = dict() + for snippet in snippets: + files_containing_snippets.setdefault(snippet.snip_from_file_spdxid, []).append(snippet.spdx_id) + + return files_containing_snippets diff --git a/tests/test_write_anything.py b/tests/test_write_anything.py index aa94ab949..2cf982188 100644 --- a/tests/test_write_anything.py +++ b/tests/test_write_anything.py @@ -19,8 +19,9 @@ dirname = os.path.join(os.path.dirname(__file__), "data", "formats") test_files = [os.path.join(dirname, fn) for fn in os.listdir(dirname)] -test_files_json_yaml_xml_tag = [filename for filename in test_files if filename.endswith("json") or - filename.endswith("yaml") or filename.endswith("xml") or filename.endswith("tag")] +test_files_json_yaml_xml_tag = [filename for filename in test_files if "TagExample" in filename] + #filename.endswith("json") or + # filename.endswith("yaml") or filename.endswith("xml") or filename.endswith("tag")] test_files_rdf = [filename for filename in test_files if filename.endswith("rdf")] UNSTABLE_CONVERSIONS = { "SPDXTagExample.tag-rdf", @@ -49,7 +50,7 @@ # https://github.com/spdx/tools-python/issues/274 -@pytest.mark.parametrize("out_format", ['rdf', 'yaml', 'xml', 'json', 'tag']) +@pytest.mark.parametrize("out_format", [ 'yaml', 'xml', 'json', 'tag']) @pytest.mark.parametrize("in_file", test_files_json_yaml_xml_tag, ids=lambda x: os.path.basename(x)) def test_write_anything_json_yaml_xml_tv(in_file, out_format, tmpdir): in_basename = os.path.basename(in_file) From 46d573e38f20d7079e0bd8a91f960fa5b9908536 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Wed, 16 Nov 2022 11:46:36 +0100 Subject: [PATCH 4/8] [issue-181] adapt tests for files at document-level Signed-off-by: Meret Behrens --- spdx/document.py | 2 +- spdx/package.py | 35 +-------- spdx/utils.py | 39 +++++++++- tests/data/doc_parse/spdx-expected.json | 94 ++++++++++++------------- tests/data/formats/SPDXTagExample.tag | 2 +- tests/test_conversion.py | 10 +++ tests/test_document.py | 45 +++++++----- tests/test_package.py | 1 + tests/test_rdf_parser.py | 2 + tests/test_rdf_writer.py | 4 ++ tests/test_write_anything.py | 7 +- 11 files changed, 136 insertions(+), 105 deletions(-) diff --git a/spdx/document.py b/spdx/document.py index afddf4e97..2e0a65950 100644 --- a/spdx/document.py +++ b/spdx/document.py @@ -126,7 +126,7 @@ def __init__( self.comment = comment self.namespace = namespace self.creation_info = CreationInfo() - self.files: Optional[List['File']] = [] + self.files: List['File'] = [] self.packages = [] if package is not None: self.packages.append(package) diff --git a/spdx/package.py b/spdx/package.py index 976e2e9b2..aaa32f895 100644 --- a/spdx/package.py +++ b/spdx/package.py @@ -9,11 +9,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import hashlib from datetime import datetime from enum import Enum from functools import reduce -from typing import List, Optional +from typing import Optional from spdx import checksum from spdx import creationinfo @@ -74,7 +73,6 @@ class Package(object): - description: Optional str. - comment: Comments about the package being described, optional one. Type: str - - files: List of files in package, at least one. - verif_exc_files: list of file names excluded from verification code or None. - ext_pkg_refs: External references referenced within the given package. Optional, one or many. Type: ExternalPackageRef @@ -113,7 +111,6 @@ def __init__( self.description = None self.comment = None self.attribution_text = None - self.files = [] self.verif_exc_files = [] self.pkg_ext_refs = [] self.primary_package_purpose: Optional[PackagePurpose] = None @@ -140,9 +137,6 @@ def checksum(self): def checksum(self, value): self.checksums[0] = value - def add_file(self, fil): - self.files.append(fil) - def add_lics_from_file(self, lics): self.licenses_from_files.append(lics) @@ -162,7 +156,6 @@ def validate(self, messages): self.validate_checksum(messages) self.validate_optional_str_fields(messages) self.validate_mandatory_str_fields(messages) - self.validate_files(messages) self.validate_pkg_ext_refs(messages) self.validate_optional_fields(messages) messages.pop_context() @@ -245,13 +238,6 @@ def validate_pkg_ext_refs(self, messages): return messages - def validate_files(self, messages): - if self.are_files_analyzed: - for file in self.files: - messages = file.validate(messages) - - return messages - def validate_optional_str_fields(self, messages): """Fields marked as optional and of type string in class docstring must be of a type that provides __str__ method. @@ -311,25 +297,6 @@ def validate_checksum(self, messages): return messages - def calc_verif_code(self): - hashes = [] - - for file_entry in self.files: - if ( - isinstance(file_entry.chksum, checksum.Algorithm) - and file_entry.chksum.identifier == "SHA1" - ): - sha1 = file_entry.chksum.value - else: - sha1 = file_entry.calc_chksum() - hashes.append(sha1) - - hashes.sort() - - sha1 = hashlib.sha1() - sha1.update("".join(hashes).encode("utf-8")) - return sha1.hexdigest() - def has_optional_field(self, field): return bool(getattr(self, field, None)) diff --git a/spdx/utils.py b/spdx/utils.py index 98c0b88c1..70f124b5a 100644 --- a/spdx/utils.py +++ b/spdx/utils.py @@ -10,12 +10,17 @@ # limitations under the License. import datetime +import hashlib import re -from typing import Dict +from typing import Dict, List from ply import lex from ply import yacc +from spdx import checksum +from spdx.file import File +from spdx.package import Package +from spdx.relationship import Relationship from spdx import license @@ -31,7 +36,6 @@ def datetime_iso_format(date): r"(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)Z", re.UNICODE ) - # Groups for retrieving values from DATE_ISO_REGEX matches. DATE_ISO_YEAR_GRP = 1 DATE_ISO_MONTH_GRP = 2 @@ -107,7 +111,6 @@ def __str__(self): class LicenseListLexer(object): - tokens = ["LP", "RP", "AND", "OR", "LICENSE"] def t_LP(self, t): @@ -211,3 +214,33 @@ def update_dict_item_with_new_item(current_state: Dict, key: str, item_to_add: s current_state[key] = [item_to_add] elif item_to_add not in current_state[key]: current_state[key].append(item_to_add) + + +def calc_verif_code(files: List[File]) -> str: + hashes = [] + + for file_entry in files: + if ( + isinstance(file_entry.chksum, checksum.Algorithm) + and file_entry.chksum.identifier == "SHA1" + ): + sha1 = file_entry.chksum.value + else: + sha1 = file_entry.calc_chksum() + hashes.append(sha1) + + hashes.sort() + + sha1 = hashlib.sha1() + sha1.update("".join(hashes).encode("utf-8")) + return sha1.hexdigest() + + +def get_files_in_package(package: Package, files: List[File], relationships: List[Relationship]) -> List[File]: + files_in_package = [] + for file in files: + if file.spdx_id in [relationship.related_spdx_element for relationship in relationships + if relationship.type == "CONTAINS" and relationship.spdx_element_id == package.spdx_id]: + files_in_package.append(file) + + return files_in_package diff --git a/tests/data/doc_parse/spdx-expected.json b/tests/data/doc_parse/spdx-expected.json index f5704427e..30b6e2ede 100644 --- a/tests/data/doc_parse/spdx-expected.json +++ b/tests/data/doc_parse/spdx-expected.json @@ -101,7 +101,53 @@ "identifier": "SHA1", "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" }, - "files": [ + "licenseInfoFromFiles": [ + { + "type": "Single", + "identifier": "Apache-1.0", + "name": "Apache License 1.0" + }, + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + { + "type": "Single", + "identifier": "LicenseRef-1", + "name": "LicenseRef-1" + }, + { + "type": "Single", + "identifier": "LicenseRef-2", + "name": "LicenseRef-2" + }, + { + "type": "Single", + "identifier": "LicenseRef-3", + "name": "CyberNeko License" + }, + { + "type": "Single", + "identifier": "LicenseRef-4", + "name": "LicenseRef-4" + }, + { + "type": "Single", + "identifier": "MPL-1.1", + "name": "Mozilla Public License 1.1" + } + ], + "verificationCode": { + "value": "4e3211c67a2d28fced849ee1bb76e7391b93feba", + "excludedFilesNames": [ + "SpdxTranslatorSpdx.rdf", + "SpdxTranslatorSpdx.txt" + ] + } + } + ], + "files": [ { "id": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-File1", "fileName": "Jenna-2.6.3/jena-2.6.3-sources.jar", @@ -167,52 +213,6 @@ "artifactOfProjectURI": [] } ], - "licenseInfoFromFiles": [ - { - "type": "Single", - "identifier": "Apache-1.0", - "name": "Apache License 1.0" - }, - { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" - }, - { - "type": "Single", - "identifier": "LicenseRef-1", - "name": "LicenseRef-1" - }, - { - "type": "Single", - "identifier": "LicenseRef-2", - "name": "LicenseRef-2" - }, - { - "type": "Single", - "identifier": "LicenseRef-3", - "name": "CyberNeko License" - }, - { - "type": "Single", - "identifier": "LicenseRef-4", - "name": "LicenseRef-4" - }, - { - "type": "Single", - "identifier": "MPL-1.1", - "name": "Mozilla Public License 1.1" - } - ], - "verificationCode": { - "value": "4e3211c67a2d28fced849ee1bb76e7391b93feba", - "excludedFilesNames": [ - "SpdxTranslatorSpdx.rdf", - "SpdxTranslatorSpdx.txt" - ] - } - } - ], "externalDocumentRefs": [ { "externalDocumentId": "DocumentRef-spdx-tool-2.1", diff --git a/tests/data/formats/SPDXTagExample.tag b/tests/data/formats/SPDXTagExample.tag index 97fd4ef9a..27471695d 100644 --- a/tests/data/formats/SPDXTagExample.tag +++ b/tests/data/formats/SPDXTagExample.tag @@ -87,7 +87,7 @@ LicenseComments: This license is used by Jena FileCopyrightText: (c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP ArtifactOfProjectName: Jena ArtifactOfProjectHomePage: http://www.openjena.org/ -ArtifactOfProjectURI: UNKNOWN +ArtifactOfProjectURI: http://www.openjena.org/doap.rdf FileComment: This file belongs to Jena ## Snippet Information diff --git a/tests/test_conversion.py b/tests/test_conversion.py index 4d61300d1..0b8e76b1f 100644 --- a/tests/test_conversion.py +++ b/tests/test_conversion.py @@ -112,30 +112,40 @@ def write_xml_file(self, document, file_name): with open(file_name, mode='w') as out: xmlwriter.write_document(document, out) + @unittest.skip("This test fails because rdf doesn't support files at document-level yet." + " https://github.com/spdx/tools-python/issues/295") def test_tagvalue_rdf(self): doc = self.parse_tagvalue_file(utils_test.get_test_loc('formats/SPDXTagExample.tag')) filename = get_temp_file('.rdf') self.write_rdf_file(doc, filename) self.parse_rdf_file(filename) + @unittest.skip("This test fails because rdf doesn't support files at document-level yet." + " https://github.com/spdx/tools-python/issues/295") def test_json_rdf(self): doc = self.parse_json_file(utils_test.get_test_loc('formats/SPDXJsonExample.json')) filename = get_temp_file('.rdf') self.write_rdf_file(doc, filename) self.parse_rdf_file(filename) + @unittest.skip("This test fails because rdf doesn't support files at document-level yet." + " https://github.com/spdx/tools-python/issues/295") def test_yaml_rdf(self): doc = self.parse_yaml_file(utils_test.get_test_loc('formats/SPDXYamlExample.yaml')) filename = get_temp_file('.rdf') self.write_rdf_file(doc, filename) self.parse_rdf_file(filename) + @unittest.skip("This test fails because rdf doesn't support files at document-level yet." + " https://github.com/spdx/tools-python/issues/295") def test_xml_rdf(self): doc = self.parse_xml_file(utils_test.get_test_loc('formats/SPDXXmlExample.xml')) filename = get_temp_file('.rdf') self.write_rdf_file(doc, filename) self.parse_rdf_file(filename) + @unittest.skip("This test fails because rdf doesn't support files at document-level yet." + " https://github.com/spdx/tools-python/issues/295") def test_rdf_rdf(self): doc = self.parse_rdf_file(utils_test.get_test_loc('formats/SPDXRdfExample.rdf')) filename = get_temp_file('.rdf') diff --git a/tests/test_document.py b/tests/test_document.py index 527f83c9c..e09b96551 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -12,6 +12,7 @@ import os import shutil import tempfile +import unittest from datetime import datetime from unittest import TestCase @@ -23,7 +24,7 @@ from spdx.file import File from spdx.package import Package, PackagePurpose from spdx.parsers.loggers import ErrorMessages -from spdx.relationship import Relationship +from spdx.relationship import Relationship, RelationshipType from spdx.utils import NoAssert from spdx.version import Version @@ -121,7 +122,9 @@ def test_document_is_valid_when_using_or_later_licenses(self): file1.add_lics(lic1) package.add_lics_from_file(lic1) - package.add_file(file1) + doc.add_file(file1) + relationship = create_relationship(package.spdx_id, RelationshipType.CONTAINS, file1.spdx_id) + doc.add_relationships(relationship) messages = ErrorMessages() messages = doc.validate(messages) assert not messages @@ -199,10 +202,10 @@ def _get_lgpl_doc(self, or_later=False): doc.add_file(file1) package.add_lics_from_file(lic1) - relationship = package.spdx_id + " " + "CONTAINS" + " " + file1.spdx_id - doc.add_relationships(Relationship(relationship)) - relationship = doc.spdx_id + " " + "DESCRIBES" + " " + package.spdx_id - doc.add_relationships(Relationship(relationship)) + relationship = create_relationship(package.spdx_id, RelationshipType.CONTAINS, file1.spdx_id) + doc.add_relationships(relationship) + relationship = create_relationship(doc.spdx_id, RelationshipType.DESCRIBES, package.spdx_id) + doc.add_relationships(relationship) return doc def _get_lgpl_multi_package_doc(self, or_later=False): @@ -257,19 +260,21 @@ def _get_lgpl_multi_package_doc(self, or_later=False): doc.add_package(package3) doc.add_file(file1) - relationship = doc.spdx_id + " " + "DESCRIBES" + " " + package1.spdx_id - doc.add_relationships(Relationship(relationship)) - relationship = doc.spdx_id + " " + "DESCRIBES" + " " + package2.spdx_id - doc.add_relationships(Relationship(relationship)) - relationship = doc.spdx_id + " " + "DESCRIBES" + " " + package3.spdx_id - doc.add_relationships(Relationship(relationship)) - relationship = package2.spdx_id + " " + "CONTAINS" + " " + file1.spdx_id - doc.add_relationships(Relationship(relationship)) - relationship = package3.spdx_id + " " + "CONTAINS" + " " + file1.spdx_id - doc.add_relationships(Relationship(relationship)) + relationship = create_relationship(doc.spdx_id, RelationshipType.DESCRIBES, package1.spdx_id) + doc.add_relationships(relationship) + relationship = create_relationship(doc.spdx_id, RelationshipType.DESCRIBES, package2.spdx_id) + doc.add_relationships(relationship) + relationship = create_relationship(doc.spdx_id, RelationshipType.DESCRIBES, package3.spdx_id) + doc.add_relationships(relationship) + relationship = create_relationship(package2.spdx_id, RelationshipType.CONTAINS, file1.spdx_id) + doc.add_relationships(relationship) + relationship = create_relationship(package3.spdx_id, RelationshipType.CONTAINS, file1.spdx_id) + doc.add_relationships(relationship) return doc + @unittest.skip("This test fails because rdf doesn't support files at document-level yet. " + "https://github.com/spdx/tools-python/issues/295") def test_write_document_rdf_with_validate(self): from spdx.writers.rdf import write_document doc = self._get_lgpl_doc() @@ -289,6 +294,8 @@ def test_write_document_rdf_with_validate(self): if temp_dir and os.path.exists(temp_dir): shutil.rmtree(temp_dir) + @unittest.skip("This test fails because rdf doesn't support files at document-level yet. " + "https://github.com/spdx/tools-python/issues/295") def test_write_document_rdf_with_or_later_with_validate(self): from spdx.writers.rdf import write_document doc = self._get_lgpl_doc(or_later=True) @@ -557,6 +564,8 @@ def test_write_document_tv_mini(self): if temp_dir and os.path.exists(temp_dir): shutil.rmtree(temp_dir) + @unittest.skip("This test fails because rdf doesn't support files at document-level yet. " + "https://github.com/spdx/tools-python/issues/295") def test_write_document_rdf_mini(self): from spdx.writers.rdf import write_document doc = self._get_mini_doc() @@ -573,6 +582,10 @@ def test_write_document_rdf_mini(self): shutil.rmtree(temp_dir) +def create_relationship(spdx_element_id: str, relationship_type: RelationshipType, related_spdx_element: str) -> Relationship: + return Relationship(spdx_element_id + " " + relationship_type.name + " " + related_spdx_element) + + class TestLicense(TestCase): maxDiff = None diff --git a/tests/test_package.py b/tests/test_package.py index 0c8529734..8438a5503 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -18,6 +18,7 @@ class TestPackage(TestCase): + @unittest.skip("https://github.com/spdx/tools-python/issues/296") def test_calc_verif_code(self): package = Package() package.calc_verif_code() diff --git a/tests/test_rdf_parser.py b/tests/test_rdf_parser.py index 8f52f58b7..4ebea4916 100644 --- a/tests/test_rdf_parser.py +++ b/tests/test_rdf_parser.py @@ -25,6 +25,8 @@ class TestParser(unittest.TestCase): maxDiff = None + @unittest.skip("This test fails because rdf doesn't support files at document-level yet. " + "https://github.com/spdx/tools-python/issues/295") def test_rdf_parser(self, regen=False): parser = rdf.Parser(RDFBuilder(), StandardLogger()) test_file = utils_test.get_test_loc('formats/SPDXRdfExample.rdf', test_data_dir=utils_test.test_data_dir) diff --git a/tests/test_rdf_writer.py b/tests/test_rdf_writer.py index 9153dbac6..a76743c52 100644 --- a/tests/test_rdf_writer.py +++ b/tests/test_rdf_writer.py @@ -23,6 +23,8 @@ def temporary_file_path() -> str: # This test is really clunky since it's hard to isolate features of the rdf writer to test. Should be improved when # that part is refactored. +@pytest.mark.skip(reason="This test fails because rdf doesn't support files at document-level yet." + " https://github.com/spdx/tools-python/issues/295") def test_accept_provided_doc_node(temporary_file_path) -> None: doc_node = URIRef("http://www.spdx.org/tools#SPDXRef-DOCUMENT") document: Document = minimal_document_with_package() @@ -40,6 +42,8 @@ def test_accept_provided_doc_node(temporary_file_path) -> None: assert parsed_document.spdx_id is None +@pytest.mark.skip(reason="This test fails because rdf doesn't support files at document-level yet." + " https://github.com/spdx/tools-python/issues/295") def test_external_package_references(temporary_file_path) -> None: document: Document = minimal_document_with_package() package: Package = document.packages[0] diff --git a/tests/test_write_anything.py b/tests/test_write_anything.py index 2cf982188..e848c4444 100644 --- a/tests/test_write_anything.py +++ b/tests/test_write_anything.py @@ -19,9 +19,8 @@ dirname = os.path.join(os.path.dirname(__file__), "data", "formats") test_files = [os.path.join(dirname, fn) for fn in os.listdir(dirname)] -test_files_json_yaml_xml_tag = [filename for filename in test_files if "TagExample" in filename] - #filename.endswith("json") or - # filename.endswith("yaml") or filename.endswith("xml") or filename.endswith("tag")] +test_files_json_yaml_xml_tag = [filename for filename in test_files if filename.endswith("json") + or filename.endswith("yaml") or filename.endswith("xml") or filename.endswith("tag")] test_files_rdf = [filename for filename in test_files if filename.endswith("rdf")] UNSTABLE_CONVERSIONS = { "SPDXTagExample.tag-rdf", @@ -60,6 +59,8 @@ def test_write_anything_json_yaml_xml_tv(in_file, out_format, tmpdir): write_anything_test(in_basename, in_file, out_format, tmpdir) +@pytest.mark.skip(reason="This test fails because rdf doesn't support files at document-level yet." + " https://github.com/spdx/tools-python/issues/295") @pytest.mark.parametrize("out_format", ['rdf']) @pytest.mark.parametrize("in_file", test_files_rdf, ids=lambda x: os.path.basename(x)) def test_write_anything_rdf(in_file, out_format, tmpdir): From fcb4961987c1fdb5b32102bf3152113cc3c5fc6b Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Wed, 16 Nov 2022 11:46:36 +0100 Subject: [PATCH 5/8] [issue-181] adapt tests for files at document-level Signed-off-by: Meret Behrens --- spdx/utils.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/spdx/utils.py b/spdx/utils.py index 70f124b5a..b1893aab8 100644 --- a/spdx/utils.py +++ b/spdx/utils.py @@ -209,6 +209,36 @@ def parse(self, data): return None +def calc_verif_code(files: List[File]) -> str: + hashes = [] + + for file_entry in files: + if ( + isinstance(file_entry.chksum, checksum.Algorithm) + and file_entry.chksum.identifier == "SHA1" + ): + sha1 = file_entry.chksum.value + else: + sha1 = file_entry.calc_chksum() + hashes.append(sha1) + + hashes.sort() + + sha1 = hashlib.sha1() + sha1.update("".join(hashes).encode("utf-8")) + return sha1.hexdigest() + + +def get_files_in_package(package: Package, files: List[File], relationships: List[Relationship]) -> List[File]: + files_in_package = [] + for file in files: + if file.spdx_id in [relationship.related_spdx_element for relationship in relationships + if relationship.type == "CONTAINS" and relationship.spdx_element_id == package.spdx_id]: + files_in_package.append(file) + + return files_in_package + + def update_dict_item_with_new_item(current_state: Dict, key: str, item_to_add: str) -> None: if key not in current_state: current_state[key] = [item_to_add] @@ -216,7 +246,7 @@ def update_dict_item_with_new_item(current_state: Dict, key: str, item_to_add: s current_state[key].append(item_to_add) -def calc_verif_code(files: List[File]) -> str: +def calc_verif_code(files: List['File']) -> str: hashes = [] for file_entry in files: @@ -236,11 +266,11 @@ def calc_verif_code(files: List[File]) -> str: return sha1.hexdigest() -def get_files_in_package(package: Package, files: List[File], relationships: List[Relationship]) -> List[File]: +def get_files_in_package(package: Package, files: List['File'], relationships: List[Relationship]) -> List['File']: files_in_package = [] for file in files: if file.spdx_id in [relationship.related_spdx_element for relationship in relationships - if relationship.type == "CONTAINS" and relationship.spdx_element_id == package.spdx_id]: + if relationship.relationship_type == "CONTAINS" and relationship.spdx_element_id == package.spdx_id]: files_in_package.append(file) return files_in_package From 986f9fa320ed7ebfb2fe4972b18ec5878734ee6c Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Wed, 16 Nov 2022 12:15:53 +0100 Subject: [PATCH 6/8] [issue-181] fix writing of snippets in tv-writer Signed-off-by: Meret Behrens --- spdx/writers/tagvalue.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index 2619dbae8..411ec7bfa 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -431,10 +431,13 @@ def write_document(document, out, validate=True): write_separator(out) -def write_snippets_contained_in_file(file, files_contain_snippets: List, out: TextIO) -> None: - if file.spdx_id in files_contain_snippets: - for snippet in files_contain_snippets[file.spdx_id]: - write_snippet(snippet, out) +def write_snippets_contained_in_file(file, files_containing_snippets: Dict, out: TextIO) -> None: + if file.spdx_id in files_containing_snippets: + for snippet_spdx_id in files_containing_snippets[file.spdx_id]: + for snippet in document.snippet: + if snippet.spdx_id == snippet_spdx_id: + write_snippet(snippet, out) + break def scan_relationships(relationships: List[Relationship], packages: List[Package], files: List[File]) -> Tuple[ From 55de3388848e33092d157a08dda7642f258eea9e Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Wed, 16 Nov 2022 13:33:53 +0100 Subject: [PATCH 7/8] [issue-181] fix rdf tests Signed-off-by: Meret Behrens --- spdx/creationinfo.py | 2 +- spdx/utils.py | 13 +- spdx/writers/rdf.py | 4 +- tests/data/doc_parse/spdx-expected.json | 587 +++++++++++----------- tests/data/doc_write/rdf-simple-plus.json | 30 +- tests/data/doc_write/rdf-simple.json | 27 +- tests/test_conversion.py | 10 - tests/test_document.py | 6 - tests/test_rdf_parser.py | 5 +- tests/test_rdf_writer.py | 4 - tests/test_write_anything.py | 2 - tests/utils_test.py | 7 +- 12 files changed, 353 insertions(+), 344 deletions(-) diff --git a/spdx/creationinfo.py b/spdx/creationinfo.py index e464bda9d..259a9fa9d 100644 --- a/spdx/creationinfo.py +++ b/spdx/creationinfo.py @@ -154,7 +154,7 @@ def remove_creator(self, creator): self.creators.remove(creator) def set_created_now(self): - self.created = datetime.utcnow() + self.created = datetime.utcnow().replace(microsecond=0) @property def created_iso_format(self): diff --git a/spdx/utils.py b/spdx/utils.py index b1893aab8..115ea3ae1 100644 --- a/spdx/utils.py +++ b/spdx/utils.py @@ -12,13 +12,14 @@ import datetime import hashlib import re -from typing import Dict, List +from typing import Dict, List, TYPE_CHECKING from ply import lex from ply import yacc from spdx import checksum -from spdx.file import File +if TYPE_CHECKING: + from spdx.file import File from spdx.package import Package from spdx.relationship import Relationship from spdx import license @@ -209,7 +210,7 @@ def parse(self, data): return None -def calc_verif_code(files: List[File]) -> str: +def calc_verif_code(files: List['File']) -> str: hashes = [] for file_entry in files: @@ -229,11 +230,13 @@ def calc_verif_code(files: List[File]) -> str: return sha1.hexdigest() -def get_files_in_package(package: Package, files: List[File], relationships: List[Relationship]) -> List[File]: +def get_files_in_package(package: Package, files: List['File'], relationships: List[Relationship]) -> List['File']: files_in_package = [] for file in files: if file.spdx_id in [relationship.related_spdx_element for relationship in relationships - if relationship.type == "CONTAINS" and relationship.spdx_element_id == package.spdx_id]: + if relationship.relationship_type == "CONTAINS" and relationship.spdx_element_id == package.spdx_id] \ + or file.spdx_id in [relationship.spdx_element_id for relationship in relationships + if relationship.relationship_type == "CONTAINED_BY" and relationship.related_spdx_element == package.spdx_id]: files_in_package.append(file) return files_in_package diff --git a/spdx/writers/rdf.py b/spdx/writers/rdf.py index 2591aabed..07d35ba6f 100644 --- a/spdx/writers/rdf.py +++ b/spdx/writers/rdf.py @@ -25,6 +25,7 @@ from spdx.package import Package from spdx.parsers.loggers import ErrorMessages from spdx.relationship import Relationship +from spdx.utils import get_files_in_package from spdx.writers.tagvalue import InvalidDocumentError import warnings @@ -885,7 +886,8 @@ def handle_package_has_file(self, package, package_node): Add hasFile triples to graph. Must be called after files have been added. """ - file_nodes = map(self.handle_package_has_file_helper, package.files) + files = get_files_in_package(package, self.document.files, self.document.relationships) + file_nodes = map(self.handle_package_has_file_helper, files) triples = [ (package_node, self.spdx_namespace.hasFile, node) for node in file_nodes ] diff --git a/tests/data/doc_parse/spdx-expected.json b/tests/data/doc_parse/spdx-expected.json index 30b6e2ede..0229a788e 100644 --- a/tests/data/doc_parse/spdx-expected.json +++ b/tests/data/doc_parse/spdx-expected.json @@ -1,314 +1,321 @@ { - "id": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-DOCUMENT", - "specVersion": { - "major": 2, - "minor": 1 - }, - "documentNamespace": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", - "name": "Sample_Document-V2.1", - "comment": "This is a sample spreadsheet", - "dataLicense": { - "type": "Single", - "identifier": "CC0-1.0", - "name": "Creative Commons Zero v1.0 Universal" - }, - "licenseListVersion": { - "major": 3, - "minor": 6 - }, - "creators": [ - { - "name": "Gary O'Neall", - "email": null, - "type": "Person" + "id": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-DOCUMENT", + "specVersion": { + "major": 2, + "minor": 1 }, - { - "name": "Source Auditor Inc.", - "email": null, - "type": "Organization" + "documentNamespace": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", + "name": "Sample_Document-V2.1", + "comment": "This is a sample spreadsheet", + "dataLicense": { + "type": "Single", + "identifier": "CC0-1.0", + "name": "Creative Commons Zero v1.0 Universal" + }, + "licenseListVersion": { + "major": 3, + "minor": 6 }, - { - "name": "SourceAuditor-V1.2", - "type": "Tool" - } - ], - "created": "2010-02-03T00:00:00Z", - "creatorComment": "This is an example of an SPDX spreadsheet format", - "packages": [ - { - "id": "SPDXRef-Package", - "name": "SPDX Translator", - "packageFileName": "spdxtranslator-1.0.zip", - "summary": "SPDX Translator utility", - "description": "This utility translates and SPDX RDF XML document to a spreadsheet, translates a spreadsheet to an SPDX RDF XML document and translates an SPDX RDFa document to an SPDX RDF XML document.", - "versionInfo": "Version 0.9.2", - "sourceInfo": "Version 1.0 of the SPDX Translator application", - "downloadLocation": "http://www.spdx.org/tools", - "homepage": null, - "originator": { - "name": "SPDX", - "email": null, - "type": "Organization" - }, - "supplier": { - "name": "Linux Foundation", - "email": null, - "type": "Organization" - }, - "licenseConcluded": { - "type": "Conjunction", - "identifier": [ - "Apache-1.0", - "Apache-2.0", - "LicenseRef-1", - "LicenseRef-2", - "LicenseRef-3", - "LicenseRef-4", - "MPL-1.1" - ], - "name": [ - "Apache License 1.0", - "Apache License 2.0", - "CyberNeko License", - "LicenseRef-1", - "LicenseRef-2", - "LicenseRef-4", - "Mozilla Public License 1.1" - ] - }, - "licenseDeclared": { - "type": "Conjunction", - "identifier": [ - "Apache-2.0", - "LicenseRef-1", - "LicenseRef-2", - "LicenseRef-3", - "LicenseRef-4", - "MPL-1.1" - ], - "name": [ - "Apache License 2.0", - "CyberNeko License", - "LicenseRef-1", - "LicenseRef-2", - "LicenseRef-4", - "Mozilla Public License 1.1" - ] - }, - "copyrightText": " Copyright 2010, 2011 Source Auditor Inc.", - "licenseComment": "The declared license information can be found in the NOTICE file at the root of the archive file", - "checksum": { - "identifier": "SHA1", - "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" - }, - "licenseInfoFromFiles": [ + "creators": [ { - "type": "Single", - "identifier": "Apache-1.0", - "name": "Apache License 1.0" + "name": "Gary O'Neall", + "email": null, + "type": "Person" }, { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" + "name": "Source Auditor Inc.", + "email": null, + "type": "Organization" }, { - "type": "Single", - "identifier": "LicenseRef-1", - "name": "LicenseRef-1" + "name": "SourceAuditor-V1.2", + "type": "Tool" + } + ], + "created": "2010-02-03T00:00:00Z", + "creatorComment": "This is an example of an SPDX spreadsheet format", + "files": [ + { + "id": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-File1", + "fileName": "Jenna-2.6.3/jena-2.6.3-sources.jar", + "type": 3, + "comment": "This file belongs to Jena", + "licenseConcluded": { + "type": "Single", + "identifier": "LicenseRef-1", + "name": "LicenseRef-1" + }, + "copyrightText": "(c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP", + "licenseComment": "This license is used by Jena", + "notice": null, + "checksum": { + "identifier": "SHA1", + "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125" + }, + "licenseInfoInFiles": [ + { + "type": "Single", + "identifier": "LicenseRef-1", + "name": "LicenseRef-1" + } + ], + "contributors": [], + "dependencies": [], + "artifactOfProjectName": [ + "Jena" + ], + "artifactOfProjectHome": [ + "http://www.openjena.org/" + ], + "artifactOfProjectURI": [] }, { - "type": "Single", - "identifier": "LicenseRef-2", - "name": "LicenseRef-2" + "id": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-File2", + "fileName": "src/org/spdx/parser/DOAPProject.java", + "type": 1, + "comment": null, + "licenseConcluded": { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + "copyrightText": "Copyright 2010, 2011 Source Auditor Inc.", + "licenseComment": null, + "notice": null, + "checksum": { + "identifier": "SHA1", + "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + }, + "licenseInfoInFiles": [ + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + } + ], + "contributors": [], + "dependencies": [], + "artifactOfProjectName": [], + "artifactOfProjectHome": [], + "artifactOfProjectURI": [] + } + ], + "packages": [ + { + "id": "SPDXRef-Package", + "name": "SPDX Translator", + "packageFileName": "spdxtranslator-1.0.zip", + "summary": "SPDX Translator utility", + "description": "This utility translates and SPDX RDF XML document to a spreadsheet, translates a spreadsheet to an SPDX RDF XML document and translates an SPDX RDFa document to an SPDX RDF XML document.", + "versionInfo": "Version 0.9.2", + "sourceInfo": "Version 1.0 of the SPDX Translator application", + "downloadLocation": "http://www.spdx.org/tools", + "homepage": null, + "originator": { + "name": "SPDX", + "email": null, + "type": "Organization" + }, + "supplier": { + "name": "Linux Foundation", + "email": null, + "type": "Organization" + }, + "licenseConcluded": { + "type": "Conjunction", + "identifier": [ + "Apache-1.0", + "Apache-2.0", + "LicenseRef-1", + "LicenseRef-2", + "LicenseRef-3", + "LicenseRef-4", + "MPL-1.1" + ], + "name": [ + "Apache License 1.0", + "Apache License 2.0", + "CyberNeko License", + "LicenseRef-1", + "LicenseRef-2", + "LicenseRef-4", + "Mozilla Public License 1.1" + ] + }, + "licenseDeclared": { + "type": "Conjunction", + "identifier": [ + "Apache-2.0", + "LicenseRef-1", + "LicenseRef-2", + "LicenseRef-3", + "LicenseRef-4", + "MPL-1.1" + ], + "name": [ + "Apache License 2.0", + "CyberNeko License", + "LicenseRef-1", + "LicenseRef-2", + "LicenseRef-4", + "Mozilla Public License 1.1" + ] + }, + "copyrightText": " Copyright 2010, 2011 Source Auditor Inc.", + "licenseComment": "The declared license information can be found in the NOTICE file at the root of the archive file", + "checksum": { + "identifier": "SHA1", + "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + }, + "licenseInfoFromFiles": [ + { + "type": "Single", + "identifier": "Apache-1.0", + "name": "Apache License 1.0" + }, + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + { + "type": "Single", + "identifier": "LicenseRef-1", + "name": "LicenseRef-1" + }, + { + "type": "Single", + "identifier": "LicenseRef-2", + "name": "LicenseRef-2" + }, + { + "type": "Single", + "identifier": "LicenseRef-3", + "name": "CyberNeko License" + }, + { + "type": "Single", + "identifier": "LicenseRef-4", + "name": "LicenseRef-4" + }, + { + "type": "Single", + "identifier": "MPL-1.1", + "name": "Mozilla Public License 1.1" + } + ], + "verificationCode": { + "value": "4e3211c67a2d28fced849ee1bb76e7391b93feba", + "excludedFilesNames": [ + "SpdxTranslatorSpdx.rdf", + "SpdxTranslatorSpdx.txt" + ] + } + } + ], + "externalDocumentRefs": [ + { + "externalDocumentId": "DocumentRef-spdx-tool-2.1", + "spdxDocument": "https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301", + "checksum": { + "identifier": "SHA1", + "value": "d6a770ba38583ed4bb4525bd96e50461655d2759" + } + } + ], + "extractedLicenses": [ + { + "name": "LicenseRef-1", + "identifier": "LicenseRef-1", + "text": "/*\n * (c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */", + "comment": null, + "cross_refs": [] }, { - "type": "Single", - "identifier": "LicenseRef-3", - "name": "CyberNeko License" + "name": "LicenseRef-2", + "identifier": "LicenseRef-2", + "text": "This package includes the GRDDL parser developed by Hewlett Packard under the following license:\n\u00a9 Copyright 2007 Hewlett-Packard Development Company, LP\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: \n\nRedistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. \nRedistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. \nThe name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. \nTHIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ", + "comment": null, + "cross_refs": [] }, { - "type": "Single", - "identifier": "LicenseRef-4", - "name": "LicenseRef-4" + "name": "CyberNeko License", + "identifier": "LicenseRef-3", + "text": "The CyberNeko Software License, Version 1.0\n\n \n(C) Copyright 2002-2005, Andy Clark. All rights reserved.\n \nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions\nare met:\n\n1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer. \n\n2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in\n the documentation and/or other materials provided with the\n distribution.\n\n3. The end-user documentation included with the redistribution,\n if any, must include the following acknowledgment: \n \"This product includes software developed by Andy Clark.\"\n Alternately, this acknowledgment may appear in the software itself,\n if and wherever such third-party acknowledgments normally appear.\n\n4. The names \"CyberNeko\" and \"NekoHTML\" must not be used to endorse\n or promote products derived from this software without prior \n written permission. For written permission, please contact \n andyc@cyberneko.net.\n\n5. Products derived from this software may not be called \"CyberNeko\",\n nor may \"CyberNeko\" appear in their name, without prior written\n permission of the author.\n\nTHIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED\nWARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\nOF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR OTHER CONTRIBUTORS\nBE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, \nOR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT \nOF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR \nBUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, \nWHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE \nOR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, \nEVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", + "comment": "This is tye CyperNeko License", + "cross_refs": [ + "http://justasample.url.com", + "http://people.apache.org/~andyc/neko/LICENSE" + ] }, { - "type": "Single", - "identifier": "MPL-1.1", - "name": "Mozilla Public License 1.1" + "name": "LicenseRef-4", + "identifier": "LicenseRef-4", + "text": "/*\n * (c) Copyright 2009 University of Bristol\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */ ", + "comment": null, + "cross_refs": [] } - ], - "verificationCode": { - "value": "4e3211c67a2d28fced849ee1bb76e7391b93feba", - "excludedFilesNames": [ - "SpdxTranslatorSpdx.rdf", - "SpdxTranslatorSpdx.txt" - ] - } - } - ], - "files": [ + ], + "annotations": [ { - "id": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-File1", - "fileName": "Jenna-2.6.3/jena-2.6.3-sources.jar", - "type": 3, - "comment": "This file belongs to Jena", - "licenseConcluded": { - "type": "Single", - "identifier": "LicenseRef-1", - "name": "LicenseRef-1" - }, - "copyrightText": "(c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP", - "licenseComment": "This license is used by Jena", - "notice": null, - "checksum": { - "identifier": "SHA1", - "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125" - }, - "licenseInfoInFiles": [ - { - "type": "Single", - "identifier": "LicenseRef-1", - "name": "LicenseRef-1" - } - ], - "contributors": [], - "dependencies": [], - "artifactOfProjectName": [ - "Jena" - ], - "artifactOfProjectHome": [ - "http://www.openjena.org/" - ], - "artifactOfProjectURI": [] + "id": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-45", + "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", + "type": "REVIEW", + "annotator": { + "name": "Jim Reviewer", + "email": null, + "type": "Person" + }, + "date": "2012-06-13T00:00:00Z" + } + ], + "reviews": [ + { + "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", + "reviewer": { + "name": "Joe Reviewer", + "email": null, + "type": "Person" + }, + "date": "2010-02-10T00:00:00Z" }, { - "id": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-File2", - "fileName": "src/org/spdx/parser/DOAPProject.java", - "type": 1, - "comment": null, - "licenseConcluded": { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" - }, - "copyrightText": "Copyright 2010, 2011 Source Auditor Inc.", - "licenseComment": null, - "notice": null, - "checksum": { - "identifier": "SHA1", - "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" - }, - "licenseInfoInFiles": [ - { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" - } - ], - "contributors": [], - "dependencies": [], - "artifactOfProjectName": [], - "artifactOfProjectHome": [], - "artifactOfProjectURI": [] + "comment": "Another example reviewer.", + "reviewer": { + "name": "Suzanne Reviewer", + "email": null, + "type": "Person" + }, + "date": "2011-03-13T00:00:00Z" } - ], - "externalDocumentRefs": [ - { - "externalDocumentId": "DocumentRef-spdx-tool-2.1", - "spdxDocument": "https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301", - "checksum": { - "identifier": "SHA1", - "value": "d6a770ba38583ed4bb4525bd96e50461655d2759" - } - } - ], - "extractedLicenses": [ - { - "name": "LicenseRef-1", - "identifier": "LicenseRef-1", - "text": "/*\n * (c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */", - "comment": null, - "cross_refs": [] - }, - { - "name": "LicenseRef-2", - "identifier": "LicenseRef-2", - "text": "This package includes the GRDDL parser developed by Hewlett Packard under the following license:\n\u00a9 Copyright 2007 Hewlett-Packard Development Company, LP\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: \n\nRedistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. \nRedistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. \nThe name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. \nTHIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ", - "comment": null, - "cross_refs": [] - }, - { - "name": "CyberNeko License", - "identifier": "LicenseRef-3", - "text": "The CyberNeko Software License, Version 1.0\n\n \n(C) Copyright 2002-2005, Andy Clark. All rights reserved.\n \nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions\nare met:\n\n1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer. \n\n2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in\n the documentation and/or other materials provided with the\n distribution.\n\n3. The end-user documentation included with the redistribution,\n if any, must include the following acknowledgment: \n \"This product includes software developed by Andy Clark.\"\n Alternately, this acknowledgment may appear in the software itself,\n if and wherever such third-party acknowledgments normally appear.\n\n4. The names \"CyberNeko\" and \"NekoHTML\" must not be used to endorse\n or promote products derived from this software without prior \n written permission. For written permission, please contact \n andyc@cyberneko.net.\n\n5. Products derived from this software may not be called \"CyberNeko\",\n nor may \"CyberNeko\" appear in their name, without prior written\n permission of the author.\n\nTHIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED\nWARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\nOF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR OTHER CONTRIBUTORS\nBE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, \nOR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT \nOF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR \nBUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, \nWHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE \nOR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, \nEVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", - "comment": "This is tye CyperNeko License", - "cross_refs": [ - "http://justasample.url.com", - "http://people.apache.org/~andyc/neko/LICENSE" - ] - }, - { - "name": "LicenseRef-4", - "identifier": "LicenseRef-4", - "text": "/*\n * (c) Copyright 2009 University of Bristol\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */ ", - "comment": null, - "cross_refs": [] - } - ], - "annotations": [ - { - "id": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-45", - "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", - "type": "REVIEW", - "annotator": { - "name": "Jim Reviewer", - "email": null, - "type": "Person" - }, - "date": "2012-06-13T00:00:00Z" - } - ], - "reviews": [ - { - "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", - "reviewer": { - "name": "Joe Reviewer", - "email": null, - "type": "Person" - }, - "date": "2010-02-10T00:00:00Z" - }, - { - "comment": "Another example reviewer.", - "reviewer": { - "name": "Suzanne Reviewer", - "email": null, - "type": "Person" - }, - "date": "2011-03-13T00:00:00Z" - } - ], - "snippets": [ - { - "id": "SPDXRef-Snippet", - "name": "from linux kernel", - "comment": "This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0-or-later.", - "copyrightText": "Copyright 2008-2010 John Smith", - "licenseComments": "The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz.", - "snippetFromFile": "SPDXRef-DoapSource", - "licenseConcluded": { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" - }, - "licenseInfoInSnippets": [ + ], + "snippets": [ + { + "id": "SPDXRef-Snippet", + "name": "from linux kernel", + "comment": "This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0-or-later.", + "copyrightText": "Copyright 2008-2010 John Smith", + "licenseComments": "The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz.", + "snippetFromFile": "SPDXRef-DoapSource", + "licenseConcluded": { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + "licenseInfoInSnippets": [ + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + } + ] + } + ], + "relationships": [ { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" + "spdx_element_id": "SPDXRef-DOCUMENT", + "relationship_type": "DESCRIBES", + "related_spdx_element": "SPDXRef-Package" } - ] - } - ] + ] } diff --git a/tests/data/doc_write/rdf-simple-plus.json b/tests/data/doc_write/rdf-simple-plus.json index c3066228c..5fa4c8793 100644 --- a/tests/data/doc_write/rdf-simple-plus.json +++ b/tests/data/doc_write/rdf-simple-plus.json @@ -62,15 +62,27 @@ } } }, - "ns1:relationship": { - "ns1:Relationship": { - "ns1:spdxElementId": "SPDXRef-DOCUMENT", - "ns1:relatedSpdxElement": "SPDXRef-Package", - "ns1:relationshipType": { - "@rdf:resource": "http://spdx.org/rdf/terms#relationshipType_describes" + "ns1:relationship": [ + { + "ns1:Relationship": { + "ns1:spdxElementId": "SPDXRef-DOCUMENT", + "ns1:relatedSpdxElement": "SPDXRef-Package", + "ns1:relationshipType": { + "@rdf:resource": "http://spdx.org/rdf/terms#relationshipType_describes" + } + } + }, + { + "ns1:Relationship": { + "ns1:spdxElementId": "SPDXRef-Package", + "ns1:relatedSpdxElement": "SPDXRef-File", + "ns1:relationshipType": { + "@rdf:resource": "http://spdx.org/rdf/terms#relationshipType_contains" + } + } } - }}, - "@rdf:about": "http://www.spdx.org/tools#SPDXRef-DOCUMENT" + ], + "@rdf:about": "http://www.spdx.org/tools#SPDXRef-DOCUMENT" + } } - } } diff --git a/tests/data/doc_write/rdf-simple.json b/tests/data/doc_write/rdf-simple.json index 5229113ef..27f99a325 100644 --- a/tests/data/doc_write/rdf-simple.json +++ b/tests/data/doc_write/rdf-simple.json @@ -62,15 +62,26 @@ "ns1:fileName": "./some/path/tofile" } }, - "ns1:relationship": { - "ns1:Relationship": { - "ns1:spdxElementId": "SPDXRef-DOCUMENT", - "ns1:relatedSpdxElement": "SPDXRef-Package", - "ns1:relationshipType": { - "@rdf:resource": "http://spdx.org/rdf/terms#relationshipType_describes" + "ns1:relationship": [ + { + "ns1:Relationship": { + "ns1:spdxElementId": "SPDXRef-DOCUMENT", + "ns1:relatedSpdxElement": "SPDXRef-Package", + "ns1:relationshipType": { + "@rdf:resource": "http://spdx.org/rdf/terms#relationshipType_describes" + } + } + }, + { + "ns1:Relationship": { + "ns1:spdxElementId": "SPDXRef-Package", + "ns1:relatedSpdxElement": "SPDXRef-File", + "ns1:relationshipType": { + "@rdf:resource": "http://spdx.org/rdf/terms#relationshipType_contains" + } + } } - } - }, + ], "@rdf:about": "http://www.spdx.org/tools#SPDXRef-DOCUMENT" } } diff --git a/tests/test_conversion.py b/tests/test_conversion.py index 0b8e76b1f..4d61300d1 100644 --- a/tests/test_conversion.py +++ b/tests/test_conversion.py @@ -112,40 +112,30 @@ def write_xml_file(self, document, file_name): with open(file_name, mode='w') as out: xmlwriter.write_document(document, out) - @unittest.skip("This test fails because rdf doesn't support files at document-level yet." - " https://github.com/spdx/tools-python/issues/295") def test_tagvalue_rdf(self): doc = self.parse_tagvalue_file(utils_test.get_test_loc('formats/SPDXTagExample.tag')) filename = get_temp_file('.rdf') self.write_rdf_file(doc, filename) self.parse_rdf_file(filename) - @unittest.skip("This test fails because rdf doesn't support files at document-level yet." - " https://github.com/spdx/tools-python/issues/295") def test_json_rdf(self): doc = self.parse_json_file(utils_test.get_test_loc('formats/SPDXJsonExample.json')) filename = get_temp_file('.rdf') self.write_rdf_file(doc, filename) self.parse_rdf_file(filename) - @unittest.skip("This test fails because rdf doesn't support files at document-level yet." - " https://github.com/spdx/tools-python/issues/295") def test_yaml_rdf(self): doc = self.parse_yaml_file(utils_test.get_test_loc('formats/SPDXYamlExample.yaml')) filename = get_temp_file('.rdf') self.write_rdf_file(doc, filename) self.parse_rdf_file(filename) - @unittest.skip("This test fails because rdf doesn't support files at document-level yet." - " https://github.com/spdx/tools-python/issues/295") def test_xml_rdf(self): doc = self.parse_xml_file(utils_test.get_test_loc('formats/SPDXXmlExample.xml')) filename = get_temp_file('.rdf') self.write_rdf_file(doc, filename) self.parse_rdf_file(filename) - @unittest.skip("This test fails because rdf doesn't support files at document-level yet." - " https://github.com/spdx/tools-python/issues/295") def test_rdf_rdf(self): doc = self.parse_rdf_file(utils_test.get_test_loc('formats/SPDXRdfExample.rdf')) filename = get_temp_file('.rdf') diff --git a/tests/test_document.py b/tests/test_document.py index e09b96551..379aaf535 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -273,8 +273,6 @@ def _get_lgpl_multi_package_doc(self, or_later=False): return doc - @unittest.skip("This test fails because rdf doesn't support files at document-level yet. " - "https://github.com/spdx/tools-python/issues/295") def test_write_document_rdf_with_validate(self): from spdx.writers.rdf import write_document doc = self._get_lgpl_doc() @@ -294,8 +292,6 @@ def test_write_document_rdf_with_validate(self): if temp_dir and os.path.exists(temp_dir): shutil.rmtree(temp_dir) - @unittest.skip("This test fails because rdf doesn't support files at document-level yet. " - "https://github.com/spdx/tools-python/issues/295") def test_write_document_rdf_with_or_later_with_validate(self): from spdx.writers.rdf import write_document doc = self._get_lgpl_doc(or_later=True) @@ -564,8 +560,6 @@ def test_write_document_tv_mini(self): if temp_dir and os.path.exists(temp_dir): shutil.rmtree(temp_dir) - @unittest.skip("This test fails because rdf doesn't support files at document-level yet. " - "https://github.com/spdx/tools-python/issues/295") def test_write_document_rdf_mini(self): from spdx.writers.rdf import write_document doc = self._get_mini_doc() diff --git a/tests/test_rdf_parser.py b/tests/test_rdf_parser.py index 4ebea4916..7963781fa 100644 --- a/tests/test_rdf_parser.py +++ b/tests/test_rdf_parser.py @@ -13,6 +13,7 @@ import io import json import unittest +from collections import OrderedDict from spdx.parsers import rdf from spdx.parsers.loggers import StandardLogger @@ -25,8 +26,6 @@ class TestParser(unittest.TestCase): maxDiff = None - @unittest.skip("This test fails because rdf doesn't support files at document-level yet. " - "https://github.com/spdx/tools-python/issues/295") def test_rdf_parser(self, regen=False): parser = rdf.Parser(RDFBuilder(), StandardLogger()) test_file = utils_test.get_test_loc('formats/SPDXRdfExample.rdf', test_data_dir=utils_test.test_data_dir) @@ -44,6 +43,6 @@ def check_document(self, document, expected_loc, regen=False): o.write(data) with io.open(expected_loc, 'r', encoding='utf-8') as ex: - expected = json.load(ex) + expected = json.load(ex, object_pairs_hook=OrderedDict) assert result == expected diff --git a/tests/test_rdf_writer.py b/tests/test_rdf_writer.py index a76743c52..9153dbac6 100644 --- a/tests/test_rdf_writer.py +++ b/tests/test_rdf_writer.py @@ -23,8 +23,6 @@ def temporary_file_path() -> str: # This test is really clunky since it's hard to isolate features of the rdf writer to test. Should be improved when # that part is refactored. -@pytest.mark.skip(reason="This test fails because rdf doesn't support files at document-level yet." - " https://github.com/spdx/tools-python/issues/295") def test_accept_provided_doc_node(temporary_file_path) -> None: doc_node = URIRef("http://www.spdx.org/tools#SPDXRef-DOCUMENT") document: Document = minimal_document_with_package() @@ -42,8 +40,6 @@ def test_accept_provided_doc_node(temporary_file_path) -> None: assert parsed_document.spdx_id is None -@pytest.mark.skip(reason="This test fails because rdf doesn't support files at document-level yet." - " https://github.com/spdx/tools-python/issues/295") def test_external_package_references(temporary_file_path) -> None: document: Document = minimal_document_with_package() package: Package = document.packages[0] diff --git a/tests/test_write_anything.py b/tests/test_write_anything.py index e848c4444..b5c1ea40c 100644 --- a/tests/test_write_anything.py +++ b/tests/test_write_anything.py @@ -59,8 +59,6 @@ def test_write_anything_json_yaml_xml_tv(in_file, out_format, tmpdir): write_anything_test(in_basename, in_file, out_format, tmpdir) -@pytest.mark.skip(reason="This test fails because rdf doesn't support files at document-level yet." - " https://github.com/spdx/tools-python/issues/295") @pytest.mark.parametrize("out_format", ['rdf']) @pytest.mark.parametrize("in_file", test_files_rdf, ids=lambda x: os.path.basename(x)) def test_write_anything_rdf(in_file, out_format, tmpdir): diff --git a/tests/utils_test.py b/tests/utils_test.py index 1a39e334b..b76d8289a 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -107,17 +107,14 @@ def sort_nested(data): new_data = {} for k, v in data.items(): if isinstance(v, list): - try: - v = sort_nested(v) - except TypeError: - pass + v = sort_nested(v) if isinstance(v, dict): v = sort_nested(v) new_data[k] = v return new_data elif isinstance(data, list): new_data = [] - for v in sorted(data): + for v in sorted(data, key=lambda x: json.dumps(x, sort_keys=True)): if isinstance(v, list): v = sort_nested(v) if isinstance(v, dict): From dfeee30aacefe7fe00e238f53bbee70cc63fc9d1 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Wed, 16 Nov 2022 16:52:12 +0100 Subject: [PATCH 8/8] [issue-181] fix tv-parser/writer Signed-off-by: Meret Behrens --- spdx/parsers/tagvalue.py | 46 +++++++++++------ spdx/parsers/tagvaluebuilders.py | 37 +++++++++----- spdx/writers/tagvalue.py | 88 ++++++++++++++++++-------------- 3 files changed, 104 insertions(+), 67 deletions(-) diff --git a/spdx/parsers/tagvalue.py b/spdx/parsers/tagvalue.py index a5772549f..b060488d2 100644 --- a/spdx/parsers/tagvalue.py +++ b/spdx/parsers/tagvalue.py @@ -575,8 +575,8 @@ def p_file_name_1(self, p): try: value = p[2] self.builder.set_file_name(self.document, value) - self.builder.set_current_name("file", value) - self.builder.set_current_id("file", None) + self.builder.set_current_file_name(value) + self.builder.set_current_file_id(None) except OrderError: self.order_error("FileName", "PackageName", p.lineno(1)) @@ -590,16 +590,30 @@ def p_file_name_2(self, p): def p_spdx_id(self, p): """spdx_id : SPDX_ID LINE""" value = p[2] - if not self.builder.doc_spdx_id_set: - self.builder.set_doc_spdx_id(self.document, value) - elif self.builder.current_package_has_name() and not self.builder.current_package_has_id(): - self.builder.set_pkg_spdx_id(self.document, value) - self.builder.set_current_id("package", value) - else: - self.builder.set_file_spdx_id(self.document, value) - self.builder.set_current_id("file", value) - if self.builder.current_package_has_name: - self.builder.add_relationship(self.document, self.builder.current_package["spdx_id"] + " CONTAINS " + value) + try: + # first parsed spdx id belongs to the document + if not self.builder.doc_spdx_id_set: + self.builder.set_doc_spdx_id(self.document, value) + + # else if a package is in scope that doesn't have an id yet, the parsed spdx id belongs to the package + elif self.builder.current_package_has_name() \ + and not self.builder.current_package_has_id(): + self.builder.set_pkg_spdx_id(self.document, value) + self.builder.set_current_package_id(value) + + # else if a file is in scope that doesn't have an id yet, the parsed spdx id belongs to the file + elif self.builder.current_file_has_name() \ + and not self.builder.current_file_has_id(): + self.builder.set_file_spdx_id(self.document, value) + self.builder.set_current_file_id(value) + if self.builder.has_current_package(): + self.builder.add_relationship(self.document, + self.builder.current_package["spdx_id"] + " CONTAINS " + value) + else: + raise SPDXValueError("SPDX ID couldn't be assigned properly. Line no. {0}") + except SPDXValueError as err: + self.error = True + self.logger.log(err.msg.format(p.lineno(2))) def p_file_comment_1(self, p): """file_comment : FILE_COMMENT text_or_line""" @@ -1075,8 +1089,6 @@ def p_pkg_file_name(self, p): try: value = p[2] self.builder.set_pkg_file_name(self.document, value) - self.builder.set_current_name("file", value) - self.builder.set_current_id("file", None) except OrderError: self.order_error("PackageFileName", "PackageName", p.lineno(1)) except CardinalityError: @@ -1109,8 +1121,10 @@ def p_package_name(self, p): try: value = p[2] self.builder.create_package(self.document, value) - self.builder.set_current_name("package", value) - self.builder.set_current_id("package", None) + self.builder.set_current_package_name(value) + self.builder.set_current_package_id(None) + self.builder.set_current_file_name(None) # start of a new package implies new file + self.builder.set_current_file_id(None) except CardinalityError: self.more_than_one_error("PackageName", p.lineno(1)) diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index 1c603bf8d..e755439bd 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -9,6 +9,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from enum import auto, Enum import re from typing import Dict @@ -1687,24 +1688,34 @@ def __init__(self): self.current_package: Dict = dict() self.current_file: Dict = dict() - def set_current_name(self, indicator: str, name: str) -> None: - if indicator == "package": - self.current_package["name"] = name - elif indicator == "file": - self.current_file["name"] = name + def set_current_package_name(self, name: str) -> None: + self.current_package["name"] = name - def set_current_id(self, indicator: str, spdx_id: str) -> None: - if indicator == "package": - self.current_package["spdx_id"] = spdx_id - elif indicator == "file": - self.current_file["spdx_id"] = spdx_id + def set_current_file_name(self, name: str) -> None: + self.current_file["name"] = name - def current_package_has_name(self): - return bool("name" in self.current_package) and (self.current_package["name"]) + def set_current_file_id(self, spdx_id: str) -> None: + self.current_file["spdx_id"] = spdx_id - def current_package_has_id(self): + def set_current_package_id(self, spdx_id: str) -> None: + self.current_package["spdx_id"] = spdx_id + + def current_package_has_name(self) -> bool: + return bool(("name" in self.current_package) and (self.current_package["name"])) + + def current_file_has_name(self) -> bool: + return bool(("name" in self.current_file) and (self.current_file["name"])) + + def current_package_has_id(self) -> bool: return bool("spdx_id" in self.current_package) and (self.current_package["spdx_id"]) + def current_file_has_id(self) -> bool: + return bool("spdx_id" in self.current_file) and (self.current_file["spdx_id"]) + + def has_current_package(self) -> bool: + return bool(self.current_package) + + def reset(self): """ Reset builder's state for building new documents. diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index 411ec7bfa..f8baf696b 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -14,6 +14,7 @@ from spdx import license, utils from spdx import file as spdx_file +from spdx.document import Document from spdx.file import File from spdx.package import Package from spdx.parsers.loggers import ErrorMessages @@ -386,11 +387,13 @@ def write_document(document, out, validate=True): write_separator(out) write_separator(out) - relationships_to_write, packages_containing_files = scan_relationships(document.relationships, document.packages, - document.files) - files_containing_snippets = determine_files_containing_snippets(document.snippet) - packaged_files = [file_spdx_id for file_spdx_id_list in packages_containing_files.values() - for file_spdx_id in file_spdx_id_list] + relationships_to_write, contained_files_by_package_id = scan_relationships(document.relationships, + document.packages, document.files) + contained_snippets_by_file_id = determine_files_containing_snippets(document.snippet, document.files) + packaged_file_ids = [file.spdx_id for files_list in contained_files_by_package_id.values() + for file in files_list] + filed_snippet_ids = [snippet.spdx_id for snippets_list in contained_snippets_by_file_id.values() + for snippet in snippets_list] # Write Relationships if relationships_to_write: @@ -399,29 +402,31 @@ def write_document(document, out, validate=True): write_relationship(relationship, out) write_separators(out) + # Write snippet info + for snippet in document.snippet: + if snippet.spdx_id not in filed_snippet_ids: + write_snippet(snippet, out) + write_separators(out) + # Write file info for file in document.files: - if file.spdx_id not in packaged_files: + if file.spdx_id not in packaged_file_ids: write_file(file, out) write_separators(out) - write_snippets_contained_in_file(file, files_containing_snippets, out) + if file.spdx_id in contained_snippets_by_file_id: + write_snippets(contained_snippets_by_file_id[file.spdx_id], out) # Write out package info for package in document.packages: write_package(package, out) write_separators(out) - if package.spdx_id in packages_containing_files: - for file_spdx_id in packages_containing_files[package.spdx_id]: - for file in document.files: - if file.spdx_id == file_spdx_id: - write_file(file, out) - write_snippets_contained_in_file(file, files_containing_snippets, out) - break - - # Write out snippet info - for snippet in document.snippet: - write_snippet(snippet, out) - write_separators(out) + if package.spdx_id in contained_files_by_package_id: + for file in contained_files_by_package_id[package.spdx_id]: + write_file(file, out) + write_separators(out) + if file.spdx_id in contained_snippets_by_file_id: + write_snippets(contained_snippets_by_file_id[file.spdx_id], out) + break if document.extracted_licenses: out.write("# Extracted Licenses\n\n") @@ -431,36 +436,43 @@ def write_document(document, out, validate=True): write_separator(out) -def write_snippets_contained_in_file(file, files_containing_snippets: Dict, out: TextIO) -> None: - if file.spdx_id in files_containing_snippets: - for snippet_spdx_id in files_containing_snippets[file.spdx_id]: - for snippet in document.snippet: - if snippet.spdx_id == snippet_spdx_id: - write_snippet(snippet, out) - break +def write_snippets(snippets_to_write: List, out: TextIO) -> None: + for snippet in snippets_to_write: + write_snippet(snippet, out) + write_separators(out) -def scan_relationships(relationships: List[Relationship], packages: List[Package], files: List[File]) -> Tuple[ - List, Dict]: - packages_containing_files = dict() +def scan_relationships(relationships: List[Relationship], packages: List[Package], files: List[File]) \ + -> Tuple[List, Dict]: + contained_files_by_package_id = dict() relationships_to_write = [] + files_by_spdx_id = {file.spdx_id: file for file in files} + packages_spdx_ids = [package.spdx_id for package in packages] for relationship in relationships: if relationship.relationship_type == "CONTAINS" and \ - relationship.spdx_element_id in [package.spdx_id for package in packages] and \ - relationship.related_spdx_element in [file.spdx_id for file in files]: - packages_containing_files.setdefault(relationship.spdx_element_id, []).append( - relationship.related_spdx_element) + relationship.spdx_element_id in packages_spdx_ids and \ + relationship.related_spdx_element in files_by_spdx_id.keys(): + contained_files_by_package_id.setdefault(relationship.spdx_element_id, []).append( + files_by_spdx_id[relationship.related_spdx_element]) + if relationship.has_comment: + relationships_to_write.append(relationship) + elif relationship.relationship_type == "CONTAINED_BY" and \ + relationship.related_spdx_element in packages_spdx_ids and \ + relationship.spdx_element_id in files_by_spdx_id: + contained_files_by_package_id.setdefault(relationship.related_spdx_element, []).append( + files_by_spdx_id[relationship.spdx_element_id]) if relationship.has_comment: relationships_to_write.append(relationship) else: relationships_to_write.append(relationship) - return relationships_to_write, packages_containing_files + return relationships_to_write, contained_files_by_package_id -def determine_files_containing_snippets(snippets: List[Snippet]) -> Dict: - files_containing_snippets = dict() +def determine_files_containing_snippets(snippets: List[Snippet], files: List[File]) -> Dict: + contained_snippets_by_file_id = dict() for snippet in snippets: - files_containing_snippets.setdefault(snippet.snip_from_file_spdxid, []).append(snippet.spdx_id) + if snippet.snip_from_file_spdxid in [file.spdx_id for file in files]: + contained_snippets_by_file_id.setdefault(snippet.snip_from_file_spdxid, []).append(snippet) - return files_containing_snippets + return contained_snippets_by_file_id