From 9314f273ab2fd37439c17c0f8300a808732ac092 Mon Sep 17 00:00:00 2001 From: bjv Date: Mon, 23 May 2022 16:11:30 -0700 Subject: [PATCH 01/18] Add unpackaged file info + tag/value writer Signed-off-by: Brandon J. Van Vaerenbergh --- spdx/document.py | 16 +++++++++++++++- spdx/writers/tagvalue.py | 4 ++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/spdx/document.py b/spdx/document.py index e0232d7cc..4440cc9f0 100644 --- a/spdx/document.py +++ b/spdx/document.py @@ -316,6 +316,7 @@ def __init__( self.packages = [] if package is not None: self.packages.append(package) + self.unpackaged_files = [] self.extracted_licenses = [] self.reviews = [] self.annotations = [] @@ -331,6 +332,9 @@ def add_annotation(self, annotation): def add_relationships(self, relationship): self.relationships.append(relationship) + def add_file(self, value): + self.unpackaged_files.append(value) + def add_extr_lic(self, lic): self.extracted_licenses.append(lic) @@ -370,10 +374,16 @@ def package(self, value): @property def files(self): + warnings.warn('document.package and document.files are deprecated; ' + 'use document.packages instead', + DeprecationWarning) return self.package.files @files.setter def files(self, value): + warnings.warn('document.package and document.files are deprecated; ' + 'use document.packages instead', + DeprecationWarning) self.package.files = value @property @@ -400,6 +410,7 @@ def validate(self, messages=None): self.validate_creation_info(messages) self.validate_packages(messages) self.validate_extracted_licenses(messages) + self.validate_unpackaged_files(messages) self.validate_reviews(messages) self.validate_snippet(messages) self.validate_annotations(messages) @@ -486,4 +497,7 @@ def validate_extracted_licenses(self, messages): messages.append( "Document extracted licenses must be of type " "spdx.document.ExtractedLicense and not " + type(lic) - ) \ No newline at end of file + ) + def validate_unpackaged_files(self, messages): + for value in self.unpackaged_files: + messages = value.validate(messages) diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index 7ffd80d4b..d37549442 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -358,6 +358,10 @@ def write_document(document, out, validate=True): write_relationship(relationship, out) write_separators(out) + # Write out unpackaged file info + for spdx_file in document.unpackaged_files: + write_file(spdx_file, out) + # Write out package info for package in document.packages: write_package(package, out) From 3acc3dc88059330ed80181665fe2a21cdc2bfbac Mon Sep 17 00:00:00 2001 From: bjv Date: Wed, 25 May 2022 14:52:03 -0700 Subject: [PATCH 02/18] add unpackaged files to tag/value parser, tests & json-yaml-xml writer --- spdx/parsers/tagvalue.py | 3 +- spdx/parsers/tagvaluebuilders.py | 133 +++++++++++++++++++++++-------- spdx/writers/jsonyamlxml.py | 3 + tests/test_tag_value_parser.py | 23 +++++- 4 files changed, 127 insertions(+), 35 deletions(-) diff --git a/spdx/parsers/tagvalue.py b/spdx/parsers/tagvalue.py index eda58fa70..4b645f385 100644 --- a/spdx/parsers/tagvalue.py +++ b/spdx/parsers/tagvalue.py @@ -564,7 +564,8 @@ def p_spdx_id(self, p): value = p[2] if not self.builder.doc_spdx_id_set: self.builder.set_doc_spdx_id(self.document, value) - elif not self.builder.package_spdx_id_set: + elif (not self.builder.package_spdx_id_set + and self.builder.package_set): self.builder.set_pkg_spdx_id(self.document, value) else: self.builder.set_file_spdx_id(self.document, value) diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index caf7f2ff9..29d8c24d9 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -162,6 +162,22 @@ def set_doc_namespace(self, doc, namespace): else: raise CardinalityError("Document::Comment") + def set_doc_pkgs_started(self, doc): + """ + Indicate Package has been set. + Raise value error if malformed value. + Raise CardinalityError if already defined. + """ + if not self.doc_pkgs_started_set: + if len(doc.packages) == "SPDXRef-DOCUMENT": + doc.spdx_id = doc_spdx_id_line + self.doc_pkgs_started_set = True + return True + else: + raise SPDXValueError("Document::SPDXID") + else: + raise CardinalityError("Document::SPDXID") + def reset_document(self): """ Reset the state to allow building new documents @@ -173,6 +189,7 @@ def reset_document(self): self.doc_data_lics_set = False self.doc_name_set = False self.doc_spdx_id_set = False + self.doc_pkgs_started_set = False class ExternalDocumentRefBuilder(object): @@ -1033,26 +1050,31 @@ def __init__(self): def set_file_name(self, doc, name): """ - Raise OrderError if no package defined. + Set the file name. """ if self.has_package(doc): doc.packages[-1].files.append(file.File(name)) - # A file name marks the start of a new file instance. - # The builder must be reset - # FIXME: this state does not make sense - self.reset_file_stat() - return True else: - raise OrderError("File::Name") + # Starting with SPDX 2.0, file entries may proceed any package information section + doc.unpackaged_files.append(file.File(name)) + # raise OrderError("File::Name") #TODO: CLEANUP + # A file name marks the start of a new file instance. + # The builder must be reset + # FIXME: this state does not make sense + self.reset_file_stat() + return True def set_file_spdx_id(self, doc, spdx_id): """ Set the file SPDX Identifier. - Raise OrderError if no package or no file defined. + Raise OrderError if no package/unpackaged file is defined. Raise SPDXValueError if malformed value. Raise CardinalityError if more than one spdx_id set. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if not self.file_spdx_id_set: self.file_spdx_id_set = True if validations.validate_file_spdx_id(spdx_id): @@ -1067,11 +1089,14 @@ def set_file_spdx_id(self, doc, spdx_id): def set_file_comment(self, doc, text): """ - Raise OrderError if no package or no file defined. + Raise OrderError if no package/unpackaged file is defined. Raise CardinalityError if more than one comment set. Raise SPDXValueError if text is not free form text. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if not self.file_comment_set: self.file_comment_set = True if validations.validate_file_comment(text): @@ -1089,7 +1114,10 @@ def set_file_attribution_text(self, doc, text): Set the file's attribution text . Raise SPDXValueError if text is not free form text. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if validations.validate_file_attribution_text(text): self.file(doc).comment = str_from_text(text) return True @@ -1098,7 +1126,7 @@ def set_file_attribution_text(self, doc, text): def set_file_type(self, doc, type_value): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package/unpackaged file is defined. Raise CardinalityError if more than one type set. Raise SPDXValueError if type is unknown. """ @@ -1108,7 +1136,10 @@ def set_file_type(self, doc, type_value): "ARCHIVE": file.FileType.ARCHIVE, "OTHER": file.FileType.OTHER, } - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if not self.file_type_set: self.file_type_set = True if type_value in type_dict.keys(): @@ -1123,10 +1154,13 @@ def set_file_type(self, doc, type_value): def set_file_chksum(self, doc, chksum): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package/unpackaged file is defined. Raise CardinalityError if more than one chksum set. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if not self.file_chksum_set: self.file_chksum_set = True self.file(doc).chk_sum = checksum_from_sha1(chksum) @@ -1138,11 +1172,14 @@ def set_file_chksum(self, doc, chksum): def set_concluded_license(self, doc, lic): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package/unpackaged file is defined. Raise CardinalityError if already set. Raise SPDXValueError if malformed. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if not self.file_conc_lics_set: self.file_conc_lics_set = True if validations.validate_lics_conc(lic): @@ -1157,10 +1194,13 @@ def set_concluded_license(self, doc, lic): def set_file_license_in_file(self, doc, lic): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package/unpackaged file is defined. Raise SPDXValueError if malformed value. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if validations.validate_file_lics_in_file(lic): self.file(doc).add_lics(lic) return True @@ -1171,11 +1211,14 @@ def set_file_license_in_file(self, doc, lic): def set_file_license_comment(self, doc, text): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package/unpackaged file is defined. Raise SPDXValueError if text is not free form text. Raise CardinalityError if more than one per file. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if not self.file_license_comment_set: self.file_license_comment_set = True if validations.validate_file_lics_comment(text): @@ -1189,11 +1232,14 @@ def set_file_license_comment(self, doc, text): def set_file_copyright(self, doc, text): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package/unpackaged file is defined. Raise SPDXValueError if not free form text or NONE or NO_ASSERT. Raise CardinalityError if more than one. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if not self.file_copytext_set: self.file_copytext_set = True if validations.validate_file_cpyright(text): @@ -1211,11 +1257,14 @@ def set_file_copyright(self, doc, text): def set_file_notice(self, doc, text): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package/unpackaged file is defined. Raise SPDXValueError if not free form text. Raise CardinalityError if more than one. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if not self.file_notice_set: self.file_notice_set = True if validations.validate_file_notice(text): @@ -1229,18 +1278,24 @@ def set_file_notice(self, doc, text): def add_file_contribution(self, doc, value): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package/unpackaged file is defined. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): self.file(doc).add_contrib(value) else: raise OrderError("File::Contributor") def add_file_dep(self, doc, value): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package/unpackaged file is defined. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): self.file(doc).add_depend(value) else: raise OrderError("File::Dependency") @@ -1248,17 +1303,23 @@ def add_file_dep(self, doc, value): def set_file_atrificat_of_project(self, doc, symbol, value): """ Set a file name, uri or home artificat. - Raise OrderError if no package or file defined. + Raise OrderError if no package/unpackaged file is defined. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): self.file(doc).add_artifact(symbol, value) else: raise OrderError("File::Artificat") def file(self, doc): """ - Return the last file in the document's package's file list. + Return the last file in document's package's file list (or + unpackaged list if no package has been set). """ + if not self.package_set: + return doc.unpackaged_files[-1] return doc.packages[-1].files[-1] def has_file(self, doc): @@ -1268,6 +1329,12 @@ def has_file(self, doc): """ return len(doc.packages[-1].files) != 0 + def has_unpackaged_file(self, doc): + """ + Return true if the document has at least one file not part of any package. + """ + return len(doc.unpackaged_files) != 0 + def has_package(self, doc): """ Return true if the document has a package. diff --git a/spdx/writers/jsonyamlxml.py b/spdx/writers/jsonyamlxml.py index faac513dd..b65a2584d 100644 --- a/spdx/writers/jsonyamlxml.py +++ b/spdx/writers/jsonyamlxml.py @@ -471,6 +471,9 @@ def create_document(self): self.document_object["SPDXID"] = self.spdx_id(self.document.spdx_id) self.document_object["name"] = self.document.name + if self.document.unpackaged_files: + self.document_object["files"] = self.document.unpackaged_files + package_objects = [] for package in self.document.packages: package_info_object = self.create_package_info(package) diff --git a/tests/test_tag_value_parser.py b/tests/test_tag_value_parser.py index 54dde115e..9c5bdc1e1 100644 --- a/tests/test_tag_value_parser.py +++ b/tests/test_tag_value_parser.py @@ -207,6 +207,17 @@ class TestParser(TestCase): 'ReviewComment: Alice was also here.' ]) + unpackaged_file_str = '\n'.join([ + 'FileName: testfile.text-info', + 'SPDXID: SPDXRef-File', + 'FileType: OTHER', + 'FileChecksum: SHA1: c940141b1f4d098f12812675e9cfa5fe72e07bab', + 'LicenseConcluded: Apache-2.0', + 'LicenseInfoInFile: Apache-2.0', + 'FileCopyrightText: Copyright 2022 Acme Inc.', + 'FileComment: Very long file' + ]) + package_str = '\n'.join([ 'PackageName: Test', 'SPDXID: SPDXRef-Package', @@ -259,7 +270,7 @@ class TestParser(TestCase): 'LicenseInfoInSnippet: Apache-2.0', ]) - complete_str = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}'.format(document_str, creation_str, review_str, package_str, file_str, snippet_str) + complete_str = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}\n{6}'.format(document_str, creation_str, review_str, unpackaged_file_str, package_str, file_str, snippet_str) def setUp(self): self.p = Parser(Builder(), StandardLogger()) @@ -319,6 +330,16 @@ def test_file(self): assert len(spdx_file.artifact_of_project_home) == 1 assert len(spdx_file.artifact_of_project_uri) == 1 + def test_unpackaged_file(self): + document, error = self.p.parse(self.complete_str) + assert document is not None + assert not error + assert len(document.files) == 1 + spdx_file = document.unpackaged_files[0] + assert spdx_file.name == 'testfile.text-info' + assert spdx_file.spdx_id == 'SPDXRef-File' + assert spdx_file.type == spdx.file.FileType.OTHER + def test_unknown_tag(self): try: From 6128dea6009b35cc00cf423da54b0cb7a8ce5fb0 Mon Sep 17 00:00:00 2001 From: bjv Date: Wed, 25 May 2022 15:34:36 -0700 Subject: [PATCH 03/18] nicer unpackaged_file validator var name + docs --- spdx/document.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spdx/document.py b/spdx/document.py index 4440cc9f0..fbaf9dc51 100644 --- a/spdx/document.py +++ b/spdx/document.py @@ -279,6 +279,7 @@ class Document(object): - documentNamespace: SPDX document specific namespace. Mandatory, one. Type: str - creation_info: SPDX file creation info. Mandatory, one. Type: CreationInfo - package: Package described by this document. Mandatory, one. Type: Package + - unpackaged_files: Files not part of any package. Optional zero or more. Type: File. - extracted_licenses: List of licenses extracted that are not part of the SPDX license list. Optional, many. Type: ExtractedLicense. - reviews: SPDX document review information, Optional zero or more. @@ -499,5 +500,5 @@ def validate_extracted_licenses(self, messages): "spdx.document.ExtractedLicense and not " + type(lic) ) def validate_unpackaged_files(self, messages): - for value in self.unpackaged_files: - messages = value.validate(messages) + for spdx_file in self.unpackaged_files: + messages = spdx_file.validate(messages) From 11b45020b11bedb569e33cd46f8f0afb0d2c09e4 Mon Sep 17 00:00:00 2001 From: bjv Date: Wed, 25 May 2022 15:52:46 -0700 Subject: [PATCH 04/18] drop unused flag --- spdx/parsers/tagvaluebuilders.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index 29d8c24d9..371241d4f 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -162,22 +162,6 @@ def set_doc_namespace(self, doc, namespace): else: raise CardinalityError("Document::Comment") - def set_doc_pkgs_started(self, doc): - """ - Indicate Package has been set. - Raise value error if malformed value. - Raise CardinalityError if already defined. - """ - if not self.doc_pkgs_started_set: - if len(doc.packages) == "SPDXRef-DOCUMENT": - doc.spdx_id = doc_spdx_id_line - self.doc_pkgs_started_set = True - return True - else: - raise SPDXValueError("Document::SPDXID") - else: - raise CardinalityError("Document::SPDXID") - def reset_document(self): """ Reset the state to allow building new documents @@ -189,7 +173,6 @@ def reset_document(self): self.doc_data_lics_set = False self.doc_name_set = False self.doc_spdx_id_set = False - self.doc_pkgs_started_set = False class ExternalDocumentRefBuilder(object): From 9072617aace1761830d1e5f656fd319cf106a422 Mon Sep 17 00:00:00 2001 From: bjv Date: Wed, 25 May 2022 15:54:05 -0700 Subject: [PATCH 05/18] cleanup --- spdx/parsers/tagvaluebuilders.py | 1 - 1 file changed, 1 deletion(-) diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index 371241d4f..9447d3097 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -1040,7 +1040,6 @@ def set_file_name(self, doc, name): else: # Starting with SPDX 2.0, file entries may proceed any package information section doc.unpackaged_files.append(file.File(name)) - # raise OrderError("File::Name") #TODO: CLEANUP # A file name marks the start of a new file instance. # The builder must be reset # FIXME: this state does not make sense From 0c633f82bee4b0b122650fae44cef88386fef078 Mon Sep 17 00:00:00 2001 From: bjv Date: Tue, 31 May 2022 14:35:02 -0700 Subject: [PATCH 06/18] Add rdf parser (partial commit) --- .../spdx-expected-unpackaged-files.json | 312 ++++++++++++++++++ .../formats/SPDXRdfUnpackagedFileExample.rdf | 250 ++++++++++++++ tests/test_rdf_parser.py | 9 + tests/test_write_anything.py | 7 +- 4 files changed, 577 insertions(+), 1 deletion(-) create mode 100644 tests/data/doc_parse/spdx-expected-unpackaged-files.json create mode 100644 tests/data/formats/SPDXRdfUnpackagedFileExample.rdf diff --git a/tests/data/doc_parse/spdx-expected-unpackaged-files.json b/tests/data/doc_parse/spdx-expected-unpackaged-files.json new file mode 100644 index 000000000..8829b1227 --- /dev/null +++ b/tests/data/doc_parse/spdx-expected-unpackaged-files.json @@ -0,0 +1,312 @@ +{ + "id": "https://spdx.org/spdxdocs/spdx-example-F16320D4-36AB-476D-9691-59E1BAD31206#SPDXRef-DOCUMENT", + "specVersion": { + "major": 2, + "minor": 1 + }, + "documentNamespace": "https://spdx.org/spdxdocs/spdx-example-F16320D4-36AB-476D-9691-59E1BAD31206", + "name": "Sample_Document-V2.1", + "comment": "This is a sample spreadsheet", + "dataLicense": { + "type": "Single", + "identifier": "CC0-1.0", + "name": "Creative Commons Zero v1.0 Universal" + }, + "licenseListVersion": { + "major": 3, + "minor": 6 + }, + "creators": [ + { + "name": "A. Developer", + "email": null, + "type": "Person" + }, + { + "name": "Sample Group Inc.", + "email": null, + "type": "Organization" + }, + { + "name": "SomeExampleTool-V1.0", + "type": "Tool" + } + ], + "created": "2022-05-31T00:00:00Z", + "creatorComment": "This is an example of an SPDX spreadsheet format", + "files": [ + { + "id": "https://spdx.org/spdxdocs/spdx-example-F16320D4-36AB-476D-9691-59E1BAD31206#SPDXRef-File1", + "name": "file.bin", + "type": 1, + "comment": null, + "licenseConcluded": { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + "copyrightText": "Copyright 2022 Example Client Inc.", + "licenseComment": null, + "notice": null, + "checksum": { + "identifier": "SHA1", + "value": "1b30f385d353620c845594d7ef96b3dd6d17a69a" + }, + "licenseInfoFromFiles": [ + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + } + ], + "contributors": [], + "dependencies": [], + "artifactOfProjectName": [], + "artifactOfProjectHome": [], + "artifactOfProjectURI": [] + } + ], + "packages": [ + { + "id": "SPDXRef-Package", + "name": "SPDX Translator", + "packageFileName": "spdxtranslator-1.0.zip", + "summary": "SPDX Translator utility", + "description": "This utility translates and SPDX RDF XML document to a spreadsheet, translates a spreadsheet to an SPDX RDF XML document and translates an SPDX RDFa document to an SPDX RDF XML document.", + "versionInfo": "Version 0.9.2", + "sourceInfo": "Version 1.0 of the SPDX Translator application", + "downloadLocation": "http://www.spdx.org/tools", + "homepage": null, + "originator": { + "name": "SPDX", + "email": null, + "type": "Organization" + }, + "supplier": { + "name": "Linux Foundation", + "email": null, + "type": "Organization" + }, + "licenseConcluded": { + "type": "Conjunction", + "identifier": [ + "Apache-1.0", + "Apache-2.0", + "LicenseRef-1", + "LicenseRef-2", + "LicenseRef-3", + "LicenseRef-4", + "MPL-1.1" + ], + "name": [ + "Apache License 1.0", + "Apache License 2.0", + "CyberNeko License", + "LicenseRef-1", + "LicenseRef-2", + "LicenseRef-4", + "Mozilla Public License 1.1" + ] + }, + "licenseDeclared": { + "type": "Conjunction", + "identifier": [ + "Apache-2.0", + "LicenseRef-1", + "LicenseRef-2", + "LicenseRef-3", + "LicenseRef-4", + "MPL-1.1" + ], + "name": [ + "Apache License 2.0", + "CyberNeko License", + "LicenseRef-1", + "LicenseRef-2", + "LicenseRef-4", + "Mozilla Public License 1.1" + ] + }, + "copyrightText": " Copyright 2010, 2011 Source Auditor Inc.", + "licenseComment": "The declared license information can be found in the NOTICE file at the root of the archive file", + "checksum": { + "identifier": "SHA1", + "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + }, + "files": [ + { + "id": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-File2", + "name": "src/org/spdx/parser/DOAPProject.java", + "type": 1, + "comment": null, + "licenseConcluded": { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + "copyrightText": "Copyright 2010, 2011 Source Auditor Inc.", + "licenseComment": null, + "notice": null, + "checksum": { + "identifier": "SHA1", + "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + }, + "licenseInfoFromFiles": [ + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + } + ], + "contributors": [], + "dependencies": [], + "artifactOfProjectName": [], + "artifactOfProjectHome": [], + "artifactOfProjectURI": [] + } + ], + "licenseInfoFromFiles": [ + { + "type": "Single", + "identifier": "Apache-1.0", + "name": "Apache License 1.0" + }, + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + { + "type": "Single", + "identifier": "LicenseRef-1", + "name": "LicenseRef-1" + }, + { + "type": "Single", + "identifier": "LicenseRef-2", + "name": "LicenseRef-2" + }, + { + "type": "Single", + "identifier": "LicenseRef-3", + "name": "CyberNeko License" + }, + { + "type": "Single", + "identifier": "LicenseRef-4", + "name": "LicenseRef-4" + }, + { + "type": "Single", + "identifier": "MPL-1.1", + "name": "Mozilla Public License 1.1" + } + ], + "verificationCode": { + "value": "4e3211c67a2d28fced849ee1bb76e7391b93feba", + "excludedFilesNames": [ + "SpdxTranslatorSpdx.rdf", + "SpdxTranslatorSpdx.txt" + ] + } + } + ], + "externalDocumentRefs": [ + { + "externalDocumentId": "DocumentRef-spdx-tool-2.1", + "spdxDocument": "https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301", + "checksum": { + "identifier": "SHA1", + "value": "d6a770ba38583ed4bb4525bd96e50461655d2759" + } + } + ], + "extractedLicenses": [ + { + "name": "LicenseRef-1", + "identifier": "LicenseRef-1", + "text": "/*\n * (c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */", + "comment": null, + "cross_refs": [] + }, + { + "name": "LicenseRef-2", + "identifier": "LicenseRef-2", + "text": "This package includes the GRDDL parser developed by Hewlett Packard under the following license:\n\u00a9 Copyright 2007 Hewlett-Packard Development Company, LP\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: \n\nRedistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. \nRedistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. \nThe name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. \nTHIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ", + "comment": null, + "cross_refs": [] + }, + { + "name": "CyberNeko License", + "identifier": "LicenseRef-3", + "text": "The CyberNeko Software License, Version 1.0\n\n \n(C) Copyright 2002-2005, Andy Clark. All rights reserved.\n \nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions\nare met:\n\n1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer. \n\n2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in\n the documentation and/or other materials provided with the\n distribution.\n\n3. The end-user documentation included with the redistribution,\n if any, must include the following acknowledgment: \n \"This product includes software developed by Andy Clark.\"\n Alternately, this acknowledgment may appear in the software itself,\n if and wherever such third-party acknowledgments normally appear.\n\n4. The names \"CyberNeko\" and \"NekoHTML\" must not be used to endorse\n or promote products derived from this software without prior \n written permission. For written permission, please contact \n andyc@cyberneko.net.\n\n5. Products derived from this software may not be called \"CyberNeko\",\n nor may \"CyberNeko\" appear in their name, without prior written\n permission of the author.\n\nTHIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED\nWARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\nOF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR OTHER CONTRIBUTORS\nBE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, \nOR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT \nOF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR \nBUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, \nWHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE \nOR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, \nEVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", + "comment": "This is tye CyperNeko License", + "cross_refs": [ + "http://justasample.url.com", + "http://people.apache.org/~andyc/neko/LICENSE" + ] + }, + { + "name": "LicenseRef-4", + "identifier": "LicenseRef-4", + "text": "/*\n * (c) Copyright 2009 University of Bristol\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */ ", + "comment": null, + "cross_refs": [] + } + ], + "annotations": [ + { + "id": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-45", + "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", + "type": "REVIEW", + "annotator": { + "name": "Jim Reviewer", + "email": null, + "type": "Person" + }, + "date": "2012-06-13T00:00:00Z" + } + ], + "reviews": [ + { + "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", + "reviewer": { + "name": "Joe Reviewer", + "email": null, + "type": "Person" + }, + "date": "2010-02-10T00:00:00Z" + }, + { + "comment": "Another example reviewer.", + "reviewer": { + "name": "Suzanne Reviewer", + "email": null, + "type": "Person" + }, + "date": "2011-03-13T00:00:00Z" + } + ], + "snippets": [ + { + "id": "SPDXRef-Snippet", + "name": "from linux kernel", + "comment": "This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0-or-later.", + "copyrightText": "Copyright 2008-2010 John Smith", + "licenseComments": "The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz.", + "fileId": "SPDXRef-DoapSource", + "licenseConcluded": { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + "licenseInfoFromSnippet": [ + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + } + ] + } + ] +} diff --git a/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf b/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf new file mode 100644 index 000000000..e6726d648 --- /dev/null +++ b/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf @@ -0,0 +1,250 @@ + + + + + + from linux kernel + Copyright 2008-2010 John Smith + The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz. + This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0-or-later. + + + + + Sample_Document-V2.1 + + + 2022-05-31T00:00:00Z + This is an example of an SPDX spreadsheet format + Tool: SomeExampleTool-V1.0 + Organization: Sample Group Inc. + Person: A. Developer + + + SPDX-2.1 + + + DocumentRef-spdx-tool-2.1 + + + + d6a770ba38583ed4bb4525bd96e50461655d2759 + + + + + + + + Copyright 2022 Example Client Inc. + + + + + + 1b30f385d353620c845594d7ef96b3dd6d17a69a + + + + file.bin + + + + + This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses + 2010-02-10T00:00:00Z + Person: Joe Reviewer + + + + + + + Another example reviewer. + 2011-03-13T00:00:00Z + Person: Suzanne Reviewer + + + + + + This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses + 2012-06-13T00:00:00Z + Person: Jim Reviewer + + + + + + + + + + + + http://www.spdx.org/tools + true + Organization:Linux Foundation + + + + + 4e3211c67a2d28fced849ee1bb76e7391b93feba + SpdxTranslatorSpdx.txt + SpdxTranslatorSpdx.rdf + + + + + + + This package includes the GRDDL parser developed by Hewlett Packard under the following license: +© Copyright 2007 Hewlett-Packard Development Company, LP + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + LicenseRef-2 + + + + + + + + /* + * (c) Copyright 2009 University of Bristol + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + LicenseRef-4 + + + + + http://justasample.url.com + http://people.apache.org/~andyc/neko/LICENSE + CyberNeko License + This is tye CyperNeko License + The CyberNeko Software License, Version 1.0 + + +(C) Copyright 2002-2005, Andy Clark. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +3. The end-user documentation included with the redistribution, + if any, must include the following acknowledgment: + "This product includes software developed by Andy Clark." + Alternately, this acknowledgment may appear in the software itself, + if and wherever such third-party acknowledgments normally appear. + +4. The names "CyberNeko" and "NekoHTML" must not be used to endorse + or promote products derived from this software without prior + written permission. For written permission, please contact + andyc@cyberneko.net. + +5. Products derived from this software may not be called "CyberNeko", + nor may "CyberNeko" appear in their name, without prior written + permission of the author. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR OTHER CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, +OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + LicenseRef-3 + + + + + Version 1.0 of the SPDX Translator application + + + 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 + + + + spdxtranslator-1.0.zip + This utility translates and SPDX RDF XML document to a spreadsheet, translates a spreadsheet to an SPDX RDF XML document and translates an SPDX RDFa document to an SPDX RDF XML document. + + SPDX Translator + Version 0.9.2 + + + + Copyright 2010, 2011 Source Auditor Inc. + + + + + + + + + + + Organization:SPDX + The declared license information can be found in the NOTICE file at the root of the archive file + SPDX Translator utility + + + + + + org.apache.commons:commons-lang:3.2.1 + NIST National Vulnerability Database (NVD) describes security vulnerabilities (CVEs) which affect Vendor Product Version acmecorp:acmenator:6.6.6 + + + + + This is a sample spreadsheet + + + + + diff --git a/tests/test_rdf_parser.py b/tests/test_rdf_parser.py index 8f52f58b7..13800fa4b 100644 --- a/tests/test_rdf_parser.py +++ b/tests/test_rdf_parser.py @@ -45,3 +45,12 @@ def check_document(self, document, expected_loc, regen=False): expected = json.load(ex) assert result == expected + + def test_rdf_parser_unpacked_files(self, regen=False): + parser = rdf.Parser(RDFBuilder(), StandardLogger()) + test_file = utils_test.get_test_loc('formats/SPDXRdfUnpackagedFileExample.rdf', test_data_dir=utils_test.test_data_dir) + with io.open(test_file, 'rb') as f: + document, _ = parser.parse(f) + expected_loc = utils_test.get_test_loc('doc_parse/spdx-expected-unpackaged-files.json', test_data_dir=utils_test.test_data_dir) + self.check_document(document, expected_loc, regen=regen) + diff --git a/tests/test_write_anything.py b/tests/test_write_anything.py index 9dceaffcb..a997c88f2 100644 --- a/tests/test_write_anything.py +++ b/tests/test_write_anything.py @@ -35,7 +35,12 @@ "SPDXRdfExample.rdf-yaml", "SPDXRdfExample.rdf-xml", "SPDXRdfExample.rdf-json", - "SPDXRdfExample.rdf-tag" + "SPDXRdfExample.rdf-tag", + "SPDXRdfUnpackagedFileExample.rdf-rdf", + "SPDXRdfUnpackagedFileExample.rdf-yaml", + "SPDXRdfUnpackagedFileExample.rdf-xml", + "SPDXRdfUnpackagedFileExample.rdf-json", + "SPDXRdfUnpackagedFileExample.rdf-tag" } @pytest.mark.parametrize("out_format", ['rdf', 'yaml', 'xml', 'json', 'tag']) From 83d7b89e3b384398c8f5c73c3aeace0b50c66372 Mon Sep 17 00:00:00 2001 From: bjv Date: Thu, 2 Jun 2022 15:49:42 -0700 Subject: [PATCH 07/18] Cleaned up rdf test --- .../spdx-expected-unpackaged-files.json | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/tests/data/doc_parse/spdx-expected-unpackaged-files.json b/tests/data/doc_parse/spdx-expected-unpackaged-files.json index 8829b1227..de5f1d190 100644 --- a/tests/data/doc_parse/spdx-expected-unpackaged-files.json +++ b/tests/data/doc_parse/spdx-expected-unpackaged-files.json @@ -92,7 +92,6 @@ "identifier": [ "Apache-1.0", "Apache-2.0", - "LicenseRef-1", "LicenseRef-2", "LicenseRef-3", "LicenseRef-4", @@ -102,7 +101,6 @@ "Apache License 1.0", "Apache License 2.0", "CyberNeko License", - "LicenseRef-1", "LicenseRef-2", "LicenseRef-4", "Mozilla Public License 1.1" @@ -112,7 +110,6 @@ "type": "Conjunction", "identifier": [ "Apache-2.0", - "LicenseRef-1", "LicenseRef-2", "LicenseRef-3", "LicenseRef-4", @@ -121,7 +118,6 @@ "name": [ "Apache License 2.0", "CyberNeko License", - "LicenseRef-1", "LicenseRef-2", "LicenseRef-4", "Mozilla Public License 1.1" @@ -135,8 +131,8 @@ }, "files": [ { - "id": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-File2", - "name": "src/org/spdx/parser/DOAPProject.java", + "id": "https://spdx.org/spdxdocs/spdx-example-F16320D4-36AB-476D-9691-59E1BAD31206#SPDXRef-File1", + "name": "file.bin", "type": 1, "comment": null, "licenseConcluded": { @@ -144,12 +140,12 @@ "identifier": "Apache-2.0", "name": "Apache License 2.0" }, - "copyrightText": "Copyright 2010, 2011 Source Auditor Inc.", + "copyrightText": "Copyright 2022 Example Client Inc.", "licenseComment": null, "notice": null, "checksum": { "identifier": "SHA1", - "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + "value": "1b30f385d353620c845594d7ef96b3dd6d17a69a" }, "licenseInfoFromFiles": [ { @@ -176,11 +172,6 @@ "identifier": "Apache-2.0", "name": "Apache License 2.0" }, - { - "type": "Single", - "identifier": "LicenseRef-1", - "name": "LicenseRef-1" - }, { "type": "Single", "identifier": "LicenseRef-2", @@ -222,13 +213,6 @@ } ], "extractedLicenses": [ - { - "name": "LicenseRef-1", - "identifier": "LicenseRef-1", - "text": "/*\n * (c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */", - "comment": null, - "cross_refs": [] - }, { "name": "LicenseRef-2", "identifier": "LicenseRef-2", From a2c506b446f2dde7116c4cee8bf79f5342312174 Mon Sep 17 00:00:00 2001 From: bjv Date: Thu, 2 Jun 2022 16:07:08 -0700 Subject: [PATCH 08/18] Render unpackaged files, for test --- tests/data/doc_parse/SBOMexpected.json | 1 + tests/data/doc_parse/expected.json | 1 + tests/data/doc_parse/spdx-expected.json | 1 + tests/utils_test.py | 1 + 4 files changed, 4 insertions(+) diff --git a/tests/data/doc_parse/SBOMexpected.json b/tests/data/doc_parse/SBOMexpected.json index 848d19ab0..436ff7815 100644 --- a/tests/data/doc_parse/SBOMexpected.json +++ b/tests/data/doc_parse/SBOMexpected.json @@ -30,6 +30,7 @@ ], "created": "2020-07-23T18:30:22Z", "creatorComment": null, + "files": [], "packages": [ { "id": "SPDXRef-Package-xyz", diff --git a/tests/data/doc_parse/expected.json b/tests/data/doc_parse/expected.json index 346de7167..58d6a1342 100644 --- a/tests/data/doc_parse/expected.json +++ b/tests/data/doc_parse/expected.json @@ -34,6 +34,7 @@ ], "created": "2010-02-03T00:00:00Z", "creatorComment": "This is an example of an SPDX spreadsheet format", + "files: [], "packages": [ { "id": "SPDXRef-Package", diff --git a/tests/data/doc_parse/spdx-expected.json b/tests/data/doc_parse/spdx-expected.json index 3514bfcc6..d57354f1d 100644 --- a/tests/data/doc_parse/spdx-expected.json +++ b/tests/data/doc_parse/spdx-expected.json @@ -34,6 +34,7 @@ ], "created": "2010-02-03T00:00:00Z", "creatorComment": "This is an example of an SPDX spreadsheet format", + "files": [], "packages": [ { "id": "SPDXRef-Package", diff --git a/tests/utils_test.py b/tests/utils_test.py index 92840f108..6e7ced09d 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -510,6 +510,7 @@ def to_dict(cls, doc): ('creators', [cls.entity_to_dict(creator) for creator in creators]), ('created', utils.datetime_iso_format(doc.creation_info.created)), ('creatorComment', doc.creation_info.comment), + ('files', cls.files_to_list(doc.unpackaged_files)), ('packages', [cls.package_to_dict(p) for p in doc.packages]), ('externalDocumentRefs', cls.ext_document_references_to_list(sorted(doc.ext_document_references))), ('extractedLicenses', cls.extracted_licenses_to_list(sorted(doc.extracted_licenses))), From 6799c308370af75fe23b3740cfe67bd14858bd2a Mon Sep 17 00:00:00 2001 From: bjv Date: Thu, 2 Jun 2022 16:43:50 -0700 Subject: [PATCH 09/18] Cleanup hanging licenseref --- tests/data/formats/SPDXRdfUnpackagedFileExample.rdf | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf b/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf index e6726d648..31f5b16ae 100644 --- a/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf +++ b/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf @@ -214,7 +214,6 @@ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SPDX Translator Version 0.9.2 - Copyright 2010, 2011 Source Auditor Inc. From 773ecad7f505403bd206407122198b25d71679d9 Mon Sep 17 00:00:00 2001 From: bjv Date: Thu, 2 Jun 2022 16:44:15 -0700 Subject: [PATCH 10/18] Use filetype 'other' for test --- tests/data/doc_parse/spdx-expected-unpackaged-files.json | 2 +- tests/data/formats/SPDXRdfUnpackagedFileExample.rdf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/doc_parse/spdx-expected-unpackaged-files.json b/tests/data/doc_parse/spdx-expected-unpackaged-files.json index de5f1d190..537753221 100644 --- a/tests/data/doc_parse/spdx-expected-unpackaged-files.json +++ b/tests/data/doc_parse/spdx-expected-unpackaged-files.json @@ -133,7 +133,7 @@ { "id": "https://spdx.org/spdxdocs/spdx-example-F16320D4-36AB-476D-9691-59E1BAD31206#SPDXRef-File1", "name": "file.bin", - "type": 1, + "type": 4, "comment": null, "licenseConcluded": { "type": "Single", diff --git a/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf b/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf index 31f5b16ae..8e1ec3d33 100644 --- a/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf +++ b/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf @@ -43,7 +43,7 @@ Copyright 2022 Example Client Inc. - + 1b30f385d353620c845594d7ef96b3dd6d17a69a From ca3588fb9c0b4174b1b784c0d76b131c05ae258f Mon Sep 17 00:00:00 2001 From: bjv Date: Fri, 3 Jun 2022 15:17:13 -0700 Subject: [PATCH 11/18] (Breaks many tests!) Add unpackaged RDF referencesFile to document unpackaged_files Test assume builder always adds File to the last RDF Package? Seems wrong --- spdx/parsers/rdf.py | 9 ++++- spdx/parsers/rdfbuilders.py | 32 +++++++++++++---- .../spdx-expected-unpackaged-files.json | 35 ++----------------- 3 files changed, 35 insertions(+), 41 deletions(-) diff --git a/spdx/parsers/rdf.py b/spdx/parsers/rdf.py index c014c3d85..43da32b50 100644 --- a/spdx/parsers/rdf.py +++ b/spdx/parsers/rdf.py @@ -17,6 +17,7 @@ from rdflib import Namespace from rdflib import RDF from rdflib import RDFS +from rdflib.term import URIRef from spdx import document from spdx import utils @@ -1299,6 +1300,11 @@ def parse(self, fil): ): self.handle_extracted_license(s) + for s, _p, o in self.graph.triples( + (URIRef(self.doc.spdx_id), self.spdx_namespace["referencesFile"], None) + ): + self.parse_file(o) # Document-level ref file (unpackaged) + for s, _p, o in self.graph.triples( (None, RDF.type, self.spdx_namespace["Package"]) ): @@ -1312,7 +1318,8 @@ def parse(self, fil): for s, _p, o in self.graph.triples( (None, self.spdx_namespace["referencesFile"], None) ): - self.parse_file(o) + if not str(s) == self.doc.spdx_id: + self.parse_file(o) # Package child ref file for s, _p, o in self.graph.triples( (None, RDF.type, self.spdx_namespace["Snippet"]) diff --git a/spdx/parsers/rdfbuilders.py b/spdx/parsers/rdfbuilders.py index 3affb7329..e6f5383fd 100644 --- a/spdx/parsers/rdfbuilders.py +++ b/spdx/parsers/rdfbuilders.py @@ -386,9 +386,12 @@ def set_file_chksum(self, doc, chk_sum): Set the file check sum, if not already set. chk_sum - A string Raise CardinalityError if already defined. - Raise OrderError if no package previously defined. + Raise OrderError if no package/unpackaged file previously defined. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if not self.file_chksum_set: self.file_chksum_set = True self.file(doc).chk_sum = checksum.Algorithm("SHA1", chk_sum) @@ -403,7 +406,10 @@ def set_file_license_comment(self, doc, text): Raise OrderError if no package or file defined. Raise CardinalityError if more than one per file. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if not self.file_license_comment_set: self.file_license_comment_set = True self.file(doc).license_comment = text @@ -417,7 +423,10 @@ def set_file_attribution_text(self, doc, text): """ Set the file's attribution text. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): self.assert_package_exists() self.file(doc).attribution_text = text return True @@ -427,7 +436,10 @@ def set_file_copyright(self, doc, text): Raise OrderError if no package or file defined. Raise CardinalityError if more than one. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if not self.file_copytext_set: self.file_copytext_set = True self.file(doc).copyright = text @@ -442,7 +454,10 @@ def set_file_comment(self, doc, text): Raise OrderError if no package or no file defined. Raise CardinalityError if more than one comment set. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if not self.file_comment_set: self.file_comment_set = True self.file(doc).comment = text @@ -457,7 +472,10 @@ def set_file_notice(self, doc, text): Raise OrderError if no package or file defined. Raise CardinalityError if more than one. """ - if self.has_package(doc) and self.has_file(doc): + if ( + (self.has_package(doc) and self.has_file(doc)) + or (self.has_unpackaged_file(doc) and not self.package_set) + ): if not self.file_notice_set: self.file_notice_set = True self.file(doc).notice = tagvaluebuilders.str_from_text(text) diff --git a/tests/data/doc_parse/spdx-expected-unpackaged-files.json b/tests/data/doc_parse/spdx-expected-unpackaged-files.json index 537753221..6297ff0e6 100644 --- a/tests/data/doc_parse/spdx-expected-unpackaged-files.json +++ b/tests/data/doc_parse/spdx-expected-unpackaged-files.json @@ -38,7 +38,7 @@ { "id": "https://spdx.org/spdxdocs/spdx-example-F16320D4-36AB-476D-9691-59E1BAD31206#SPDXRef-File1", "name": "file.bin", - "type": 1, + "type": 4, "comment": null, "licenseConcluded": { "type": "Single", @@ -129,38 +129,7 @@ "identifier": "SHA1", "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" }, - "files": [ - { - "id": "https://spdx.org/spdxdocs/spdx-example-F16320D4-36AB-476D-9691-59E1BAD31206#SPDXRef-File1", - "name": "file.bin", - "type": 4, - "comment": null, - "licenseConcluded": { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" - }, - "copyrightText": "Copyright 2022 Example Client Inc.", - "licenseComment": null, - "notice": null, - "checksum": { - "identifier": "SHA1", - "value": "1b30f385d353620c845594d7ef96b3dd6d17a69a" - }, - "licenseInfoFromFiles": [ - { - "type": "Single", - "identifier": "Apache-2.0", - "name": "Apache License 2.0" - } - ], - "contributors": [], - "dependencies": [], - "artifactOfProjectName": [], - "artifactOfProjectHome": [], - "artifactOfProjectURI": [] - } - ], + "files": [], "licenseInfoFromFiles": [ { "type": "Single", From 850d525afb10e483292d6234787e0f82de8581f1 Mon Sep 17 00:00:00 2001 From: bjv Date: Wed, 8 Jun 2022 10:10:05 -0700 Subject: [PATCH 12/18] parse rdf package hasFile refs, to distribute doc referencesFile among packages --- spdx/parsers/rdf.py | 11 ++++++++++- spdx/writers/jsonyamlxml.py | 10 ++++++---- tests/data/doc_parse/expected.json | 2 +- tests/data/formats/SPDXRdfExample.rdf | 4 ++-- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/spdx/parsers/rdf.py b/spdx/parsers/rdf.py index 43da32b50..17490d176 100644 --- a/spdx/parsers/rdf.py +++ b/spdx/parsers/rdf.py @@ -366,6 +366,7 @@ def parse_package(self, p_term): self.p_pkg_summary(p_term, self.spdx_namespace["summary"]) self.p_pkg_descr(p_term, self.spdx_namespace["description"]) self.p_pkg_comment(p_term, self.spdx_namespace["comment"]) + self.p_pkg_has_file(p_term, self.spdx_namespace["hasFile"]) def p_pkg_cr_text(self, p_term, predicate): try: @@ -397,6 +398,13 @@ def p_pkg_comment(self, p_term, predicate): except CardinalityError: self.more_than_one_error("package comment") + def p_pkg_has_file(self, p_term, predicate): + for _, _, has_file in self.graph.triples((p_term, predicate, None)): + for index, spdx_file in enumerate(self.doc.unpackaged_files): + if str(has_file) == spdx_file.spdx_id: + self.doc.packages[-1].files.append(spdx_file) + del self.doc.unpackaged_files[index] + def p_pkg_attribution_text(self, p_term, predicate): try: for _, _, attribute_text in self.graph.triples((p_term, predicate, None)): @@ -1319,7 +1327,8 @@ def parse(self, fil): (None, self.spdx_namespace["referencesFile"], None) ): if not str(s) == self.doc.spdx_id: - self.parse_file(o) # Package child ref file + # self.parse_file(o) # TODO: Package child ref file? + raise NotImplementedError(f"Processing {s} referencesFile") for s, _p, o in self.graph.triples( (None, RDF.type, self.spdx_namespace["Snippet"]) diff --git a/spdx/writers/jsonyamlxml.py b/spdx/writers/jsonyamlxml.py index b65a2584d..7bc49b31a 100644 --- a/spdx/writers/jsonyamlxml.py +++ b/spdx/writers/jsonyamlxml.py @@ -182,7 +182,10 @@ def create_artifact_info(self, file): return artifact_of_objects - def create_file_info(self, package): + def create_pkg_file_info(self, package): + return self.create_file_info(package.files) + + def create_file_info(self, files): file_types = { 1: "fileType_source", 2: "fileType_binary", @@ -190,7 +193,6 @@ def create_file_info(self, package): 4: "fileType_other", } file_objects = [] - files = package.files for file in files: file_object = dict() @@ -472,12 +474,12 @@ def create_document(self): self.document_object["name"] = self.document.name if self.document.unpackaged_files: - self.document_object["files"] = self.document.unpackaged_files + self.document_object["files"] = self.create_file_info(self.document.unpackaged_files) package_objects = [] for package in self.document.packages: package_info_object = self.create_package_info(package) - package_info_object["files"] = self.create_file_info(package) + package_info_object["files"] = self.create_pkg_file_info(package) package_objects.append({"Package": package_info_object}) self.document_object["documentDescribes"] = package_objects diff --git a/tests/data/doc_parse/expected.json b/tests/data/doc_parse/expected.json index 58d6a1342..c09768a87 100644 --- a/tests/data/doc_parse/expected.json +++ b/tests/data/doc_parse/expected.json @@ -34,7 +34,7 @@ ], "created": "2010-02-03T00:00:00Z", "creatorComment": "This is an example of an SPDX spreadsheet format", - "files: [], + "files": [], "packages": [ { "id": "SPDXRef-Package", diff --git a/tests/data/formats/SPDXRdfExample.rdf b/tests/data/formats/SPDXRdfExample.rdf index 4637b4ca9..3049188bd 100644 --- a/tests/data/formats/SPDXRdfExample.rdf +++ b/tests/data/formats/SPDXRdfExample.rdf @@ -39,7 +39,7 @@ - > + /* @@ -142,7 +142,7 @@ http://www.spdx.org/tools true Organization:Linux Foundation - + From f5050b2a362bf40f51d579083d17beea33034ffa Mon Sep 17 00:00:00 2001 From: bjv Date: Mon, 13 Jun 2022 07:54:02 -0700 Subject: [PATCH 13/18] fix more test RDF data --- tests/data/formats/SPDXRdfExample.rdf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/formats/SPDXRdfExample.rdf b/tests/data/formats/SPDXRdfExample.rdf index 3049188bd..4d5a778d2 100644 --- a/tests/data/formats/SPDXRdfExample.rdf +++ b/tests/data/formats/SPDXRdfExample.rdf @@ -269,7 +269,7 @@ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SPDX Translator Version 0.9.2 - + Copyright 2010, 2011 Source Auditor Inc. From a7a5e6e51621c98d5d961dea2f2b59ae4b7ca84c Mon Sep 17 00:00:00 2001 From: bjv Date: Mon, 13 Jun 2022 08:20:30 -0700 Subject: [PATCH 14/18] work around https://github.com/spdx/tools-python/issues/149 --- .../spdx-expected-unpackaged-files.json | 33 ++++++++++++++++++- .../formats/SPDXRdfUnpackagedFileExample.rdf | 18 ++++++++-- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/tests/data/doc_parse/spdx-expected-unpackaged-files.json b/tests/data/doc_parse/spdx-expected-unpackaged-files.json index 6297ff0e6..f47714a67 100644 --- a/tests/data/doc_parse/spdx-expected-unpackaged-files.json +++ b/tests/data/doc_parse/spdx-expected-unpackaged-files.json @@ -129,7 +129,38 @@ "identifier": "SHA1", "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" }, - "files": [], + "files": [ + { + "id": "https://spdx.org/spdxdocs/spdx-example-F16320D4-36AB-476D-9691-59E1BAD31206#SPDXRef-File2", + "name": "etc/message.dat", + "type": 4, + "comment": null, + "licenseConcluded": { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + "copyrightText": "Copyright 2022 Example Client Inc.", + "licenseComment": null, + "notice": null, + "checksum": { + "identifier": "SHA1", + "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + }, + "licenseInfoFromFiles": [ + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + } + ], + "contributors": [], + "dependencies": [], + "artifactOfProjectName": [], + "artifactOfProjectHome": [], + "artifactOfProjectURI": [] + } + ], "licenseInfoFromFiles": [ { "type": "Single", diff --git a/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf b/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf index 8e1ec3d33..d5443ddfc 100644 --- a/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf +++ b/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf @@ -83,13 +83,28 @@ + + + Copyright 2022 Example Client Inc. + + + + + + 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 + + + + etc/message.dat + + http://www.spdx.org/tools true Organization:Linux Foundation - + @@ -214,7 +229,6 @@ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SPDX Translator Version 0.9.2 - Copyright 2010, 2011 Source Auditor Inc. From b65966a4fc8b5cd1e5d4e2226ef83e25fa14addb Mon Sep 17 00:00:00 2001 From: bjv Date: Mon, 13 Jun 2022 09:12:28 -0700 Subject: [PATCH 15/18] DRY out parser builders, a bit --- spdx/parsers/rdfbuilders.py | 30 +++---------- spdx/parsers/tagvaluebuilders.py | 75 ++++++++++---------------------- 2 files changed, 29 insertions(+), 76 deletions(-) diff --git a/spdx/parsers/rdfbuilders.py b/spdx/parsers/rdfbuilders.py index e6f5383fd..a49b8d933 100644 --- a/spdx/parsers/rdfbuilders.py +++ b/spdx/parsers/rdfbuilders.py @@ -388,10 +388,7 @@ def set_file_chksum(self, doc, chk_sum): Raise CardinalityError if already defined. Raise OrderError if no package/unpackaged file previously defined. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if not self.file_chksum_set: self.file_chksum_set = True self.file(doc).chk_sum = checksum.Algorithm("SHA1", chk_sum) @@ -406,10 +403,7 @@ def set_file_license_comment(self, doc, text): Raise OrderError if no package or file defined. Raise CardinalityError if more than one per file. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if not self.file_license_comment_set: self.file_license_comment_set = True self.file(doc).license_comment = text @@ -423,10 +417,7 @@ def set_file_attribution_text(self, doc, text): """ Set the file's attribution text. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): self.assert_package_exists() self.file(doc).attribution_text = text return True @@ -436,10 +427,7 @@ def set_file_copyright(self, doc, text): Raise OrderError if no package or file defined. Raise CardinalityError if more than one. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if not self.file_copytext_set: self.file_copytext_set = True self.file(doc).copyright = text @@ -454,10 +442,7 @@ def set_file_comment(self, doc, text): Raise OrderError if no package or no file defined. Raise CardinalityError if more than one comment set. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if not self.file_comment_set: self.file_comment_set = True self.file(doc).comment = text @@ -472,10 +457,7 @@ def set_file_notice(self, doc, text): Raise OrderError if no package or file defined. Raise CardinalityError if more than one. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if not self.file_notice_set: self.file_notice_set = True self.file(doc).notice = tagvaluebuilders.str_from_text(text) diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index 9447d3097..2f59a2af0 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -1031,6 +1031,16 @@ def __init__(self): # FIXME: this state does not make sense self.reset_file_stat() + def _build_file_valid(self, doc): + """ + Return true if building a File for referenced document is valid. + """ + if self.has_package(doc) and self.has_file(doc): + return True + if not self.package_set and self.has_unpackaged_file(doc): + return True + return False + def set_file_name(self, doc, name): """ Set the file name. @@ -1053,10 +1063,7 @@ def set_file_spdx_id(self, doc, spdx_id): Raise SPDXValueError if malformed value. Raise CardinalityError if more than one spdx_id set. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if not self.file_spdx_id_set: self.file_spdx_id_set = True if validations.validate_file_spdx_id(spdx_id): @@ -1075,10 +1082,7 @@ def set_file_comment(self, doc, text): Raise CardinalityError if more than one comment set. Raise SPDXValueError if text is not free form text. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if not self.file_comment_set: self.file_comment_set = True if validations.validate_file_comment(text): @@ -1096,10 +1100,7 @@ def set_file_attribution_text(self, doc, text): Set the file's attribution text . Raise SPDXValueError if text is not free form text. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if validations.validate_file_attribution_text(text): self.file(doc).comment = str_from_text(text) return True @@ -1118,10 +1119,7 @@ def set_file_type(self, doc, type_value): "ARCHIVE": file.FileType.ARCHIVE, "OTHER": file.FileType.OTHER, } - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if not self.file_type_set: self.file_type_set = True if type_value in type_dict.keys(): @@ -1139,10 +1137,7 @@ def set_file_chksum(self, doc, chksum): Raise OrderError if no package/unpackaged file is defined. Raise CardinalityError if more than one chksum set. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if not self.file_chksum_set: self.file_chksum_set = True self.file(doc).chk_sum = checksum_from_sha1(chksum) @@ -1158,10 +1153,7 @@ def set_concluded_license(self, doc, lic): Raise CardinalityError if already set. Raise SPDXValueError if malformed. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if not self.file_conc_lics_set: self.file_conc_lics_set = True if validations.validate_lics_conc(lic): @@ -1179,10 +1171,7 @@ def set_file_license_in_file(self, doc, lic): Raise OrderError if no package/unpackaged file is defined. Raise SPDXValueError if malformed value. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if validations.validate_file_lics_in_file(lic): self.file(doc).add_lics(lic) return True @@ -1197,10 +1186,7 @@ def set_file_license_comment(self, doc, text): Raise SPDXValueError if text is not free form text. Raise CardinalityError if more than one per file. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if not self.file_license_comment_set: self.file_license_comment_set = True if validations.validate_file_lics_comment(text): @@ -1218,10 +1204,7 @@ def set_file_copyright(self, doc, text): Raise SPDXValueError if not free form text or NONE or NO_ASSERT. Raise CardinalityError if more than one. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if not self.file_copytext_set: self.file_copytext_set = True if validations.validate_file_cpyright(text): @@ -1243,10 +1226,7 @@ def set_file_notice(self, doc, text): Raise SPDXValueError if not free form text. Raise CardinalityError if more than one. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): if not self.file_notice_set: self.file_notice_set = True if validations.validate_file_notice(text): @@ -1262,10 +1242,7 @@ def add_file_contribution(self, doc, value): """ Raise OrderError if no package/unpackaged file is defined. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): self.file(doc).add_contrib(value) else: raise OrderError("File::Contributor") @@ -1274,10 +1251,7 @@ def add_file_dep(self, doc, value): """ Raise OrderError if no package/unpackaged file is defined. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): self.file(doc).add_depend(value) else: raise OrderError("File::Dependency") @@ -1287,10 +1261,7 @@ def set_file_atrificat_of_project(self, doc, symbol, value): Set a file name, uri or home artificat. Raise OrderError if no package/unpackaged file is defined. """ - if ( - (self.has_package(doc) and self.has_file(doc)) - or (self.has_unpackaged_file(doc) and not self.package_set) - ): + if (self._build_file_valid(doc)): self.file(doc).add_artifact(symbol, value) else: raise OrderError("File::Artificat") From 2794f0f8a82fa119f25a0fb0565c3e7fbad0cd71 Mon Sep 17 00:00:00 2001 From: bjv Date: Mon, 13 Jun 2022 09:18:04 -0700 Subject: [PATCH 16/18] add unpackaged Files tag/value separator --- spdx/writers/tagvalue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index d37549442..092754ac7 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -361,6 +361,7 @@ def write_document(document, out, validate=True): # Write out unpackaged file info for spdx_file in document.unpackaged_files: write_file(spdx_file, out) + write_separators(out) # Write out package info for package in document.packages: From 441301e6aeb60dcc3f1d25ca23bd6b49cb3d15bf Mon Sep 17 00:00:00 2001 From: bjv Date: Mon, 13 Jun 2022 13:18:13 -0700 Subject: [PATCH 17/18] docs & whitespace cleanup --- spdx/document.py | 1 + spdx/parsers/rdfbuilders.py | 2 +- spdx/parsers/tagvaluebuilders.py | 24 ++++++++++++------------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/spdx/document.py b/spdx/document.py index fbaf9dc51..1396b1732 100644 --- a/spdx/document.py +++ b/spdx/document.py @@ -499,6 +499,7 @@ def validate_extracted_licenses(self, messages): "Document extracted licenses must be of type " "spdx.document.ExtractedLicense and not " + type(lic) ) + def validate_unpackaged_files(self, messages): for spdx_file in self.unpackaged_files: messages = spdx_file.validate(messages) diff --git a/spdx/parsers/rdfbuilders.py b/spdx/parsers/rdfbuilders.py index a49b8d933..7f3356fc2 100644 --- a/spdx/parsers/rdfbuilders.py +++ b/spdx/parsers/rdfbuilders.py @@ -386,7 +386,7 @@ def set_file_chksum(self, doc, chk_sum): Set the file check sum, if not already set. chk_sum - A string Raise CardinalityError if already defined. - Raise OrderError if no package/unpackaged file previously defined. + Raise OrderError if no package, package file, or unpackaged file is defined. """ if (self._build_file_valid(doc)): if not self.file_chksum_set: diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index 2f59a2af0..7c1ceccde 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -1059,7 +1059,7 @@ def set_file_name(self, doc, name): def set_file_spdx_id(self, doc, spdx_id): """ Set the file SPDX Identifier. - Raise OrderError if no package/unpackaged file is defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise SPDXValueError if malformed value. Raise CardinalityError if more than one spdx_id set. """ @@ -1078,7 +1078,7 @@ def set_file_spdx_id(self, doc, spdx_id): def set_file_comment(self, doc, text): """ - Raise OrderError if no package/unpackaged file is defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise CardinalityError if more than one comment set. Raise SPDXValueError if text is not free form text. """ @@ -1109,7 +1109,7 @@ def set_file_attribution_text(self, doc, text): def set_file_type(self, doc, type_value): """ - Raise OrderError if no package/unpackaged file is defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise CardinalityError if more than one type set. Raise SPDXValueError if type is unknown. """ @@ -1134,7 +1134,7 @@ def set_file_type(self, doc, type_value): def set_file_chksum(self, doc, chksum): """ - Raise OrderError if no package/unpackaged file is defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise CardinalityError if more than one chksum set. """ if (self._build_file_valid(doc)): @@ -1149,7 +1149,7 @@ def set_file_chksum(self, doc, chksum): def set_concluded_license(self, doc, lic): """ - Raise OrderError if no package/unpackaged file is defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise CardinalityError if already set. Raise SPDXValueError if malformed. """ @@ -1168,7 +1168,7 @@ def set_concluded_license(self, doc, lic): def set_file_license_in_file(self, doc, lic): """ - Raise OrderError if no package/unpackaged file is defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise SPDXValueError if malformed value. """ if (self._build_file_valid(doc)): @@ -1182,7 +1182,7 @@ def set_file_license_in_file(self, doc, lic): def set_file_license_comment(self, doc, text): """ - Raise OrderError if no package/unpackaged file is defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise SPDXValueError if text is not free form text. Raise CardinalityError if more than one per file. """ @@ -1200,7 +1200,7 @@ def set_file_license_comment(self, doc, text): def set_file_copyright(self, doc, text): """ - Raise OrderError if no package/unpackaged file is defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise SPDXValueError if not free form text or NONE or NO_ASSERT. Raise CardinalityError if more than one. """ @@ -1222,7 +1222,7 @@ def set_file_copyright(self, doc, text): def set_file_notice(self, doc, text): """ - Raise OrderError if no package/unpackaged file is defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise SPDXValueError if not free form text. Raise CardinalityError if more than one. """ @@ -1240,7 +1240,7 @@ def set_file_notice(self, doc, text): def add_file_contribution(self, doc, value): """ - Raise OrderError if no package/unpackaged file is defined. + Raise OrderError if no package, package file, or unpackaged file is defined. """ if (self._build_file_valid(doc)): self.file(doc).add_contrib(value) @@ -1249,7 +1249,7 @@ def add_file_contribution(self, doc, value): def add_file_dep(self, doc, value): """ - Raise OrderError if no package/unpackaged file is defined. + Raise OrderError if no package, package file, or unpackaged file is defined. """ if (self._build_file_valid(doc)): self.file(doc).add_depend(value) @@ -1259,7 +1259,7 @@ def add_file_dep(self, doc, value): def set_file_atrificat_of_project(self, doc, symbol, value): """ Set a file name, uri or home artificat. - Raise OrderError if no package/unpackaged file is defined. + Raise OrderError if no package, package file, or unpackaged file is defined. """ if (self._build_file_valid(doc)): self.file(doc).add_artifact(symbol, value) From c450e9d20f991d8c3bcef5bff5f6c88f5dd42ba2 Mon Sep 17 00:00:00 2001 From: bjv Date: Mon, 13 Jun 2022 13:41:15 -0700 Subject: [PATCH 18/18] change var names & comparison order, for clarity --- spdx/parsers/rdf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spdx/parsers/rdf.py b/spdx/parsers/rdf.py index 17490d176..575b69321 100644 --- a/spdx/parsers/rdf.py +++ b/spdx/parsers/rdf.py @@ -399,9 +399,10 @@ def p_pkg_comment(self, p_term, predicate): self.more_than_one_error("package comment") def p_pkg_has_file(self, p_term, predicate): - for _, _, has_file in self.graph.triples((p_term, predicate, None)): + for _, _, has_file_object in self.graph.triples((p_term, predicate, None)): for index, spdx_file in enumerate(self.doc.unpackaged_files): - if str(has_file) == spdx_file.spdx_id: + object_spdx_id = str(has_file_object) + if spdx_file.spdx_id == object_spdx_id: self.doc.packages[-1].files.append(spdx_file) del self.doc.unpackaged_files[index]