From a687220a125afa917f328e023913a0a0aebe0d41 Mon Sep 17 00:00:00 2001 From: Jeffrey Otterson Date: Fri, 26 Nov 2021 09:47:49 -0500 Subject: [PATCH 01/13] add support for multiple file checksums Signed-off-by: Jeffrey Otterson --- spdx/checksum.py | 14 ++++++++ spdx/file.py | 40 ++++++++++++--------- spdx/parsers/jsonyamlxml.py | 8 +++-- spdx/parsers/jsonyamlxmlbuilders.py | 18 ++++++++++ spdx/parsers/lexers/tagvalue.py | 2 +- spdx/parsers/rdf.py | 24 ++++++++++--- spdx/parsers/rdfbuilders.py | 14 ++++---- spdx/parsers/tagvaluebuilders.py | 41 +++++++++++----------- spdx/writers/jsonyamlxml.py | 11 +++--- spdx/writers/rdf.py | 18 +++++----- spdx/writers/tagvalue.py | 3 +- tests/data/doc_parse/expected.json | 11 +++--- tests/data/doc_parse/spdx-expected.json | 11 +++--- tests/data/doc_write/json-simple-plus.json | 16 +++++---- tests/data/doc_write/json-simple.json | 6 +++- tests/data/doc_write/rdf-simple-plus.json | 19 ++++++---- tests/data/doc_write/rdf-simple.json | 19 ++++++---- tests/data/doc_write/tv-simple-plus.tv | 1 + tests/data/doc_write/tv-simple.tv | 1 + tests/data/doc_write/yaml-simple-plus.yaml | 2 ++ tests/data/doc_write/yaml-simple.yaml | 2 ++ tests/data/formats/SPDXJsonExample.json | 12 ++++--- tests/data/formats/SPDXRdfExample.rdf | 12 +++++++ tests/data/formats/SPDXTagExample.tag | 4 +++ tests/data/formats/SPDXYamlExample.yaml | 2 ++ tests/test_document.py | 5 +-- tests/utils_test.py | 10 ++++-- 27 files changed, 224 insertions(+), 102 deletions(-) diff --git a/spdx/checksum.py b/spdx/checksum.py index d770f5a84..673b35edf 100644 --- a/spdx/checksum.py +++ b/spdx/checksum.py @@ -9,11 +9,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +CHECKSUM_ALGORITHMS = ['SHA1', 'SHA256', 'SHA512'] +CHECKSUM_ALGORITHM_FROM_XML_DICT = { + 'checksumAlgorithm_sha1': 'SHA1', + 'checksumAlgorithm_sha256': 'SHA256', + 'checksumAlgorithm_sha512': 'SHA512', +} +CHECKSUM_ALGORITHM_TO_XML_DICT = { + 'SHA1': 'checksumAlgorithm_sha1', + 'SHA256': 'checksumAlgorithm_sha256', + 'SHA512': 'checksumAlgorithm_sha512', +} + class Algorithm(object): """Generic checksum algorithm.""" def __init__(self, identifier, value): + if identifier not in CHECKSUM_ALGORITHMS: + raise ValueError('checksum algorithm: {}'.format(identifier)) self.identifier = identifier self.value = value diff --git a/spdx/file.py b/spdx/file.py index eecf8b8f7..a6d794253 100644 --- a/spdx/file.py +++ b/spdx/file.py @@ -58,7 +58,7 @@ class File(object): referenced by other elements. Mandatory, one. Type: str. - comment: File comment str, Optional zero or one. - file_types: list of file types. Cardinality 0..* - - chksum: SHA1, Mandatory one. + - chk_sums: list of checksums, there must be a SHA1 hash, at least. - conc_lics: Mandatory one. license.License or utils.NoAssert or utils.SPDXNone. - licenses_in_file: list of licenses found in file, mandatory one or more. document.License or utils.SPDXNone or utils.NoAssert. @@ -74,12 +74,12 @@ class File(object): -attribution_text: optional string. """ - def __init__(self, name, spdx_id=None, chksum=None): + def __init__(self, name, spdx_id=None): self.name = name self.spdx_id = spdx_id self.comment = None self.file_types = [] - self.checksums = [chksum] + self.chk_sums = [] self.conc_lics = None self.licenses_in_file = [] self.license_comment = None @@ -203,28 +203,36 @@ def validate_file_types(self, messages): return messages def validate_checksum(self, messages): - if not isinstance(self.chksum, checksum.Algorithm): - messages.append( - "File checksum must be instance of spdx.checksum.Algorithm" - ) - else: - if not self.chksum.identifier == "SHA1": - messages.append("File checksum algorithm must be SHA1") - + if self.get_checksum() is None: + messages.append("At least one file checksum algorithm must be SHA1") return messages - def calc_chksum(self): + def calculate_checksum(self, hash_algorithm='SHA1'): BUFFER_SIZE = 65536 - file_sha1 = hashlib.sha1() + file_hash = hashlib.new(hash_algorithm.lower()) with open(self.name, "rb") as file_handle: while True: data = file_handle.read(BUFFER_SIZE) if not data: break - file_sha1.update(data) - - return file_sha1.hexdigest() + file_hash.update(data) + + return file_hash.hexdigest() + + def get_checksum(self, hash_algorithm='SHA1'): + for chk_sum in self.chk_sums: + if chk_sum.identifier == hash_algorithm: + return chk_sum + return None + + def set_checksum(self, chk_sum): + if isinstance(chk_sum, checksum.Algorithm): + for file_chk_sum in self.chk_sums: + if file_chk_sum.identifier == chk_sum.identifier: + file_chk_sum.value = chk_sum.value + return + self.chk_sums.append(chk_sum) def has_optional_field(self, field): return bool (getattr(self, field, None)) diff --git a/spdx/parsers/jsonyamlxml.py b/spdx/parsers/jsonyamlxml.py index 2cdbbdc54..a11df1d4c 100644 --- a/spdx/parsers/jsonyamlxml.py +++ b/spdx/parsers/jsonyamlxml.py @@ -823,7 +823,7 @@ def parse_file(self, file): self.parse_file_attribution_text(file.get("attributionTexts")) self.parse_file_dependencies(file.get("fileDependencies")) self.parse_annotations(file.get("annotations"), spdx_id=file.get("SPDXID")) - self.parse_file_chksum(file.get("sha1")) + self.parse_file_chksum(file.get("checksums")) else: self.value_error("FILE", file) @@ -1095,9 +1095,13 @@ def _handle_file_dependency(self, file_dependency): def parse_file_chksum(self, file_chksum): """ - Parse File checksum + Parse File checksums - file_chksum: Python str/unicode """ + if isinstance(file_chksum, list): + for chk_sum in file_chksum: + self.builder.set_file_chksum(self.document, chk_sum) + return True if isinstance(file_chksum, str): try: return self.builder.set_file_chksum(self.document, file_chksum) diff --git a/spdx/parsers/jsonyamlxmlbuilders.py b/spdx/parsers/jsonyamlxmlbuilders.py index a0565b470..8d590a5b1 100644 --- a/spdx/parsers/jsonyamlxmlbuilders.py +++ b/spdx/parsers/jsonyamlxmlbuilders.py @@ -12,6 +12,7 @@ from spdx.parsers import rdfbuilders from spdx.parsers import tagvaluebuilders from spdx.parsers import validations +from spdx import checksum from spdx.parsers.builderexceptions import SPDXValueError from spdx.parsers.builderexceptions import CardinalityError from spdx.parsers.builderexceptions import OrderError @@ -159,6 +160,23 @@ class FileBuilder(rdfbuilders.FileBuilder): def __init__(self): super(FileBuilder, self).__init__() + def set_file_chksum(self, doc, chk_sum): + """ + Set the file check sum, if not already set. + chk_sum - A string + Raise CardinalityError if already defined. + Raise OrderError if no package previously defined. + """ + if self.has_package(doc) and self.has_file(doc): + if isinstance(chk_sum, dict): + algo = checksum.CHECKSUM_ALGORITHM_FROM_XML_DICT.get(chk_sum.get('algorithm') or 'SHA1') + self.file(doc).set_checksum(checksum.Algorithm(algo, chk_sum.get('checksumValue'))) + elif isinstance(chk_sum, checksum.Algorithm): + self.file(doc).set_checksum(chk_sum) + else: + self.file(doc).set_checksum(checksum.Algorithm("SHA1", chk_sum)) + return True + def set_file_notice(self, doc, text): """ Set file notice diff --git a/spdx/parsers/lexers/tagvalue.py b/spdx/parsers/lexers/tagvalue.py index ee27d5e7a..08762c3ff 100644 --- a/spdx/parsers/lexers/tagvalue.py +++ b/spdx/parsers/lexers/tagvalue.py @@ -165,7 +165,7 @@ def t_text_error(self, t): print("Lexer error in text state") def t_CHKSUM(self, t): - r":\s*SHA1:\s*[a-f0-9]{40,40}" + r":\s*(SHA.*):\s*([a-f0-9]{40,128})" t.value = t.value[1:].strip() return t diff --git a/spdx/parsers/rdf.py b/spdx/parsers/rdf.py index a745a3697..cde43405a 100644 --- a/spdx/parsers/rdf.py +++ b/spdx/parsers/rdf.py @@ -21,6 +21,7 @@ from spdx import document from spdx import license from spdx import utils +from spdx import checksum from spdx.parsers.builderexceptions import CardinalityError from spdx.parsers.builderexceptions import SPDXValueError from spdx.parsers.loggers import ErrorMessages @@ -66,6 +67,16 @@ } +def convert_rdf_checksum_algorithm(algo): + ss = algo.split('#') + if len(ss) != 2: + raise SPDXValueError('Unknown checksum algorithm {}'.format(algo)) + algo = checksum.CHECKSUM_ALGORITHM_FROM_XML_DICT.get(ss[1]) + if algo is None: + raise SPDXValueError('Unknown checksum algorithm {}'.format(algo)) + return algo + + class BaseParser(object): """ Base class for all parsers. @@ -771,14 +782,19 @@ def p_file_type(self, f_term, predicate): def p_file_chk_sum(self, f_term, predicate): """ - Set file checksum. Assumes SHA1 algorithm without checking. + Set file checksum. """ try: - for _s, _p, checksum in self.graph.triples((f_term, predicate, None)): + for _s, _p, file_checksum in self.graph.triples((f_term, predicate, None)): for _, _, value in self.graph.triples( - (checksum, self.spdx_namespace["checksumValue"], None) + (file_checksum, self.spdx_namespace["checksumValue"], None) ): - self.builder.set_file_chksum(self.doc, str(value)) + for _, _, algo in self.graph.triples( + (file_checksum, self.spdx_namespace["algorithm"], None) + ): + algo = convert_rdf_checksum_algorithm(str(algo)) + chk_sum = checksum.Algorithm(str(algo), str(value)) + self.builder.set_file_chksum(self.doc, chk_sum) except CardinalityError: self.more_than_one_error("File checksum") diff --git a/spdx/parsers/rdfbuilders.py b/spdx/parsers/rdfbuilders.py index 78bf0f4ea..31ae853ef 100644 --- a/spdx/parsers/rdfbuilders.py +++ b/spdx/parsers/rdfbuilders.py @@ -388,14 +388,14 @@ def set_file_chksum(self, doc, chk_sum): Raise OrderError if no package previously defined. """ if self.has_package(doc) and self.has_file(doc): - if not self.file_chksum_set: - self.file_chksum_set = True - self.file(doc).chksum = checksum.Algorithm("SHA1", chk_sum) - return True + if isinstance(chk_sum, dict): + self.file(doc).set_checksum(checksum.Algorithm(chk_sum.get('algorithm'), + chk_sum.get('checksumValue'))) + elif isinstance(chk_sum, checksum.Algorithm): + self.file(doc).set_checksum(chk_sum) else: - raise CardinalityError("File::CheckSum") - else: - raise OrderError("File::CheckSum") + self.file(doc).set_checksum(checksum.Algorithm("SHA1", chk_sum)) + return True def set_file_license_comment(self, doc, text): """ diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index 33bfdad83..7686c5275 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -33,16 +33,11 @@ from spdx.parsers import validations -def checksum_from_sha1(value): - """ - Return an spdx.checksum.Algorithm instance representing the SHA1 - checksum or None if does not match CHECKSUM_RE. - """ - # More constrained regex at lexer level - CHECKSUM_RE = re.compile("SHA1:\\s*([\\S]+)", re.UNICODE) +def checksum_algorithm_from_string(value): + CHECKSUM_RE = re.compile("(SHA1|SHA256|SHA512):\\s*([a-f0-9]*)") match = CHECKSUM_RE.match(value) if match: - return checksum.Algorithm(identifier="SHA1", value=match.group(1)) + return checksum.Algorithm(identifier=match.group(1), value=match.group(2)) else: return None @@ -202,7 +197,7 @@ def set_chksum(self, doc, chksum): """ Set the `check_sum` attribute of the `ExternalDocumentRef` object. """ - doc.ext_document_references[-1].check_sum = checksum_from_sha1(chksum) + doc.ext_document_references[-1].check_sum = checksum_algorithm_from_string(chksum) def add_ext_doc_refs(self, doc, ext_doc_id, spdx_doc_uri, chksum): self.set_ext_doc_id(doc, ext_doc_id) @@ -788,13 +783,13 @@ def set_pkg_chk_sum(self, doc, chk_sum): Raise OrderError if no package previously defined. """ self.assert_package_exists() - if self.package_chk_sum_set: + if not self.package_chk_sum_set: + self.package_chk_sum_set = True + doc.packages[-1].check_sum = checksum_algorithm_from_string(chk_sum) + return True + else: raise CardinalityError("Package::CheckSum") - self.package_chk_sum_set = True - doc.packages[-1].checksum = checksum_from_sha1(chk_sum) - return True - def set_pkg_source_info(self, doc, text): """ Set the package's source information, if not already set. @@ -1202,14 +1197,18 @@ def set_file_chksum(self, doc, chksum): Raise OrderError if no package or file defined. Raise CardinalityError if more than one chksum set. """ - if not self.has_file(doc): + if self.has_package(doc) and self.has_file(doc): + self.file_chksum_set = False + chk_sums = doc.files[-1].chk_sums + chk_sums.append(checksum_algorithm_from_string(chksum)) + doc.files[-1].chk_sums = chk_sums + for chk_sum in self.file(doc).chk_sums: + if chk_sum.identifier == 'SHA1': + self.file_chksum_set = True + if not self.file_chksum_set: + raise CardinalityError("File::CheckSum") + else: raise OrderError("File::CheckSum") - - if self.file_chksum_set: - raise CardinalityError("File::CheckSum") - - self.file_chksum_set = True - self.file(doc).chksum = checksum_from_sha1(chksum) return True def set_concluded_license(self, doc, lic): diff --git a/spdx/writers/jsonyamlxml.py b/spdx/writers/jsonyamlxml.py index 9a612ee1d..006e6e228 100644 --- a/spdx/writers/jsonyamlxml.py +++ b/spdx/writers/jsonyamlxml.py @@ -49,11 +49,8 @@ def checksum(self, checksum_field): """ Return a dictionary representation of a spdx.checksum.Algorithm object """ - checksum_object = dict() - checksum_object["algorithm"] = ( - checksum_field.identifier.upper() - ) - checksum_object["checksumValue"] = checksum_field.value + checksum_object = {'algorithm': "checksumAlgorithm_" + checksum_field.identifier.lower(), + 'checksumValue': checksum_field.value} return checksum_object def spdx_id(self, spdx_id_field): @@ -235,7 +232,9 @@ def create_file_info(self, file, annotations_by_spdx_id): file_object["fileName"] = file.name file_object["SPDXID"] = self.spdx_id(file.spdx_id) - file_object["checksums"] = [self.checksum(file.chksum)] + file_object["checksums"] = [] + for chk_sum in file.chk_sums: + file_object["checksums"].append(self.checksum(chk_sum)) if file.has_optional_field("conc_lics"): file_object["licenseConcluded"] = self.license(file.conc_lics) diff --git a/spdx/writers/rdf.py b/spdx/writers/rdf.py index 6d99dcb80..7528ee259 100644 --- a/spdx/writers/rdf.py +++ b/spdx/writers/rdf.py @@ -17,7 +17,7 @@ from rdflib import RDFS from rdflib import URIRef from rdflib.compare import to_isomorphic - +from spdx import checksum from spdx import config from spdx import file from spdx import license @@ -48,13 +48,14 @@ def create_checksum_node(self, chksum): """ Return a node representing spdx.checksum. """ + algo = checksum.CHECKSUM_ALGORITHM_TO_XML_DICT.get(chksum.identifier) or 'checksumAlgorithm_sha1' chksum_node = BNode() type_triple = (chksum_node, RDF.type, self.spdx_namespace.Checksum) self.graph.add(type_triple) algorithm_triple = ( chksum_node, self.spdx_namespace.algorithm, - Literal(chksum.identifier), + Literal('http://spdx.org/rdf/terms#' + algo), ) self.graph.add(algorithm_triple) value_triple = ( @@ -264,13 +265,14 @@ def create_file_node(self, doc_file): ftype_triple = (file_node, self.spdx_namespace.fileType, ftype) self.graph.add(ftype_triple) - self.graph.add( - ( - file_node, - self.spdx_namespace.checksum, - self.create_checksum_node(doc_file.chksum), + for chk_sum in doc_file.chk_sums: + self.graph.add( + ( + file_node, + self.spdx_namespace.checksum, + self.create_checksum_node(chk_sum), + ) ) - ) conc_lic_node = self.license_or_special(doc_file.conc_lics) conc_lic_triple = ( diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index d60c01bdc..7fa22e9e5 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -124,7 +124,8 @@ def write_file(spdx_file, out): write_value("SPDXID", spdx_file.spdx_id, out) for file_type in spdx_file.file_types: write_file_type(file_type.name, out) - write_value("FileChecksum", spdx_file.chksum.to_tv(), out) + for file_chk_sum in spdx_file.chk_sums: + write_value("FileChecksum", file_chk_sum.to_tv(), out) if spdx_file.has_optional_field("conc_lics"): if isinstance( spdx_file.conc_lics, (license.LicenseConjunction, license.LicenseDisjunction) diff --git a/tests/data/doc_parse/expected.json b/tests/data/doc_parse/expected.json index a7466e1a2..5153d2e9c 100644 --- a/tests/data/doc_parse/expected.json +++ b/tests/data/doc_parse/expected.json @@ -48,10 +48,13 @@ "copyrightText": "(c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP", "licenseComment": "This license is used by Jena", "notice": null, - "checksum": { - "identifier": "SHA1", - "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125" - }, + "checksums": [{ + "identifier": "SHA1", + "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125" + },{ + "identifier": "SHA256", + "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f1250000000000000000" + }], "licenseInfoInFiles": [ { "type": "Single", diff --git a/tests/data/doc_parse/spdx-expected.json b/tests/data/doc_parse/spdx-expected.json index 154840727..2269092e4 100644 --- a/tests/data/doc_parse/spdx-expected.json +++ b/tests/data/doc_parse/spdx-expected.json @@ -48,10 +48,13 @@ "copyrightText": "(c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP", "licenseComment": "This license is used by Jena", "notice": null, - "checksum": { - "identifier": "SHA1", - "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125" - }, + "checksums": [{ + "identifier": "SHA1", + "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125" + },{ + "identifier": "SHA256", + "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f1250000000000000000" + }], "licenseInfoInFiles": [ { "type": "Single", diff --git a/tests/data/doc_write/json-simple-plus.json b/tests/data/doc_write/json-simple-plus.json index d414159db..c78f3bb3e 100644 --- a/tests/data/doc_write/json-simple-plus.json +++ b/tests/data/doc_write/json-simple-plus.json @@ -46,12 +46,16 @@ "files": [ { "SPDXID": "SPDXRef-File", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "SOME-SHA1" - } - ], + "checksums": [ + { + "algorithm": "checksumAlgorithm_sha1", + "checksumValue": "SOME-SHA1" + }, + { + "algorithm": "checksumAlgorithm_sha256", + "checksumValue": "SOME-SHA256" + } + ], "licenseConcluded": "NOASSERTION", "copyrightText": "NOASSERTION", "fileName": "./some/path/tofile", diff --git a/tests/data/doc_write/json-simple.json b/tests/data/doc_write/json-simple.json index 760ef5e3c..d029da5f3 100644 --- a/tests/data/doc_write/json-simple.json +++ b/tests/data/doc_write/json-simple.json @@ -48,8 +48,12 @@ "SPDXID": "SPDXRef-File", "checksums": [ { - "algorithm": "SHA1", + "algorithm": "checksumAlgorithm_sha1", "checksumValue": "SOME-SHA1" + }, + { + "algorithm": "checksumAlgorithm_sha256", + "checksumValue": "SOME-SHA256" } ], "licenseConcluded": "NOASSERTION", diff --git a/tests/data/doc_write/rdf-simple-plus.json b/tests/data/doc_write/rdf-simple-plus.json index 022479e80..66bc8f8e8 100644 --- a/tests/data/doc_write/rdf-simple-plus.json +++ b/tests/data/doc_write/rdf-simple-plus.json @@ -28,7 +28,7 @@ "ns1:copyrightText": "Some copyright", "ns1:checksum": { "ns1:Checksum": { - "ns1:algorithm": "SHA1", + "ns1:algorithm": "http://spdx.org/rdf/terms#checksumAlgorithm_sha1", "ns1:checksumValue": "SOME-SHA1" } } @@ -45,12 +45,19 @@ "ns1:File": { "@rdf:about": "http://www.spdx.org/files#SPDXRef-File", "ns1:fileName": "./some/path/tofile", - "ns1:checksum": { - "ns1:Checksum": { - "ns1:checksumValue": "SOME-SHA1", - "ns1:algorithm": "SHA1" + "ns1:checksum": [ + { + "ns1:Checksum": { + "ns1:checksumValue": "SOME-SHA1", + "ns1:algorithm": "http://spdx.org/rdf/terms#checksumAlgorithm_sha1" + } + }, { + "ns1:Checksum": { + "ns1:checksumValue": "SOME-SHA256", + "ns1:algorithm": "http://spdx.org/rdf/terms#checksumAlgorithm_sha256" + } } - }, + ], "ns1:licenseConcluded": { "@rdf:resource": "http://spdx.org/rdf/terms#noassertion" }, diff --git a/tests/data/doc_write/rdf-simple.json b/tests/data/doc_write/rdf-simple.json index 21004560c..63860a80c 100644 --- a/tests/data/doc_write/rdf-simple.json +++ b/tests/data/doc_write/rdf-simple.json @@ -28,7 +28,7 @@ "ns1:copyrightText": "Some copyright", "ns1:checksum": { "ns1:Checksum": { - "ns1:algorithm": "SHA1", + "ns1:algorithm": "http://spdx.org/rdf/terms#checksumAlgorithm_sha1", "ns1:checksumValue": "SOME-SHA1" } } @@ -47,12 +47,19 @@ "ns1:licenseInfoInFile": { "@rdf:resource": "http://spdx.org/licenses/LGPL-2.1-only" }, - "ns1:checksum": { - "ns1:Checksum": { - "ns1:checksumValue": "SOME-SHA1", - "ns1:algorithm": "SHA1" + "ns1:checksum": [ + { + "ns1:Checksum": { + "ns1:checksumValue": "SOME-SHA1", + "ns1:algorithm": "http://spdx.org/rdf/terms#checksumAlgorithm_sha1" + } + }, { + "ns1:Checksum": { + "ns1:checksumValue": "SOME-SHA256", + "ns1:algorithm": "http://spdx.org/rdf/terms#checksumAlgorithm_sha256" } - }, + } + ], "ns1:licenseConcluded": { "@rdf:resource": "http://spdx.org/rdf/terms#noassertion" }, diff --git a/tests/data/doc_write/tv-simple-plus.tv b/tests/data/doc_write/tv-simple-plus.tv index 5258e8a98..ecbf581bd 100644 --- a/tests/data/doc_write/tv-simple-plus.tv +++ b/tests/data/doc_write/tv-simple-plus.tv @@ -27,6 +27,7 @@ SPDXID: SPDXRef-File FileType: OTHER FileType: SOURCE FileChecksum: SHA1: SOME-SHA1 +FileChecksum: SHA256: SOME-SHA256 LicenseConcluded: NOASSERTION LicenseInfoInFile: LGPL-2.1-or-later FileCopyrightText: NOASSERTION diff --git a/tests/data/doc_write/tv-simple.tv b/tests/data/doc_write/tv-simple.tv index 0ffa2833f..2e38b67e6 100644 --- a/tests/data/doc_write/tv-simple.tv +++ b/tests/data/doc_write/tv-simple.tv @@ -27,6 +27,7 @@ SPDXID: SPDXRef-File FileType: OTHER FileType: SOURCE FileChecksum: SHA1: SOME-SHA1 +FileChecksum: SHA256: SOME-SHA256 LicenseConcluded: NOASSERTION LicenseInfoInFile: LGPL-2.1-only FileCopyrightText: NOASSERTION diff --git a/tests/data/doc_write/yaml-simple-plus.yaml b/tests/data/doc_write/yaml-simple-plus.yaml index a5391c072..637cbebdc 100644 --- a/tests/data/doc_write/yaml-simple-plus.yaml +++ b/tests/data/doc_write/yaml-simple-plus.yaml @@ -13,6 +13,8 @@ files: checksums: - algorithm: SHA1 checksumValue: SOME-SHA1 + - algorithm: checksumAlgorithm_sha256 + checksumValue: SOME-SHA256 copyrightText: NOASSERTION fileName: ./some/path/tofile licenseConcluded: NOASSERTION diff --git a/tests/data/doc_write/yaml-simple.yaml b/tests/data/doc_write/yaml-simple.yaml index 1ed7c6977..3060228b7 100644 --- a/tests/data/doc_write/yaml-simple.yaml +++ b/tests/data/doc_write/yaml-simple.yaml @@ -13,6 +13,8 @@ files: checksums: - algorithm: SHA1 checksumValue: SOME-SHA1 + - algorithm: "checksumAlgorithm_sha256" + checksumValue: "SOME-SHA256" copyrightText: NOASSERTION fileName: ./some/path/tofile licenseConcluded: NOASSERTION diff --git a/tests/data/formats/SPDXJsonExample.json b/tests/data/formats/SPDXJsonExample.json index 7d14fdace..7e23c846a 100644 --- a/tests/data/formats/SPDXJsonExample.json +++ b/tests/data/formats/SPDXJsonExample.json @@ -72,12 +72,16 @@ ], "licenseConcluded": "LicenseRef-1", "licenseComments": "This license is used by Jena", - "checksums": [ + "checksums": [ { - "checksumValue": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125", - "algorithm": "SHA1" + "checksumValue": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125", + "algorithm": "checksumAlgorithm_sha1" + }, + { + "checksumValue": "3ab4e1c67a2d28fced849ee1bb76e7391b93f1250000000000000000", + "algorithm": "checksumAlgorithm_sha256" } - ], + ], "fileTypes": [ "ARCHIVE", "OTHER" diff --git a/tests/data/formats/SPDXRdfExample.rdf b/tests/data/formats/SPDXRdfExample.rdf index 7446bc71a..cc6ba73fe 100644 --- a/tests/data/formats/SPDXRdfExample.rdf +++ b/tests/data/formats/SPDXRdfExample.rdf @@ -90,6 +90,12 @@ + + + 3ab4e1c67a2d28fced849ee1bb76e7391b93f1250000000000000000 + + + @@ -135,6 +141,12 @@ + + + 2fd4e1c67a2d28fced849ee1bb76e7391b93eb120000000000000000 + + + src/org/spdx/parser/DOAPProject.java diff --git a/tests/data/formats/SPDXTagExample.tag b/tests/data/formats/SPDXTagExample.tag index 4a886bea1..8d9c52847 100644 --- a/tests/data/formats/SPDXTagExample.tag +++ b/tests/data/formats/SPDXTagExample.tag @@ -74,6 +74,8 @@ SPDXID: SPDXRef-File1 FileType: SOURCE FileType: TEXT FileChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 +FileChecksum: SHA256: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb120000000000000000 +FileType: SOURCE LicenseConcluded: Apache-2.0 LicenseInfoInFile: Apache-2.0 FileCopyrightText: Copyright 2010, 2011 Source Auditor Inc. @@ -83,6 +85,8 @@ SPDXID: SPDXRef-File2 FileType: ARCHIVE FileType: OTHER FileChecksum: SHA1: 3ab4e1c67a2d28fced849ee1bb76e7391b93f125 +FileChecksum: SHA256: 3ab4e1c67a2d28fced849ee1bb76e7391b93f1250000000000000000 +FileType: ARCHIVE LicenseConcluded: LicenseRef-1 LicenseInfoInFile: LicenseRef-1 LicenseComments: This license is used by Jena diff --git a/tests/data/formats/SPDXYamlExample.yaml b/tests/data/formats/SPDXYamlExample.yaml index 96a8fe505..4124a530b 100644 --- a/tests/data/formats/SPDXYamlExample.yaml +++ b/tests/data/formats/SPDXYamlExample.yaml @@ -235,6 +235,8 @@ Document: checksums: - algorithm: checksumAlgorithm_sha1 checksumValue: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 + - algorithm: checksumAlgorithm_sha256 + checksumValue: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb120000000000000000 copyrightText: Copyright 2010, 2011 Source Auditor Inc. fileTypes: - SOURCE diff --git a/tests/test_document.py b/tests/test_document.py index 4d1a93027..092614c9b 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -115,7 +115,7 @@ def test_document_is_valid_when_using_or_later_licenses(self): file1.name = './some/path/tofile' file1.spdx_id = 'SPDXRef-File' file1.file_types = [FileType.OTHER] - file1.chksum = Algorithm('SHA1', 'SOME-SHA1') + file1.set_checksum(Algorithm('SHA1', 'SOME-SHA1')) file1.conc_lics = NoAssert() file1.copyright = NoAssert() @@ -191,7 +191,8 @@ def _get_lgpl_doc(self, or_later=False): file1 = File('./some/path/tofile') file1.name = './some/path/tofile' file1.spdx_id = 'SPDXRef-File' - file1.chksum = Algorithm('SHA1', 'SOME-SHA1') + file1.set_checksum(Algorithm('SHA1', 'SOME-SHA1')) + file1.set_checksum(Algorithm('SHA256', 'SOME-SHA256')) file1.conc_lics = NoAssert() file1.copyright = NoAssert() file1.file_types = [FileType.OTHER, FileType.SOURCE] diff --git a/tests/utils_test.py b/tests/utils_test.py index e9b4b09c7..d61902ff2 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -368,10 +368,10 @@ def package_to_dict(cls, package): if package.built_date: package_dict['builtDate'] = utils.datetime_iso_format(package.built_date) - + if package.release_date: package_dict['releaseDate'] = utils.datetime_iso_format(package.release_date) - + if package.valid_until_date: package_dict['validUntilDate'] = utils.datetime_iso_format(package.valid_until_date) @@ -391,6 +391,10 @@ def files_to_list(cls, files): for file in files: lics_in_files = sorted(file.licenses_in_file, key=lambda lic: lic.identifier) contributors = sorted(file.contributors, key=lambda c: c.name) + chk_sums = [] + for chk_sum in file.chk_sums: + chk_sums.append(cls.checksum_to_dict(chk_sum)) + file_dict = OrderedDict([ ('id', file.spdx_id), ('fileName', file.name), @@ -400,7 +404,7 @@ def files_to_list(cls, files): ('copyrightText', file.copyright), ('licenseComment', file.license_comment), ('notice', file.notice), - ('checksum', cls.checksum_to_dict(file.chksum)), + ('checksums', chk_sums), ('licenseInfoInFiles', [cls.license_to_dict(lic) for lic in lics_in_files]), ('contributors', [cls.entity_to_dict(contributor) for contributor in contributors]), ('dependencies', sorted(file.dependencies)), From 32a54020c744cb1ff4bea803be32f4bbc1e74aa2 Mon Sep 17 00:00:00 2001 From: Jeffrey Otterson Date: Thu, 2 Dec 2021 14:04:59 +0100 Subject: [PATCH 02/13] allow multiple checksums for package, too. Signed-off-by: Jeffrey Otterson --- spdx/file.py | 19 +++++----- spdx/package.py | 39 +++++++++++++++------ spdx/parsers/jsonyamlxml.py | 6 +++- spdx/parsers/jsonyamlxmlbuilders.py | 4 ++- spdx/parsers/rdf.py | 16 +++++---- spdx/parsers/rdfbuilders.py | 14 +++++--- spdx/parsers/tagvaluebuilders.py | 15 ++++---- spdx/writers/jsonyamlxml.py | 24 +++++++------ spdx/writers/rdf.py | 2 +- spdx/writers/tagvalue.py | 4 +-- tests/data/doc_parse/SBOMexpected.json | 15 ++++++-- tests/data/doc_write/json-simple-plus.json | 25 +++++++------ tests/data/doc_write/json-simple.json | 14 +++++--- tests/data/doc_write/tv-simple-plus.tv | 1 + tests/data/doc_write/tv-simple.tv | 1 + tests/data/doc_write/yaml-simple-plus.yaml | 4 ++- tests/data/doc_write/yaml-simple.yaml | 6 ++-- tests/data/formats/SPDXRdfExample.rdf | 6 ++++ tests/data/formats/SPDXSBOMExample.spdx.yml | 9 +++++ tests/data/formats/SPDXYamlExample.yaml | 4 +-- tests/test_document.py | 9 ++--- tests/utils_test.py | 2 +- 22 files changed, 153 insertions(+), 86 deletions(-) diff --git a/spdx/file.py b/spdx/file.py index a6d794253..bdfa3533b 100644 --- a/spdx/file.py +++ b/spdx/file.py @@ -79,7 +79,7 @@ def __init__(self, name, spdx_id=None): self.spdx_id = spdx_id self.comment = None self.file_types = [] - self.chk_sums = [] + self.checksums = [] self.conc_lics = None self.licenses_in_file = [] self.license_comment = None @@ -99,16 +99,15 @@ def __lt__(self, other): return self.name < other.name @property - def chksum(self): + def chk_sum(self): """ Backwards compatibility, return first checksum. """ - # NOTE Package.check_sum but File.chk_sum - return self.checksums[0] + return self.get_checksum('SHA1') - @chksum.setter - def chksum(self, value): - self.checksums[0] = value + @chk_sum.setter + def chk_sum(self, value): + self.set_checksum(value) def add_lics(self, lics): self.licenses_in_file.append(lics) @@ -221,18 +220,18 @@ def calculate_checksum(self, hash_algorithm='SHA1'): return file_hash.hexdigest() def get_checksum(self, hash_algorithm='SHA1'): - for chk_sum in self.chk_sums: + for chk_sum in self.checksums: if chk_sum.identifier == hash_algorithm: return chk_sum return None def set_checksum(self, chk_sum): if isinstance(chk_sum, checksum.Algorithm): - for file_chk_sum in self.chk_sums: + for file_chk_sum in self.checksums: if file_chk_sum.identifier == chk_sum.identifier: file_chk_sum.value = chk_sum.value return - self.chk_sums.append(chk_sum) + self.checksums.append(chk_sum) def has_optional_field(self, field): return bool (getattr(self, field, None)) diff --git a/spdx/package.py b/spdx/package.py index aaa32f895..d5e8b93c3 100644 --- a/spdx/package.py +++ b/spdx/package.py @@ -100,7 +100,7 @@ def __init__( self.files_analyzed = None self.homepage = None self.verif_code = None - self.checksums = [None] + self.checksums = [] self.source_info = None self.conc_lics = None self.license_declared = None @@ -119,6 +119,17 @@ def __init__( self.valid_until_date: Optional[datetime] = None + @property + def check_sum(self): + """ + Backwards compatibility, return first checksum. + """ + return self.get_checksum('SHA1') + + @check_sum.setter + def check_sum(self, value): + self.set_checksum(value) + @property def are_files_analyzed(self): return self.files_analyzed is not False @@ -285,18 +296,24 @@ def validate_str_fields(self, fields, optional, messages): return messages def validate_checksum(self, messages): - if self.checksum is not None: - if not isinstance(self.checksum, checksum.Algorithm): - messages.append( - "Package checksum must be instance of spdx.checksum.Algorithm" - ) - elif not self.checksum.identifier == "SHA1": - messages.append( - "First checksum in package must be SHA1." - ) - + if self.get_checksum() is None: + messages.append("At least one package checksum algorithm must be SHA1") return messages + def get_checksum(self, hash_algorithm='SHA1'): + for chk_sum in self.checksums: + if chk_sum.identifier == hash_algorithm: + return chk_sum + return None + + def set_checksum(self, new_checksum): + if isinstance(new_checksum, checksum.Algorithm): + for c in self.checksums: + if c.identifier == new_checksum.identifier: + c.value = new_checksum.value + return + self.checksums.append(new_checksum) + def has_optional_field(self, field): return bool(getattr(self, field, None)) diff --git a/spdx/parsers/jsonyamlxml.py b/spdx/parsers/jsonyamlxml.py index a11df1d4c..e005e1720 100644 --- a/spdx/parsers/jsonyamlxml.py +++ b/spdx/parsers/jsonyamlxml.py @@ -1162,7 +1162,7 @@ def parse_package(self, package: Package, method_to_parse_relationship: Callable self.parse_annotations(package.get("annotations"), spdx_id=package.get("SPDXID")) self.parse_pkg_attribution_text(package.get("attributionTexts")) self.parse_pkg_files(package.get("hasFiles"), method_to_parse_relationship) - self.parse_pkg_chksum(package.get("sha1")) + self.parse_pkg_chksum(package.get("checksums")) self.parse_package_external_refs(package.get("externalRefs")) self.parse_primary_package_purpose(package.get("primaryPackagePurpose")) self.parse_release_date(package.get("releaseDate")) @@ -1597,6 +1597,10 @@ def parse_pkg_chksum(self, pkg_chksum): Parse Package checksum - pkg_chksum: Python str/unicode """ + if isinstance(pkg_chksum, list): + for chk_sum in pkg_chksum: + self.builder.set_pkg_chk_sum(self.document, chk_sum) + return True if isinstance(pkg_chksum, str): try: return self.builder.set_pkg_chk_sum(self.document, pkg_chksum) diff --git a/spdx/parsers/jsonyamlxmlbuilders.py b/spdx/parsers/jsonyamlxmlbuilders.py index 8d590a5b1..89e1b44a4 100644 --- a/spdx/parsers/jsonyamlxmlbuilders.py +++ b/spdx/parsers/jsonyamlxmlbuilders.py @@ -169,7 +169,9 @@ def set_file_chksum(self, doc, chk_sum): """ if self.has_package(doc) and self.has_file(doc): if isinstance(chk_sum, dict): - algo = checksum.CHECKSUM_ALGORITHM_FROM_XML_DICT.get(chk_sum.get('algorithm') or 'SHA1') + + #algo = checksum.CHECKSUM_ALGORITHM_FROM_XML_DICT.get(chk_sum.get('algorithm') or 'SHA1') + algo = chk_sum.get('algorithm') or 'SHA1' self.file(doc).set_checksum(checksum.Algorithm(algo, chk_sum.get('checksumValue'))) elif isinstance(chk_sum, checksum.Algorithm): self.file(doc).set_checksum(chk_sum) diff --git a/spdx/parsers/rdf.py b/spdx/parsers/rdf.py index cde43405a..2b496e9ab 100644 --- a/spdx/parsers/rdf.py +++ b/spdx/parsers/rdf.py @@ -515,15 +515,17 @@ def p_pkg_src_info(self, p_term, predicate): break def p_pkg_chk_sum(self, p_term, predicate): - for _s, _p, checksum in self.graph.triples((p_term, predicate, None)): + for _s, _p, pkg_checksum in self.graph.triples((p_term, predicate, None)): for _, _, value in self.graph.triples( - (checksum, self.spdx_namespace["checksumValue"], None) + (pkg_checksum, self.spdx_namespace["checksumValue"], None) ): - try: - self.builder.set_pkg_chk_sum(self.doc, str(value)) - except CardinalityError: - self.more_than_one_error("Package checksum") - break + for _, _, algo in self.graph.triples( + (pkg_checksum, self.spdx_namespace["algorithm"], None) + ): + algo = convert_rdf_checksum_algorithm(str(algo)) + chk_sum = checksum.Algorithm(str(algo), str(value)) + self.builder.set_pkg_chk_sum(self.doc, chk_sum) + def p_pkg_homepg(self, p_term, predicate): for _s, _p, o in self.graph.triples((p_term, predicate, None)): diff --git a/spdx/parsers/rdfbuilders.py b/spdx/parsers/rdfbuilders.py index 31ae853ef..7da702f2b 100644 --- a/spdx/parsers/rdfbuilders.py +++ b/spdx/parsers/rdfbuilders.py @@ -192,11 +192,15 @@ def set_pkg_chk_sum(self, doc, chk_sum): Raise OrderError if no package previously defined. """ self.assert_package_exists() - if not self.package_chk_sum_set: - self.package_chk_sum_set = True - doc.packages[-1].checksum = checksum.Algorithm("SHA1", chk_sum) - else: - raise CardinalityError("Package::CheckSum") + self.package_chk_sum_set = True + if isinstance(chk_sum, dict): + algo = chk_sum.get('algorithm') or 'SHA1' + if algo.startswith('checksumAlgorithm_'): + algo = checksum.CHECKSUM_ALGORITHM_FROM_XML_DICT.get(algo) or 'SHA1' + doc.packages[-1].set_checksum(checksum.Algorithm(identifier=algo, + value=chk_sum.get('checksumValue'))) + elif isinstance(chk_sum, checksum.Algorithm): + doc.packages[-1].set_checksum(chk_sum) def set_pkg_source_info(self, doc, text): """ diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index 7686c5275..4a3295555 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -783,12 +783,9 @@ def set_pkg_chk_sum(self, doc, chk_sum): Raise OrderError if no package previously defined. """ self.assert_package_exists() - if not self.package_chk_sum_set: - self.package_chk_sum_set = True - doc.packages[-1].check_sum = checksum_algorithm_from_string(chk_sum) - return True - else: - raise CardinalityError("Package::CheckSum") + self.package_chk_sum_set = True + doc.packages[-1].set_checksum(checksum_algorithm_from_string(chk_sum)) + return True def set_pkg_source_info(self, doc, text): """ @@ -1199,10 +1196,10 @@ def set_file_chksum(self, doc, chksum): """ if self.has_package(doc) and self.has_file(doc): self.file_chksum_set = False - chk_sums = doc.files[-1].chk_sums + chk_sums = doc.files[-1].checksums chk_sums.append(checksum_algorithm_from_string(chksum)) - doc.files[-1].chk_sums = chk_sums - for chk_sum in self.file(doc).chk_sums: + doc.files[-1].checksums = chk_sums + for chk_sum in self.file(doc).checksums: if chk_sum.identifier == 'SHA1': self.file_chksum_set = True if not self.file_chksum_set: diff --git a/spdx/writers/jsonyamlxml.py b/spdx/writers/jsonyamlxml.py index 006e6e228..294838499 100644 --- a/spdx/writers/jsonyamlxml.py +++ b/spdx/writers/jsonyamlxml.py @@ -45,13 +45,14 @@ def license(self, license_field): license_str = license_field.__str__() return license_str - def checksum(self, checksum_field): + def checksum_to_dict(self, checksum_field): """ Return a dictionary representation of a spdx.checksum.Algorithm object """ - checksum_object = {'algorithm': "checksumAlgorithm_" + checksum_field.identifier.lower(), - 'checksumValue': checksum_field.value} - return checksum_object + #checksum_object = {'algorithm': "checksumAlgorithm_" + checksum_field.identifier.lower(), + # 'checksumValue': checksum_field.value} + #return checksum_object + return {'algorithm': checksum_field.identifier, 'checksumValue': checksum_field.value} def spdx_id(self, spdx_id_field): return spdx_id_field.__str__().split("#")[-1] @@ -168,8 +169,12 @@ def create_package_info(self, package, annotations_by_spdx_id): if package.has_optional_field("originator"): package_object["originator"] = package.originator.to_value() - if package.has_optional_field("checksum"): - package_object["checksums"] = [self.checksum(checksum) for checksum in package.checksums if checksum] + package_object["checksums"] = [] + for chk_sum in package.checksums: + package_object["checksums"].append(self.checksum_to_dict(chk_sum)) + sha1 = package.get_checksum('SHA1') + if sha1 is not None: + package_object['sha1'] = sha1.value if package.has_optional_field("description"): package_object["description"] = package.description @@ -233,9 +238,8 @@ def create_file_info(self, file, annotations_by_spdx_id): file_object["fileName"] = file.name file_object["SPDXID"] = self.spdx_id(file.spdx_id) file_object["checksums"] = [] - for chk_sum in file.chk_sums: - file_object["checksums"].append(self.checksum(chk_sum)) - + for chk_sum in file.checksums: + file_object["checksums"].append(self.checksum_to_dict(chk_sum)) if file.has_optional_field("conc_lics"): file_object["licenseConcluded"] = self.license(file.conc_lics) @@ -521,7 +525,7 @@ def create_ext_document_references(self): "spdxDocument" ] = ext_document_reference.spdx_document_uri - ext_document_reference_object["checksum"] = self.checksum( + ext_document_reference_object["checksum"] = self.checksum_to_dict( ext_document_reference.check_sum ) diff --git a/spdx/writers/rdf.py b/spdx/writers/rdf.py index 7528ee259..cca19edb8 100644 --- a/spdx/writers/rdf.py +++ b/spdx/writers/rdf.py @@ -265,7 +265,7 @@ def create_file_node(self, doc_file): ftype_triple = (file_node, self.spdx_namespace.fileType, ftype) self.graph.add(ftype_triple) - for chk_sum in doc_file.chk_sums: + for chk_sum in doc_file.checksums: self.graph.add( ( file_node, diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index 7fa22e9e5..7a2c1a3a6 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -240,8 +240,8 @@ def write_package(package, out): if package.has_optional_field("originator"): write_value("PackageOriginator", package.originator, out) - if package.has_optional_field("checksum"): - write_value("PackageChecksum", package.checksum.to_tv(), out) + for package_checksum in package.checksums: + write_value("PackageChecksum", package_checksum.to_tv(), out) if package.has_optional_field("verif_code"): write_value("PackageVerificationCode", format_verif_code(package), out) diff --git a/tests/data/doc_parse/SBOMexpected.json b/tests/data/doc_parse/SBOMexpected.json index 6fdfbe7d0..a473606aa 100644 --- a/tests/data/doc_parse/SBOMexpected.json +++ b/tests/data/doc_parse/SBOMexpected.json @@ -64,7 +64,10 @@ }, "copyrightText": "copyright 2004-2020 Example Inc. All Rights Reserved.", "licenseComment": null, - "checksum": null, + "checksum": { + "identifier": "SHA1", + "value": "SOME-SHA1" + }, "licenseInfoFromFiles": [], "verificationCode": { "value": null, @@ -95,7 +98,10 @@ }, "copyrightText": "Copyright (c) 1996 - 2020, Daniel Stenberg, , and many contributors, see the THANKS file.", "licenseComment": null, - "checksum": null, + "checksum": { + "identifier": "SHA1", + "value": "SOME-SHA1" + }, "licenseInfoFromFiles": [], "verificationCode": { "value": null, @@ -126,7 +132,10 @@ }, "copyrightText": "copyright 2004-2020 The OpenSSL Project Authors. All Rights Reserved.", "licenseComment": null, - "checksum": null, + "checksum": { + "identifier": "SHA1", + "value": "SOME-SHA1" + }, "licenseInfoFromFiles": [], "verificationCode": { "value": null, diff --git a/tests/data/doc_write/json-simple-plus.json b/tests/data/doc_write/json-simple-plus.json index c78f3bb3e..fae11a34d 100644 --- a/tests/data/doc_write/json-simple-plus.json +++ b/tests/data/doc_write/json-simple-plus.json @@ -14,12 +14,11 @@ "documentDescribes": [ "SPDXRef-Package" ], - "packages": [ - { - "SPDXID": "SPDXRef-Package", - "name": "some/path", - "downloadLocation": "NOASSERTION", - "copyrightText": "Some copyright", + "packages": [ { + "SPDXID": "SPDXRef-Package", + "name": "some/path", + "downloadLocation": "NOASSERTION", + "copyrightText": "Some copyright", "packageVerificationCode": { "packageVerificationCodeValue": "SOME code" }, @@ -27,9 +26,13 @@ { "algorithm": "SHA1", "checksumValue": "SOME-SHA1" - } - ], - "licenseDeclared": "NOASSERTION", + }, + { + "algorithm": "SHA256", + "checksumValue": "SOME-SHA256" + } + ], + "licenseDeclared": "NOASSERTION", "licenseConcluded": "NOASSERTION", "licenseInfoFromFiles": [ "LGPL-2.1-or-later" @@ -48,11 +51,11 @@ "SPDXID": "SPDXRef-File", "checksums": [ { - "algorithm": "checksumAlgorithm_sha1", + "algorithm": "SHA1", "checksumValue": "SOME-SHA1" }, { - "algorithm": "checksumAlgorithm_sha256", + "algorithm": "SHA256", "checksumValue": "SOME-SHA256" } ], diff --git a/tests/data/doc_write/json-simple.json b/tests/data/doc_write/json-simple.json index d029da5f3..b2a5fa461 100644 --- a/tests/data/doc_write/json-simple.json +++ b/tests/data/doc_write/json-simple.json @@ -27,9 +27,13 @@ { "algorithm": "SHA1", "checksumValue": "SOME-SHA1" - } - ], - "licenseDeclared": "NOASSERTION", + }, + { + "algorithm": "SHA256", + "checksumValue": "SOME-SHA256" + } + ], + "licenseDeclared": "NOASSERTION", "licenseConcluded": "NOASSERTION", "licenseInfoFromFiles": [ "LGPL-2.1-only" @@ -48,11 +52,11 @@ "SPDXID": "SPDXRef-File", "checksums": [ { - "algorithm": "checksumAlgorithm_sha1", + "algorithm": "SHA1", "checksumValue": "SOME-SHA1" }, { - "algorithm": "checksumAlgorithm_sha256", + "algorithm": "SHA256", "checksumValue": "SOME-SHA256" } ], diff --git a/tests/data/doc_write/tv-simple-plus.tv b/tests/data/doc_write/tv-simple-plus.tv index ecbf581bd..19b458f6c 100644 --- a/tests/data/doc_write/tv-simple-plus.tv +++ b/tests/data/doc_write/tv-simple-plus.tv @@ -12,6 +12,7 @@ PackageName: some/path SPDXID: SPDXRef-Package PackageDownloadLocation: NOASSERTION PackageChecksum: SHA1: SOME-SHA1 +PackageChecksum: SHA256: SOME-SHA256 PackageVerificationCode: SOME code PackageLicenseDeclared: NOASSERTION PackageLicenseConcluded: NOASSERTION diff --git a/tests/data/doc_write/tv-simple.tv b/tests/data/doc_write/tv-simple.tv index 2e38b67e6..d7cd21426 100644 --- a/tests/data/doc_write/tv-simple.tv +++ b/tests/data/doc_write/tv-simple.tv @@ -12,6 +12,7 @@ PackageName: some/path SPDXID: SPDXRef-Package PackageDownloadLocation: NOASSERTION PackageChecksum: SHA1: SOME-SHA1 +PackageChecksum: SHA256: SOME-SHA256 PackageVerificationCode: SOME code PackageLicenseDeclared: NOASSERTION PackageLicenseConcluded: NOASSERTION diff --git a/tests/data/doc_write/yaml-simple-plus.yaml b/tests/data/doc_write/yaml-simple-plus.yaml index 637cbebdc..007d6cdd9 100644 --- a/tests/data/doc_write/yaml-simple-plus.yaml +++ b/tests/data/doc_write/yaml-simple-plus.yaml @@ -13,7 +13,7 @@ files: checksums: - algorithm: SHA1 checksumValue: SOME-SHA1 - - algorithm: checksumAlgorithm_sha256 + - algorithm: SHA256 checksumValue: SOME-SHA256 copyrightText: NOASSERTION fileName: ./some/path/tofile @@ -29,6 +29,8 @@ packages: checksums: - algorithm: SHA1 checksumValue: SOME-SHA1 + - algorithm: SHA256 + checksumValue: SOME-SHA256 copyrightText: Some copyright downloadLocation: NOASSERTION hasFiles: diff --git a/tests/data/doc_write/yaml-simple.yaml b/tests/data/doc_write/yaml-simple.yaml index 3060228b7..98e6edb42 100644 --- a/tests/data/doc_write/yaml-simple.yaml +++ b/tests/data/doc_write/yaml-simple.yaml @@ -13,8 +13,8 @@ files: checksums: - algorithm: SHA1 checksumValue: SOME-SHA1 - - algorithm: "checksumAlgorithm_sha256" - checksumValue: "SOME-SHA256" + - algorithm: SHA256 + checksumValue: SOME-SHA256 copyrightText: NOASSERTION fileName: ./some/path/tofile licenseConcluded: NOASSERTION @@ -30,6 +30,8 @@ packages: checksums: - algorithm: SHA1 checksumValue: SOME-SHA1 + - algorithm: SHA256 + checksumValue: SOME-SHA256 copyrightText: Some copyright downloadLocation: NOASSERTION hasFiles: diff --git a/tests/data/formats/SPDXRdfExample.rdf b/tests/data/formats/SPDXRdfExample.rdf index cc6ba73fe..d5912d8a8 100644 --- a/tests/data/formats/SPDXRdfExample.rdf +++ b/tests/data/formats/SPDXRdfExample.rdf @@ -165,6 +165,12 @@ SpdxTranslatorSpdx.rdf + + + 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 + + + diff --git a/tests/data/formats/SPDXSBOMExample.spdx.yml b/tests/data/formats/SPDXSBOMExample.spdx.yml index a5664c3eb..5655dae55 100644 --- a/tests/data/formats/SPDXSBOMExample.spdx.yml +++ b/tests/data/formats/SPDXSBOMExample.spdx.yml @@ -16,6 +16,9 @@ documentDescribes: packages: - SPDXID: "SPDXRef-Package-xyz" summary: "Awesome product created by Example Inc." + checksums: + - algorithm: SHA1 + checksumValue: SOME-SHA1 copyrightText: "copyright 2004-2020 Example Inc. All Rights Reserved." downloadLocation: "git+ssh://gitlab.example.com:3389/products/xyz.git@b2c358080011af6a366d2512a25a379fbe7b1f78" filesAnalyzed: false @@ -28,6 +31,9 @@ packages: description: "A command line tool and library for transferring data with URL syntax, supporting \ HTTP, HTTPS, FTP, FTPS, GOPHER, TFTP, SCP, SFTP, SMB, TELNET, DICT, LDAP, LDAPS, MQTT, FILE, \ IMAP, SMTP, POP3, RTSP and RTMP. libcurl offers a myriad of powerful features." + checksums: + - algorithm: SHA1 + checksumValue: SOME-SHA1 copyrightText: "Copyright (c) 1996 - 2020, Daniel Stenberg, , and many contributors, see the THANKS file." downloadLocation: "https://github.com/curl/curl/releases/download/curl-7_70_0/curl-7.70.0.tar.gz" @@ -41,6 +47,9 @@ packages: - SPDXID: "SPDXRef-Package-openssl" description: "OpenSSL is a robust, commercial-grade, full-featured Open Source Toolkit for the Transport Layer Security (TLS) protocol formerly known as the Secure Sockets Layer (SSL) protocol. The protocol implementation is based on a full-strength general purpose cryptographic library, which can also be used stand-alone." copyrightText: "copyright 2004-2020 The OpenSSL Project Authors. All Rights Reserved." + checksums: + - algorithm: SHA1 + checksumValue: SOME-SHA1 downloadLocation: "git+ssh://github.com/openssl/openssl.git@e2e09d9fba1187f8d6aafaa34d4172f56f1ffb72" filesAnalyzed: false homepage: "https://www.openssl.org/" diff --git a/tests/data/formats/SPDXYamlExample.yaml b/tests/data/formats/SPDXYamlExample.yaml index 4124a530b..f6b544ed9 100644 --- a/tests/data/formats/SPDXYamlExample.yaml +++ b/tests/data/formats/SPDXYamlExample.yaml @@ -20,7 +20,7 @@ Document: packages: - SPDXID: SPDXRef-Package checksums: - - algorithm: checksumAlgorithm_sha1 + - algorithm: SHA1 checksumValue: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 copyrightText: ' Copyright 2010, 2011 Source Auditor Inc.' description: This utility translates and SPDX RDF XML document to a spreadsheet, @@ -68,7 +68,7 @@ Document: validUntilDate: '2022-01-01T12:00:00Z' externalDocumentRefs: - checksum: - algorithm: checksumAlgorithm_sha1 + algorithm: SHA1 checksumValue: d6a770ba38583ed4bb4525bd96e50461655d2759 externalDocumentId: DocumentRef-spdx-tool-2.1 spdxDocument: https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301 diff --git a/tests/test_document.py b/tests/test_document.py index 092614c9b..52ddf4eae 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -80,7 +80,7 @@ def test_document_validate_failures_returns_informative_messages(self): 'Sample_Document-V2.1', spdx_id='SPDXRef-DOCUMENT', namespace='https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301') pack = doc.package = Package('some/path', NoAssert()) - pack.checksum = 'SOME-SHA1' + pack.check_sum = Algorithm('SHA256', 'SOME-SHA256') file1 = File('./some/path/tofile') file1.name = './some/path/tofile' file1.spdx_id = 'SPDXRef-File' @@ -92,8 +92,7 @@ def test_document_validate_failures_returns_informative_messages(self): messages = doc.validate(messages) expected = ['Sample_Document-V2.1: Creation info missing created date.', 'Sample_Document-V2.1: No creators defined, must have at least one.', - 'Sample_Document-V2.1: some/path: Package checksum must be instance of ' - 'spdx.checksum.Algorithm', + 'Sample_Document-V2.1: some/path: At least one package checksum algorithm must be SHA1', 'Sample_Document-V2.1: some/path: Package download_location can not be None.'] assert sorted(expected) == sorted(messages) @@ -107,6 +106,7 @@ def test_document_is_valid_when_using_or_later_licenses(self): package = doc.package = Package(name='some/path', download_location=NoAssert()) package.spdx_id = 'SPDXRef-Package' package.cr_text = 'Some copyright' + package.set_checksum(Algorithm('SHA1', 'SOME-SHA1')) package.verif_code = 'SOME code' package.license_declared = NoAssert() package.conc_lics = NoAssert() @@ -179,7 +179,8 @@ def _get_lgpl_doc(self, or_later=False): package.spdx_id = 'SPDXRef-Package' package.cr_text = 'Some copyright' package.verif_code = 'SOME code' - package.checksum = Algorithm('SHA1', 'SOME-SHA1') + package.set_checksum(Algorithm('SHA1', 'SOME-SHA1')) + package.set_checksum(Algorithm('SHA256', 'SOME-SHA256')) package.license_declared = NoAssert() package.conc_lics = NoAssert() package.primary_package_purpose = PackagePurpose.FILE diff --git a/tests/utils_test.py b/tests/utils_test.py index d61902ff2..e77159fa3 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -392,7 +392,7 @@ def files_to_list(cls, files): lics_in_files = sorted(file.licenses_in_file, key=lambda lic: lic.identifier) contributors = sorted(file.contributors, key=lambda c: c.name) chk_sums = [] - for chk_sum in file.chk_sums: + for chk_sum in file.checksums: chk_sums.append(cls.checksum_to_dict(chk_sum)) file_dict = OrderedDict([ From e17952559e7f8ebc514d25398814d4ace3897e3c Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Wed, 23 Nov 2022 14:51:30 +0100 Subject: [PATCH 03/13] [issue-185] fix failing tests Signed-off-by: Meret Behrens --- spdx/file.py | 4 +++- spdx/package.py | 9 +++++++-- spdx/utils.py | 6 +++--- spdx/writers/json.py | 1 + spdx/writers/jsonyamlxml.py | 9 ++------- spdx/writers/tagvalue.py | 2 +- tests/data/doc_parse/expected.json | 4 ++-- tests/data/doc_parse/spdx-expected.json | 13 +++++++++---- tests/data/doc_write/xml-simple-multi-package.xml | 2 +- tests/data/doc_write/xml-simple-plus.xml | 8 ++++++++ tests/data/doc_write/xml-simple.xml | 8 ++++++++ tests/data/formats/SPDXJsonExample.json | 4 ++-- tests/data/formats/SPDXXmlExample.xml | 12 ++++++++---- tests/data/formats/SPDXYamlExample.yaml | 8 ++++---- tests/test_document.py | 2 +- tests/test_jsonyamlxml_writer.py | 3 ++- tests/test_package.py | 2 +- 17 files changed, 63 insertions(+), 34 deletions(-) diff --git a/spdx/file.py b/spdx/file.py index bdfa3533b..3b05d5b02 100644 --- a/spdx/file.py +++ b/spdx/file.py @@ -202,9 +202,11 @@ def validate_file_types(self, messages): return messages def validate_checksum(self, messages): + if not self.checksums: + return messages if self.get_checksum() is None: messages.append("At least one file checksum algorithm must be SHA1") - return messages + return messages def calculate_checksum(self, hash_algorithm='SHA1'): BUFFER_SIZE = 65536 diff --git a/spdx/package.py b/spdx/package.py index d5e8b93c3..f2885777f 100644 --- a/spdx/package.py +++ b/spdx/package.py @@ -142,7 +142,10 @@ def checksum(self): Backwards compatibility, return first checksum. """ # NOTE Package.check_sum but File.chk_sum - return self.checksums[0] + if self.checksums: + return self.checksums[0] + else: + return None @checksum.setter def checksum(self, value): @@ -296,9 +299,11 @@ def validate_str_fields(self, fields, optional, messages): return messages def validate_checksum(self, messages): + if not self.checksums: + return messages if self.get_checksum() is None: messages.append("At least one package checksum algorithm must be SHA1") - return messages + return messages def get_checksum(self, hash_algorithm='SHA1'): for chk_sum in self.checksums: diff --git a/spdx/utils.py b/spdx/utils.py index 115ea3ae1..c1ebae9ec 100644 --- a/spdx/utils.py +++ b/spdx/utils.py @@ -20,7 +20,7 @@ from spdx import checksum if TYPE_CHECKING: from spdx.file import File -from spdx.package import Package + from spdx.package import Package from spdx.relationship import Relationship from spdx import license @@ -230,7 +230,7 @@ def calc_verif_code(files: List['File']) -> str: return sha1.hexdigest() -def get_files_in_package(package: Package, files: List['File'], relationships: List[Relationship]) -> List['File']: +def get_files_in_package(package: 'Package', files: List['File'], relationships: List[Relationship]) -> List['File']: files_in_package = [] for file in files: if file.spdx_id in [relationship.related_spdx_element for relationship in relationships @@ -269,7 +269,7 @@ def calc_verif_code(files: List['File']) -> str: return sha1.hexdigest() -def get_files_in_package(package: Package, files: List['File'], relationships: List[Relationship]) -> List['File']: +def get_files_in_package(package: 'Package', files: List['File'], relationships: List[Relationship]) -> List['File']: files_in_package = [] for file in files: if file.spdx_id in [relationship.related_spdx_element for relationship in relationships diff --git a/spdx/writers/json.py b/spdx/writers/json.py index a2b226fdf..732b6900f 100644 --- a/spdx/writers/json.py +++ b/spdx/writers/json.py @@ -30,6 +30,7 @@ def write_document(document, out, validate=True): messages = ErrorMessages() messages = document.validate(messages) if messages: + print(messages.messages) raise InvalidDocumentError(messages) writer = Writer(document) diff --git a/spdx/writers/jsonyamlxml.py b/spdx/writers/jsonyamlxml.py index 294838499..cfbdaae5e 100644 --- a/spdx/writers/jsonyamlxml.py +++ b/spdx/writers/jsonyamlxml.py @@ -169,12 +169,8 @@ def create_package_info(self, package, annotations_by_spdx_id): if package.has_optional_field("originator"): package_object["originator"] = package.originator.to_value() - package_object["checksums"] = [] for chk_sum in package.checksums: - package_object["checksums"].append(self.checksum_to_dict(chk_sum)) - sha1 = package.get_checksum('SHA1') - if sha1 is not None: - package_object['sha1'] = sha1.value + package_object.setdefault("checksums", []).append(self.checksum_to_dict(chk_sum)) if package.has_optional_field("description"): package_object["description"] = package.description @@ -237,9 +233,8 @@ def create_file_info(self, file, annotations_by_spdx_id): file_object["fileName"] = file.name file_object["SPDXID"] = self.spdx_id(file.spdx_id) - file_object["checksums"] = [] for chk_sum in file.checksums: - file_object["checksums"].append(self.checksum_to_dict(chk_sum)) + file_object.setdefault("checksums",[]).append(self.checksum_to_dict(chk_sum)) if file.has_optional_field("conc_lics"): file_object["licenseConcluded"] = self.license(file.conc_lics) diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index 7a2c1a3a6..4ff728da6 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -124,7 +124,7 @@ def write_file(spdx_file, out): write_value("SPDXID", spdx_file.spdx_id, out) for file_type in spdx_file.file_types: write_file_type(file_type.name, out) - for file_chk_sum in spdx_file.chk_sums: + for file_chk_sum in spdx_file.checksums: write_value("FileChecksum", file_chk_sum.to_tv(), out) if spdx_file.has_optional_field("conc_lics"): if isinstance( diff --git a/tests/data/doc_parse/expected.json b/tests/data/doc_parse/expected.json index 5153d2e9c..0900dc983 100644 --- a/tests/data/doc_parse/expected.json +++ b/tests/data/doc_parse/expected.json @@ -87,10 +87,10 @@ "copyrightText": "Copyright 2010, 2011 Source Auditor Inc.", "licenseComment": null, "notice": null, - "checksum": { + "checksums": [{ "identifier": "SHA1", "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" - }, + }], "licenseInfoInFiles": [ { "type": "Single", diff --git a/tests/data/doc_parse/spdx-expected.json b/tests/data/doc_parse/spdx-expected.json index 2269092e4..05e73da2a 100644 --- a/tests/data/doc_parse/spdx-expected.json +++ b/tests/data/doc_parse/spdx-expected.json @@ -85,10 +85,15 @@ "copyrightText": "Copyright 2010, 2011 Source Auditor Inc.", "licenseComment": null, "notice": null, - "checksum": { - "identifier": "SHA1", - "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" - }, + "checksums": [ + { + "identifier": "SHA1", + "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + }, + { + "identifier": "SHA256", + "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb120000000000000000" + }], "licenseInfoInFiles": [ { "type": "Single", diff --git a/tests/data/doc_write/xml-simple-multi-package.xml b/tests/data/doc_write/xml-simple-multi-package.xml index 4b7e6e85c..7552c6dcc 100644 --- a/tests/data/doc_write/xml-simple-multi-package.xml +++ b/tests/data/doc_write/xml-simple-multi-package.xml @@ -56,4 +56,4 @@ LGPL-2.1-or-later NOASSERTION - \ No newline at end of file + diff --git a/tests/data/doc_write/xml-simple-plus.xml b/tests/data/doc_write/xml-simple-plus.xml index 7fb595fca..82f20efe7 100644 --- a/tests/data/doc_write/xml-simple-plus.xml +++ b/tests/data/doc_write/xml-simple-plus.xml @@ -18,6 +18,10 @@ SOME-SHA1 SHA1 + + SOME-SHA256 + SHA256 + NOASSERTION NOASSERTION LGPL-2.1-or-later @@ -34,6 +38,10 @@ SOME-SHA1 SHA1 + + SOME-SHA256 + SHA256 + NOASSERTION NOASSERTION LGPL-2.1-or-later diff --git a/tests/data/doc_write/xml-simple.xml b/tests/data/doc_write/xml-simple.xml index e10c36bc4..961fb9ad4 100644 --- a/tests/data/doc_write/xml-simple.xml +++ b/tests/data/doc_write/xml-simple.xml @@ -18,6 +18,10 @@ SOME-SHA1 SHA1 + + SOME-SHA256 + SHA256 + NOASSERTION NOASSERTION LGPL-2.1-only @@ -34,6 +38,10 @@ SOME-SHA1 SHA1 + + SOME-SHA256 + SHA256 + NOASSERTION NOASSERTION LGPL-2.1-only diff --git a/tests/data/formats/SPDXJsonExample.json b/tests/data/formats/SPDXJsonExample.json index 7e23c846a..c70308667 100644 --- a/tests/data/formats/SPDXJsonExample.json +++ b/tests/data/formats/SPDXJsonExample.json @@ -75,11 +75,11 @@ "checksums": [ { "checksumValue": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125", - "algorithm": "checksumAlgorithm_sha1" + "algorithm": "SHA1" }, { "checksumValue": "3ab4e1c67a2d28fced849ee1bb76e7391b93f1250000000000000000", - "algorithm": "checksumAlgorithm_sha256" + "algorithm": "SHA256" } ], "fileTypes": [ diff --git a/tests/data/formats/SPDXXmlExample.xml b/tests/data/formats/SPDXXmlExample.xml index f31894e17..b703fb5f2 100644 --- a/tests/data/formats/SPDXXmlExample.xml +++ b/tests/data/formats/SPDXXmlExample.xml @@ -31,7 +31,7 @@ Organization: Linux Foundation 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 - checksumAlgorithm_sha1 + SHA1 Version 0.9.2 (LicenseRef-3 AND LicenseRef-2 AND Apache-2.0 AND MPL-1.1 AND LicenseRef-1 AND LicenseRef-4) @@ -53,7 +53,7 @@ d6a770ba38583ed4bb4525bd96e50461655d2759 - checksumAlgorithm_sha1 + SHA1 https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301 DocumentRef-spdx-tool-2.1 @@ -206,7 +206,7 @@ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 - checksumAlgorithm_sha1 + SHA1 SOURCE TEXT @@ -226,7 +226,11 @@ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. This license is used by Jena 3ab4e1c67a2d28fced849ee1bb76e7391b93f125 - checksumAlgorithm_sha1 + SHA1 + + + 3ab4e1c67a2d28fced849ee1bb76e7391b93f1250000000000000000 + SHA256 ARCHIVE OTHER diff --git a/tests/data/formats/SPDXYamlExample.yaml b/tests/data/formats/SPDXYamlExample.yaml index f6b544ed9..0aeec67e0 100644 --- a/tests/data/formats/SPDXYamlExample.yaml +++ b/tests/data/formats/SPDXYamlExample.yaml @@ -213,8 +213,10 @@ Document: files: - SPDXID: SPDXRef-File1 checksums: - - algorithm: checksumAlgorithm_sha1 + - algorithm: SHA1 checksumValue: 3ab4e1c67a2d28fced849ee1bb76e7391b93f125 + - algorithm: SHA256 + checksumValue: 3ab4e1c67a2d28fced849ee1bb76e7391b93f1250000000000000000 comment: This file belongs to Jena copyrightText: (c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP @@ -233,10 +235,8 @@ Document: - SPDXID: SPDXRef-File2 checksums: - - algorithm: checksumAlgorithm_sha1 + - algorithm: SHA1 checksumValue: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 - - algorithm: checksumAlgorithm_sha256 - checksumValue: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb120000000000000000 copyrightText: Copyright 2010, 2011 Source Auditor Inc. fileTypes: - SOURCE diff --git a/tests/test_document.py b/tests/test_document.py index 52ddf4eae..f1c1ea89c 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -247,7 +247,7 @@ def _get_lgpl_multi_package_doc(self, or_later=False): file1 = File('./some/path/tofile') file1.name = './some/path/tofile' file1.spdx_id = 'SPDXRef-File' - file1.chksum = Algorithm('SHA1', 'SOME-SHA1') + file1.set_checksum(Algorithm('SHA1', 'SOME-SHA1')) file1.conc_lics = NoAssert() file1.copyright = NoAssert() diff --git a/tests/test_jsonyamlxml_writer.py b/tests/test_jsonyamlxml_writer.py index 291ed1386..ff1ebbbbf 100644 --- a/tests/test_jsonyamlxml_writer.py +++ b/tests/test_jsonyamlxml_writer.py @@ -147,7 +147,8 @@ def minimal_document(): def minimal_file(): - file = File(name="Example File", spdx_id="SPDXRef-File", chksum=Algorithm('SHA1', 'SOME-SHA1')) + file = File(name="Example File", spdx_id="SPDXRef-File") + file.set_checksum(Algorithm('SHA1', 'some-sha1-value')) return file diff --git a/tests/test_package.py b/tests/test_package.py index 8438a5503..944f2516d 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -25,7 +25,7 @@ def test_calc_verif_code(self): def test_package_with_non_sha1_check_sum(self): package = Package() - package.checksum = Algorithm("SHA256", '') + package.set_checksum(Algorithm("SHA256", '')) # Make sure that validation still works despite the checksum not being SHA1 messages = [] From f03ae97c6b8c4e33223198d779c46e01746b4cb6 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Wed, 23 Nov 2022 16:34:45 +0100 Subject: [PATCH 04/13] [issue-185] review fixes Signed-off-by: Meret Behrens --- spdx/cli_tools/parser.py | 4 +-- spdx/file.py | 6 ++--- spdx/package.py | 4 +-- spdx/utils.py | 56 +++++++++------------------------------- 4 files changed, 18 insertions(+), 52 deletions(-) diff --git a/spdx/cli_tools/parser.py b/spdx/cli_tools/parser.py index 97d3897e9..8d46baae2 100755 --- a/spdx/cli_tools/parser.py +++ b/spdx/cli_tools/parser.py @@ -51,8 +51,8 @@ def main(file, force): "Package Download Location: {0}".format(package.download_location) ) print("Package Homepage: {0}".format(package.homepage)) - if package.checksum: - print("Package Checksum: {0}".format(package.checksum.value)) + for checksum in doc.package.checksums: + print("Package Checksum: {0}".format(checksum.value)) print("Package Attribution Text: {0}".format(package.attribution_text)) print("Package verification code: {0}".format(package.verif_code)) print( diff --git a/spdx/file.py b/spdx/file.py index 3b05d5b02..da2181111 100644 --- a/spdx/file.py +++ b/spdx/file.py @@ -58,7 +58,7 @@ class File(object): referenced by other elements. Mandatory, one. Type: str. - comment: File comment str, Optional zero or one. - file_types: list of file types. Cardinality 0..* - - chk_sums: list of checksums, there must be a SHA1 hash, at least. + - checksums: List of checksums, there must be a SHA1 hash, at least. - conc_lics: Mandatory one. license.License or utils.NoAssert or utils.SPDXNone. - licenses_in_file: list of licenses found in file, mandatory one or more. document.License or utils.SPDXNone or utils.NoAssert. @@ -202,9 +202,7 @@ def validate_file_types(self, messages): return messages def validate_checksum(self, messages): - if not self.checksums: - return messages - if self.get_checksum() is None: + if self.get_checksum("SHA1") is None: messages.append("At least one file checksum algorithm must be SHA1") return messages diff --git a/spdx/package.py b/spdx/package.py index f2885777f..bddc3d20d 100644 --- a/spdx/package.py +++ b/spdx/package.py @@ -59,7 +59,7 @@ class Package(object): whenever files_analyzed is True or None (omitted) and Must be None (omitted) if files_analyzed is False. However, as a convenience within this library, we allow this to be Optional even when files_analyzed is True/None. - - check_sum: Optional , spdx.checksum.Algorithm. + - checksums: Optional , List of spdx.checksum.Algorithm. - source_info: Optional string. - conc_lics: Mandatory license.License or utils.SPDXNone or utils.NoAssert. @@ -301,7 +301,7 @@ def validate_str_fields(self, fields, optional, messages): def validate_checksum(self, messages): if not self.checksums: return messages - if self.get_checksum() is None: + if self.get_checksum("SHA1") is None: messages.append("At least one package checksum algorithm must be SHA1") return messages diff --git a/spdx/utils.py b/spdx/utils.py index c1ebae9ec..03b526df6 100644 --- a/spdx/utils.py +++ b/spdx/utils.py @@ -211,23 +211,21 @@ def parse(self, data): def calc_verif_code(files: List['File']) -> str: - hashes = [] - - for file_entry in files: - if ( - isinstance(file_entry.chksum, checksum.Algorithm) - and file_entry.chksum.identifier == "SHA1" - ): - sha1 = file_entry.chksum.value + list_of_file_hashes = [] + hash_algo_name = "SHA1" + for f in files: + file_chksum = f.get_checksum(hash_algo_name) + if file_chksum is not None: + file_ch = file_chksum.value else: - sha1 = file_entry.calc_chksum() - hashes.append(sha1) + file_ch = f.calculate_checksum(hash_algo_name) + list_of_file_hashes.append(file_ch) - hashes.sort() + list_of_file_hashes.sort() - sha1 = hashlib.sha1() - sha1.update("".join(hashes).encode("utf-8")) - return sha1.hexdigest() + hasher = hashlib.new(hash_algo_name.lower()) + hasher.update("".join(list_of_file_hashes).encode("utf-8")) + return hasher.hexdigest() def get_files_in_package(package: 'Package', files: List['File'], relationships: List[Relationship]) -> List['File']: @@ -247,33 +245,3 @@ def update_dict_item_with_new_item(current_state: Dict, key: str, item_to_add: s current_state[key] = [item_to_add] elif item_to_add not in current_state[key]: current_state[key].append(item_to_add) - - -def calc_verif_code(files: List['File']) -> str: - hashes = [] - - for file_entry in files: - if ( - isinstance(file_entry.chksum, checksum.Algorithm) - and file_entry.chksum.identifier == "SHA1" - ): - sha1 = file_entry.chksum.value - else: - sha1 = file_entry.calc_chksum() - hashes.append(sha1) - - hashes.sort() - - sha1 = hashlib.sha1() - sha1.update("".join(hashes).encode("utf-8")) - return sha1.hexdigest() - - -def get_files_in_package(package: 'Package', files: List['File'], relationships: List[Relationship]) -> List['File']: - files_in_package = [] - for file in files: - if file.spdx_id in [relationship.related_spdx_element for relationship in relationships - if relationship.relationship_type == "CONTAINS" and relationship.spdx_element_id == package.spdx_id]: - files_in_package.append(file) - - return files_in_package From bdad9d4383ca1b77b598875bea209f2d4bb3b678 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Wed, 23 Nov 2022 16:35:50 +0100 Subject: [PATCH 05/13] [issue-185] add Enum for checksumAlgorithm Signed-off-by: Meret Behrens --- spdx/checksum.py | 45 +++++++++++++++++++++++++++++++-------------- spdx/file.py | 34 +++++++++++++++++++++++----------- spdx/package.py | 32 ++++++++++++++++++++++---------- spdx/parsers/rdf.py | 3 ++- spdx/writers/rdf.py | 3 ++- 5 files changed, 80 insertions(+), 37 deletions(-) diff --git a/spdx/checksum.py b/spdx/checksum.py index 673b35edf..d3bc2a813 100644 --- a/spdx/checksum.py +++ b/spdx/checksum.py @@ -8,26 +8,43 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from enum import Enum, auto -CHECKSUM_ALGORITHMS = ['SHA1', 'SHA256', 'SHA512'] -CHECKSUM_ALGORITHM_FROM_XML_DICT = { - 'checksumAlgorithm_sha1': 'SHA1', - 'checksumAlgorithm_sha256': 'SHA256', - 'checksumAlgorithm_sha512': 'SHA512', -} -CHECKSUM_ALGORITHM_TO_XML_DICT = { - 'SHA1': 'checksumAlgorithm_sha1', - 'SHA256': 'checksumAlgorithm_sha256', - 'SHA512': 'checksumAlgorithm_sha512', -} + +class ChecksumAlgorithmIdentifier(Enum): + SHA1 = auto() + SHA224 = auto() + SHA256 = auto() + SHA384 = auto() + SHA512 = auto() + SHA3_256 = auto() + SHA3_384 = auto() + SHA3_512 = auto() + BLAKE2B_256 = auto() + BLAKE2B_384 = auto() + BLAKE2B_512 = auto() + BLAKE3 = auto() + MD2 = auto() + MD4 = auto() + MD5 = auto() + MD6 = auto() + ADLER32 = auto() + + def checksum_to_rdf(self): + return "checksumAlgorithm_" + self.name.lower() + + @classmethod + def checksum_from_rdf(cls, identifier: str) -> str: + identifier = identifier.split('_')[-1].upper() + return identifier class Algorithm(object): """Generic checksum algorithm.""" - def __init__(self, identifier, value): - if identifier not in CHECKSUM_ALGORITHMS: - raise ValueError('checksum algorithm: {}'.format(identifier)) + def __init__(self, identifier: str, value): + if identifier not in ChecksumAlgorithmIdentifier.__members__: + raise ValueError('Invalid algorithm for Checksum: {}'.format(identifier)) self.identifier = identifier self.value = value diff --git a/spdx/file.py b/spdx/file.py index da2181111..80debf2d3 100644 --- a/spdx/file.py +++ b/spdx/file.py @@ -10,10 +10,11 @@ # limitations under the License. from enum import Enum, auto +import warnings from functools import total_ordering import hashlib -from spdx import checksum +from spdx.checksum import Algorithm, ChecksumAlgorithmIdentifier from spdx import utils from spdx.license import License from spdx.parsers.builderexceptions import SPDXValueError @@ -101,13 +102,21 @@ def __lt__(self, other): @property def chk_sum(self): """ - Backwards compatibility, return first checksum. + Backwards compatibility, return SHA1 checksum. """ + warnings.warn("This property is deprecated. Use get_checksum instead.") return self.get_checksum('SHA1') @chk_sum.setter def chk_sum(self, value): - self.set_checksum(value) + """ + Backwards compatibility, set checksum. + """ + warnings.warn("This property is deprecated. Use set_checksum instead.") + if isinstance(value, str): + self.set_checksum(Algorithm(ChecksumAlgorithmIdentifier.SHA1, value)) + elif isinstance(value, Algorithm): + self.set_checksum(value) def add_lics(self, lics): self.licenses_in_file.append(lics) @@ -207,6 +216,8 @@ def validate_checksum(self, messages): return messages def calculate_checksum(self, hash_algorithm='SHA1'): + if hash_algorithm not in ChecksumAlgorithmIdentifier.__members__: + raise ValueError BUFFER_SIZE = 65536 file_hash = hashlib.new(hash_algorithm.lower()) @@ -219,19 +230,20 @@ def calculate_checksum(self, hash_algorithm='SHA1'): return file_hash.hexdigest() - def get_checksum(self, hash_algorithm='SHA1'): + def get_checksum(self, hash_algorithm: ChecksumAlgorithmIdentifier = ChecksumAlgorithmIdentifier.SHA1) -> Algorithm: for chk_sum in self.checksums: if chk_sum.identifier == hash_algorithm: return chk_sum return None - def set_checksum(self, chk_sum): - if isinstance(chk_sum, checksum.Algorithm): - for file_chk_sum in self.checksums: - if file_chk_sum.identifier == chk_sum.identifier: - file_chk_sum.value = chk_sum.value - return - self.checksums.append(chk_sum) + def set_checksum(self, chk_sum: Algorithm): + if not isinstance(chk_sum, Algorithm): + raise SPDXValueError + for file_chk_sum in self.checksums: + if file_chk_sum.identifier == chk_sum.identifier: + file_chk_sum.value = chk_sum.value + return + self.checksums.append(chk_sum) def has_optional_field(self, field): return bool (getattr(self, field, None)) diff --git a/spdx/package.py b/spdx/package.py index bddc3d20d..fe256a981 100644 --- a/spdx/package.py +++ b/spdx/package.py @@ -8,7 +8,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import warnings from datetime import datetime from enum import Enum from functools import reduce @@ -18,6 +18,8 @@ from spdx import creationinfo from spdx import license from spdx import utils +from spdx.checksum import Algorithm, ChecksumAlgorithmIdentifier +from spdx.parsers.builderexceptions import SPDXValueError from spdx.parsers.loggers import ErrorMessages @@ -122,13 +124,21 @@ def __init__( @property def check_sum(self): """ - Backwards compatibility, return first checksum. + Backwards compatibility, return SHA1 checksum. """ + warnings.warn("This property is deprecated. Use get_checksum instead.") return self.get_checksum('SHA1') @check_sum.setter def check_sum(self, value): - self.set_checksum(value) + """ + Backwards compatibility, set SHA1 checksum. + """ + warnings.warn("This property is deprecated. Use set_checksum instead.") + if isinstance(value, str): + self.set_checksum(Algorithm(ChecksumAlgorithmIdentifier.SHA1, value)) + elif isinstance(value, Algorithm): + self.set_checksum(value) @property def are_files_analyzed(self): @@ -305,19 +315,21 @@ def validate_checksum(self, messages): messages.append("At least one package checksum algorithm must be SHA1") return messages - def get_checksum(self, hash_algorithm='SHA1'): + def get_checksum(self, hash_algorithm: ChecksumAlgorithmIdentifier = ChecksumAlgorithmIdentifier.SHA1): for chk_sum in self.checksums: if chk_sum.identifier == hash_algorithm: return chk_sum return None def set_checksum(self, new_checksum): - if isinstance(new_checksum, checksum.Algorithm): - for c in self.checksums: - if c.identifier == new_checksum.identifier: - c.value = new_checksum.value - return - self.checksums.append(new_checksum) + if not isinstance(new_checksum, Algorithm): + raise SPDXValueError + + for checksum in self.checksums: + if checksum.identifier == new_checksum.identifier: + checksum.value = new_checksum.value + return + self.checksums.append(new_checksum) def has_optional_field(self, field): return bool(getattr(self, field, None)) diff --git a/spdx/parsers/rdf.py b/spdx/parsers/rdf.py index 2b496e9ab..39a2bca29 100644 --- a/spdx/parsers/rdf.py +++ b/spdx/parsers/rdf.py @@ -22,6 +22,7 @@ from spdx import license from spdx import utils from spdx import checksum +from spdx.checksum import ChecksumAlgorithmIdentifier from spdx.parsers.builderexceptions import CardinalityError from spdx.parsers.builderexceptions import SPDXValueError from spdx.parsers.loggers import ErrorMessages @@ -71,7 +72,7 @@ def convert_rdf_checksum_algorithm(algo): ss = algo.split('#') if len(ss) != 2: raise SPDXValueError('Unknown checksum algorithm {}'.format(algo)) - algo = checksum.CHECKSUM_ALGORITHM_FROM_XML_DICT.get(ss[1]) + algo = ChecksumAlgorithmIdentifier.checksum_from_rdf(ss[1]) if algo is None: raise SPDXValueError('Unknown checksum algorithm {}'.format(algo)) return algo diff --git a/spdx/writers/rdf.py b/spdx/writers/rdf.py index cca19edb8..3f5fdf25b 100644 --- a/spdx/writers/rdf.py +++ b/spdx/writers/rdf.py @@ -22,6 +22,7 @@ from spdx import file from spdx import license from spdx import utils +from spdx.checksum import ChecksumAlgorithmIdentifier from spdx.package import Package from spdx.parsers.loggers import ErrorMessages from spdx.relationship import Relationship @@ -48,7 +49,7 @@ def create_checksum_node(self, chksum): """ Return a node representing spdx.checksum. """ - algo = checksum.CHECKSUM_ALGORITHM_TO_XML_DICT.get(chksum.identifier) or 'checksumAlgorithm_sha1' + algo = ChecksumAlgorithmIdentifier[chksum.identifier].checksum_to_rdf() or 'checksumAlgorithm_sha1' chksum_node = BNode() type_triple = (chksum_node, RDF.type, self.spdx_namespace.Checksum) self.graph.add(type_triple) From 9d05af87c56b9174563f65a08c6d1a6d41a7eb90 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Thu, 24 Nov 2022 09:07:57 +0100 Subject: [PATCH 06/13] [issue-185] support multiple checksums for tv-files Signed-off-by: Meret Behrens --- spdx/checksum.py | 2 +- spdx/cli_tools/parser.py | 7 ++++--- spdx/package.py | 2 +- spdx/parsers/lexers/tagvalue.py | 6 +++--- spdx/parsers/tagvaluebuilders.py | 16 ++++++++-------- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/spdx/checksum.py b/spdx/checksum.py index d3bc2a813..c40cc99c1 100644 --- a/spdx/checksum.py +++ b/spdx/checksum.py @@ -43,7 +43,7 @@ class Algorithm(object): """Generic checksum algorithm.""" def __init__(self, identifier: str, value): - if identifier not in ChecksumAlgorithmIdentifier.__members__: + if identifier.upper().replace('-','_') not in ChecksumAlgorithmIdentifier.__members__: raise ValueError('Invalid algorithm for Checksum: {}'.format(identifier)) self.identifier = identifier self.value = value diff --git a/spdx/cli_tools/parser.py b/spdx/cli_tools/parser.py index 8d46baae2..891a3017a 100755 --- a/spdx/cli_tools/parser.py +++ b/spdx/cli_tools/parser.py @@ -51,8 +51,8 @@ def main(file, force): "Package Download Location: {0}".format(package.download_location) ) print("Package Homepage: {0}".format(package.homepage)) - for checksum in doc.package.checksums: - print("Package Checksum: {0}".format(checksum.value)) + for checksum in doc.package.checksums.values(): + print("Package Checksum: {0} {1}".format(checksum.identifier, checksum.value)) print("Package Attribution Text: {0}".format(package.attribution_text)) print("Package verification code: {0}".format(package.verif_code)) print( @@ -82,7 +82,8 @@ def main(file, force): print("\tFile name: {0}".format(f.name)) for file_type in f.file_types: print("\tFile type: {0}".format(file_type.name)) - print("\tFile Checksum: {0}".format(f.chksum.value)) + for file_checksum in f.checksums: + print("\tFile Checksum: {0} {1}".format(file_checksum.identifier, file_checksum.value)) print("\tFile license concluded: {0}".format(f.conc_lics)) print( "\tFile license info in file: {0}".format( diff --git a/spdx/package.py b/spdx/package.py index fe256a981..6be9d5ecc 100644 --- a/spdx/package.py +++ b/spdx/package.py @@ -323,7 +323,7 @@ def get_checksum(self, hash_algorithm: ChecksumAlgorithmIdentifier = ChecksumAlg def set_checksum(self, new_checksum): if not isinstance(new_checksum, Algorithm): - raise SPDXValueError + raise SPDXValueError("Package::Checksum") for checksum in self.checksums: if checksum.identifier == new_checksum.identifier: diff --git a/spdx/parsers/lexers/tagvalue.py b/spdx/parsers/lexers/tagvalue.py index 08762c3ff..ba7aa62a9 100644 --- a/spdx/parsers/lexers/tagvalue.py +++ b/spdx/parsers/lexers/tagvalue.py @@ -151,7 +151,7 @@ def t_text(self, t): def t_text_end(self, t): r"\s*" t.type = "TEXT" - t.value = t.lexer.lexdata[t.lexer.text_start : t.lexer.lexpos] + t.value = t.lexer.lexdata[t.lexer.text_start: t.lexer.lexpos] t.lexer.lineno += t.value.count("\n") t.value = t.value.strip() t.lexer.begin("INITIAL") @@ -165,7 +165,8 @@ def t_text_error(self, t): print("Lexer error in text state") def t_CHKSUM(self, t): - r":\s*(SHA.*):\s*([a-f0-9]{40,128})" + r":\s*(ADLER32|BLAKE2b-256|BLAKE2b-384|BLAKE2b-512|BLAKE3|MD2|MD4|MD5|MD6|" \ + "SHA1|SHA224|SHA256|SHA384|SHA512|SHA3-256|SHA3-384|SHA3-512):\s*([a-f0-9]*)" t.value = t.value[1:].strip() return t @@ -174,7 +175,6 @@ def t_RANGE(self, t): t.value = t.value[1:].strip() return t - def t_DOC_REF_ID(self, t): r":\s*DocumentRef-([A-Za-z0-9\+\.\-]+)" t.value = t.value[1:].strip() diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index 4a3295555..045fd2ab8 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -34,7 +34,8 @@ def checksum_algorithm_from_string(value): - CHECKSUM_RE = re.compile("(SHA1|SHA256|SHA512):\\s*([a-f0-9]*)") + CHECKSUM_RE = re.compile("(ADLER32|BLAKE2b-256|BLAKE2b-384|BLAKE2b-512|BLAKE3|MD2|MD4|MD5|MD6|" \ + "SHA1|SHA224|SHA256|SHA384|SHA512|SHA3-256|SHA3-384|SHA3-512):\\s*([a-f0-9]*)") match = CHECKSUM_RE.match(value) if match: return checksum.Algorithm(identifier=match.group(1), value=match.group(2)) @@ -777,9 +778,8 @@ def set_pkg_verif_code(self, doc, code): def set_pkg_chk_sum(self, doc, chk_sum): """ - Set the package check sum, if not already set. + Set the package checksum, if not already set. chk_sum - A string - Raise CardinalityError if already defined. Raise OrderError if no package previously defined. """ self.assert_package_exists() @@ -1191,14 +1191,14 @@ def set_file_type(self, doc, type_value): def set_file_chksum(self, doc, chksum): """ - Raise OrderError if no package or file defined. - Raise CardinalityError if more than one chksum set. + Raise OrderError if no file defined. + Raise CardinalityError if no SHA1 checksum set. """ - if self.has_package(doc) and self.has_file(doc): + if self.has_file(doc): self.file_chksum_set = False - chk_sums = doc.files[-1].checksums + chk_sums = self.file(doc).checksums chk_sums.append(checksum_algorithm_from_string(chksum)) - doc.files[-1].checksums = chk_sums + self.file(doc).checksums = chk_sums for chk_sum in self.file(doc).checksums: if chk_sum.identifier == 'SHA1': self.file_chksum_set = True From a068c8d6feca6fca17730463b5080dfdc25b24e5 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Thu, 24 Nov 2022 09:30:06 +0100 Subject: [PATCH 07/13] [issue-185] refactor chk_sum to checksum Signed-off-by: Meret Behrens --- spdx/file.py | 19 ++++++++--------- spdx/package.py | 7 +++---- spdx/parsers/jsonyamlxml.py | 32 ++++++++++++++--------------- spdx/parsers/jsonyamlxmlbuilders.py | 20 ++++++++---------- spdx/parsers/rdf.py | 4 ++-- spdx/parsers/rdfbuilders.py | 30 ++++++++++++++------------- spdx/parsers/tagvalue.py | 4 ++-- spdx/parsers/tagvaluebuilders.py | 19 ++++++++--------- spdx/writers/jsonyamlxml.py | 8 ++++---- spdx/writers/tagvalue.py | 4 ++-- tests/test_builder.py | 2 +- 11 files changed, 73 insertions(+), 76 deletions(-) diff --git a/spdx/file.py b/spdx/file.py index 80debf2d3..bf2547457 100644 --- a/spdx/file.py +++ b/spdx/file.py @@ -231,19 +231,18 @@ def calculate_checksum(self, hash_algorithm='SHA1'): return file_hash.hexdigest() def get_checksum(self, hash_algorithm: ChecksumAlgorithmIdentifier = ChecksumAlgorithmIdentifier.SHA1) -> Algorithm: - for chk_sum in self.checksums: - if chk_sum.identifier == hash_algorithm: - return chk_sum - return None + for checksum in self.checksums: + if checksum.identifier == hash_algorithm: + return checksum - def set_checksum(self, chk_sum: Algorithm): - if not isinstance(chk_sum, Algorithm): + def set_checksum(self, checksum: Algorithm): + if not isinstance(checksum, Algorithm): raise SPDXValueError - for file_chk_sum in self.checksums: - if file_chk_sum.identifier == chk_sum.identifier: - file_chk_sum.value = chk_sum.value + for file_checksum in self.checksums: + if file_checksum.identifier == checksum.identifier: + file_checksum.value = checksum.value return - self.checksums.append(chk_sum) + self.checksums.append(checksum) def has_optional_field(self, field): return bool (getattr(self, field, None)) diff --git a/spdx/package.py b/spdx/package.py index 6be9d5ecc..cc028e88e 100644 --- a/spdx/package.py +++ b/spdx/package.py @@ -14,7 +14,6 @@ from functools import reduce from typing import Optional -from spdx import checksum from spdx import creationinfo from spdx import license from spdx import utils @@ -316,9 +315,9 @@ def validate_checksum(self, messages): return messages def get_checksum(self, hash_algorithm: ChecksumAlgorithmIdentifier = ChecksumAlgorithmIdentifier.SHA1): - for chk_sum in self.checksums: - if chk_sum.identifier == hash_algorithm: - return chk_sum + for checksum in self.checksums: + if checksum.identifier == hash_algorithm: + return checksum return None def set_checksum(self, new_checksum): diff --git a/spdx/parsers/jsonyamlxml.py b/spdx/parsers/jsonyamlxml.py index e005e1720..d9baf3748 100644 --- a/spdx/parsers/jsonyamlxml.py +++ b/spdx/parsers/jsonyamlxml.py @@ -1093,24 +1093,24 @@ def _handle_file_dependency(self, file_dependency): return None return None - def parse_file_chksum(self, file_chksum): + def parse_file_chksum(self, file_checksum): """ Parse File checksums - - file_chksum: Python str/unicode + - file_checksum: Python str/unicode """ - if isinstance(file_chksum, list): - for chk_sum in file_chksum: - self.builder.set_file_chksum(self.document, chk_sum) + if isinstance(file_checksum, list): + for checksum in file_checksum: + self.builder.set_file_checksum(self.document, checksum) return True - if isinstance(file_chksum, str): + if isinstance(file_checksum, str): try: - return self.builder.set_file_chksum(self.document, file_chksum) + return self.builder.set_file_checksum(self.document, file_checksum) except CardinalityError: self.more_than_one_error("FILE_CHECKSUM") except OrderError: self.order_error("FILE_CHECKSUM", "FILE_NAME") else: - self.value_error("FILE_CHECKSUM", file_chksum) + self.value_error("FILE_CHECKSUM", file_checksum) def parse_files(self, files: List[Dict]) -> None: if files is None: @@ -1592,24 +1592,24 @@ def parse_pkg_files(self, pkg_has_files: List[str], method_to_parse_relationship elif pkg_has_files is not None: self.value_error("PKG_HAS_FILES", pkg_has_files) - def parse_pkg_chksum(self, pkg_chksum): + def parse_pkg_chksum(self, pkg_checksum): """ Parse Package checksum - pkg_chksum: Python str/unicode """ - if isinstance(pkg_chksum, list): - for chk_sum in pkg_chksum: - self.builder.set_pkg_chk_sum(self.document, chk_sum) + if isinstance(pkg_checksum, list): + for checksum in pkg_checksum: + self.builder.set_pkg_checksum(self.document, checksum) return True - if isinstance(pkg_chksum, str): + if isinstance(pkg_checksum, str): try: - return self.builder.set_pkg_chk_sum(self.document, pkg_chksum) + return self.builder.set_pkg_checksum(self.document, pkg_checksum) except CardinalityError: self.more_than_one_error("PKG_CHECKSUM") except OrderError: self.order_error("PKG_CHECKSUM", "PKG_NAME") - elif pkg_chksum is not None: - self.value_error("PKG_CHECKSUM", pkg_chksum) + elif pkg_checksum is not None: + self.value_error("PKG_CHECKSUM", pkg_checksum) def parse_package_external_refs(self, external_refs: List[Dict]): if external_refs is None: diff --git a/spdx/parsers/jsonyamlxmlbuilders.py b/spdx/parsers/jsonyamlxmlbuilders.py index 89e1b44a4..7e521a35e 100644 --- a/spdx/parsers/jsonyamlxmlbuilders.py +++ b/spdx/parsers/jsonyamlxmlbuilders.py @@ -12,7 +12,7 @@ from spdx.parsers import rdfbuilders from spdx.parsers import tagvaluebuilders from spdx.parsers import validations -from spdx import checksum +from spdx.checksum import Algorithm from spdx.parsers.builderexceptions import SPDXValueError from spdx.parsers.builderexceptions import CardinalityError from spdx.parsers.builderexceptions import OrderError @@ -160,23 +160,21 @@ class FileBuilder(rdfbuilders.FileBuilder): def __init__(self): super(FileBuilder, self).__init__() - def set_file_chksum(self, doc, chk_sum): + def set_file_checksum(self, doc, checksum): """ Set the file check sum, if not already set. - chk_sum - A string + checksum - A string Raise CardinalityError if already defined. Raise OrderError if no package previously defined. """ if self.has_package(doc) and self.has_file(doc): - if isinstance(chk_sum, dict): - - #algo = checksum.CHECKSUM_ALGORITHM_FROM_XML_DICT.get(chk_sum.get('algorithm') or 'SHA1') - algo = chk_sum.get('algorithm') or 'SHA1' - self.file(doc).set_checksum(checksum.Algorithm(algo, chk_sum.get('checksumValue'))) - elif isinstance(chk_sum, checksum.Algorithm): - self.file(doc).set_checksum(chk_sum) + if isinstance(checksum, dict): + algo = checksum.get('algorithm') or 'SHA1' + self.file(doc).set_checksum(Algorithm(algo, checksum.get('checksumValue'))) + elif isinstance(checksum, Algorithm): + self.file(doc).set_checksum(checksum) else: - self.file(doc).set_checksum(checksum.Algorithm("SHA1", chk_sum)) + self.file(doc).set_checksum(Algorithm("SHA1", checksum)) return True def set_file_notice(self, doc, text): diff --git a/spdx/parsers/rdf.py b/spdx/parsers/rdf.py index 39a2bca29..668602829 100644 --- a/spdx/parsers/rdf.py +++ b/spdx/parsers/rdf.py @@ -525,7 +525,7 @@ def p_pkg_chk_sum(self, p_term, predicate): ): algo = convert_rdf_checksum_algorithm(str(algo)) chk_sum = checksum.Algorithm(str(algo), str(value)) - self.builder.set_pkg_chk_sum(self.doc, chk_sum) + self.builder.set_pkg_checksum(self.doc, chk_sum) def p_pkg_homepg(self, p_term, predicate): @@ -797,7 +797,7 @@ def p_file_chk_sum(self, f_term, predicate): ): algo = convert_rdf_checksum_algorithm(str(algo)) chk_sum = checksum.Algorithm(str(algo), str(value)) - self.builder.set_file_chksum(self.doc, chk_sum) + self.builder.set_file_checksum(self.doc, chk_sum) except CardinalityError: self.more_than_one_error("File checksum") diff --git a/spdx/parsers/rdfbuilders.py b/spdx/parsers/rdfbuilders.py index 7da702f2b..3a9c38386 100644 --- a/spdx/parsers/rdfbuilders.py +++ b/spdx/parsers/rdfbuilders.py @@ -11,15 +11,17 @@ import re -from spdx import checksum, file +from spdx import file from spdx import license from spdx import package from spdx import version +from spdx.checksum import Algorithm from spdx.parsers.builderexceptions import CardinalityError from spdx.parsers.builderexceptions import OrderError from spdx.parsers.builderexceptions import SPDXValueError from spdx.parsers import tagvaluebuilders from spdx.parsers import validations +from spdx.parsers.rdf import convert_rdf_checksum_algorithm class DocBuilder(object): @@ -141,7 +143,7 @@ def set_chksum(self, doc, chk_sum): chk_sum - The checksum value in the form of a string. """ if chk_sum: - doc.ext_document_references[-1].check_sum = checksum.Algorithm( + doc.ext_document_references[-1].check_sum = Algorithm( "SHA1", chk_sum ) else: @@ -184,7 +186,7 @@ class PackageBuilder(tagvaluebuilders.PackageBuilder): def __init__(self): super(PackageBuilder, self).__init__() - def set_pkg_chk_sum(self, doc, chk_sum): + def set_pkg_checksum(self, doc, checksum): """ Set the package check sum, if not already set. chk_sum - A string @@ -193,14 +195,14 @@ def set_pkg_chk_sum(self, doc, chk_sum): """ self.assert_package_exists() self.package_chk_sum_set = True - if isinstance(chk_sum, dict): - algo = chk_sum.get('algorithm') or 'SHA1' + if isinstance(checksum, dict): + algo = checksum.get('algorithm') or 'SHA1' if algo.startswith('checksumAlgorithm_'): - algo = checksum.CHECKSUM_ALGORITHM_FROM_XML_DICT.get(algo) or 'SHA1' - doc.packages[-1].set_checksum(checksum.Algorithm(identifier=algo, - value=chk_sum.get('checksumValue'))) - elif isinstance(chk_sum, checksum.Algorithm): - doc.packages[-1].set_checksum(chk_sum) + algo = convert_rdf_checksum_algorithm(algo) or 'SHA1' + doc.packages[-1].set_checksum(Algorithm(identifier=algo, + value=checksum.get('checksumValue'))) + elif isinstance(checksum, Algorithm): + doc.packages[-1].set_checksum(checksum) def set_pkg_source_info(self, doc, text): """ @@ -384,7 +386,7 @@ class FileBuilder(tagvaluebuilders.FileBuilder): def __init__(self): super(FileBuilder, self).__init__() - def set_file_chksum(self, doc, chk_sum): + def set_file_checksum(self, doc, chk_sum): """ Set the file check sum, if not already set. chk_sum - A string @@ -393,12 +395,12 @@ def set_file_chksum(self, doc, chk_sum): """ if self.has_package(doc) and self.has_file(doc): if isinstance(chk_sum, dict): - self.file(doc).set_checksum(checksum.Algorithm(chk_sum.get('algorithm'), + self.file(doc).set_checksum(Algorithm(chk_sum.get('algorithm'), chk_sum.get('checksumValue'))) - elif isinstance(chk_sum, checksum.Algorithm): + elif isinstance(chk_sum, Algorithm): self.file(doc).set_checksum(chk_sum) else: - self.file(doc).set_checksum(checksum.Algorithm("SHA1", chk_sum)) + self.file(doc).set_checksum(Algorithm("SHA1", chk_sum)) return True def set_file_license_comment(self, doc, text): diff --git a/spdx/parsers/tagvalue.py b/spdx/parsers/tagvalue.py index 9444db1e6..ec83e92c7 100644 --- a/spdx/parsers/tagvalue.py +++ b/spdx/parsers/tagvalue.py @@ -650,7 +650,7 @@ def p_file_chksum_1(self, p): """file_chksum : FILE_CHKSUM CHKSUM""" try: value = p[2] - self.builder.set_file_chksum(self.document, value) + self.builder.set_file_checksum(self.document, value) except OrderError: self.order_error("FileChecksum", "FileName", p.lineno(1)) except CardinalityError: @@ -934,7 +934,7 @@ def p_pkg_chksum_1(self, p): """pkg_chksum : PKG_CHKSUM CHKSUM""" try: value = p[2] - self.builder.set_pkg_chk_sum(self.document, value) + self.builder.set_pkg_checksum(self.document, value) except OrderError: self.order_error("PackageChecksum", "PackageFileName", p.lineno(1)) except CardinalityError: diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index 045fd2ab8..bb478d776 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -24,6 +24,7 @@ from spdx import snippet from spdx import utils from spdx import version +from spdx.checksum import Algorithm from spdx.document import ExternalDocumentRef from spdx.package import PackagePurpose @@ -38,7 +39,7 @@ def checksum_algorithm_from_string(value): "SHA1|SHA224|SHA256|SHA384|SHA512|SHA3-256|SHA3-384|SHA3-512):\\s*([a-f0-9]*)") match = CHECKSUM_RE.match(value) if match: - return checksum.Algorithm(identifier=match.group(1), value=match.group(2)) + return Algorithm(identifier=match.group(1), value=match.group(2)) else: return None @@ -776,15 +777,15 @@ def set_pkg_verif_code(self, doc, code): ).split(",") return True - def set_pkg_chk_sum(self, doc, chk_sum): + def set_pkg_checksum(self, doc, checksum): """ Set the package checksum, if not already set. - chk_sum - A string + checksum - A string Raise OrderError if no package previously defined. """ self.assert_package_exists() self.package_chk_sum_set = True - doc.packages[-1].set_checksum(checksum_algorithm_from_string(chk_sum)) + doc.packages[-1].set_checksum(checksum_algorithm_from_string(checksum)) return True def set_pkg_source_info(self, doc, text): @@ -1189,18 +1190,16 @@ def set_file_type(self, doc, type_value): spdx_file.file_types.append(file_type) - def set_file_chksum(self, doc, chksum): + def set_file_checksum(self, doc, checksum): """ Raise OrderError if no file defined. Raise CardinalityError if no SHA1 checksum set. """ if self.has_file(doc): self.file_chksum_set = False - chk_sums = self.file(doc).checksums - chk_sums.append(checksum_algorithm_from_string(chksum)) - self.file(doc).checksums = chk_sums - for chk_sum in self.file(doc).checksums: - if chk_sum.identifier == 'SHA1': + self.file(doc).checksums.append(checksum_algorithm_from_string(checksum)) + for file_checksum in self.file(doc).checksums: + if file_checksum.identifier == 'SHA1': self.file_chksum_set = True if not self.file_chksum_set: raise CardinalityError("File::CheckSum") diff --git a/spdx/writers/jsonyamlxml.py b/spdx/writers/jsonyamlxml.py index cfbdaae5e..6e0909352 100644 --- a/spdx/writers/jsonyamlxml.py +++ b/spdx/writers/jsonyamlxml.py @@ -169,8 +169,8 @@ def create_package_info(self, package, annotations_by_spdx_id): if package.has_optional_field("originator"): package_object["originator"] = package.originator.to_value() - for chk_sum in package.checksums: - package_object.setdefault("checksums", []).append(self.checksum_to_dict(chk_sum)) + for checksum in package.checksums: + package_object.setdefault("checksums", []).append(self.checksum_to_dict(checksum)) if package.has_optional_field("description"): package_object["description"] = package.description @@ -233,8 +233,8 @@ def create_file_info(self, file, annotations_by_spdx_id): file_object["fileName"] = file.name file_object["SPDXID"] = self.spdx_id(file.spdx_id) - for chk_sum in file.checksums: - file_object.setdefault("checksums",[]).append(self.checksum_to_dict(chk_sum)) + for checksum in file.checksums: + file_object.setdefault("checksums",[]).append(self.checksum_to_dict(checksum)) if file.has_optional_field("conc_lics"): file_object["licenseConcluded"] = self.license(file.conc_lics) diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index 4ff728da6..93f57c45d 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -124,8 +124,8 @@ def write_file(spdx_file, out): write_value("SPDXID", spdx_file.spdx_id, out) for file_type in spdx_file.file_types: write_file_type(file_type.name, out) - for file_chk_sum in spdx_file.checksums: - write_value("FileChecksum", file_chk_sum.to_tv(), out) + for file_checksum in spdx_file.checksums: + write_value("FileChecksum", file_checksum.to_tv(), out) if spdx_file.has_optional_field("conc_lics"): if isinstance( spdx_file.conc_lics, (license.LicenseConjunction, license.LicenseDisjunction) diff --git a/tests/test_builder.py b/tests/test_builder.py index 30719e6a6..bac2aedda 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -516,7 +516,7 @@ def test_pkg_verif_order(self): @testing_utils.raises(builders.OrderError) def test_pkg_chksum_order(self): - self.builder.set_pkg_chk_sum(self.document, "some code") + self.builder.set_pkg_checksum(self.document, "some code") @testing_utils.raises(builders.OrderError) def test_pkg_source_info_order(self): From 1b0f04ea1635aa63cb8be590ef45b2cbb3941968 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Thu, 24 Nov 2022 10:43:33 +0100 Subject: [PATCH 08/13] [issue-185] add mult. checksums support for jsonyamlxml Signed-off-by: Meret Behrens --- spdx/file.py | 2 +- spdx/package.py | 2 +- spdx/parsers/jsonyamlxml.py | 34 +++++++++++++++-------------- spdx/parsers/jsonyamlxmlbuilders.py | 8 +++---- spdx/writers/jsonyamlxml.py | 2 +- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/spdx/file.py b/spdx/file.py index bf2547457..28bf7ec97 100644 --- a/spdx/file.py +++ b/spdx/file.py @@ -245,4 +245,4 @@ def set_checksum(self, checksum: Algorithm): self.checksums.append(checksum) def has_optional_field(self, field): - return bool (getattr(self, field, None)) + return bool(getattr(self, field, None)) diff --git a/spdx/package.py b/spdx/package.py index cc028e88e..673dfef10 100644 --- a/spdx/package.py +++ b/spdx/package.py @@ -320,7 +320,7 @@ def get_checksum(self, hash_algorithm: ChecksumAlgorithmIdentifier = ChecksumAlg return checksum return None - def set_checksum(self, new_checksum): + def set_checksum(self, new_checksum: Algorithm): if not isinstance(new_checksum, Algorithm): raise SPDXValueError("Package::Checksum") diff --git a/spdx/parsers/jsonyamlxml.py b/spdx/parsers/jsonyamlxml.py index d9baf3748..815cba485 100644 --- a/spdx/parsers/jsonyamlxml.py +++ b/spdx/parsers/jsonyamlxml.py @@ -823,7 +823,7 @@ def parse_file(self, file): self.parse_file_attribution_text(file.get("attributionTexts")) self.parse_file_dependencies(file.get("fileDependencies")) self.parse_annotations(file.get("annotations"), spdx_id=file.get("SPDXID")) - self.parse_file_chksum(file.get("checksums")) + self.parse_file_checksums(file.get("checksums")) else: self.value_error("FILE", file) @@ -1093,24 +1093,25 @@ def _handle_file_dependency(self, file_dependency): return None return None - def parse_file_chksum(self, file_checksum): + def parse_file_checksums(self, file_checksums): """ Parse File checksums - - file_checksum: Python str/unicode + - file_checksum: Python List """ - if isinstance(file_checksum, list): - for checksum in file_checksum: + if isinstance(file_checksums, list): + for checksum in file_checksums: self.builder.set_file_checksum(self.document, checksum) return True - if isinstance(file_checksum, str): + if isinstance(file_checksums, str): + # kept for backwards compatibility try: - return self.builder.set_file_checksum(self.document, file_checksum) + return self.builder.set_file_checksum(self.document, file_checksums) except CardinalityError: self.more_than_one_error("FILE_CHECKSUM") except OrderError: self.order_error("FILE_CHECKSUM", "FILE_NAME") else: - self.value_error("FILE_CHECKSUM", file_checksum) + self.value_error("FILE_CHECKSUM", file_checksums) def parse_files(self, files: List[Dict]) -> None: if files is None: @@ -1162,7 +1163,7 @@ def parse_package(self, package: Package, method_to_parse_relationship: Callable self.parse_annotations(package.get("annotations"), spdx_id=package.get("SPDXID")) self.parse_pkg_attribution_text(package.get("attributionTexts")) self.parse_pkg_files(package.get("hasFiles"), method_to_parse_relationship) - self.parse_pkg_chksum(package.get("checksums")) + self.parse_pkg_checksums(package.get("checksums")) self.parse_package_external_refs(package.get("externalRefs")) self.parse_primary_package_purpose(package.get("primaryPackagePurpose")) self.parse_release_date(package.get("releaseDate")) @@ -1592,24 +1593,25 @@ def parse_pkg_files(self, pkg_has_files: List[str], method_to_parse_relationship elif pkg_has_files is not None: self.value_error("PKG_HAS_FILES", pkg_has_files) - def parse_pkg_chksum(self, pkg_checksum): + def parse_pkg_checksums(self, pkg_checksums): """ Parse Package checksum - pkg_chksum: Python str/unicode """ - if isinstance(pkg_checksum, list): - for checksum in pkg_checksum: + if isinstance(pkg_checksums, list): + for checksum in pkg_checksums: self.builder.set_pkg_checksum(self.document, checksum) return True - if isinstance(pkg_checksum, str): + if isinstance(pkg_checksums, str): + # kept for backwards compatibility try: - return self.builder.set_pkg_checksum(self.document, pkg_checksum) + return self.builder.set_pkg_checksum(self.document, pkg_checksums) except CardinalityError: self.more_than_one_error("PKG_CHECKSUM") except OrderError: self.order_error("PKG_CHECKSUM", "PKG_NAME") - elif pkg_checksum is not None: - self.value_error("PKG_CHECKSUM", pkg_checksum) + elif pkg_checksums is not None: + self.value_error("PKG_CHECKSUM", pkg_checksums) def parse_package_external_refs(self, external_refs: List[Dict]): if external_refs is None: diff --git a/spdx/parsers/jsonyamlxmlbuilders.py b/spdx/parsers/jsonyamlxmlbuilders.py index 7e521a35e..189e7e2e1 100644 --- a/spdx/parsers/jsonyamlxmlbuilders.py +++ b/spdx/parsers/jsonyamlxmlbuilders.py @@ -162,12 +162,10 @@ def __init__(self): def set_file_checksum(self, doc, checksum): """ - Set the file check sum, if not already set. + Set the file checksum. checksum - A string - Raise CardinalityError if already defined. - Raise OrderError if no package previously defined. """ - if self.has_package(doc) and self.has_file(doc): + if self.has_file(doc): if isinstance(checksum, dict): algo = checksum.get('algorithm') or 'SHA1' self.file(doc).set_checksum(Algorithm(algo, checksum.get('checksumValue'))) @@ -183,7 +181,7 @@ def set_file_notice(self, doc, text): Raise OrderError if no package or file defined. Raise CardinalityError if more than one. """ - if self.has_package(doc) and self.has_file(doc): + if self.has_file(doc): self.file_notice_set = True self.file(doc).notice = text return True diff --git a/spdx/writers/jsonyamlxml.py b/spdx/writers/jsonyamlxml.py index 6e0909352..e62056da7 100644 --- a/spdx/writers/jsonyamlxml.py +++ b/spdx/writers/jsonyamlxml.py @@ -234,7 +234,7 @@ def create_file_info(self, file, annotations_by_spdx_id): file_object["fileName"] = file.name file_object["SPDXID"] = self.spdx_id(file.spdx_id) for checksum in file.checksums: - file_object.setdefault("checksums",[]).append(self.checksum_to_dict(checksum)) + file_object.setdefault("checksums", []).append(self.checksum_to_dict(checksum)) if file.has_optional_field("conc_lics"): file_object["licenseConcluded"] = self.license(file.conc_lics) From f99ac9683ad16e1ae86c73f0e7ead02010a37fb9 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Thu, 24 Nov 2022 11:16:15 +0100 Subject: [PATCH 09/13] [issue-185] review comments rdf Signed-off-by: Meret Behrens --- spdx/checksum.py | 7 ++++++- spdx/parsers/rdf.py | 39 ++++++++++++++++--------------------- spdx/parsers/rdfbuilders.py | 15 +++++++------- spdx/writers/rdf.py | 18 ++++++++--------- 4 files changed, 40 insertions(+), 39 deletions(-) diff --git a/spdx/checksum.py b/spdx/checksum.py index c40cc99c1..b8350a4bf 100644 --- a/spdx/checksum.py +++ b/spdx/checksum.py @@ -10,6 +10,9 @@ # limitations under the License. from enum import Enum, auto +Exceptions_rdf = {"SHA3256": "SHA3_256", "SHA3384": "SHA3_384", "SHA3512": "SHA3_512", "BLAKE2B256": "BLAKE2B_256", + "BLAKE2B384": "BLAKE2B_384", "BLAKE2V512": "BLAKE2B_512"} + class ChecksumAlgorithmIdentifier(Enum): SHA1 = auto() @@ -36,6 +39,8 @@ def checksum_to_rdf(self): @classmethod def checksum_from_rdf(cls, identifier: str) -> str: identifier = identifier.split('_')[-1].upper() + if identifier in Exceptions_rdf: + return Exceptions_rdf[identifier] return identifier @@ -43,7 +48,7 @@ class Algorithm(object): """Generic checksum algorithm.""" def __init__(self, identifier: str, value): - if identifier.upper().replace('-','_') not in ChecksumAlgorithmIdentifier.__members__: + if identifier.upper().replace('-', '_') not in ChecksumAlgorithmIdentifier.__members__: raise ValueError('Invalid algorithm for Checksum: {}'.format(identifier)) self.identifier = identifier self.value = value diff --git a/spdx/parsers/rdf.py b/spdx/parsers/rdf.py index 668602829..389b5ca5f 100644 --- a/spdx/parsers/rdf.py +++ b/spdx/parsers/rdf.py @@ -21,8 +21,7 @@ from spdx import document from spdx import license from spdx import utils -from spdx import checksum -from spdx.checksum import ChecksumAlgorithmIdentifier +from spdx.checksum import Algorithm, ChecksumAlgorithmIdentifier from spdx.parsers.builderexceptions import CardinalityError from spdx.parsers.builderexceptions import SPDXValueError from spdx.parsers.loggers import ErrorMessages @@ -364,7 +363,7 @@ def parse_package(self, p_term): self.p_pkg_down_loc(p_term, self.spdx_namespace["downloadLocation"]) self.p_pkg_files_analyzed(p_term, self.spdx_namespace["filesAnalyzed"]) self.p_pkg_homepg(p_term, self.doap_namespace["homepage"]) - self.p_pkg_chk_sum(p_term, self.spdx_namespace["checksum"]) + self.p_pkg_checksum(p_term, self.spdx_namespace["checksum"]) self.p_pkg_src_info(p_term, self.spdx_namespace["sourceInfo"]) self.p_pkg_verif_code(p_term, self.spdx_namespace["packageVerificationCode"]) self.p_pkg_attribution_text(p_term, self.spdx_namespace["attributionText"]) @@ -515,7 +514,7 @@ def p_pkg_src_info(self, p_term, predicate): self.more_than_one_error("package source info") break - def p_pkg_chk_sum(self, p_term, predicate): + def p_pkg_checksum(self, p_term, predicate): for _s, _p, pkg_checksum in self.graph.triples((p_term, predicate, None)): for _, _, value in self.graph.triples( (pkg_checksum, self.spdx_namespace["checksumValue"], None) @@ -523,10 +522,9 @@ def p_pkg_chk_sum(self, p_term, predicate): for _, _, algo in self.graph.triples( (pkg_checksum, self.spdx_namespace["algorithm"], None) ): - algo = convert_rdf_checksum_algorithm(str(algo)) - chk_sum = checksum.Algorithm(str(algo), str(value)) - self.builder.set_pkg_checksum(self.doc, chk_sum) - + algorithm_identifier = convert_rdf_checksum_algorithm(str(algo)) + checksum = Algorithm(algorithm_identifier, str(value)) + self.builder.set_pkg_checksum(self.doc, checksum) def p_pkg_homepg(self, p_term, predicate): for _s, _p, o in self.graph.triples((p_term, predicate, None)): @@ -629,7 +627,7 @@ def parse_file(self, f_term): self.p_file_spdx_id(f_term, self.spdx_namespace["File"]) self.p_file_type(f_term, self.spdx_namespace["fileType"]) - self.p_file_chk_sum(f_term, self.spdx_namespace["checksum"]) + self.p_file_checksum(f_term, self.spdx_namespace["checksum"]) self.p_file_lic_conc(f_term, self.spdx_namespace["licenseConcluded"]) self.p_file_lic_info(f_term, self.spdx_namespace["licenseInfoInFile"]) self.p_file_comments_on_lics(f_term, self.spdx_namespace["licenseComments"]) @@ -783,23 +781,20 @@ def p_file_type(self, f_term, predicate): except CardinalityError: self.more_than_one_error("file type") - def p_file_chk_sum(self, f_term, predicate): + def p_file_checksum(self, f_term, predicate): """ Set file checksum. """ - try: - for _s, _p, file_checksum in self.graph.triples((f_term, predicate, None)): - for _, _, value in self.graph.triples( - (file_checksum, self.spdx_namespace["checksumValue"], None) + for _s, _p, file_checksum in self.graph.triples((f_term, predicate, None)): + for _, _, value in self.graph.triples( + (file_checksum, self.spdx_namespace["checksumValue"], None) + ): + for _, _, algo in self.graph.triples( + (file_checksum, self.spdx_namespace["algorithm"], None) ): - for _, _, algo in self.graph.triples( - (file_checksum, self.spdx_namespace["algorithm"], None) - ): - algo = convert_rdf_checksum_algorithm(str(algo)) - chk_sum = checksum.Algorithm(str(algo), str(value)) - self.builder.set_file_checksum(self.doc, chk_sum) - except CardinalityError: - self.more_than_one_error("File checksum") + algorithm_identifier = convert_rdf_checksum_algorithm(str(algo)) + checksum = Algorithm(algorithm_identifier, str(value)) + self.builder.set_file_checksum(self.doc, checksum) def p_file_lic_conc(self, f_term, predicate): """ diff --git a/spdx/parsers/rdfbuilders.py b/spdx/parsers/rdfbuilders.py index 3a9c38386..ef812f4b3 100644 --- a/spdx/parsers/rdfbuilders.py +++ b/spdx/parsers/rdfbuilders.py @@ -10,6 +10,7 @@ # limitations under the License. import re +from typing import Dict from spdx import file from spdx import license @@ -186,23 +187,23 @@ class PackageBuilder(tagvaluebuilders.PackageBuilder): def __init__(self): super(PackageBuilder, self).__init__() - def set_pkg_checksum(self, doc, checksum): + def set_pkg_checksum(self, doc, checksum: [Algorithm, Dict]): """ - Set the package check sum, if not already set. - chk_sum - A string - Raise CardinalityError if already defined. + Set the package checksum. + checksum - A string + Raise SPDXValueError if checksum type invalid. Raise OrderError if no package previously defined. """ self.assert_package_exists() - self.package_chk_sum_set = True if isinstance(checksum, dict): algo = checksum.get('algorithm') or 'SHA1' if algo.startswith('checksumAlgorithm_'): algo = convert_rdf_checksum_algorithm(algo) or 'SHA1' - doc.packages[-1].set_checksum(Algorithm(identifier=algo, - value=checksum.get('checksumValue'))) + doc.packages[-1].set_checksum(Algorithm(identifier=algo, value=checksum.get('checksumValue'))) elif isinstance(checksum, Algorithm): doc.packages[-1].set_checksum(checksum) + else: + raise SPDXValueError("Invalid value for package checksum.") def set_pkg_source_info(self, doc, text): """ diff --git a/spdx/writers/rdf.py b/spdx/writers/rdf.py index 3f5fdf25b..42bc8667a 100644 --- a/spdx/writers/rdf.py +++ b/spdx/writers/rdf.py @@ -17,7 +17,7 @@ from rdflib import RDFS from rdflib import URIRef from rdflib.compare import to_isomorphic -from spdx import checksum + from spdx import config from spdx import file from spdx import license @@ -45,27 +45,27 @@ def __init__(self, document, out): self.spdx_namespace = Namespace("http://spdx.org/rdf/terms#") self.graph = Graph() - def create_checksum_node(self, chksum): + def create_checksum_node(self, checksum): """ Return a node representing spdx.checksum. """ - algo = ChecksumAlgorithmIdentifier[chksum.identifier].checksum_to_rdf() or 'checksumAlgorithm_sha1' - chksum_node = BNode() - type_triple = (chksum_node, RDF.type, self.spdx_namespace.Checksum) + algo = ChecksumAlgorithmIdentifier[checksum.identifier].checksum_to_rdf() or 'checksumAlgorithm_sha1' + checksum_node = BNode() + type_triple = (checksum_node, RDF.type, self.spdx_namespace.Checksum) self.graph.add(type_triple) algorithm_triple = ( - chksum_node, + checksum_node, self.spdx_namespace.algorithm, Literal('http://spdx.org/rdf/terms#' + algo), ) self.graph.add(algorithm_triple) value_triple = ( - chksum_node, + checksum_node, self.spdx_namespace.checksumValue, - Literal(chksum.value), + Literal(checksum.value), ) self.graph.add(value_triple) - return chksum_node + return checksum_node def to_special_value(self, value): """ From b5f647c6f75666fca53c5c07c5c480fc55f3aabe Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Thu, 24 Nov 2022 12:00:09 +0100 Subject: [PATCH 10/13] [issue-185] refactor checksums in package and file to dicts Signed-off-by: Meret Behrens --- spdx/file.py | 27 +++++++++++-------------- spdx/package.py | 26 ++++++++++-------------- spdx/parsers/tagvaluebuilders.py | 10 +++------ spdx/writers/jsonyamlxml.py | 4 ++-- spdx/writers/rdf.py | 2 +- spdx/writers/tagvalue.py | 4 ++-- tests/data/doc_parse/SBOMexpected.json | 12 +++++------ tests/data/doc_parse/expected.json | 4 ++-- tests/data/doc_parse/spdx-expected.json | 4 ++-- tests/utils_test.py | 4 ++-- 10 files changed, 43 insertions(+), 54 deletions(-) diff --git a/spdx/file.py b/spdx/file.py index 28bf7ec97..219c9d6bd 100644 --- a/spdx/file.py +++ b/spdx/file.py @@ -59,7 +59,8 @@ class File(object): referenced by other elements. Mandatory, one. Type: str. - comment: File comment str, Optional zero or one. - file_types: list of file types. Cardinality 0..* - - checksums: List of checksums, there must be a SHA1 hash, at least. + - checksums: Dict with identifier of spdx.checksum.Algorithm as key and spdx.checksum.Algorithm as value, + there must be a SHA1 hash, at least. - conc_lics: Mandatory one. license.License or utils.NoAssert or utils.SPDXNone. - licenses_in_file: list of licenses found in file, mandatory one or more. document.License or utils.SPDXNone or utils.NoAssert. @@ -80,7 +81,7 @@ def __init__(self, name, spdx_id=None): self.spdx_id = spdx_id self.comment = None self.file_types = [] - self.checksums = [] + self.checksums = {} self.conc_lics = None self.licenses_in_file = [] self.license_comment = None @@ -105,7 +106,7 @@ def chk_sum(self): Backwards compatibility, return SHA1 checksum. """ warnings.warn("This property is deprecated. Use get_checksum instead.") - return self.get_checksum('SHA1') + return self.get_checksum(ChecksumAlgorithmIdentifier.SHA1) @chk_sum.setter def chk_sum(self, value): @@ -114,7 +115,7 @@ def chk_sum(self, value): """ warnings.warn("This property is deprecated. Use set_checksum instead.") if isinstance(value, str): - self.set_checksum(Algorithm(ChecksumAlgorithmIdentifier.SHA1, value)) + self.set_checksum(Algorithm("SHA1", value)) elif isinstance(value, Algorithm): self.set_checksum(value) @@ -211,7 +212,7 @@ def validate_file_types(self, messages): return messages def validate_checksum(self, messages): - if self.get_checksum("SHA1") is None: + if self.get_checksum(ChecksumAlgorithmIdentifier.SHA1) is None: messages.append("At least one file checksum algorithm must be SHA1") return messages @@ -231,18 +232,14 @@ def calculate_checksum(self, hash_algorithm='SHA1'): return file_hash.hexdigest() def get_checksum(self, hash_algorithm: ChecksumAlgorithmIdentifier = ChecksumAlgorithmIdentifier.SHA1) -> Algorithm: - for checksum in self.checksums: - if checksum.identifier == hash_algorithm: - return checksum + return self.checksums[hash_algorithm.name] - def set_checksum(self, checksum: Algorithm): - if not isinstance(checksum, Algorithm): + def set_checksum(self, new_checksum: Algorithm): + if not isinstance(new_checksum, Algorithm): raise SPDXValueError - for file_checksum in self.checksums: - if file_checksum.identifier == checksum.identifier: - file_checksum.value = checksum.value - return - self.checksums.append(checksum) + if new_checksum.identifier in self.checksums: + return + self.checksums[new_checksum.identifier] = new_checksum def has_optional_field(self, field): return bool(getattr(self, field, None)) diff --git a/spdx/package.py b/spdx/package.py index 673dfef10..e1b329efc 100644 --- a/spdx/package.py +++ b/spdx/package.py @@ -60,7 +60,7 @@ class Package(object): whenever files_analyzed is True or None (omitted) and Must be None (omitted) if files_analyzed is False. However, as a convenience within this library, we allow this to be Optional even when files_analyzed is True/None. - - checksums: Optional , List of spdx.checksum.Algorithm. + - checksums: Optional, Dict with identifier of spdx.checksum.Algorithm as key and spdx.checksum.Algorithm as value. - source_info: Optional string. - conc_lics: Mandatory license.License or utils.SPDXNone or utils.NoAssert. @@ -101,7 +101,7 @@ def __init__( self.files_analyzed = None self.homepage = None self.verif_code = None - self.checksums = [] + self.checksums = {} self.source_info = None self.conc_lics = None self.license_declared = None @@ -152,7 +152,7 @@ def checksum(self): """ # NOTE Package.check_sum but File.chk_sum if self.checksums: - return self.checksums[0] + return self.checksums["SHA1"] else: return None @@ -310,25 +310,21 @@ def validate_str_fields(self, fields, optional, messages): def validate_checksum(self, messages): if not self.checksums: return messages - if self.get_checksum("SHA1") is None: + try: + self.get_checksum(ChecksumAlgorithmIdentifier.SHA1) + except KeyError: messages.append("At least one package checksum algorithm must be SHA1") return messages - def get_checksum(self, hash_algorithm: ChecksumAlgorithmIdentifier = ChecksumAlgorithmIdentifier.SHA1): - for checksum in self.checksums: - if checksum.identifier == hash_algorithm: - return checksum - return None + def get_checksum(self, hash_algorithm: ChecksumAlgorithmIdentifier = ChecksumAlgorithmIdentifier.SHA1) -> Algorithm: + return self.checksums[hash_algorithm.name] def set_checksum(self, new_checksum: Algorithm): if not isinstance(new_checksum, Algorithm): raise SPDXValueError("Package::Checksum") - - for checksum in self.checksums: - if checksum.identifier == new_checksum.identifier: - checksum.value = new_checksum.value - return - self.checksums.append(new_checksum) + if new_checksum.identifier in self.checksums: + return + self.checksums[new_checksum.identifier] = new_checksum def has_optional_field(self, field): return bool(getattr(self, field, None)) diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index bb478d776..6564f5e25 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -1196,13 +1196,9 @@ def set_file_checksum(self, doc, checksum): Raise CardinalityError if no SHA1 checksum set. """ if self.has_file(doc): - self.file_chksum_set = False - self.file(doc).checksums.append(checksum_algorithm_from_string(checksum)) - for file_checksum in self.file(doc).checksums: - if file_checksum.identifier == 'SHA1': - self.file_chksum_set = True - if not self.file_chksum_set: - raise CardinalityError("File::CheckSum") + new_checksum = checksum_algorithm_from_string(checksum) + self.file(doc).set_checksum(new_checksum) + self.file_chksum_set = True else: raise OrderError("File::CheckSum") return True diff --git a/spdx/writers/jsonyamlxml.py b/spdx/writers/jsonyamlxml.py index e62056da7..b77224716 100644 --- a/spdx/writers/jsonyamlxml.py +++ b/spdx/writers/jsonyamlxml.py @@ -169,7 +169,7 @@ def create_package_info(self, package, annotations_by_spdx_id): if package.has_optional_field("originator"): package_object["originator"] = package.originator.to_value() - for checksum in package.checksums: + for checksum in package.checksums.values(): package_object.setdefault("checksums", []).append(self.checksum_to_dict(checksum)) if package.has_optional_field("description"): @@ -233,7 +233,7 @@ def create_file_info(self, file, annotations_by_spdx_id): file_object["fileName"] = file.name file_object["SPDXID"] = self.spdx_id(file.spdx_id) - for checksum in file.checksums: + for checksum in file.checksums.values(): file_object.setdefault("checksums", []).append(self.checksum_to_dict(checksum)) if file.has_optional_field("conc_lics"): file_object["licenseConcluded"] = self.license(file.conc_lics) diff --git a/spdx/writers/rdf.py b/spdx/writers/rdf.py index 42bc8667a..28ac52dba 100644 --- a/spdx/writers/rdf.py +++ b/spdx/writers/rdf.py @@ -266,7 +266,7 @@ def create_file_node(self, doc_file): ftype_triple = (file_node, self.spdx_namespace.fileType, ftype) self.graph.add(ftype_triple) - for chk_sum in doc_file.checksums: + for chk_sum in doc_file.checksums.values(): self.graph.add( ( file_node, diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index 93f57c45d..c717206e8 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -124,7 +124,7 @@ def write_file(spdx_file, out): write_value("SPDXID", spdx_file.spdx_id, out) for file_type in spdx_file.file_types: write_file_type(file_type.name, out) - for file_checksum in spdx_file.checksums: + for file_checksum in spdx_file.checksums.values(): write_value("FileChecksum", file_checksum.to_tv(), out) if spdx_file.has_optional_field("conc_lics"): if isinstance( @@ -240,7 +240,7 @@ def write_package(package, out): if package.has_optional_field("originator"): write_value("PackageOriginator", package.originator, out) - for package_checksum in package.checksums: + for package_checksum in package.checksums.values(): write_value("PackageChecksum", package_checksum.to_tv(), out) if package.has_optional_field("verif_code"): diff --git a/tests/data/doc_parse/SBOMexpected.json b/tests/data/doc_parse/SBOMexpected.json index a473606aa..3d0504388 100644 --- a/tests/data/doc_parse/SBOMexpected.json +++ b/tests/data/doc_parse/SBOMexpected.json @@ -64,10 +64,10 @@ }, "copyrightText": "copyright 2004-2020 Example Inc. All Rights Reserved.", "licenseComment": null, - "checksum": { + "checksums": [{ "identifier": "SHA1", "value": "SOME-SHA1" - }, + }], "licenseInfoFromFiles": [], "verificationCode": { "value": null, @@ -98,10 +98,10 @@ }, "copyrightText": "Copyright (c) 1996 - 2020, Daniel Stenberg, , and many contributors, see the THANKS file.", "licenseComment": null, - "checksum": { + "checksums": [{ "identifier": "SHA1", "value": "SOME-SHA1" - }, + }], "licenseInfoFromFiles": [], "verificationCode": { "value": null, @@ -132,10 +132,10 @@ }, "copyrightText": "copyright 2004-2020 The OpenSSL Project Authors. All Rights Reserved.", "licenseComment": null, - "checksum": { + "checksums": [{ "identifier": "SHA1", "value": "SOME-SHA1" - }, + }], "licenseInfoFromFiles": [], "verificationCode": { "value": null, diff --git a/tests/data/doc_parse/expected.json b/tests/data/doc_parse/expected.json index 0900dc983..84f6689d1 100644 --- a/tests/data/doc_parse/expected.json +++ b/tests/data/doc_parse/expected.json @@ -168,10 +168,10 @@ }, "copyrightText": " Copyright 2010, 2011 Source Auditor Inc.", "licenseComment": "The declared license information can be found in the NOTICE file at the root of the archive file", - "checksum": { + "checksums": [{ "identifier": "SHA1", "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" - }, + }], "licenseInfoFromFiles": [ { "type": "Single", diff --git a/tests/data/doc_parse/spdx-expected.json b/tests/data/doc_parse/spdx-expected.json index 05e73da2a..ee30bd895 100644 --- a/tests/data/doc_parse/spdx-expected.json +++ b/tests/data/doc_parse/spdx-expected.json @@ -171,10 +171,10 @@ }, "copyrightText": " Copyright 2010, 2011 Source Auditor Inc.", "licenseComment": "The declared license information can be found in the NOTICE file at the root of the archive file", - "checksum": { + "checksums": [{ "identifier": "SHA1", "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" - }, + }], "licenseInfoFromFiles": [ { "type": "Single", diff --git a/tests/utils_test.py b/tests/utils_test.py index e77159fa3..ae3362f22 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -358,7 +358,7 @@ def package_to_dict(cls, package): ('licenseDeclared', cls.license_to_dict(package.license_declared)), ('copyrightText', package.cr_text), ('licenseComment', package.license_comment), - ('checksum', cls.checksum_to_dict(package.checksum)), + ('checksums', [cls.checksum_to_dict(checksum) for checksum in package.checksums.values()]), ('licenseInfoFromFiles', [cls.license_to_dict(lic) for lic in lics_from_files]), ('verificationCode', OrderedDict([ ('value', package.verif_code), @@ -392,7 +392,7 @@ def files_to_list(cls, files): lics_in_files = sorted(file.licenses_in_file, key=lambda lic: lic.identifier) contributors = sorted(file.contributors, key=lambda c: c.name) chk_sums = [] - for chk_sum in file.checksums: + for _, chk_sum in file.checksums.items(): chk_sums.append(cls.checksum_to_dict(chk_sum)) file_dict = OrderedDict([ From 938cb10da062e3ce60f1b069de121d6a716dfc9a Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Thu, 24 Nov 2022 12:10:08 +0100 Subject: [PATCH 11/13] [issue-185] delete duplicated checksum property Signed-off-by: Meret Behrens --- spdx/package.py | 20 ++------------------ spdx/writers/rdf.py | 7 ++++--- tests/data/doc_write/rdf-simple-plus.json | 20 ++++++++++++++------ tests/data/doc_write/rdf-simple.json | 20 ++++++++++++++------ 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/spdx/package.py b/spdx/package.py index e1b329efc..3f0095ab9 100644 --- a/spdx/package.py +++ b/spdx/package.py @@ -119,14 +119,13 @@ def __init__( self.built_date: Optional[datetime] = None self.valid_until_date: Optional[datetime] = None - @property def check_sum(self): """ Backwards compatibility, return SHA1 checksum. """ warnings.warn("This property is deprecated. Use get_checksum instead.") - return self.get_checksum('SHA1') + return self.get_checksum(ChecksumAlgorithmIdentifier.SHA1) @check_sum.setter def check_sum(self, value): @@ -135,7 +134,7 @@ def check_sum(self, value): """ warnings.warn("This property is deprecated. Use set_checksum instead.") if isinstance(value, str): - self.set_checksum(Algorithm(ChecksumAlgorithmIdentifier.SHA1, value)) + self.set_checksum(Algorithm("SHA1", value)) elif isinstance(value, Algorithm): self.set_checksum(value) @@ -145,21 +144,6 @@ def are_files_analyzed(self): # as default None Value is False, previous line is simplification of # return self.files_analyzed or self.files_analyzed is None - @property - def checksum(self): - """ - Backwards compatibility, return first checksum. - """ - # NOTE Package.check_sum but File.chk_sum - if self.checksums: - return self.checksums["SHA1"] - else: - return None - - @checksum.setter - def checksum(self, value): - self.checksums[0] = value - def add_lics_from_file(self, lics): self.licenses_from_files.append(lics) diff --git a/spdx/writers/rdf.py b/spdx/writers/rdf.py index 28ac52dba..23936f969 100644 --- a/spdx/writers/rdf.py +++ b/spdx/writers/rdf.py @@ -773,9 +773,10 @@ def handle_pkg_optional_fields(self, package: Package, package_node): package, package_node, self.spdx_namespace.filesAnalyzed, "files_analyzed" ) - if package.has_optional_field("checksum"): - checksum_node = self.create_checksum_node(package.checksum) - self.graph.add((package_node, self.spdx_namespace.checksum, checksum_node)) + if package.has_optional_field("checksums"): + for checksum in package.checksums.values(): + checksum_node = self.create_checksum_node(checksum) + self.graph.add((package_node, self.spdx_namespace.checksum, checksum_node)) if package.has_optional_field("homepage"): homepage_node = URIRef(self.to_special_value(package.homepage)) diff --git a/tests/data/doc_write/rdf-simple-plus.json b/tests/data/doc_write/rdf-simple-plus.json index 66bc8f8e8..d2b65aeed 100644 --- a/tests/data/doc_write/rdf-simple-plus.json +++ b/tests/data/doc_write/rdf-simple-plus.json @@ -26,12 +26,20 @@ "@rdf:resource": "http://spdx.org/rdf/terms#noassertion" }, "ns1:copyrightText": "Some copyright", - "ns1:checksum": { - "ns1:Checksum": { - "ns1:algorithm": "http://spdx.org/rdf/terms#checksumAlgorithm_sha1", - "ns1:checksumValue": "SOME-SHA1" - } - } + "ns1:checksum": [ + { + "ns1:Checksum": { + "ns1:algorithm": "http://spdx.org/rdf/terms#checksumAlgorithm_sha1", + "ns1:checksumValue": "SOME-SHA1" + } + }, + { + "ns1:Checksum": { + "ns1:algorithm": "http://spdx.org/rdf/terms#checksumAlgorithm_sha256", + "ns1:checksumValue": "SOME-SHA256" + } + } + ] } }, "ns1:specVersion": "SPDX-2.1", diff --git a/tests/data/doc_write/rdf-simple.json b/tests/data/doc_write/rdf-simple.json index 63860a80c..00064a345 100644 --- a/tests/data/doc_write/rdf-simple.json +++ b/tests/data/doc_write/rdf-simple.json @@ -26,12 +26,20 @@ "@rdf:resource": "http://spdx.org/rdf/terms#noassertion" }, "ns1:copyrightText": "Some copyright", - "ns1:checksum": { - "ns1:Checksum": { - "ns1:algorithm": "http://spdx.org/rdf/terms#checksumAlgorithm_sha1", - "ns1:checksumValue": "SOME-SHA1" - } - } + "ns1:checksum": [ + { + "ns1:Checksum": { + "ns1:algorithm": "http://spdx.org/rdf/terms#checksumAlgorithm_sha1", + "ns1:checksumValue": "SOME-SHA1" + } + }, + { + "ns1:Checksum": { + "ns1:algorithm": "http://spdx.org/rdf/terms#checksumAlgorithm_sha256", + "ns1:checksumValue": "SOME-SHA256" + } + } + ] } }, "ns1:specVersion": "SPDX-2.1", From 9143c97623ea202d53809031ceff7066cae666a2 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Thu, 24 Nov 2022 13:44:03 +0100 Subject: [PATCH 12/13] [issue-185] use regex to match blake checksum algorithms in rdf files Signed-off-by: Meret Behrens --- spdx/checksum.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/spdx/checksum.py b/spdx/checksum.py index b8350a4bf..121e58321 100644 --- a/spdx/checksum.py +++ b/spdx/checksum.py @@ -8,10 +8,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import re from enum import Enum, auto -Exceptions_rdf = {"SHA3256": "SHA3_256", "SHA3384": "SHA3_384", "SHA3512": "SHA3_512", "BLAKE2B256": "BLAKE2B_256", - "BLAKE2B384": "BLAKE2B_384", "BLAKE2V512": "BLAKE2B_512"} class ChecksumAlgorithmIdentifier(Enum): @@ -38,9 +37,11 @@ def checksum_to_rdf(self): @classmethod def checksum_from_rdf(cls, identifier: str) -> str: - identifier = identifier.split('_')[-1].upper() - if identifier in Exceptions_rdf: - return Exceptions_rdf[identifier] + identifier = identifier.split('_', 1)[-1].upper() + blake_checksum = re.compile(r"(BLAKE2B)\s*(256|384|512)", re.UNICODE) + match = blake_checksum.match(identifier) + if match: + return match[1] + '_' + match[2] return identifier From af36fb5103c3a85f7419f5705bb0570c3bb7a069 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Mon, 28 Nov 2022 11:20:02 +0100 Subject: [PATCH 13/13] [issue-185] squash review commits Signed-off-by: Meret Behrens --- examples/write_tv.py | 8 ++-- spdx/checksum.py | 46 +++++++++++++------ spdx/cli_tools/parser.py | 6 +-- spdx/document.py | 12 ++--- spdx/file.py | 43 +++++++++--------- spdx/package.py | 45 +++++++++---------- spdx/parsers/jsonyamlxml.py | 12 ++--- spdx/parsers/jsonyamlxmlbuilders.py | 30 ++++++++----- spdx/parsers/lexers/tagvalue.py | 2 +- spdx/parsers/rdf.py | 20 ++++----- spdx/parsers/rdfbuilders.py | 46 ++++++++++--------- spdx/parsers/tagvaluebuilders.py | 26 +++-------- spdx/utils.py | 19 ++++---- spdx/writers/json.py | 1 - spdx/writers/jsonyamlxml.py | 12 +++-- spdx/writers/rdf.py | 8 ++-- spdx/writers/tagvalue.py | 2 +- tests/data/doc_parse/spdx-expected.json | 30 ++++++++----- tests/data/doc_write/json-simple-plus.json | 51 +++++++++++----------- tests/data/doc_write/json-simple.json | 20 ++++----- tests/data/formats/SPDXTagExample.tag | 4 +- tests/test_builder.py | 4 +- tests/test_checksum.py | 48 ++++++++++++++++++++ tests/test_document.py | 27 ++++++------ tests/test_jsonyamlxml_writer.py | 4 +- tests/test_package.py | 6 +-- tests/utils_test.py | 8 ++-- 27 files changed, 303 insertions(+), 237 deletions(-) create mode 100644 tests/test_checksum.py diff --git a/examples/write_tv.py b/examples/write_tv.py index 47c8cd494..3273b3b2d 100755 --- a/examples/write_tv.py +++ b/examples/write_tv.py @@ -15,7 +15,7 @@ from spdx.review import Review from spdx.package import Package from spdx.file import File, FileType - from spdx.checksum import Algorithm + from spdx.checksum import Checksum from spdx.utils import SPDXNone, NoAssert, UnKnown doc = Document() @@ -36,7 +36,7 @@ testfile1.type = FileType.BINARY testfile1.spdx_id = "TestFilet#SPDXRef-FILE" testfile1.comment = "This is a test file." - testfile1.chksum = Algorithm("SHA1", "c537c5d99eca5333f23491d47ededd083fefb7ad") + testfile1.chksum = Checksum("SHA1", "c537c5d99eca5333f23491d47ededd083fefb7ad") testfile1.conc_lics = License.from_identifier("BSD-2-Clause") testfile1.add_lics(License.from_identifier("BSD-2-Clause")) testfile1.copyright = SPDXNone() @@ -48,7 +48,7 @@ testfile2.type = FileType.SOURCE testfile2.spdx_id = "TestFile2#SPDXRef-FILE" testfile2.comment = "This is a test file." - testfile2.chksum = Algorithm("SHA1", "bb154f28d1cf0646ae21bb0bec6c669a2b90e113") + testfile2.chksum = Checksum("SHA1", "bb154f28d1cf0646ae21bb0bec6c669a2b90e113") testfile2.conc_lics = License.from_identifier("Apache-2.0") testfile2.add_lics(License.from_identifier("Apache-2.0")) testfile2.copyright = NoAssert() @@ -62,7 +62,7 @@ package.file_name = "twt.jar" package.spdx_id = 'TestPackage#SPDXRef-PACKAGE' package.download_location = "http://www.tagwritetest.test/download" - package.checksum = Algorithm("SHA1", "c537c5d99eca5333f23491d47ededd083fefb7ad") + package.checksum = Checksum("SHA1", "c537c5d99eca5333f23491d47ededd083fefb7ad") package.homepage = SPDXNone() package.verif_code = "4e3211c67a2d28fced849ee1bb76e7391b93feba" license_set = LicenseConjunction( diff --git a/spdx/checksum.py b/spdx/checksum.py index 121e58321..c17361818 100644 --- a/spdx/checksum.py +++ b/spdx/checksum.py @@ -10,10 +10,10 @@ # limitations under the License. import re from enum import Enum, auto +from typing import Optional - -class ChecksumAlgorithmIdentifier(Enum): +class ChecksumAlgorithm(Enum): SHA1 = auto() SHA224 = auto() SHA256 = auto() @@ -32,27 +32,47 @@ class ChecksumAlgorithmIdentifier(Enum): MD6 = auto() ADLER32 = auto() - def checksum_to_rdf(self): - return "checksumAlgorithm_" + self.name.lower() + def algorithm_to_rdf_representation(self) -> str: + if self.name.startswith("BLAKE2B"): + return "checksumAlgorithm_" + self.name.replace("_", "").lower() + else: + return "checksumAlgorithm_" + self.name.lower() @classmethod - def checksum_from_rdf(cls, identifier: str) -> str: + def checksum_from_rdf(cls, identifier: str) -> 'ChecksumAlgorithm': identifier = identifier.split('_', 1)[-1].upper() - blake_checksum = re.compile(r"(BLAKE2B)\s*(256|384|512)", re.UNICODE) + blake_checksum = re.compile(r"^(BLAKE2B)(256|384|512)$", re.UNICODE) match = blake_checksum.match(identifier) if match: - return match[1] + '_' + match[2] - return identifier + identifier = match[1] + '_' + match[2] + if identifier not in ChecksumAlgorithm.__members__: + raise ValueError(f"Invalid algorithm for checksum: {identifier}") + return ChecksumAlgorithm[identifier] + + @classmethod + def checksum_algorithm_from_string(cls, identifier: str) -> 'ChecksumAlgorithm': + identifier.replace("-", "_").upper() + if identifier not in ChecksumAlgorithm.__members__: + raise ValueError(f"Invalid algorithm for checksum: {identifier}") + return ChecksumAlgorithm[identifier] -class Algorithm(object): +class Checksum(object): """Generic checksum algorithm.""" - def __init__(self, identifier: str, value): - if identifier.upper().replace('-', '_') not in ChecksumAlgorithmIdentifier.__members__: - raise ValueError('Invalid algorithm for Checksum: {}'.format(identifier)) + def __init__(self, identifier: ChecksumAlgorithm, value: str): self.identifier = identifier self.value = value + @classmethod + def checksum_from_string(cls, value: str) -> 'Checksum': + CHECKSUM_RE = re.compile("(ADLER32|BLAKE2b-256|BLAKE2b-384|BLAKE2b-512|BLAKE3|MD2|MD4|MD5|MD6|" \ + "SHA1|SHA224|SHA256|SHA384|SHA512|SHA3-256|SHA3-384|SHA3-512):\\s*([a-fA-F0-9]*)") + match = CHECKSUM_RE.match(value) + if match is None or match.group(1) is None or match.group(2) is None: + raise ValueError(f"Invalid checksum: {value}") + identifier = ChecksumAlgorithm.checksum_algorithm_from_string(match.group(1)) + return Checksum(identifier=identifier, value=match.group(2)) + def to_tv(self): - return "{0}: {1}".format(self.identifier, self.value) + return "{0}: {1}".format(self.identifier.name, self.value) diff --git a/spdx/cli_tools/parser.py b/spdx/cli_tools/parser.py index 891a3017a..4b5d45a21 100755 --- a/spdx/cli_tools/parser.py +++ b/spdx/cli_tools/parser.py @@ -52,7 +52,7 @@ def main(file, force): ) print("Package Homepage: {0}".format(package.homepage)) for checksum in doc.package.checksums.values(): - print("Package Checksum: {0} {1}".format(checksum.identifier, checksum.value)) + print("Package Checksum: {0} {1}".format(checksum.identifier.name, checksum.value)) print("Package Attribution Text: {0}".format(package.attribution_text)) print("Package verification code: {0}".format(package.verif_code)) print( @@ -82,8 +82,8 @@ def main(file, force): print("\tFile name: {0}".format(f.name)) for file_type in f.file_types: print("\tFile type: {0}".format(file_type.name)) - for file_checksum in f.checksums: - print("\tFile Checksum: {0} {1}".format(file_checksum.identifier, file_checksum.value)) + for file_checksum in f.checksums.values(): + print("\tFile Checksum: {0} {1}".format(file_checksum.identifier.name, file_checksum.value)) print("\tFile license concluded: {0}".format(f.conc_lics)) print( "\tFile license info in file: {0}".format( diff --git a/spdx/document.py b/spdx/document.py index 2e0a65950..c14e891d7 100644 --- a/spdx/document.py +++ b/spdx/document.py @@ -34,25 +34,25 @@ class ExternalDocumentRef(object): """ def __init__( - self, external_document_id=None, spdx_document_uri=None, check_sum=None + self, external_document_id=None, spdx_document_uri=None, checksum=None ): self.external_document_id = external_document_id self.spdx_document_uri = spdx_document_uri - self.check_sum = check_sum + self.checksum = checksum def __eq__(self, other): return ( isinstance(other, ExternalDocumentRef) and self.external_document_id == other.external_document_id and self.spdx_document_uri == other.spdx_document_uri - and self.check_sum == other.check_sum + and self.checksum == other.checksum ) def __lt__(self, other): - return (self.external_document_id, self.spdx_document_uri, self.check_sum) < ( + return (self.external_document_id, self.spdx_document_uri, self.checksum) < ( other.external_document_id, other.spdx_document_uri, - other.check_sum, + other.checksum, ) def validate(self, messages: ErrorMessages) -> ErrorMessages: @@ -74,7 +74,7 @@ def validate_spdx_doc_uri(self, messages): messages.append("ExternalDocumentRef has no SPDX Document URI.") def validate_checksum(self, messages): - if not self.check_sum: + if not self.checksum: messages.append("ExternalDocumentRef has no Checksum.") diff --git a/spdx/file.py b/spdx/file.py index 219c9d6bd..7781b7ea7 100644 --- a/spdx/file.py +++ b/spdx/file.py @@ -14,10 +14,11 @@ from functools import total_ordering import hashlib -from spdx.checksum import Algorithm, ChecksumAlgorithmIdentifier -from spdx import utils from spdx.license import License +from spdx import utils +from spdx.checksum import Checksum, ChecksumAlgorithm from spdx.parsers.builderexceptions import SPDXValueError +from spdx.parsers.loggers import ErrorMessages class FileType(Enum): @@ -59,7 +60,7 @@ class File(object): referenced by other elements. Mandatory, one. Type: str. - comment: File comment str, Optional zero or one. - file_types: list of file types. Cardinality 0..* - - checksums: Dict with identifier of spdx.checksum.Algorithm as key and spdx.checksum.Algorithm as value, + - checksums: Dict with checksum.ChecksumAlgorithm as key and checksum.Checksum as value, there must be a SHA1 hash, at least. - conc_lics: Mandatory one. license.License or utils.NoAssert or utils.SPDXNone. - licenses_in_file: list of licenses found in file, mandatory one or more. @@ -101,22 +102,22 @@ def __lt__(self, other): return self.name < other.name @property - def chk_sum(self): + def checksum(self): """ Backwards compatibility, return SHA1 checksum. """ warnings.warn("This property is deprecated. Use get_checksum instead.") - return self.get_checksum(ChecksumAlgorithmIdentifier.SHA1) + return self.get_checksum(ChecksumAlgorithm.SHA1) - @chk_sum.setter - def chk_sum(self, value): + @checksum.setter + def checksum(self, value): """ Backwards compatibility, set checksum. """ warnings.warn("This property is deprecated. Use set_checksum instead.") if isinstance(value, str): - self.set_checksum(Algorithm("SHA1", value)) - elif isinstance(value, Algorithm): + self.set_checksum(Checksum("SHA1", value)) + elif isinstance(value, Checksum): self.set_checksum(value) def add_lics(self, lics): @@ -144,7 +145,7 @@ def validate(self, messages): messages.push_context(self.name) self.validate_concluded_license(messages) self.validate_file_types(messages) - self.validate_checksum(messages) + self.validate_checksums(messages) self.validate_licenses_in_file(messages) self.validate_copyright(messages) self.validate_artifacts(messages) @@ -211,13 +212,16 @@ def validate_file_types(self, messages): messages.append(f"{file_type} is not of type FileType.") return messages - def validate_checksum(self, messages): - if self.get_checksum(ChecksumAlgorithmIdentifier.SHA1) is None: + def validate_checksums(self, messages: ErrorMessages): + for checksum in self.checksums.values(): + if not isinstance(checksum, Checksum): + messages.append("File checksum must be instance of spdx.checksum.Checksum.") + + if self.get_checksum(ChecksumAlgorithm.SHA1) is None: messages.append("At least one file checksum algorithm must be SHA1") - return messages def calculate_checksum(self, hash_algorithm='SHA1'): - if hash_algorithm not in ChecksumAlgorithmIdentifier.__members__: + if hash_algorithm not in ChecksumAlgorithm.__members__: raise ValueError BUFFER_SIZE = 65536 @@ -231,14 +235,13 @@ def calculate_checksum(self, hash_algorithm='SHA1'): return file_hash.hexdigest() - def get_checksum(self, hash_algorithm: ChecksumAlgorithmIdentifier = ChecksumAlgorithmIdentifier.SHA1) -> Algorithm: - return self.checksums[hash_algorithm.name] + def get_checksum(self, hash_algorithm: ChecksumAlgorithm = ChecksumAlgorithm.SHA1) -> Checksum: + return self.checksums[hash_algorithm] - def set_checksum(self, new_checksum: Algorithm): - if not isinstance(new_checksum, Algorithm): + def set_checksum(self, new_checksum: Checksum): + if not isinstance(new_checksum, Checksum): raise SPDXValueError - if new_checksum.identifier in self.checksums: - return + self.checksums[new_checksum.identifier] = new_checksum def has_optional_field(self, field): diff --git a/spdx/package.py b/spdx/package.py index 3f0095ab9..fdb909c36 100644 --- a/spdx/package.py +++ b/spdx/package.py @@ -17,7 +17,7 @@ from spdx import creationinfo from spdx import license from spdx import utils -from spdx.checksum import Algorithm, ChecksumAlgorithmIdentifier +from spdx.checksum import Checksum, ChecksumAlgorithm from spdx.parsers.builderexceptions import SPDXValueError from spdx.parsers.loggers import ErrorMessages @@ -60,7 +60,7 @@ class Package(object): whenever files_analyzed is True or None (omitted) and Must be None (omitted) if files_analyzed is False. However, as a convenience within this library, we allow this to be Optional even when files_analyzed is True/None. - - checksums: Optional, Dict with identifier of spdx.checksum.Algorithm as key and spdx.checksum.Algorithm as value. + - checksums: Optional, Dict with checksum.ChecksumAlgorithm as key and checksum.Checksum as value. - source_info: Optional string. - conc_lics: Mandatory license.License or utils.SPDXNone or utils.NoAssert. @@ -120,22 +120,22 @@ def __init__( self.valid_until_date: Optional[datetime] = None @property - def check_sum(self): + def checksum(self): """ Backwards compatibility, return SHA1 checksum. """ warnings.warn("This property is deprecated. Use get_checksum instead.") - return self.get_checksum(ChecksumAlgorithmIdentifier.SHA1) + return self.get_checksum(ChecksumAlgorithm.SHA1) - @check_sum.setter - def check_sum(self, value): + @checksum.setter + def checksum(self, value): """ Backwards compatibility, set SHA1 checksum. """ warnings.warn("This property is deprecated. Use set_checksum instead.") if isinstance(value, str): - self.set_checksum(Algorithm("SHA1", value)) - elif isinstance(value, Algorithm): + self.set_checksum(Checksum("SHA1", value)) + elif isinstance(value, Checksum): self.set_checksum(value) @property @@ -160,7 +160,7 @@ def validate(self, messages): """ messages.push_context(self.name) self.validate_files_analyzed(messages) - self.validate_checksum(messages) + self.validate_checksums(messages) self.validate_optional_str_fields(messages) self.validate_mandatory_str_fields(messages) self.validate_pkg_ext_refs(messages) @@ -291,23 +291,20 @@ def validate_str_fields(self, fields, optional, messages): return messages - def validate_checksum(self, messages): + def validate_checksums(self, messages: ErrorMessages): if not self.checksums: - return messages - try: - self.get_checksum(ChecksumAlgorithmIdentifier.SHA1) - except KeyError: - messages.append("At least one package checksum algorithm must be SHA1") - return messages - - def get_checksum(self, hash_algorithm: ChecksumAlgorithmIdentifier = ChecksumAlgorithmIdentifier.SHA1) -> Algorithm: - return self.checksums[hash_algorithm.name] - - def set_checksum(self, new_checksum: Algorithm): - if not isinstance(new_checksum, Algorithm): - raise SPDXValueError("Package::Checksum") - if new_checksum.identifier in self.checksums: return + for checksum in self.checksums.values(): + if not isinstance(checksum, Checksum): + messages.append("Package checksum must be instance of spdx.checksum.Checksum") + + def get_checksum(self, hash_algorithm: ChecksumAlgorithm = ChecksumAlgorithm.SHA1) -> Checksum: + return self.checksums[hash_algorithm] + + def set_checksum(self, new_checksum: Checksum): + if not isinstance(new_checksum, Checksum): + raise SPDXValueError("Package::Checksum") + self.checksums[new_checksum.identifier] = new_checksum def has_optional_field(self, field): diff --git a/spdx/parsers/jsonyamlxml.py b/spdx/parsers/jsonyamlxml.py index 815cba485..0e8916a6f 100644 --- a/spdx/parsers/jsonyamlxml.py +++ b/spdx/parsers/jsonyamlxml.py @@ -10,7 +10,7 @@ # limitations under the License. from datetime import datetime from enum import Enum, auto -from typing import List, Dict, Tuple, Callable +from typing import List, Dict, Tuple, Callable, Optional from spdx import document from spdx import utils @@ -1093,10 +1093,10 @@ def _handle_file_dependency(self, file_dependency): return None return None - def parse_file_checksums(self, file_checksums): + def parse_file_checksums(self, file_checksums: List[Dict]) -> Optional[bool]: """ Parse File checksums - - file_checksum: Python List + - file_checksums: Python List """ if isinstance(file_checksums, list): for checksum in file_checksums: @@ -1593,10 +1593,10 @@ def parse_pkg_files(self, pkg_has_files: List[str], method_to_parse_relationship elif pkg_has_files is not None: self.value_error("PKG_HAS_FILES", pkg_has_files) - def parse_pkg_checksums(self, pkg_checksums): + def parse_pkg_checksums(self, pkg_checksums: List[Dict]) -> Optional[bool]: """ - Parse Package checksum - - pkg_chksum: Python str/unicode + Parse Package checksums + - pkg_chksums: Python List """ if isinstance(pkg_checksums, list): for checksum in pkg_checksums: diff --git a/spdx/parsers/jsonyamlxmlbuilders.py b/spdx/parsers/jsonyamlxmlbuilders.py index 189e7e2e1..d514af7ab 100644 --- a/spdx/parsers/jsonyamlxmlbuilders.py +++ b/spdx/parsers/jsonyamlxmlbuilders.py @@ -8,11 +8,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from spdx import file +from typing import Dict, Union + +from spdx.document import Document from spdx.parsers import rdfbuilders from spdx.parsers import tagvaluebuilders from spdx.parsers import validations -from spdx.checksum import Algorithm +from spdx.checksum import Checksum, ChecksumAlgorithm from spdx.parsers.builderexceptions import SPDXValueError from spdx.parsers.builderexceptions import CardinalityError from spdx.parsers.builderexceptions import OrderError @@ -160,20 +162,24 @@ class FileBuilder(rdfbuilders.FileBuilder): def __init__(self): super(FileBuilder, self).__init__() - def set_file_checksum(self, doc, checksum): + def set_file_checksum(self, doc: Document, checksum: Union[Dict, Checksum, str]) -> bool: """ Set the file checksum. checksum - A string + raise OrderError if no file defined. """ - if self.has_file(doc): - if isinstance(checksum, dict): - algo = checksum.get('algorithm') or 'SHA1' - self.file(doc).set_checksum(Algorithm(algo, checksum.get('checksumValue'))) - elif isinstance(checksum, Algorithm): - self.file(doc).set_checksum(checksum) - else: - self.file(doc).set_checksum(Algorithm("SHA1", checksum)) - return True + if not self.has_file(doc): + raise OrderError("No file for checksum defined.") + + if isinstance(checksum, dict): + algo = checksum.get('algorithm') or 'SHA1' + identifier = ChecksumAlgorithm.checksum_algorithm_from_string(algo) + self.file(doc).set_checksum(Checksum(identifier, checksum.get('checksumValue'))) + elif isinstance(checksum, Checksum): + self.file(doc).set_checksum(checksum) + elif isinstance(checksum, str): + self.file(doc).set_checksum(Checksum(ChecksumAlgorithm.SHA1, checksum)) + return True def set_file_notice(self, doc, text): """ diff --git a/spdx/parsers/lexers/tagvalue.py b/spdx/parsers/lexers/tagvalue.py index ba7aa62a9..83c709a0d 100644 --- a/spdx/parsers/lexers/tagvalue.py +++ b/spdx/parsers/lexers/tagvalue.py @@ -166,7 +166,7 @@ def t_text_error(self, t): def t_CHKSUM(self, t): r":\s*(ADLER32|BLAKE2b-256|BLAKE2b-384|BLAKE2b-512|BLAKE3|MD2|MD4|MD5|MD6|" \ - "SHA1|SHA224|SHA256|SHA384|SHA512|SHA3-256|SHA3-384|SHA3-512):\s*([a-f0-9]*)" + "SHA1|SHA224|SHA256|SHA384|SHA512|SHA3-256|SHA3-384|SHA3-512):\s*([a-fA-F0-9]*)" t.value = t.value[1:].strip() return t diff --git a/spdx/parsers/rdf.py b/spdx/parsers/rdf.py index 389b5ca5f..279156867 100644 --- a/spdx/parsers/rdf.py +++ b/spdx/parsers/rdf.py @@ -21,7 +21,7 @@ from spdx import document from spdx import license from spdx import utils -from spdx.checksum import Algorithm, ChecksumAlgorithmIdentifier +from spdx.checksum import Checksum, ChecksumAlgorithm from spdx.parsers.builderexceptions import CardinalityError from spdx.parsers.builderexceptions import SPDXValueError from spdx.parsers.loggers import ErrorMessages @@ -67,14 +67,12 @@ } -def convert_rdf_checksum_algorithm(algo): - ss = algo.split('#') - if len(ss) != 2: - raise SPDXValueError('Unknown checksum algorithm {}'.format(algo)) - algo = ChecksumAlgorithmIdentifier.checksum_from_rdf(ss[1]) - if algo is None: - raise SPDXValueError('Unknown checksum algorithm {}'.format(algo)) - return algo +def convert_rdf_checksum_algorithm(rdf_checksum_algorithm: str) -> ChecksumAlgorithm: + split_string = rdf_checksum_algorithm.split('#') + if len(split_string) != 2: + raise SPDXValueError('Unknown checksum algorithm {}'.format(rdf_checksum_algorithm)) + checksum_algorithm = ChecksumAlgorithm.checksum_from_rdf(split_string[1]) + return checksum_algorithm class BaseParser(object): @@ -523,7 +521,7 @@ def p_pkg_checksum(self, p_term, predicate): (pkg_checksum, self.spdx_namespace["algorithm"], None) ): algorithm_identifier = convert_rdf_checksum_algorithm(str(algo)) - checksum = Algorithm(algorithm_identifier, str(value)) + checksum = Checksum(algorithm_identifier, str(value)) self.builder.set_pkg_checksum(self.doc, checksum) def p_pkg_homepg(self, p_term, predicate): @@ -793,7 +791,7 @@ def p_file_checksum(self, f_term, predicate): (file_checksum, self.spdx_namespace["algorithm"], None) ): algorithm_identifier = convert_rdf_checksum_algorithm(str(algo)) - checksum = Algorithm(algorithm_identifier, str(value)) + checksum = Checksum(algorithm_identifier, str(value)) self.builder.set_file_checksum(self.doc, checksum) def p_file_lic_conc(self, f_term, predicate): diff --git a/spdx/parsers/rdfbuilders.py b/spdx/parsers/rdfbuilders.py index ef812f4b3..c7845ae4d 100644 --- a/spdx/parsers/rdfbuilders.py +++ b/spdx/parsers/rdfbuilders.py @@ -10,13 +10,14 @@ # limitations under the License. import re -from typing import Dict +from typing import Dict, Union from spdx import file from spdx import license from spdx import package from spdx import version -from spdx.checksum import Algorithm +from spdx.checksum import Checksum, ChecksumAlgorithm +from spdx.document import Document from spdx.parsers.builderexceptions import CardinalityError from spdx.parsers.builderexceptions import OrderError from spdx.parsers.builderexceptions import SPDXValueError @@ -144,8 +145,8 @@ def set_chksum(self, doc, chk_sum): chk_sum - The checksum value in the form of a string. """ if chk_sum: - doc.ext_document_references[-1].check_sum = Algorithm( - "SHA1", chk_sum + doc.ext_document_references[-1].checksum = Checksum( + ChecksumAlgorithm.SHA1, chk_sum ) else: raise SPDXValueError("ExternalDocumentRef::Checksum") @@ -187,21 +188,26 @@ class PackageBuilder(tagvaluebuilders.PackageBuilder): def __init__(self): super(PackageBuilder, self).__init__() - def set_pkg_checksum(self, doc, checksum: [Algorithm, Dict]): + def set_pkg_checksum(self, doc, checksum: Union[Checksum, Dict]): """ Set the package checksum. - checksum - A string + checksum - A Checksum or a Dict Raise SPDXValueError if checksum type invalid. Raise OrderError if no package previously defined. """ self.assert_package_exists() if isinstance(checksum, dict): - algo = checksum.get('algorithm') or 'SHA1' + algo = checksum.get('algorithm') or ChecksumAlgorithm.SHA1 if algo.startswith('checksumAlgorithm_'): - algo = convert_rdf_checksum_algorithm(algo) or 'SHA1' - doc.packages[-1].set_checksum(Algorithm(identifier=algo, value=checksum.get('checksumValue'))) - elif isinstance(checksum, Algorithm): + algo = convert_rdf_checksum_algorithm(algo) or ChecksumAlgorithm.SHA1 + else: + algo = ChecksumAlgorithm.checksum_algorithm_from_string(algo) + doc.packages[-1].set_checksum(Checksum(identifier=algo, value=checksum.get('checksumValue'))) + elif isinstance(checksum, Checksum): doc.packages[-1].set_checksum(checksum) + elif isinstance(checksum, str): + # kept for backwards compatibility + doc.packages[-1].set_checksum(Checksum(identifier=ChecksumAlgorithm.SHA1, value=checksum)) else: raise SPDXValueError("Invalid value for package checksum.") @@ -387,21 +393,21 @@ class FileBuilder(tagvaluebuilders.FileBuilder): def __init__(self): super(FileBuilder, self).__init__() - def set_file_checksum(self, doc, chk_sum): + def set_file_checksum(self, doc: Document, chk_sum: Union[Checksum, Dict, str]): """ Set the file check sum, if not already set. - chk_sum - A string - Raise CardinalityError if already defined. - Raise OrderError if no package previously defined. + chk_sum - A checksum.Checksum or a dict """ - if self.has_package(doc) and self.has_file(doc): + if self.has_file(doc): if isinstance(chk_sum, dict): - self.file(doc).set_checksum(Algorithm(chk_sum.get('algorithm'), - chk_sum.get('checksumValue'))) - elif isinstance(chk_sum, Algorithm): + identifier = ChecksumAlgorithm.checksum_algorithm_from_string(chk_sum.get('algorithm')) + self.file(doc).set_checksum(Checksum(identifier, + chk_sum.get('checksumValue'))) + elif isinstance(chk_sum, Checksum): self.file(doc).set_checksum(chk_sum) - else: - self.file(doc).set_checksum(Algorithm("SHA1", chk_sum)) + elif isinstance(chk_sum, str): + # kept for backwards compatibility + self.file(doc).set_checksum(Checksum(ChecksumAlgorithm.SHA1, chk_sum)) return True def set_file_license_comment(self, doc, text): diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index 6564f5e25..5f03816c4 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -9,12 +9,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from enum import auto, Enum import re from typing import Dict from spdx import annotation -from spdx import checksum from spdx import creationinfo from spdx import file from spdx import license @@ -24,9 +22,9 @@ from spdx import snippet from spdx import utils from spdx import version -from spdx.checksum import Algorithm +from spdx.checksum import Checksum -from spdx.document import ExternalDocumentRef +from spdx.document import ExternalDocumentRef, Document from spdx.package import PackagePurpose from spdx.parsers.builderexceptions import CardinalityError from spdx.parsers.builderexceptions import OrderError @@ -34,16 +32,6 @@ from spdx.parsers import validations -def checksum_algorithm_from_string(value): - CHECKSUM_RE = re.compile("(ADLER32|BLAKE2b-256|BLAKE2b-384|BLAKE2b-512|BLAKE3|MD2|MD4|MD5|MD6|" \ - "SHA1|SHA224|SHA256|SHA384|SHA512|SHA3-256|SHA3-384|SHA3-512):\\s*([a-f0-9]*)") - match = CHECKSUM_RE.match(value) - if match: - return Algorithm(identifier=match.group(1), value=match.group(2)) - else: - return None - - def str_from_text(text) -> str: """ Return content of a free form text block as a string. @@ -199,7 +187,7 @@ def set_chksum(self, doc, chksum): """ Set the `check_sum` attribute of the `ExternalDocumentRef` object. """ - doc.ext_document_references[-1].check_sum = checksum_algorithm_from_string(chksum) + doc.ext_document_references[-1].checksum = Checksum.checksum_from_string(chksum) def add_ext_doc_refs(self, doc, ext_doc_id, spdx_doc_uri, chksum): self.set_ext_doc_id(doc, ext_doc_id) @@ -785,7 +773,7 @@ def set_pkg_checksum(self, doc, checksum): """ self.assert_package_exists() self.package_chk_sum_set = True - doc.packages[-1].set_checksum(checksum_algorithm_from_string(checksum)) + doc.packages[-1].set_checksum(Checksum.checksum_from_string(checksum)) return True def set_pkg_source_info(self, doc, text): @@ -1190,15 +1178,13 @@ def set_file_type(self, doc, type_value): spdx_file.file_types.append(file_type) - def set_file_checksum(self, doc, checksum): + def set_file_checksum(self, doc: Document, checksum: str): """ Raise OrderError if no file defined. - Raise CardinalityError if no SHA1 checksum set. """ if self.has_file(doc): - new_checksum = checksum_algorithm_from_string(checksum) + new_checksum = Checksum.checksum_from_string(checksum) self.file(doc).set_checksum(new_checksum) - self.file_chksum_set = True else: raise OrderError("File::CheckSum") return True diff --git a/spdx/utils.py b/spdx/utils.py index 03b526df6..3695cd0bd 100644 --- a/spdx/utils.py +++ b/spdx/utils.py @@ -17,7 +17,8 @@ from ply import lex from ply import yacc -from spdx import checksum +from spdx.checksum import ChecksumAlgorithm + if TYPE_CHECKING: from spdx.file import File from spdx.package import Package @@ -212,18 +213,18 @@ def parse(self, data): def calc_verif_code(files: List['File']) -> str: list_of_file_hashes = [] - hash_algo_name = "SHA1" - for f in files: - file_chksum = f.get_checksum(hash_algo_name) - if file_chksum is not None: - file_ch = file_chksum.value + hash_algorithm_name = ChecksumAlgorithm.SHA1 + for file in files: + file_checksum = file.get_checksum(hash_algorithm_name) + if file_checksum is not None: + file_checksum_value = file_checksum.value else: - file_ch = f.calculate_checksum(hash_algo_name) - list_of_file_hashes.append(file_ch) + file_checksum_value = file.calculate_checksum(hash_algorithm_name) + list_of_file_hashes.append(file_checksum_value) list_of_file_hashes.sort() - hasher = hashlib.new(hash_algo_name.lower()) + hasher = hashlib.new(hash_algorithm_name.name.lower()) hasher.update("".join(list_of_file_hashes).encode("utf-8")) return hasher.hexdigest() diff --git a/spdx/writers/json.py b/spdx/writers/json.py index 732b6900f..a2b226fdf 100644 --- a/spdx/writers/json.py +++ b/spdx/writers/json.py @@ -30,7 +30,6 @@ def write_document(document, out, validate=True): messages = ErrorMessages() messages = document.validate(messages) if messages: - print(messages.messages) raise InvalidDocumentError(messages) writer = Writer(document) diff --git a/spdx/writers/jsonyamlxml.py b/spdx/writers/jsonyamlxml.py index b77224716..3f0f8bac8 100644 --- a/spdx/writers/jsonyamlxml.py +++ b/spdx/writers/jsonyamlxml.py @@ -13,6 +13,7 @@ from rdflib import Literal from spdx import license, utils +from spdx.checksum import Checksum from spdx.package import ExternalPackageRef from spdx.relationship import Relationship from spdx.utils import update_dict_item_with_new_item @@ -45,14 +46,11 @@ def license(self, license_field): license_str = license_field.__str__() return license_str - def checksum_to_dict(self, checksum_field): + def checksum_to_dict(self, checksum_field: Checksum) -> Dict: """ - Return a dictionary representation of a spdx.checksum.Algorithm object + Return a dictionary representation of a checksum.Checksum object """ - #checksum_object = {'algorithm': "checksumAlgorithm_" + checksum_field.identifier.lower(), - # 'checksumValue': checksum_field.value} - #return checksum_object - return {'algorithm': checksum_field.identifier, 'checksumValue': checksum_field.value} + return {'algorithm': checksum_field.identifier.name, 'checksumValue': checksum_field.value} def spdx_id(self, spdx_id_field): return spdx_id_field.__str__().split("#")[-1] @@ -521,7 +519,7 @@ def create_ext_document_references(self): ] = ext_document_reference.spdx_document_uri ext_document_reference_object["checksum"] = self.checksum_to_dict( - ext_document_reference.check_sum + ext_document_reference.checksum ) ext_document_reference_objects.append(ext_document_reference_object) diff --git a/spdx/writers/rdf.py b/spdx/writers/rdf.py index 23936f969..193c46658 100644 --- a/spdx/writers/rdf.py +++ b/spdx/writers/rdf.py @@ -22,7 +22,7 @@ from spdx import file from spdx import license from spdx import utils -from spdx.checksum import ChecksumAlgorithmIdentifier +from spdx.checksum import Checksum from spdx.package import Package from spdx.parsers.loggers import ErrorMessages from spdx.relationship import Relationship @@ -45,11 +45,11 @@ def __init__(self, document, out): self.spdx_namespace = Namespace("http://spdx.org/rdf/terms#") self.graph = Graph() - def create_checksum_node(self, checksum): + def create_checksum_node(self, checksum: Checksum) -> BNode: """ Return a node representing spdx.checksum. """ - algo = ChecksumAlgorithmIdentifier[checksum.identifier].checksum_to_rdf() or 'checksumAlgorithm_sha1' + algo = checksum.identifier.algorithm_to_rdf_representation() or 'checksumAlgorithm_sha1' checksum_node = BNode() type_triple = (checksum_node, RDF.type, self.spdx_namespace.Checksum) self.graph.add(type_triple) @@ -668,7 +668,7 @@ def create_external_document_ref_node(self, ext_document_references): doc_uri_triple = (ext_doc_ref_node, self.spdx_namespace.spdxDocument, doc_uri) self.graph.add(doc_uri_triple) - checksum_node = self.create_checksum_node(ext_document_references.check_sum) + checksum_node = self.create_checksum_node(ext_document_references.checksum) self.graph.add((ext_doc_ref_node, self.spdx_namespace.checksum, checksum_node)) return ext_doc_ref_node diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index c717206e8..f3abbbd39 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -357,7 +357,7 @@ def write_document(document, out, validate=True): [ doc_ref.external_document_id, doc_ref.spdx_document_uri, - doc_ref.check_sum.identifier + ":" + doc_ref.check_sum.value, + doc_ref.checksum.identifier.name + ": " + doc_ref.checksum.value, ] ) write_value("ExternalDocumentRef", doc_ref_str, out) diff --git a/tests/data/doc_parse/spdx-expected.json b/tests/data/doc_parse/spdx-expected.json index ee30bd895..fce55cb2c 100644 --- a/tests/data/doc_parse/spdx-expected.json +++ b/tests/data/doc_parse/spdx-expected.json @@ -48,13 +48,16 @@ "copyrightText": "(c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP", "licenseComment": "This license is used by Jena", "notice": null, - "checksums": [{ - "identifier": "SHA1", - "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125" - },{ - "identifier": "SHA256", - "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f1250000000000000000" - }], + "checksums": [ + { + "identifier": "SHA1", + "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125" + }, + { + "identifier": "SHA256", + "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f1250000000000000000" + } + ], "licenseInfoInFiles": [ { "type": "Single", @@ -93,7 +96,8 @@ { "identifier": "SHA256", "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb120000000000000000" - }], + } + ], "licenseInfoInFiles": [ { "type": "Single", @@ -171,10 +175,12 @@ }, "copyrightText": " Copyright 2010, 2011 Source Auditor Inc.", "licenseComment": "The declared license information can be found in the NOTICE file at the root of the archive file", - "checksums": [{ - "identifier": "SHA1", - "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" - }], + "checksums": [ + { + "identifier": "SHA1", + "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + } + ], "licenseInfoFromFiles": [ { "type": "Single", diff --git a/tests/data/doc_write/json-simple-plus.json b/tests/data/doc_write/json-simple-plus.json index fae11a34d..e9be761eb 100644 --- a/tests/data/doc_write/json-simple-plus.json +++ b/tests/data/doc_write/json-simple-plus.json @@ -14,11 +14,12 @@ "documentDescribes": [ "SPDXRef-Package" ], - "packages": [ { - "SPDXID": "SPDXRef-Package", - "name": "some/path", - "downloadLocation": "NOASSERTION", - "copyrightText": "Some copyright", + "packages": [ + { + "SPDXID": "SPDXRef-Package", + "name": "some/path", + "downloadLocation": "NOASSERTION", + "copyrightText": "Some copyright", "packageVerificationCode": { "packageVerificationCodeValue": "SOME code" }, @@ -27,12 +28,12 @@ "algorithm": "SHA1", "checksumValue": "SOME-SHA1" }, - { - "algorithm": "SHA256", - "checksumValue": "SOME-SHA256" - } - ], - "licenseDeclared": "NOASSERTION", + { + "algorithm": "SHA256", + "checksumValue": "SOME-SHA256" + } + ], + "licenseDeclared": "NOASSERTION", "licenseConcluded": "NOASSERTION", "licenseInfoFromFiles": [ "LGPL-2.1-or-later" @@ -40,25 +41,25 @@ "hasFiles": [ "SPDXRef-File" ], - "primaryPackagePurpose" : "FILE", - "releaseDate" : "2021-01-01T12:00:00Z", - "builtDate" : "2021-01-01T12:00:00Z", - "validUntilDate" : "2022-01-01T12:00:00Z" + "primaryPackagePurpose": "FILE", + "releaseDate": "2021-01-01T12:00:00Z", + "builtDate": "2021-01-01T12:00:00Z", + "validUntilDate": "2022-01-01T12:00:00Z" } ], "files": [ { "SPDXID": "SPDXRef-File", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "SOME-SHA1" - }, - { - "algorithm": "SHA256", - "checksumValue": "SOME-SHA256" - } - ], + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "SOME-SHA1" + }, + { + "algorithm": "SHA256", + "checksumValue": "SOME-SHA256" + } + ], "licenseConcluded": "NOASSERTION", "copyrightText": "NOASSERTION", "fileName": "./some/path/tofile", diff --git a/tests/data/doc_write/json-simple.json b/tests/data/doc_write/json-simple.json index b2a5fa461..30c1b8943 100644 --- a/tests/data/doc_write/json-simple.json +++ b/tests/data/doc_write/json-simple.json @@ -28,12 +28,12 @@ "algorithm": "SHA1", "checksumValue": "SOME-SHA1" }, - { - "algorithm": "SHA256", - "checksumValue": "SOME-SHA256" - } - ], - "licenseDeclared": "NOASSERTION", + { + "algorithm": "SHA256", + "checksumValue": "SOME-SHA256" + } + ], + "licenseDeclared": "NOASSERTION", "licenseConcluded": "NOASSERTION", "licenseInfoFromFiles": [ "LGPL-2.1-only" @@ -41,10 +41,10 @@ "hasFiles": [ "SPDXRef-File" ], - "primaryPackagePurpose" : "FILE", - "releaseDate" : "2021-01-01T12:00:00Z", - "builtDate" : "2021-01-01T12:00:00Z", - "validUntilDate" : "2022-01-01T12:00:00Z" + "primaryPackagePurpose": "FILE", + "releaseDate": "2021-01-01T12:00:00Z", + "builtDate": "2021-01-01T12:00:00Z", + "validUntilDate": "2022-01-01T12:00:00Z" } ], "files": [ diff --git a/tests/data/formats/SPDXTagExample.tag b/tests/data/formats/SPDXTagExample.tag index 8d9c52847..855ba417a 100644 --- a/tests/data/formats/SPDXTagExample.tag +++ b/tests/data/formats/SPDXTagExample.tag @@ -73,9 +73,8 @@ FileName: src/org/spdx/parser/DOAPProject.java SPDXID: SPDXRef-File1 FileType: SOURCE FileType: TEXT -FileChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 +FileChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eB12 FileChecksum: SHA256: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb120000000000000000 -FileType: SOURCE LicenseConcluded: Apache-2.0 LicenseInfoInFile: Apache-2.0 FileCopyrightText: Copyright 2010, 2011 Source Auditor Inc. @@ -86,7 +85,6 @@ FileType: ARCHIVE FileType: OTHER FileChecksum: SHA1: 3ab4e1c67a2d28fced849ee1bb76e7391b93f125 FileChecksum: SHA256: 3ab4e1c67a2d28fced849ee1bb76e7391b93f1250000000000000000 -FileType: ARCHIVE LicenseConcluded: LicenseRef-1 LicenseInfoInFile: LicenseRef-1 LicenseComments: This license is used by Jena diff --git a/tests/test_builder.py b/tests/test_builder.py index bac2aedda..810d431e0 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -131,7 +131,7 @@ def test_checksum(self): chksum_val = "d6a770ba38583ed4bb4525bd96e50461655d2759" self.builder.set_ext_doc_id(self.document, "DocumentRef-spdx-tool-2.1") self.builder.set_chksum(self.document, chksum) - assert self.document.ext_document_references[-1].check_sum.value == chksum_val + assert self.document.ext_document_references[-1].checksum.value == chksum_val def test_add_ext_doc_refs(self): ext_doc_id_val = "DocumentRef-spdx-tool-2.1" @@ -149,7 +149,7 @@ def test_add_ext_doc_refs(self): assert ( self.document.ext_document_references[-1].spdx_document_uri == spdx_doc_uri ) - assert self.document.ext_document_references[-1].check_sum.value == chksum_val + assert self.document.ext_document_references[-1].checksum.value == chksum_val class TestEntityBuilder(TestCase): diff --git a/tests/test_checksum.py b/tests/test_checksum.py new file mode 100644 index 000000000..ebaaaba2a --- /dev/null +++ b/tests/test_checksum.py @@ -0,0 +1,48 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + +from spdx.checksum import ChecksumAlgorithm + + +@pytest.mark.parametrize("algorithm,expected", + [("SHA1", "checksumAlgorithm_sha1"), ("SHA224", "checksumAlgorithm_sha224"), + ("SHA3_256", "checksumAlgorithm_sha3_256"), ("BLAKE2B_256", "checksumAlgorithm_blake2b256"), + ("MD5", "checksumAlgorithm_md5")]) +def test_checksum_to_rdf(algorithm, expected): + test_algorithm = ChecksumAlgorithm[algorithm] + rdf_algorithm = test_algorithm.algorithm_to_rdf_representation() + + assert rdf_algorithm == expected + + +@pytest.mark.parametrize("expected,rdf_algorithm", + [(ChecksumAlgorithm.SHA1, "checksumAlgorithm_sha1"), + (ChecksumAlgorithm.SHA224, "checksumAlgorithm_sha224"), + (ChecksumAlgorithm.SHA3_256, "checksumAlgorithm_sha3_256"), + (ChecksumAlgorithm.BLAKE2B_256, "checksumAlgorithm_blake2b256"), + (ChecksumAlgorithm.MD5, "checksumAlgorithm_md5")]) +def test_checksum_from_rdf(rdf_algorithm, expected): + algorithm = ChecksumAlgorithm.checksum_from_rdf(rdf_algorithm) + + assert algorithm == expected + + +@pytest.mark.parametrize("rdf_algorithm", + ["_checksumAlgorithm_sha1", "checksumAlgorithm_sha_224", "checksumAlgorithm_sha3256", + "checksumAlgorithm_blake2b 256", "checksumAlgorithm_blake2b-256", + "checksumAlgorithm_bblake2b 256"]) +def test_checksum_from_wrong_rdf(rdf_algorithm): + with pytest.raises(ValueError) as error: + ChecksumAlgorithm.checksum_from_rdf(rdf_algorithm) + + assert str(error.value).startswith("Invalid algorithm for checksum") + diff --git a/tests/test_document.py b/tests/test_document.py index f1c1ea89c..7f9a1d677 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -16,7 +16,7 @@ from datetime import datetime from unittest import TestCase -from spdx.checksum import Algorithm +from spdx.checksum import Checksum, ChecksumAlgorithm from spdx.config import LICENSE_MAP, EXCEPTION_MAP from spdx.creationinfo import Tool from spdx.document import Document, ExternalDocumentRef @@ -65,26 +65,26 @@ def test_creation(self): document.add_ext_document_reference( ExternalDocumentRef('DocumentRef-spdx-tool-2.1', 'https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301', - Algorithm('SHA1', 'SOME-SHA1')) + Checksum(ChecksumAlgorithm.SHA1, 'SOME-SHA1')) ) assert document.comment is None assert document.version == Version(2, 1) assert document.data_license.identifier == 'AFL-1.1' assert document.ext_document_references[-1].external_document_id == 'DocumentRef-spdx-tool-2.1' assert document.ext_document_references[-1].spdx_document_uri == 'https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301' - assert document.ext_document_references[-1].check_sum.identifier == 'SHA1' - assert document.ext_document_references[-1].check_sum.value == 'SOME-SHA1' + assert document.ext_document_references[-1].checksum.identifier.name == 'SHA1' + assert document.ext_document_references[-1].checksum.value == 'SOME-SHA1' def test_document_validate_failures_returns_informative_messages(self): doc = Document(Version(2, 1), License.from_identifier('CC0-1.0'), 'Sample_Document-V2.1', spdx_id='SPDXRef-DOCUMENT', namespace='https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301') pack = doc.package = Package('some/path', NoAssert()) - pack.check_sum = Algorithm('SHA256', 'SOME-SHA256') + pack.set_checksum(Checksum(ChecksumAlgorithm.SHA256, 'SOME-SHA256')) file1 = File('./some/path/tofile') file1.name = './some/path/tofile' file1.spdx_id = 'SPDXRef-File' - file1.chksum = Algorithm('SHA1', 'SOME-SHA1') + file1.set_checksum(Checksum(ChecksumAlgorithm.SHA1, 'SOME-SHA1')) lic1 = License.from_identifier('LGPL-2.1-only') file1.add_lics(lic1) pack.add_lics_from_file(lic1) @@ -92,7 +92,6 @@ def test_document_validate_failures_returns_informative_messages(self): messages = doc.validate(messages) expected = ['Sample_Document-V2.1: Creation info missing created date.', 'Sample_Document-V2.1: No creators defined, must have at least one.', - 'Sample_Document-V2.1: some/path: At least one package checksum algorithm must be SHA1', 'Sample_Document-V2.1: some/path: Package download_location can not be None.'] assert sorted(expected) == sorted(messages) @@ -106,7 +105,7 @@ def test_document_is_valid_when_using_or_later_licenses(self): package = doc.package = Package(name='some/path', download_location=NoAssert()) package.spdx_id = 'SPDXRef-Package' package.cr_text = 'Some copyright' - package.set_checksum(Algorithm('SHA1', 'SOME-SHA1')) + package.set_checksum(Checksum(ChecksumAlgorithm.SHA1, 'SOME-SHA1')) package.verif_code = 'SOME code' package.license_declared = NoAssert() package.conc_lics = NoAssert() @@ -115,7 +114,7 @@ def test_document_is_valid_when_using_or_later_licenses(self): file1.name = './some/path/tofile' file1.spdx_id = 'SPDXRef-File' file1.file_types = [FileType.OTHER] - file1.set_checksum(Algorithm('SHA1', 'SOME-SHA1')) + file1.set_checksum(Checksum(ChecksumAlgorithm.SHA1, 'SOME-SHA1')) file1.conc_lics = NoAssert() file1.copyright = NoAssert() @@ -179,8 +178,8 @@ def _get_lgpl_doc(self, or_later=False): package.spdx_id = 'SPDXRef-Package' package.cr_text = 'Some copyright' package.verif_code = 'SOME code' - package.set_checksum(Algorithm('SHA1', 'SOME-SHA1')) - package.set_checksum(Algorithm('SHA256', 'SOME-SHA256')) + package.set_checksum(Checksum(ChecksumAlgorithm.SHA1, 'SOME-SHA1')) + package.set_checksum(Checksum(ChecksumAlgorithm.SHA256, 'SOME-SHA256')) package.license_declared = NoAssert() package.conc_lics = NoAssert() package.primary_package_purpose = PackagePurpose.FILE @@ -192,8 +191,8 @@ def _get_lgpl_doc(self, or_later=False): file1 = File('./some/path/tofile') file1.name = './some/path/tofile' file1.spdx_id = 'SPDXRef-File' - file1.set_checksum(Algorithm('SHA1', 'SOME-SHA1')) - file1.set_checksum(Algorithm('SHA256', 'SOME-SHA256')) + file1.set_checksum(Checksum(ChecksumAlgorithm.SHA1, 'SOME-SHA1')) + file1.set_checksum(Checksum(ChecksumAlgorithm.SHA256, 'SOME-SHA256')) file1.conc_lics = NoAssert() file1.copyright = NoAssert() file1.file_types = [FileType.OTHER, FileType.SOURCE] @@ -247,7 +246,7 @@ def _get_lgpl_multi_package_doc(self, or_later=False): file1 = File('./some/path/tofile') file1.name = './some/path/tofile' file1.spdx_id = 'SPDXRef-File' - file1.set_checksum(Algorithm('SHA1', 'SOME-SHA1')) + file1.set_checksum(Checksum(ChecksumAlgorithm.SHA1, 'SOME-SHA1')) file1.conc_lics = NoAssert() file1.copyright = NoAssert() diff --git a/tests/test_jsonyamlxml_writer.py b/tests/test_jsonyamlxml_writer.py index ff1ebbbbf..0ed226e36 100644 --- a/tests/test_jsonyamlxml_writer.py +++ b/tests/test_jsonyamlxml_writer.py @@ -5,7 +5,7 @@ import pytest -from spdx.checksum import Algorithm +from spdx.checksum import Checksum, ChecksumAlgorithm from spdx.document import Document from spdx.file import File from spdx.license import License @@ -148,7 +148,7 @@ def minimal_document(): def minimal_file(): file = File(name="Example File", spdx_id="SPDXRef-File") - file.set_checksum(Algorithm('SHA1', 'some-sha1-value')) + file.set_checksum(Checksum(ChecksumAlgorithm.SHA1, 'some-sha1-value')) return file diff --git a/tests/test_package.py b/tests/test_package.py index 944f2516d..3b8325b02 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -12,7 +12,7 @@ import unittest from unittest import TestCase -from spdx.checksum import Algorithm +from spdx.checksum import Checksum from spdx.package import Package @@ -25,11 +25,11 @@ def test_calc_verif_code(self): def test_package_with_non_sha1_check_sum(self): package = Package() - package.set_checksum(Algorithm("SHA256", '')) + package.set_checksum(Checksum("SHA256", '')) # Make sure that validation still works despite the checksum not being SHA1 messages = [] - messages = package.validate_checksum(messages) + package.validate_checksums(messages) if __name__ == '__main__': diff --git a/tests/utils_test.py b/tests/utils_test.py index ae3362f22..c196168f1 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -331,7 +331,7 @@ def checksum_to_dict(cls, checksum): if checksum is None: return None return OrderedDict([ - ('identifier', checksum.identifier), + ('identifier', checksum.identifier.name), ('value', checksum.value)]) @classmethod @@ -392,8 +392,8 @@ def files_to_list(cls, files): lics_in_files = sorted(file.licenses_in_file, key=lambda lic: lic.identifier) contributors = sorted(file.contributors, key=lambda c: c.name) chk_sums = [] - for _, chk_sum in file.checksums.items(): - chk_sums.append(cls.checksum_to_dict(chk_sum)) + for checksum in file.checksums.values(): + chk_sums.append(cls.checksum_to_dict(checksum)) file_dict = OrderedDict([ ('id', file.spdx_id), @@ -427,7 +427,7 @@ def ext_document_references_to_list(cls, ext_doc_refs): ext_doc_ref_dict = OrderedDict([ ('externalDocumentId', ext_doc_ref.external_document_id), ('spdxDocument', ext_doc_ref.spdx_document_uri), - ('checksum', cls.checksum_to_dict(ext_doc_ref.check_sum)), + ('checksum', cls.checksum_to_dict(ext_doc_ref.checksum)), ]) ext_doc_refs_list.append(ext_doc_ref_dict)