Add unpackaged files

spdx/document.py

@@ -279,6 +279,7 @@ class Document(object):
     - documentNamespace: SPDX document specific namespace. Mandatory, one. Type: str
     - creation_info: SPDX file creation info. Mandatory, one. Type: CreationInfo
     - package: Package described by this document. Mandatory, one. Type: Package
+    - unpackaged_files: Files not part of any package. Optional zero or more. Type: File.
     - extracted_licenses: List of licenses extracted that are not part of the
       SPDX license list. Optional, many. Type: ExtractedLicense.
     - reviews: SPDX document review information, Optional zero or more.
@@ -316,6 +317,7 @@ def __init__(
         self.packages = []
         if package is not None:
             self.packages.append(package)
+        self.unpackaged_files = []
         self.extracted_licenses = []
         self.reviews = []
         self.annotations = []
@@ -331,6 +333,9 @@ def add_annotation(self, annotation):
     def add_relationships(self, relationship):
         self.relationships.append(relationship)
 
+    def add_file(self, value):
+        self.unpackaged_files.append(value)
+
     def add_extr_lic(self, lic):
         self.extracted_licenses.append(lic)
 
@@ -370,10 +375,16 @@ def package(self, value):
 
     @property
     def files(self):
+        warnings.warn('document.package and document.files are deprecated; '
+                      'use document.packages instead',
+                      DeprecationWarning)
         return self.package.files
 
     @files.setter
     def files(self, value):
+        warnings.warn('document.package and document.files are deprecated; '
+                      'use document.packages instead',
+                      DeprecationWarning)
         self.package.files = value
 
     @property
@@ -400,6 +411,7 @@ def validate(self, messages=None):
         self.validate_creation_info(messages)
         self.validate_packages(messages)
         self.validate_extracted_licenses(messages)
+        self.validate_unpackaged_files(messages)
         self.validate_reviews(messages)
         self.validate_snippet(messages)
         self.validate_annotations(messages)
@@ -486,4 +498,8 @@ def validate_extracted_licenses(self, messages):
                 messages.append(
                     "Document extracted licenses must be of type "
                     "spdx.document.ExtractedLicense and not " + type(lic)
-                )
+                )
+
+    def validate_unpackaged_files(self, messages):
+        for spdx_file in self.unpackaged_files:
+            messages = spdx_file.validate(messages)

spdx/parsers/rdf.py

@@ -17,6 +17,7 @@
 from rdflib import Namespace
 from rdflib import RDF
 from rdflib import RDFS
+from rdflib.term import URIRef
 
 from spdx import document
 from spdx import utils
@@ -365,6 +366,7 @@ def parse_package(self, p_term):
         self.p_pkg_summary(p_term, self.spdx_namespace["summary"])
         self.p_pkg_descr(p_term, self.spdx_namespace["description"])
         self.p_pkg_comment(p_term, self.spdx_namespace["comment"])
+        self.p_pkg_has_file(p_term, self.spdx_namespace["hasFile"])
 
     def p_pkg_cr_text(self, p_term, predicate):
         try:
@@ -396,6 +398,14 @@ def p_pkg_comment(self, p_term, predicate):
         except CardinalityError:
             self.more_than_one_error("package comment")
 
+    def p_pkg_has_file(self, p_term, predicate):
+        for _, _, has_file_object in self.graph.triples((p_term, predicate, None)):
+            for index, spdx_file in enumerate(self.doc.unpackaged_files):
+                object_spdx_id = str(has_file_object)
+                if spdx_file.spdx_id == object_spdx_id:
+                    self.doc.packages[-1].files.append(spdx_file)
+                    del self.doc.unpackaged_files[index]
+
     def p_pkg_attribution_text(self, p_term, predicate):
         try:
             for _, _, attribute_text in self.graph.triples((p_term, predicate, None)):
@@ -1299,6 +1309,11 @@ def parse(self, fil):
         ):
             self.handle_extracted_license(s)
 
+        for s, _p, o in self.graph.triples(
+            (URIRef(self.doc.spdx_id), self.spdx_namespace["referencesFile"], None)
+        ):
+            self.parse_file(o)  # Document-level ref file (unpackaged)
+
         for s, _p, o in self.graph.triples(
             (None, RDF.type, self.spdx_namespace["Package"])
         ):
@@ -1312,7 +1327,9 @@ def parse(self, fil):
         for s, _p, o in self.graph.triples(
             (None, self.spdx_namespace["referencesFile"], None)
         ):
-            self.parse_file(o)
+            if not str(s) == self.doc.spdx_id:
+                # self.parse_file(o)  # TODO: Package child ref file?
+                raise NotImplementedError(f"Processing {s} referencesFile")
 
         for s, _p, o in self.graph.triples(
             (None, RDF.type, self.spdx_namespace["Snippet"])

spdx/parsers/rdfbuilders.py

@@ -386,9 +386,9 @@ def set_file_chksum(self, doc, chk_sum):
         Set the file check sum, if not already set.
         chk_sum - A string
         Raise CardinalityError if already defined.
-        Raise OrderError if no package previously defined.
+        Raise OrderError if no package, package file, or unpackaged file is defined.
         """
-        if self.has_package(doc) and self.has_file(doc):
+        if (self._build_file_valid(doc)):
             if not self.file_chksum_set:
                 self.file_chksum_set = True
                 self.file(doc).chk_sum = checksum.Algorithm("SHA1", chk_sum)
@@ -403,7 +403,7 @@ def set_file_license_comment(self, doc, text):
         Raise OrderError if no package or file defined.
         Raise CardinalityError if more than one per file.
         """
-        if self.has_package(doc) and self.has_file(doc):
+        if (self._build_file_valid(doc)):
             if not self.file_license_comment_set:
                 self.file_license_comment_set = True
                 self.file(doc).license_comment = text
@@ -417,7 +417,7 @@ def set_file_attribution_text(self, doc, text):
         """
         Set the file's attribution text.
         """
-        if self.has_package(doc) and self.has_file(doc):
+        if (self._build_file_valid(doc)):
             self.assert_package_exists()
             self.file(doc).attribution_text = text
             return True
@@ -427,7 +427,7 @@ def set_file_copyright(self, doc, text):
         Raise OrderError if no package or file defined.
         Raise CardinalityError if more than one.
         """
-        if self.has_package(doc) and self.has_file(doc):
+        if (self._build_file_valid(doc)):
             if not self.file_copytext_set:
                 self.file_copytext_set = True
                 self.file(doc).copyright = text
@@ -442,7 +442,7 @@ def set_file_comment(self, doc, text):
         Raise OrderError if no package or no file defined.
         Raise CardinalityError if more than one comment set.
         """
-        if self.has_package(doc) and self.has_file(doc):
+        if (self._build_file_valid(doc)):
             if not self.file_comment_set:
                 self.file_comment_set = True
                 self.file(doc).comment = text
@@ -457,7 +457,7 @@ def set_file_notice(self, doc, text):
         Raise OrderError if no package or file defined.
         Raise CardinalityError if more than one.
         """
-        if self.has_package(doc) and self.has_file(doc):
+        if (self._build_file_valid(doc)):
             if not self.file_notice_set:
                 self.file_notice_set = True
                 self.file(doc).notice = tagvaluebuilders.str_from_text(text)

spdx/parsers/tagvalue.py

@@ -564,7 +564,8 @@ def p_spdx_id(self, p):
         value = p[2]
         if not self.builder.doc_spdx_id_set:
             self.builder.set_doc_spdx_id(self.document, value)
-        elif not self.builder.package_spdx_id_set:
+        elif (not self.builder.package_spdx_id_set
+                and self.builder.package_set):
             self.builder.set_pkg_spdx_id(self.document, value)
         else:
             self.builder.set_file_spdx_id(self.document, value)