[issue-181] add files at document-level

spdx/creationinfo.py

@@ -154,7 +154,7 @@ def remove_creator(self, creator):
         self.creators.remove(creator)
 
     def set_created_now(self):
-        self.created = datetime.utcnow()
+        self.created = datetime.utcnow().replace(microsecond=0)
 
     @property
     def created_iso_format(self):

spdx/document.py

@@ -8,12 +8,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from typing import List, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from spdx.file import File
 from spdx.license import ExtractedLicense
 from spdx.parsers.loggers import ErrorMessages
 
 import warnings
 
 from functools import total_ordering
+from spdx.relationship import Relationship
+
+from spdx.relationship import Relationship
 
 
 @total_ordering
@@ -48,14 +55,15 @@ def __lt__(self, other):
             other.check_sum,
         )
 
-    def validate(self, messages):
+    def validate(self, messages: ErrorMessages) -> ErrorMessages:
         """
         Check that all the fields are valid.
         Appends any error messages to messages parameter shall be a ErrorMessages.
         """
         self.validate_ext_doc_id(messages)
         self.validate_spdx_doc_uri(messages)
         self.validate_checksum(messages)
+        return messages
 
     def validate_ext_doc_id(self, messages):
         if not self.external_document_id:
@@ -118,13 +126,14 @@ def __init__(
         self.comment = comment
         self.namespace = namespace
         self.creation_info = CreationInfo()
+        self.files: List['File'] = []
         self.packages = []
         if package is not None:
             self.packages.append(package)
         self.extracted_licenses = []
         self.reviews = []
         self.annotations = []
-        self.relationships = []
+        self.relationships: List[Relationship] = []
         self.snippet = []
 
     def add_review(self, review):
@@ -148,6 +157,9 @@ def add_snippet(self, snip):
     def add_package(self, package):
         self.packages.append(package)
 
+    def add_file(self, file: 'File') -> None:
+        self.files.append(file)
+
     # For backwards compatibility with older versions, we support a
     # mode where the first package in a document may be referred to as
     # the document's "package", and the files it contains may be
@@ -173,17 +185,6 @@ def package(self, value):
         else:
             self.packages[0] = value
 
-    @property
-    def files(self):
-        if self.packages:
-            return self.package.files
-        else:
-            return []
-
-    @files.setter
-    def files(self, value):
-        self.package.files = value
-
     @property
     def has_comment(self):
         return self.comment is not None
@@ -206,6 +207,7 @@ def validate(self, messages=None):
         self.validate_namespace(messages)
         self.validate_ext_document_references(messages)
         self.validate_creation_info(messages)
+        self.validate_files(messages)
         self.validate_packages(messages)
         self.validate_extracted_licenses(messages)
         self.validate_reviews(messages)
@@ -251,10 +253,15 @@ def validate_ext_document_references(self, messages):
                     "External document references must be of the type "
                     "spdx.document.ExternalDocumentRef and not " + str(type(doc))
                 ]
+
     def validate_reviews(self, messages):
         for review in self.reviews:
             messages = review.validate(messages)
 
+    def validate_files(self, messages: ErrorMessages) -> None:
+        for file in self.files:
+            messages = file.validate(messages)
+
     def validate_annotations(self, messages):
         for annotation in self.annotations:
             messages = annotation.validate(messages)

spdx/package.py

@@ -9,11 +9,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import hashlib
 from datetime import datetime
 from enum import Enum
 from functools import reduce
-from typing import List, Optional
+from typing import Optional
 
 from spdx import checksum
 from spdx import creationinfo
@@ -74,7 +73,6 @@ class Package(object):
      - description: Optional str.
      - comment: Comments about the package being described, optional one.
      Type: str
-     - files: List of files in package, at least one.
      - verif_exc_files: list of file names excluded from verification code or None.
      - ext_pkg_refs: External references referenced within the given package.
      Optional, one or many. Type: ExternalPackageRef
@@ -113,7 +111,6 @@ def __init__(
         self.description = None
         self.comment = None
         self.attribution_text = None
-        self.files = []
         self.verif_exc_files = []
         self.pkg_ext_refs = []
         self.primary_package_purpose: Optional[PackagePurpose] = None
@@ -140,9 +137,6 @@ def checksum(self):
     def checksum(self, value):
         self.checksums[0] = value
 
-    def add_file(self, fil):
-        self.files.append(fil)
-
     def add_lics_from_file(self, lics):
         self.licenses_from_files.append(lics)
 
@@ -162,7 +156,6 @@ def validate(self, messages):
         self.validate_checksum(messages)
         self.validate_optional_str_fields(messages)
         self.validate_mandatory_str_fields(messages)
-        self.validate_files(messages)
         self.validate_pkg_ext_refs(messages)
         self.validate_optional_fields(messages)
         messages.pop_context()
@@ -245,13 +238,6 @@ def validate_pkg_ext_refs(self, messages):
 
         return messages
 
-    def validate_files(self, messages):
-        if self.are_files_analyzed:
-            for file in self.files:
-                messages = file.validate(messages)
-
-        return messages
-
     def validate_optional_str_fields(self, messages):
         """Fields marked as optional and of type string in class
         docstring must be of a type that provides __str__ method.
@@ -311,25 +297,6 @@ def validate_checksum(self, messages):
 
         return messages
 
-    def calc_verif_code(self):
-        hashes = []
-
-        for file_entry in self.files:
-            if (
-                isinstance(file_entry.chksum, checksum.Algorithm)
-                and file_entry.chksum.identifier == "SHA1"
-            ):
-                sha1 = file_entry.chksum.value
-            else:
-                sha1 = file_entry.calc_chksum()
-            hashes.append(sha1)
-
-        hashes.sort()
-
-        sha1 = hashlib.sha1()
-        sha1.update("".join(hashes).encode("utf-8"))
-        return sha1.hexdigest()
-
     def has_optional_field(self, field):
         return bool(getattr(self, field, None))
 

spdx/parsers/jsonyamlxml.py

@@ -10,12 +10,12 @@
 # limitations under the License.
 from datetime import datetime
 from enum import Enum, auto
-from typing import List, Dict, Tuple
+from typing import List, Dict, Tuple, Callable
 
 from spdx import document
 from spdx import utils
 from spdx.license import LicenseConjunction, LicenseDisjunction
-from spdx.package import ExternalPackageRef, PackagePurpose
+from spdx.package import ExternalPackageRef, PackagePurpose, Package
 from spdx.parsers import rdf
 from spdx.parsers.builderexceptions import SPDXValueError, CardinalityError, OrderError
 from spdx.parsers.loggers import ErrorMessages
@@ -1110,6 +1110,16 @@ def parse_file_chksum(self, file_chksum):
         else:
             self.value_error("FILE_CHECKSUM", file_chksum)
 
+    def parse_files(self, files: List[Dict]) -> None:
+        if files is None:
+            return
+        if isinstance(files, list):
+            for file in files:
+                self.parse_file(file)
+        else:
+            self.value_error("FILES", files)
+
+
 
 class PackageParser(BaseParser):
     def __init__(self, builder, logger):
@@ -1120,7 +1130,7 @@ def package(self):
         # current package being parsed is the last one
         return self.document.packages[-1]
 
-    def parse_package(self, package):
+    def parse_package(self, package: Package, method_to_parse_relationship: Callable):
         """
         Parse Package Information fields
         - package: Python dict with Package Information fields in it
@@ -1149,7 +1159,7 @@ def parse_package(self, package):
             self.parse_pkg_description(package.get("description"))
             self.parse_annotations(package.get("annotations"), spdx_id=package.get("SPDXID"))
             self.parse_pkg_attribution_text(package.get("attributionTexts"))
-            self.parse_pkg_files(package.get("files"))
+            self.parse_pkg_files(package.get("hasFiles"), method_to_parse_relationship)
             self.parse_pkg_chksum(package.get("sha1"))
             self.parse_package_external_refs(package.get("externalRefs"))
             self.parse_primary_package_purpose(package.get("primaryPackagePurpose"))
@@ -1561,24 +1571,24 @@ def parse_pkg_description(self, pkg_description):
         elif pkg_description is not None:
             self.value_error("PKG_DESCRIPTION", pkg_description)
 
-    def parse_pkg_files(self, pkg_files):
+    def parse_pkg_files(self, pkg_has_files: List[str], method_to_parse_relationship: Callable) -> None:
         """
         Parse Package files
-        - pkg_files: Python list of dicts as in FileParser.parse_file
+        - pkg_has_files: Python list of spdx_ids
         """
         if not self.package.are_files_analyzed:
-            if pkg_files is not None:
-                self.value_error("PKG_FILES", pkg_files)
+            if pkg_has_files is not None:
+                self.value_error("PKG_FILES", pkg_has_files)
             return
 
-        if isinstance(pkg_files, list):
-            for pkg_file in pkg_files:
-                if isinstance(pkg_file, dict):
-                    self.parse_file(pkg_file.get("File"))
+        if isinstance(pkg_has_files, list):
+            for pkg_file_spdx_id in pkg_has_files:
+                if isinstance(pkg_file_spdx_id, str):
+                    method_to_parse_relationship(self.package.spdx_id, "CONTAINS", pkg_file_spdx_id)
                 else:
-                    self.value_error("PKG_FILE", pkg_file)
-        elif pkg_files is not None:
-            self.value_error("PKG_FILES", pkg_files)
+                    self.value_error("PKG_FILE", pkg_file_spdx_id)
+        elif pkg_has_files is not None:
+            self.value_error("PKG_HAS_FILES", pkg_has_files)
 
     def parse_pkg_chksum(self, pkg_chksum):
         """
@@ -1730,6 +1740,7 @@ def parse(self):
         self.parse_snippets(self.document_object.get("snippets"))
 
         self.parse_packages(self.document_object.get("packages"))
+        self.parse_files(self.document_object.get("files"))
 
         if self.document_object.get("documentDescribes"):
             self.parse_doc_described_objects(self.document_object.get("documentDescribes"))
@@ -1853,7 +1864,7 @@ def parse_packages(self, packages):
             return
         if isinstance(packages, list):
             for package in packages:
-                self.parse_package(package)
+                self.parse_package(package, self.parse_relationship)
             return True
         else:
             self.value_error("PACKAGES", packages)

spdx/parsers/tagvalue.py

@@ -575,6 +575,9 @@ def p_file_name_1(self, p):
         try:
             value = p[2]
             self.builder.set_file_name(self.document, value)
+            self.builder.set_current_file_name(value)
+            self.builder.set_current_file_id(None)
+
         except OrderError:
             self.order_error("FileName", "PackageName", p.lineno(1))
 
@@ -587,12 +590,30 @@ def p_file_name_2(self, p):
     def p_spdx_id(self, p):
         """spdx_id : SPDX_ID LINE"""
         value = p[2]
-        if not self.builder.doc_spdx_id_set:
-            self.builder.set_doc_spdx_id(self.document, value)
-        elif not self.builder.package_spdx_id_set:
-            self.builder.set_pkg_spdx_id(self.document, value)
-        else:
-            self.builder.set_file_spdx_id(self.document, value)
+        try:
+            # first parsed spdx id belongs to the document
+            if not self.builder.doc_spdx_id_set:
+                self.builder.set_doc_spdx_id(self.document, value)
+
+            # else if a package is in scope that doesn't have an id yet, the parsed spdx id belongs to the package
+            elif self.builder.current_package_has_name() \
+                    and not self.builder.current_package_has_id():
+                self.builder.set_pkg_spdx_id(self.document, value)
+                self.builder.set_current_package_id(value)
+
+            # else if a file is in scope that doesn't have an id yet, the parsed spdx id belongs to the file
+            elif self.builder.current_file_has_name() \
+                    and not self.builder.current_file_has_id():
+                self.builder.set_file_spdx_id(self.document, value)
+                self.builder.set_current_file_id(value)
+                if self.builder.has_current_package():
+                    self.builder.add_relationship(self.document,
+                                                  self.builder.current_package["spdx_id"] + " CONTAINS " + value)
+            else:
+                raise SPDXValueError("SPDX ID couldn't be assigned properly.  Line no. {0}")
+        except SPDXValueError as err:
+            self.error = True
+            self.logger.log(err.msg.format(p.lineno(2)))
 
     def p_file_comment_1(self, p):
         """file_comment : FILE_COMMENT text_or_line"""
@@ -1100,6 +1121,10 @@ def p_package_name(self, p):
         try:
             value = p[2]
             self.builder.create_package(self.document, value)
+            self.builder.set_current_package_name(value)
+            self.builder.set_current_package_id(None)
+            self.builder.set_current_file_name(None)  # start of a new package implies new file
+            self.builder.set_current_file_id(None)
         except CardinalityError:
             self.more_than_one_error("PackageName", p.lineno(1))