add byte range and line ranges to snippet

spdx/parsers/jsonyamlxml.py

@@ -9,7 +9,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from datetime import datetime
-from typing import List, Dict
+from enum import Enum, auto
+from typing import List, Dict, Tuple
 
 from spdx import document
 from spdx import utils
@@ -19,6 +20,7 @@
 from spdx.parsers import rdf
 from spdx.parsers.builderexceptions import SPDXValueError, CardinalityError, OrderError
 from spdx.parsers.loggers import ErrorMessages
+from spdx.snippet import Snippet
 from spdx.utils import UnKnown
 
 ERROR_MESSAGES = rdf.ERROR_MESSAGES
@@ -486,10 +488,19 @@ def parse_relationship_comment(self, relationship_comment):
             self.value_error("RELATIONSHIP_COMMENT", relationship_comment)
 
 
+class RangeType(Enum):
+    BYTE = auto()
+    LINE = auto()
+
+
 class SnippetParser(BaseParser):
     def __init__(self, builder, logger):
         super(SnippetParser, self).__init__(builder, logger)
 
+    @property
+    def snippet(self) -> Snippet:
+        return self.document.snippet[-1]
+
     def parse_snippets(self, snippets):
         """
         Parse Snippet Information fields
@@ -516,6 +527,7 @@ def parse_snippets(self, snippets):
                             snippet.get("licenseInfoFromSnippet")
                         )
                         self.parse_annotations(snippet.get("annotations"), spdx_id=snippet.get("SPDXID"))
+                        self.parse_snippet_ranges(snippet.get("ranges"))
                 else:
                     self.value_error("SNIPPET", snippet)
 
@@ -667,6 +679,61 @@ def parse_snippet_license_info_from_snippet(self, license_info_from_snippet):
         elif license_info_from_snippet is not None:
             self.value_error("SNIPPET_LIC_INFO_FIELD", license_info_from_snippet)
 
+    def parse_snippet_ranges(self, ranges_from_snippet: List[Dict]) -> None:
+        """
+        Parse ranges (byte range and optional line range) from snippet
+        - ranges_from_snippet; Python list of dict
+        """
+        if not isinstance(ranges_from_snippet, list):
+            self.value_error("SNIPPET_RANGES", ranges_from_snippet)
+            return
+
+        for range_dict in ranges_from_snippet:
+            try:
+                range_type = self.validate_range_and_get_type(range_dict)
+                start_end_tuple: Tuple[int, int] = SnippetParser.get_start_end_tuple(range_dict, range_type)
+            except SPDXValueError:
+                self.value_error("SNIPPET_RANGE", range_dict)
+                return
+
+            if range_type == RangeType.BYTE:
+                self.snippet.byte_range = start_end_tuple
+            elif range_type == RangeType.LINE:
+                self.snippet.line_range = start_end_tuple
+
+    @staticmethod
+    def get_start_end_tuple(range_dict: Dict, range_type: RangeType) -> Tuple[int, int]:
+        end_pointer = range_dict["endPointer"]
+        start_pointer = range_dict["startPointer"]
+        if range_type == RangeType.BYTE:
+            start = int(start_pointer["offset"])
+            end = int(end_pointer["offset"])
+        else:
+            start = int(start_pointer["lineNumber"])
+            end = int(end_pointer["lineNumber"])
+        if start > end:
+            raise SPDXValueError("Snippet::ranges")
+
+        return start, end
+
+    def validate_range_and_get_type(self, range_dict: Dict) -> RangeType:
+        if ("startPointer" not in range_dict) or ("endPointer" not in range_dict):
+            raise SPDXValueError("Snippet::ranges")
+        start_pointer_type = self.validate_pointer_and_get_type(range_dict["startPointer"])
+        end_pointer_type = self.validate_pointer_and_get_type(range_dict["endPointer"])
+        if start_pointer_type != end_pointer_type:
+            raise SPDXValueError("Snippet::ranges")
+        return start_pointer_type
+
+    def validate_pointer_and_get_type(self, pointer: Dict) -> RangeType:
+        if self.snippet.snip_from_file_spdxid != pointer["reference"]:
+            raise SPDXValueError("Snippet::ranges")
+        if ("offset" in pointer and "lineNumber" in pointer) or (
+                "offset" not in pointer and "lineNumber" not in pointer):
+            raise SPDXValueError("Snippet::ranges")
+
+        return RangeType.BYTE if "offset" in pointer else RangeType.LINE
+
 
 class ReviewParser(BaseParser):
     def __init__(self, builder, logger):
@@ -1574,7 +1641,7 @@ def parse_built_date(self, built_date: str):
             self.package.built_date = parsed_date
         else:
             self.value_error("BUILT_DATE", built_date)
-            
+
     def parse_valid_until_date(self, valid_until_date: str):
         if valid_until_date is None:
             return

spdx/parsers/lexers/tagvalue.py

@@ -95,6 +95,8 @@ class Lexer(object):
         "SnippetLicenseConcluded": "SNIPPET_LICS_CONC",
         "LicenseInfoInSnippet": "SNIPPET_LICS_INFO",
         "SnippetAttributionText": "SNIPPET_ATTRIBUTION_TEXT",
+        "SnippetByteRange": "SNIPPET_BYTE_RANGE",
+        "SnippetLineRange": "SNIPPET_LINE_RANGE",
         # Common
         "NOASSERTION": "NO_ASSERT",
         "UNKNOWN": "UN_KNOWN",
@@ -115,6 +117,7 @@ class Lexer(object):
         "PERSON_VALUE",
         "DATE",
         "LINE",
+        "RANGE",
         "CHKSUM",
         "DOC_REF_ID",
         "DOC_URI",
@@ -147,6 +150,12 @@ def t_CHKSUM(self, t):
         t.value = t.value[1:].strip()
         return t
 
+    def t_RANGE(self, t):
+        r":\s*\d+:\d+"
+        t.value = t.value[1:].strip()
+        return t
+
+
     def t_DOC_REF_ID(self, t):
         r":\s*DocumentRef-([A-Za-z0-9\+\.\-]+)"
         t.value = t.value[1:].strip()

spdx/parsers/parse_anything.py

@@ -46,6 +46,6 @@ def parse_file(fn):
     with open(fn) as f:
         if read_data:
             data = f.read()
-            return  p.parse(data)
+            return p.parse(data)
         else:
-            return  p.parse(f)
+            return p.parse(f)

spdx/parsers/tagvalue.py

@@ -204,6 +204,8 @@ def p_attrib(self, p):
                   | snip_file_spdx_id
                   | snip_lics_conc
                   | snip_lics_info
+                  | snip_byte_range
+                  | snip_line_range
                   | extr_lic_id
                   | extr_lic_text
                   | extr_lic_name
@@ -1251,6 +1253,42 @@ def p_snippet_lics_info_1(self, p):
         msg = ERROR_MESSAGES["SNIP_LICS_INFO_VALUE"].format(p.lineno(1))
         self.logger.log(msg)
 
+    def p_snippet_byte_range(self, p):
+        """snip_byte_range : SNIPPET_BYTE_RANGE RANGE"""
+        try:
+            self.builder.set_snippet_byte_range(self.document, p[2])
+        except OrderError:
+            self.order_error("SnippetByteRange", "SnippetSPDXID", p.lineno(1))
+        except SPDXValueError:
+            self.error = True
+            msg = "Value for Snippet ByteRange invalid in line {}.".format(p.lineno(1))
+            self.logger.log(msg)
+
+    def p_snippet_byte_range_1(self, p):
+        """snip_byte_range : SNIPPET_BYTE_RANGE error"""
+
+        self.error = True
+        msg = "Reading of SnippetByteRange failed for line {}.".format(p.lineno(1))
+        self.logger.log(msg)
+
+    def p_snippet_line_range(self, p):
+        """snip_line_range : SNIPPET_LINE_RANGE RANGE"""
+        try:
+            self.builder.set_snippet_line_range(self.document, p[2])
+        except OrderError:
+            self.order_error("SnippetLineRange", "SnippetSPDXID", p.lineno(1))
+        except SPDXValueError:
+            self.error = True
+            msg = "Value for Snippet LineRange invalid in line {}.".format(p.lineno(1))
+            self.logger.log(msg)
+
+    def p_snippet_line_range_1(self, p):
+        """snip_line_range : SNIPPET_LINE_RANGE error"""
+
+        self.error = True
+        msg = "Reading of SnippetLineRange failed for line {}.".format(p.lineno(1))
+        self.logger.log(msg)
+
     def p_snip_lic_info_value_1(self, p):
         """snip_lic_info_value : NONE"""
         p[0] = utils.SPDXNone()

spdx/parsers/tagvaluebuilders.py

@@ -1542,6 +1542,32 @@ def set_snippet_lics_info(self, doc, lics_info):
         else:
             raise SPDXValueError("Snippet::LicenseInfoInSnippet")
 
+    def set_snippet_byte_range(self, doc, parsed):
+        """
+        Raise OrderError if no snippet previously defined.
+        Raise SPDXValueError if the data is malformed.
+        """
+        self.assert_snippet_exists()
+        startpoint = int(parsed.split(":")[0])
+        endpoint = int(parsed.split(":")[-1])
+        if startpoint <= endpoint:
+            doc.snippet[-1].byte_range = (startpoint, endpoint)
+        else:
+            raise SPDXValueError("Snippet::ByteRange")
+
+    def set_snippet_line_range(self, doc, parsed):
+        """
+        Raise OrderError if no snippet previously defined.
+        Raise SPDXValueError if the data is malformed.
+        """
+        self.assert_snippet_exists()
+        startpoint = int(parsed.split(":")[0])
+        endpoint = int(parsed.split(":")[-1])
+        if startpoint <= endpoint:
+            doc.snippet[-1].line_range = (startpoint, endpoint)
+        else:
+            raise SPDXValueError("Snippet::LineRange")
+
     def reset_snippet(self):
         # FIXME: this state does not make sense
         self.snippet_spdx_id_set = False

spdx/parsers/xmlparser.py

@@ -47,7 +47,8 @@ def __init__(self, builder, logger):
             "packages",
             "checksums",
             "hasFiles",
-            "externalRefs"
+            "externalRefs",
+            "ranges"
         }
 
     def parse(self, file):

spdx/snippet.py

@@ -8,6 +8,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from typing import Tuple, Optional
 
 from spdx import document
 from spdx import utils
@@ -35,10 +36,14 @@ class Snippet(object):
      Mandatory, one or more. Type: document.License or utils.SPDXNone or
      utils.NoAssert.
      - attribution_text: optional string.
+     - byte_range: Defines the byte range in the original host file that the
+     snippet information applies to. Mandatory, one. Type (int, int)
+     - line_range: Defines the line range in the original host file that the
+     snippet information applies to. Optional, one. Type (int, int)
     """
 
     def __init__(
-        self, spdx_id=None, copyright=None, snip_from_file_spdxid=None, conc_lics=None
+        self, spdx_id=None, copyright=None, snip_from_file_spdxid=None, conc_lics=None, byte_range=None
     ):
         self.spdx_id = spdx_id
         self.name = None
@@ -49,6 +54,8 @@ def __init__(
         self.snip_from_file_spdxid = snip_from_file_spdxid
         self.conc_lics = conc_lics
         self.licenses_in_snippet = []
+        self.byte_range: Optional[Tuple[int, int]] = byte_range
+        self.line_range: Optional[Tuple[int, int]] = None
 
     def add_lics(self, lics):
         self.licenses_in_snippet.append(lics)
@@ -105,4 +112,4 @@ def validate_licenses_in_snippet(self, messages):
                 )
 
     def has_optional_field(self, field):
-        return bool (getattr(self, field, None))
+        return bool(getattr(self, field, None))

spdx/writers/jsonyamlxml.py

@@ -399,9 +399,10 @@ def create_snippet_info(self, annotations_by_spdx_id):
         snippets = self.document.snippet
 
         for snippet in snippets:
+            snippet_from_file_spdx_id = self.spdx_id(snippet.snip_from_file_spdxid)
             snippet_object = dict()
             snippet_object["SPDXID"] = self.spdx_id(snippet.spdx_id)
-            snippet_object["fileId"] = self.spdx_id(snippet.snip_from_file_spdxid)
+            snippet_object["fileId"] = snippet_from_file_spdx_id
 
             if snippet.has_optional_field("copyright"):
                 snippet_object["copyrightText"] = snippet.copyright
@@ -413,6 +414,9 @@ def create_snippet_info(self, annotations_by_spdx_id):
                 snippet_object["licenseInfoFromSnippet"] = list(
                     map(self.license, snippet.licenses_in_snippet)
                 )
+            byte_range = {"endPointer": {"offset": snippet.byte_range[1], "reference": snippet_from_file_spdx_id},
+                          "startPointer": {"offset": snippet.byte_range[0], "reference": snippet_from_file_spdx_id}}
+            snippet_object["ranges"] = [byte_range]
 
             if snippet.has_optional_field("name"):
                 snippet_object["name"] = snippet.name
@@ -429,11 +433,18 @@ def create_snippet_info(self, annotations_by_spdx_id):
             if snippet.spdx_id in annotations_by_spdx_id:
                 snippet_object["annotations"] = annotations_by_spdx_id[snippet.spdx_id]
 
+            if snippet.has_optional_field("line_range"):
+                line_range = {
+                    "endPointer": {"lineNumber": snippet.line_range[1], "reference": snippet_from_file_spdx_id},
+                    "startPointer": {"lineNumber": snippet.line_range[0], "reference": snippet_from_file_spdx_id}}
+                snippet_object["ranges"].append(line_range)
+
             snippet_info_objects.append(snippet_object)
 
         return snippet_info_objects
 
 
+
 class ExtractedLicenseWriter(BaseWriter):
     """
     Represent spdx.document.ExtractedLicense as json-serializable objects

spdx/writers/tagvalue.py

@@ -39,6 +39,9 @@ def format_verif_code(package):
 def write_value(tag, value, out):
     out.write("{0}: {1}\n".format(tag, value))
 
+def write_range(tag, value, out):
+    out.write("{0}: {1}:{2}\n".format(tag, value[0], value[1]))
+
 
 def write_text_value(tag, value, out):
     value = "{0}: <text>{1}</text>\n".format(tag, value)
@@ -170,6 +173,10 @@ def write_snippet(snippet, out):
     write_value("SnippetSPDXID", snippet.spdx_id, out)
     write_value("SnippetFromFileSPDXID", snippet.snip_from_file_spdxid, out)
     write_text_value("SnippetCopyrightText", snippet.copyright, out)
+    write_range("SnippetByteRange", snippet.byte_range, out)
+    if snippet.has_optional_field("line_range"):
+        write_range("SnippetLineRange", snippet.line_range, out)
+
     if snippet.has_optional_field("name"):
         write_value("SnippetName", snippet.name, out)
     if snippet.has_optional_field("comment"):

tests/data/formats/SPDXJsonExample.json

@@ -177,7 +177,29 @@
       ],
       "licenseComments": "The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz.",
       "SPDXID": "SPDXRef-Snippet",
-      "fileId": "SPDXRef-DoapSource"
+      "fileId": "SPDXRef-DoapSource",
+        "ranges": [
+            {
+                "endPointer": {
+                    "offset": 420,
+                    "reference": "SPDXRef-DoapSource"
+                },
+                "startPointer": {
+                    "offset": 310,
+                    "reference": "SPDXRef-DoapSource"
+                }
+            },
+            {
+                "endPointer": {
+                    "lineNumber": 23,
+                    "reference": "SPDXRef-DoapSource"
+                },
+                "startPointer": {
+                    "lineNumber": 5,
+                    "reference": "SPDXRef-DoapSource"
+                }
+            }
+        ]
     }
   ]
 }

tests/data/formats/SPDXTagExample.tag

@@ -99,6 +99,7 @@ SnippetComment: <text>This snippet was identified as significant and highlighted
 SnippetName: from linux kernel
 SnippetLicenseConcluded: Apache-2.0
 LicenseInfoInSnippet: Apache-2.0
+SnippetByteRange: 310:420
 
 ## License Information
 LicenseID: LicenseRef-3