From 5a799747e23faa82ec3cac7f3f2174a8614ebed2 Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Fri, 13 Jun 2014 21:48:09 -0400 Subject: [PATCH 01/14] Changed the behavior of VocabString.lookup_class() to return the VocabString class when the lookup fails --- stix/common/vocabs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix/common/vocabs.py b/stix/common/vocabs.py index eaaaf0ea..14d11409 100644 --- a/stix/common/vocabs.py +++ b/stix/common/vocabs.py @@ -67,7 +67,7 @@ def lookup_class(xsi_type): if xsi_type in k: return v - raise ValueError("Unregistered xsi:type %s" % xsi_type) + return VocabString def to_obj(self, return_obj=None): if not return_obj: From 564e8f36ce3a563ed4fca9e3e64552119ed7e408 Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Fri, 13 Jun 2014 22:02:06 -0400 Subject: [PATCH 02/14] Changed nsparser get_namespaces() logic to play nicer with documents that utilize non-STIX namespaces. Tests fine with the STIX sample documents, but could use some more testing. --- stix/utils/nsparser.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/stix/utils/nsparser.py b/stix/utils/nsparser.py index e10effcf..d95bd55a 100644 --- a/stix/utils/nsparser.py +++ b/stix/utils/nsparser.py @@ -52,7 +52,7 @@ def get_namespaces(self, entity, ns_dict=None): import stix.utils.idgen as idgen if not ns_dict: ns_dict = {} - all_ns_dict = {'http://www.w3.org/2001/XMLSchema-instance': 'xsi', + entity_ns_dict = {'http://www.w3.org/2001/XMLSchema-instance': 'xsi', 'http://stix.mitre.org/stix-1': 'stix', 'http://stix.mitre.org/common-1': 'stixCommon', 'http://stix.mitre.org/default_vocabularies-1': 'stixVocabs', @@ -60,19 +60,22 @@ def get_namespaces(self, entity, ns_dict=None): default_cybox_namespaces = dict((ns,alias) for (ns,alias,schemaloc) in cybox_nsparser.NS_LIST) default_stix_namespaces = dict(default_cybox_namespaces.items() + XML_NAMESPACES.items() + DEFAULT_STIX_NS_TO_PREFIX.items() + DEFAULT_EXT_TO_PREFIX.items()) - - ns_set = self._get_namespace_set(entity) - for ns in ns_set: - all_ns_dict[ns] = default_stix_namespaces[ns] - - # add additional @ns_dict and parsed, input namespaces - all_ns_dict.update(ns_dict) + if hasattr(entity, "__input_namespaces__"): for ns,alias in entity.__input_namespaces__.iteritems(): if ns not in (default_stix_namespaces): - all_ns_dict[ns] = alias + entity_ns_dict[ns] = alias + + entity_ns_set = self._get_namespace_set(entity) + for ns in entity_ns_set: + if ns not in entity_ns_dict: + entity_ns_dict[ns] = default_stix_namespaces[ns] + + # add additional @ns_dict and parsed + entity_ns_dict.update(ns_dict) + - return all_ns_dict + return entity_ns_dict def _get_input_schemalocations(self, entity): all_schemalocations = {} From 75c13e4eec3678edbf6aad0445163efc3b0b6cac Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Sat, 14 Jun 2014 18:07:27 -0400 Subject: [PATCH 03/14] Updated nsparser code to detect namespaces on user-defined controlled vocabuarlies. Significant changes to code that walks the python object model to retrieve namespaces and xsi_type attribute values --- stix/base.py | 14 +++++----- stix/utils/nsparser.py | 63 +++++++++++++++++++++++------------------- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/stix/base.py b/stix/base.py index d9759ad9..47c28e32 100644 --- a/stix/base.py +++ b/stix/base.py @@ -28,9 +28,9 @@ def _get_namespaces(self, ns_dict): all_ns_dict = namespace_parser.get_namespaces(self, ns_dict=ns_dict) return all_ns_dict - def _get_schema_locations(self, ns_dict=None): + def _get_schema_locations(self, namespaces=None): import stix.utils.nsparser as nsparser - schemaloc_dict = nsparser.NamespaceParser().get_namespace_schemalocation_dict(self, ns_dict=ns_dict) + schemaloc_dict = nsparser.NamespaceParser().get_namespace_schemalocation_dict(self, namespaces=namespaces) return schemaloc_dict def to_xml(self, include_namespaces=True, ns_dict=None, pretty=True): @@ -41,16 +41,16 @@ def to_xml(self, include_namespaces=True, ns_dict=None, pretty=True): import stix.utils.nsparser as nsparser if include_namespaces: if not ns_dict: ns_dict = {} - all_ns_dict = self._get_namespaces(ns_dict) - schemaloc_dict = self._get_schema_locations(all_ns_dict) - namespace_def = nsparser.NamespaceParser().get_namespace_def_str(all_ns_dict, schemaloc_dict) + all_namespaces = self._get_namespaces(ns_dict) + schemaloc_dict = self._get_schema_locations(set(ns for ns,_ in all_namespaces)) + namespace_def = nsparser.NamespaceParser().get_namespace_def_str(all_namespaces, schemaloc_dict) else: - all_ns_dict = dict(nsparser.DEFAULT_STIX_NS_TO_PREFIX.items() + nsparser.DEFAULT_EXT_TO_PREFIX.items()) + all_namespaces = dict(nsparser.DEFAULT_STIX_NS_TO_PREFIX.items() + nsparser.DEFAULT_EXT_TO_PREFIX.items()) if not pretty: namespace_def = namespace_def.replace('\n\t', ' ') - self.to_obj().export(s, 0, all_ns_dict, pretty_print=pretty, namespacedef_=namespace_def) + self.to_obj().export(s, 0, dict(all_namespaces), pretty_print=pretty, namespacedef_=namespace_def) return s.getvalue() def _get_children(self): diff --git a/stix/utils/nsparser.py b/stix/utils/nsparser.py index d95bd55a..4e4f1a2c 100644 --- a/stix/utils/nsparser.py +++ b/stix/utils/nsparser.py @@ -14,8 +14,8 @@ def __init__(self): def _get_observable_namespaces(self, obs): '''Returns namespaces used within a CybOX Observable''' obs_namespaces = obs._get_namespaces() - namespaces = [x.name for x in obs_namespaces] - namespaces.append('http://cybox.mitre.org/default_vocabularies-2') + namespaces = [(x.name,x.prefix) for x in obs_namespaces] + namespaces.append(('http://cybox.mitre.org/default_vocabularies-2','cyboxVocabs')) return namespaces def _get_namespace_set(self, entity): @@ -31,7 +31,11 @@ def _get_namespace_set(self, entity): for child in entity._get_children(): all_namespaces.update(self._get_namespace_set(child)) elif hasattr(entity, "_namespace"): - all_namespaces.add(entity._namespace) + if hasattr(entity, "_XSI_TYPE") and entity._XSI_TYPE: + ns_alias = entity._XSI_TYPE.split(":")[0] + all_namespaces.add((entity._namespace, ns_alias)) + else: + all_namespaces.add((entity._namespace, None)) for child in entity._get_children(): if not hasattr(child, "nsparser_touched"): @@ -42,7 +46,7 @@ def _get_namespace_set(self, entity): return all_namespaces def get_namespaces(self, entity, ns_dict=None): - """Returns a dictionary of namespace=>alias for all namespaces + """Returns a set of namespace for all namespaces used within the supplied entity. Arguments: @@ -52,11 +56,12 @@ def get_namespaces(self, entity, ns_dict=None): import stix.utils.idgen as idgen if not ns_dict: ns_dict = {} - entity_ns_dict = {'http://www.w3.org/2001/XMLSchema-instance': 'xsi', - 'http://stix.mitre.org/stix-1': 'stix', - 'http://stix.mitre.org/common-1': 'stixCommon', - 'http://stix.mitre.org/default_vocabularies-1': 'stixVocabs', - idgen.get_id_namespace() : idgen.get_id_namespace_alias()} + entity_namespaces = \ + set([('http://www.w3.org/2001/XMLSchema-instance', 'xsi'), + ('http://stix.mitre.org/stix-1', 'stix'), + ('http://stix.mitre.org/common-1', 'stixCommon'), + ('http://stix.mitre.org/default_vocabularies-1', 'stixVocabs'), + (idgen.get_id_namespace(), idgen.get_id_namespace_alias())]) default_cybox_namespaces = dict((ns,alias) for (ns,alias,schemaloc) in cybox_nsparser.NS_LIST) default_stix_namespaces = dict(default_cybox_namespaces.items() + XML_NAMESPACES.items() + DEFAULT_STIX_NS_TO_PREFIX.items() + DEFAULT_EXT_TO_PREFIX.items()) @@ -64,18 +69,19 @@ def get_namespaces(self, entity, ns_dict=None): if hasattr(entity, "__input_namespaces__"): for ns,alias in entity.__input_namespaces__.iteritems(): if ns not in (default_stix_namespaces): - entity_ns_dict[ns] = alias - - entity_ns_set = self._get_namespace_set(entity) - for ns in entity_ns_set: - if ns not in entity_ns_dict: - entity_ns_dict[ns] = default_stix_namespaces[ns] + entity_namespaces.add((ns,alias)) + + entity_ns_tuples = self._get_namespace_set(entity) + for ns,alias in entity_ns_tuples: + if alias: + entity_namespaces.add((ns, alias)) + elif ns not in entity_namespaces: + default_alias = default_stix_namespaces[ns] + entity_namespaces.add((ns, default_alias)) # add additional @ns_dict and parsed - entity_ns_dict.update(ns_dict) - - - return entity_ns_dict + entity_namespaces.update(ns_dict.items()) + return entity_namespaces def _get_input_schemalocations(self, entity): all_schemalocations = {} @@ -93,18 +99,19 @@ def _get_input_schemalocations(self, entity): del entity.nsparser_touched return all_schemalocations - def get_namespace_schemalocation_dict(self, entity, ns_dict=None): + def get_namespace_schemalocation_dict(self, entity, namespaces=None): d = {} - if ns_dict: - ns_set = ns_dict.iterkeys() + if namespaces: + ns_set = set(namespaces) else: - ns_set = self.get_namespaces(entity).iterkeys() + namespaces = self.get_namespaces(entity) # returns a [(ns,alias)*] + ns_set = set(ns for ns,_ in namespaces) input_schemalocations = self._get_input_schemalocations(entity) d.update(input_schemalocations) default_cybox_schemaloc_dict = {} - for (ns,alias,schemaloc) in cybox_nsparser.NS_LIST: + for (ns,_,schemaloc) in cybox_nsparser.NS_LIST: if schemaloc: default_cybox_schemaloc_dict[ns] = schemaloc default_stix_schemaloc_dict = dict(STIX_NS_TO_SCHEMALOCATION.items() + EXT_NS_TO_SCHEMALOCATION.items() + default_cybox_schemaloc_dict.items()) @@ -118,8 +125,8 @@ def get_namespace_schemalocation_dict(self, entity, ns_dict=None): return d - def _get_xmlns_str(self, ns_dict): - return "\n\t".join(['xmlns:%s="%s"' % (alias,ns) for (ns,alias) in sorted(ns_dict.iteritems())]) + def _get_xmlns_str(self, namespaces): + return "\n\t".join(['xmlns:%s="%s"' % (alias,ns) for ns,alias in sorted(namespaces)]) def _get_schemaloc_str(self, schemaloc_dict): schemaloc_str_start = 'xsi:schemaLocation="\n\t' @@ -127,8 +134,8 @@ def _get_schemaloc_str(self, schemaloc_dict): schemaloc_str_content = "\n\t".join(["%s %s" % (ns, loc) for ns,loc in sorted(schemaloc_dict.iteritems())]) return schemaloc_str_start + schemaloc_str_content + schemaloc_str_end - def get_namespace_def_str(self, ns_dict, schemaloc_dict): - return "\n\t" + self._get_xmlns_str(ns_dict) + "\n\t" + self._get_schemaloc_str(schemaloc_dict) + def get_namespace_def_str(self, namespaces, schemaloc_dict): + return "\n\t" + self._get_xmlns_str(namespaces) + "\n\t" + self._get_schemaloc_str(schemaloc_dict) XML_NAMESPACES = {'http://www.w3.org/2001/XMLSchema-instance' : 'xsi', 'http://www.w3.org/2001/XMLSchema' : 'xs', From 5604e3921ded9ec53a1074b80ef42ebab6e9f1a9 Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Mon, 11 Aug 2014 18:20:40 -0400 Subject: [PATCH 04/14] Updated nsparser to maintain public interfaces. Input schemalocations are attached to a STIX entity on from_xml() call and exported during to_xml() unless overridden by a default. --- stix/base.py | 33 ++++-- stix/utils/nsparser.py | 254 +++++++++++++++++++++++------------------ stix/utils/parser.py | 24 +++- 3 files changed, 187 insertions(+), 124 deletions(-) diff --git a/stix/base.py b/stix/base.py index a976ecbe..9b38788f 100644 --- a/stix/base.py +++ b/stix/base.py @@ -22,15 +22,17 @@ def from_obj(cls, obj): """Create an object from a binding object""" raise NotImplementedError() - def _get_namespaces(self, ns_dict): + def _get_namespaces(self, ns_dict=None): import stix.utils.nsparser as nsparser - namespace_parser = nsparser.NamespaceParser() - all_ns_dict = namespace_parser.get_namespaces(self, ns_dict=ns_dict) + parser = nsparser.NamespaceParser() + all_ns_dict = parser.get_namespaces(self, ns_dict=ns_dict) return all_ns_dict - def _get_schema_locations(self, namespaces=None): + def _get_schema_locations(self, ns_dict=None): import stix.utils.nsparser as nsparser - schemaloc_dict = nsparser.NamespaceParser().get_namespace_schemalocation_dict(self, namespaces=namespaces) + parser = nsparser.NamespaceParser() + schemaloc_dict = \ + parser.get_namespace_schemalocation_dict(self, ns_dict=ns_dict) return schemaloc_dict def to_xml(self, include_namespaces=True, ns_dict=None, pretty=True): @@ -40,17 +42,22 @@ def to_xml(self, include_namespaces=True, ns_dict=None, pretty=True): import stix.utils.nsparser as nsparser if include_namespaces: - if not ns_dict: ns_dict = {} - all_namespaces = self._get_namespaces(ns_dict) - schemaloc_dict = self._get_schema_locations(set(ns for ns,_ in all_namespaces)) - namespace_def = nsparser.NamespaceParser().get_namespace_def_str(all_namespaces, schemaloc_dict) + all_ns_dict = self._get_namespaces(ns_dict) + schemaloc_dict = \ + self._get_schema_locations(all_ns_dict) + + parser = nsparser.NamespaceParser() + namespace_def = parser.get_namespace_def_str(all_ns_dict, + schemaloc_dict) else: - all_namespaces = dict(nsparser.DEFAULT_STIX_NS_TO_PREFIX.items() + nsparser.DEFAULT_EXT_TO_PREFIX.items()) + all_ns_dict = dict(nsparser.DEFAULT_STIX_NS_TO_PREFIX.items() + + nsparser.DEFAULT_EXT_TO_PREFIX.items()) if not pretty: namespace_def = namespace_def.replace('\n\t', ' ') - self.to_obj().export(s, 0, dict(all_namespaces), pretty_print=pretty, namespacedef_=namespace_def) + self.to_obj().export(s, 0, all_ns_dict, pretty_print=pretty, + namespacedef_=namespace_def) return s.getvalue() def _get_children(self): @@ -62,7 +69,9 @@ def _get_children(self): yield obj elif isinstance(obj, (tuple, collections.MutableSequence)): for item in obj: - if isinstance(item, Entity) or isinstance(item, Observable) or isinstance(item, cybox.Entity): + if (isinstance(item, Entity) or + isinstance(item, Observable) or + isinstance(item, cybox.Entity)): yield item def to_json(self): diff --git a/stix/utils/nsparser.py b/stix/utils/nsparser.py index 4e4f1a2c..acd5ee3d 100644 --- a/stix/utils/nsparser.py +++ b/stix/utils/nsparser.py @@ -2,6 +2,7 @@ # See LICENSE.txt for complete terms. import inspect +import warnings import stix import cybox from cybox.core import Observables, Observable @@ -11,36 +12,39 @@ class NamespaceParser(object): def __init__(self): pass - def _get_observable_namespaces(self, obs): - '''Returns namespaces used within a CybOX Observable''' + def _get_observable_namespace_dict(self, obs): + '''Returns a dict of namespaces used within a CybOX Observable''' + namespaces = {'http://cybox.mitre.org/default_vocabularies-2': 'cyboxVocabs'} + obs_namespaces = obs._get_namespaces() - namespaces = [(x.name,x.prefix) for x in obs_namespaces] - namespaces.append(('http://cybox.mitre.org/default_vocabularies-2','cyboxVocabs')) + for namespace in obs_namespaces: + namespaces[namespace.name] = namespace.prefix + return namespaces - def _get_namespace_set(self, entity): - all_namespaces = set() + def _get_namespace_dict(self, entity): + all_namespaces = {} if not isinstance(entity, (stix.Entity, cybox.Entity)): raise ValueError("Must provide an instance of stix.Entity or cybox.core.Observable") entity.nsparser_touched = True if isinstance(entity, Observable): - all_namespaces.update(self._get_observable_namespaces(entity)) + all_namespaces.update(self._get_observable_namespace_dict(entity)) elif isinstance(entity, Observables): for child in entity._get_children(): - all_namespaces.update(self._get_namespace_set(child)) + all_namespaces.update(self._get_namespace_dict(child)) elif hasattr(entity, "_namespace"): if hasattr(entity, "_XSI_TYPE") and entity._XSI_TYPE: ns_alias = entity._XSI_TYPE.split(":")[0] - all_namespaces.add((entity._namespace, ns_alias)) + all_namespaces[entity._namespace] = ns_alias else: - all_namespaces.add((entity._namespace, None)) + all_namespaces[entity._namespace] = None for child in entity._get_children(): if not hasattr(child, "nsparser_touched"): if hasattr(child, "_namespace") or isinstance(child, Observable): - all_namespaces.update(self._get_namespace_set(child)) + all_namespaces.update(self._get_namespace_dict(child)) del entity.nsparser_touched return all_namespaces @@ -57,30 +61,52 @@ def get_namespaces(self, entity, ns_dict=None): if not ns_dict: ns_dict = {} entity_namespaces = \ - set([('http://www.w3.org/2001/XMLSchema-instance', 'xsi'), - ('http://stix.mitre.org/stix-1', 'stix'), - ('http://stix.mitre.org/common-1', 'stixCommon'), - ('http://stix.mitre.org/default_vocabularies-1', 'stixVocabs'), - (idgen.get_id_namespace(), idgen.get_id_namespace_alias())]) + {'http://www.w3.org/2001/XMLSchema-instance': 'xsi', + 'http://stix.mitre.org/stix-1': 'stix', + 'http://stix.mitre.org/common-1': 'stixCommon', + 'http://stix.mitre.org/default_vocabularies-1': 'stixVocabs', + idgen.get_id_namespace(): idgen.get_id_namespace_alias()} - default_cybox_namespaces = dict((ns,alias) for (ns,alias,schemaloc) in cybox_nsparser.NS_LIST) - default_stix_namespaces = dict(default_cybox_namespaces.items() + XML_NAMESPACES.items() + DEFAULT_STIX_NS_TO_PREFIX.items() + DEFAULT_EXT_TO_PREFIX.items()) + default_cybox_namespaces = dict((ns, alias) for (ns, alias, _) in cybox_nsparser.NS_LIST) + default_stix_namespaces = dict(default_cybox_namespaces.items() + + XML_NAMESPACES.items() + + DEFAULT_STIX_NS_TO_PREFIX.items() + + DEFAULT_EXT_TO_PREFIX.items()) if hasattr(entity, "__input_namespaces__"): - for ns,alias in entity.__input_namespaces__.iteritems(): + for ns, alias in entity.__input_namespaces__.iteritems(): if ns not in (default_stix_namespaces): - entity_namespaces.add((ns,alias)) + entity_namespaces[ns] = alias - entity_ns_tuples = self._get_namespace_set(entity) - for ns,alias in entity_ns_tuples: + entity_ns_dict = self._get_namespace_dict(entity) + for ns, alias in entity_ns_dict.iteritems(): if alias: - entity_namespaces.add((ns, alias)) + entity_namespaces[ns] = alias elif ns not in entity_namespaces: default_alias = default_stix_namespaces[ns] - entity_namespaces.add((ns, default_alias)) + entity_namespaces[ns] = default_alias # add additional @ns_dict and parsed - entity_namespaces.update(ns_dict.items()) + entity_namespaces.update(ns_dict) + + # Remove "http://example.com/" namespace if "http://example.com" exists + # A bit of a hack to resolve issues with the STIX samples and our + # own default ID namespace + if all((entity_namespaces.get('http://example.com/'), + entity_namespaces.get('http://example.com'))): + del entity_namespaces['http://example.com/'] + + # sanity check namespaces for things like duplicate aliases + aliases = {} + for ns, alias in entity_namespaces.iteritems(): + if alias not in aliases: + aliases[alias] = ns + else: + # TODO: Should we just throw an exception here? + # The XML will be invalid if there is a duplicate ns alias + warnings.warn("namespace alias '%s' mapped to '%s' and '%s'" % + (alias, ns, aliases[alias])) + return entity_namespaces def _get_input_schemalocations(self, entity): @@ -99,120 +125,126 @@ def _get_input_schemalocations(self, entity): del entity.nsparser_touched return all_schemalocations - def get_namespace_schemalocation_dict(self, entity, namespaces=None): + def get_namespace_schemalocation_dict(self, entity, ns_dict=None): d = {} - if namespaces: - ns_set = set(namespaces) - else: - namespaces = self.get_namespaces(entity) # returns a [(ns,alias)*] - ns_set = set(ns for ns,_ in namespaces) + if not ns_dict: + ns_dict = self.get_namespaces(entity) input_schemalocations = self._get_input_schemalocations(entity) d.update(input_schemalocations) - - default_cybox_schemaloc_dict = {} - for (ns,_,schemaloc) in cybox_nsparser.NS_LIST: - if schemaloc: default_cybox_schemaloc_dict[ns] = schemaloc - - default_stix_schemaloc_dict = dict(STIX_NS_TO_SCHEMALOCATION.items() + EXT_NS_TO_SCHEMALOCATION.items() + default_cybox_schemaloc_dict.items()) - for ns in ns_set: + # Build default schemaLocation dictionary for CybOX + default_cybox_schemaloc_dict = {} + for ns, _, schemaloc in cybox_nsparser.NS_LIST: + if schemaloc: + default_cybox_schemaloc_dict[ns] = schemaloc + + # Build default schemaLocation dict for STIX and CybOX and extension + default_stix_schemaloc_dict = dict(STIX_NS_TO_SCHEMALOCATION.items() + + EXT_NS_TO_SCHEMALOCATION.items() + + default_cybox_schemaloc_dict.items()) + + # Iterate over input/discovered namespaces for document and attempt + # to map them to schemalocations. Warn if unable to map ns to schemaloc. + for ns in ns_dict.iterkeys(): if ns in default_stix_schemaloc_dict: schemalocation = default_stix_schemaloc_dict[ns] d[ns] = schemalocation else: - continue - + warnings.warn("Unable to map namespace '%s' to " + "schemaLocation" % ns) return d - def _get_xmlns_str(self, namespaces): - return "\n\t".join(['xmlns:%s="%s"' % (alias,ns) for ns,alias in sorted(namespaces)]) + def _get_xmlns_str(self, ns_dict): + return "\n\t".join(['xmlns:%s="%s"' % + (alias, ns) for ns, alias in sorted(ns_dict.iteritems())]) def _get_schemaloc_str(self, schemaloc_dict): schemaloc_str_start = 'xsi:schemaLocation="\n\t' schemaloc_str_end = '"' - schemaloc_str_content = "\n\t".join(["%s %s" % (ns, loc) for ns,loc in sorted(schemaloc_dict.iteritems())]) + schemaloc_str_content = "\n\t".join(["%s %s" % + (ns, loc) for ns, loc in sorted(schemaloc_dict.iteritems())]) return schemaloc_str_start + schemaloc_str_content + schemaloc_str_end def get_namespace_def_str(self, namespaces, schemaloc_dict): return "\n\t" + self._get_xmlns_str(namespaces) + "\n\t" + self._get_schemaloc_str(schemaloc_dict) -XML_NAMESPACES = {'http://www.w3.org/2001/XMLSchema-instance' : 'xsi', - 'http://www.w3.org/2001/XMLSchema' : 'xs', - 'http://www.w3.org/1999/xlink' : 'xlink', +XML_NAMESPACES = {'http://www.w3.org/2001/XMLSchema-instance': 'xsi', + 'http://www.w3.org/2001/XMLSchema': 'xs', + 'http://www.w3.org/1999/xlink': 'xlink', 'http://www.w3.org/2000/09/xmldsig#': 'ds'} # Schema locations for namespaces defined by the STIX language STIX_NS_TO_SCHEMALOCATION = { - 'http://data-marking.mitre.org/Marking-1' : 'http://stix.mitre.org/XMLSchema/data_marking/1.1.1/data_marking.xsd', - 'http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1' : 'http://stix.mitre.org/XMLSchema/extensions/marking/simple/1.1.1/simple_marking.xsd', - 'http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1' : 'http://stix.mitre.org/XMLSchema/extensions/marking/tlp/1.1.1/tlp_marking.xsd', - 'http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1' : 'http://stix.mitre.org/XMLSchema/extensions/marking/terms_of_use/1.0/terms_of_use_marking.xsd', - 'http://stix.mitre.org/Campaign-1' : 'http://stix.mitre.org/XMLSchema/campaign/1.1.1/campaign.xsd', - 'http://stix.mitre.org/CourseOfAction-1' : 'http://stix.mitre.org/XMLSchema/course_of_action/1.1.1/course_of_action.xsd', - 'http://stix.mitre.org/ExploitTarget-1' : 'http://stix.mitre.org/XMLSchema/exploit_target/1.1.1/exploit_target.xsd', - 'http://stix.mitre.org/Incident-1' : 'http://stix.mitre.org/XMLSchema/incident/1.1.1/incident.xsd', - 'http://stix.mitre.org/Indicator-2' : 'http://stix.mitre.org/XMLSchema/indicator/2.1.1/indicator.xsd', - 'http://stix.mitre.org/TTP-1' : 'http://stix.mitre.org/XMLSchema/ttp/1.1.1/ttp.xsd', - 'http://stix.mitre.org/ThreatActor-1' : 'http://stix.mitre.org/XMLSchema/threat_actor/1.1.1/threat_actor.xsd', - 'http://stix.mitre.org/common-1' : 'http://stix.mitre.org/XMLSchema/common/1.1.1/stix_common.xsd', - 'http://stix.mitre.org/default_vocabularies-1' : 'http://stix.mitre.org/XMLSchema/default_vocabularies/1.1.1/stix_default_vocabularies.xsd', - 'http://stix.mitre.org/extensions/AP#CAPEC2.7-1' : 'http://stix.mitre.org/XMLSchema/extensions/attack_pattern/capec_2.7/1.0.1/capec_2.7_attack_pattern.xsd', - 'http://stix.mitre.org/extensions/Address#CIQAddress3.0-1' : 'http://stix.mitre.org/XMLSchema/extensions/address/ciq_3.0/1.1.1/ciq_3.0_address.xsd', - 'http://stix.mitre.org/extensions/Identity#CIQIdentity3.0-1' : 'http://stix.mitre.org/XMLSchema/extensions/identity/ciq_3.0/1.1.1/ciq_3.0_identity.xsd', - 'http://stix.mitre.org/extensions/Malware#MAEC4.1-1' : 'http://stix.mitre.org/XMLSchema/extensions/malware/maec_4.1/1.0.1/maec_4.1_malware.xsd', - 'http://stix.mitre.org/extensions/StructuredCOA#Generic-1' : 'http://stix.mitre.org/XMLSchema/extensions/structured_coa/generic/1.1.1/generic_structured_coa.xsd', - 'http://stix.mitre.org/extensions/TestMechanism#Generic-1' : 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/generic/1.1.1/generic_test_mechanism.xsd', - 'http://stix.mitre.org/extensions/TestMechanism#OVAL5.10-1' : 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/oval_5.10/1.1.1/oval_5.10_test_mechanism.xsd', - 'http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1' : 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/open_ioc_2010/1.1.1/open_ioc_2010_test_mechanism.xsd', - 'http://stix.mitre.org/extensions/TestMechanism#Snort-1' : 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/snort/1.1.1/snort_test_mechanism.xsd', - 'http://stix.mitre.org/extensions/TestMechanism#YARA-1' : 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/yara/1.1.1/yara_test_mechanism.xsd', - 'http://stix.mitre.org/extensions/Vulnerability#CVRF-1' : 'http://stix.mitre.org/XMLSchema/extensions/vulnerability/cvrf_1.1/1.1.1/cvrf_1.1_vulnerability.xsd', - 'http://stix.mitre.org/stix-1' : 'http://stix.mitre.org/XMLSchema/core/1.1.1/stix_core.xsd'} + 'http://data-marking.mitre.org/Marking-1': 'http://stix.mitre.org/XMLSchema/data_marking/1.1.1/data_marking.xsd', + 'http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1': 'http://stix.mitre.org/XMLSchema/extensions/marking/simple/1.1.1/simple_marking.xsd', + 'http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1': 'http://stix.mitre.org/XMLSchema/extensions/marking/tlp/1.1.1/tlp_marking.xsd', + 'http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1': 'http://stix.mitre.org/XMLSchema/extensions/marking/terms_of_use/1.0/terms_of_use_marking.xsd', + 'http://stix.mitre.org/Campaign-1': 'http://stix.mitre.org/XMLSchema/campaign/1.1.1/campaign.xsd', + 'http://stix.mitre.org/CourseOfAction-1': 'http://stix.mitre.org/XMLSchema/course_of_action/1.1.1/course_of_action.xsd', + 'http://stix.mitre.org/ExploitTarget-1': 'http://stix.mitre.org/XMLSchema/exploit_target/1.1.1/exploit_target.xsd', + 'http://stix.mitre.org/Incident-1': 'http://stix.mitre.org/XMLSchema/incident/1.1.1/incident.xsd', + 'http://stix.mitre.org/Indicator-2': 'http://stix.mitre.org/XMLSchema/indicator/2.1.1/indicator.xsd', + 'http://stix.mitre.org/TTP-1': 'http://stix.mitre.org/XMLSchema/ttp/1.1.1/ttp.xsd', + 'http://stix.mitre.org/ThreatActor-1': 'http://stix.mitre.org/XMLSchema/threat_actor/1.1.1/threat_actor.xsd', + 'http://stix.mitre.org/common-1': 'http://stix.mitre.org/XMLSchema/common/1.1.1/stix_common.xsd', + 'http://stix.mitre.org/default_vocabularies-1': 'http://stix.mitre.org/XMLSchema/default_vocabularies/1.1.1/stix_default_vocabularies.xsd', + 'http://stix.mitre.org/extensions/AP#CAPEC2.7-1': 'http://stix.mitre.org/XMLSchema/extensions/attack_pattern/capec_2.7/1.0.1/capec_2.7_attack_pattern.xsd', + 'http://stix.mitre.org/extensions/Address#CIQAddress3.0-1': 'http://stix.mitre.org/XMLSchema/extensions/address/ciq_3.0/1.1.1/ciq_3.0_address.xsd', + 'http://stix.mitre.org/extensions/Identity#CIQIdentity3.0-1': 'http://stix.mitre.org/XMLSchema/extensions/identity/ciq_3.0/1.1.1/ciq_3.0_identity.xsd', + 'http://stix.mitre.org/extensions/Malware#MAEC4.1-1': 'http://stix.mitre.org/XMLSchema/extensions/malware/maec_4.1/1.0.1/maec_4.1_malware.xsd', + 'http://stix.mitre.org/extensions/StructuredCOA#Generic-1': 'http://stix.mitre.org/XMLSchema/extensions/structured_coa/generic/1.1.1/generic_structured_coa.xsd', + 'http://stix.mitre.org/extensions/TestMechanism#Generic-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/generic/1.1.1/generic_test_mechanism.xsd', + 'http://stix.mitre.org/extensions/TestMechanism#OVAL5.10-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/oval_5.10/1.1.1/oval_5.10_test_mechanism.xsd', + 'http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/open_ioc_2010/1.1.1/open_ioc_2010_test_mechanism.xsd', + 'http://stix.mitre.org/extensions/TestMechanism#Snort-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/snort/1.1.1/snort_test_mechanism.xsd', + 'http://stix.mitre.org/extensions/TestMechanism#YARA-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/yara/1.1.1/yara_test_mechanism.xsd', + 'http://stix.mitre.org/extensions/Vulnerability#CVRF-1': 'http://stix.mitre.org/XMLSchema/extensions/vulnerability/cvrf_1.1/1.1.1/cvrf_1.1_vulnerability.xsd', + 'http://stix.mitre.org/stix-1': 'http://stix.mitre.org/XMLSchema/core/1.1.1/stix_core.xsd'} # Schema locations for namespaces not defined by STIX, but hosted on the STIX website -EXT_NS_TO_SCHEMALOCATION = {'urn:oasis:names:tc:ciq:xal:3' : 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xAL.xsd', - 'urn:oasis:names:tc:ciq:xpil:3' : 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xPIL.xsd', - 'urn:oasis:names:tc:ciq:xnl:3' : 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xNL.xsd'} +EXT_NS_TO_SCHEMALOCATION = {'urn:oasis:names:tc:ciq:xal:3': 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xAL.xsd', + 'urn:oasis:names:tc:ciq:xpil:3': 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xPIL.xsd', + 'urn:oasis:names:tc:ciq:xnl:3': 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xNL.xsd'} # Default namespace->alias mappings. These can be overriden by user-provided dictionaries on export DEFAULT_STIX_NS_TO_PREFIX = { - 'http://cybox.mitre.org/common-2' : 'cyboxCommon', - 'http://cybox.mitre.org/cybox-2' : 'cybox', - 'http://data-marking.mitre.org/Marking-1' : 'marking', - 'http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1' : 'simpleMarking', - 'http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1' : 'tlpMarking', - 'http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1' : 'TOUMarking', - 'http://stix.mitre.org/Campaign-1' : 'campaign', - 'http://stix.mitre.org/CourseOfAction-1' : 'coa', - 'http://stix.mitre.org/ExploitTarget-1' : 'et', - 'http://stix.mitre.org/Incident-1' : 'incident', - 'http://stix.mitre.org/Indicator-2' : 'indicator', - 'http://stix.mitre.org/TTP-1' : 'ttp', - 'http://stix.mitre.org/ThreatActor-1' : 'ta', - 'http://stix.mitre.org/stix-1' : 'stix', - 'http://stix.mitre.org/common-1' : 'stixCommon', - 'http://stix.mitre.org/default_vocabularies-1' : 'stixVocabs', - 'http://stix.mitre.org/extensions/AP#CAPEC2.7-1' : 'stix-capec', - 'http://stix.mitre.org/extensions/Address#CIQAddress3.0-1' : 'stix-ciqaddress', - 'http://stix.mitre.org/extensions/Identity#CIQIdentity3.0-1' : 'stix-ciqidentity', - 'http://stix.mitre.org/extensions/Malware#MAEC4.1-1' : 'stix-maec', - 'http://stix.mitre.org/extensions/StructuredCOA#Generic-1' : 'genericStructuredCOA', - 'http://stix.mitre.org/extensions/TestMechanism#Generic-1' : 'genericTM', - 'http://stix.mitre.org/extensions/TestMechanism#OVAL5.10-1' : 'stix-oval', - 'http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1' : 'stix-openioc', - 'http://stix.mitre.org/extensions/TestMechanism#Snort-1' : 'snortTM', - 'http://stix.mitre.org/extensions/TestMechanism#YARA-1' : 'yaraTM', - 'http://stix.mitre.org/extensions/Vulnerability#CVRF-1' : 'stix-cvrf'} + 'http://cybox.mitre.org/common-2': 'cyboxCommon', + 'http://cybox.mitre.org/cybox-2': 'cybox', + 'http://data-marking.mitre.org/Marking-1': 'marking', + 'http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1': 'simpleMarking', + 'http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1': 'tlpMarking', + 'http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1': 'TOUMarking', + 'http://stix.mitre.org/Campaign-1': 'campaign', + 'http://stix.mitre.org/CourseOfAction-1': 'coa', + 'http://stix.mitre.org/ExploitTarget-1': 'et', + 'http://stix.mitre.org/Incident-1': 'incident', + 'http://stix.mitre.org/Indicator-2': 'indicator', + 'http://stix.mitre.org/TTP-1': 'ttp', + 'http://stix.mitre.org/ThreatActor-1': 'ta', + 'http://stix.mitre.org/stix-1': 'stix', + 'http://stix.mitre.org/common-1': 'stixCommon', + 'http://stix.mitre.org/default_vocabularies-1': 'stixVocabs', + 'http://stix.mitre.org/extensions/AP#CAPEC2.7-1': 'stix-capec', + 'http://stix.mitre.org/extensions/Address#CIQAddress3.0-1': 'stix-ciqaddress', + 'http://stix.mitre.org/extensions/Identity#CIQIdentity3.0-1': 'stix-ciqidentity', + 'http://stix.mitre.org/extensions/Malware#MAEC4.1-1': 'stix-maec', + 'http://stix.mitre.org/extensions/StructuredCOA#Generic-1': 'genericStructuredCOA', + 'http://stix.mitre.org/extensions/TestMechanism#Generic-1': 'genericTM', + 'http://stix.mitre.org/extensions/TestMechanism#OVAL5.10-1': 'stix-oval', + 'http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1': 'stix-openioc', + 'http://stix.mitre.org/extensions/TestMechanism#Snort-1': 'snortTM', + 'http://stix.mitre.org/extensions/TestMechanism#YARA-1': 'yaraTM', + 'http://stix.mitre.org/extensions/Vulnerability#CVRF-1': 'stix-cvrf'} DEFAULT_EXT_TO_PREFIX = { - 'http://capec.mitre.org/capec-2' : 'capec', - 'http://maec.mitre.org/XMLSchema/maec-package-2' : 'maec', - 'http://oval.mitre.org/XMLSchema/oval-definitions-5' : 'oval-def', - 'http://oval.mitre.org/XMLSchema/oval-variables-5' : 'oval-var', - 'http://schemas.mandiant.com/2010/ioc' : 'ioc', - 'http://schemas.mandiant.com/2010/ioc/TR/' : 'ioc-tr', - 'http://www.icasi.org/CVRF/schema/cvrf/1.1' : 'cvrf', - 'urn:oasis:names:tc:ciq:xal:3' : 'xal', - 'urn:oasis:names:tc:ciq:xpil:3' : 'xpil', - 'urn:oasis:names:tc:ciq:xnl:3' : 'xnl'} + 'http://capec.mitre.org/capec-2': 'capec', + 'http://maec.mitre.org/XMLSchema/maec-package-2': 'maec', + 'http://oval.mitre.org/XMLSchema/oval-definitions-5': 'oval-def', + 'http://oval.mitre.org/XMLSchema/oval-variables-5': 'oval-var', + 'http://schemas.mandiant.com/2010/ioc': 'ioc', + 'http://schemas.mandiant.com/2010/ioc/TR/': 'ioc-tr', + 'http://www.icasi.org/CVRF/schema/cvrf/1.1': 'cvrf', + 'urn:oasis:names:tc:ciq:xal:3': 'xal', + 'urn:oasis:names:tc:ciq:xpil:3': 'xpil', + 'urn:oasis:names:tc:ciq:xnl:3': 'xnl'} diff --git a/stix/utils/parser.py b/stix/utils/parser.py index 22365635..836b6f4e 100644 --- a/stix/utils/parser.py +++ b/stix/utils/parser.py @@ -2,6 +2,7 @@ # See LICENSE.txt for complete terms. import stix +from itertools import izip from distutils.version import StrictVersion from lxml import etree @@ -56,9 +57,29 @@ def _apply_input_namespaces(self, tree, entity): root = tree entity.__input_namespaces__ = {} - for alias,ns in root.nsmap.iteritems(): + for alias, ns in root.nsmap.iteritems(): entity.__input_namespaces__[ns] = alias + def _apply_input_schemalocations(self, tree, entity): + """ + Attaches an __input_schemalocations__ dictionary to the input entity. + + :param tree: The input etree instance + :param entity: The entity to attach the schemlocation dictionary to + """ + try: + root = tree.getroot() # is tree an lxml.Element or lxml.ElementTree + except AttributeError: + root = tree + + schemaloc_attrib_name = \ + '{http://www.w3.org/2001/XMLSchema-instance}schemaLocation' + if root.attrib.get(schemaloc_attrib_name): + schemalocs = root.attrib[schemaloc_attrib_name].split() + pairs = izip(schemalocs[::2], schemalocs[1::2]) + entity.__input_schemalocations__ = dict(pairs) + + def parse_xml_to_obj(self, xml_file, check_version=True, check_root=True): """Creates a STIX binding object from the supplied xml file. @@ -108,5 +129,6 @@ def parse_xml(self, xml_file, check_version=True, check_root=True): from stix.core import STIXPackage # resolve circular dependencies stix_package = STIXPackage().from_obj(stix_package_obj) self._apply_input_namespaces(tree, stix_package) + self._apply_input_schemalocations(tree, stix_package) return stix_package From 3da966cf830bca4a3c486fb3fc55cdc254c0f876 Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Mon, 11 Aug 2014 18:46:41 -0400 Subject: [PATCH 05/14] Updated to_xml() to accept schemaloc_dict parameter which allows users to pass in third-party schemalocations --- stix/base.py | 16 ++++++++-------- stix/utils/nsparser.py | 14 +++++++++++--- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/stix/base.py b/stix/base.py index 9b38788f..a69cfcde 100644 --- a/stix/base.py +++ b/stix/base.py @@ -28,14 +28,14 @@ def _get_namespaces(self, ns_dict=None): all_ns_dict = parser.get_namespaces(self, ns_dict=ns_dict) return all_ns_dict - def _get_schema_locations(self, ns_dict=None): + def _get_schema_locations(self, ns_dict=None, schemaloc_dict=None): import stix.utils.nsparser as nsparser parser = nsparser.NamespaceParser() - schemaloc_dict = \ - parser.get_namespace_schemalocation_dict(self, ns_dict=ns_dict) - return schemaloc_dict + all_schemaloc_dict = \ + parser.get_namespace_schemalocation_dict(self, ns_dict=ns_dict, schemaloc_dict=schemaloc_dict) + return all_schemaloc_dict - def to_xml(self, include_namespaces=True, ns_dict=None, pretty=True): + def to_xml(self, include_namespaces=True, ns_dict=None, schemaloc_dict=None, pretty=True): """Export an object as an XML String""" s = StringIO() namespace_def = "" @@ -43,12 +43,12 @@ def to_xml(self, include_namespaces=True, ns_dict=None, pretty=True): import stix.utils.nsparser as nsparser if include_namespaces: all_ns_dict = self._get_namespaces(ns_dict) - schemaloc_dict = \ - self._get_schema_locations(all_ns_dict) + all_schemaloc_dict = \ + self._get_schema_locations(all_ns_dict, schemaloc_dict) parser = nsparser.NamespaceParser() namespace_def = parser.get_namespace_def_str(all_ns_dict, - schemaloc_dict) + all_schemaloc_dict) else: all_ns_dict = dict(nsparser.DEFAULT_STIX_NS_TO_PREFIX.items() + nsparser.DEFAULT_EXT_TO_PREFIX.items()) diff --git a/stix/utils/nsparser.py b/stix/utils/nsparser.py index acd5ee3d..1c53b06a 100644 --- a/stix/utils/nsparser.py +++ b/stix/utils/nsparser.py @@ -125,11 +125,14 @@ def _get_input_schemalocations(self, entity): del entity.nsparser_touched return all_schemalocations - def get_namespace_schemalocation_dict(self, entity, ns_dict=None): + def get_namespace_schemalocation_dict(self, entity, ns_dict=None, schemaloc_dict=None): d = {} if not ns_dict: ns_dict = self.get_namespaces(entity) + if not schemaloc_dict: + schemaloc_dict = {} + input_schemalocations = self._get_input_schemalocations(entity) d.update(input_schemalocations) @@ -151,8 +154,13 @@ def get_namespace_schemalocation_dict(self, entity, ns_dict=None): schemalocation = default_stix_schemaloc_dict[ns] d[ns] = schemalocation else: - warnings.warn("Unable to map namespace '%s' to " - "schemaLocation" % ns) + if not ((ns in schemaloc_dict) or + (ns in d) or + (ns in XML_NAMESPACES)): + warnings.warn("Unable to map namespace '%s' to " + "schemaLocation" % ns) + + d.update(schemaloc_dict) return d def _get_xmlns_str(self, ns_dict): From 8aa5e84ac3a9c72d065433a9709d184acae557a2 Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Mon, 11 Aug 2014 22:05:45 -0400 Subject: [PATCH 06/14] Added PackageIntents class to handle issue where invalid types could be passed into package_intents list --- stix/core/stix_header.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/stix/core/stix_header.py b/stix/core/stix_header.py index 02f06648..cb88085c 100644 --- a/stix/core/stix_header.py +++ b/stix/core/stix_header.py @@ -62,7 +62,7 @@ def package_intents(self): @package_intents.setter def package_intents(self, value): - self._package_intents = [] + self._package_intents = PackageIntents() if not value: return elif isinstance(value, list): @@ -168,3 +168,10 @@ def to_dict(self): d['profiles'] = self.profiles return d + + +class PackageIntents(stix.EntityList): + _contained_type = VocabString + + def _fix_value(self, value): + return PackageIntent(value) From 8efa8c2ef28df50a1ec9015441a44a608ad17eba Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Mon, 11 Aug 2014 22:08:02 -0400 Subject: [PATCH 07/14] Updated Indicator.indicator_types to use IndicatorTypes entitylist --- stix/indicator/indicator.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/stix/indicator/indicator.py b/stix/indicator/indicator.py index a677314e..70362ef1 100644 --- a/stix/indicator/indicator.py +++ b/stix/indicator/indicator.py @@ -704,4 +704,9 @@ def from_dict(cls, dict_repr, return_obj=None): super(CompositeIndicatorExpression, cls).from_dict(dict_repr, return_obj=return_obj) return_obj.operator = dict_repr.get('operator') return return_obj - + +class IndicatorTypes(stix.EntityList): + _contained_type = VocabString + + def _fix_value(self, value): + return IndicatorType(value) From cf172b0e44d70ca2a31cb0e94214924f7029216f Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Mon, 11 Aug 2014 22:11:40 -0400 Subject: [PATCH 08/14] Set indicator_types property to IndicatorTypes instance by default --- stix/indicator/indicator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix/indicator/indicator.py b/stix/indicator/indicator.py index 70362ef1..2588cc3b 100644 --- a/stix/indicator/indicator.py +++ b/stix/indicator/indicator.py @@ -221,7 +221,7 @@ def indicator_types(self): @indicator_types.setter def indicator_types(self, value): - self._indicator_types = [] + self._indicator_types = IndicatorTypes() if not value: return elif isinstance(value, list): From 1d12697ad92b616e499d54b9c99267ee8aa173b8 Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Tue, 12 Aug 2014 11:20:28 -0400 Subject: [PATCH 09/14] Fixed potential issue with namespace parser not descending into VocabString EntityLists in STIXHeader and Indicator --- stix/core/stix_header.py | 1 + stix/indicator/indicator.py | 1 + 2 files changed, 2 insertions(+) diff --git a/stix/core/stix_header.py b/stix/core/stix_header.py index cb88085c..c4a30560 100644 --- a/stix/core/stix_header.py +++ b/stix/core/stix_header.py @@ -171,6 +171,7 @@ def to_dict(self): class PackageIntents(stix.EntityList): + _namespace = "http://stix.mitre.org/stix-1" _contained_type = VocabString def _fix_value(self, value): diff --git a/stix/indicator/indicator.py b/stix/indicator/indicator.py index 2588cc3b..7833e9e0 100644 --- a/stix/indicator/indicator.py +++ b/stix/indicator/indicator.py @@ -706,6 +706,7 @@ def from_dict(cls, dict_repr, return_obj=None): return return_obj class IndicatorTypes(stix.EntityList): + _namespace = "http://stix.mitre.org/Indicator-2" _contained_type = VocabString def _fix_value(self, value): From a3e0f8424eeea968053654df6353b687e19a1996 Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Tue, 12 Aug 2014 11:20:59 -0400 Subject: [PATCH 10/14] Added VocabString EntityLists for Incident class --- stix/incident/__init__.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/stix/incident/__init__.py b/stix/incident/__init__.py index 5679fcc8..2ff98178 100644 --- a/stix/incident/__init__.py +++ b/stix/incident/__init__.py @@ -180,7 +180,7 @@ def intended_effects(self): @intended_effects.setter def intended_effects(self, value): - self._intended_effects = [] + self._intended_effects = IntendedEffects() if not value: return elif isinstance(value, list): @@ -227,7 +227,7 @@ def categories(self): @categories.setter def categories(self, value): - self._categories = [] + self._categories = IncidentCategories() if not value: return elif isinstance(value, list): @@ -272,7 +272,7 @@ def discovery_methods(self): @discovery_methods.setter def discovery_methods(self, value): - self._discovery_methods = [] + self._discovery_methods = DiscoveryMethods() if not value: return elif isinstance(value, list): @@ -716,3 +716,25 @@ class RelatedIncidents(GenericRelationshipList): _contained_type = RelatedIncident _inner_name = "incidents" + +class DiscoveryMethods(stix.EntityList): + _namespace = "http://stix.mitre.org/Incident-1" + _contained_type = VocabString + + def _fix_value(self, value): + return DiscoveryMethod(value) + + +class IncidentCategories(stix.EntityList): + _namespace = "http://stix.mitre.org/Incident-1" + _contained_type = VocabString + + def _fix_value(self, value): + return IncidentCategory(value) + +class IntendedEffects(stix.EntityList): + _namespace = "http://stix.mitre.org/Incident-1" + _contained_type = VocabString + + def _fix_value(self, value): + return IntendedEffect(value) From a8fcd8088c8421ea46b8c4d7d3c443929bf35728 Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Tue, 12 Aug 2014 11:27:01 -0400 Subject: [PATCH 11/14] Added Roles VocabString EntityList impl to InformationSource --- stix/common/information_source.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/stix/common/information_source.py b/stix/common/information_source.py index e883e948..1488f453 100644 --- a/stix/common/information_source.py +++ b/stix/common/information_source.py @@ -135,7 +135,7 @@ def roles(self): @roles.setter def roles(self, value): - self._roles = [] + self._roles = Roles() if not value: return @@ -243,3 +243,10 @@ class ContributingSources(stix.EntityList): _contained_type = InformationSource _inner_name = "sources" + +class Roles(stix.EntityList): + _namespace = "http://stix.mitre.org/common-1" + _contained_type = VocabString + + def _fix_value(self, value): + return InformationSourceRole(value) \ No newline at end of file From 23db20beef48ed00c1143f62df0cd0d3bcee25bb Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Tue, 12 Aug 2014 11:36:19 -0400 Subject: [PATCH 12/14] Added VocabString EntityLists to TTP classes and modules --- stix/ttp/infrastructure.py | 10 +++++++++- stix/ttp/malware_instance.py | 19 +++++++++++++++++-- stix/ttp/victim_targeting.py | 20 ++++++++++++++++++-- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/stix/ttp/infrastructure.py b/stix/ttp/infrastructure.py index dbca7b01..4350b309 100644 --- a/stix/ttp/infrastructure.py +++ b/stix/ttp/infrastructure.py @@ -63,7 +63,7 @@ def types(self): @types.setter def types(self, value): - self._types = [] + self._types = InfraStructureTypes() if not value: return elif isinstance(value, list): @@ -161,3 +161,11 @@ def from_dict(cls, dict_repr, return_obj=None): return_obj.observable_characterization = Observables.from_dict(dict_repr.get('observable_characterization')) return return_obj + + +class InfraStructureTypes(stix.EntityList): + _namespace = "http://stix.mitre.org/TTP-1" + _contained_type = VocabString + + def _fix_value(self, value): + return AttackerInfrastructureType(value) diff --git a/stix/ttp/malware_instance.py b/stix/ttp/malware_instance.py index 1825020b..5bc08d70 100644 --- a/stix/ttp/malware_instance.py +++ b/stix/ttp/malware_instance.py @@ -62,7 +62,7 @@ def names(self): @names.setter def names(self, value): - self._names = [] + self._names = MalwareNames() if not value: return elif isinstance(value, list): @@ -85,7 +85,7 @@ def types(self): @types.setter def types(self, value): - self._types = [] + self._types = MalwareTypes() if not value: return elif isinstance(value, list): @@ -195,6 +195,21 @@ def from_dict(cls, dict_repr, return_obj=None): return return_obj + + +class MalwareNames(stix.EntityList): + _namespace = "http://stix.mitre.org/TTP-1" + _contained_type = VocabString + + +class MalwareTypes(stix.EntityList): + _namespace = "http://stix.mitre.org/TTP-1" + _contained_type = VocabString + + def _fix_value(self, value): + return MalwareType(value) + + _EXTENSION_MAP = {} def add_extension(cls): _EXTENSION_MAP[cls._XSI_TYPE] = cls diff --git a/stix/ttp/victim_targeting.py b/stix/ttp/victim_targeting.py index 3ded998f..3570d16d 100644 --- a/stix/ttp/victim_targeting.py +++ b/stix/ttp/victim_targeting.py @@ -24,7 +24,7 @@ def targeted_systems(self): @targeted_systems.setter def targeted_systems(self, value): - self._targeted_systems = [] + self._targeted_systems = TargetedSystems() if not value: return elif isinstance(value, list): @@ -47,7 +47,7 @@ def targeted_information(self): @targeted_information.setter def targeted_information(self, value): - self._targeted_information = [] + self._targeted_information = TargetedInformation() if not value: return elif isinstance(value, list): @@ -135,3 +135,19 @@ def from_dict(cls, dict_repr, return_obj=None): return_obj.targeted_technical_details = Observables.from_dict(dict_repr.get('targeted_technical_details')) return return_obj + + +class TargetedSystems(stix.EntityList): + _namespace = "http://stix.mitre.org/TTP-1" + _contained_type = VocabString + + def _fix_value(self, value): + return SystemType(value) + + +class TargetedInformation(stix.EntityList): + _namespace = "http://stix.mitre.org/TTP-1" + _contained_type = VocabString + + def _fix_value(self, value): + return InformationType(value) From 522189a44a54d938646f890248c78553d80d9738 Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Tue, 12 Aug 2014 13:44:09 -0400 Subject: [PATCH 13/14] Added sample demonstrating VocabString use. --- examples/vocabstrings.py | 53 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 examples/vocabstrings.py diff --git a/examples/vocabstrings.py b/examples/vocabstrings.py new file mode 100644 index 00000000..4c9e0544 --- /dev/null +++ b/examples/vocabstrings.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# Copyright (c) 2014, The MITRE Corporation. All rights reserved. +# See LICENSE.txt for complete terms. + +''' +File: vocabstrings. + +Demonstrates uses of VocabString for STIX Controlled Vocabularies +''' +import sys +from stix.common.vocabs import VocabString + +def main(): + # Adding a Package_Intent field to a STIX_Header. + from stix.core import STIXHeader + from stix.common.vocabs import PackageIntent + + # Create STIXHeader instance + header = STIXHeader() + header.package_intents.append(PackageIntent.TERM_INDICATORS) + + # To add a Package_Intent value that exists outside of the + # PackageIntentVocab controlled vocabulary, we pass in an + # instance of VocabString. + # + # This will create a new Package_Intent field without an + # xsi:type and will not perform any validation of input terms. + # + # Passing in an instance of VocabString works for every + # ControlledVocabularyStringType field (or in python-stix, + # every VocabString field). + + non_default_value = VocabString("NON STANDARD PACKAGE INTENT") + header.package_intents.append(non_default_value) + + print header.to_xml() + + # NOTE: Passing in a str value that is not included in the list + # of default CV terms will raise a ValueError. This is why we pass + # in a VocabString instance. + # + # Example: + try: + print("[-] Attempting to add an str instance that does not exist " + "in the PackageIntent ALLOWED_VALUES list") + header.package_intents.append("THIS WILL THROW A VALUEERROR") + except Exception as ex: + print "[!] As expected, that failed. Here's the Exception message:" + print "[!]", str(ex) + +if __name__ == '__main__': + main() + From d4f30f233b7f8410f1396633897cf7372d7cee6f Mon Sep 17 00:00:00 2001 From: Bryan Worrell Date: Tue, 12 Aug 2014 15:21:42 -0400 Subject: [PATCH 14/14] Added examples section to documentation --- docs/examples/index.rst | 195 ++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 196 insertions(+) create mode 100644 docs/examples/index.rst diff --git a/docs/examples/index.rst b/docs/examples/index.rst new file mode 100644 index 00000000..cdd7533b --- /dev/null +++ b/docs/examples/index.rst @@ -0,0 +1,195 @@ +Examples +======== + +This page includes some basic examples of creating and parsing STIX content. + +There are a couple things we do in these examples for purposes of demonstration +that shouldn't be done in production code: + +* When calling ``to_xml()``, we use ``include_namespaces=False``. This is to + make the example output easier to read, but means the resulting output + cannot be successfully parsed. The XML parser doesn't know what namespaces + to use if they aren't included. In production code, you should explicitly + set ``include_namespaces`` to ``True`` or omit it entirely (``True`` is the + default). + +* In some examples, we use ``set_id_method(IDGenerator.METHOD_INT)`` to make + IDs for STIX constructs easier to read and cross-reference within the XML + document. In production code, you should omit this statement, which causes + random UUIDs to be created instead, or create explicit IDs yourself for STIX + constructs. + +See the `STIX Idioms `_ +documentation for more great examples of how to use ``python-stix``. + + +Creating a STIX Package +----------------------- +.. testcode:: + + from stix.core import STIXPackage, STIXHeader + from stix.utils import IDGenerator, set_id_method + + set_id_method(IDGenerator.METHOD_INT) # For testing and demonstration only! + + stix_package = STIXPackage() + stix_header = STIXHeader() + stix_header.description = "Getting Started!" + stix_package.stix_header = stix_header + + print stix_package.to_xml(include_namespaces=False) + +Which outputs: + +.. testoutput:: + + + + Getting Started! + + + + +ID Namespaces +------------- +By default, ``python-stix`` sets the default ID namespace to +``http://example.com`` with an alias of ``example``. This results in STIX +id declarations that look like +``id="example:Package-2813128d-f45e-41f7-b10a-20a5656e3785"``. + +To change this, use the ``stix.utils.set_id_namespace()`` method which takes +a dictionary as a parameter. + +.. testcode:: + + from stix.core import STIXPackage + from stix.utils import set_id_namespace + + NAMESPACE = {"http://MY-NAMESPACE.com" : "myNS"} + set_id_namespace(NAMESPACE) # new ids will be prefixed by "myNS" + + stix_package = STIXPackage() # id will be created automatically + print stix_package.to_xml() + +Which outputs: + +.. testoutput:: + + + +Success! The ``xmlns:myNS="http://MY-NAMESPACE.com"`` matches our ``NAMESPACE`` +dictionary and the ``id`` attribute includes the ``myNS`` namespace alias. + +Controlled Vocabularies: VocabString +------------------------------------ +Many fields in STIX leverage the ``stixCommon:ControlledVocabularyStringType``, +which acts as a base type for controlled vocabulary implementations. The STIX +language defines a set of default controlled vocabularies which are found in +the ``stix_default_vocabs.xsd`` XML Schema file. + +The ``python-stix`` library contains a ``stix.common.vocabs`` module, which +defines the ``VocabString`` class implementation of the schema +``ControlledVocabularyStringType`` as well as ``VocabString`` implementations +which correspond to default controlled vocabularies. + +For example, the ``stix_default_vocabularies.xsd`` schema defines a controlled +vocabulary for STIX Package Intents: ``PackageIntentVocab-1.0``. The +``stix.common.vocabs`` module contains an analogous ``PackageIntent`` class, +which acts as a derivation of ``VocabString``. + +Each ``VocabString`` implementation contains: + +* A static list of class-level term attributes, each beginning with ``TERM_` + (e.g., ``TERM_INDICATORS``) + +* A tuple containing all allowed vocabulary terms: ``ALLOWED_VALUES``, which is + use for input validation + +* Methods found on ``stix.Entity``, such as ``to_xml()``, ``to_dict()``, + ``from_dict()``, etc. + + +Interacting With VocabString Fields +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The following sections define ways of interacting with VocabString fields. + +Default Vocabulary Terms +######################## +The STIX Language often suggested a default controlled vocabulary type for a +given controlled vocabulary field. Each controlled vocabulary contains an +enumeration of allowed terms. + +Each ``VocabString`` implementation found in the ``stix.common.vocabs`` module +contains static class-level attributes for each vocabulary term. When setting +controlled vocabulary field values, it is recommended that users take advantage +of these class-level attributes. + +The following demonstrates setting the ``Package_Intent`` field with a default +vocabulary term. Note that the ``STIXHeader.package_intents`` property returns +a list. As such, we use the ``append()`` method to add terms. Other STIX +controlled vocabulary fields may only allow one value rather than a list of +values. + +.. testcode:: + + from stix.core import STIXHeader + from stix.common.vocabs import PackageIntent + + header = STIXHeader() + header.package_intents.append(PackageIntent.TERM_INDICATORS) + + print header.to_xml(include_namespaces=False) + +Which outputs: + +.. testoutput:: + + + Indicators + + +Non-Default Vocabulary Terms +############################ +Though it is suggested, STIX content authors are not required to use the default +controlled vocabulary for a given field. As such, ``python-stix`` allows users +to pass in non-default values for controlled vocabulary fields. + +To set a controlled vocabulary to a non-default vocabulary term, pass a +``VocabString`` instance into a controlled vocabulary field. + +A raw ``VocabString`` field will contain no ``xsi:type`` information or +``ALLOWED_VALUES`` members, which removes the input and schema validation +requirements. + +.. testcode:: + + from stix.core import STIXHeader + from stix.common.vocabs import VocabString, PackageIntent + + header = STIXHeader() + non_default_term = VocabString("NON-DEFAULT VOCABULARY TERM") + header.package_intents.append(non_default_term) + + print header.to_xml(include_namespaces=False) + +Which outputs: + +.. testoutput:: + + + NON-DEFAULT VOCABULARY TERM + + +Notice that the ```` field does not have an ``xsi:type`` +attribute. As such, this field can contain any string value and is not bound +by a controlled vocabulary enumeration of terms. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 08b0336a..41ef069c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -43,6 +43,7 @@ Contents installation getting_started + examples/index api_vs_bindings/index API Reference