diff --git a/docs/examples/index.rst b/docs/examples/index.rst
new file mode 100644
index 00000000..cdd7533b
--- /dev/null
+++ b/docs/examples/index.rst
@@ -0,0 +1,195 @@
+Examples
+========
+
+This page includes some basic examples of creating and parsing STIX content.
+
+There are a couple things we do in these examples for purposes of demonstration
+that shouldn't be done in production code:
+
+* When calling ``to_xml()``, we use ``include_namespaces=False``. This is to
+ make the example output easier to read, but means the resulting output
+ cannot be successfully parsed. The XML parser doesn't know what namespaces
+ to use if they aren't included. In production code, you should explicitly
+ set ``include_namespaces`` to ``True`` or omit it entirely (``True`` is the
+ default).
+
+* In some examples, we use ``set_id_method(IDGenerator.METHOD_INT)`` to make
+ IDs for STIX constructs easier to read and cross-reference within the XML
+ document. In production code, you should omit this statement, which causes
+ random UUIDs to be created instead, or create explicit IDs yourself for STIX
+ constructs.
+
+See the `STIX Idioms `_
+documentation for more great examples of how to use ``python-stix``.
+
+
+Creating a STIX Package
+-----------------------
+.. testcode::
+
+ from stix.core import STIXPackage, STIXHeader
+ from stix.utils import IDGenerator, set_id_method
+
+ set_id_method(IDGenerator.METHOD_INT) # For testing and demonstration only!
+
+ stix_package = STIXPackage()
+ stix_header = STIXHeader()
+ stix_header.description = "Getting Started!"
+ stix_package.stix_header = stix_header
+
+ print stix_package.to_xml(include_namespaces=False)
+
+Which outputs:
+
+.. testoutput::
+
+
+
+ Getting Started!
+
+
+
+
+ID Namespaces
+-------------
+By default, ``python-stix`` sets the default ID namespace to
+``http://example.com`` with an alias of ``example``. This results in STIX
+id declarations that look like
+``id="example:Package-2813128d-f45e-41f7-b10a-20a5656e3785"``.
+
+To change this, use the ``stix.utils.set_id_namespace()`` method which takes
+a dictionary as a parameter.
+
+.. testcode::
+
+ from stix.core import STIXPackage
+ from stix.utils import set_id_namespace
+
+ NAMESPACE = {"http://MY-NAMESPACE.com" : "myNS"}
+ set_id_namespace(NAMESPACE) # new ids will be prefixed by "myNS"
+
+ stix_package = STIXPackage() # id will be created automatically
+ print stix_package.to_xml()
+
+Which outputs:
+
+.. testoutput::
+
+
+
+Success! The ``xmlns:myNS="http://MY-NAMESPACE.com"`` matches our ``NAMESPACE``
+dictionary and the ``id`` attribute includes the ``myNS`` namespace alias.
+
+Controlled Vocabularies: VocabString
+------------------------------------
+Many fields in STIX leverage the ``stixCommon:ControlledVocabularyStringType``,
+which acts as a base type for controlled vocabulary implementations. The STIX
+language defines a set of default controlled vocabularies which are found in
+the ``stix_default_vocabs.xsd`` XML Schema file.
+
+The ``python-stix`` library contains a ``stix.common.vocabs`` module, which
+defines the ``VocabString`` class implementation of the schema
+``ControlledVocabularyStringType`` as well as ``VocabString`` implementations
+which correspond to default controlled vocabularies.
+
+For example, the ``stix_default_vocabularies.xsd`` schema defines a controlled
+vocabulary for STIX Package Intents: ``PackageIntentVocab-1.0``. The
+``stix.common.vocabs`` module contains an analogous ``PackageIntent`` class,
+which acts as a derivation of ``VocabString``.
+
+Each ``VocabString`` implementation contains:
+
+* A static list of class-level term attributes, each beginning with ``TERM_`
+ (e.g., ``TERM_INDICATORS``)
+
+* A tuple containing all allowed vocabulary terms: ``ALLOWED_VALUES``, which is
+ use for input validation
+
+* Methods found on ``stix.Entity``, such as ``to_xml()``, ``to_dict()``,
+ ``from_dict()``, etc.
+
+
+Interacting With VocabString Fields
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The following sections define ways of interacting with VocabString fields.
+
+Default Vocabulary Terms
+########################
+The STIX Language often suggested a default controlled vocabulary type for a
+given controlled vocabulary field. Each controlled vocabulary contains an
+enumeration of allowed terms.
+
+Each ``VocabString`` implementation found in the ``stix.common.vocabs`` module
+contains static class-level attributes for each vocabulary term. When setting
+controlled vocabulary field values, it is recommended that users take advantage
+of these class-level attributes.
+
+The following demonstrates setting the ``Package_Intent`` field with a default
+vocabulary term. Note that the ``STIXHeader.package_intents`` property returns
+a list. As such, we use the ``append()`` method to add terms. Other STIX
+controlled vocabulary fields may only allow one value rather than a list of
+values.
+
+.. testcode::
+
+ from stix.core import STIXHeader
+ from stix.common.vocabs import PackageIntent
+
+ header = STIXHeader()
+ header.package_intents.append(PackageIntent.TERM_INDICATORS)
+
+ print header.to_xml(include_namespaces=False)
+
+Which outputs:
+
+.. testoutput::
+
+
+ Indicators
+
+
+Non-Default Vocabulary Terms
+############################
+Though it is suggested, STIX content authors are not required to use the default
+controlled vocabulary for a given field. As such, ``python-stix`` allows users
+to pass in non-default values for controlled vocabulary fields.
+
+To set a controlled vocabulary to a non-default vocabulary term, pass a
+``VocabString`` instance into a controlled vocabulary field.
+
+A raw ``VocabString`` field will contain no ``xsi:type`` information or
+``ALLOWED_VALUES`` members, which removes the input and schema validation
+requirements.
+
+.. testcode::
+
+ from stix.core import STIXHeader
+ from stix.common.vocabs import VocabString, PackageIntent
+
+ header = STIXHeader()
+ non_default_term = VocabString("NON-DEFAULT VOCABULARY TERM")
+ header.package_intents.append(non_default_term)
+
+ print header.to_xml(include_namespaces=False)
+
+Which outputs:
+
+.. testoutput::
+
+
+ NON-DEFAULT VOCABULARY TERM
+
+
+Notice that the ```` field does not have an ``xsi:type``
+attribute. As such, this field can contain any string value and is not bound
+by a controlled vocabulary enumeration of terms.
\ No newline at end of file
diff --git a/docs/index.rst b/docs/index.rst
index 08b0336a..41ef069c 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -43,6 +43,7 @@ Contents
installation
getting_started
+ examples/index
api_vs_bindings/index
API Reference
diff --git a/examples/vocabstrings.py b/examples/vocabstrings.py
new file mode 100644
index 00000000..4c9e0544
--- /dev/null
+++ b/examples/vocabstrings.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# Copyright (c) 2014, The MITRE Corporation. All rights reserved.
+# See LICENSE.txt for complete terms.
+
+'''
+File: vocabstrings.
+
+Demonstrates uses of VocabString for STIX Controlled Vocabularies
+'''
+import sys
+from stix.common.vocabs import VocabString
+
+def main():
+ # Adding a Package_Intent field to a STIX_Header.
+ from stix.core import STIXHeader
+ from stix.common.vocabs import PackageIntent
+
+ # Create STIXHeader instance
+ header = STIXHeader()
+ header.package_intents.append(PackageIntent.TERM_INDICATORS)
+
+ # To add a Package_Intent value that exists outside of the
+ # PackageIntentVocab controlled vocabulary, we pass in an
+ # instance of VocabString.
+ #
+ # This will create a new Package_Intent field without an
+ # xsi:type and will not perform any validation of input terms.
+ #
+ # Passing in an instance of VocabString works for every
+ # ControlledVocabularyStringType field (or in python-stix,
+ # every VocabString field).
+
+ non_default_value = VocabString("NON STANDARD PACKAGE INTENT")
+ header.package_intents.append(non_default_value)
+
+ print header.to_xml()
+
+ # NOTE: Passing in a str value that is not included in the list
+ # of default CV terms will raise a ValueError. This is why we pass
+ # in a VocabString instance.
+ #
+ # Example:
+ try:
+ print("[-] Attempting to add an str instance that does not exist "
+ "in the PackageIntent ALLOWED_VALUES list")
+ header.package_intents.append("THIS WILL THROW A VALUEERROR")
+ except Exception as ex:
+ print "[!] As expected, that failed. Here's the Exception message:"
+ print "[!]", str(ex)
+
+if __name__ == '__main__':
+ main()
+
diff --git a/stix/base.py b/stix/base.py
index 6eddd8f1..a69cfcde 100644
--- a/stix/base.py
+++ b/stix/base.py
@@ -22,35 +22,42 @@ def from_obj(cls, obj):
"""Create an object from a binding object"""
raise NotImplementedError()
- def _get_namespaces(self, ns_dict):
+ def _get_namespaces(self, ns_dict=None):
import stix.utils.nsparser as nsparser
- namespace_parser = nsparser.NamespaceParser()
- all_ns_dict = namespace_parser.get_namespaces(self, ns_dict=ns_dict)
+ parser = nsparser.NamespaceParser()
+ all_ns_dict = parser.get_namespaces(self, ns_dict=ns_dict)
return all_ns_dict
- def _get_schema_locations(self, ns_dict=None):
+ def _get_schema_locations(self, ns_dict=None, schemaloc_dict=None):
import stix.utils.nsparser as nsparser
- schemaloc_dict = nsparser.NamespaceParser().get_namespace_schemalocation_dict(self, ns_dict=ns_dict)
- return schemaloc_dict
+ parser = nsparser.NamespaceParser()
+ all_schemaloc_dict = \
+ parser.get_namespace_schemalocation_dict(self, ns_dict=ns_dict, schemaloc_dict=schemaloc_dict)
+ return all_schemaloc_dict
- def to_xml(self, include_namespaces=True, ns_dict=None, pretty=True):
+ def to_xml(self, include_namespaces=True, ns_dict=None, schemaloc_dict=None, pretty=True):
"""Export an object as an XML String"""
s = StringIO()
namespace_def = ""
import stix.utils.nsparser as nsparser
if include_namespaces:
- if not ns_dict: ns_dict = {}
all_ns_dict = self._get_namespaces(ns_dict)
- schemaloc_dict = self._get_schema_locations(all_ns_dict)
- namespace_def = nsparser.NamespaceParser().get_namespace_def_str(all_ns_dict, schemaloc_dict)
+ all_schemaloc_dict = \
+ self._get_schema_locations(all_ns_dict, schemaloc_dict)
+
+ parser = nsparser.NamespaceParser()
+ namespace_def = parser.get_namespace_def_str(all_ns_dict,
+ all_schemaloc_dict)
else:
- all_ns_dict = dict(nsparser.DEFAULT_STIX_NS_TO_PREFIX.items() + nsparser.DEFAULT_EXT_TO_PREFIX.items())
+ all_ns_dict = dict(nsparser.DEFAULT_STIX_NS_TO_PREFIX.items() +
+ nsparser.DEFAULT_EXT_TO_PREFIX.items())
if not pretty:
namespace_def = namespace_def.replace('\n\t', ' ')
- self.to_obj().export(s, 0, all_ns_dict, pretty_print=pretty, namespacedef_=namespace_def)
+ self.to_obj().export(s, 0, all_ns_dict, pretty_print=pretty,
+ namespacedef_=namespace_def)
return s.getvalue()
def _get_children(self):
@@ -62,7 +69,9 @@ def _get_children(self):
yield obj
elif isinstance(obj, (tuple, collections.MutableSequence)):
for item in obj:
- if isinstance(item, Entity) or isinstance(item, Observable) or isinstance(item, cybox.Entity):
+ if (isinstance(item, Entity) or
+ isinstance(item, Observable) or
+ isinstance(item, cybox.Entity)):
yield item
def to_json(self):
diff --git a/stix/common/information_source.py b/stix/common/information_source.py
index e883e948..1488f453 100644
--- a/stix/common/information_source.py
+++ b/stix/common/information_source.py
@@ -135,7 +135,7 @@ def roles(self):
@roles.setter
def roles(self, value):
- self._roles = []
+ self._roles = Roles()
if not value:
return
@@ -243,3 +243,10 @@ class ContributingSources(stix.EntityList):
_contained_type = InformationSource
_inner_name = "sources"
+
+class Roles(stix.EntityList):
+ _namespace = "http://stix.mitre.org/common-1"
+ _contained_type = VocabString
+
+ def _fix_value(self, value):
+ return InformationSourceRole(value)
\ No newline at end of file
diff --git a/stix/common/vocabs.py b/stix/common/vocabs.py
index eaaaf0ea..14d11409 100644
--- a/stix/common/vocabs.py
+++ b/stix/common/vocabs.py
@@ -67,7 +67,7 @@ def lookup_class(xsi_type):
if xsi_type in k:
return v
- raise ValueError("Unregistered xsi:type %s" % xsi_type)
+ return VocabString
def to_obj(self, return_obj=None):
if not return_obj:
diff --git a/stix/core/stix_header.py b/stix/core/stix_header.py
index 02f06648..c4a30560 100644
--- a/stix/core/stix_header.py
+++ b/stix/core/stix_header.py
@@ -62,7 +62,7 @@ def package_intents(self):
@package_intents.setter
def package_intents(self, value):
- self._package_intents = []
+ self._package_intents = PackageIntents()
if not value:
return
elif isinstance(value, list):
@@ -168,3 +168,11 @@ def to_dict(self):
d['profiles'] = self.profiles
return d
+
+
+class PackageIntents(stix.EntityList):
+ _namespace = "http://stix.mitre.org/stix-1"
+ _contained_type = VocabString
+
+ def _fix_value(self, value):
+ return PackageIntent(value)
diff --git a/stix/incident/__init__.py b/stix/incident/__init__.py
index 5679fcc8..2ff98178 100644
--- a/stix/incident/__init__.py
+++ b/stix/incident/__init__.py
@@ -180,7 +180,7 @@ def intended_effects(self):
@intended_effects.setter
def intended_effects(self, value):
- self._intended_effects = []
+ self._intended_effects = IntendedEffects()
if not value:
return
elif isinstance(value, list):
@@ -227,7 +227,7 @@ def categories(self):
@categories.setter
def categories(self, value):
- self._categories = []
+ self._categories = IncidentCategories()
if not value:
return
elif isinstance(value, list):
@@ -272,7 +272,7 @@ def discovery_methods(self):
@discovery_methods.setter
def discovery_methods(self, value):
- self._discovery_methods = []
+ self._discovery_methods = DiscoveryMethods()
if not value:
return
elif isinstance(value, list):
@@ -716,3 +716,25 @@ class RelatedIncidents(GenericRelationshipList):
_contained_type = RelatedIncident
_inner_name = "incidents"
+
+class DiscoveryMethods(stix.EntityList):
+ _namespace = "http://stix.mitre.org/Incident-1"
+ _contained_type = VocabString
+
+ def _fix_value(self, value):
+ return DiscoveryMethod(value)
+
+
+class IncidentCategories(stix.EntityList):
+ _namespace = "http://stix.mitre.org/Incident-1"
+ _contained_type = VocabString
+
+ def _fix_value(self, value):
+ return IncidentCategory(value)
+
+class IntendedEffects(stix.EntityList):
+ _namespace = "http://stix.mitre.org/Incident-1"
+ _contained_type = VocabString
+
+ def _fix_value(self, value):
+ return IntendedEffect(value)
diff --git a/stix/indicator/indicator.py b/stix/indicator/indicator.py
index a677314e..7833e9e0 100644
--- a/stix/indicator/indicator.py
+++ b/stix/indicator/indicator.py
@@ -221,7 +221,7 @@ def indicator_types(self):
@indicator_types.setter
def indicator_types(self, value):
- self._indicator_types = []
+ self._indicator_types = IndicatorTypes()
if not value:
return
elif isinstance(value, list):
@@ -704,4 +704,10 @@ def from_dict(cls, dict_repr, return_obj=None):
super(CompositeIndicatorExpression, cls).from_dict(dict_repr, return_obj=return_obj)
return_obj.operator = dict_repr.get('operator')
return return_obj
-
+
+class IndicatorTypes(stix.EntityList):
+ _namespace = "http://stix.mitre.org/Indicator-2"
+ _contained_type = VocabString
+
+ def _fix_value(self, value):
+ return IndicatorType(value)
diff --git a/stix/ttp/infrastructure.py b/stix/ttp/infrastructure.py
index dbca7b01..4350b309 100644
--- a/stix/ttp/infrastructure.py
+++ b/stix/ttp/infrastructure.py
@@ -63,7 +63,7 @@ def types(self):
@types.setter
def types(self, value):
- self._types = []
+ self._types = InfraStructureTypes()
if not value:
return
elif isinstance(value, list):
@@ -161,3 +161,11 @@ def from_dict(cls, dict_repr, return_obj=None):
return_obj.observable_characterization = Observables.from_dict(dict_repr.get('observable_characterization'))
return return_obj
+
+
+class InfraStructureTypes(stix.EntityList):
+ _namespace = "http://stix.mitre.org/TTP-1"
+ _contained_type = VocabString
+
+ def _fix_value(self, value):
+ return AttackerInfrastructureType(value)
diff --git a/stix/ttp/malware_instance.py b/stix/ttp/malware_instance.py
index 1825020b..5bc08d70 100644
--- a/stix/ttp/malware_instance.py
+++ b/stix/ttp/malware_instance.py
@@ -62,7 +62,7 @@ def names(self):
@names.setter
def names(self, value):
- self._names = []
+ self._names = MalwareNames()
if not value:
return
elif isinstance(value, list):
@@ -85,7 +85,7 @@ def types(self):
@types.setter
def types(self, value):
- self._types = []
+ self._types = MalwareTypes()
if not value:
return
elif isinstance(value, list):
@@ -195,6 +195,21 @@ def from_dict(cls, dict_repr, return_obj=None):
return return_obj
+
+
+class MalwareNames(stix.EntityList):
+ _namespace = "http://stix.mitre.org/TTP-1"
+ _contained_type = VocabString
+
+
+class MalwareTypes(stix.EntityList):
+ _namespace = "http://stix.mitre.org/TTP-1"
+ _contained_type = VocabString
+
+ def _fix_value(self, value):
+ return MalwareType(value)
+
+
_EXTENSION_MAP = {}
def add_extension(cls):
_EXTENSION_MAP[cls._XSI_TYPE] = cls
diff --git a/stix/ttp/victim_targeting.py b/stix/ttp/victim_targeting.py
index 3ded998f..3570d16d 100644
--- a/stix/ttp/victim_targeting.py
+++ b/stix/ttp/victim_targeting.py
@@ -24,7 +24,7 @@ def targeted_systems(self):
@targeted_systems.setter
def targeted_systems(self, value):
- self._targeted_systems = []
+ self._targeted_systems = TargetedSystems()
if not value:
return
elif isinstance(value, list):
@@ -47,7 +47,7 @@ def targeted_information(self):
@targeted_information.setter
def targeted_information(self, value):
- self._targeted_information = []
+ self._targeted_information = TargetedInformation()
if not value:
return
elif isinstance(value, list):
@@ -135,3 +135,19 @@ def from_dict(cls, dict_repr, return_obj=None):
return_obj.targeted_technical_details = Observables.from_dict(dict_repr.get('targeted_technical_details'))
return return_obj
+
+
+class TargetedSystems(stix.EntityList):
+ _namespace = "http://stix.mitre.org/TTP-1"
+ _contained_type = VocabString
+
+ def _fix_value(self, value):
+ return SystemType(value)
+
+
+class TargetedInformation(stix.EntityList):
+ _namespace = "http://stix.mitre.org/TTP-1"
+ _contained_type = VocabString
+
+ def _fix_value(self, value):
+ return InformationType(value)
diff --git a/stix/utils/nsparser.py b/stix/utils/nsparser.py
index e10effcf..1c53b06a 100644
--- a/stix/utils/nsparser.py
+++ b/stix/utils/nsparser.py
@@ -2,6 +2,7 @@
# See LICENSE.txt for complete terms.
import inspect
+import warnings
import stix
import cybox
from cybox.core import Observables, Observable
@@ -11,38 +12,45 @@ class NamespaceParser(object):
def __init__(self):
pass
- def _get_observable_namespaces(self, obs):
- '''Returns namespaces used within a CybOX Observable'''
+ def _get_observable_namespace_dict(self, obs):
+ '''Returns a dict of namespaces used within a CybOX Observable'''
+ namespaces = {'http://cybox.mitre.org/default_vocabularies-2': 'cyboxVocabs'}
+
obs_namespaces = obs._get_namespaces()
- namespaces = [x.name for x in obs_namespaces]
- namespaces.append('http://cybox.mitre.org/default_vocabularies-2')
+ for namespace in obs_namespaces:
+ namespaces[namespace.name] = namespace.prefix
+
return namespaces
- def _get_namespace_set(self, entity):
- all_namespaces = set()
+ def _get_namespace_dict(self, entity):
+ all_namespaces = {}
if not isinstance(entity, (stix.Entity, cybox.Entity)):
raise ValueError("Must provide an instance of stix.Entity or cybox.core.Observable")
entity.nsparser_touched = True
if isinstance(entity, Observable):
- all_namespaces.update(self._get_observable_namespaces(entity))
+ all_namespaces.update(self._get_observable_namespace_dict(entity))
elif isinstance(entity, Observables):
for child in entity._get_children():
- all_namespaces.update(self._get_namespace_set(child))
+ all_namespaces.update(self._get_namespace_dict(child))
elif hasattr(entity, "_namespace"):
- all_namespaces.add(entity._namespace)
+ if hasattr(entity, "_XSI_TYPE") and entity._XSI_TYPE:
+ ns_alias = entity._XSI_TYPE.split(":")[0]
+ all_namespaces[entity._namespace] = ns_alias
+ else:
+ all_namespaces[entity._namespace] = None
for child in entity._get_children():
if not hasattr(child, "nsparser_touched"):
if hasattr(child, "_namespace") or isinstance(child, Observable):
- all_namespaces.update(self._get_namespace_set(child))
+ all_namespaces.update(self._get_namespace_dict(child))
del entity.nsparser_touched
return all_namespaces
def get_namespaces(self, entity, ns_dict=None):
- """Returns a dictionary of namespace=>alias for all namespaces
+ """Returns a set of namespace for all namespaces
used within the supplied entity.
Arguments:
@@ -52,27 +60,54 @@ def get_namespaces(self, entity, ns_dict=None):
import stix.utils.idgen as idgen
if not ns_dict: ns_dict = {}
- all_ns_dict = {'http://www.w3.org/2001/XMLSchema-instance': 'xsi',
- 'http://stix.mitre.org/stix-1': 'stix',
- 'http://stix.mitre.org/common-1': 'stixCommon',
- 'http://stix.mitre.org/default_vocabularies-1': 'stixVocabs',
- idgen.get_id_namespace() : idgen.get_id_namespace_alias()}
-
- default_cybox_namespaces = dict((ns,alias) for (ns,alias,schemaloc) in cybox_nsparser.NS_LIST)
- default_stix_namespaces = dict(default_cybox_namespaces.items() + XML_NAMESPACES.items() + DEFAULT_STIX_NS_TO_PREFIX.items() + DEFAULT_EXT_TO_PREFIX.items())
-
- ns_set = self._get_namespace_set(entity)
- for ns in ns_set:
- all_ns_dict[ns] = default_stix_namespaces[ns]
+ entity_namespaces = \
+ {'http://www.w3.org/2001/XMLSchema-instance': 'xsi',
+ 'http://stix.mitre.org/stix-1': 'stix',
+ 'http://stix.mitre.org/common-1': 'stixCommon',
+ 'http://stix.mitre.org/default_vocabularies-1': 'stixVocabs',
+ idgen.get_id_namespace(): idgen.get_id_namespace_alias()}
- # add additional @ns_dict and parsed, input namespaces
- all_ns_dict.update(ns_dict)
+ default_cybox_namespaces = dict((ns, alias) for (ns, alias, _) in cybox_nsparser.NS_LIST)
+ default_stix_namespaces = dict(default_cybox_namespaces.items() +
+ XML_NAMESPACES.items() +
+ DEFAULT_STIX_NS_TO_PREFIX.items() +
+ DEFAULT_EXT_TO_PREFIX.items())
+
if hasattr(entity, "__input_namespaces__"):
- for ns,alias in entity.__input_namespaces__.iteritems():
+ for ns, alias in entity.__input_namespaces__.iteritems():
if ns not in (default_stix_namespaces):
- all_ns_dict[ns] = alias
+ entity_namespaces[ns] = alias
+
+ entity_ns_dict = self._get_namespace_dict(entity)
+ for ns, alias in entity_ns_dict.iteritems():
+ if alias:
+ entity_namespaces[ns] = alias
+ elif ns not in entity_namespaces:
+ default_alias = default_stix_namespaces[ns]
+ entity_namespaces[ns] = default_alias
- return all_ns_dict
+ # add additional @ns_dict and parsed
+ entity_namespaces.update(ns_dict)
+
+ # Remove "http://example.com/" namespace if "http://example.com" exists
+ # A bit of a hack to resolve issues with the STIX samples and our
+ # own default ID namespace
+ if all((entity_namespaces.get('http://example.com/'),
+ entity_namespaces.get('http://example.com'))):
+ del entity_namespaces['http://example.com/']
+
+ # sanity check namespaces for things like duplicate aliases
+ aliases = {}
+ for ns, alias in entity_namespaces.iteritems():
+ if alias not in aliases:
+ aliases[alias] = ns
+ else:
+ # TODO: Should we just throw an exception here?
+ # The XML will be invalid if there is a duplicate ns alias
+ warnings.warn("namespace alias '%s' mapped to '%s' and '%s'" %
+ (alias, ns, aliases[alias]))
+
+ return entity_namespaces
def _get_input_schemalocations(self, entity):
all_schemalocations = {}
@@ -90,119 +125,134 @@ def _get_input_schemalocations(self, entity):
del entity.nsparser_touched
return all_schemalocations
- def get_namespace_schemalocation_dict(self, entity, ns_dict=None):
+ def get_namespace_schemalocation_dict(self, entity, ns_dict=None, schemaloc_dict=None):
d = {}
- if ns_dict:
- ns_set = ns_dict.iterkeys()
- else:
- ns_set = self.get_namespaces(entity).iterkeys()
+ if not ns_dict:
+ ns_dict = self.get_namespaces(entity)
+
+ if not schemaloc_dict:
+ schemaloc_dict = {}
input_schemalocations = self._get_input_schemalocations(entity)
d.update(input_schemalocations)
-
+
+ # Build default schemaLocation dictionary for CybOX
default_cybox_schemaloc_dict = {}
- for (ns,alias,schemaloc) in cybox_nsparser.NS_LIST:
- if schemaloc: default_cybox_schemaloc_dict[ns] = schemaloc
-
- default_stix_schemaloc_dict = dict(STIX_NS_TO_SCHEMALOCATION.items() + EXT_NS_TO_SCHEMALOCATION.items() + default_cybox_schemaloc_dict.items())
+ for ns, _, schemaloc in cybox_nsparser.NS_LIST:
+ if schemaloc:
+ default_cybox_schemaloc_dict[ns] = schemaloc
+
+ # Build default schemaLocation dict for STIX and CybOX and extension
+ default_stix_schemaloc_dict = dict(STIX_NS_TO_SCHEMALOCATION.items() +
+ EXT_NS_TO_SCHEMALOCATION.items() +
+ default_cybox_schemaloc_dict.items())
- for ns in ns_set:
+ # Iterate over input/discovered namespaces for document and attempt
+ # to map them to schemalocations. Warn if unable to map ns to schemaloc.
+ for ns in ns_dict.iterkeys():
if ns in default_stix_schemaloc_dict:
schemalocation = default_stix_schemaloc_dict[ns]
d[ns] = schemalocation
else:
- continue
+ if not ((ns in schemaloc_dict) or
+ (ns in d) or
+ (ns in XML_NAMESPACES)):
+ warnings.warn("Unable to map namespace '%s' to "
+ "schemaLocation" % ns)
+ d.update(schemaloc_dict)
return d
def _get_xmlns_str(self, ns_dict):
- return "\n\t".join(['xmlns:%s="%s"' % (alias,ns) for (ns,alias) in sorted(ns_dict.iteritems())])
+ return "\n\t".join(['xmlns:%s="%s"' %
+ (alias, ns) for ns, alias in sorted(ns_dict.iteritems())])
def _get_schemaloc_str(self, schemaloc_dict):
schemaloc_str_start = 'xsi:schemaLocation="\n\t'
schemaloc_str_end = '"'
- schemaloc_str_content = "\n\t".join(["%s %s" % (ns, loc) for ns,loc in sorted(schemaloc_dict.iteritems())])
+ schemaloc_str_content = "\n\t".join(["%s %s" %
+ (ns, loc) for ns, loc in sorted(schemaloc_dict.iteritems())])
return schemaloc_str_start + schemaloc_str_content + schemaloc_str_end
- def get_namespace_def_str(self, ns_dict, schemaloc_dict):
- return "\n\t" + self._get_xmlns_str(ns_dict) + "\n\t" + self._get_schemaloc_str(schemaloc_dict)
+ def get_namespace_def_str(self, namespaces, schemaloc_dict):
+ return "\n\t" + self._get_xmlns_str(namespaces) + "\n\t" + self._get_schemaloc_str(schemaloc_dict)
-XML_NAMESPACES = {'http://www.w3.org/2001/XMLSchema-instance' : 'xsi',
- 'http://www.w3.org/2001/XMLSchema' : 'xs',
- 'http://www.w3.org/1999/xlink' : 'xlink',
+XML_NAMESPACES = {'http://www.w3.org/2001/XMLSchema-instance': 'xsi',
+ 'http://www.w3.org/2001/XMLSchema': 'xs',
+ 'http://www.w3.org/1999/xlink': 'xlink',
'http://www.w3.org/2000/09/xmldsig#': 'ds'}
# Schema locations for namespaces defined by the STIX language
STIX_NS_TO_SCHEMALOCATION = {
- 'http://data-marking.mitre.org/Marking-1' : 'http://stix.mitre.org/XMLSchema/data_marking/1.1.1/data_marking.xsd',
- 'http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1' : 'http://stix.mitre.org/XMLSchema/extensions/marking/simple/1.1.1/simple_marking.xsd',
- 'http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1' : 'http://stix.mitre.org/XMLSchema/extensions/marking/tlp/1.1.1/tlp_marking.xsd',
- 'http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1' : 'http://stix.mitre.org/XMLSchema/extensions/marking/terms_of_use/1.0/terms_of_use_marking.xsd',
- 'http://stix.mitre.org/Campaign-1' : 'http://stix.mitre.org/XMLSchema/campaign/1.1.1/campaign.xsd',
- 'http://stix.mitre.org/CourseOfAction-1' : 'http://stix.mitre.org/XMLSchema/course_of_action/1.1.1/course_of_action.xsd',
- 'http://stix.mitre.org/ExploitTarget-1' : 'http://stix.mitre.org/XMLSchema/exploit_target/1.1.1/exploit_target.xsd',
- 'http://stix.mitre.org/Incident-1' : 'http://stix.mitre.org/XMLSchema/incident/1.1.1/incident.xsd',
- 'http://stix.mitre.org/Indicator-2' : 'http://stix.mitre.org/XMLSchema/indicator/2.1.1/indicator.xsd',
- 'http://stix.mitre.org/TTP-1' : 'http://stix.mitre.org/XMLSchema/ttp/1.1.1/ttp.xsd',
- 'http://stix.mitre.org/ThreatActor-1' : 'http://stix.mitre.org/XMLSchema/threat_actor/1.1.1/threat_actor.xsd',
- 'http://stix.mitre.org/common-1' : 'http://stix.mitre.org/XMLSchema/common/1.1.1/stix_common.xsd',
- 'http://stix.mitre.org/default_vocabularies-1' : 'http://stix.mitre.org/XMLSchema/default_vocabularies/1.1.1/stix_default_vocabularies.xsd',
- 'http://stix.mitre.org/extensions/AP#CAPEC2.7-1' : 'http://stix.mitre.org/XMLSchema/extensions/attack_pattern/capec_2.7/1.0.1/capec_2.7_attack_pattern.xsd',
- 'http://stix.mitre.org/extensions/Address#CIQAddress3.0-1' : 'http://stix.mitre.org/XMLSchema/extensions/address/ciq_3.0/1.1.1/ciq_3.0_address.xsd',
- 'http://stix.mitre.org/extensions/Identity#CIQIdentity3.0-1' : 'http://stix.mitre.org/XMLSchema/extensions/identity/ciq_3.0/1.1.1/ciq_3.0_identity.xsd',
- 'http://stix.mitre.org/extensions/Malware#MAEC4.1-1' : 'http://stix.mitre.org/XMLSchema/extensions/malware/maec_4.1/1.0.1/maec_4.1_malware.xsd',
- 'http://stix.mitre.org/extensions/StructuredCOA#Generic-1' : 'http://stix.mitre.org/XMLSchema/extensions/structured_coa/generic/1.1.1/generic_structured_coa.xsd',
- 'http://stix.mitre.org/extensions/TestMechanism#Generic-1' : 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/generic/1.1.1/generic_test_mechanism.xsd',
- 'http://stix.mitre.org/extensions/TestMechanism#OVAL5.10-1' : 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/oval_5.10/1.1.1/oval_5.10_test_mechanism.xsd',
- 'http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1' : 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/open_ioc_2010/1.1.1/open_ioc_2010_test_mechanism.xsd',
- 'http://stix.mitre.org/extensions/TestMechanism#Snort-1' : 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/snort/1.1.1/snort_test_mechanism.xsd',
- 'http://stix.mitre.org/extensions/TestMechanism#YARA-1' : 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/yara/1.1.1/yara_test_mechanism.xsd',
- 'http://stix.mitre.org/extensions/Vulnerability#CVRF-1' : 'http://stix.mitre.org/XMLSchema/extensions/vulnerability/cvrf_1.1/1.1.1/cvrf_1.1_vulnerability.xsd',
- 'http://stix.mitre.org/stix-1' : 'http://stix.mitre.org/XMLSchema/core/1.1.1/stix_core.xsd'}
+ 'http://data-marking.mitre.org/Marking-1': 'http://stix.mitre.org/XMLSchema/data_marking/1.1.1/data_marking.xsd',
+ 'http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1': 'http://stix.mitre.org/XMLSchema/extensions/marking/simple/1.1.1/simple_marking.xsd',
+ 'http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1': 'http://stix.mitre.org/XMLSchema/extensions/marking/tlp/1.1.1/tlp_marking.xsd',
+ 'http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1': 'http://stix.mitre.org/XMLSchema/extensions/marking/terms_of_use/1.0/terms_of_use_marking.xsd',
+ 'http://stix.mitre.org/Campaign-1': 'http://stix.mitre.org/XMLSchema/campaign/1.1.1/campaign.xsd',
+ 'http://stix.mitre.org/CourseOfAction-1': 'http://stix.mitre.org/XMLSchema/course_of_action/1.1.1/course_of_action.xsd',
+ 'http://stix.mitre.org/ExploitTarget-1': 'http://stix.mitre.org/XMLSchema/exploit_target/1.1.1/exploit_target.xsd',
+ 'http://stix.mitre.org/Incident-1': 'http://stix.mitre.org/XMLSchema/incident/1.1.1/incident.xsd',
+ 'http://stix.mitre.org/Indicator-2': 'http://stix.mitre.org/XMLSchema/indicator/2.1.1/indicator.xsd',
+ 'http://stix.mitre.org/TTP-1': 'http://stix.mitre.org/XMLSchema/ttp/1.1.1/ttp.xsd',
+ 'http://stix.mitre.org/ThreatActor-1': 'http://stix.mitre.org/XMLSchema/threat_actor/1.1.1/threat_actor.xsd',
+ 'http://stix.mitre.org/common-1': 'http://stix.mitre.org/XMLSchema/common/1.1.1/stix_common.xsd',
+ 'http://stix.mitre.org/default_vocabularies-1': 'http://stix.mitre.org/XMLSchema/default_vocabularies/1.1.1/stix_default_vocabularies.xsd',
+ 'http://stix.mitre.org/extensions/AP#CAPEC2.7-1': 'http://stix.mitre.org/XMLSchema/extensions/attack_pattern/capec_2.7/1.0.1/capec_2.7_attack_pattern.xsd',
+ 'http://stix.mitre.org/extensions/Address#CIQAddress3.0-1': 'http://stix.mitre.org/XMLSchema/extensions/address/ciq_3.0/1.1.1/ciq_3.0_address.xsd',
+ 'http://stix.mitre.org/extensions/Identity#CIQIdentity3.0-1': 'http://stix.mitre.org/XMLSchema/extensions/identity/ciq_3.0/1.1.1/ciq_3.0_identity.xsd',
+ 'http://stix.mitre.org/extensions/Malware#MAEC4.1-1': 'http://stix.mitre.org/XMLSchema/extensions/malware/maec_4.1/1.0.1/maec_4.1_malware.xsd',
+ 'http://stix.mitre.org/extensions/StructuredCOA#Generic-1': 'http://stix.mitre.org/XMLSchema/extensions/structured_coa/generic/1.1.1/generic_structured_coa.xsd',
+ 'http://stix.mitre.org/extensions/TestMechanism#Generic-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/generic/1.1.1/generic_test_mechanism.xsd',
+ 'http://stix.mitre.org/extensions/TestMechanism#OVAL5.10-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/oval_5.10/1.1.1/oval_5.10_test_mechanism.xsd',
+ 'http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/open_ioc_2010/1.1.1/open_ioc_2010_test_mechanism.xsd',
+ 'http://stix.mitre.org/extensions/TestMechanism#Snort-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/snort/1.1.1/snort_test_mechanism.xsd',
+ 'http://stix.mitre.org/extensions/TestMechanism#YARA-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/yara/1.1.1/yara_test_mechanism.xsd',
+ 'http://stix.mitre.org/extensions/Vulnerability#CVRF-1': 'http://stix.mitre.org/XMLSchema/extensions/vulnerability/cvrf_1.1/1.1.1/cvrf_1.1_vulnerability.xsd',
+ 'http://stix.mitre.org/stix-1': 'http://stix.mitre.org/XMLSchema/core/1.1.1/stix_core.xsd'}
# Schema locations for namespaces not defined by STIX, but hosted on the STIX website
-EXT_NS_TO_SCHEMALOCATION = {'urn:oasis:names:tc:ciq:xal:3' : 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xAL.xsd',
- 'urn:oasis:names:tc:ciq:xpil:3' : 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xPIL.xsd',
- 'urn:oasis:names:tc:ciq:xnl:3' : 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xNL.xsd'}
+EXT_NS_TO_SCHEMALOCATION = {'urn:oasis:names:tc:ciq:xal:3': 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xAL.xsd',
+ 'urn:oasis:names:tc:ciq:xpil:3': 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xPIL.xsd',
+ 'urn:oasis:names:tc:ciq:xnl:3': 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xNL.xsd'}
# Default namespace->alias mappings. These can be overriden by user-provided dictionaries on export
DEFAULT_STIX_NS_TO_PREFIX = {
- 'http://cybox.mitre.org/common-2' : 'cyboxCommon',
- 'http://cybox.mitre.org/cybox-2' : 'cybox',
- 'http://data-marking.mitre.org/Marking-1' : 'marking',
- 'http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1' : 'simpleMarking',
- 'http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1' : 'tlpMarking',
- 'http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1' : 'TOUMarking',
- 'http://stix.mitre.org/Campaign-1' : 'campaign',
- 'http://stix.mitre.org/CourseOfAction-1' : 'coa',
- 'http://stix.mitre.org/ExploitTarget-1' : 'et',
- 'http://stix.mitre.org/Incident-1' : 'incident',
- 'http://stix.mitre.org/Indicator-2' : 'indicator',
- 'http://stix.mitre.org/TTP-1' : 'ttp',
- 'http://stix.mitre.org/ThreatActor-1' : 'ta',
- 'http://stix.mitre.org/stix-1' : 'stix',
- 'http://stix.mitre.org/common-1' : 'stixCommon',
- 'http://stix.mitre.org/default_vocabularies-1' : 'stixVocabs',
- 'http://stix.mitre.org/extensions/AP#CAPEC2.7-1' : 'stix-capec',
- 'http://stix.mitre.org/extensions/Address#CIQAddress3.0-1' : 'stix-ciqaddress',
- 'http://stix.mitre.org/extensions/Identity#CIQIdentity3.0-1' : 'stix-ciqidentity',
- 'http://stix.mitre.org/extensions/Malware#MAEC4.1-1' : 'stix-maec',
- 'http://stix.mitre.org/extensions/StructuredCOA#Generic-1' : 'genericStructuredCOA',
- 'http://stix.mitre.org/extensions/TestMechanism#Generic-1' : 'genericTM',
- 'http://stix.mitre.org/extensions/TestMechanism#OVAL5.10-1' : 'stix-oval',
- 'http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1' : 'stix-openioc',
- 'http://stix.mitre.org/extensions/TestMechanism#Snort-1' : 'snortTM',
- 'http://stix.mitre.org/extensions/TestMechanism#YARA-1' : 'yaraTM',
- 'http://stix.mitre.org/extensions/Vulnerability#CVRF-1' : 'stix-cvrf'}
+ 'http://cybox.mitre.org/common-2': 'cyboxCommon',
+ 'http://cybox.mitre.org/cybox-2': 'cybox',
+ 'http://data-marking.mitre.org/Marking-1': 'marking',
+ 'http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1': 'simpleMarking',
+ 'http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1': 'tlpMarking',
+ 'http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1': 'TOUMarking',
+ 'http://stix.mitre.org/Campaign-1': 'campaign',
+ 'http://stix.mitre.org/CourseOfAction-1': 'coa',
+ 'http://stix.mitre.org/ExploitTarget-1': 'et',
+ 'http://stix.mitre.org/Incident-1': 'incident',
+ 'http://stix.mitre.org/Indicator-2': 'indicator',
+ 'http://stix.mitre.org/TTP-1': 'ttp',
+ 'http://stix.mitre.org/ThreatActor-1': 'ta',
+ 'http://stix.mitre.org/stix-1': 'stix',
+ 'http://stix.mitre.org/common-1': 'stixCommon',
+ 'http://stix.mitre.org/default_vocabularies-1': 'stixVocabs',
+ 'http://stix.mitre.org/extensions/AP#CAPEC2.7-1': 'stix-capec',
+ 'http://stix.mitre.org/extensions/Address#CIQAddress3.0-1': 'stix-ciqaddress',
+ 'http://stix.mitre.org/extensions/Identity#CIQIdentity3.0-1': 'stix-ciqidentity',
+ 'http://stix.mitre.org/extensions/Malware#MAEC4.1-1': 'stix-maec',
+ 'http://stix.mitre.org/extensions/StructuredCOA#Generic-1': 'genericStructuredCOA',
+ 'http://stix.mitre.org/extensions/TestMechanism#Generic-1': 'genericTM',
+ 'http://stix.mitre.org/extensions/TestMechanism#OVAL5.10-1': 'stix-oval',
+ 'http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1': 'stix-openioc',
+ 'http://stix.mitre.org/extensions/TestMechanism#Snort-1': 'snortTM',
+ 'http://stix.mitre.org/extensions/TestMechanism#YARA-1': 'yaraTM',
+ 'http://stix.mitre.org/extensions/Vulnerability#CVRF-1': 'stix-cvrf'}
DEFAULT_EXT_TO_PREFIX = {
- 'http://capec.mitre.org/capec-2' : 'capec',
- 'http://maec.mitre.org/XMLSchema/maec-package-2' : 'maec',
- 'http://oval.mitre.org/XMLSchema/oval-definitions-5' : 'oval-def',
- 'http://oval.mitre.org/XMLSchema/oval-variables-5' : 'oval-var',
- 'http://schemas.mandiant.com/2010/ioc' : 'ioc',
- 'http://schemas.mandiant.com/2010/ioc/TR/' : 'ioc-tr',
- 'http://www.icasi.org/CVRF/schema/cvrf/1.1' : 'cvrf',
- 'urn:oasis:names:tc:ciq:xal:3' : 'xal',
- 'urn:oasis:names:tc:ciq:xpil:3' : 'xpil',
- 'urn:oasis:names:tc:ciq:xnl:3' : 'xnl'}
+ 'http://capec.mitre.org/capec-2': 'capec',
+ 'http://maec.mitre.org/XMLSchema/maec-package-2': 'maec',
+ 'http://oval.mitre.org/XMLSchema/oval-definitions-5': 'oval-def',
+ 'http://oval.mitre.org/XMLSchema/oval-variables-5': 'oval-var',
+ 'http://schemas.mandiant.com/2010/ioc': 'ioc',
+ 'http://schemas.mandiant.com/2010/ioc/TR/': 'ioc-tr',
+ 'http://www.icasi.org/CVRF/schema/cvrf/1.1': 'cvrf',
+ 'urn:oasis:names:tc:ciq:xal:3': 'xal',
+ 'urn:oasis:names:tc:ciq:xpil:3': 'xpil',
+ 'urn:oasis:names:tc:ciq:xnl:3': 'xnl'}
diff --git a/stix/utils/parser.py b/stix/utils/parser.py
index 22365635..836b6f4e 100644
--- a/stix/utils/parser.py
+++ b/stix/utils/parser.py
@@ -2,6 +2,7 @@
# See LICENSE.txt for complete terms.
import stix
+from itertools import izip
from distutils.version import StrictVersion
from lxml import etree
@@ -56,9 +57,29 @@ def _apply_input_namespaces(self, tree, entity):
root = tree
entity.__input_namespaces__ = {}
- for alias,ns in root.nsmap.iteritems():
+ for alias, ns in root.nsmap.iteritems():
entity.__input_namespaces__[ns] = alias
+ def _apply_input_schemalocations(self, tree, entity):
+ """
+ Attaches an __input_schemalocations__ dictionary to the input entity.
+
+ :param tree: The input etree instance
+ :param entity: The entity to attach the schemlocation dictionary to
+ """
+ try:
+ root = tree.getroot() # is tree an lxml.Element or lxml.ElementTree
+ except AttributeError:
+ root = tree
+
+ schemaloc_attrib_name = \
+ '{http://www.w3.org/2001/XMLSchema-instance}schemaLocation'
+ if root.attrib.get(schemaloc_attrib_name):
+ schemalocs = root.attrib[schemaloc_attrib_name].split()
+ pairs = izip(schemalocs[::2], schemalocs[1::2])
+ entity.__input_schemalocations__ = dict(pairs)
+
+
def parse_xml_to_obj(self, xml_file, check_version=True, check_root=True):
"""Creates a STIX binding object from the supplied xml file.
@@ -108,5 +129,6 @@ def parse_xml(self, xml_file, check_version=True, check_root=True):
from stix.core import STIXPackage # resolve circular dependencies
stix_package = STIXPackage().from_obj(stix_package_obj)
self._apply_input_namespaces(tree, stix_package)
+ self._apply_input_schemalocations(tree, stix_package)
return stix_package