From 259ce48fa13402a8f32df59ace2e21f60f9ac1bc Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 31 Jan 2023 16:54:07 +0100 Subject: [PATCH 01/27] Initial version of getting cloud context from AWS. --- sentry_sdk/integrations/cloudcontext.py | 109 ++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 sentry_sdk/integrations/cloudcontext.py diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py new file mode 100644 index 0000000000..2276d11a68 --- /dev/null +++ b/sentry_sdk/integrations/cloudcontext.py @@ -0,0 +1,109 @@ +import json + +import os +import urllib3 # type: ignore + +from sentry_sdk.integrations import Integration +from sentry_sdk.api import set_context + +from sentry_sdk._types import MYPY + +if MYPY: + from typing import Dict + + +CONTEXT_TYPE = "cloud" + +AWS_METADATA_HOST = "169.254.169.254" +AWS_TOKEN_URL = "http://{}/latest/api/token".format(AWS_METADATA_HOST) +AWS_METADATA_URL = "http://{}/latest/dynamic/instance-identity/document".format( + AWS_METADATA_HOST +) + + +class CLOUD_PROVIDER: + ALIBABA = "alibaba_cloud" + AWS = "aws" + AZURE = "azure" + GCP = "gcp" + IBM = "ibm_cloud" + TENCENT = "tencent_cloud" + + +class CLOUD_PLATFORM: + AWS_EC2 = "aws_ec2" + + +class CloudContextIntegration(Integration): + """ + Adds cloud context to the Senty scope + """ + + identifier = "cloudcontext" + + @classmethod + def _is_aws(cls): + # type: () -> bool + return True if os.environ.get("AWS_DEFAULT_REGION") is not None else False + + @classmethod + def _get_aws_context(cls): + # type: () -> Dict[str, str] + ctx = { + "cloud.provider": CLOUD_PROVIDER.AWS, + "cloud.platform": CLOUD_PLATFORM.AWS_EC2, + "cloud.region": os.environ.get("AWS_REGION", ""), + } + + try: + http = urllib3.PoolManager() + r = http.request( + "PUT", + AWS_TOKEN_URL, + headers={"X-aws-ec2-metadata-token-ttl-seconds": "60"}, + ) + token = r.data + + r = http.request( + "GET", AWS_METADATA_URL, headers={"X-aws-ec2-metadata-token:": token} + ) + if r.status == 200: + data = json.loads(r.data.decode("utf-8")) + ctx.update( + { + "cloud.account.id": data["accountId"], + "cloud.availability_zone": data["availabilityZone"], + } + ) + except Exception: + pass + + return ctx + + @classmethod + def _get_cloud_provider(cls): + # type: () -> str + if cls._is_aws(): + return CLOUD_PROVIDER.AWS + + return "" + + @classmethod + def _get_cloud_context(cls): + # type: () -> Dict[str, str] + getters = { + CLOUD_PROVIDER.AWS: cls._get_aws_context, + } + + cloud_provider = CloudContextIntegration._get_cloud_provider() + if cloud_provider in getters.keys(): + return getters[cloud_provider]() + + return {} + + @staticmethod + def setup_once(): + # type: () -> None + context = CloudContextIntegration._get_cloud_context() + if context != {}: + set_context(CONTEXT_TYPE, context) From 8af586df427bd5efeb590278d25a11c549b7b6ca Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 1 Feb 2023 11:28:36 +0100 Subject: [PATCH 02/27] Fixing guessing if we are in AWS --- sentry_sdk/integrations/cloudcontext.py | 31 ++++++++++++++----------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py index 2276d11a68..7cc2666603 100644 --- a/sentry_sdk/integrations/cloudcontext.py +++ b/sentry_sdk/integrations/cloudcontext.py @@ -1,6 +1,4 @@ import json - -import os import urllib3 # type: ignore from sentry_sdk.integrations import Integration @@ -41,10 +39,22 @@ class CloudContextIntegration(Integration): identifier = "cloudcontext" + aws_token = None + http = urllib3.PoolManager() + @classmethod def _is_aws(cls): # type: () -> bool - return True if os.environ.get("AWS_DEFAULT_REGION") is not None else False + try: + r = cls.http.request( + "PUT", + AWS_TOKEN_URL, + headers={"X-aws-ec2-metadata-token-ttl-seconds": "60"}, + ) + cls.aws_token = r.data + return True + except Exception: + return False @classmethod def _get_aws_context(cls): @@ -52,27 +62,22 @@ def _get_aws_context(cls): ctx = { "cloud.provider": CLOUD_PROVIDER.AWS, "cloud.platform": CLOUD_PLATFORM.AWS_EC2, - "cloud.region": os.environ.get("AWS_REGION", ""), } try: - http = urllib3.PoolManager() - r = http.request( - "PUT", - AWS_TOKEN_URL, - headers={"X-aws-ec2-metadata-token-ttl-seconds": "60"}, + r = cls.http.request( + "GET", + AWS_METADATA_URL, + headers={"X-aws-ec2-metadata-token": cls.aws_token}, ) - token = r.data - r = http.request( - "GET", AWS_METADATA_URL, headers={"X-aws-ec2-metadata-token:": token} - ) if r.status == 200: data = json.loads(r.data.decode("utf-8")) ctx.update( { "cloud.account.id": data["accountId"], "cloud.availability_zone": data["availabilityZone"], + "cloud.region": data["region"], } ) except Exception: From d2f7f8b135e7e9fe85e302c59f067a6f0134b24a Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 6 Feb 2023 15:37:36 +0100 Subject: [PATCH 03/27] Added option for specifying the cloud provider, to prevent autodetection --- sentry_sdk/integrations/cloudcontext.py | 40 ++++++++++++++++++++----- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py index 7cc2666603..43fb0bf94c 100644 --- a/sentry_sdk/integrations/cloudcontext.py +++ b/sentry_sdk/integrations/cloudcontext.py @@ -3,6 +3,7 @@ from sentry_sdk.integrations import Integration from sentry_sdk.api import set_context +from sentry_sdk.utils import logger from sentry_sdk._types import MYPY @@ -39,9 +40,15 @@ class CloudContextIntegration(Integration): identifier = "cloudcontext" - aws_token = None + cloud_provider = "" + + aws_token = "" http = urllib3.PoolManager() + def __init__(self, cloud_provider=""): + # type: (str) -> None + CloudContextIntegration.cloud_provider = cloud_provider + @classmethod def _is_aws(cls): # type: () -> bool @@ -96,19 +103,36 @@ def _get_cloud_provider(cls): @classmethod def _get_cloud_context(cls): # type: () -> Dict[str, str] - getters = { - CLOUD_PROVIDER.AWS: cls._get_aws_context, - } - - cloud_provider = CloudContextIntegration._get_cloud_provider() - if cloud_provider in getters.keys(): - return getters[cloud_provider]() + cloud_provider = ( + cls.cloud_provider + if cls.cloud_provider != "" + else CloudContextIntegration._get_cloud_provider() + ) + if cloud_provider in context_getters.keys(): + return context_getters[cloud_provider]() return {} @staticmethod def setup_once(): # type: () -> None + cloud_provider = CloudContextIntegration.cloud_provider + unsupported_cloud_provider = ( + cloud_provider != "" and cloud_provider not in context_getters.keys() + ) + + if unsupported_cloud_provider: + logger.warning( + "Invalid value for cloud_provider: %s (must be in %s). Falling back to autodetection...", + CloudContextIntegration.cloud_provider, + list(context_getters.keys()), + ) + context = CloudContextIntegration._get_cloud_context() if context != {}: set_context(CONTEXT_TYPE, context) + + +context_getters = { + CLOUD_PROVIDER.AWS: CloudContextIntegration._get_aws_context, +} From fb23b64bef197be027520d568d5df07cbc5f407a Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 6 Feb 2023 16:33:31 +0100 Subject: [PATCH 04/27] Added GCP support --- sentry_sdk/integrations/cloudcontext.py | 60 +++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py index 43fb0bf94c..4163382a1d 100644 --- a/sentry_sdk/integrations/cloudcontext.py +++ b/sentry_sdk/integrations/cloudcontext.py @@ -19,8 +19,18 @@ AWS_METADATA_HOST ) +GCP_METADATA_HOST = "metadata.google.internal" +GCP_METADATA_URL = "http://{}/computeMetadata/v1/?recursive=true".format( + GCP_METADATA_HOST +) + class CLOUD_PROVIDER: + """ + Name of the cloud provider. + see https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud/ + """ + ALIBABA = "alibaba_cloud" AWS = "aws" AZURE = "azure" @@ -30,7 +40,13 @@ class CLOUD_PROVIDER: class CLOUD_PLATFORM: + """ + The cloud platform. + see https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud/ + """ + AWS_EC2 = "aws_ec2" + GCP_COMPUTE_ENGINE = "gcp_compute_engine" class CloudContextIntegration(Integration): @@ -45,6 +61,8 @@ class CloudContextIntegration(Integration): aws_token = "" http = urllib3.PoolManager() + gcp_metadata = {} + def __init__(self, cloud_provider=""): # type: (str) -> None CloudContextIntegration.cloud_provider = cloud_provider @@ -60,6 +78,7 @@ def _is_aws(cls): ) cls.aws_token = r.data return True + except Exception: return False @@ -92,6 +111,46 @@ def _get_aws_context(cls): return ctx + @classmethod + def _is_gcp(cls): + # type: () -> bool + try: + r = cls.http.request( + "GET", + GCP_METADATA_URL, + headers={"Metadata-Flavor": "Google"}, + ) + cls.gcp_metadata = json.loads(r.data.decode("utf-8")) + return True + + except Exception: + return False + + @classmethod + def _get_gcp_context(cls): + # type: () -> Dict[str, str] + ctx = { + "cloud.provider": CLOUD_PROVIDER.GCP, + "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, + } + + try: + ctx["cloud.account.id"] = cls.gcp_metadata["project"]["projectId"] + except Exception: + pass + + try: + ctx["cloud.zone"] = cls.gcp_metadata["instance"]["zone"].split("/")[-1] + except Exception: + pass + + try: + ctx["cloud.region"] = cls.gcp_metadata["instance"]["region"].split("/")[-1] + except Exception: + pass + + return ctx + @classmethod def _get_cloud_provider(cls): # type: () -> str @@ -135,4 +194,5 @@ def setup_once(): context_getters = { CLOUD_PROVIDER.AWS: CloudContextIntegration._get_aws_context, + CLOUD_PROVIDER.GCP: CloudContextIntegration._get_gcp_context, } From 3d64837dca6c602df9262ffb45a0347c0b06ffe7 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 6 Feb 2023 17:46:11 +0100 Subject: [PATCH 05/27] Added some debug output --- sentry_sdk/integrations/cloudcontext.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py index 4163382a1d..bed9c4cf17 100644 --- a/sentry_sdk/integrations/cloudcontext.py +++ b/sentry_sdk/integrations/cloudcontext.py @@ -61,7 +61,7 @@ class CloudContextIntegration(Integration): aws_token = "" http = urllib3.PoolManager() - gcp_metadata = {} + gcp_metadata = None def __init__(self, cloud_provider=""): # type: (str) -> None @@ -120,6 +120,11 @@ def _is_gcp(cls): GCP_METADATA_URL, headers={"Metadata-Flavor": "Google"}, ) + print(r.data) + print("-----------------") + print(r.data.decode("utf-8")) + print("-----------------") + print(json.loads(r.data.decode("utf-8"))) cls.gcp_metadata = json.loads(r.data.decode("utf-8")) return True @@ -134,6 +139,19 @@ def _get_gcp_context(cls): "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, } + if cls.gcp_metadata is None: + r = cls.http.request( + "GET", + GCP_METADATA_URL, + headers={"Metadata-Flavor": "Google"}, + ) + print(r.data) + print("-----------------") + print(r.data.decode("utf-8")) + print("-----------------") + print(json.loads(r.data.decode("utf-8"))) + cls.gcp_metadata = json.loads(r.data.decode("utf-8")) + try: ctx["cloud.account.id"] = cls.gcp_metadata["project"]["projectId"] except Exception: @@ -157,6 +175,9 @@ def _get_cloud_provider(cls): if cls._is_aws(): return CLOUD_PROVIDER.AWS + if cls._is_gcp(): + return CLOUD_PROVIDER.GCP + return "" @classmethod From bb3266484a3fc272cdf6136828616df33ad5049f Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 6 Feb 2023 17:52:16 +0100 Subject: [PATCH 06/27] Better logging --- sentry_sdk/integrations/cloudcontext.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py index bed9c4cf17..6c05e82be5 100644 --- a/sentry_sdk/integrations/cloudcontext.py +++ b/sentry_sdk/integrations/cloudcontext.py @@ -120,11 +120,11 @@ def _is_gcp(cls): GCP_METADATA_URL, headers={"Metadata-Flavor": "Google"}, ) - print(r.data) - print("-----------------") - print(r.data.decode("utf-8")) - print("-----------------") - print(json.loads(r.data.decode("utf-8"))) + logger.warning(r.data) + logger.warning("-----------------") + logger.warning(r.data.decode("utf-8")) + logger.warning("-----------------") + logger.warning(json.loads(r.data.decode("utf-8"))) cls.gcp_metadata = json.loads(r.data.decode("utf-8")) return True @@ -145,11 +145,11 @@ def _get_gcp_context(cls): GCP_METADATA_URL, headers={"Metadata-Flavor": "Google"}, ) - print(r.data) - print("-----------------") - print(r.data.decode("utf-8")) - print("-----------------") - print(json.loads(r.data.decode("utf-8"))) + logger.warning(r.data) + logger.warning("-----------------") + logger.warning(r.data.decode("utf-8")) + logger.warning("-----------------") + logger.warning(json.loads(r.data.decode("utf-8"))) cls.gcp_metadata = json.loads(r.data.decode("utf-8")) try: From f248f1f39b8c1cf3df02cc42aceb636456b880e0 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 7 Feb 2023 09:10:05 +0100 Subject: [PATCH 07/27] Check for correct status response --- sentry_sdk/integrations/cloudcontext.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py index 6c05e82be5..8e76d6a003 100644 --- a/sentry_sdk/integrations/cloudcontext.py +++ b/sentry_sdk/integrations/cloudcontext.py @@ -76,6 +76,10 @@ def _is_aws(cls): AWS_TOKEN_URL, headers={"X-aws-ec2-metadata-token-ttl-seconds": "60"}, ) + + if r.status != 200: + return False + cls.aws_token = r.data return True From 444aa27d11b84a6275f855d816d1d8a5892c5567 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 7 Feb 2023 09:17:58 +0100 Subject: [PATCH 08/27] Cleanup --- sentry_sdk/integrations/cloudcontext.py | 39 +++++++++++++------------ 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py index 8e76d6a003..19c09c86ef 100644 --- a/sentry_sdk/integrations/cloudcontext.py +++ b/sentry_sdk/integrations/cloudcontext.py @@ -101,15 +101,17 @@ def _get_aws_context(cls): headers={"X-aws-ec2-metadata-token": cls.aws_token}, ) - if r.status == 200: - data = json.loads(r.data.decode("utf-8")) - ctx.update( - { - "cloud.account.id": data["accountId"], - "cloud.availability_zone": data["availabilityZone"], - "cloud.region": data["region"], - } - ) + if r.status != 200: + return ctx + + data = json.loads(r.data.decode("utf-8")) + ctx.update( + { + "cloud.account.id": data["accountId"], + "cloud.availability_zone": data["availabilityZone"], + "cloud.region": data["region"], + } + ) except Exception: pass @@ -124,11 +126,10 @@ def _is_gcp(cls): GCP_METADATA_URL, headers={"Metadata-Flavor": "Google"}, ) - logger.warning(r.data) - logger.warning("-----------------") - logger.warning(r.data.decode("utf-8")) - logger.warning("-----------------") - logger.warning(json.loads(r.data.decode("utf-8"))) + + if r.status != 200: + return False + cls.gcp_metadata = json.loads(r.data.decode("utf-8")) return True @@ -149,11 +150,10 @@ def _get_gcp_context(cls): GCP_METADATA_URL, headers={"Metadata-Flavor": "Google"}, ) - logger.warning(r.data) - logger.warning("-----------------") - logger.warning(r.data.decode("utf-8")) - logger.warning("-----------------") - logger.warning(json.loads(r.data.decode("utf-8"))) + + if r.status != 200: + return ctx + cls.gcp_metadata = json.loads(r.data.decode("utf-8")) try: @@ -167,6 +167,7 @@ def _get_gcp_context(cls): pass try: + # only populated in google cloud run ctx["cloud.region"] = cls.gcp_metadata["instance"]["region"].split("/")[-1] except Exception: pass From 1e4748cb0420f4b034fcaca36e3e50ce8050a8a4 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 7 Feb 2023 09:20:17 +0100 Subject: [PATCH 09/27] Some doc strings --- sentry_sdk/integrations/cloudcontext.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py index 19c09c86ef..2846aff322 100644 --- a/sentry_sdk/integrations/cloudcontext.py +++ b/sentry_sdk/integrations/cloudcontext.py @@ -218,6 +218,8 @@ def setup_once(): set_context(CONTEXT_TYPE, context) +# Map with the currently supported cloud providers +# mapping to functions extracting the context context_getters = { CLOUD_PROVIDER.AWS: CloudContextIntegration._get_aws_context, CLOUD_PROVIDER.GCP: CloudContextIntegration._get_gcp_context, From fd948aa5aef5ae479d9827970502ca78f1758c3b Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 7 Feb 2023 09:27:32 +0100 Subject: [PATCH 10/27] Added GCE host id --- sentry_sdk/integrations/cloudcontext.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py index 2846aff322..2d3316da12 100644 --- a/sentry_sdk/integrations/cloudcontext.py +++ b/sentry_sdk/integrations/cloudcontext.py @@ -172,6 +172,11 @@ def _get_gcp_context(cls): except Exception: pass + try: + ctx["host.id"] = cls.gcp_metadata["instance"]["id"] + except Exception: + pass + return ctx @classmethod From 0fece862ea353b1be4c20e30ff297a60cf2e6a10 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 7 Feb 2023 09:40:38 +0100 Subject: [PATCH 11/27] Added more data to AWS context --- sentry_sdk/integrations/cloudcontext.py | 33 +++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py index 2d3316da12..4bfcb8478d 100644 --- a/sentry_sdk/integrations/cloudcontext.py +++ b/sentry_sdk/integrations/cloudcontext.py @@ -105,13 +105,32 @@ def _get_aws_context(cls): return ctx data = json.loads(r.data.decode("utf-8")) - ctx.update( - { - "cloud.account.id": data["accountId"], - "cloud.availability_zone": data["availabilityZone"], - "cloud.region": data["region"], - } - ) + + try: + ctx["cloud.account.id"] = data["accountId"] + except Exception: + pass + + try: + ctx["cloud.availability_zone"] = data["availabilityZone"] + except Exception: + pass + + try: + ctx["cloud.region"] = data["region"] + except Exception: + pass + + try: + ctx["host.id"] = data["instanceId"] + except Exception: + pass + + try: + ctx["host.type"] = data["instanceType"] + except Exception: + pass + except Exception: pass From 0ec6c60ecca7a8f16e2f89a547b67bae896a63ef Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 7 Feb 2023 16:06:38 +0100 Subject: [PATCH 12/27] Fix linting error --- sentry_sdk/integrations/cloudcontext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py index 4bfcb8478d..e4f7bff868 100644 --- a/sentry_sdk/integrations/cloudcontext.py +++ b/sentry_sdk/integrations/cloudcontext.py @@ -25,7 +25,7 @@ ) -class CLOUD_PROVIDER: +class CLOUD_PROVIDER: # noqa: N801 """ Name of the cloud provider. see https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud/ @@ -39,7 +39,7 @@ class CLOUD_PROVIDER: TENCENT = "tencent_cloud" -class CLOUD_PLATFORM: +class CLOUD_PLATFORM: # noqa: N801 """ The cloud platform. see https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud/ From b4bafba623eca178b2585479c0e2aadb5caa0a7c Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 8 Feb 2023 11:27:20 +0100 Subject: [PATCH 13/27] Added tests (and fixed a discovered bug) --- sentry_sdk/integrations/cloudcontext.py | 54 +-- tests/integrations/cloudcontext/__init__.py | 0 .../cloudcontext/test_cloudcontext.py | 340 ++++++++++++++++++ 3 files changed, 370 insertions(+), 24 deletions(-) create mode 100644 tests/integrations/cloudcontext/__init__.py create mode 100644 tests/integrations/cloudcontext/test_cloudcontext.py diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py index e4f7bff868..f9b55d4f18 100644 --- a/sentry_sdk/integrations/cloudcontext.py +++ b/sentry_sdk/integrations/cloudcontext.py @@ -163,36 +163,42 @@ def _get_gcp_context(cls): "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, } - if cls.gcp_metadata is None: - r = cls.http.request( - "GET", - GCP_METADATA_URL, - headers={"Metadata-Flavor": "Google"}, - ) + try: + if cls.gcp_metadata is None: + r = cls.http.request( + "GET", + GCP_METADATA_URL, + headers={"Metadata-Flavor": "Google"}, + ) - if r.status != 200: - return ctx + if r.status != 200: + return ctx - cls.gcp_metadata = json.loads(r.data.decode("utf-8")) + cls.gcp_metadata = json.loads(r.data.decode("utf-8")) - try: - ctx["cloud.account.id"] = cls.gcp_metadata["project"]["projectId"] - except Exception: - pass + try: + ctx["cloud.account.id"] = cls.gcp_metadata["project"]["projectId"] + except Exception: + pass - try: - ctx["cloud.zone"] = cls.gcp_metadata["instance"]["zone"].split("/")[-1] - except Exception: - pass + try: + ctx["cloud.zone"] = cls.gcp_metadata["instance"]["zone"].split("/")[-1] + except Exception: + pass - try: - # only populated in google cloud run - ctx["cloud.region"] = cls.gcp_metadata["instance"]["region"].split("/")[-1] - except Exception: - pass + try: + # only populated in google cloud run + ctx["cloud.region"] = cls.gcp_metadata["instance"]["region"].split("/")[ + -1 + ] + except Exception: + pass + + try: + ctx["host.id"] = cls.gcp_metadata["instance"]["id"] + except Exception: + pass - try: - ctx["host.id"] = cls.gcp_metadata["instance"]["id"] except Exception: pass diff --git a/tests/integrations/cloudcontext/__init__.py b/tests/integrations/cloudcontext/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integrations/cloudcontext/test_cloudcontext.py b/tests/integrations/cloudcontext/test_cloudcontext.py new file mode 100644 index 0000000000..2840d7e8f7 --- /dev/null +++ b/tests/integrations/cloudcontext/test_cloudcontext.py @@ -0,0 +1,340 @@ +import json + +import pytest +import mock +from mock import MagicMock + +from sentry_sdk.integrations.cloudcontext import CLOUD_PLATFORM, CLOUD_PROVIDER + +AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD = { + "accountId": "298817902971", + "architecture": "x86_64", + "availabilityZone": "us-east-1b", + "billingProducts": None, + "devpayProductCodes": None, + "marketplaceProductCodes": None, + "imageId": "ami-00874d747dde344fa", + "instanceId": "i-07d3301297fe0a55a", + "instanceType": "t2.small", + "kernelId": None, + "pendingTime": "2023-02-08T07:54:05Z", + "privateIp": "171.131.65.115", + "ramdiskId": None, + "region": "us-east-1", + "version": "2017-09-30", +} + +GCP_GCE_EXAMPLE_METADATA_PLAYLOAD = { + "instance": { + "attributes": {}, + "cpuPlatform": "Intel Broadwell", + "description": "", + "disks": [ + { + "deviceName": "tests-cloud-contexts-in-python-sdk", + "index": 0, + "interface": "SCSI", + "mode": "READ_WRITE", + "type": "PERSISTENT-BALANCED", + } + ], + "guestAttributes": {}, + "hostname": "tests-cloud-contexts-in-python-sdk.c.client-infra-internal.internal", + "id": 1535324527892303790, + "image": "projects/debian-cloud/global/images/debian-11-bullseye-v20221206", + "licenses": [{"id": "2853224013536823851"}], + "machineType": "projects/542054129475/machineTypes/e2-medium", + "maintenanceEvent": "NONE", + "name": "tests-cloud-contexts-in-python-sdk", + "networkInterfaces": [ + { + "accessConfigs": [ + {"externalIp": "134.30.53.15", "type": "ONE_TO_ONE_NAT"} + ], + "dnsServers": ["169.254.169.254"], + "forwardedIps": [], + "gateway": "10.188.0.1", + "ip": "10.188.0.3", + "ipAliases": [], + "mac": "42:01:0c:7c:00:13", + "mtu": 1460, + "network": "projects/544954029479/networks/default", + "subnetmask": "255.255.240.0", + "targetInstanceIps": [], + } + ], + "preempted": "FALSE", + "remainingCpuTime": -1, + "scheduling": { + "automaticRestart": "TRUE", + "onHostMaintenance": "MIGRATE", + "preemptible": "FALSE", + }, + "serviceAccounts": {}, + "tags": ["http-server", "https-server"], + "virtualClock": {"driftToken": "0"}, + "zone": "projects/142954069479/zones/northamerica-northeast2-b", + }, + "oslogin": {"authenticate": {"sessions": {}}}, + "project": { + "attributes": {}, + "numericProjectId": 204954049439, + "projectId": "my-project-internal", + }, +} + + +def test_is_aws_http_error(): + from sentry_sdk.integrations.cloudcontext import CloudContextIntegration + + response = MagicMock() + response.status = 405 + + CloudContextIntegration.http = MagicMock() + CloudContextIntegration.http.request = MagicMock(return_value=response) + + assert CloudContextIntegration._is_aws() is False + assert CloudContextIntegration.aws_token == "" + + +def test_is_aws_ok(): + from sentry_sdk.integrations.cloudcontext import CloudContextIntegration + + response = MagicMock() + response.status = 200 + response.data = b"something" + CloudContextIntegration.http = MagicMock() + CloudContextIntegration.http.request = MagicMock(return_value=response) + + assert CloudContextIntegration._is_aws() is True + assert CloudContextIntegration.aws_token == b"something" + + CloudContextIntegration.http.request = MagicMock(side_effect=Exception("Test")) + assert CloudContextIntegration._is_aws() is False + + +def test_is_aw_exception(): + from sentry_sdk.integrations.cloudcontext import CloudContextIntegration + + CloudContextIntegration.http = MagicMock() + CloudContextIntegration.http.request = MagicMock(side_effect=Exception("Test")) + + assert CloudContextIntegration._is_aws() is False + + +@pytest.mark.parametrize( + "http_status, response_data, expected_context", + [ + [ + 405, + b"", + { + "cloud.provider": CLOUD_PROVIDER.AWS, + "cloud.platform": CLOUD_PLATFORM.AWS_EC2, + }, + ], + [ + 200, + b"something-but-not-json", + { + "cloud.provider": CLOUD_PROVIDER.AWS, + "cloud.platform": CLOUD_PLATFORM.AWS_EC2, + }, + ], + [ + 200, + bytes(json.dumps(AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD), "utf-8"), + { + "cloud.provider": "aws", + "cloud.platform": "aws_ec2", + "cloud.account.id": "298817902971", + "cloud.availability_zone": "us-east-1b", + "cloud.region": "us-east-1", + "host.id": "i-07d3301297fe0a55a", + "host.type": "t2.small", + }, + ], + ], +) +def test_get_aws_context(http_status, response_data, expected_context): + from sentry_sdk.integrations.cloudcontext import CloudContextIntegration + + response = MagicMock() + response.status = http_status + response.data = response_data + + CloudContextIntegration.http = MagicMock() + CloudContextIntegration.http.request = MagicMock(return_value=response) + + assert CloudContextIntegration._get_aws_context() == expected_context + + +def test_is_gcp_http_error(): + from sentry_sdk.integrations.cloudcontext import CloudContextIntegration + + response = MagicMock() + response.status = 405 + response.data = b'{"some": "json"}' + CloudContextIntegration.http = MagicMock() + CloudContextIntegration.http.request = MagicMock(return_value=response) + + assert CloudContextIntegration._is_gcp() is False + assert CloudContextIntegration.gcp_metadata is None + + +def test_is_gcp_ok(): + from sentry_sdk.integrations.cloudcontext import CloudContextIntegration + + response = MagicMock() + response.status = 200 + response.data = b'{"some": "json"}' + CloudContextIntegration.http = MagicMock() + CloudContextIntegration.http.request = MagicMock(return_value=response) + + assert CloudContextIntegration._is_gcp() is True + assert CloudContextIntegration.gcp_metadata == {"some": "json"} + + +def test_is_gcp_exception(): + from sentry_sdk.integrations.cloudcontext import CloudContextIntegration + + CloudContextIntegration.http = MagicMock() + CloudContextIntegration.http.request = MagicMock(side_effect=Exception("Test")) + assert CloudContextIntegration._is_gcp() is False + + +@pytest.mark.parametrize( + "http_status, response_data, expected_context", + [ + [ + 405, + None, + { + "cloud.provider": CLOUD_PROVIDER.GCP, + "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, + }, + ], + [ + 200, + b"something-but-not-json", + { + "cloud.provider": CLOUD_PROVIDER.GCP, + "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, + }, + ], + [ + 200, + bytes(json.dumps(GCP_GCE_EXAMPLE_METADATA_PLAYLOAD), "utf-8"), + { + "cloud.provider": "gcp", + "cloud.platform": "gcp_compute_engine", + "cloud.account.id": "my-project-internal", + "cloud.zone": "northamerica-northeast2-b", + "host.id": 1535324527892303790, + }, + ], + ], +) +def test_get_gcp_context(http_status, response_data, expected_context): + from sentry_sdk.integrations.cloudcontext import CloudContextIntegration + + CloudContextIntegration.gcp_metadata = None + + response = MagicMock() + response.status = http_status + response.data = response_data + + CloudContextIntegration.http = MagicMock() + CloudContextIntegration.http.request = MagicMock(return_value=response) + + assert CloudContextIntegration._get_gcp_context() == expected_context + + +@pytest.mark.parametrize( + "is_aws, is_gcp, expected_provider", + [ + [False, False, ""], + [False, True, CLOUD_PROVIDER.GCP], + [True, False, CLOUD_PROVIDER.AWS], + [True, True, CLOUD_PROVIDER.AWS], + ], +) +def test_get_cloud_provider(is_aws, is_gcp, expected_provider): + from sentry_sdk.integrations.cloudcontext import CloudContextIntegration + + CloudContextIntegration._is_aws = MagicMock(return_value=is_aws) + CloudContextIntegration._is_gcp = MagicMock(return_value=is_gcp) + + assert CloudContextIntegration._get_cloud_provider() == expected_provider + + +@pytest.mark.parametrize( + "cloud_provider", + [ + CLOUD_PROVIDER.ALIBABA, + CLOUD_PROVIDER.AZURE, + CLOUD_PROVIDER.IBM, + CLOUD_PROVIDER.TENCENT, + ], +) +def test_get_cloud_context_unsupported_providers(cloud_provider): + from sentry_sdk.integrations.cloudcontext import CloudContextIntegration + + CloudContextIntegration._get_cloud_provider = MagicMock(return_value=cloud_provider) + + assert CloudContextIntegration._get_cloud_context() == {} + + +@pytest.mark.parametrize( + "cloud_provider", + [ + CLOUD_PROVIDER.AWS, + CLOUD_PROVIDER.GCP, + ], +) +def test_get_cloud_context_supported_providers(cloud_provider): + from sentry_sdk.integrations.cloudcontext import CloudContextIntegration + + CloudContextIntegration._get_cloud_provider = MagicMock(return_value=cloud_provider) + + assert CloudContextIntegration._get_cloud_context() != {} + + +@pytest.mark.parametrize( + "cloud_provider, cloud_context, warning_called, set_context_called", + [ + ["", {}, False, False], + [CLOUD_PROVIDER.AWS, {}, False, False], + [CLOUD_PROVIDER.GCP, {}, False, False], + [CLOUD_PROVIDER.AZURE, {}, True, False], + [CLOUD_PROVIDER.ALIBABA, {}, True, False], + [CLOUD_PROVIDER.IBM, {}, True, False], + [CLOUD_PROVIDER.TENCENT, {}, True, False], + ["", {"some": "context"}, False, True], + [CLOUD_PROVIDER.AWS, {"some": "context"}, False, True], + [CLOUD_PROVIDER.GCP, {"some": "context"}, False, True], + ], +) +def test_setup_once(cloud_provider, cloud_context, warning_called, set_context_called): + from sentry_sdk.integrations.cloudcontext import CloudContextIntegration + + CloudContextIntegration.cloud_provider = cloud_provider + CloudContextIntegration._get_cloud_context = MagicMock(return_value=cloud_context) + + with mock.patch( + "sentry_sdk.integrations.cloudcontext.set_context" + ) as fake_set_context: + with mock.patch( + "sentry_sdk.integrations.cloudcontext.logger.warning" + ) as fake_warning: + CloudContextIntegration.setup_once() + + if set_context_called: + fake_set_context.assert_called_once_with("cloud", cloud_context) + else: + fake_set_context.assert_not_called() + + if warning_called: + fake_warning.assert_called_once() + else: + fake_warning.assert_not_called() From 8abbdcda4e4e690cd5912664a0240faea855bf28 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 8 Feb 2023 11:47:29 +0100 Subject: [PATCH 14/27] Added cloudcontext to test suite --- .../test-integration-cloudcontext.yml | 73 +++++++++++++++++++ tox.ini | 12 ++- 2 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test-integration-cloudcontext.yml diff --git a/.github/workflows/test-integration-cloudcontext.yml b/.github/workflows/test-integration-cloudcontext.yml new file mode 100644 index 0000000000..58f2f71110 --- /dev/null +++ b/.github/workflows/test-integration-cloudcontext.yml @@ -0,0 +1,73 @@ +name: Test cloudcontext + +on: + push: + branches: + - master + - release/** + + pull_request: + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + BUILD_CACHE_KEY: ${{ github.sha }} + CACHED_BUILD_PATHS: | + ${{ github.workspace }}/dist-serverless + +jobs: + test: + name: cloudcontext, python ${{ matrix.python-version }}, ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + + strategy: + fail-fast: false + matrix: + python-version: ["2.7","3.6","3.7","3.8","3.9","3.10","3.11"] + # python3.6 reached EOL and is no longer being supported on + # new versions of hosted runners on Github Actions + # ubuntu-20.04 is the last version that supported python3.6 + # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 + os: [ubuntu-20.04] + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup Test Env + run: | + pip install codecov "tox>=3,<4" + + - name: Test cloudcontext + timeout-minutes: 45 + shell: bash + run: | + set -x # print commands that are executed + coverage erase + + ./scripts/runtox.sh "${{ matrix.python-version }}-cloudcontext" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch + coverage combine .coverage* + coverage xml -i + codecov --file coverage.xml + + check_required_tests: + name: All cloudcontext tests passed or skipped + needs: test + # Always run this, even if a dependent job failed + if: always() + runs-on: ubuntu-20.04 + steps: + - name: Check for failures + if: contains(needs.test.result, 'failure') + run: | + echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1 diff --git a/tox.ini b/tox.ini index cda2e6ccf6..cc0f00c316 100644 --- a/tox.ini +++ b/tox.ini @@ -49,6 +49,9 @@ envlist = # Chalice {py3.6,py3.7,py3.8}-chalice-v{1.16,1.17,1.18,1.19,1.20} + # CloudContext + {py2.7,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-cloudcontext + # Django # - Django 1.x {py2.7,py3.5}-django-v{1.8,1.9,1.10} @@ -79,9 +82,9 @@ envlist = # HTTPX {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-httpx-v{0.16,0.17} - + # Huey - {py2.7,py3.5,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-huey-2 + {py2.7,py3.5,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-huey-2 # OpenTelemetry (OTel) {py3.7,py3.8,py3.9,py3.10,py3.11}-opentelemetry @@ -264,10 +267,10 @@ deps = # HTTPX httpx-v0.16: httpx>=0.16,<0.17 httpx-v0.17: httpx>=0.17,<0.18 - + # Huey huey-2: huey>=2.0 - + # OpenTelemetry (OTel) opentelemetry: opentelemetry-distro @@ -383,6 +386,7 @@ setenv = bottle: TESTPATH=tests/integrations/bottle celery: TESTPATH=tests/integrations/celery chalice: TESTPATH=tests/integrations/chalice + cloudcontext: TESTPATH=tests/integrations/cloudcontext django: TESTPATH=tests/integrations/django falcon: TESTPATH=tests/integrations/falcon fastapi: TESTPATH=tests/integrations/fastapi From 48e0c098ab19a774efdbf9ed526177bbd9876553 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 8 Feb 2023 12:01:19 +0100 Subject: [PATCH 15/27] Removed Python 2.7 --- .github/workflows/test-integration-cloudcontext.yml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-integration-cloudcontext.yml b/.github/workflows/test-integration-cloudcontext.yml index 58f2f71110..91f8a60621 100644 --- a/.github/workflows/test-integration-cloudcontext.yml +++ b/.github/workflows/test-integration-cloudcontext.yml @@ -31,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["2.7","3.6","3.7","3.8","3.9","3.10","3.11"] + python-version: ["3.6","3.7","3.8","3.9","3.10","3.11"] # python3.6 reached EOL and is no longer being supported on # new versions of hosted runners on Github Actions # ubuntu-20.04 is the last version that supported python3.6 diff --git a/tox.ini b/tox.ini index cc0f00c316..cf1764632f 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,7 @@ envlist = {py3.6,py3.7,py3.8}-chalice-v{1.16,1.17,1.18,1.19,1.20} # CloudContext - {py2.7,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-cloudcontext + {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-cloudcontext # Django # - Django 1.x From f0c92990a2efaaa99c6c37837de412afec13f715 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 8 Feb 2023 17:45:23 +0100 Subject: [PATCH 16/27] Renamed cloud context to cloud_resource context for better description of what it can contain. --- .../test-integration-cloudcontext.yml | 73 ---- sentry_sdk/integrations/cloudcontext.py | 256 ------------- tests/integrations/cloudcontext/__init__.py | 0 .../cloudcontext/test_cloudcontext.py | 340 ------------------ tox.ini | 6 +- 5 files changed, 3 insertions(+), 672 deletions(-) delete mode 100644 .github/workflows/test-integration-cloudcontext.yml delete mode 100644 sentry_sdk/integrations/cloudcontext.py delete mode 100644 tests/integrations/cloudcontext/__init__.py delete mode 100644 tests/integrations/cloudcontext/test_cloudcontext.py diff --git a/.github/workflows/test-integration-cloudcontext.yml b/.github/workflows/test-integration-cloudcontext.yml deleted file mode 100644 index 91f8a60621..0000000000 --- a/.github/workflows/test-integration-cloudcontext.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Test cloudcontext - -on: - push: - branches: - - master - - release/** - - pull_request: - -# Cancel in progress workflows on pull_requests. -# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -env: - BUILD_CACHE_KEY: ${{ github.sha }} - CACHED_BUILD_PATHS: | - ${{ github.workspace }}/dist-serverless - -jobs: - test: - name: cloudcontext, python ${{ matrix.python-version }}, ${{ matrix.os }} - runs-on: ${{ matrix.os }} - timeout-minutes: 45 - - strategy: - fail-fast: false - matrix: - python-version: ["3.6","3.7","3.8","3.9","3.10","3.11"] - # python3.6 reached EOL and is no longer being supported on - # new versions of hosted runners on Github Actions - # ubuntu-20.04 is the last version that supported python3.6 - # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 - os: [ubuntu-20.04] - - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup Test Env - run: | - pip install codecov "tox>=3,<4" - - - name: Test cloudcontext - timeout-minutes: 45 - shell: bash - run: | - set -x # print commands that are executed - coverage erase - - ./scripts/runtox.sh "${{ matrix.python-version }}-cloudcontext" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch - coverage combine .coverage* - coverage xml -i - codecov --file coverage.xml - - check_required_tests: - name: All cloudcontext tests passed or skipped - needs: test - # Always run this, even if a dependent job failed - if: always() - runs-on: ubuntu-20.04 - steps: - - name: Check for failures - if: contains(needs.test.result, 'failure') - run: | - echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1 diff --git a/sentry_sdk/integrations/cloudcontext.py b/sentry_sdk/integrations/cloudcontext.py deleted file mode 100644 index f9b55d4f18..0000000000 --- a/sentry_sdk/integrations/cloudcontext.py +++ /dev/null @@ -1,256 +0,0 @@ -import json -import urllib3 # type: ignore - -from sentry_sdk.integrations import Integration -from sentry_sdk.api import set_context -from sentry_sdk.utils import logger - -from sentry_sdk._types import MYPY - -if MYPY: - from typing import Dict - - -CONTEXT_TYPE = "cloud" - -AWS_METADATA_HOST = "169.254.169.254" -AWS_TOKEN_URL = "http://{}/latest/api/token".format(AWS_METADATA_HOST) -AWS_METADATA_URL = "http://{}/latest/dynamic/instance-identity/document".format( - AWS_METADATA_HOST -) - -GCP_METADATA_HOST = "metadata.google.internal" -GCP_METADATA_URL = "http://{}/computeMetadata/v1/?recursive=true".format( - GCP_METADATA_HOST -) - - -class CLOUD_PROVIDER: # noqa: N801 - """ - Name of the cloud provider. - see https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud/ - """ - - ALIBABA = "alibaba_cloud" - AWS = "aws" - AZURE = "azure" - GCP = "gcp" - IBM = "ibm_cloud" - TENCENT = "tencent_cloud" - - -class CLOUD_PLATFORM: # noqa: N801 - """ - The cloud platform. - see https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud/ - """ - - AWS_EC2 = "aws_ec2" - GCP_COMPUTE_ENGINE = "gcp_compute_engine" - - -class CloudContextIntegration(Integration): - """ - Adds cloud context to the Senty scope - """ - - identifier = "cloudcontext" - - cloud_provider = "" - - aws_token = "" - http = urllib3.PoolManager() - - gcp_metadata = None - - def __init__(self, cloud_provider=""): - # type: (str) -> None - CloudContextIntegration.cloud_provider = cloud_provider - - @classmethod - def _is_aws(cls): - # type: () -> bool - try: - r = cls.http.request( - "PUT", - AWS_TOKEN_URL, - headers={"X-aws-ec2-metadata-token-ttl-seconds": "60"}, - ) - - if r.status != 200: - return False - - cls.aws_token = r.data - return True - - except Exception: - return False - - @classmethod - def _get_aws_context(cls): - # type: () -> Dict[str, str] - ctx = { - "cloud.provider": CLOUD_PROVIDER.AWS, - "cloud.platform": CLOUD_PLATFORM.AWS_EC2, - } - - try: - r = cls.http.request( - "GET", - AWS_METADATA_URL, - headers={"X-aws-ec2-metadata-token": cls.aws_token}, - ) - - if r.status != 200: - return ctx - - data = json.loads(r.data.decode("utf-8")) - - try: - ctx["cloud.account.id"] = data["accountId"] - except Exception: - pass - - try: - ctx["cloud.availability_zone"] = data["availabilityZone"] - except Exception: - pass - - try: - ctx["cloud.region"] = data["region"] - except Exception: - pass - - try: - ctx["host.id"] = data["instanceId"] - except Exception: - pass - - try: - ctx["host.type"] = data["instanceType"] - except Exception: - pass - - except Exception: - pass - - return ctx - - @classmethod - def _is_gcp(cls): - # type: () -> bool - try: - r = cls.http.request( - "GET", - GCP_METADATA_URL, - headers={"Metadata-Flavor": "Google"}, - ) - - if r.status != 200: - return False - - cls.gcp_metadata = json.loads(r.data.decode("utf-8")) - return True - - except Exception: - return False - - @classmethod - def _get_gcp_context(cls): - # type: () -> Dict[str, str] - ctx = { - "cloud.provider": CLOUD_PROVIDER.GCP, - "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, - } - - try: - if cls.gcp_metadata is None: - r = cls.http.request( - "GET", - GCP_METADATA_URL, - headers={"Metadata-Flavor": "Google"}, - ) - - if r.status != 200: - return ctx - - cls.gcp_metadata = json.loads(r.data.decode("utf-8")) - - try: - ctx["cloud.account.id"] = cls.gcp_metadata["project"]["projectId"] - except Exception: - pass - - try: - ctx["cloud.zone"] = cls.gcp_metadata["instance"]["zone"].split("/")[-1] - except Exception: - pass - - try: - # only populated in google cloud run - ctx["cloud.region"] = cls.gcp_metadata["instance"]["region"].split("/")[ - -1 - ] - except Exception: - pass - - try: - ctx["host.id"] = cls.gcp_metadata["instance"]["id"] - except Exception: - pass - - except Exception: - pass - - return ctx - - @classmethod - def _get_cloud_provider(cls): - # type: () -> str - if cls._is_aws(): - return CLOUD_PROVIDER.AWS - - if cls._is_gcp(): - return CLOUD_PROVIDER.GCP - - return "" - - @classmethod - def _get_cloud_context(cls): - # type: () -> Dict[str, str] - cloud_provider = ( - cls.cloud_provider - if cls.cloud_provider != "" - else CloudContextIntegration._get_cloud_provider() - ) - if cloud_provider in context_getters.keys(): - return context_getters[cloud_provider]() - - return {} - - @staticmethod - def setup_once(): - # type: () -> None - cloud_provider = CloudContextIntegration.cloud_provider - unsupported_cloud_provider = ( - cloud_provider != "" and cloud_provider not in context_getters.keys() - ) - - if unsupported_cloud_provider: - logger.warning( - "Invalid value for cloud_provider: %s (must be in %s). Falling back to autodetection...", - CloudContextIntegration.cloud_provider, - list(context_getters.keys()), - ) - - context = CloudContextIntegration._get_cloud_context() - if context != {}: - set_context(CONTEXT_TYPE, context) - - -# Map with the currently supported cloud providers -# mapping to functions extracting the context -context_getters = { - CLOUD_PROVIDER.AWS: CloudContextIntegration._get_aws_context, - CLOUD_PROVIDER.GCP: CloudContextIntegration._get_gcp_context, -} diff --git a/tests/integrations/cloudcontext/__init__.py b/tests/integrations/cloudcontext/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/integrations/cloudcontext/test_cloudcontext.py b/tests/integrations/cloudcontext/test_cloudcontext.py deleted file mode 100644 index 2840d7e8f7..0000000000 --- a/tests/integrations/cloudcontext/test_cloudcontext.py +++ /dev/null @@ -1,340 +0,0 @@ -import json - -import pytest -import mock -from mock import MagicMock - -from sentry_sdk.integrations.cloudcontext import CLOUD_PLATFORM, CLOUD_PROVIDER - -AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD = { - "accountId": "298817902971", - "architecture": "x86_64", - "availabilityZone": "us-east-1b", - "billingProducts": None, - "devpayProductCodes": None, - "marketplaceProductCodes": None, - "imageId": "ami-00874d747dde344fa", - "instanceId": "i-07d3301297fe0a55a", - "instanceType": "t2.small", - "kernelId": None, - "pendingTime": "2023-02-08T07:54:05Z", - "privateIp": "171.131.65.115", - "ramdiskId": None, - "region": "us-east-1", - "version": "2017-09-30", -} - -GCP_GCE_EXAMPLE_METADATA_PLAYLOAD = { - "instance": { - "attributes": {}, - "cpuPlatform": "Intel Broadwell", - "description": "", - "disks": [ - { - "deviceName": "tests-cloud-contexts-in-python-sdk", - "index": 0, - "interface": "SCSI", - "mode": "READ_WRITE", - "type": "PERSISTENT-BALANCED", - } - ], - "guestAttributes": {}, - "hostname": "tests-cloud-contexts-in-python-sdk.c.client-infra-internal.internal", - "id": 1535324527892303790, - "image": "projects/debian-cloud/global/images/debian-11-bullseye-v20221206", - "licenses": [{"id": "2853224013536823851"}], - "machineType": "projects/542054129475/machineTypes/e2-medium", - "maintenanceEvent": "NONE", - "name": "tests-cloud-contexts-in-python-sdk", - "networkInterfaces": [ - { - "accessConfigs": [ - {"externalIp": "134.30.53.15", "type": "ONE_TO_ONE_NAT"} - ], - "dnsServers": ["169.254.169.254"], - "forwardedIps": [], - "gateway": "10.188.0.1", - "ip": "10.188.0.3", - "ipAliases": [], - "mac": "42:01:0c:7c:00:13", - "mtu": 1460, - "network": "projects/544954029479/networks/default", - "subnetmask": "255.255.240.0", - "targetInstanceIps": [], - } - ], - "preempted": "FALSE", - "remainingCpuTime": -1, - "scheduling": { - "automaticRestart": "TRUE", - "onHostMaintenance": "MIGRATE", - "preemptible": "FALSE", - }, - "serviceAccounts": {}, - "tags": ["http-server", "https-server"], - "virtualClock": {"driftToken": "0"}, - "zone": "projects/142954069479/zones/northamerica-northeast2-b", - }, - "oslogin": {"authenticate": {"sessions": {}}}, - "project": { - "attributes": {}, - "numericProjectId": 204954049439, - "projectId": "my-project-internal", - }, -} - - -def test_is_aws_http_error(): - from sentry_sdk.integrations.cloudcontext import CloudContextIntegration - - response = MagicMock() - response.status = 405 - - CloudContextIntegration.http = MagicMock() - CloudContextIntegration.http.request = MagicMock(return_value=response) - - assert CloudContextIntegration._is_aws() is False - assert CloudContextIntegration.aws_token == "" - - -def test_is_aws_ok(): - from sentry_sdk.integrations.cloudcontext import CloudContextIntegration - - response = MagicMock() - response.status = 200 - response.data = b"something" - CloudContextIntegration.http = MagicMock() - CloudContextIntegration.http.request = MagicMock(return_value=response) - - assert CloudContextIntegration._is_aws() is True - assert CloudContextIntegration.aws_token == b"something" - - CloudContextIntegration.http.request = MagicMock(side_effect=Exception("Test")) - assert CloudContextIntegration._is_aws() is False - - -def test_is_aw_exception(): - from sentry_sdk.integrations.cloudcontext import CloudContextIntegration - - CloudContextIntegration.http = MagicMock() - CloudContextIntegration.http.request = MagicMock(side_effect=Exception("Test")) - - assert CloudContextIntegration._is_aws() is False - - -@pytest.mark.parametrize( - "http_status, response_data, expected_context", - [ - [ - 405, - b"", - { - "cloud.provider": CLOUD_PROVIDER.AWS, - "cloud.platform": CLOUD_PLATFORM.AWS_EC2, - }, - ], - [ - 200, - b"something-but-not-json", - { - "cloud.provider": CLOUD_PROVIDER.AWS, - "cloud.platform": CLOUD_PLATFORM.AWS_EC2, - }, - ], - [ - 200, - bytes(json.dumps(AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD), "utf-8"), - { - "cloud.provider": "aws", - "cloud.platform": "aws_ec2", - "cloud.account.id": "298817902971", - "cloud.availability_zone": "us-east-1b", - "cloud.region": "us-east-1", - "host.id": "i-07d3301297fe0a55a", - "host.type": "t2.small", - }, - ], - ], -) -def test_get_aws_context(http_status, response_data, expected_context): - from sentry_sdk.integrations.cloudcontext import CloudContextIntegration - - response = MagicMock() - response.status = http_status - response.data = response_data - - CloudContextIntegration.http = MagicMock() - CloudContextIntegration.http.request = MagicMock(return_value=response) - - assert CloudContextIntegration._get_aws_context() == expected_context - - -def test_is_gcp_http_error(): - from sentry_sdk.integrations.cloudcontext import CloudContextIntegration - - response = MagicMock() - response.status = 405 - response.data = b'{"some": "json"}' - CloudContextIntegration.http = MagicMock() - CloudContextIntegration.http.request = MagicMock(return_value=response) - - assert CloudContextIntegration._is_gcp() is False - assert CloudContextIntegration.gcp_metadata is None - - -def test_is_gcp_ok(): - from sentry_sdk.integrations.cloudcontext import CloudContextIntegration - - response = MagicMock() - response.status = 200 - response.data = b'{"some": "json"}' - CloudContextIntegration.http = MagicMock() - CloudContextIntegration.http.request = MagicMock(return_value=response) - - assert CloudContextIntegration._is_gcp() is True - assert CloudContextIntegration.gcp_metadata == {"some": "json"} - - -def test_is_gcp_exception(): - from sentry_sdk.integrations.cloudcontext import CloudContextIntegration - - CloudContextIntegration.http = MagicMock() - CloudContextIntegration.http.request = MagicMock(side_effect=Exception("Test")) - assert CloudContextIntegration._is_gcp() is False - - -@pytest.mark.parametrize( - "http_status, response_data, expected_context", - [ - [ - 405, - None, - { - "cloud.provider": CLOUD_PROVIDER.GCP, - "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, - }, - ], - [ - 200, - b"something-but-not-json", - { - "cloud.provider": CLOUD_PROVIDER.GCP, - "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, - }, - ], - [ - 200, - bytes(json.dumps(GCP_GCE_EXAMPLE_METADATA_PLAYLOAD), "utf-8"), - { - "cloud.provider": "gcp", - "cloud.platform": "gcp_compute_engine", - "cloud.account.id": "my-project-internal", - "cloud.zone": "northamerica-northeast2-b", - "host.id": 1535324527892303790, - }, - ], - ], -) -def test_get_gcp_context(http_status, response_data, expected_context): - from sentry_sdk.integrations.cloudcontext import CloudContextIntegration - - CloudContextIntegration.gcp_metadata = None - - response = MagicMock() - response.status = http_status - response.data = response_data - - CloudContextIntegration.http = MagicMock() - CloudContextIntegration.http.request = MagicMock(return_value=response) - - assert CloudContextIntegration._get_gcp_context() == expected_context - - -@pytest.mark.parametrize( - "is_aws, is_gcp, expected_provider", - [ - [False, False, ""], - [False, True, CLOUD_PROVIDER.GCP], - [True, False, CLOUD_PROVIDER.AWS], - [True, True, CLOUD_PROVIDER.AWS], - ], -) -def test_get_cloud_provider(is_aws, is_gcp, expected_provider): - from sentry_sdk.integrations.cloudcontext import CloudContextIntegration - - CloudContextIntegration._is_aws = MagicMock(return_value=is_aws) - CloudContextIntegration._is_gcp = MagicMock(return_value=is_gcp) - - assert CloudContextIntegration._get_cloud_provider() == expected_provider - - -@pytest.mark.parametrize( - "cloud_provider", - [ - CLOUD_PROVIDER.ALIBABA, - CLOUD_PROVIDER.AZURE, - CLOUD_PROVIDER.IBM, - CLOUD_PROVIDER.TENCENT, - ], -) -def test_get_cloud_context_unsupported_providers(cloud_provider): - from sentry_sdk.integrations.cloudcontext import CloudContextIntegration - - CloudContextIntegration._get_cloud_provider = MagicMock(return_value=cloud_provider) - - assert CloudContextIntegration._get_cloud_context() == {} - - -@pytest.mark.parametrize( - "cloud_provider", - [ - CLOUD_PROVIDER.AWS, - CLOUD_PROVIDER.GCP, - ], -) -def test_get_cloud_context_supported_providers(cloud_provider): - from sentry_sdk.integrations.cloudcontext import CloudContextIntegration - - CloudContextIntegration._get_cloud_provider = MagicMock(return_value=cloud_provider) - - assert CloudContextIntegration._get_cloud_context() != {} - - -@pytest.mark.parametrize( - "cloud_provider, cloud_context, warning_called, set_context_called", - [ - ["", {}, False, False], - [CLOUD_PROVIDER.AWS, {}, False, False], - [CLOUD_PROVIDER.GCP, {}, False, False], - [CLOUD_PROVIDER.AZURE, {}, True, False], - [CLOUD_PROVIDER.ALIBABA, {}, True, False], - [CLOUD_PROVIDER.IBM, {}, True, False], - [CLOUD_PROVIDER.TENCENT, {}, True, False], - ["", {"some": "context"}, False, True], - [CLOUD_PROVIDER.AWS, {"some": "context"}, False, True], - [CLOUD_PROVIDER.GCP, {"some": "context"}, False, True], - ], -) -def test_setup_once(cloud_provider, cloud_context, warning_called, set_context_called): - from sentry_sdk.integrations.cloudcontext import CloudContextIntegration - - CloudContextIntegration.cloud_provider = cloud_provider - CloudContextIntegration._get_cloud_context = MagicMock(return_value=cloud_context) - - with mock.patch( - "sentry_sdk.integrations.cloudcontext.set_context" - ) as fake_set_context: - with mock.patch( - "sentry_sdk.integrations.cloudcontext.logger.warning" - ) as fake_warning: - CloudContextIntegration.setup_once() - - if set_context_called: - fake_set_context.assert_called_once_with("cloud", cloud_context) - else: - fake_set_context.assert_not_called() - - if warning_called: - fake_warning.assert_called_once() - else: - fake_warning.assert_not_called() diff --git a/tox.ini b/tox.ini index cf1764632f..3d48d82971 100644 --- a/tox.ini +++ b/tox.ini @@ -49,8 +49,8 @@ envlist = # Chalice {py3.6,py3.7,py3.8}-chalice-v{1.16,1.17,1.18,1.19,1.20} - # CloudContext - {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-cloudcontext + # Cloud Resource Context + {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-cloudresourcecontext # Django # - Django 1.x @@ -386,7 +386,7 @@ setenv = bottle: TESTPATH=tests/integrations/bottle celery: TESTPATH=tests/integrations/celery chalice: TESTPATH=tests/integrations/chalice - cloudcontext: TESTPATH=tests/integrations/cloudcontext + cloudresourcecontext: TESTPATH=tests/integrations/cloudresourcecontext django: TESTPATH=tests/integrations/django falcon: TESTPATH=tests/integrations/falcon fastapi: TESTPATH=tests/integrations/fastapi From cd5f89064b8f96c60756674d5c4b077124458586 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 8 Feb 2023 17:51:58 +0100 Subject: [PATCH 17/27] Added missing files --- .../test-integration-cloudresourcecontext.yml | 73 ++++ .../integrations/cloudresourcecontext.py | 256 ++++++++++++ .../cloudresourcecontext/__init__.py | 0 .../test_cloudresourcecontext.py | 380 ++++++++++++++++++ tests/test_handle_in_app_impl.py | 167 ++++++++ ..._impl.test_handle_in_app_impl.approved.txt | 1 + 6 files changed, 877 insertions(+) create mode 100644 .github/workflows/test-integration-cloudresourcecontext.yml create mode 100644 sentry_sdk/integrations/cloudresourcecontext.py create mode 100644 tests/integrations/cloudresourcecontext/__init__.py create mode 100644 tests/integrations/cloudresourcecontext/test_cloudresourcecontext.py create mode 100644 tests/test_handle_in_app_impl.py create mode 100644 tests/test_handle_in_app_impl.test_handle_in_app_impl.approved.txt diff --git a/.github/workflows/test-integration-cloudresourcecontext.yml b/.github/workflows/test-integration-cloudresourcecontext.yml new file mode 100644 index 0000000000..f477a13060 --- /dev/null +++ b/.github/workflows/test-integration-cloudresourcecontext.yml @@ -0,0 +1,73 @@ +name: Test cloudresourcecontext + +on: + push: + branches: + - master + - release/** + + pull_request: + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + BUILD_CACHE_KEY: ${{ github.sha }} + CACHED_BUILD_PATHS: | + ${{ github.workspace }}/dist-serverless + +jobs: + test: + name: cloudresourcecontext, python ${{ matrix.python-version }}, ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + + strategy: + fail-fast: false + matrix: + python-version: ["3.6","3.7","3.8","3.9","3.10","3.11"] + # python3.6 reached EOL and is no longer being supported on + # new versions of hosted runners on Github Actions + # ubuntu-20.04 is the last version that supported python3.6 + # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 + os: [ubuntu-20.04] + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup Test Env + run: | + pip install codecov "tox>=3,<4" + + - name: Test cloudresourcecontext + timeout-minutes: 45 + shell: bash + run: | + set -x # print commands that are executed + coverage erase + + ./scripts/runtox.sh "${{ matrix.python-version }}-cloudresourcecontext" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch + coverage combine .coverage* + coverage xml -i + codecov --file coverage.xml + + check_required_tests: + name: All cloudresourcecontext tests passed or skipped + needs: test + # Always run this, even if a dependent job failed + if: always() + runs-on: ubuntu-20.04 + steps: + - name: Check for failures + if: contains(needs.test.result, 'failure') + run: | + echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1 diff --git a/sentry_sdk/integrations/cloudresourcecontext.py b/sentry_sdk/integrations/cloudresourcecontext.py new file mode 100644 index 0000000000..1c8a640395 --- /dev/null +++ b/sentry_sdk/integrations/cloudresourcecontext.py @@ -0,0 +1,256 @@ +import json +import urllib3 # type: ignore + +from sentry_sdk.integrations import Integration +from sentry_sdk.api import set_context +from sentry_sdk.utils import logger + +from sentry_sdk._types import MYPY + +if MYPY: + from typing import Dict + + +CONTEXT_TYPE = "cloud_resource" + +AWS_METADATA_HOST = "169.254.169.254" +AWS_TOKEN_URL = "http://{}/latest/api/token".format(AWS_METADATA_HOST) +AWS_METADATA_URL = "http://{}/latest/dynamic/instance-identity/document".format( + AWS_METADATA_HOST +) + +GCP_METADATA_HOST = "metadata.google.internal" +GCP_METADATA_URL = "http://{}/computeMetadata/v1/?recursive=true".format( + GCP_METADATA_HOST +) + + +class CLOUD_PROVIDER: # noqa: N801 + """ + Name of the cloud provider. + see https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud/ + """ + + ALIBABA = "alibaba_cloud" + AWS = "aws" + AZURE = "azure" + GCP = "gcp" + IBM = "ibm_cloud" + TENCENT = "tencent_cloud" + + +class CLOUD_PLATFORM: # noqa: N801 + """ + The cloud platform. + see https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud/ + """ + + AWS_EC2 = "aws_ec2" + GCP_COMPUTE_ENGINE = "gcp_compute_engine" + + +class CloudResourceContextIntegration(Integration): + """ + Adds cloud resource context to the Senty scope + """ + + identifier = "cloudresourcecontext" + + cloud_provider = "" + + aws_token = "" + http = urllib3.PoolManager() + + gcp_metadata = None + + def __init__(self, cloud_provider=""): + # type: (str) -> None + CloudResourceContextIntegration.cloud_provider = cloud_provider + + @classmethod + def _is_aws(cls): + # type: () -> bool + try: + r = cls.http.request( + "PUT", + AWS_TOKEN_URL, + headers={"X-aws-ec2-metadata-token-ttl-seconds": "60"}, + ) + + if r.status != 200: + return False + + cls.aws_token = r.data + return True + + except Exception: + return False + + @classmethod + def _get_aws_context(cls): + # type: () -> Dict[str, str] + ctx = { + "cloud.provider": CLOUD_PROVIDER.AWS, + "cloud.platform": CLOUD_PLATFORM.AWS_EC2, + } + + try: + r = cls.http.request( + "GET", + AWS_METADATA_URL, + headers={"X-aws-ec2-metadata-token": cls.aws_token}, + ) + + if r.status != 200: + return ctx + + data = json.loads(r.data.decode("utf-8")) + + try: + ctx["cloud.account.id"] = data["accountId"] + except Exception: + pass + + try: + ctx["cloud.availability_zone"] = data["availabilityZone"] + except Exception: + pass + + try: + ctx["cloud.region"] = data["region"] + except Exception: + pass + + try: + ctx["host.id"] = data["instanceId"] + except Exception: + pass + + try: + ctx["host.type"] = data["instanceType"] + except Exception: + pass + + except Exception: + pass + + return ctx + + @classmethod + def _is_gcp(cls): + # type: () -> bool + try: + r = cls.http.request( + "GET", + GCP_METADATA_URL, + headers={"Metadata-Flavor": "Google"}, + ) + + if r.status != 200: + return False + + cls.gcp_metadata = json.loads(r.data.decode("utf-8")) + return True + + except Exception: + return False + + @classmethod + def _get_gcp_context(cls): + # type: () -> Dict[str, str] + ctx = { + "cloud.provider": CLOUD_PROVIDER.GCP, + "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, + } + + try: + if cls.gcp_metadata is None: + r = cls.http.request( + "GET", + GCP_METADATA_URL, + headers={"Metadata-Flavor": "Google"}, + ) + + if r.status != 200: + return ctx + + cls.gcp_metadata = json.loads(r.data.decode("utf-8")) + + try: + ctx["cloud.account.id"] = cls.gcp_metadata["project"]["projectId"] + except Exception: + pass + + try: + ctx["cloud.zone"] = cls.gcp_metadata["instance"]["zone"].split("/")[-1] + except Exception: + pass + + try: + # only populated in google cloud run + ctx["cloud.region"] = cls.gcp_metadata["instance"]["region"].split("/")[ + -1 + ] + except Exception: + pass + + try: + ctx["host.id"] = cls.gcp_metadata["instance"]["id"] + except Exception: + pass + + except Exception: + pass + + return ctx + + @classmethod + def _get_cloud_provider(cls): + # type: () -> str + if cls._is_aws(): + return CLOUD_PROVIDER.AWS + + if cls._is_gcp(): + return CLOUD_PROVIDER.GCP + + return "" + + @classmethod + def _get_cloud_resource_context(cls): + # type: () -> Dict[str, str] + cloud_provider = ( + cls.cloud_provider + if cls.cloud_provider != "" + else CloudResourceContextIntegration._get_cloud_provider() + ) + if cloud_provider in context_getters.keys(): + return context_getters[cloud_provider]() + + return {} + + @staticmethod + def setup_once(): + # type: () -> None + cloud_provider = CloudResourceContextIntegration.cloud_provider + unsupported_cloud_provider = ( + cloud_provider != "" and cloud_provider not in context_getters.keys() + ) + + if unsupported_cloud_provider: + logger.warning( + "Invalid value for cloud_provider: %s (must be in %s). Falling back to autodetection...", + CloudResourceContextIntegration.cloud_provider, + list(context_getters.keys()), + ) + + context = CloudResourceContextIntegration._get_cloud_resource_context() + if context != {}: + set_context(CONTEXT_TYPE, context) + + +# Map with the currently supported cloud providers +# mapping to functions extracting the context +context_getters = { + CLOUD_PROVIDER.AWS: CloudResourceContextIntegration._get_aws_context, + CLOUD_PROVIDER.GCP: CloudResourceContextIntegration._get_gcp_context, +} diff --git a/tests/integrations/cloudresourcecontext/__init__.py b/tests/integrations/cloudresourcecontext/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integrations/cloudresourcecontext/test_cloudresourcecontext.py b/tests/integrations/cloudresourcecontext/test_cloudresourcecontext.py new file mode 100644 index 0000000000..4400f3496f --- /dev/null +++ b/tests/integrations/cloudresourcecontext/test_cloudresourcecontext.py @@ -0,0 +1,380 @@ +import json + +import pytest +import mock +from mock import MagicMock + +from sentry_sdk.integrations.cloudresourcecontext import CLOUD_PLATFORM, CLOUD_PROVIDER + +AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD = { + "accountId": "298817902971", + "architecture": "x86_64", + "availabilityZone": "us-east-1b", + "billingProducts": None, + "devpayProductCodes": None, + "marketplaceProductCodes": None, + "imageId": "ami-00874d747dde344fa", + "instanceId": "i-07d3301297fe0a55a", + "instanceType": "t2.small", + "kernelId": None, + "pendingTime": "2023-02-08T07:54:05Z", + "privateIp": "171.131.65.115", + "ramdiskId": None, + "region": "us-east-1", + "version": "2017-09-30", +} + +GCP_GCE_EXAMPLE_METADATA_PLAYLOAD = { + "instance": { + "attributes": {}, + "cpuPlatform": "Intel Broadwell", + "description": "", + "disks": [ + { + "deviceName": "tests-cloud-contexts-in-python-sdk", + "index": 0, + "interface": "SCSI", + "mode": "READ_WRITE", + "type": "PERSISTENT-BALANCED", + } + ], + "guestAttributes": {}, + "hostname": "tests-cloud-contexts-in-python-sdk.c.client-infra-internal.internal", + "id": 1535324527892303790, + "image": "projects/debian-cloud/global/images/debian-11-bullseye-v20221206", + "licenses": [{"id": "2853224013536823851"}], + "machineType": "projects/542054129475/machineTypes/e2-medium", + "maintenanceEvent": "NONE", + "name": "tests-cloud-contexts-in-python-sdk", + "networkInterfaces": [ + { + "accessConfigs": [ + {"externalIp": "134.30.53.15", "type": "ONE_TO_ONE_NAT"} + ], + "dnsServers": ["169.254.169.254"], + "forwardedIps": [], + "gateway": "10.188.0.1", + "ip": "10.188.0.3", + "ipAliases": [], + "mac": "42:01:0c:7c:00:13", + "mtu": 1460, + "network": "projects/544954029479/networks/default", + "subnetmask": "255.255.240.0", + "targetInstanceIps": [], + } + ], + "preempted": "FALSE", + "remainingCpuTime": -1, + "scheduling": { + "automaticRestart": "TRUE", + "onHostMaintenance": "MIGRATE", + "preemptible": "FALSE", + }, + "serviceAccounts": {}, + "tags": ["http-server", "https-server"], + "virtualClock": {"driftToken": "0"}, + "zone": "projects/142954069479/zones/northamerica-northeast2-b", + }, + "oslogin": {"authenticate": {"sessions": {}}}, + "project": { + "attributes": {}, + "numericProjectId": 204954049439, + "projectId": "my-project-internal", + }, +} + + +def test_is_aws_http_error(): + from sentry_sdk.integrations.cloudresourcecontext import ( + CloudResourceContextIntegration, + ) + + response = MagicMock() + response.status = 405 + + CloudResourceContextIntegration.http = MagicMock() + CloudResourceContextIntegration.http.request = MagicMock(return_value=response) + + assert CloudResourceContextIntegration._is_aws() is False + assert CloudResourceContextIntegration.aws_token == "" + + +def test_is_aws_ok(): + from sentry_sdk.integrations.cloudresourcecontext import ( + CloudResourceContextIntegration, + ) + + response = MagicMock() + response.status = 200 + response.data = b"something" + CloudResourceContextIntegration.http = MagicMock() + CloudResourceContextIntegration.http.request = MagicMock(return_value=response) + + assert CloudResourceContextIntegration._is_aws() is True + assert CloudResourceContextIntegration.aws_token == b"something" + + CloudResourceContextIntegration.http.request = MagicMock( + side_effect=Exception("Test") + ) + assert CloudResourceContextIntegration._is_aws() is False + + +def test_is_aw_exception(): + from sentry_sdk.integrations.cloudresourcecontext import ( + CloudResourceContextIntegration, + ) + + CloudResourceContextIntegration.http = MagicMock() + CloudResourceContextIntegration.http.request = MagicMock( + side_effect=Exception("Test") + ) + + assert CloudResourceContextIntegration._is_aws() is False + + +@pytest.mark.parametrize( + "http_status, response_data, expected_context", + [ + [ + 405, + b"", + { + "cloud.provider": CLOUD_PROVIDER.AWS, + "cloud.platform": CLOUD_PLATFORM.AWS_EC2, + }, + ], + [ + 200, + b"something-but-not-json", + { + "cloud.provider": CLOUD_PROVIDER.AWS, + "cloud.platform": CLOUD_PLATFORM.AWS_EC2, + }, + ], + [ + 200, + bytes(json.dumps(AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD), "utf-8"), + { + "cloud.provider": "aws", + "cloud.platform": "aws_ec2", + "cloud.account.id": "298817902971", + "cloud.availability_zone": "us-east-1b", + "cloud.region": "us-east-1", + "host.id": "i-07d3301297fe0a55a", + "host.type": "t2.small", + }, + ], + ], +) +def test_get_aws_context(http_status, response_data, expected_context): + from sentry_sdk.integrations.cloudresourcecontext import ( + CloudResourceContextIntegration, + ) + + response = MagicMock() + response.status = http_status + response.data = response_data + + CloudResourceContextIntegration.http = MagicMock() + CloudResourceContextIntegration.http.request = MagicMock(return_value=response) + + assert CloudResourceContextIntegration._get_aws_context() == expected_context + + +def test_is_gcp_http_error(): + from sentry_sdk.integrations.cloudresourcecontext import ( + CloudResourceContextIntegration, + ) + + response = MagicMock() + response.status = 405 + response.data = b'{"some": "json"}' + CloudResourceContextIntegration.http = MagicMock() + CloudResourceContextIntegration.http.request = MagicMock(return_value=response) + + assert CloudResourceContextIntegration._is_gcp() is False + assert CloudResourceContextIntegration.gcp_metadata is None + + +def test_is_gcp_ok(): + from sentry_sdk.integrations.cloudresourcecontext import ( + CloudResourceContextIntegration, + ) + + response = MagicMock() + response.status = 200 + response.data = b'{"some": "json"}' + CloudResourceContextIntegration.http = MagicMock() + CloudResourceContextIntegration.http.request = MagicMock(return_value=response) + + assert CloudResourceContextIntegration._is_gcp() is True + assert CloudResourceContextIntegration.gcp_metadata == {"some": "json"} + + +def test_is_gcp_exception(): + from sentry_sdk.integrations.cloudresourcecontext import ( + CloudResourceContextIntegration, + ) + + CloudResourceContextIntegration.http = MagicMock() + CloudResourceContextIntegration.http.request = MagicMock( + side_effect=Exception("Test") + ) + assert CloudResourceContextIntegration._is_gcp() is False + + +@pytest.mark.parametrize( + "http_status, response_data, expected_context", + [ + [ + 405, + None, + { + "cloud.provider": CLOUD_PROVIDER.GCP, + "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, + }, + ], + [ + 200, + b"something-but-not-json", + { + "cloud.provider": CLOUD_PROVIDER.GCP, + "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, + }, + ], + [ + 200, + bytes(json.dumps(GCP_GCE_EXAMPLE_METADATA_PLAYLOAD), "utf-8"), + { + "cloud.provider": "gcp", + "cloud.platform": "gcp_compute_engine", + "cloud.account.id": "my-project-internal", + "cloud.zone": "northamerica-northeast2-b", + "host.id": 1535324527892303790, + }, + ], + ], +) +def test_get_gcp_context(http_status, response_data, expected_context): + from sentry_sdk.integrations.cloudresourcecontext import ( + CloudResourceContextIntegration, + ) + + CloudResourceContextIntegration.gcp_metadata = None + + response = MagicMock() + response.status = http_status + response.data = response_data + + CloudResourceContextIntegration.http = MagicMock() + CloudResourceContextIntegration.http.request = MagicMock(return_value=response) + + assert CloudResourceContextIntegration._get_gcp_context() == expected_context + + +@pytest.mark.parametrize( + "is_aws, is_gcp, expected_provider", + [ + [False, False, ""], + [False, True, CLOUD_PROVIDER.GCP], + [True, False, CLOUD_PROVIDER.AWS], + [True, True, CLOUD_PROVIDER.AWS], + ], +) +def test_get_cloud_provider(is_aws, is_gcp, expected_provider): + from sentry_sdk.integrations.cloudresourcecontext import ( + CloudResourceContextIntegration, + ) + + CloudResourceContextIntegration._is_aws = MagicMock(return_value=is_aws) + CloudResourceContextIntegration._is_gcp = MagicMock(return_value=is_gcp) + + assert CloudResourceContextIntegration._get_cloud_provider() == expected_provider + + +@pytest.mark.parametrize( + "cloud_provider", + [ + CLOUD_PROVIDER.ALIBABA, + CLOUD_PROVIDER.AZURE, + CLOUD_PROVIDER.IBM, + CLOUD_PROVIDER.TENCENT, + ], +) +def test_get_cloud_resource_context_unsupported_providers(cloud_provider): + from sentry_sdk.integrations.cloudresourcecontext import ( + CloudResourceContextIntegration, + ) + + CloudResourceContextIntegration._get_cloud_provider = MagicMock( + return_value=cloud_provider + ) + + assert CloudResourceContextIntegration._get_cloud_resource_context() == {} + + +@pytest.mark.parametrize( + "cloud_provider", + [ + CLOUD_PROVIDER.AWS, + CLOUD_PROVIDER.GCP, + ], +) +def test_get_cloud_resource_context_supported_providers(cloud_provider): + from sentry_sdk.integrations.cloudresourcecontext import ( + CloudResourceContextIntegration, + ) + + CloudResourceContextIntegration._get_cloud_provider = MagicMock( + return_value=cloud_provider + ) + + assert CloudResourceContextIntegration._get_cloud_resource_context() != {} + + +@pytest.mark.parametrize( + "cloud_provider, cloud_resource_context, warning_called, set_context_called", + [ + ["", {}, False, False], + [CLOUD_PROVIDER.AWS, {}, False, False], + [CLOUD_PROVIDER.GCP, {}, False, False], + [CLOUD_PROVIDER.AZURE, {}, True, False], + [CLOUD_PROVIDER.ALIBABA, {}, True, False], + [CLOUD_PROVIDER.IBM, {}, True, False], + [CLOUD_PROVIDER.TENCENT, {}, True, False], + ["", {"some": "context"}, False, True], + [CLOUD_PROVIDER.AWS, {"some": "context"}, False, True], + [CLOUD_PROVIDER.GCP, {"some": "context"}, False, True], + ], +) +def test_setup_once( + cloud_provider, cloud_resource_context, warning_called, set_context_called +): + from sentry_sdk.integrations.cloudresourcecontext import ( + CloudResourceContextIntegration, + ) + + CloudResourceContextIntegration.cloud_provider = cloud_provider + CloudResourceContextIntegration._get_cloud_resource_context = MagicMock( + return_value=cloud_resource_context + ) + + with mock.patch( + "sentry_sdk.integrations.cloudresourcecontext.set_context" + ) as fake_set_context: + with mock.patch( + "sentry_sdk.integrations.cloudresourcecontext.logger.warning" + ) as fake_warning: + CloudResourceContextIntegration.setup_once() + + if set_context_called: + fake_set_context.assert_called_once_with( + "cloud", cloud_resource_context + ) + else: + fake_set_context.assert_not_called() + + if warning_called: + fake_warning.assert_called_once() + else: + fake_warning.assert_not_called() diff --git a/tests/test_handle_in_app_impl.py b/tests/test_handle_in_app_impl.py new file mode 100644 index 0000000000..70e0e74749 --- /dev/null +++ b/tests/test_handle_in_app_impl.py @@ -0,0 +1,167 @@ +import pytest +from approvaltests.approvals import verify + +from sentry_sdk.utils import handle_in_app_impl + +SAMPLE_FRAMES = [ + { + "function": "__call__", + "filename": "starlette/middleware/exceptions.py", + "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", + "in_app": True, + }, + { + "function": "__call__", + "filename": "fastapi/middleware/asyncexitstack.py", + "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", + "in_app": False, + }, + { + "function": "__call__", + "filename": "fastapi/middleware/asyncexitstack.py", + "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", + "in_app": None, + }, + { + "function": "__call__", + "filename": "starlette/routing.py", + "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/routing.py", + }, + { + "function": "handle", + "module": "starlette.routing", + "filename": "starlette/routing.py", + "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/routing.py", + "in_app": True, + }, + { + "function": "app", + "module": "starlette.routing2", + "filename": "starlette/routing.py", + "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/routing.py", + "in_app": False, + }, + { + "function": "app", + "module": "fastapi.routing3", + "filename": "fastapi/routing.py", + "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/routing.py", + "in_app": None, + }, + { + "function": "run_endpoint_function", + "module": "fastapi.routing4", + "filename": "fastapi/routing.py", + "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/routing.py", + }, + { + "function": "trigger_error", + "module": "something", + "filename": "something.py", + "abs_path": "/home/ubuntu/fastapi/something.py", + "in_app": True, + }, + { + "function": "trigger_error", + "module": "bla", + "filename": "bla.py", + "abs_path": "/home/ubuntu/fastapi/bla/bla.py", + "in_app": False, + }, + { + "function": "trigger_error", + "module": "main", + "filename": "main.py", + "abs_path": "/home/ubuntu/fastapi/main.py", + "in_app": None, + }, + { + "function": "trigger_error", + "module": "blub", + "filename": "blub.py", + "abs_path": "/home/ubuntu/fastapi/something/blub.py", + }, + { + "function": "trigger_error", + "filename": "something.py", + "abs_path": "/home/ubuntu/fastapi/something.py", + "in_app": True, + }, + { + "function": "trigger_error", + "filename": "bla.py", + "abs_path": "/home/ubuntu/fastapi/bla/bla.py", + "in_app": False, + }, + { + "function": "trigger_error", + "filename": "main.py", + "abs_path": "/home/ubuntu/fastapi/main.py", + "in_app": None, + }, + { + "function": "trigger_error", + "filename": "blub.py", + "abs_path": "/home/ubuntu/fastapi/something/blub.py", + }, +] + + +@pytest.mark.parametrize( + "frames, in_app_exclude, in_app_include, default_in_app", + [ + [ + SAMPLE_FRAMES, + [], + [], + False, + ], + [ + SAMPLE_FRAMES, + [], + [], + True, + ], + [ + SAMPLE_FRAMES, + ["starlette.routing", "bla"], + [], + False, + ], + [ + SAMPLE_FRAMES, + ["starlette.routing", "bla"], + [], + False, + ], + [ + SAMPLE_FRAMES, + [], + ["fastapi.routing3", "blub"], + False, + ], + [ + SAMPLE_FRAMES, + [], + ["fastapi.routing3", "blub"], + True, + ], + [ + SAMPLE_FRAMES, + ["starlette.routing", "bla"], + ["fastapi.routing3", "blub"], + False, + ], + [ + SAMPLE_FRAMES, + ["starlette.routing", "bla"], + ["fastapi.routing3", "blub"], + True, + ], + ], +) +def test_handle_in_app_impl(frames, in_app_exclude, in_app_include, default_in_app): + new_frames = handle_in_app_impl( + frames, in_app_exclude, in_app_include, default_in_app + ) + verify(new_frames) diff --git a/tests/test_handle_in_app_impl.test_handle_in_app_impl.approved.txt b/tests/test_handle_in_app_impl.test_handle_in_app_impl.approved.txt new file mode 100644 index 0000000000..c947d783d3 --- /dev/null +++ b/tests/test_handle_in_app_impl.test_handle_in_app_impl.approved.txt @@ -0,0 +1 @@ +[{'function': '__call__', 'filename': 'starlette/middleware/exceptions.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py', 'in_app': True}, {'function': '__call__', 'filename': 'fastapi/middleware/asyncexitstack.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py', 'in_app': False}, {'function': '__call__', 'filename': 'fastapi/middleware/asyncexitstack.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py', 'in_app': None}, {'function': '__call__', 'filename': 'starlette/routing.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/routing.py'}, {'function': 'handle', 'module': 'starlette.routing', 'filename': 'starlette/routing.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/routing.py', 'in_app': True}, {'function': 'app', 'module': 'starlette.routing2', 'filename': 'starlette/routing.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/routing.py', 'in_app': False}, {'function': 'app', 'module': 'fastapi.routing3', 'filename': 'fastapi/routing.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/routing.py', 'in_app': True}, {'function': 'run_endpoint_function', 'module': 'fastapi.routing4', 'filename': 'fastapi/routing.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/routing.py'}, {'function': 'trigger_error', 'module': 'something', 'filename': 'something.py', 'abs_path': '/home/ubuntu/fastapi/something.py', 'in_app': True}, {'function': 'trigger_error', 'module': 'bla', 'filename': 'bla.py', 'abs_path': '/home/ubuntu/fastapi/bla/bla.py', 'in_app': False}, {'function': 'trigger_error', 'module': 'main', 'filename': 'main.py', 'abs_path': '/home/ubuntu/fastapi/main.py', 'in_app': None}, {'function': 'trigger_error', 'module': 'blub', 'filename': 'blub.py', 'abs_path': '/home/ubuntu/fastapi/something/blub.py', 'in_app': True}, {'function': 'trigger_error', 'filename': 'something.py', 'abs_path': '/home/ubuntu/fastapi/something.py', 'in_app': True}, {'function': 'trigger_error', 'filename': 'bla.py', 'abs_path': '/home/ubuntu/fastapi/bla/bla.py', 'in_app': False}, {'function': 'trigger_error', 'filename': 'main.py', 'abs_path': '/home/ubuntu/fastapi/main.py', 'in_app': None}, {'function': 'trigger_error', 'filename': 'blub.py', 'abs_path': '/home/ubuntu/fastapi/something/blub.py'}] From 96dea1391178f6d7662ced9c5535a2f27238277b Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 8 Feb 2023 18:29:52 +0100 Subject: [PATCH 18/27] Removed stuff that should not be there. --- sentry_sdk/utils.py | 19 ++ tests/test_handle_in_app_impl.py | 167 ------------------ ..._impl.test_handle_in_app_impl.approved.txt | 1 - 3 files changed, 19 insertions(+), 168 deletions(-) delete mode 100644 tests/test_handle_in_app_impl.py delete mode 100644 tests/test_handle_in_app_impl.test_handle_in_app_impl.approved.txt diff --git a/sentry_sdk/utils.py b/sentry_sdk/utils.py index 4fd53e927d..11f1d04b80 100644 --- a/sentry_sdk/utils.py +++ b/sentry_sdk/utils.py @@ -779,6 +779,7 @@ def handle_in_app_impl(frames, in_app_exclude, in_app_include, default_in_app=Tr return None any_in_app = False + for frame in frames: in_app = frame.get("in_app") if in_app is not None: @@ -789,9 +790,27 @@ def handle_in_app_impl(frames, in_app_exclude, in_app_include, default_in_app=Tr module = frame.get("module") if not module: continue + + # if frame["abs_path"] contains os.environ.get("VIRTUAL_ENV") then in_app = False + # if frame["abs_path"] matches [\/](?:dist|site)-packages[\/] then in_app = False + + # check in go app for in app: + # if strings.Contains(f.Path, "/site-packages/") || + # strings.Contains(f.Path, "/dist-packages/") || + # strings.Contains(f.Path, "\\site-packages\\") || + # strings.Contains(f.Path, "\\dist-packages\\") { + # return false + # } + + # in app in the sentry backend: + # if ctx["platform"] == "python": + # # This allows detecting a file that belongs to Python's 3rd party modules + # scope.set_tag("stacktrace_link.in_app", "site-packages" not in str(ctx["abs_path"])) + elif _module_in_set(module, in_app_include): frame["in_app"] = True any_in_app = True + elif _module_in_set(module, in_app_exclude): frame["in_app"] = False diff --git a/tests/test_handle_in_app_impl.py b/tests/test_handle_in_app_impl.py deleted file mode 100644 index 70e0e74749..0000000000 --- a/tests/test_handle_in_app_impl.py +++ /dev/null @@ -1,167 +0,0 @@ -import pytest -from approvaltests.approvals import verify - -from sentry_sdk.utils import handle_in_app_impl - -SAMPLE_FRAMES = [ - { - "function": "__call__", - "filename": "starlette/middleware/exceptions.py", - "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", - "in_app": True, - }, - { - "function": "__call__", - "filename": "fastapi/middleware/asyncexitstack.py", - "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", - "in_app": False, - }, - { - "function": "__call__", - "filename": "fastapi/middleware/asyncexitstack.py", - "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", - "in_app": None, - }, - { - "function": "__call__", - "filename": "starlette/routing.py", - "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/routing.py", - }, - { - "function": "handle", - "module": "starlette.routing", - "filename": "starlette/routing.py", - "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/routing.py", - "in_app": True, - }, - { - "function": "app", - "module": "starlette.routing2", - "filename": "starlette/routing.py", - "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/routing.py", - "in_app": False, - }, - { - "function": "app", - "module": "fastapi.routing3", - "filename": "fastapi/routing.py", - "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/routing.py", - "in_app": None, - }, - { - "function": "run_endpoint_function", - "module": "fastapi.routing4", - "filename": "fastapi/routing.py", - "abs_path": "/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/routing.py", - }, - { - "function": "trigger_error", - "module": "something", - "filename": "something.py", - "abs_path": "/home/ubuntu/fastapi/something.py", - "in_app": True, - }, - { - "function": "trigger_error", - "module": "bla", - "filename": "bla.py", - "abs_path": "/home/ubuntu/fastapi/bla/bla.py", - "in_app": False, - }, - { - "function": "trigger_error", - "module": "main", - "filename": "main.py", - "abs_path": "/home/ubuntu/fastapi/main.py", - "in_app": None, - }, - { - "function": "trigger_error", - "module": "blub", - "filename": "blub.py", - "abs_path": "/home/ubuntu/fastapi/something/blub.py", - }, - { - "function": "trigger_error", - "filename": "something.py", - "abs_path": "/home/ubuntu/fastapi/something.py", - "in_app": True, - }, - { - "function": "trigger_error", - "filename": "bla.py", - "abs_path": "/home/ubuntu/fastapi/bla/bla.py", - "in_app": False, - }, - { - "function": "trigger_error", - "filename": "main.py", - "abs_path": "/home/ubuntu/fastapi/main.py", - "in_app": None, - }, - { - "function": "trigger_error", - "filename": "blub.py", - "abs_path": "/home/ubuntu/fastapi/something/blub.py", - }, -] - - -@pytest.mark.parametrize( - "frames, in_app_exclude, in_app_include, default_in_app", - [ - [ - SAMPLE_FRAMES, - [], - [], - False, - ], - [ - SAMPLE_FRAMES, - [], - [], - True, - ], - [ - SAMPLE_FRAMES, - ["starlette.routing", "bla"], - [], - False, - ], - [ - SAMPLE_FRAMES, - ["starlette.routing", "bla"], - [], - False, - ], - [ - SAMPLE_FRAMES, - [], - ["fastapi.routing3", "blub"], - False, - ], - [ - SAMPLE_FRAMES, - [], - ["fastapi.routing3", "blub"], - True, - ], - [ - SAMPLE_FRAMES, - ["starlette.routing", "bla"], - ["fastapi.routing3", "blub"], - False, - ], - [ - SAMPLE_FRAMES, - ["starlette.routing", "bla"], - ["fastapi.routing3", "blub"], - True, - ], - ], -) -def test_handle_in_app_impl(frames, in_app_exclude, in_app_include, default_in_app): - new_frames = handle_in_app_impl( - frames, in_app_exclude, in_app_include, default_in_app - ) - verify(new_frames) diff --git a/tests/test_handle_in_app_impl.test_handle_in_app_impl.approved.txt b/tests/test_handle_in_app_impl.test_handle_in_app_impl.approved.txt deleted file mode 100644 index c947d783d3..0000000000 --- a/tests/test_handle_in_app_impl.test_handle_in_app_impl.approved.txt +++ /dev/null @@ -1 +0,0 @@ -[{'function': '__call__', 'filename': 'starlette/middleware/exceptions.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py', 'in_app': True}, {'function': '__call__', 'filename': 'fastapi/middleware/asyncexitstack.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py', 'in_app': False}, {'function': '__call__', 'filename': 'fastapi/middleware/asyncexitstack.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py', 'in_app': None}, {'function': '__call__', 'filename': 'starlette/routing.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/routing.py'}, {'function': 'handle', 'module': 'starlette.routing', 'filename': 'starlette/routing.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/routing.py', 'in_app': True}, {'function': 'app', 'module': 'starlette.routing2', 'filename': 'starlette/routing.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/starlette/routing.py', 'in_app': False}, {'function': 'app', 'module': 'fastapi.routing3', 'filename': 'fastapi/routing.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/routing.py', 'in_app': True}, {'function': 'run_endpoint_function', 'module': 'fastapi.routing4', 'filename': 'fastapi/routing.py', 'abs_path': '/home/ubuntu/fastapi/.venv/lib/python3.10/site-packages/fastapi/routing.py'}, {'function': 'trigger_error', 'module': 'something', 'filename': 'something.py', 'abs_path': '/home/ubuntu/fastapi/something.py', 'in_app': True}, {'function': 'trigger_error', 'module': 'bla', 'filename': 'bla.py', 'abs_path': '/home/ubuntu/fastapi/bla/bla.py', 'in_app': False}, {'function': 'trigger_error', 'module': 'main', 'filename': 'main.py', 'abs_path': '/home/ubuntu/fastapi/main.py', 'in_app': None}, {'function': 'trigger_error', 'module': 'blub', 'filename': 'blub.py', 'abs_path': '/home/ubuntu/fastapi/something/blub.py', 'in_app': True}, {'function': 'trigger_error', 'filename': 'something.py', 'abs_path': '/home/ubuntu/fastapi/something.py', 'in_app': True}, {'function': 'trigger_error', 'filename': 'bla.py', 'abs_path': '/home/ubuntu/fastapi/bla/bla.py', 'in_app': False}, {'function': 'trigger_error', 'filename': 'main.py', 'abs_path': '/home/ubuntu/fastapi/main.py', 'in_app': None}, {'function': 'trigger_error', 'filename': 'blub.py', 'abs_path': '/home/ubuntu/fastapi/something/blub.py'}] From 845786d6bd87db84a00a1e63cdca44237e3fa559 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 8 Feb 2023 18:31:14 +0100 Subject: [PATCH 19/27] Cleanup --- sentry_sdk/utils.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/sentry_sdk/utils.py b/sentry_sdk/utils.py index 11f1d04b80..5edf834472 100644 --- a/sentry_sdk/utils.py +++ b/sentry_sdk/utils.py @@ -779,7 +779,6 @@ def handle_in_app_impl(frames, in_app_exclude, in_app_include, default_in_app=Tr return None any_in_app = False - for frame in frames: in_app = frame.get("in_app") if in_app is not None: @@ -791,22 +790,6 @@ def handle_in_app_impl(frames, in_app_exclude, in_app_include, default_in_app=Tr if not module: continue - # if frame["abs_path"] contains os.environ.get("VIRTUAL_ENV") then in_app = False - # if frame["abs_path"] matches [\/](?:dist|site)-packages[\/] then in_app = False - - # check in go app for in app: - # if strings.Contains(f.Path, "/site-packages/") || - # strings.Contains(f.Path, "/dist-packages/") || - # strings.Contains(f.Path, "\\site-packages\\") || - # strings.Contains(f.Path, "\\dist-packages\\") { - # return false - # } - - # in app in the sentry backend: - # if ctx["platform"] == "python": - # # This allows detecting a file that belongs to Python's 3rd party modules - # scope.set_tag("stacktrace_link.in_app", "site-packages" not in str(ctx["abs_path"])) - elif _module_in_set(module, in_app_include): frame["in_app"] = True any_in_app = True From f08b9430109e77c6c767b489313e7c57bcbd3852 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 8 Feb 2023 18:32:01 +0100 Subject: [PATCH 20/27] Cleanup --- sentry_sdk/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sentry_sdk/utils.py b/sentry_sdk/utils.py index 5edf834472..4fd53e927d 100644 --- a/sentry_sdk/utils.py +++ b/sentry_sdk/utils.py @@ -789,11 +789,9 @@ def handle_in_app_impl(frames, in_app_exclude, in_app_include, default_in_app=Tr module = frame.get("module") if not module: continue - elif _module_in_set(module, in_app_include): frame["in_app"] = True any_in_app = True - elif _module_in_set(module, in_app_exclude): frame["in_app"] = False From 8157ec2838d27bd346009421f3ba0934b474fd22 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Fri, 10 Feb 2023 14:36:09 +0100 Subject: [PATCH 21/27] Renamed cloudresourceconetxt to could_resource_context --- ...st-integration-cloudre_source_context.yml} | 10 +++--- ...cecontext.py => cloud_resource_context.py} | 0 .../__init__.py | 0 .../test_cloud_resource_context.py} | 33 ++++++++++--------- tox.ini | 4 +-- 5 files changed, 25 insertions(+), 22 deletions(-) rename .github/workflows/{test-integration-cloudresourcecontext.yml => test-integration-cloudre_source_context.yml} (86%) rename sentry_sdk/integrations/{cloudresourcecontext.py => cloud_resource_context.py} (100%) rename tests/integrations/{cloudresourcecontext => cloud_resource_context}/__init__.py (100%) rename tests/integrations/{cloudresourcecontext/test_cloudresourcecontext.py => cloud_resource_context/test_cloud_resource_context.py} (91%) diff --git a/.github/workflows/test-integration-cloudresourcecontext.yml b/.github/workflows/test-integration-cloudre_source_context.yml similarity index 86% rename from .github/workflows/test-integration-cloudresourcecontext.yml rename to .github/workflows/test-integration-cloudre_source_context.yml index f477a13060..ab814191e6 100644 --- a/.github/workflows/test-integration-cloudresourcecontext.yml +++ b/.github/workflows/test-integration-cloudre_source_context.yml @@ -1,4 +1,4 @@ -name: Test cloudresourcecontext +name: Test cloudre_source_context on: push: @@ -24,7 +24,7 @@ env: jobs: test: - name: cloudresourcecontext, python ${{ matrix.python-version }}, ${{ matrix.os }} + name: cloudre_source_context, python ${{ matrix.python-version }}, ${{ matrix.os }} runs-on: ${{ matrix.os }} timeout-minutes: 45 @@ -48,20 +48,20 @@ jobs: run: | pip install codecov "tox>=3,<4" - - name: Test cloudresourcecontext + - name: Test cloudre_source_context timeout-minutes: 45 shell: bash run: | set -x # print commands that are executed coverage erase - ./scripts/runtox.sh "${{ matrix.python-version }}-cloudresourcecontext" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch + ./scripts/runtox.sh "${{ matrix.python-version }}-cloudre_source_context" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch coverage combine .coverage* coverage xml -i codecov --file coverage.xml check_required_tests: - name: All cloudresourcecontext tests passed or skipped + name: All cloudre_source_context tests passed or skipped needs: test # Always run this, even if a dependent job failed if: always() diff --git a/sentry_sdk/integrations/cloudresourcecontext.py b/sentry_sdk/integrations/cloud_resource_context.py similarity index 100% rename from sentry_sdk/integrations/cloudresourcecontext.py rename to sentry_sdk/integrations/cloud_resource_context.py diff --git a/tests/integrations/cloudresourcecontext/__init__.py b/tests/integrations/cloud_resource_context/__init__.py similarity index 100% rename from tests/integrations/cloudresourcecontext/__init__.py rename to tests/integrations/cloud_resource_context/__init__.py diff --git a/tests/integrations/cloudresourcecontext/test_cloudresourcecontext.py b/tests/integrations/cloud_resource_context/test_cloud_resource_context.py similarity index 91% rename from tests/integrations/cloudresourcecontext/test_cloudresourcecontext.py rename to tests/integrations/cloud_resource_context/test_cloud_resource_context.py index 4400f3496f..8deb2603ef 100644 --- a/tests/integrations/cloudresourcecontext/test_cloudresourcecontext.py +++ b/tests/integrations/cloud_resource_context/test_cloud_resource_context.py @@ -4,7 +4,10 @@ import mock from mock import MagicMock -from sentry_sdk.integrations.cloudresourcecontext import CLOUD_PLATFORM, CLOUD_PROVIDER +from sentry_sdk.integrations.cloud_resource_context import ( + CLOUD_PLATFORM, + CLOUD_PROVIDER, +) AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD = { "accountId": "298817902971", @@ -85,7 +88,7 @@ def test_is_aws_http_error(): - from sentry_sdk.integrations.cloudresourcecontext import ( + from sentry_sdk.integrations.cloud_resource_context import ( CloudResourceContextIntegration, ) @@ -100,7 +103,7 @@ def test_is_aws_http_error(): def test_is_aws_ok(): - from sentry_sdk.integrations.cloudresourcecontext import ( + from sentry_sdk.integrations.cloud_resource_context import ( CloudResourceContextIntegration, ) @@ -120,7 +123,7 @@ def test_is_aws_ok(): def test_is_aw_exception(): - from sentry_sdk.integrations.cloudresourcecontext import ( + from sentry_sdk.integrations.cloud_resource_context import ( CloudResourceContextIntegration, ) @@ -167,7 +170,7 @@ def test_is_aw_exception(): ], ) def test_get_aws_context(http_status, response_data, expected_context): - from sentry_sdk.integrations.cloudresourcecontext import ( + from sentry_sdk.integrations.cloud_resource_context import ( CloudResourceContextIntegration, ) @@ -182,7 +185,7 @@ def test_get_aws_context(http_status, response_data, expected_context): def test_is_gcp_http_error(): - from sentry_sdk.integrations.cloudresourcecontext import ( + from sentry_sdk.integrations.cloud_resource_context import ( CloudResourceContextIntegration, ) @@ -197,7 +200,7 @@ def test_is_gcp_http_error(): def test_is_gcp_ok(): - from sentry_sdk.integrations.cloudresourcecontext import ( + from sentry_sdk.integrations.cloud_resource_context import ( CloudResourceContextIntegration, ) @@ -212,7 +215,7 @@ def test_is_gcp_ok(): def test_is_gcp_exception(): - from sentry_sdk.integrations.cloudresourcecontext import ( + from sentry_sdk.integrations.cloud_resource_context import ( CloudResourceContextIntegration, ) @@ -256,7 +259,7 @@ def test_is_gcp_exception(): ], ) def test_get_gcp_context(http_status, response_data, expected_context): - from sentry_sdk.integrations.cloudresourcecontext import ( + from sentry_sdk.integrations.cloud_resource_context import ( CloudResourceContextIntegration, ) @@ -282,7 +285,7 @@ def test_get_gcp_context(http_status, response_data, expected_context): ], ) def test_get_cloud_provider(is_aws, is_gcp, expected_provider): - from sentry_sdk.integrations.cloudresourcecontext import ( + from sentry_sdk.integrations.cloud_resource_context import ( CloudResourceContextIntegration, ) @@ -302,7 +305,7 @@ def test_get_cloud_provider(is_aws, is_gcp, expected_provider): ], ) def test_get_cloud_resource_context_unsupported_providers(cloud_provider): - from sentry_sdk.integrations.cloudresourcecontext import ( + from sentry_sdk.integrations.cloud_resource_context import ( CloudResourceContextIntegration, ) @@ -321,7 +324,7 @@ def test_get_cloud_resource_context_unsupported_providers(cloud_provider): ], ) def test_get_cloud_resource_context_supported_providers(cloud_provider): - from sentry_sdk.integrations.cloudresourcecontext import ( + from sentry_sdk.integrations.cloud_resource_context import ( CloudResourceContextIntegration, ) @@ -350,7 +353,7 @@ def test_get_cloud_resource_context_supported_providers(cloud_provider): def test_setup_once( cloud_provider, cloud_resource_context, warning_called, set_context_called ): - from sentry_sdk.integrations.cloudresourcecontext import ( + from sentry_sdk.integrations.cloud_resource_context import ( CloudResourceContextIntegration, ) @@ -360,10 +363,10 @@ def test_setup_once( ) with mock.patch( - "sentry_sdk.integrations.cloudresourcecontext.set_context" + "sentry_sdk.integrations.cloud_resource_context.set_context" ) as fake_set_context: with mock.patch( - "sentry_sdk.integrations.cloudresourcecontext.logger.warning" + "sentry_sdk.integrations.cloud_resource_context.logger.warning" ) as fake_warning: CloudResourceContextIntegration.setup_once() diff --git a/tox.ini b/tox.ini index 3d48d82971..c9fee96a20 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,7 @@ envlist = {py3.6,py3.7,py3.8}-chalice-v{1.16,1.17,1.18,1.19,1.20} # Cloud Resource Context - {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-cloudresourcecontext + {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-cloudre_source_context # Django # - Django 1.x @@ -386,7 +386,7 @@ setenv = bottle: TESTPATH=tests/integrations/bottle celery: TESTPATH=tests/integrations/celery chalice: TESTPATH=tests/integrations/chalice - cloudresourcecontext: TESTPATH=tests/integrations/cloudresourcecontext + cloud_resource_context: TESTPATH=tests/integrations/cloud_resource_context django: TESTPATH=tests/integrations/django falcon: TESTPATH=tests/integrations/falcon fastapi: TESTPATH=tests/integrations/fastapi From f52de3ce5fd4cfa5e13dca0a08d591f4b672f4fe Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 15 Feb 2023 15:26:44 +0100 Subject: [PATCH 22/27] Do not use separate fields for availability zone of GCP and AWS --- sentry_sdk/integrations/cloud_resource_context.py | 4 +++- .../cloud_resource_context/test_cloud_resource_context.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/cloud_resource_context.py b/sentry_sdk/integrations/cloud_resource_context.py index 1c8a640395..c7b96c35a8 100644 --- a/sentry_sdk/integrations/cloud_resource_context.py +++ b/sentry_sdk/integrations/cloud_resource_context.py @@ -182,7 +182,9 @@ def _get_gcp_context(cls): pass try: - ctx["cloud.zone"] = cls.gcp_metadata["instance"]["zone"].split("/")[-1] + ctx["cloud.availability_zone"] = cls.gcp_metadata["instance"][ + "zone" + ].split("/")[-1] except Exception: pass diff --git a/tests/integrations/cloud_resource_context/test_cloud_resource_context.py b/tests/integrations/cloud_resource_context/test_cloud_resource_context.py index 8deb2603ef..cfdd46940b 100644 --- a/tests/integrations/cloud_resource_context/test_cloud_resource_context.py +++ b/tests/integrations/cloud_resource_context/test_cloud_resource_context.py @@ -252,7 +252,7 @@ def test_is_gcp_exception(): "cloud.provider": "gcp", "cloud.platform": "gcp_compute_engine", "cloud.account.id": "my-project-internal", - "cloud.zone": "northamerica-northeast2-b", + "cloud.availability_zone": "northamerica-northeast2-b", "host.id": 1535324527892303790, }, ], From f3d6bedb964abc65925e453d4769abe0f1a2242d Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 15 Feb 2023 15:39:17 +0100 Subject: [PATCH 23/27] Fixed test after renaming context --- .../cloud_resource_context/test_cloud_resource_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrations/cloud_resource_context/test_cloud_resource_context.py b/tests/integrations/cloud_resource_context/test_cloud_resource_context.py index cfdd46940b..2f75ef560e 100644 --- a/tests/integrations/cloud_resource_context/test_cloud_resource_context.py +++ b/tests/integrations/cloud_resource_context/test_cloud_resource_context.py @@ -372,7 +372,7 @@ def test_setup_once( if set_context_called: fake_set_context.assert_called_once_with( - "cloud", cloud_resource_context + "cloud_resource", cloud_resource_context ) else: fake_set_context.assert_not_called() From 37586c5d461ff39c7d4bb6a833f97a3cbd370455 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 15 Feb 2023 16:40:39 +0100 Subject: [PATCH 24/27] Renamed keys to make ingestion possible --- .../integrations/cloud_resource_context.py | 26 ++++++------ .../test_cloud_resource_context.py | 40 +++++++++---------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/sentry_sdk/integrations/cloud_resource_context.py b/sentry_sdk/integrations/cloud_resource_context.py index c7b96c35a8..67547da051 100644 --- a/sentry_sdk/integrations/cloud_resource_context.py +++ b/sentry_sdk/integrations/cloud_resource_context.py @@ -90,8 +90,8 @@ def _is_aws(cls): def _get_aws_context(cls): # type: () -> Dict[str, str] ctx = { - "cloud.provider": CLOUD_PROVIDER.AWS, - "cloud.platform": CLOUD_PLATFORM.AWS_EC2, + "cloud_provider": CLOUD_PROVIDER.AWS, + "cloud_platform": CLOUD_PLATFORM.AWS_EC2, } try: @@ -107,27 +107,27 @@ def _get_aws_context(cls): data = json.loads(r.data.decode("utf-8")) try: - ctx["cloud.account.id"] = data["accountId"] + ctx["cloud_account_id"] = data["accountId"] except Exception: pass try: - ctx["cloud.availability_zone"] = data["availabilityZone"] + ctx["cloud_availability_zone"] = data["availabilityZone"] except Exception: pass try: - ctx["cloud.region"] = data["region"] + ctx["cloud_region"] = data["region"] except Exception: pass try: - ctx["host.id"] = data["instanceId"] + ctx["host_id"] = data["instanceId"] except Exception: pass try: - ctx["host.type"] = data["instanceType"] + ctx["host_type"] = data["instanceType"] except Exception: pass @@ -159,8 +159,8 @@ def _is_gcp(cls): def _get_gcp_context(cls): # type: () -> Dict[str, str] ctx = { - "cloud.provider": CLOUD_PROVIDER.GCP, - "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, + "cloud_provider": CLOUD_PROVIDER.GCP, + "cloud_platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, } try: @@ -177,12 +177,12 @@ def _get_gcp_context(cls): cls.gcp_metadata = json.loads(r.data.decode("utf-8")) try: - ctx["cloud.account.id"] = cls.gcp_metadata["project"]["projectId"] + ctx["cloud_account_id"] = cls.gcp_metadata["project"]["projectId"] except Exception: pass try: - ctx["cloud.availability_zone"] = cls.gcp_metadata["instance"][ + ctx["cloud_availability_zone"] = cls.gcp_metadata["instance"][ "zone" ].split("/")[-1] except Exception: @@ -190,14 +190,14 @@ def _get_gcp_context(cls): try: # only populated in google cloud run - ctx["cloud.region"] = cls.gcp_metadata["instance"]["region"].split("/")[ + ctx["cloud_region"] = cls.gcp_metadata["instance"]["region"].split("/")[ -1 ] except Exception: pass try: - ctx["host.id"] = cls.gcp_metadata["instance"]["id"] + ctx["host_id"] = cls.gcp_metadata["instance"]["id"] except Exception: pass diff --git a/tests/integrations/cloud_resource_context/test_cloud_resource_context.py b/tests/integrations/cloud_resource_context/test_cloud_resource_context.py index 2f75ef560e..494be501ee 100644 --- a/tests/integrations/cloud_resource_context/test_cloud_resource_context.py +++ b/tests/integrations/cloud_resource_context/test_cloud_resource_context.py @@ -142,29 +142,29 @@ def test_is_aw_exception(): 405, b"", { - "cloud.provider": CLOUD_PROVIDER.AWS, - "cloud.platform": CLOUD_PLATFORM.AWS_EC2, + "cloud_provider": CLOUD_PROVIDER.AWS, + "cloud_platform": CLOUD_PLATFORM.AWS_EC2, }, ], [ 200, b"something-but-not-json", { - "cloud.provider": CLOUD_PROVIDER.AWS, - "cloud.platform": CLOUD_PLATFORM.AWS_EC2, + "cloud_provider": CLOUD_PROVIDER.AWS, + "cloud_platform": CLOUD_PLATFORM.AWS_EC2, }, ], [ 200, bytes(json.dumps(AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD), "utf-8"), { - "cloud.provider": "aws", - "cloud.platform": "aws_ec2", - "cloud.account.id": "298817902971", - "cloud.availability_zone": "us-east-1b", - "cloud.region": "us-east-1", - "host.id": "i-07d3301297fe0a55a", - "host.type": "t2.small", + "cloud_provider": "aws", + "cloud_platform": "aws_ec2", + "cloud_account_id": "298817902971", + "cloud_availability_zone": "us-east-1b", + "cloud_region": "us-east-1", + "host_id": "i-07d3301297fe0a55a", + "host_type": "t2.small", }, ], ], @@ -233,27 +233,27 @@ def test_is_gcp_exception(): 405, None, { - "cloud.provider": CLOUD_PROVIDER.GCP, - "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, + "cloud_provider": CLOUD_PROVIDER.GCP, + "cloud_platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, }, ], [ 200, b"something-but-not-json", { - "cloud.provider": CLOUD_PROVIDER.GCP, - "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, + "cloud_provider": CLOUD_PROVIDER.GCP, + "cloud_platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, }, ], [ 200, bytes(json.dumps(GCP_GCE_EXAMPLE_METADATA_PLAYLOAD), "utf-8"), { - "cloud.provider": "gcp", - "cloud.platform": "gcp_compute_engine", - "cloud.account.id": "my-project-internal", - "cloud.availability_zone": "northamerica-northeast2-b", - "host.id": 1535324527892303790, + "cloud_provider": "gcp", + "cloud_platform": "gcp_compute_engine", + "cloud_account_id": "my-project-internal", + "cloud_availability_zone": "northamerica-northeast2-b", + "host_id": 1535324527892303790, }, ], ], From 610f918f518ca82f75245323a1563af9d51ffe2e Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 16 Feb 2023 08:15:16 +0100 Subject: [PATCH 25/27] Revert "Renamed keys to make ingestion possible" This reverts commit 37586c5d461ff39c7d4bb6a833f97a3cbd370455. --- .../integrations/cloud_resource_context.py | 26 ++++++------ .../test_cloud_resource_context.py | 40 +++++++++---------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/sentry_sdk/integrations/cloud_resource_context.py b/sentry_sdk/integrations/cloud_resource_context.py index 67547da051..c7b96c35a8 100644 --- a/sentry_sdk/integrations/cloud_resource_context.py +++ b/sentry_sdk/integrations/cloud_resource_context.py @@ -90,8 +90,8 @@ def _is_aws(cls): def _get_aws_context(cls): # type: () -> Dict[str, str] ctx = { - "cloud_provider": CLOUD_PROVIDER.AWS, - "cloud_platform": CLOUD_PLATFORM.AWS_EC2, + "cloud.provider": CLOUD_PROVIDER.AWS, + "cloud.platform": CLOUD_PLATFORM.AWS_EC2, } try: @@ -107,27 +107,27 @@ def _get_aws_context(cls): data = json.loads(r.data.decode("utf-8")) try: - ctx["cloud_account_id"] = data["accountId"] + ctx["cloud.account.id"] = data["accountId"] except Exception: pass try: - ctx["cloud_availability_zone"] = data["availabilityZone"] + ctx["cloud.availability_zone"] = data["availabilityZone"] except Exception: pass try: - ctx["cloud_region"] = data["region"] + ctx["cloud.region"] = data["region"] except Exception: pass try: - ctx["host_id"] = data["instanceId"] + ctx["host.id"] = data["instanceId"] except Exception: pass try: - ctx["host_type"] = data["instanceType"] + ctx["host.type"] = data["instanceType"] except Exception: pass @@ -159,8 +159,8 @@ def _is_gcp(cls): def _get_gcp_context(cls): # type: () -> Dict[str, str] ctx = { - "cloud_provider": CLOUD_PROVIDER.GCP, - "cloud_platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, + "cloud.provider": CLOUD_PROVIDER.GCP, + "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, } try: @@ -177,12 +177,12 @@ def _get_gcp_context(cls): cls.gcp_metadata = json.loads(r.data.decode("utf-8")) try: - ctx["cloud_account_id"] = cls.gcp_metadata["project"]["projectId"] + ctx["cloud.account.id"] = cls.gcp_metadata["project"]["projectId"] except Exception: pass try: - ctx["cloud_availability_zone"] = cls.gcp_metadata["instance"][ + ctx["cloud.availability_zone"] = cls.gcp_metadata["instance"][ "zone" ].split("/")[-1] except Exception: @@ -190,14 +190,14 @@ def _get_gcp_context(cls): try: # only populated in google cloud run - ctx["cloud_region"] = cls.gcp_metadata["instance"]["region"].split("/")[ + ctx["cloud.region"] = cls.gcp_metadata["instance"]["region"].split("/")[ -1 ] except Exception: pass try: - ctx["host_id"] = cls.gcp_metadata["instance"]["id"] + ctx["host.id"] = cls.gcp_metadata["instance"]["id"] except Exception: pass diff --git a/tests/integrations/cloud_resource_context/test_cloud_resource_context.py b/tests/integrations/cloud_resource_context/test_cloud_resource_context.py index 494be501ee..2f75ef560e 100644 --- a/tests/integrations/cloud_resource_context/test_cloud_resource_context.py +++ b/tests/integrations/cloud_resource_context/test_cloud_resource_context.py @@ -142,29 +142,29 @@ def test_is_aw_exception(): 405, b"", { - "cloud_provider": CLOUD_PROVIDER.AWS, - "cloud_platform": CLOUD_PLATFORM.AWS_EC2, + "cloud.provider": CLOUD_PROVIDER.AWS, + "cloud.platform": CLOUD_PLATFORM.AWS_EC2, }, ], [ 200, b"something-but-not-json", { - "cloud_provider": CLOUD_PROVIDER.AWS, - "cloud_platform": CLOUD_PLATFORM.AWS_EC2, + "cloud.provider": CLOUD_PROVIDER.AWS, + "cloud.platform": CLOUD_PLATFORM.AWS_EC2, }, ], [ 200, bytes(json.dumps(AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD), "utf-8"), { - "cloud_provider": "aws", - "cloud_platform": "aws_ec2", - "cloud_account_id": "298817902971", - "cloud_availability_zone": "us-east-1b", - "cloud_region": "us-east-1", - "host_id": "i-07d3301297fe0a55a", - "host_type": "t2.small", + "cloud.provider": "aws", + "cloud.platform": "aws_ec2", + "cloud.account.id": "298817902971", + "cloud.availability_zone": "us-east-1b", + "cloud.region": "us-east-1", + "host.id": "i-07d3301297fe0a55a", + "host.type": "t2.small", }, ], ], @@ -233,27 +233,27 @@ def test_is_gcp_exception(): 405, None, { - "cloud_provider": CLOUD_PROVIDER.GCP, - "cloud_platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, + "cloud.provider": CLOUD_PROVIDER.GCP, + "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, }, ], [ 200, b"something-but-not-json", { - "cloud_provider": CLOUD_PROVIDER.GCP, - "cloud_platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, + "cloud.provider": CLOUD_PROVIDER.GCP, + "cloud.platform": CLOUD_PLATFORM.GCP_COMPUTE_ENGINE, }, ], [ 200, bytes(json.dumps(GCP_GCE_EXAMPLE_METADATA_PLAYLOAD), "utf-8"), { - "cloud_provider": "gcp", - "cloud_platform": "gcp_compute_engine", - "cloud_account_id": "my-project-internal", - "cloud_availability_zone": "northamerica-northeast2-b", - "host_id": 1535324527892303790, + "cloud.provider": "gcp", + "cloud.platform": "gcp_compute_engine", + "cloud.account.id": "my-project-internal", + "cloud.availability_zone": "northamerica-northeast2-b", + "host.id": 1535324527892303790, }, ], ], From f6d0489976caf5125112932fda778bcdbbc250ef Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 16 Feb 2023 09:05:32 +0100 Subject: [PATCH 26/27] Fixed typo --- ...yml => test-integration-cloud_resource_context.yml} | 10 +++++----- tox.ini | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) rename .github/workflows/{test-integration-cloudre_source_context.yml => test-integration-cloud_resource_context.yml} (87%) diff --git a/.github/workflows/test-integration-cloudre_source_context.yml b/.github/workflows/test-integration-cloud_resource_context.yml similarity index 87% rename from .github/workflows/test-integration-cloudre_source_context.yml rename to .github/workflows/test-integration-cloud_resource_context.yml index ab814191e6..d4e2a25be8 100644 --- a/.github/workflows/test-integration-cloudre_source_context.yml +++ b/.github/workflows/test-integration-cloud_resource_context.yml @@ -1,4 +1,4 @@ -name: Test cloudre_source_context +name: Test cloud_resource_context on: push: @@ -24,7 +24,7 @@ env: jobs: test: - name: cloudre_source_context, python ${{ matrix.python-version }}, ${{ matrix.os }} + name: cloud_resource_context, python ${{ matrix.python-version }}, ${{ matrix.os }} runs-on: ${{ matrix.os }} timeout-minutes: 45 @@ -48,20 +48,20 @@ jobs: run: | pip install codecov "tox>=3,<4" - - name: Test cloudre_source_context + - name: Test cloud_resource_context timeout-minutes: 45 shell: bash run: | set -x # print commands that are executed coverage erase - ./scripts/runtox.sh "${{ matrix.python-version }}-cloudre_source_context" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch + ./scripts/runtox.sh "${{ matrix.python-version }}-cloud_resource_context" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch coverage combine .coverage* coverage xml -i codecov --file coverage.xml check_required_tests: - name: All cloudre_source_context tests passed or skipped + name: All cloud_resource_context tests passed or skipped needs: test # Always run this, even if a dependent job failed if: always() diff --git a/tox.ini b/tox.ini index c9fee96a20..c4fb834910 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,7 @@ envlist = {py3.6,py3.7,py3.8}-chalice-v{1.16,1.17,1.18,1.19,1.20} # Cloud Resource Context - {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-cloudre_source_context + {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-cloud_resource_context # Django # - Django 1.x From fc2d051279b0776c86c9fcdd253620aaa3512fb7 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 22 Feb 2023 15:47:48 +0100 Subject: [PATCH 27/27] Made it support Python 2 --- .../test_cloud_resource_context.py | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/integrations/cloud_resource_context/test_cloud_resource_context.py b/tests/integrations/cloud_resource_context/test_cloud_resource_context.py index 2f75ef560e..b1efd97f3f 100644 --- a/tests/integrations/cloud_resource_context/test_cloud_resource_context.py +++ b/tests/integrations/cloud_resource_context/test_cloud_resource_context.py @@ -27,6 +27,17 @@ "version": "2017-09-30", } +try: + # Python 3 + AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD_BYTES = bytes( + json.dumps(AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD), "utf-8" + ) +except TypeError: + # Python 2 + AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD_BYTES = bytes( + json.dumps(AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD) + ).encode("utf-8") + GCP_GCE_EXAMPLE_METADATA_PLAYLOAD = { "instance": { "attributes": {}, @@ -86,6 +97,17 @@ }, } +try: + # Python 3 + GCP_GCE_EXAMPLE_METADATA_PLAYLOAD_BYTES = bytes( + json.dumps(GCP_GCE_EXAMPLE_METADATA_PLAYLOAD), "utf-8" + ) +except TypeError: + # Python 2 + GCP_GCE_EXAMPLE_METADATA_PLAYLOAD_BYTES = bytes( + json.dumps(GCP_GCE_EXAMPLE_METADATA_PLAYLOAD) + ).encode("utf-8") + def test_is_aws_http_error(): from sentry_sdk.integrations.cloud_resource_context import ( @@ -156,7 +178,7 @@ def test_is_aw_exception(): ], [ 200, - bytes(json.dumps(AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD), "utf-8"), + AWS_EC2_EXAMPLE_IMDSv2_PAYLOAD_BYTES, { "cloud.provider": "aws", "cloud.platform": "aws_ec2", @@ -247,7 +269,7 @@ def test_is_gcp_exception(): ], [ 200, - bytes(json.dumps(GCP_GCE_EXAMPLE_METADATA_PLAYLOAD), "utf-8"), + GCP_GCE_EXAMPLE_METADATA_PLAYLOAD_BYTES, { "cloud.provider": "gcp", "cloud.platform": "gcp_compute_engine",