From f6c0ccbb9d3049919bb3a426c50e0a875ddf8094 Mon Sep 17 00:00:00 2001 From: clavedeluna Date: Mon, 19 Aug 2024 09:37:27 -0300 Subject: [PATCH 1/5] add core codemod --- src/core_codemods/__init__.py | 2 + src/core_codemods/timezone_aware_datetime.py | 116 ++++++++++++++++++ .../codemods/test_timezone_aware_datetime.py | 77 ++++++++++++ 3 files changed, 195 insertions(+) create mode 100644 src/core_codemods/timezone_aware_datetime.py create mode 100644 tests/codemods/test_timezone_aware_datetime.py diff --git a/src/core_codemods/__init__.py b/src/core_codemods/__init__.py index ac717cb6..1155e821 100644 --- a/src/core_codemods/__init__.py +++ b/src/core_codemods/__init__.py @@ -95,6 +95,7 @@ from .str_concat_in_seq_literal import StrConcatInSeqLiteral from .subprocess_shell_false import SubprocessShellFalse from .tempfile_mktemp import TempfileMktemp +from .timezone_aware_datetime import TimezoneAwareDatetime from .upgrade_sslcontext_minimum_version import UpgradeSSLContextMinimumVersion from .upgrade_sslcontext_tls import UpgradeSSLContextTLS from .url_sandbox import UrlSandbox @@ -136,6 +137,7 @@ UseDefusedXml, UseGenerator, UseSetLiteral, + TimezoneAwareDatetime, UseWalrusIf, WithThreadingLock, SQLQueryParameterization, diff --git a/src/core_codemods/timezone_aware_datetime.py b/src/core_codemods/timezone_aware_datetime.py new file mode 100644 index 00000000..d3d84592 --- /dev/null +++ b/src/core_codemods/timezone_aware_datetime.py @@ -0,0 +1,116 @@ +import libcst as cst +from libcst import matchers + +from codemodder.codemods.libcst_transformer import ( + LibcstResultTransformer, + LibcstTransformerPipeline, + NewArg, +) +from codemodder.codemods.utils_mixin import NameResolutionMixin +from core_codemods.api import CoreCodemod, Metadata, Reference, ReviewGuidance + + +class TransformDatetimeWithTimezone(LibcstResultTransformer, NameResolutionMixin): + + change_description = "Add `tz=datetime.timezone.utc` to datetime call" + need_kwarg = ( + "datetime.datetime", + "datetime.datetime.now", + "datetime.datetime.fromtimestamp", + ) + _module_name = "datetime" + + def leave_Call(self, original_node: cst.Call, updated_node: cst.Call): + if not self.node_is_selected(original_node): + return updated_node + + match self.find_base_name(original_node): + case "datetime.datetime.utcnow": + self.report_change(original_node) + maybe_name, kwarg_val, module = self._determine_module_and_kwarg( + original_node + ) + new_args = self.replace_args( + original_node, + [ + NewArg( + name="tz", + value=kwarg_val, + add_if_missing=True, + ) + ], + ) + return self.update_call_target( + updated_node, module, "now", replacement_args=new_args + ) + case "datetime.datetime.utcfromtimestamp": + self.report_change(original_node) + maybe_name, kwarg_val, module = self._determine_module_and_kwarg( + original_node + ) + if len(original_node.args) != 2 and not self._has_timezone_arg( + original_node, "tz" + ): + new_args = self.replace_args( + original_node, + [ + NewArg( + name="tz", + value=kwarg_val, + add_if_missing=True, + ) + ], + ) + else: + new_args = original_node.args + + return self.update_call_target( + updated_node, + module, + "fromtimestamp", + replacement_args=new_args, + ) + + return updated_node + + def _determine_module_and_kwarg(self, original_node: cst.Call): + maybe_name = self.get_aliased_prefix_name(original_node, self._module_name) + + if maybe_name: + # it's a regular import OR alias import + if maybe_name == self._module_name: + module = "datetime.datetime" + else: + module = f"{maybe_name}.datetime" + kwarg_val = f"{maybe_name}.timezone.utc" + else: + # it's from import so timezone should also be from import + self.add_needed_import("datetime", "timezone") + kwarg_val = "timezone.utc" + module = ( + "datetime" + if (curr_module := original_node.func.value.value) + in (self._module_name, "date") + else curr_module + ) + + return maybe_name, kwarg_val, module + + def _has_timezone_arg(self, original_node: cst.Call, name: str) -> bool: + return any( + matchers.matches(arg, matchers.Arg(keyword=matchers.Name(name))) + for arg in original_node.args + ) + + +TimezoneAwareDatetime = CoreCodemod( + metadata=Metadata( + name="timezone-aware-datetime", + summary="Make `datetime` Calls Timezone-Aware", + review_guidance=ReviewGuidance.MERGE_AFTER_REVIEW, + references=[ + Reference(url="https://docs.python.org/3/library/datetime.html"), + ], + ), + transformer=LibcstTransformerPipeline(TransformDatetimeWithTimezone), +) diff --git a/tests/codemods/test_timezone_aware_datetime.py b/tests/codemods/test_timezone_aware_datetime.py new file mode 100644 index 00000000..7e4d48df --- /dev/null +++ b/tests/codemods/test_timezone_aware_datetime.py @@ -0,0 +1,77 @@ +from codemodder.codemods.test import BaseCodemodTest +from core_codemods.timezone_aware_datetime import TimezoneAwareDatetime + + +class TestTimezoneAwareDatetimeNeedKwarg(BaseCodemodTest): + codemod = TimezoneAwareDatetime + + def test_name(self): + assert self.codemod.name == "timezone-aware-datetime" + + def test_import(self, tmpdir): + input_code = """ + import datetime + import time + + datetime.datetime.utcnow() + datetime.datetime.utcfromtimestamp(time.time()) + """ + expected = """ + import datetime + import time + + datetime.datetime.now(tz=datetime.timezone.utc) + datetime.datetime.fromtimestamp(time.time(), tz=datetime.timezone.utc) + """ + self.run_and_assert(tmpdir, input_code, expected, num_changes=2) + + def test_import_alias(self, tmpdir): + input_code = """ + import datetime as mydate + import time + + mydate.datetime.utcnow() + mydate.datetime.utcfromtimestamp(time.time()) + """ + expected = """ + import datetime as mydate + import time + + mydate.datetime.now(tz=mydate.timezone.utc) + mydate.datetime.fromtimestamp(time.time(), tz=mydate.timezone.utc) + """ + self.run_and_assert(tmpdir, input_code, expected, num_changes=2) + + def test_import_from(self, tmpdir): + input_code = """ + from datetime import datetime + import time + + datetime.utcnow() + datetime.utcfromtimestamp(time.time()) + """ + expected = """ + from datetime import timezone, datetime + import time + + datetime.now(tz=timezone.utc) + datetime.fromtimestamp(time.time(), tz=timezone.utc) + """ + self.run_and_assert(tmpdir, input_code, expected, num_changes=2) + + def test_import_from_alias(self, tmpdir): + input_code = """ + from datetime import datetime as mydate + import time + + mydate.utcnow() + mydate.utcfromtimestamp(time.time()) + """ + expected = """ + from datetime import timezone, datetime as mydate + import time + + mydate.now(tz=timezone.utc) + mydate.fromtimestamp(time.time(), tz=timezone.utc) + """ + self.run_and_assert(tmpdir, input_code, expected, num_changes=2) From 1a86c7501559105d575879b0dccafae5fcd304d9 Mon Sep 17 00:00:00 2001 From: clavedeluna Date: Mon, 19 Aug 2024 09:39:39 -0300 Subject: [PATCH 2/5] document --- src/codemodder/scripts/generate_docs.py | 4 ++++ .../docs/pixee_python_timezone-aware-datetime.md | 14 ++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 src/core_codemods/docs/pixee_python_timezone-aware-datetime.md diff --git a/src/codemodder/scripts/generate_docs.py b/src/codemodder/scripts/generate_docs.py index 10c574a9..6867eb78 100644 --- a/src/codemodder/scripts/generate_docs.py +++ b/src/codemodder/scripts/generate_docs.py @@ -291,6 +291,10 @@ class DocMetadata: guidance_explained="This change is generally safe and will prevent deserialization vulnerabilities.", need_sarif="Yes (DefectDojo)", ), + "timezone-aware-datetime": DocMetadata( + importance="Medium", + guidance_explained="This change makes your code more accurate with regards to timezones. However, it's possible you wish to specify a different timezone for your application needs.", + ), } SONAR_CODEMOD_NAMES = [ diff --git a/src/core_codemods/docs/pixee_python_timezone-aware-datetime.md b/src/core_codemods/docs/pixee_python_timezone-aware-datetime.md new file mode 100644 index 00000000..9e20b3bc --- /dev/null +++ b/src/core_codemods/docs/pixee_python_timezone-aware-datetime.md @@ -0,0 +1,14 @@ +Some `datetime` object calls use the machine's local timezone instead of a reasonable default like UTC. This may be okay in some cases, but it can lead to bugs. Misinterpretation of dates have been the culprit for serious issues in banking, satellite communications, and other industries. + +The `datetime` [documentation](https://docs.python.org/3/library/datetime.html#datetime.datetime.utcnow) explicitly encourages using timezone aware objects to prevent bugs. + +Our changes look like the following: +```diff + from datetime import datetime + import time + +- datetime.utcnow() +- datetime.utcfromtimestamp(time.time()) ++ datetime.now(tz=timezone.utc) ++ datetime.fromtimestamp(time.time(), tz=timezone.utc) +``` From ccc7f753d137200907b1b2e92ddd16f09ac0b664 Mon Sep 17 00:00:00 2001 From: clavedeluna Date: Mon, 19 Aug 2024 10:01:06 -0300 Subject: [PATCH 3/5] add sonar codemod --- src/codemodder/scripts/generate_docs.py | 12 ++-- src/core_codemods/__init__.py | 2 + .../sonar/sonar_timezone_aware_datetime.py | 9 +++ .../test_sonar_timezone_aware_datetime.py | 64 +++++++++++++++++++ 4 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 src/core_codemods/sonar/sonar_timezone_aware_datetime.py create mode 100644 tests/codemods/sonar/test_sonar_timezone_aware_datetime.py diff --git a/src/codemodder/scripts/generate_docs.py b/src/codemodder/scripts/generate_docs.py index 6867eb78..d039e2bc 100644 --- a/src/codemodder/scripts/generate_docs.py +++ b/src/codemodder/scripts/generate_docs.py @@ -279,6 +279,10 @@ class DocMetadata: importance="Low", guidance_explained="We believe this replacement is safe and should not result in any issues.", ), + "timezone-aware-datetime": DocMetadata( + importance="Medium", + guidance_explained="This change makes your code more accurate with regards to timezones. However, it's possible you wish to specify a different timezone for your application needs.", + ), } DEFECTDOJO_CODEMODS = { "django-secure-set-cookie": DocMetadata( @@ -291,10 +295,6 @@ class DocMetadata: guidance_explained="This change is generally safe and will prevent deserialization vulnerabilities.", need_sarif="Yes (DefectDojo)", ), - "timezone-aware-datetime": DocMetadata( - importance="Medium", - guidance_explained="This change makes your code more accurate with regards to timezones. However, it's possible you wish to specify a different timezone for your application needs.", - ), } SONAR_CODEMOD_NAMES = [ @@ -318,7 +318,11 @@ class DocMetadata: "django-model-without-dunder-str", "break-or-continue-out-of-loop", "disable-graphql-introspection", +<<<<<<< HEAD "invert-boolean-check", +======= + "timezone-aware-datetime", +>>>>>>> 0e9e1643 (add sonar codemod) ] SONAR_CODEMODS = { name: DocMetadata( diff --git a/src/core_codemods/__init__.py b/src/core_codemods/__init__.py index 1155e821..1abf97b7 100644 --- a/src/core_codemods/__init__.py +++ b/src/core_codemods/__init__.py @@ -90,6 +90,7 @@ from .sonar.sonar_secure_random import SonarSecureRandom from .sonar.sonar_sql_parameterization import SonarSQLParameterization from .sonar.sonar_tempfile_mktemp import SonarTempfileMktemp +from .sonar.sonar_timezone_aware_datetime import SonarTimezoneAwareDatetime from .sonar.sonar_url_sandbox import SonarUrlSandbox from .sql_parameterization import SQLQueryParameterization from .str_concat_in_seq_literal import StrConcatInSeqLiteral @@ -199,6 +200,7 @@ SonarBreakOrContinueOutOfLoop, SonarDisableGraphQLIntrospection, SonarInvertedBooleanCheck, + SonarTimezoneAwareDatetime, ], ) diff --git a/src/core_codemods/sonar/sonar_timezone_aware_datetime.py b/src/core_codemods/sonar/sonar_timezone_aware_datetime.py new file mode 100644 index 00000000..f900e252 --- /dev/null +++ b/src/core_codemods/sonar/sonar_timezone_aware_datetime.py @@ -0,0 +1,9 @@ +from core_codemods.sonar.api import SonarCodemod +from core_codemods.timezone_aware_datetime import TimezoneAwareDatetime + +SonarTimezoneAwareDatetime = SonarCodemod.from_core_codemod( + name="timezone-aware-datetime", + other=TimezoneAwareDatetime, + rule_id="python:S6903", + rule_name='Using timezone-aware "datetime" objects should be preferred over using "datetime.datetime.utcnow" and "datetime.datetime.utcfromtimestamp"', +) diff --git a/tests/codemods/sonar/test_sonar_timezone_aware_datetime.py b/tests/codemods/sonar/test_sonar_timezone_aware_datetime.py new file mode 100644 index 00000000..f93e4ed7 --- /dev/null +++ b/tests/codemods/sonar/test_sonar_timezone_aware_datetime.py @@ -0,0 +1,64 @@ +import json + +from codemodder.codemods.test import BaseSASTCodemodTest +from core_codemods.sonar.sonar_timezone_aware_datetime import SonarTimezoneAwareDatetime + + +class TestSonarSQLParameterization(BaseSASTCodemodTest): + codemod = SonarTimezoneAwareDatetime + tool = "sonar" + + def test_name(self): + assert self.codemod.name == "timezone-aware-datetime" + + def test_simple(self, tmpdir): + input_code = """\ + import datetime + + datetime.datetime.utcnow() + timestamp = 1571595618.0 + datetime.datetime.utcfromtimestamp(timestamp) + """ + expected = """\ + import datetime + + datetime.datetime.now(tz=datetime.timezone.utc) + timestamp = 1571595618.0 + datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc) + """ + issues = { + "issues": [ + { + "key": "AZFcGzHT5VEY3NanjlD7", + "rule": "python:S6903", + "severity": "MAJOR", + "component": "code.py", + "hash": "92aca3da1e08f944a3c408df27c54b28", + "textRange": { + "startLine": 3, + "endLine": 3, + "startOffset": 0, + "endOffset": 26, + }, + "status": "OPEN", + "message": "Don't use `datetime.datetime.utcnow` to create this datetime object.", + }, + { + "key": "AZFcGzHT5VEY3NanjlD8", + "rule": "python:S6903", + "severity": "MAJOR", + "component": "code.py", + "textRange": { + "startLine": 5, + "endLine": 5, + "startOffset": 0, + "endOffset": 45, + }, + "status": "OPEN", + "message": "Don't use `datetime.datetime.utcfromtimestamp` to create this datetime object.", + }, + ] + } + self.run_and_assert( + tmpdir, input_code, expected, results=json.dumps(issues), num_changes=2 + ) From 88cf78422b2504d9e42cc833ac61ecbcd7368edb Mon Sep 17 00:00:00 2001 From: clavedeluna Date: Mon, 19 Aug 2024 11:02:38 -0300 Subject: [PATCH 4/5] update pygoat findings --- ci_tests/test_pygoat_findings.py | 1 + src/codemodder/scripts/generate_docs.py | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ci_tests/test_pygoat_findings.py b/ci_tests/test_pygoat_findings.py index 7602758f..ea3c6dfb 100644 --- a/ci_tests/test_pygoat_findings.py +++ b/ci_tests/test_pygoat_findings.py @@ -14,6 +14,7 @@ "pixee:python/url-sandbox", "pixee:python/use-defusedxml", "pixee:python/use-walrus-if", + "pixee:python/timezone-aware-datetime", ] diff --git a/src/codemodder/scripts/generate_docs.py b/src/codemodder/scripts/generate_docs.py index d039e2bc..42e03503 100644 --- a/src/codemodder/scripts/generate_docs.py +++ b/src/codemodder/scripts/generate_docs.py @@ -318,11 +318,8 @@ class DocMetadata: "django-model-without-dunder-str", "break-or-continue-out-of-loop", "disable-graphql-introspection", -<<<<<<< HEAD "invert-boolean-check", -======= "timezone-aware-datetime", ->>>>>>> 0e9e1643 (add sonar codemod) ] SONAR_CODEMODS = { name: DocMetadata( From e6da90f7623d7cf572e2e0c8c9701f4c30c87968 Mon Sep 17 00:00:00 2001 From: "pixeebot[bot]" <104101892+pixeebot[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 14:32:55 -0300 Subject: [PATCH 5/5] Hardening suggestions for codemodder-python / sonar-timezone (#818) * update pygoat findings * Use Assignment Expression (Walrus) In Conditional --------- Co-authored-by: clavedeluna Co-authored-by: pixeebot[bot] <104101892+pixeebot[bot]@users.noreply.github.com> --- src/core_codemods/timezone_aware_datetime.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core_codemods/timezone_aware_datetime.py b/src/core_codemods/timezone_aware_datetime.py index d3d84592..bca4103a 100644 --- a/src/core_codemods/timezone_aware_datetime.py +++ b/src/core_codemods/timezone_aware_datetime.py @@ -74,9 +74,8 @@ def leave_Call(self, original_node: cst.Call, updated_node: cst.Call): return updated_node def _determine_module_and_kwarg(self, original_node: cst.Call): - maybe_name = self.get_aliased_prefix_name(original_node, self._module_name) - if maybe_name: + if maybe_name := self.get_aliased_prefix_name(original_node, self._module_name): # it's a regular import OR alias import if maybe_name == self._module_name: module = "datetime.datetime"