From b416dd0ed7d183004914903804cff1f7f2f76dd4 Mon Sep 17 00:00:00 2001 From: tighwm Date: Wed, 3 Sep 2025 14:52:29 +0300 Subject: [PATCH 1/7] added recursive_filtering_body --- vcr/filters.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/vcr/filters.py b/vcr/filters.py index 7f33155e..26266490 100644 --- a/vcr/filters.py +++ b/vcr/filters.py @@ -69,6 +69,21 @@ def remove_query_parameters(request, query_parameters_to_remove): return replace_query_parameters(request, replacements) +def recursive_filtering_body(request, body_data, replacements): + for k, ov in list(body_data.items()): + if isinstance(ov, dict): + recursive_filtering_body(request, ov, replacements) + if not ov: + body_data.pop(k) + if k in replacements: + body_data.pop(k) + rv = replacements[k] + if callable(rv): + rv = rv(key=k, value=ov, request=request) + if rv is not None: + body_data[k] = rv + + def replace_post_data_parameters(request, replacements): """Replace post data in request--either form data or json--according to replacements. @@ -86,23 +101,11 @@ def replace_post_data_parameters(request, replacements): if request.method == "POST" and not isinstance(request.body, BytesIO): if isinstance(request.body, dict): new_body = request.body.copy() - for k, rv in replacements.items(): - if k in new_body: - ov = new_body.pop(k) - if callable(rv): - rv = rv(key=k, value=ov, request=request) - if rv is not None: - new_body[k] = rv + recursive_filtering_body(request, new_body, replacements) request.body = new_body elif request.headers.get("Content-Type") == "application/json": json_data = json.loads(request.body) - for k, rv in replacements.items(): - if k in json_data: - ov = json_data.pop(k) - if callable(rv): - rv = rv(key=k, value=ov, request=request) - if rv is not None: - json_data[k] = rv + recursive_filtering_body(request, json_data, replacements) request.body = json.dumps(json_data).encode("utf-8") else: if isinstance(request.body, str): From 7c34ffe3947593b2c47558a64b1baa82afc6b659 Mon Sep 17 00:00:00 2001 From: tighwm Date: Wed, 3 Sep 2025 14:54:48 +0300 Subject: [PATCH 2/7] added unit test with nested request body --- tests/unit/test_filters.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/test_filters.py b/tests/unit/test_filters.py index 8849e737..1767270a 100644 --- a/tests/unit/test_filters.py +++ b/tests/unit/test_filters.py @@ -202,6 +202,22 @@ def test_replace_json_post_data_parameters(): assert request_data == expected_data +def test_replace_nested_post_data_parameters(): + body = b'{"one": {"key": "secret", "nested": "change"}, "two": "keep", "three": {"key": "secret"}}' + request = Request("POST", "http://google.com", body, {}) + request.headers["Content-Type"] = "application/json" + replace_post_data_parameters( + request, + [ + ("key", None), + ("nested", "aboba") + ], + ) + request_data = json.loads(request.body) + expected_data = json.loads('{"one": {"nested": "aboba"}, "two": "keep"}') + assert request_data == expected_data + + def test_remove_json_post_data_parameters(): # Test the backward-compatible API wrapper. body = b'{"id": "secret", "foo": "bar", "baz": "qux"}' From 19f5c24f0dec1ea9dab313a66e74d1fa0f8a1401 Mon Sep 17 00:00:00 2001 From: tighwm Date: Wed, 3 Sep 2025 16:00:07 +0300 Subject: [PATCH 3/7] fixed key error --- tests/unit/test_filters.py | 7 ++++--- vcr/filters.py | 15 ++++++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_filters.py b/tests/unit/test_filters.py index 1767270a..8d688083 100644 --- a/tests/unit/test_filters.py +++ b/tests/unit/test_filters.py @@ -203,18 +203,19 @@ def test_replace_json_post_data_parameters(): def test_replace_nested_post_data_parameters(): - body = b'{"one": {"key": "secret", "nested": "change"}, "two": "keep", "three": {"key": "secret"}}' + body = b'{"nested": "same", "another": "same", "one": {"key": "secret", "nested": {"key": "secret"}}}' request = Request("POST", "http://google.com", body, {}) request.headers["Content-Type"] = "application/json" replace_post_data_parameters( request, [ ("key", None), - ("nested", "aboba") + ("nested", "aboba"), + ("another", None) ], ) request_data = json.loads(request.body) - expected_data = json.loads('{"one": {"nested": "aboba"}, "two": "keep"}') + expected_data = json.loads('{"nested": "aboba", "one": {"nested": "aboba"}}') assert request_data == expected_data diff --git a/vcr/filters.py b/vcr/filters.py index 26266490..f7d4771e 100644 --- a/vcr/filters.py +++ b/vcr/filters.py @@ -70,18 +70,19 @@ def remove_query_parameters(request, query_parameters_to_remove): def recursive_filtering_body(request, body_data, replacements): + filtered_v = [] for k, ov in list(body_data.items()): - if isinstance(ov, dict): - recursive_filtering_body(request, ov, replacements) - if not ov: - body_data.pop(k) if k in replacements: - body_data.pop(k) + filtered_v.append(body_data.pop(k)) rv = replacements[k] if callable(rv): rv = rv(key=k, value=ov, request=request) if rv is not None: body_data[k] = rv + if isinstance(ov, dict) and ov not in filtered_v: + recursive_filtering_body(request, ov, replacements) + if not ov: + body_data.pop(k) def replace_post_data_parameters(request, replacements): @@ -105,7 +106,11 @@ def replace_post_data_parameters(request, replacements): request.body = new_body elif request.headers.get("Content-Type") == "application/json": json_data = json.loads(request.body) + print() + print(json_data) + print(replacements) recursive_filtering_body(request, json_data, replacements) + print(json_data) request.body = json.dumps(json_data).encode("utf-8") else: if isinstance(request.body, str): From 233c2aaf77a552df9331d63273f3a4100ef293f6 Mon Sep 17 00:00:00 2001 From: tighwm Date: Wed, 3 Sep 2025 16:25:36 +0300 Subject: [PATCH 4/7] excluded useless list --- vcr/filters.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vcr/filters.py b/vcr/filters.py index f7d4771e..89952bc8 100644 --- a/vcr/filters.py +++ b/vcr/filters.py @@ -70,18 +70,18 @@ def remove_query_parameters(request, query_parameters_to_remove): def recursive_filtering_body(request, body_data, replacements): - filtered_v = [] for k, ov in list(body_data.items()): + if isinstance(ov, dict): + recursive_filtering_body(request, ov, replacements) + if not ov: + body_data.pop(k) if k in replacements: - filtered_v.append(body_data.pop(k)) rv = replacements[k] if callable(rv): rv = rv(key=k, value=ov, request=request) if rv is not None: body_data[k] = rv - if isinstance(ov, dict) and ov not in filtered_v: - recursive_filtering_body(request, ov, replacements) - if not ov: + else: body_data.pop(k) From 0fba86a4e74aa075879a06fb8fb93daa59f797aa Mon Sep 17 00:00:00 2001 From: tighwm Date: Wed, 3 Sep 2025 16:28:34 +0300 Subject: [PATCH 5/7] my bad... --- vcr/filters.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/vcr/filters.py b/vcr/filters.py index 89952bc8..5e8498aa 100644 --- a/vcr/filters.py +++ b/vcr/filters.py @@ -106,11 +106,7 @@ def replace_post_data_parameters(request, replacements): request.body = new_body elif request.headers.get("Content-Type") == "application/json": json_data = json.loads(request.body) - print() - print(json_data) - print(replacements) recursive_filtering_body(request, json_data, replacements) - print(json_data) request.body = json.dumps(json_data).encode("utf-8") else: if isinstance(request.body, str): From 1b579b4c3f624ae099336d5513f375e792ad182e Mon Sep 17 00:00:00 2001 From: tighwm Date: Thu, 4 Sep 2025 12:39:49 +0300 Subject: [PATCH 6/7] added recursive_filter flag, another key error fix and short docs --- tests/unit/test_filters.py | 6 +++--- vcr/config.py | 7 ++++++- vcr/filters.py | 26 ++++++++++++++++++++++---- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_filters.py b/tests/unit/test_filters.py index 8d688083..6b530dd2 100644 --- a/tests/unit/test_filters.py +++ b/tests/unit/test_filters.py @@ -202,8 +202,8 @@ def test_replace_json_post_data_parameters(): assert request_data == expected_data -def test_replace_nested_post_data_parameters(): - body = b'{"nested": "same", "another": "same", "one": {"key": "secret", "nested": {"key": "secret"}}}' +def test_replace_recursive_post_data_parameters(): + body = b'{"nested": "change", "one": {"key": "secret", "nested": {"key": "secret"}}}' request = Request("POST", "http://google.com", body, {}) request.headers["Content-Type"] = "application/json" replace_post_data_parameters( @@ -211,8 +211,8 @@ def test_replace_nested_post_data_parameters(): [ ("key", None), ("nested", "aboba"), - ("another", None) ], + True ) request_data = json.loads(request.body) expected_data = json.loads('{"nested": "aboba", "one": {"nested": "aboba"}}') diff --git a/vcr/config.py b/vcr/config.py index dbd105e5..e8e24a02 100644 --- a/vcr/config.py +++ b/vcr/config.py @@ -206,9 +206,14 @@ def _build_before_record_request(self, options): functools.partial(filters.replace_query_parameters, replacements=replacements), ) if filter_post_data_parameters: + recursive = options.get("recursive_filter", False) replacements = [p if isinstance(p, tuple) else (p, None) for p in filter_post_data_parameters] filter_functions.append( - functools.partial(filters.replace_post_data_parameters, replacements=replacements), + functools.partial( + filters.replace_post_data_parameters, + replacements=replacements, + recursive=recursive, + ), ) hosts_to_ignore = set(ignore_hosts) diff --git a/vcr/filters.py b/vcr/filters.py index 5e8498aa..abc5c21a 100644 --- a/vcr/filters.py +++ b/vcr/filters.py @@ -69,7 +69,19 @@ def remove_query_parameters(request, query_parameters_to_remove): return replace_query_parameters(request, replacements) +def filtering_body(request, body_data, replacements): + """Filtering the request body by default to only high level keys""" + for k, rv in replacements.items(): + if k in body_data: + ov = body_data.pop(k) + if callable(rv): + rv = rv(key=k, value=ov, request=request) + if rv is not None: + body_data[k] = rv + + def recursive_filtering_body(request, body_data, replacements): + """Recursive filtering the request body with nested keys""" for k, ov in list(body_data.items()): if isinstance(ov, dict): recursive_filtering_body(request, ov, replacements) @@ -81,11 +93,11 @@ def recursive_filtering_body(request, body_data, replacements): rv = rv(key=k, value=ov, request=request) if rv is not None: body_data[k] = rv - else: + elif k in body_data: body_data.pop(k) -def replace_post_data_parameters(request, replacements): +def replace_post_data_parameters(request, replacements, recursive=False): """Replace post data in request--either form data or json--according to replacements. The replacements should be a list of (key, value) pairs where the value can be any of: @@ -102,11 +114,17 @@ def replace_post_data_parameters(request, replacements): if request.method == "POST" and not isinstance(request.body, BytesIO): if isinstance(request.body, dict): new_body = request.body.copy() - recursive_filtering_body(request, new_body, replacements) + if recursive: + recursive_filtering_body(request, new_body, replacements) + else: + filtering_body(request, new_body, replacements) request.body = new_body elif request.headers.get("Content-Type") == "application/json": json_data = json.loads(request.body) - recursive_filtering_body(request, json_data, replacements) + if recursive: + recursive_filtering_body(request, json_data, replacements) + else: + filtering_body(request, json_data, replacements) request.body = json.dumps(json_data).encode("utf-8") else: if isinstance(request.body, str): From 8b1c40f28f4b443a773f341e5d405347b86ef7c5 Mon Sep 17 00:00:00 2001 From: Blen <106350452+tighwm@users.noreply.github.com> Date: Thu, 4 Sep 2025 16:36:17 +0300 Subject: [PATCH 7/7] Apply suggestions from code review Co-authored-by: vEpiphyte --- tests/unit/test_filters.py | 2 +- vcr/filters.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_filters.py b/tests/unit/test_filters.py index 6b530dd2..c302ddb1 100644 --- a/tests/unit/test_filters.py +++ b/tests/unit/test_filters.py @@ -212,7 +212,7 @@ def test_replace_recursive_post_data_parameters(): ("key", None), ("nested", "aboba"), ], - True + recursive=True ) request_data = json.loads(request.body) expected_data = json.loads('{"nested": "aboba", "one": {"nested": "aboba"}}') diff --git a/vcr/filters.py b/vcr/filters.py index abc5c21a..8f0b2f86 100644 --- a/vcr/filters.py +++ b/vcr/filters.py @@ -70,7 +70,7 @@ def remove_query_parameters(request, query_parameters_to_remove): def filtering_body(request, body_data, replacements): - """Filtering the request body by default to only high level keys""" + """Filtering the request body by default to only top level keys""" for k, rv in replacements.items(): if k in body_data: ov = body_data.pop(k)