Skip to content

Commit 545ef31

Browse files
voithvoith
committed
Added tests for filters and fix a bug in PlainValuesFilter
1 parent 6fbe310 commit 545ef31

File tree

2 files changed

+126
-9
lines changed

2 files changed

+126
-9
lines changed

‎frontera/logger/filters/__init__.py

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,22 @@
11
from __future__ import absolute_import
22
import logging
33
import six
4-
5-
6-
def format_str(s):
7-
if isinstance(s, six.text_type):
8-
return s.encode('ascii', 'ignore')
9-
return str(s)
4+
from w3lib.util import to_native_str
105

116

127
class PlainValuesFilter(logging.Filter):
138
def __init__(self, separator=None, excluded_fields=None, msg_max_length=0):
149
super(PlainValuesFilter, self).__init__()
15-
self.separator = separator or " "
10+
self.separator = to_native_str(separator or " ")
1611
self.excluded_fields = excluded_fields or []
1712
self.msg_max_length = msg_max_length
1813

1914
def filter(self, record):
2015
if isinstance(record.msg, dict):
2116
for field_name in self.excluded_fields:
2217
setattr(record, field_name, record.msg.get(field_name, ''))
23-
record.msg = self.separator.join([format_str(value)
24-
for key, value in record.msg.items()
18+
record.msg = self.separator.join([to_native_str(value)
19+
for key, value in six.iteritems(record.msg)
2520
if key not in self.excluded_fields])
2621
if self.msg_max_length and len(record.msg) > self.msg_max_length:
2722
record.msg = record.msg[0:self.msg_max_length-3] + "..."

‎tests/test_filters.py

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
import unittest
2+
3+
from frontera.logger.filters import PLAINVALUES, INCLUDEFIELDS, EXCLUDEFIELDS
4+
from tests.utils import LoggingCaptureMixin, SetupDefaultLoggingMixin
5+
6+
7+
class BaseTestFilters(SetupDefaultLoggingMixin, LoggingCaptureMixin, unittest.TestCase):
8+
def tearDown(self):
9+
super(BaseTestFilters, self).setUp()
10+
self.logger.handlers[0].filters = []
11+
12+
def addFilter(self, filter):
13+
self.logger.handlers[0].addFilter(filter)
14+
15+
16+
class TestFilterPlainValues(BaseTestFilters):
17+
def test_plain_values_exclude_fields(self):
18+
filter = PLAINVALUES(excluded_fields=['event'])
19+
self.addFilter(filter)
20+
self.logger.debug({'message1': 'logging', 'message2': 'debug', 'event': 'value'})
21+
log_msg = self.logger_output.getvalue()
22+
assert log_msg == 'logging debug\n' or log_msg == 'debug logging\n'
23+
24+
def test_plain_values_separator(self):
25+
filter = PLAINVALUES(separator=',')
26+
self.addFilter(filter)
27+
self.logger.debug({'message1': 'logging', 'message2': 'debug'})
28+
log_msg = self.logger_output.getvalue()
29+
assert log_msg == 'logging,debug\n' or log_msg == 'debug,logging\n'
30+
31+
def test_plain_values_msg_max_length(self):
32+
filter = PLAINVALUES(msg_max_length=10)
33+
self.addFilter(filter)
34+
self.logger.debug({'message1': '1' * 10, 'message2': '2' * 10})
35+
log_msg = self.logger_output.getvalue()
36+
assert log_msg == '%s...\n' % ('1' * 7) or log_msg == '%s...\n' % ('2' * 7)
37+
38+
def test_plain_values_str_msg(self):
39+
filter = PLAINVALUES(msg_max_length=10)
40+
self.addFilter(filter)
41+
self.logger.debug('debug message')
42+
self.assertEqual(self.logger_output.getvalue(), 'debug message\n')
43+
44+
45+
class TestIncludeFields(BaseTestFilters):
46+
def test_include_fields_matching_values(self):
47+
filter = INCLUDEFIELDS(field_name='event', included_values=['page_crawled'])
48+
self.addFilter(filter)
49+
self.logger.debug('crawled page P', extra={'event': 'page_crawled'})
50+
self.assertEqual(self.logger_output.getvalue(), 'crawled page P\n')
51+
52+
def test_include_fields_non_matching_values(self):
53+
filter = INCLUDEFIELDS(field_name='event', included_values=['links_extracted'])
54+
self.addFilter(filter)
55+
self.logger.debug('crawled page P', extra={'event': 'page_crawled'})
56+
self.assertEqual(self.logger_output.getvalue(), '')
57+
58+
def test_include_fields_dict_msg_matching_values(self):
59+
filter = INCLUDEFIELDS(field_name='event', included_values=['page_crawled'])
60+
self.addFilter(filter)
61+
self.logger.debug({'message': 'debug message', 'event': 'page_crawled'})
62+
log_msg = self.logger_output.getvalue()
63+
assert log_msg == "{'event': 'page_crawled', 'message': 'debug message'}\n" or \
64+
log_msg == "{'message': 'debug message', 'event': 'page_crawled'}\n"
65+
66+
def test_include_fields_dict_msg_non_matching_values(self):
67+
filter = INCLUDEFIELDS(field_name='event', included_values=['links_extracted'])
68+
self.addFilter(filter)
69+
self.logger.debug({'message': 'debug message', 'event': 'page_crawled'})
70+
self.assertEqual(self.logger_output.getvalue(), '')
71+
72+
def test_include_fields_field_name_none(self):
73+
filter = INCLUDEFIELDS(field_name=None, included_values=[])
74+
self.addFilter(filter)
75+
self.logger.debug('debug message')
76+
self.assertEqual(self.logger_output.getvalue(), 'debug message\n')
77+
78+
def test_include_fields_list_message(self):
79+
filter = INCLUDEFIELDS(field_name='event', included_values=['page_crawled'])
80+
self.addFilter(filter)
81+
self.logger.debug(['debug message'])
82+
self.assertEqual(self.logger_output.getvalue(), "['debug message']\n")
83+
84+
85+
class TestExcludeFields(BaseTestFilters):
86+
def test_exclude_fields_matching_values(self):
87+
filter = EXCLUDEFIELDS(field_name='event', excluded_fields=['page_crawled'])
88+
self.addFilter(filter)
89+
self.logger.debug('crawled page P', extra={'event': 'page_crawled'})
90+
self.assertEqual(self.logger_output.getvalue(), '')
91+
92+
def test_exclude_fields_non_matching_values(self):
93+
filter = EXCLUDEFIELDS(field_name='event', excluded_fields=['links_extracted'])
94+
self.addFilter(filter)
95+
self.logger.debug('crawled page P', extra={'event': 'page_crawled'})
96+
self.assertEqual(self.logger_output.getvalue(), 'crawled page P\n')
97+
98+
def test_exclude_fields_dict_msg_matching_values(self):
99+
filter = EXCLUDEFIELDS(field_name='event', excluded_fields='page_crawled')
100+
self.addFilter(filter)
101+
self.logger.debug({'message': 'debug message', 'event': 'page_crawled'})
102+
self.assertEqual(self.logger_output.getvalue(), '')
103+
104+
def test_exclude_fields_dict_msg_non_matching_values(self):
105+
filter = EXCLUDEFIELDS(field_name='event', excluded_fields='links_extracted')
106+
self.addFilter(filter)
107+
self.logger.debug({'message': 'debug message', 'event': 'page_crawled'})
108+
log_msg = self.logger_output.getvalue()
109+
assert log_msg == "{'event': 'page_crawled', 'message': 'debug message'}\n" or \
110+
log_msg == "{'message': 'debug message', 'event': 'page_crawled'}\n"
111+
112+
def test_include_fields_field_name_none(self):
113+
filter = EXCLUDEFIELDS(field_name=None, excluded_fields=[])
114+
self.addFilter(filter)
115+
self.logger.debug('debug message')
116+
self.assertEqual(self.logger_output.getvalue(), 'debug message\n')
117+
118+
def test_include_fields_list_message(self):
119+
filter = EXCLUDEFIELDS(field_name='event', excluded_fields=['page_crawled'])
120+
self.addFilter(filter)
121+
self.logger.debug(['debug message'])
122+
self.assertEqual(self.logger_output.getvalue(), "['debug message']\n")

0 commit comments

Comments
 (0)