Skip to content

Commit 1c25e27

Browse files
mateusz834mateusz834
authored
Merge pull request #639 from splunk/dev/MP/deflake-collection
Deflake test_collection.py
2 parents a84c951 + a12f605 commit 1c25e27

File tree

1 file changed

+97
-62
lines changed

1 file changed

+97
-62
lines changed

‎tests/test_collection.py

Lines changed: 97 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -82,61 +82,77 @@ def test_metadata(self):
8282
)
8383

8484
def test_list(self):
85-
for coll_name in collections:
85+
def test(coll_name):
8686
coll = getattr(self.service, coll_name)
8787
expected = [ent.name for ent in coll.list(count=10, sort_mode="auto")]
88-
if len(expected) == 0:
89-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
9088
found = [ent.name for ent in coll.list()][:10]
91-
self.assertEqual(
92-
expected,
93-
found,
94-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
95-
)
89+
if expected != found:
90+
logging.warning(
91+
f"on {coll_name} (expected: {expected}, found: {found})",
92+
)
93+
return False
94+
95+
return True
9696

97-
def test_list_with_count(self):
98-
N = 5
9997
for coll_name in collections:
98+
self.assertEventuallyTrue(lambda: test(coll_name))
99+
100+
def test_list_with_count(self):
101+
def test(coll_name):
102+
N = 5
100103
coll = getattr(self.service, coll_name)
101104
expected = [ent.name for ent in coll.list(count=N + 5)][:N]
102105
N = len(expected) # in case there are <N elements
103106
found = [ent.name for ent in coll.list(count=N)]
104-
self.assertEqual(
105-
expected,
106-
found,
107-
msg=f"on {coll_name} (expected {expected}, found {found})",
108-
)
107+
if expected != found:
108+
logging.warning(
109+
f"on {coll_name} (expected: {expected}, found: {found})",
110+
)
111+
return False
112+
113+
return True
114+
115+
for coll_name in collections:
116+
self.assertEventuallyTrue(lambda: test(coll_name))
109117

110118
def test_list_with_offset(self):
119+
def test(coll_name, offset):
120+
coll = getattr(self.service, coll_name)
121+
expected = [ent.name for ent in coll.list(count=offset + 10)][offset:]
122+
found = [ent.name for ent in coll.list(offset=offset, count=10)]
123+
if expected != found:
124+
logging.warning(
125+
f"on {coll_name} (expected: {expected}, found: {found})",
126+
)
127+
return False
128+
129+
return True
130+
111131
import random
112132

113133
for offset in [random.randint(3, 50) for x in range(5)]:
114134
for coll_name in collections:
115-
coll = getattr(self.service, coll_name)
116-
expected = [ent.name for ent in coll.list(count=offset + 10)][offset:]
117-
found = [ent.name for ent in coll.list(offset=offset, count=10)]
118-
self.assertEqual(
119-
expected,
120-
found,
121-
msg=f"on {coll_name} (expected {expected}, found {found})",
122-
)
135+
self.assertEventuallyTrue(lambda: test(coll_name, offset))
123136

124137
def test_list_with_search(self):
125-
for coll_name in collections:
138+
def test(coll_name):
126139
coll = getattr(self.service, coll_name)
127140
expected = [ent.name for ent in coll.list()]
128-
if len(expected) == 0:
129-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
130141
# TODO: DVPL-5868 - This should use a real search instead of *. Otherwise the test passes trivially.
131142
found = [ent.name for ent in coll.list(search="*")]
132-
self.assertEqual(
133-
expected,
134-
found,
135-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
136-
)
143+
if expected != found:
144+
logging.warning(
145+
f"on {coll_name} (expected: {expected}, found: {found})",
146+
)
147+
return False
148+
149+
return True
137150

138-
def test_list_with_sort_dir(self):
139151
for coll_name in collections:
152+
self.assertEventuallyTrue(lambda: test(coll_name))
153+
154+
def test_list_with_sort_dir(self):
155+
def test(coll_name):
140156
coll = getattr(self.service, coll_name)
141157
expected_kwargs = {"sort_dir": "desc"}
142158
found_kwargs = {"sort_dir": "asc"}
@@ -150,18 +166,24 @@ def test_list_with_sort_dir(self):
150166
logging.debug(f"No entities in collection {coll_name}; skipping test.")
151167
found = [ent.name for ent in coll.list(**found_kwargs)]
152168

153-
self.assertEqual(
154-
sorted(expected),
155-
sorted(found),
156-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
157-
)
169+
if expected != found:
170+
logging.warning(
171+
f"on {coll_name} (expected: {expected}, found: {found})",
172+
)
173+
return False
174+
175+
return True
176+
177+
for coll_name in collections:
178+
self.assertEventuallyTrue(lambda: test(coll_name))
158179

159180
def test_list_with_sort_mode_auto(self):
160181
# The jobs collection requires special handling. The sort_dir kwarg is
161182
# needed because the default sorting direction for jobs is "desc", not
162183
# "asc". The sort_key kwarg is required because there is no default
163184
# sort_key for jobs in Splunk 6.
164-
for coll_name in collections:
185+
186+
def test(coll_name):
165187
coll = getattr(self.service, coll_name)
166188
if coll_name == "jobs":
167189
expected = [
@@ -181,11 +203,16 @@ def test_list_with_sort_mode_auto(self):
181203
else:
182204
found = [ent.name for ent in coll.list()]
183205

184-
self.assertEqual(
185-
expected,
186-
found,
187-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
188-
)
206+
if expected != found:
207+
logging.warning(
208+
f"on {coll_name} (expected: {expected}, found: {found})",
209+
)
210+
return False
211+
212+
return True
213+
214+
for coll_name in collections:
215+
self.assertEventuallyTrue(lambda: test(coll_name))
189216

190217
def test_list_with_sort_mode_alpha_case(self):
191218
for coll_name in collections:
@@ -227,27 +254,28 @@ def test_list_with_sort_mode_alpha(self):
227254
)
228255

229256
def test_iteration(self):
230-
for coll_name in collections:
257+
def test(coll_name):
231258
coll = getattr(self.service, coll_name)
232259
expected = [ent.name for ent in coll.list(count=10)]
233-
if len(expected) == 0:
234-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
235260
total = len(expected)
236261
found = []
237262
for ent in coll.iter(pagesize=max(int(total / 5.0), 1), count=10):
238263
found.append(ent.name)
239-
self.assertEqual(
240-
expected,
241-
found,
242-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
243-
)
264+
if expected != found:
265+
logging.warning(
266+
f"on {coll_name} (expected: {expected}, found: {found})",
267+
)
268+
return False
269+
270+
return True
244271

245-
def test_paging(self):
246272
for coll_name in collections:
273+
self.assertEventuallyTrue(lambda: test(coll_name))
274+
275+
def test_paging(self):
276+
def test(coll_name):
247277
coll = getattr(self.service, coll_name)
248278
expected = [ent.name for ent in coll.list(count=30)]
249-
if len(expected) == 0:
250-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
251279
total = len(expected)
252280
page_size = max(int(total / 5.0), 1)
253281
found = []
@@ -256,16 +284,23 @@ def test_paging(self):
256284
page = coll.list(offset=offset, count=page_size)
257285
count = len(page)
258286
offset += count
259-
self.assertTrue(
260-
count == page_size or offset == total, msg=f"on {coll_name}"
261-
)
287+
if count != page_size and offset != total:
288+
logging.warning(
289+
f"on {coll_name} (count = {count}, page_size = {page_size}, offset = {offset}, total = {total})",
290+
)
291+
return False
262292
found.extend([ent.name for ent in page])
263293
logging.debug("Iterate: offset=%d/%d", offset, total)
264-
self.assertEqual(
265-
expected,
266-
found,
267-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
268-
)
294+
if expected != found:
295+
logging.warning(
296+
f"on {coll_name} (expected: {expected}, found: {found})",
297+
)
298+
return False
299+
300+
return True
301+
302+
for coll_name in collections:
303+
self.assertEventuallyTrue(lambda: test(coll_name))
269304

270305
def test_getitem_with_nonsense(self):
271306
for coll_name in collections:

0 commit comments

Comments
 (0)