MemoryRandomOverusedBackend used in grequests example.

Author: sibiryakov

crawlfrontier/contrib/backends/memory/__init__.py

@@ -113,6 +113,18 @@ def get_next_requests(self, max_n_requests, downloader_info):
         return self._buffer.get_next_requests(max_n_requests, downloader_info)
 
 
+class MemoryRandomOverusedBackend(MemoryRandomBackend):
+    component_name = 'Random Memory Backend taking into account overused slots'
+
+    def __init__(self, manager):
+        super(MemoryRandomOverusedBackend, self).__init__(manager)
+        self._buffer = OverusedBuffer(super(MemoryRandomOverusedBackend, self).get_next_requests,
+                                      manager.logger.manager.debug)
+
+    def get_next_requests(self, max_n_requests, downloader_info):
+        return self._buffer.get_next_requests(max_n_requests, downloader_info)
+
+
 BASE = MemoryBaseBackend
 FIFO = MemoryFIFOBackend
 LIFO = MemoryLIFOBackend