Added Lifo, fifo, dfs, bfs backend and correspoding tests for it

voith · voith · commit 1455b6b55cc2 · 2016-11-11T22:08:43.000+05:30
diff --git a/frontera/contrib/backends/__init__.py b/frontera/contrib/backends/__init__.py
@@ -92,15 +92,6 @@ def finished(self):
 
 class CommonStorageBackend(CommonBackend):
 
-    def _create_queue(self, settings):
-        if not issubclass(self.queue_component, BaseQueue):
-            raise TypeError('expected queue_component to '
-                            'belong to class: %s, got %s instead' % (type(BaseQueue).__name__,
-                                                                     type(self.queue_component).__name__))
-        return self.queue_component(self.session,
-                                    self.models['QueueModel'],
-                                    settings.get('SPIDER_FEED_PARTITIONS'))
-
     @property
     def queue(self):
         return self._queue
diff --git a/frontera/contrib/backends/cassandra/__init__.py b/frontera/contrib/backends/cassandra/__init__.py
@@ -14,8 +14,6 @@
 
 class CassandraBackend(CommonStorageBackend):
 
-    queue_component = Queue
-
     def __init__(self, manager):
         self.manager = manager
         settings = manager.settings
@@ -28,35 +26,70 @@ def __init__(self, manager):
         self.models = dict([(name, load_object(cls)) for name, cls in six.iteritems(models)])
         cluster_kwargs = {
             'port': cluster_port,
-            'compression': True
+            'compression': True,
         }
-        self.cluster = Cluster(contact_points=cluster_hosts, **cluster_kwargs)
-        self.session = self.cluster.connect(keyspace)
-        connection.setup(cluster_hosts, keyspace, **cluster_kwargs)
-        self.session.default_timeout = connection.session.default_timeout = \
-            settings.get('CASSANDRABACKEND_REQUEST_TIMEOUT')
+        if not connection.cluster:
+            connection.setup(cluster_hosts, keyspace, **cluster_kwargs)
+            connection.session.default_timeout = settings.get('CASSANDRABACKEND_REQUEST_TIMEOUT')
 
         if drop_all_tables:
             for name, table in six.iteritems(self.models):
                 drop_table(table)
 
-        for name, table in six.iteritems(self.models):
-            sync_table(table)
-
-        self._metadata = Metadata(self.session,
-                                  self.models['MetadataModel'],
-                                  settings.get('CASSANDRABACKEND_CACHE_SIZE'))
-        self._states = States(self.session,
-                              self.models['StateModel'],
-                              settings.get('STATE_CACHE_SIZE_LIMIT'))
+        self._metadata = Metadata(self.models['MetadataModel'], settings.get('CASSANDRABACKEND_CACHE_SIZE'))
+        self._states = States(self.models['StateModel'], settings.get('STATE_CACHE_SIZE_LIMIT'))
         self._queue = self._create_queue(settings)
 
     def frontier_stop(self):
         self.states.flush()
-        self.session.shutdown()
+
+    def _create_queue(self, settings):
+        return Queue(self.models['QueueModel'], settings.get('SPIDER_FEED_PARTITIONS'))
+
+
+class FIFOBackend(CassandraBackend):
+    component_name = 'Cassandra FIFO Backend'
+
+    def _create_queue(self, settings):
+        return Queue(self.models['FifoOrLIfoQueueModel'],
+                     settings.get('SPIDER_FEED_PARTITIONS'),
+                     ordering='created')
+
+
+class LIFOBackend(CassandraBackend):
+    component_name = 'Cassandra LIFO Backend'
+
+    def _create_queue(self, settings):
+        return Queue(self.models['FifoOrLIfoQueueModel'],
+                     settings.get('SPIDER_FEED_PARTITIONS'),
+                     ordering='created_desc')
+
+
+class DFSBackend(CassandraBackend):
+    component_name = 'Cassandra DFS Backend'
+
+    def _create_queue(self, settings):
+        return Queue(self.models['QueueModel'], settings.get('SPIDER_FEED_PARTITIONS'))
+
+    def _get_score(self, obj):
+        return -obj.meta[b'depth']
+
+
+class BFSBackend(CassandraBackend):
+    component_name = 'Cassandra BFS Backend'
+
+    def _create_queue(self, settings):
+        return Queue(self.models['QueueModel'], settings.get('SPIDER_FEED_PARTITIONS'))
+
+    def _get_score(self, obj):
+        return obj.meta[b'depth']
 
 
 BASE = CassandraBackend
+LIFO = LIFOBackend
+FIFO = FIFOBackend
+DFS = DFSBackend
+BFS = BFSBackend
 
 
 class Distributed(CommonDistributedStorageBackend):
@@ -111,6 +144,6 @@ def db_worker(cls, manager):
         sync_table(metadata_m)
         sync_table(queue_m)
 
-        b._metadata = Metadata(b.session, metadata_m)
-        b._queue = Queue(b.session, queue_m, settings.get('SPIDER_FEED_PARTITIONS'))
+        b._metadata = Metadata(metadata_m)
+        b._queue = Queue(queue_m, settings.get('SPIDER_FEED_PARTITIONS'))
         return b
diff --git a/frontera/contrib/backends/cassandra/components.py b/frontera/contrib/backends/cassandra/components.py
@@ -1,15 +1,13 @@
 # -*- coding: utf-8 -*-
 import logging
-import six
-import sys
-import traceback
 import uuid
 from time import time
 
+import six
 from cachetools import LRUCache
-from cassandra import (OperationTimedOut, ReadFailure, ReadTimeout,
-                       WriteFailure, WriteTimeout)
+from cassandra.cqlengine.management import sync_table
 from cassandra.cqlengine.query import BatchQuery
+from w3lib.util import to_bytes, to_native_str
 
 from frontera.contrib.backends import CreateOrModifyPageMixin
 from frontera.contrib.backends.memory import MemoryStates
@@ -20,36 +18,15 @@
 from frontera.utils.misc import chunks, get_crc32
 from frontera.utils.url import parse_domain_from_url_fast
 
-from w3lib.util import to_bytes, to_native_str
-
-
-def _retry(func):
-    def func_wrapper(self, *args, **kwargs):
-        tries = 5
-        count = 0
-        while count < tries:
-            try:
-                return func(self, *args, **kwargs)
-            except (OperationTimedOut, ReadTimeout, ReadFailure, WriteTimeout, WriteFailure) as exc:
-                ex_type, ex, tb = sys.exc_info()
-                tries += 1
-                self.logger.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
-                del tb
-                self.logger.info("Tries left %i" % tries - count)
-
-        raise exc
-
-    return func_wrapper
-
 
 class Metadata(BaseMetadata, CreateOrModifyPageMixin):
 
-    def __init__(self, session, model_cls, cache_size):
-        self.session = session
+    def __init__(self, model_cls, cache_size):
         self.model = model_cls
         self.cache = LRUCache(cache_size)
         self.batch = BatchQuery()
         self.logger = logging.getLogger("frontera.contrib.backends.cassandra.components.Metadata")
+        sync_table(model_cls)
 
     def frontier_stop(self):
         pass
@@ -80,7 +57,9 @@ def links_extracted(self, request, links):
         self.batch.execute()
 
     def update_score(self, batch):
-        for fprint, (score, url, schedule) in six.iteritems(batch):
+        if isinstance(batch, dict):
+            batch = [(fprint, score, url, schedule) for fprint, (score, url, schedule) in six.iteritems(batch)]
+        for fprint, score, url, schedule in batch:
             page = self.cache[fprint]
             page.fingerprint = to_native_str(fprint)
             page.score = score
@@ -93,20 +72,20 @@ def _add_to_batch_and_update_cache(self, page):
 
 class States(MemoryStates):
 
-    def __init__(self, session, model_cls, cache_size_limit):
+    def __init__(self, model_cls, cache_size_limit):
         super(States, self).__init__(cache_size_limit)
-        self.session = session
         self.model = model_cls
         self.batch = BatchQuery()
         self.logger = logging.getLogger("frontera.contrib.backends.cassandra.components.States")
+        sync_table(model_cls)
 
     def frontier_stop(self):
         self.flush()
 
     def fetch(self, fingerprints):
         to_fetch = [to_native_str(f) for f in fingerprints if f not in self._cache]
         self.logger.debug("cache size %s", len(self._cache))
-        self.logger.debug("to fetch %d from %d", (len(to_fetch), len(fingerprints)))
+        self.logger.debug("to fetch %d from %d", len(to_fetch), len(fingerprints))
 
         for chunk in chunks(to_fetch, 128):
             for state in self.model.objects.filter(fingerprint__in=chunk):
@@ -123,14 +102,14 @@ def flush(self, force_clear=False):
 
 class Queue(BaseQueue):
 
-    def __init__(self, session, queue_cls, partitions, ordering='default'):
-        self.session = session
+    def __init__(self, queue_cls, partitions, ordering='default'):
         self.queue_model = queue_cls
         self.logger = logging.getLogger("frontera.contrib.backends.cassandra.components.Queue")
         self.partitions = [i for i in range(0, partitions)]
         self.partitioner = Crc32NamePartitioner(self.partitions)
         self.ordering = ordering
         self.batch = BatchQuery()
+        sync_table(queue_cls)
 
     def frontier_stop(self):
         pass
@@ -228,7 +207,7 @@ def get_next_requests(self, max_n_requests, partition_id, **kwargs):
                               tries, limit, count, len(queue.keys()))
             queue.clear()
             count = 0
-            for item in self._order_by(self.queue_model.filter(partition_id=partition_id)).limit(max_n_requests):
+            for item in self._order_by(self.queue_model.filter(partition_id=partition_id).allow_filtering()).limit(max_n_requests):
                 if item.host_crc32 not in queue:
                     queue[item.host_crc32] = []
                 if max_requests_per_host is not None and len(queue[item.host_crc32]) > max_requests_per_host:
diff --git a/frontera/contrib/backends/cassandra/models.py b/frontera/contrib/backends/cassandra/models.py
@@ -88,3 +88,27 @@ class QueueModel(Model):
 
     def __repr__(self):
         return '<Queue:%s (%s)>' % (self.url, self.id)
+
+
+class FifoOrLIfoQueueModel(Model):
+    # Separate models are needed as
+    # order_by is supported on columns
+    # only in the order, the clustering
+    # keys were created
+
+    # Also Inheriting model has some runtime issues
+    # mostly a bug in the driver
+    # Hence the duplicate code
+
+    partition_id = Integer(primary_key=True)
+    score = Float(required=True)
+    created_at = BigInt(primary_key=True)
+    id = UUID(primary_key=True)
+    url = Text(required=True)
+    fingerprint = Text(required=True)
+    host_crc32 = Integer(required=True)
+    meta = PickleDict()
+    headers = PickleDict()
+    cookies = PickleDict()
+    method = Text()
+    depth = SmallInt()
diff --git a/frontera/contrib/backends/cassandra/revisiting.py b/frontera/contrib/backends/cassandra/revisiting.py
@@ -2,7 +2,7 @@
 import json
 import logging
 from datetime import datetime, timedelta
-from time import sleep, time
+from time import time
 
 from cassandra.cqlengine import columns
 from cassandra.cqlengine.models import Model
@@ -22,24 +22,6 @@ class RevisitingQueueModel(Model):
     crawl_at = columns.DateTime(required=True, default=datetime.now(), index=True)
 
 
-def retry_and_rollback(func):
-    def func_wrapper(self, *args, **kwargs):
-        tries = 5
-        while True:
-            try:
-                return func(self, *args, **kwargs)
-            except Exception as exc:
-                self.logger.exception(exc)
-                sleep(5)
-                tries -= 1
-                if tries > 0:
-                    self.logger.info("Tries left %i" % tries)
-                    continue
-                else:
-                    raise exc
-    return func_wrapper
-
-
 class RevisitingQueue(BaseQueue):
     def __init__(self, session, queue_cls, partitions):
         self.session = session()
@@ -64,7 +46,6 @@ def get_next_requests(self, max_n_requests, partition_id, **kwargs):
             self.logger.exception(exc)
         return results
 
-    @retry_and_rollback
     def schedule(self, batch):
         for fprint, score, request, schedule_at in batch:
             if schedule_at:
@@ -107,7 +88,6 @@ def _create_queue(self, obj, fingerprint, score, partition_id, host_crc32, creat
 
         return db_queue
 
-    @retry_and_rollback
     def count(self):
         return self.session.query(self.queue_model).count()
 
diff --git a/frontera/contrib/backends/sqlalchemy/__init__.py b/frontera/contrib/backends/sqlalchemy/__init__.py
@@ -12,8 +12,6 @@
 
 class SQLAlchemyBackend(CommonStorageBackend):
 
-    queue_component = Queue
-
     def __init__(self, manager):
         self.manager = manager
         settings = manager.settings
@@ -48,6 +46,9 @@ def frontier_stop(self):
         super(SQLAlchemyBackend, self).frontier_stop()
         self.engine.dispose()
 
+    def _create_queue(self, settings):
+        return Queue(self.session, self.models['QueueModel'], settings.get('SPIDER_FEED_PARTITIONS'))
+
 
 class FIFOBackend(SQLAlchemyBackend):
     component_name = 'SQLAlchemy FIFO Backend'
diff --git a/frontera/settings/default_settings.py b/frontera/settings/default_settings.py
@@ -14,7 +14,8 @@
 CASSANDRABACKEND_MODELS = {
     'MetadataModel': 'frontera.contrib.backends.cassandra.models.MetadataModel',
     'StateModel': 'frontera.contrib.backends.cassandra.models.StateModel',
-    'QueueModel': 'frontera.contrib.backends.cassandra.models.QueueModel'
+    'QueueModel': 'frontera.contrib.backends.cassandra.models.QueueModel',
+    'FifoOrLIfoQueueModel': 'frontera.contrib.backends.cassandra.models.FifoOrLIfoQueueModel',
 }
 CASSANDRABACKEND_REVISIT_INTERVAL = timedelta(days=1)
 CASSANDRABACKEND_CLUSTER_HOSTS = ['127.0.0.1']
diff --git a/setup.py b/setup.py
@@ -73,7 +73,8 @@
             'Twisted'
         ],
         'cassandra': [
-            'cassandra-driver==3.7.0'
+            'cassandra-driver==3.7.0',
+            'cachetools'
         ]
     },
     tests_require=[
diff --git a/tests/contrib/backends/cassandra/test_backend_cassandra.py b/tests/contrib/backends/cassandra/test_backend_cassandra.py
