diff --git a/dbdb/api/pagination.py b/dbdb/api/pagination.py new file mode 100644 index 0000000..8376c3b --- /dev/null +++ b/dbdb/api/pagination.py @@ -0,0 +1,12 @@ +# third-party imports +from rest_framework.pagination import LimitOffsetPagination + + +# classes + +class StandardPagination(LimitOffsetPagination): + + default_limit = 10 + max_limit = 100 + + pass diff --git a/dbdb/api/urls.py b/dbdb/api/urls.py new file mode 100644 index 0000000..328c1ad --- /dev/null +++ b/dbdb/api/urls.py @@ -0,0 +1,9 @@ +# django imports +from django.urls import path +from django.urls import include +from django.conf.urls import url + + +urlpatterns = [ + path('v202004/', include('dbdb.api.v202004.urls')), +] diff --git a/dbdb/api/v202004/__init__.py b/dbdb/api/v202004/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dbdb/api/v202004/apps.py b/dbdb/api/v202004/apps.py new file mode 100644 index 0000000..86aae4f --- /dev/null +++ b/dbdb/api/v202004/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class ApiV202004Config(AppConfig): + name = 'api_v202004' diff --git a/dbdb/api/v202004/serializers.py b/dbdb/api/v202004/serializers.py new file mode 100644 index 0000000..8ee2ca3 --- /dev/null +++ b/dbdb/api/v202004/serializers.py @@ -0,0 +1,177 @@ +# stdlib imports +import collections +# third-party imports +from rest_framework import serializers +from rest_framework.reverse import reverse +# project imports +from dbdb.core.models import FeatureOption +from dbdb.core.models import System +from dbdb.core.models import SystemFeature + + +# abstracts + +class AbstractSystemSerializer: + + + def get_acquired_by(self, obj): + current = obj.get_current() + return current.acquired_by + + def get_countries(self, obj): + current = obj.get_current() + return list( map(str, current.countries) ) + + def get_description(self, obj): + current = obj.get_current() + return current.description + + def get_developer(self, obj): + current = obj.get_current() + return current.developer + + def get_features(self, obj): + current = obj.get_current() + + sysfeatures = SystemFeature.objects \ + .filter(system=current) \ + .select_related('feature') \ + .order_by('feature__slug') + + items = collections.OrderedDict() + for sysfeature in sysfeatures: + empty = True + + f = { + 'name': sysfeature.feature.label, + } + + if sysfeature.feature.multivalued: + f['values'] = [] + pass + else: + f['value'] = None + pass + + if sysfeature.feature.multivalued: + for option in sysfeature.options.all().order_by('slug'): + f['values'].append(option.value) + empty = False + else: + option = sysfeature.options.first() + if option: + f['value'] = option.value + empty = False + pass + + if not empty: + items[sysfeature.feature_id] = f + pass + + return list( items.values() ) + + def get_former_names(self, obj): + current = obj.get_current() + return current.former_names if current.former_names else None + + def get_history(self, obj): + current = obj.get_current() + return current.history + + def get_href(self, obj): + request = self.context['request'] + return reverse('system', args=[obj.slug], request=request) + + def get_end_year(self, obj): + current = obj.get_current() + return current.end_year + + def get_project_types(self, obj): + current = obj.get_current() + return list( map(str, current.project_types.all()) ) + + def get_urls(self, obj): + current = obj.get_current() + data = { + 'docs': None if not current.tech_docs else current.tech_docs, + 'homepage': None if not current.url else current.url, + 'source': None if not current.source_url else current.source_url, + 'wikipedia': None if not current.wikipedia_url else current.wikipedia_url, + } + return data + + def get_start_year(self, obj): + current = obj.get_current() + return current.start_year + + pass + + +# serializers + +class SystemBriefSerializer(AbstractSystemSerializer, serializers.HyperlinkedModelSerializer): + + class Meta: + model = System + fields = [ + 'url', + 'href', + 'name', + 'start_year', + 'end_year', + 'description', + 'history', + 'acquired_by', + 'developer', + 'project_types', + ] + + href = serializers.SerializerMethodField() + start_year = serializers.SerializerMethodField() + end_year = serializers.SerializerMethodField() + description = serializers.SerializerMethodField() + history = serializers.SerializerMethodField() + acquired_by = serializers.SerializerMethodField() + developer = serializers.SerializerMethodField() + project_types = serializers.SerializerMethodField() + url = serializers.HyperlinkedIdentityField(lookup_field='slug', read_only=True, view_name='api_v202004:systems_view') + + pass + +class SystemSerializer(AbstractSystemSerializer, serializers.HyperlinkedModelSerializer): + + class Meta: + model = System + fields = [ + 'href', + 'name', + 'former_names', + 'start_year', + 'end_year', + 'description', + 'history', + 'acquired_by', + 'developer', + 'countries', + 'features', + 'project_types', + 'urls', + 'version', + ] + + href = serializers.SerializerMethodField() + former_names = serializers.SerializerMethodField() + start_year = serializers.SerializerMethodField() + end_year = serializers.SerializerMethodField() + description = serializers.SerializerMethodField() + history = serializers.SerializerMethodField() + acquired_by = serializers.SerializerMethodField() + developer = serializers.SerializerMethodField() + countries = serializers.SerializerMethodField() + features = serializers.SerializerMethodField() + project_types = serializers.SerializerMethodField() + urls = serializers.SerializerMethodField() + version = serializers.IntegerField(source='ver', read_only=True) + + pass + diff --git a/dbdb/api/v202004/urls.py b/dbdb/api/v202004/urls.py new file mode 100644 index 0000000..c00c542 --- /dev/null +++ b/dbdb/api/v202004/urls.py @@ -0,0 +1,17 @@ +# django imports +from django.urls import path +# third-party imports +from rest_framework.urlpatterns import format_suffix_patterns +# local imports +from . import views + + +app_name = 'api_v202004' + +urlpatterns = [ + path('', views.APIRootView.as_view(), name='root'), + path('systems', views.SystemsView.as_view(), name='systems'), + path('systems/', views.SystemView.as_view(), name='systems_view'), +] + +urlpatterns = format_suffix_patterns(urlpatterns) diff --git a/dbdb/api/v202004/views.py b/dbdb/api/v202004/views.py new file mode 100644 index 0000000..a71b7a0 --- /dev/null +++ b/dbdb/api/v202004/views.py @@ -0,0 +1,59 @@ +# django imports +from django.shortcuts import render +# third-party imports +from rest_framework import generics +from rest_framework.response import Response +from rest_framework.reverse import reverse +from rest_framework.views import APIView +# project imports +from dbdb.core.models import System +from dbdb.core.models import SystemVersion +# local imports +from .serializers import SystemBriefSerializer +from .serializers import SystemSerializer + + +# class based views + +class APIRootView(APIView): + """ + Welcome to Database of Databases RESTful API, a public API for accessing out data. + """ + + def get(self, request): + + data = { + 'systems': reverse('api_v202004:systems', request=request) + } + + return Response(data) + +class SystemView(generics.RetrieveAPIView): + + lookup_field = 'slug' + queryset = System.objects.all() + serializer_class = SystemSerializer + + pass + +class SystemsView(generics.ListAPIView): + + queryset = System.objects.all() + serializer_class = SystemBriefSerializer + + def paginate_queryset(self, queryset): + if self.paginator is None: + return None + + items = self.paginator.paginate_queryset(queryset, self.request, view=self) + + # db optimizations - versions + ids = { item.id for item in items } + versions = SystemVersion.objects.filter(system_id__in=ids, is_current=True) + versions_map = { sv.system_id : sv for sv in versions } + for item in items: + item._current = versions_map[item.id] + + return items + + pass diff --git a/dbdb/core/models.py b/dbdb/core/models.py index 7c5c15f..871d9bb 100644 --- a/dbdb/core/models.py +++ b/dbdb/core/models.py @@ -209,6 +209,16 @@ def current(self): def get_absolute_url(self): return reverse('system', args=[self.slug]) + def get_current(self): + if not hasattr(self, '_current'): + if self.id is None: + self._current = SystemVersion(system=self) + else: + self._current = self.versions.get(is_current=True) + pass + + return self._current + pass # ============================================== @@ -459,6 +469,7 @@ def create_twitter_card(self): text_size = [0, 0] for line in name.split("\n"): line_size = font.getsize(line) + text_size[0] = max(text_size[0], line_size[0]) text_size[1] += line_size[1] + 5 diff --git a/dbdb/core/tests.py b/dbdb/core/tests.py index 09cec17..7cf2444 100644 --- a/dbdb/core/tests.py +++ b/dbdb/core/tests.py @@ -125,16 +125,16 @@ def test_autocom_valid_parameters(self): target = "SQLite" for i in range(1, len(target)): query = {'q': target[:i+1]} - #pprint(query) + response = self.client.get(reverse('search_autocomplete'), data=query) - #pprint(response.json()) + self.assertContains(response, 'SQLite', html=False) return def test_autocom_invalid_parameters(self): query = {'q': "YYY"} response = self.client.get(reverse('search_autocomplete'), data=query) - #pprint(response.json()) + self.assertEquals(len(response.json()), 0) return @@ -142,6 +142,7 @@ def test_autocom_no_parameters(self): response = self.client.get(reverse('search_autocomplete')) self.assertEquals(len(response.json()), 0) return + pass # ============================================== @@ -160,6 +161,7 @@ def test_counter(self): target = "SQLite" system = System.objects.get(name=target) + orig_visits = SystemVisit.objects.filter(system=system).count() data = {"token": CounterView.build_token('system', pk=system.id)} @@ -171,7 +173,6 @@ def test_counter(self): # Check that we got added a SystemVisit new_visits = SystemVisit.objects.filter(system=system).count() self.assertEquals(new_visits, orig_visits+1) - return def test_bot_block(self): @@ -193,7 +194,6 @@ def test_bot_block(self): self.assertEquals(new_count, orig_count) return - pass # ============================================== @@ -220,7 +220,7 @@ def test_inputs_quantity(self): filtergroups = d('div.filter-group') # Add two for the year filtergroups # Add nine for country, OS, project type, PL, inspired, derived, embedded compatiable, licenses - #pprint(filtergroups) + self.assertEquals(quantity + 2 + 9, len(filtergroups)) return @@ -229,7 +229,7 @@ def test_search_with_insuficient_data(self): 'feature1': ['option1'], } response = self.client.get(reverse('browse'), data=data) - #pprint(response.content) + self.assertContains(response, 'No databases found') return @@ -337,6 +337,7 @@ def test_can_create_database(self): response = self.client.post(reverse('create_database'), data=data) self.assertRedirects(response, reverse('system', kwargs={'slug': 'testdb'})) return + pass # ============================================== @@ -384,6 +385,7 @@ def test_buttons_shows_when_superuser(self): ) self.client.logout() return + pass # ============================================== @@ -421,6 +423,5 @@ def test_cant_login_with_wrong_data(self): 'Please enter a correct username and password. Note that both fields may be case-sensitive.' ) return - pass - + pass diff --git a/dbdb/core/views.py b/dbdb/core/views.py index b49f058..8fb2d272 100644 --- a/dbdb/core/views.py +++ b/dbdb/core/views.py @@ -947,7 +947,6 @@ def build_features(self, feature_form): @never_cache def get(self, request, slug=None): - # If there is no slug, then they are trying to create a new database. # Only superusers are allowed to do that. if slug is None: diff --git a/dbdb/settings.py b/dbdb/settings.py index d229941..ee15f77 100644 --- a/dbdb/settings.py +++ b/dbdb/settings.py @@ -10,6 +10,7 @@ BASE_DIR = root() DEBUG = env('DEBUG') # False if not in os.environ + # Application definition INSTALLED_APPS = [ @@ -21,13 +22,14 @@ 'django.contrib.messages', 'django.contrib.staticfiles', - #'autoslug', 'bootstrap4', - 'easy_thumbnails', 'django_countries', + 'easy_thumbnails', 'haystack', # django-haystack + 'rest_framework', # djangorestframework - 'dbdb.core' + 'dbdb.core', + 'dbdb.api.v202004', ] MIDDLEWARE = [ @@ -81,35 +83,27 @@ 'default': env.db( default='sqlite:///{}'.format( root.path('data/db.sqlite3') ) ) } + # Password validation # https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, + { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator' }, + { 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator' }, + { 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator' }, + { 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator' }, ] + # CACHE + CACHES = { 'default': { - #'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', - #'LOCATION': 'dbdb_io_cache', + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', } } - # Haystack # https://django-haystack.readthedocs.io/ @@ -131,10 +125,6 @@ 'PATH': root.path('data/xapian')(), 'FLAGS': HAYSTACK_XAPIAN_FLAGS, }, - # 'default': { - # 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', - # 'PATH': root.path('data/whoosh')(), - # }, } @@ -157,6 +147,25 @@ MEDIA_ROOT = root.path('media')() MEDIA_URL = '/media/' + +# Rest Framework + +REST_FRAMEWORK = { + # https://www.django-rest-framework.org/api-guide/pagination/ + 'DEFAULT_PAGINATION_CLASS': 'dbdb.api.pagination.StandardPagination', + + # https://www.django-rest-framework.org/api-guide/throttling/ + 'DEFAULT_THROTTLE_CLASSES': [ + 'rest_framework.throttling.AnonRateThrottle', + 'rest_framework.throttling.UserRateThrottle' + ], + 'DEFAULT_THROTTLE_RATES': { + 'anon': '4/second', + 'user': '4/second' + } +} + + # Security ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', default=['*']) @@ -200,9 +209,11 @@ }, } + # Django Countries COUNTRIES_FIRST = ['US'] + # Django Invisible reCaptcha NORECAPTCHA_SITE_KEY = '6Lfo8VwUAAAAAEHNqeL01PSkiRul7ImQ8Bsw8Nqc' NORECAPTCHA_SECRET_KEY = '6Lfo8VwUAAAAALFGUrGKqrzCR94pfgFahtd56WY9' diff --git a/dbdb/urls.py b/dbdb/urls.py index 1962aff..9261b00 100644 --- a/dbdb/urls.py +++ b/dbdb/urls.py @@ -8,6 +8,7 @@ urlpatterns = [ url(r'^', include('django.contrib.auth.urls')), url(r'^', include('dbdb.core.urls')), + url(r'^api/', include('dbdb.api.urls')), url(r'^admin/', admin.site.urls), ] diff --git a/requirements-dev.txt b/requirements-dev.txt index eb2ea52..bcb7d9e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ django-debug-toolbar==1.8 -django-extensions==1.9.6 \ No newline at end of file +django-extensions==1.9.6 diff --git a/templates/rest_framework/api.html b/templates/rest_framework/api.html new file mode 100644 index 0000000..bc97678 --- /dev/null +++ b/templates/rest_framework/api.html @@ -0,0 +1,11 @@ +{% extends "rest_framework/base.html" %} + + +{% block title %}{{ name }} – Database of Databases RESTful API{% endblock %} + + +{% block branding %} + + Database of Databases RESTful API (v{{ request.path|slice:"6:"|slice:":6" }}) + +{% endblock %} diff --git a/xapian_backend.py b/xapian_backend.py index 7bb0b80..515b6d8 100644 --- a/xapian_backend.py +++ b/xapian_backend.py @@ -7,6 +7,7 @@ import shutil import sys +import six from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.utils.encoding import force_text @@ -19,16 +20,21 @@ from haystack.models import SearchResult from haystack.utils import get_identifier, get_model_ct -import six NGRAM_MIN_LENGTH = 2 NGRAM_MAX_LENGTH = 15 + try: import xapian except ImportError: - raise MissingDependency("The 'xapian' backend requires the installation of 'Xapian'. " - "Please refer to the documentation.") + raise MissingDependency("The 'xapian' backend requires the installation of 'Xapian'. ") + + +if sys.version_info[0] == 2: + DirectoryExistsException = OSError +elif sys.version_info[0] == 3: + DirectoryExistsException = FileExistsError class NotSupportedError(Exception): @@ -194,8 +200,11 @@ def __init__(self, connection_alias, **connection_options): self.path = connection_options.get('PATH') - if self.path != MEMORY_DB_NAME and not os.path.exists(self.path): - os.makedirs(self.path) + if self.path != MEMORY_DB_NAME: + try: + os.makedirs(self.path) + except DirectoryExistsException: + pass self.flags = connection_options.get('FLAGS', DEFAULT_XAPIAN_FLAGS) self.language = getattr(settings, 'HAYSTACK_XAPIAN_LANGUAGE', 'english')