Fix performence in dashboard (#264)

* Limit results in dashboard

* Add service loction list view

* Increase RESULTS_LIMIT

shynet/core/models.py

@@ -19,6 +19,7 @@ from django.utils.translation import gettext_lazy as _
 ACTIVE_USER_TIMEDELTA = timezone.timedelta(
     milliseconds=settings.SCRIPT_HEARTBEAT_FREQUENCY * 2
 )
+RESULTS_LIMIT = 300
 
 
 def _default_uuid():
@@ -176,7 +177,7 @@ class Service(models.Model):
         locations = (
             hits.values("location")
             .annotate(count=models.Count("location"))
-            .order_by("-count")
+            .order_by("-count")[:RESULTS_LIMIT]
         )
 
         referrer_ignore = self.get_ignored_referrer_regex()
@@ -186,7 +187,7 @@ class Service(models.Model):
                 hits.filter(initial=True)
                 .values("referrer")
                 .annotate(count=models.Count("referrer"))
-                .order_by("-count")
+                .order_by("-count")[:RESULTS_LIMIT]
             )
             if not referrer_ignore.match(referrer["referrer"])
         ]
@@ -194,29 +195,31 @@ class Service(models.Model):
         countries = (
             sessions.values("country")
             .annotate(count=models.Count("country"))
-            .order_by("-count")
+            .order_by("-count")[:RESULTS_LIMIT]
         )
 
         operating_systems = (
-            sessions.values("os").annotate(count=models.Count("os")).order_by("-count")
+            sessions.values("os")
+            .annotate(count=models.Count("os"))
+            .order_by("-count")[:RESULTS_LIMIT]
         )
 
         browsers = (
             sessions.values("browser")
             .annotate(count=models.Count("browser"))
-            .order_by("-count")
+            .order_by("-count")[:RESULTS_LIMIT]
         )
 
         device_types = (
             sessions.values("device_type")
             .annotate(count=models.Count("device_type"))
-            .order_by("-count")
+            .order_by("-count")[:RESULTS_LIMIT]
         )
 
         devices = (
             sessions.values("device")
             .annotate(count=models.Count("device"))
-            .order_by("-count")
+            .order_by("-count")[:RESULTS_LIMIT]
         )
 
         avg_load_time = hits.aggregate(load_time__avg=models.Avg("load_time"))[

shynet/dashboard/templates/dashboard/pages/service.html

@@ -135,6 +135,12 @@
                 {% endfor %}
             </tbody>
         </table>
+        {% if stats.locations.count == RESULTS_LIMIT %}
+            <hr class="sep h-8 md:h-12">
+            <a href="{% contextual_url 'dashboard:service_location_list' service.uuid %}" class="button ~neutral w-auto mb-2">
+                {% trans 'View more locations' %} &rarr;
+            </a>
+        {% endif %}
     </div>
     <div class="geo-map card ~neutral !low py-2 overflow-y-hidden">
         <p class="text-sm font-semibold p-2 border-b mb-2" style="color: var(--color-title)">

shynet/dashboard/templates/dashboard/pages/service_location_list.html

@@ -0,0 +1,47 @@
+{% extends "dashboard/service_base.html" %}
+
+{% load i18n a17t_tags pagination humanize helpers %}
+
+{% block head_title %}{{object.name}} {% trans 'Locations' %}{% endblock %}
+
+{% block service_actions %}
+<div class="mr-2">{% include 'dashboard/includes/date_range.html' %}</div>
+<a href="{% contextual_url 'dashboard:service' object.uuid %}" class="button field ~neutral !low bg-neutral-000 w-auto">{% trans 'Analytics' %} &rarr;</a>
+{% endblock %}
+
+{% block service_content %}
+<div class="card ~neutral !low mb-8 pt-2 max-w-full overflow-x-auto">
+    <table class="table">
+        <thead class="text-sm">
+            <tr>
+                <th>{% trans 'Location' %}</th>
+                <th class="rf">{% trans 'Hits' %}</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for location in object_list %}
+            <tr>
+                <td class="truncate w-full max-w-0 relative">
+                    <div class="relative flex items-center">
+                        {{location.location|default:"Unknown"|urldisplay}}
+                    </div>
+                </td>
+                <td>
+                    <div class="flex justify-end items-center">
+                        {{location.count|intcomma}}
+                        <span class="text-xs rf min-w-48">
+                            ({{location.count|percent:hit_count}})
+                        </span>
+                    </div>
+                </td>
+            </tr>
+            {% empty %}
+            <tr>
+                <td><span class="text-gray-600">{% trans 'No data yet...' %}</span></td>
+            </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+</div>
+{% pagination page_obj request %}
+{% endblock %}

shynet/dashboard/templates/dashboard/pages/service_session_list.html

@@ -14,4 +14,4 @@
     {% include 'dashboard/includes/session_list.html' %}
 </div>
 {% pagination page_obj request %}
-{% endblock %}
+{% endblock %}

shynet/dashboard/urls.py

@@ -26,6 +26,11 @@ urlpatterns = [
         views.ServiceSessionView.as_view(),
         name="service_session",
     ),
+    path(
+        "service/<pk>/locations/",
+        views.ServiceLocationsListView.as_view(),
+        name="service_location_list",
+    ),
     path(
         "api-token-refresh/",
         views.RefreshApiTokenView.as_view(),

shynet/dashboard/views.py

@@ -2,21 +2,20 @@ from django.conf import settings
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.contrib.messages.views import SuccessMessageMixin
 from django.core.cache import cache
-from django.db.models import Q
+from django.db.models import Q, Count
 from django.shortcuts import get_object_or_404, reverse, redirect
 from django.views.generic import (
     CreateView,
     DeleteView,
     DetailView,
     ListView,
-    TemplateView,
     UpdateView,
     View,
 )
 from rules.contrib.views import PermissionRequiredMixin
 
-from analytics.models import Session
+from analytics.models import Session, Hit
-from core.models import Service, _default_api_token
+from core.models import Service, _default_api_token, RESULTS_LIMIT
 
 from .forms import ServiceForm
 from .mixins import DateRangeMixin
@@ -68,6 +67,7 @@ class ServiceView(
         data = super().get_context_data(**kwargs)
         data["script_protocol"] = "https://" if settings.SCRIPT_USE_HTTPS else "http://"
         data["stats"] = self.object.get_core_stats(data["start_date"], data["end_date"])
+        data["RESULTS_LIMIT"] = RESULTS_LIMIT
         data["object_list"] = Session.objects.filter(
             service=self.get_object(),
             start_time__lt=self.get_end_date(),
@@ -141,6 +141,36 @@ class ServiceSessionsListView(
         return data
 
 
+class ServiceLocationsListView(
+    LoginRequiredMixin, PermissionRequiredMixin, DateRangeMixin, ListView
+):
+    model = Hit
+    template_name = "dashboard/pages/service_location_list.html"
+    paginate_by = RESULTS_LIMIT
+    permission_required = "core.view_service"
+
+    def get_object(self):
+        return get_object_or_404(Service, pk=self.kwargs.get("pk"))
+
+    def get_queryset(self):
+        hits = Hit.objects.filter(
+            service=self.get_object(),
+            start_time__lt=self.get_end_date(),
+            start_time__gt=self.get_start_date(),
+        )
+        self.hit_count = hits.count()
+
+        return (
+            hits.values("location").annotate(count=Count("location")).order_by("-count")
+        )
+
+    def get_context_data(self, **kwargs):
+        data = super().get_context_data(**kwargs)
+        data["object"] = self.get_object()
+        data["hit_count"] = self.hit_count
+        return data
+
+
 class ServiceSessionView(LoginRequiredMixin, PermissionRequiredMixin, DetailView):
     model = Session
     template_name = "dashboard/pages/service_session.html"