From 71431fcaa63eef4e02be6ca6c89a7eff3149e17e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Jastrz=C4=99bski?= Date: Fri, 23 Apr 2021 10:50:34 +0200 Subject: [PATCH 01/19] Fix currently_online Make currently_online aware of SCRIPT_HEARTBEAT_FREQUENCY --- shynet/analytics/models.py | 8 ++------ shynet/core/models.py | 8 +++++++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/shynet/analytics/models.py b/shynet/analytics/models.py index 92d9106..68e9bac 100644 --- a/shynet/analytics/models.py +++ b/shynet/analytics/models.py @@ -1,12 +1,10 @@ -import json import uuid -from django.conf import settings from django.db import models from django.shortcuts import reverse from django.utils import timezone -from core.models import Service +from core.models import Service, ACTIVE_USER_TIMEDELTA def _default_uuid(): @@ -61,9 +59,7 @@ class Session(models.Model): @property def is_currently_active(self): - return timezone.now() - self.last_seen < timezone.timedelta( - milliseconds=settings.SCRIPT_HEARTBEAT_FREQUENCY * 2 - ) + return timezone.now() - self.last_seen < ACTIVE_USER_TIMEDELTA @property def duration(self): diff --git a/shynet/core/models.py b/shynet/core/models.py index e67f614..0e30510 100644 --- a/shynet/core/models.py +++ b/shynet/core/models.py @@ -4,6 +4,7 @@ import re import uuid from django.apps import apps +from django.conf import settings from django.contrib.auth.models import AbstractUser from django.core.exceptions import ValidationError from django.db import models @@ -13,6 +14,11 @@ from django.shortcuts import reverse from django.utils import timezone +ACTIVE_USER_TIMEDELTA = timezone.timedelta( + milliseconds=settings.SCRIPT_HEARTBEAT_FREQUENCY * 2 +) + + def _default_uuid(): return str(uuid.uuid4()) @@ -120,7 +126,7 @@ class Service(models.Model): Hit = apps.get_model("analytics", "Hit") currently_online = Session.objects.filter( - service=self, last_seen__gt=timezone.now() - timezone.timedelta(seconds=10) + service=self, last_seen__gt=timezone.now() - ACTIVE_USER_TIMEDELTA ).count() sessions = Session.objects.filter( From 94c53d2ab56adec58b1ca6e284ee0b002e362c92 Mon Sep 17 00:00:00 2001 From: CasperVerswijvelt Date: Sat, 24 Apr 2021 12:51:56 +0200 Subject: [PATCH 02/19] Show snippet on service page when not hits are recorded yet --- shynet/core/models.py | 5 +++++ .../templates/dashboard/includes/service_snippet.html | 5 +++++ .../dashboard/templates/dashboard/pages/service.html | 11 +++++++++++ .../templates/dashboard/pages/service_update.html | 7 +------ 4 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 shynet/dashboard/templates/dashboard/includes/service_snippet.html diff --git a/shynet/core/models.py b/shynet/core/models.py index e67f614..7c8122c 100644 --- a/shynet/core/models.py +++ b/shynet/core/models.py @@ -132,6 +132,10 @@ class Service(models.Model): service=self, start_time__lt=end_time, start_time__gt=start_time ) hit_count = hits.count() + hits = Hit.objects.filter( + service=self + ) + has_hits = hits.exists() bounces = sessions.filter(is_bounce=True) bounce_count = bounces.count() @@ -218,6 +222,7 @@ class Service(models.Model): "currently_online": currently_online, "session_count": session_count, "hit_count": hit_count, + "has_hits": has_hits, "avg_hits_per_session": hit_count / (max(session_count, 1)), "bounce_rate_pct": bounce_count * 100 / session_count if session_count > 0 diff --git a/shynet/dashboard/templates/dashboard/includes/service_snippet.html b/shynet/dashboard/templates/dashboard/includes/service_snippet.html new file mode 100644 index 0000000..cfdce40 --- /dev/null +++ b/shynet/dashboard/templates/dashboard/includes/service_snippet.html @@ -0,0 +1,5 @@ +
{% filter force_escape %} +{% endfilter %} +
\ No newline at end of file diff --git a/shynet/dashboard/templates/dashboard/pages/service.html b/shynet/dashboard/templates/dashboard/pages/service.html index 8a0dec0..f8d67da 100644 --- a/shynet/dashboard/templates/dashboard/pages/service.html +++ b/shynet/dashboard/templates/dashboard/pages/service.html @@ -11,6 +11,17 @@ {% endblock %} {% block service_content %} +{% if not stats.has_hits %} +
+
Get started
+

+ It seems that you have not recorded any data yet! +
+ Get started by placing the following snippet at the end of the <body> tag on any page you'd like to track. +

+ {% include 'dashboard/includes/service_snippet.html' %} +
+{% endif %}
{% with classes="text-sm font-semibold" good_classes="text-positive-400" bad_classes="text-critical-400" neutral_classes="text-gray-400" %}
diff --git a/shynet/dashboard/templates/dashboard/pages/service_update.html b/shynet/dashboard/templates/dashboard/pages/service_update.html index 53483cf..097335f 100644 --- a/shynet/dashboard/templates/dashboard/pages/service_update.html +++ b/shynet/dashboard/templates/dashboard/pages/service_update.html @@ -12,12 +12,7 @@
Installation

Place the following snippet at the end of the <body> tag on any page you'd like to track.

-
- {% filter force_escape %} - - {% endfilter %} -
+ {% include 'dashboard/includes/service_snippet.html' %}
Settings
From 46176b19fc91d1d3e455a94e026381f1c6f5ebc0 Mon Sep 17 00:00:00 2001 From: CasperVerswijvelt Date: Sat, 24 Apr 2021 13:04:21 +0200 Subject: [PATCH 03/19] Merge 2 steps --- shynet/core/models.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/shynet/core/models.py b/shynet/core/models.py index 7c8122c..9fe575d 100644 --- a/shynet/core/models.py +++ b/shynet/core/models.py @@ -132,10 +132,8 @@ class Service(models.Model): service=self, start_time__lt=end_time, start_time__gt=start_time ) hit_count = hits.count() - hits = Hit.objects.filter( - service=self - ) - has_hits = hits.exists() + + has_hits = Hit.objects.filter(service=self).exists() bounces = sessions.filter(is_bounce=True) bounce_count = bounces.count() From fef531efa9288411bd73a881174813d54eb7ceaa Mon Sep 17 00:00:00 2001 From: "R. Miles McCain" Date: Sat, 24 Apr 2021 17:06:19 +0000 Subject: [PATCH 04/19] Improve homepage when no services exist --- shynet/dashboard/templates/dashboard/pages/dashboard.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shynet/dashboard/templates/dashboard/pages/dashboard.html b/shynet/dashboard/templates/dashboard/pages/dashboard.html index 3cff3eb..cda47d1 100644 --- a/shynet/dashboard/templates/dashboard/pages/dashboard.html +++ b/shynet/dashboard/templates/dashboard/pages/dashboard.html @@ -21,9 +21,11 @@ {% for object in object_list|dictsortreversed:"stats.session_count" %} {% include 'dashboard/includes/service_overview.html' %} {% empty %} -

You don't have any services on {{request.site.name|default:"Shynet"}} yet.

+

You don't have any services yet. {% if can_create %}Get started by creating one.{% endif %}

{% endfor %} +{% if object_list %} {% pagination page_obj request %} +{% endif %} {% endblock %} \ No newline at end of file From 68945df17dc0e89449c6a79b00ad84f0a9e35f78 Mon Sep 17 00:00:00 2001 From: "R. Miles McCain" Date: Sat, 24 Apr 2021 17:06:32 +0000 Subject: [PATCH 05/19] Improve service page when no hits are recorded --- shynet/dashboard/templates/dashboard/pages/service.html | 8 +++----- shynet/dashboard/views.py | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/shynet/dashboard/templates/dashboard/pages/service.html b/shynet/dashboard/templates/dashboard/pages/service.html index f8d67da..0005d64 100644 --- a/shynet/dashboard/templates/dashboard/pages/service.html +++ b/shynet/dashboard/templates/dashboard/pages/service.html @@ -13,15 +13,12 @@ {% block service_content %} {% if not stats.has_hits %}
-
Get started

- It seems that you have not recorded any data yet! -
- Get started by placing the following snippet at the end of the <body> tag on any page you'd like to track. + This service hasn't collected any data yet. To get started, place the following code snippet at the end of the <body> tag on any page you'd like to track.

{% include 'dashboard/includes/service_snippet.html' %}
-{% endif %} +{% else %}
{% with classes="text-sm font-semibold" good_classes="text-positive-400" bad_classes="text-critical-400" neutral_classes="text-gray-400" %}
@@ -96,6 +93,7 @@
{% endwith %}
+{% endif %}
{% include 'dashboard/includes/time_chart.html' with data=stats.session_chart_data %}
diff --git a/shynet/dashboard/views.py b/shynet/dashboard/views.py index c071143..483fee0 100644 --- a/shynet/dashboard/views.py +++ b/shynet/dashboard/views.py @@ -66,6 +66,7 @@ class ServiceView( def get_context_data(self, **kwargs): data = super().get_context_data(**kwargs) + data["script_protocol"] = "https://" if settings.SCRIPT_USE_HTTPS else "http://" data["stats"] = self.object.get_core_stats(data["start_date"], data["end_date"]) data["object_list"] = Session.objects.filter( service=self.get_object(), From 26c1ae2bceb8375955c28f8eea7b7818e95386f7 Mon Sep 17 00:00:00 2001 From: "R. Miles McCain" Date: Sat, 24 Apr 2021 17:21:29 +0000 Subject: [PATCH 06/19] Fix ingest when MMDB not found --- shynet/analytics/tasks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/shynet/analytics/tasks.py b/shynet/analytics/tasks.py index c28727c..5ea5bf4 100644 --- a/shynet/analytics/tasks.py +++ b/shynet/analytics/tasks.py @@ -39,6 +39,9 @@ def _geoip2_lookup(ip): } except geoip2.errors.AddressNotFoundError: return {} + except FileNotFoundError as e: + log.exception("Unable to perform GeoIP lookup: %s", e) + return {} @shared_task @@ -58,6 +61,7 @@ def ingress_request( log.debug(f"Linked to service {service}") if dnt and service.respect_dnt: + log.debug("Ignoring because of DNT") return try: @@ -67,6 +71,7 @@ def ingress_request( ignored_network.version == remote_ip.version and ignored_network.supernet_of(remote_ip) ): + log.debug("Ignoring because of ignored IP") return except ValueError as e: log.exception(e) @@ -197,4 +202,5 @@ def ingress_request( ) except Exception as e: log.exception(e) + print(e) raise e From 03f88af03c09f71c9d3a034a87fbbbe47fb86339 Mon Sep 17 00:00:00 2001 From: "R. Miles McCain" Date: Sat, 24 Apr 2021 17:35:55 +0000 Subject: [PATCH 07/19] Update test for my new instance --- tests/js.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/js.html b/tests/js.html index d974bfa..b745c68 100644 --- a/tests/js.html +++ b/tests/js.html @@ -3,11 +3,14 @@ - Pixel test + JS test - + + From 023e0fde1529300e197da75e8e6ef645cb0a5f51 Mon Sep 17 00:00:00 2001 From: CasperVerswijvelt Date: Thu, 6 May 2021 21:23:05 +0200 Subject: [PATCH 08/19] Preserve date range query parameters --- shynet/dashboard/mixins.py | 9 +++++++-- .../templates/dashboard/includes/service_overview.html | 2 +- .../templates/dashboard/includes/session_list.html | 2 +- shynet/dashboard/templates/dashboard/pages/service.html | 2 +- .../templates/dashboard/pages/service_session_list.html | 2 +- shynet/dashboard/templates/dashboard/service_base.html | 2 +- shynet/dashboard/templatetags/helpers.py | 4 ++++ 7 files changed, 16 insertions(+), 7 deletions(-) diff --git a/shynet/dashboard/mixins.py b/shynet/dashboard/mixins.py index ed374a1..6ec65ce 100644 --- a/shynet/dashboard/mixins.py +++ b/shynet/dashboard/mixins.py @@ -5,14 +5,16 @@ from django.utils import timezone class DateRangeMixin: - def get_start_date(self): + def get_start_date(self, use_default=True): if self.request.GET.get("startDate") != None: found_time = timezone.datetime.strptime( self.request.GET.get("startDate"), "%Y-%m-%d" ) return timezone.make_aware(datetime.combine(found_time, time.min)) - else: + elif use_default == True: return timezone.now() - timezone.timedelta(days=30) + else: + return None def get_end_date(self): if self.request.GET.get("endDate") != None: @@ -23,8 +25,11 @@ class DateRangeMixin: else: return timezone.now() + def get_context_data(self, **kwargs): data = super().get_context_data(**kwargs) data["start_date"] = self.get_start_date() data["end_date"] = self.get_end_date() + start_date = self.get_start_date(False) + data["date_query_params"] = f"?startDate={start_date.strftime('%Y-%m-%d')}&endDate={self.get_end_date().strftime('%Y-%m-%d')}" if start_date is not None else "" return data diff --git a/shynet/dashboard/templates/dashboard/includes/service_overview.html b/shynet/dashboard/templates/dashboard/includes/service_overview.html index 1996ddb..6024170 100644 --- a/shynet/dashboard/templates/dashboard/includes/service_overview.html +++ b/shynet/dashboard/templates/dashboard/includes/service_overview.html @@ -1,6 +1,6 @@ {% load humanize helpers %} - + {% with stats=object.stats %}
diff --git a/shynet/dashboard/templates/dashboard/includes/session_list.html b/shynet/dashboard/templates/dashboard/includes/session_list.html index a59d484..07aa33e 100644 --- a/shynet/dashboard/templates/dashboard/includes/session_list.html +++ b/shynet/dashboard/templates/dashboard/includes/session_list.html @@ -12,7 +12,7 @@ {% for session in object_list %} - {{session.start_time|date:"M j Y, g:i a"|capfirst}} {% if session.is_currently_active %} diff --git a/shynet/dashboard/templates/dashboard/pages/service.html b/shynet/dashboard/templates/dashboard/pages/service.html index 0005d64..8fa55c4 100644 --- a/shynet/dashboard/templates/dashboard/pages/service.html +++ b/shynet/dashboard/templates/dashboard/pages/service.html @@ -235,7 +235,7 @@
{% include 'dashboard/includes/session_list.html' %}
- View more + View more sessions →
diff --git a/shynet/dashboard/templates/dashboard/pages/service_session_list.html b/shynet/dashboard/templates/dashboard/pages/service_session_list.html index 1218682..eee85e2 100644 --- a/shynet/dashboard/templates/dashboard/pages/service_session_list.html +++ b/shynet/dashboard/templates/dashboard/pages/service_session_list.html @@ -6,7 +6,7 @@ {% block service_actions %}
{% include 'dashboard/includes/date_range.html' %}
-Analytics → +Analytics → {% endblock %} {% block service_content %} diff --git a/shynet/dashboard/templates/dashboard/service_base.html b/shynet/dashboard/templates/dashboard/service_base.html index 3ce009d..54e0c99 100644 --- a/shynet/dashboard/templates/dashboard/service_base.html +++ b/shynet/dashboard/templates/dashboard/service_base.html @@ -6,7 +6,7 @@ {% block content %}
- +

{{object.link|iconify}} {{object.name}} diff --git a/shynet/dashboard/templatetags/helpers.py b/shynet/dashboard/templatetags/helpers.py index a494c2c..78be195 100644 --- a/shynet/dashboard/templatetags/helpers.py +++ b/shynet/dashboard/templatetags/helpers.py @@ -184,3 +184,7 @@ def urldisplay(url): ) else: return url + +@register.filter +def add_string(arg1, arg2): + return f"{str(arg1)}{str(arg2)}" From 2b003b8fa91158217ad341b70ce02028e7902fc5 Mon Sep 17 00:00:00 2001 From: CasperVerswijvelt Date: Sat, 8 May 2021 12:17:29 +0200 Subject: [PATCH 09/19] Preserve date range in urls in side menu --- shynet/dashboard/mixins.py | 1 - shynet/dashboard/templates/base.html | 4 +++- shynet/dashboard/templatetags/helpers.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/shynet/dashboard/mixins.py b/shynet/dashboard/mixins.py index 6ec65ce..7a56d3b 100644 --- a/shynet/dashboard/mixins.py +++ b/shynet/dashboard/mixins.py @@ -25,7 +25,6 @@ class DateRangeMixin: else: return timezone.now() - def get_context_data(self, **kwargs): data = super().get_context_data(**kwargs) data["start_date"] = self.get_start_date() diff --git a/shynet/dashboard/templates/base.html b/shynet/dashboard/templates/base.html index e33a5f3..a3b5379 100644 --- a/shynet/dashboard/templates/base.html +++ b/shynet/dashboard/templates/base.html @@ -42,7 +42,9 @@ {% for service in user.owning_services.all %} {% url 'dashboard:service' service.uuid as url %} - {% include 'dashboard/includes/sidebar_portal.html' with label=service.name|truncatechars:16 url=url icon=service.link|iconify %} + {% with url_full=date_query_params|default:''|prepend_string:url %} + {% include 'dashboard/includes/sidebar_portal.html' with label=service.name|truncatechars:16 url=url_full icon=service.link|iconify %} + {% endwith %} {% endfor %} {% endif %} diff --git a/shynet/dashboard/templatetags/helpers.py b/shynet/dashboard/templatetags/helpers.py index 78be195..4919f9e 100644 --- a/shynet/dashboard/templatetags/helpers.py +++ b/shynet/dashboard/templatetags/helpers.py @@ -186,5 +186,5 @@ def urldisplay(url): return url @register.filter -def add_string(arg1, arg2): - return f"{str(arg1)}{str(arg2)}" +def prepend_string(arg1, arg2): + return f"{str(arg2)}{str(arg1)}" From a6a508899a73d77532d90e9a5b9a85cd3105b494 Mon Sep 17 00:00:00 2001 From: "R. Miles McCain" Date: Fri, 14 May 2021 14:50:04 +0000 Subject: [PATCH 10/19] Contextual date improvements --- shynet/dashboard/mixins.py | 8 +--- shynet/dashboard/templates/base.html | 8 ++-- .../dashboard/includes/date_range.html | 4 +- .../dashboard/includes/service_overview.html | 2 +- .../dashboard/includes/session_list.html | 2 +- .../templates/dashboard/pages/service.html | 4 +- .../dashboard/pages/service_session.html | 2 +- .../dashboard/pages/service_session_list.html | 2 +- .../templates/dashboard/service_base.html | 2 +- shynet/dashboard/templatetags/helpers.py | 44 +++++++++++++++++-- 10 files changed, 55 insertions(+), 23 deletions(-) diff --git a/shynet/dashboard/mixins.py b/shynet/dashboard/mixins.py index 7a56d3b..ed374a1 100644 --- a/shynet/dashboard/mixins.py +++ b/shynet/dashboard/mixins.py @@ -5,16 +5,14 @@ from django.utils import timezone class DateRangeMixin: - def get_start_date(self, use_default=True): + def get_start_date(self): if self.request.GET.get("startDate") != None: found_time = timezone.datetime.strptime( self.request.GET.get("startDate"), "%Y-%m-%d" ) return timezone.make_aware(datetime.combine(found_time, time.min)) - elif use_default == True: - return timezone.now() - timezone.timedelta(days=30) else: - return None + return timezone.now() - timezone.timedelta(days=30) def get_end_date(self): if self.request.GET.get("endDate") != None: @@ -29,6 +27,4 @@ class DateRangeMixin: data = super().get_context_data(**kwargs) data["start_date"] = self.get_start_date() data["end_date"] = self.get_end_date() - start_date = self.get_start_date(False) - data["date_query_params"] = f"?startDate={start_date.strftime('%Y-%m-%d')}&endDate={self.get_end_date().strftime('%Y-%m-%d')}" if start_date is not None else "" return data diff --git a/shynet/dashboard/templates/base.html b/shynet/dashboard/templates/base.html index a3b5379..cff7d83 100644 --- a/shynet/dashboard/templates/base.html +++ b/shynet/dashboard/templates/base.html @@ -41,10 +41,8 @@

Services

{% for service in user.owning_services.all %} - {% url 'dashboard:service' service.uuid as url %} - {% with url_full=date_query_params|default:''|prepend_string:url %} - {% include 'dashboard/includes/sidebar_portal.html' with label=service.name|truncatechars:16 url=url_full icon=service.link|iconify %} - {% endwith %} + {% contextual_url 'dashboard:service' service.uuid as url %} + {% include 'dashboard/includes/sidebar_portal.html' with label=service.name|truncatechars:16 url=url icon=service.link|iconify %} {% endfor %} {% endif %} @@ -62,7 +60,7 @@

Collaborations

{% for service in user.collaborating_services.all %} - {% url 'dashboard:service' service.uuid as url %} + {% contextual_url 'dashboard:service' service.uuid as url %} {% include 'dashboard/includes/sidebar_portal.html' with label=service.name|truncatechars:20 url=url %} {% endfor %} diff --git a/shynet/dashboard/templates/dashboard/includes/date_range.html b/shynet/dashboard/templates/dashboard/includes/date_range.html index 1591f04..c4b0741 100644 --- a/shynet/dashboard/templates/dashboard/includes/date_range.html +++ b/shynet/dashboard/templates/dashboard/includes/date_range.html @@ -2,7 +2,7 @@ - +