From c84dac6b01adca2f2a4038437457ac744091c01b Mon Sep 17 00:00:00 2001 From: "R. Miles McCain" Date: Thu, 7 May 2020 17:44:39 -0400 Subject: [PATCH] Add referrer hiding support (closes #26) --- .../0006_service_hide_referrer_regex.py | 19 +++++++++ shynet/core/models.py | 40 ++++++++++++++++--- shynet/dashboard/forms.py | 4 ++ .../dashboard/includes/service_form.html | 3 +- 4 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 shynet/core/migrations/0006_service_hide_referrer_regex.py diff --git a/shynet/core/migrations/0006_service_hide_referrer_regex.py b/shynet/core/migrations/0006_service_hide_referrer_regex.py new file mode 100644 index 0000000..5aca80e --- /dev/null +++ b/shynet/core/migrations/0006_service_hide_referrer_regex.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.6 on 2020-05-07 21:23 + +import core.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0005_service_ignored_ips'), + ] + + operations = [ + migrations.AddField( + model_name='service', + name='hide_referrer_regex', + field=models.TextField(blank=True, default='', validators=[core.models._validate_regex]), + ), + ] diff --git a/shynet/core/models.py b/shynet/core/models.py index 775f1f3..74b8b64 100644 --- a/shynet/core/models.py +++ b/shynet/core/models.py @@ -1,5 +1,6 @@ import ipaddress import json +import re import uuid from django.apps import apps @@ -23,6 +24,13 @@ def _validate_network_list(networks: str): raise ValidationError(str(e)) +def _validate_regex(regex: str): + try: + re.compile(regex) + except re.error: + raise ValidationError(f"'{regex}' is not valid RegEx") + + def _parse_network_list(networks: str): if len(networks.strip()) == 0: return [] @@ -61,6 +69,9 @@ class Service(models.Model): ignored_ips = models.TextField( default="", blank=True, validators=[_validate_network_list] ) + hide_referrer_regex = models.TextField( + default="", blank=True, validators=[_validate_regex] + ) class Meta: ordering = ["name", "uuid"] @@ -71,6 +82,18 @@ class Service(models.Model): def get_ignored_networks(self): return _parse_network_list(self.ignored_ips) + def get_ignored_referrer_regex(self): + if len(self.hide_referrer_regex.strip()) == 0: + return re.compile(r".^") # matches nothing + else: + try: + return re.compile(self.hide_referrer_regex) + except re.error: + # Regexes are validated in the form, but this is an important + # fallback to prevent form validation and malformed source + # data from causing all service pages to error + return re.compile(r".^") + def get_daily_stats(self): return self.get_core_stats( start_time=timezone.now() - timezone.timedelta(days=1) @@ -117,12 +140,17 @@ class Service(models.Model): .order_by("-count") ) - referrers = ( - hits.filter(initial=True) - .values("referrer") - .annotate(count=models.Count("referrer")) - .order_by("-count") - ) + referrer_ignore = self.get_ignored_referrer_regex() + referrers = [ + referrer + for referrer in ( + hits.filter(initial=True) + .values("referrer") + .annotate(count=models.Count("referrer")) + .order_by("-count") + ) + if not referrer_ignore.match(referrer["referrer"]) + ] countries = ( sessions.values("country") diff --git a/shynet/dashboard/forms.py b/shynet/dashboard/forms.py index d0f4163..d402f1b 100644 --- a/shynet/dashboard/forms.py +++ b/shynet/dashboard/forms.py @@ -14,6 +14,7 @@ class ServiceForm(forms.ModelForm): "respect_dnt", "collect_ips", "ignored_ips", + "hide_referrer_regex", "origins", "collaborators", ] @@ -23,12 +24,14 @@ class ServiceForm(forms.ModelForm): "ignored_ips": forms.TextInput(), "respect_dnt": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]), "collect_ips": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]), + "hide_referrer_regex": forms.TextInput(), } labels = { "origins": "Allowed Hostnames", "respect_dnt": "Respect DNT", "collect_ips": "Collect IP addresses", "ignored_ips": "Ignored IP addresses", + "hide_referrer_regex": "Hide specific referrers", } help_texts = { "name": _("What should the service be called?"), @@ -39,6 +42,7 @@ class ServiceForm(forms.ModelForm): "respect_dnt": "Should visitors who have enabled Do Not Track be excluded from all data?", "collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected.", "ignored_ips": "A comma-separated list of IP addresses or IP ranges (IPv4 and IPv6) to exclude from tracking (e.g., '192.168.0.2, 127.0.0.1/32').", + "hide_referrer_regex": "Any referrers that match this RegEx will not be listed in the referrer summary. Sessions will still be tracked normally. No effect if left blank.", } collaborators = forms.CharField( diff --git a/shynet/dashboard/templates/dashboard/includes/service_form.html b/shynet/dashboard/templates/dashboard/includes/service_form.html index 2413dd7..0af3f60 100644 --- a/shynet/dashboard/templates/dashboard/includes/service_form.html +++ b/shynet/dashboard/templates/dashboard/includes/service_form.html @@ -4,11 +4,12 @@ {{form.link|a17t}} {{form.collaborators|a17t}} -
+
Advanced settings
{{form.respect_dnt|a17t}} {{form.collect_ips|a17t}} {{form.ignored_ips|a17t}} + {{form.hide_referrer_regex|a17t}} {{form.origins|a17t}}
\ No newline at end of file