Add referrer hiding support (closes #26)

shynet/core/migrations/0006_service_hide_referrer_regex.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.0.6 on 2020-05-07 21:23
+
+import core.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0005_service_ignored_ips'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='service',
+            name='hide_referrer_regex',
+            field=models.TextField(blank=True, default='', validators=[core.models._validate_regex]),
+        ),
+    ]

shynet/core/models.py

@@ -1,5 +1,6 @@
 import ipaddress
 import json
+import re
 import uuid
 
 from django.apps import apps
@@ -23,6 +24,13 @@ def _validate_network_list(networks: str):
         raise ValidationError(str(e))
 
 
+def _validate_regex(regex: str):
+    try:
+        re.compile(regex)
+    except re.error:
+        raise ValidationError(f"'{regex}' is not valid RegEx")
+
+
 def _parse_network_list(networks: str):
     if len(networks.strip()) == 0:
         return []
@@ -61,6 +69,9 @@ class Service(models.Model):
     ignored_ips = models.TextField(
         default="", blank=True, validators=[_validate_network_list]
     )
+    hide_referrer_regex = models.TextField(
+        default="", blank=True, validators=[_validate_regex]
+    )
 
     class Meta:
         ordering = ["name", "uuid"]
@@ -71,6 +82,18 @@ class Service(models.Model):
     def get_ignored_networks(self):
         return _parse_network_list(self.ignored_ips)
 
+    def get_ignored_referrer_regex(self):
+        if len(self.hide_referrer_regex.strip()) == 0:
+            return re.compile(r".^") # matches nothing
+        else:
+            try:
+                return re.compile(self.hide_referrer_regex)
+            except re.error:
+                # Regexes are validated in the form, but this is an important
+                # fallback to prevent form validation and malformed source
+                # data from causing all service pages to error
+                return re.compile(r".^")
+
     def get_daily_stats(self):
         return self.get_core_stats(
             start_time=timezone.now() - timezone.timedelta(days=1)
@@ -117,12 +140,17 @@ class Service(models.Model):
             .order_by("-count")
         )
 
-        referrers = (
+        referrer_ignore = self.get_ignored_referrer_regex()
-            hits.filter(initial=True)
+        referrers = [
-            .values("referrer")
+            referrer
-            .annotate(count=models.Count("referrer"))
+            for referrer in (
-            .order_by("-count")
+                hits.filter(initial=True)
-        )
+                .values("referrer")
+                .annotate(count=models.Count("referrer"))
+                .order_by("-count")
+            )
+            if not referrer_ignore.match(referrer["referrer"])
+        ]
 
         countries = (
             sessions.values("country")

shynet/dashboard/forms.py

@@ -14,6 +14,7 @@ class ServiceForm(forms.ModelForm):
             "respect_dnt",
             "collect_ips",
             "ignored_ips",
+            "hide_referrer_regex",
             "origins",
             "collaborators",
         ]
@@ -23,12 +24,14 @@ class ServiceForm(forms.ModelForm):
             "ignored_ips": forms.TextInput(),
             "respect_dnt": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
             "collect_ips": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
+            "hide_referrer_regex": forms.TextInput(),
         }
         labels = {
             "origins": "Allowed Hostnames",
             "respect_dnt": "Respect DNT",
             "collect_ips": "Collect IP addresses",
             "ignored_ips": "Ignored IP addresses",
+            "hide_referrer_regex": "Hide specific referrers",
         }
         help_texts = {
             "name": _("What should the service be called?"),
@@ -39,6 +42,7 @@ class ServiceForm(forms.ModelForm):
             "respect_dnt": "Should visitors who have enabled <a href='https://en.wikipedia.org/wiki/Do_Not_Track'>Do Not Track</a> be excluded from all data?",
             "collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected.",
             "ignored_ips": "A comma-separated list of IP addresses or IP ranges (IPv4 and IPv6) to exclude from tracking (e.g., '192.168.0.2, 127.0.0.1/32').",
+            "hide_referrer_regex": "Any referrers that match this <a href='https://regexr.com/'>RegEx</a> will not be listed in the referrer summary. Sessions will still be tracked normally. No effect if left blank.",
         }
 
     collaborators = forms.CharField(

shynet/dashboard/templates/dashboard/includes/service_form.html

@@ -4,11 +4,12 @@
 {{form.link|a17t}}
 {{form.collaborators|a17t}}
 
-<details class="p-4 border rounded" {% if form.errors %}open{% endif %}>
+<details {% if form.errors %}open{% endif %}>
     <summary class="cursor-pointer text-sm">Advanced settings</summary>
     <hr class="sep h-4">
     {{form.respect_dnt|a17t}}
     {{form.collect_ips|a17t}}
     {{form.ignored_ips|a17t}}
+    {{form.hide_referrer_regex|a17t}}
     {{form.origins|a17t}}
 </details>