Add option to ignore robots

shynet/analytics/tasks.py

@@ -110,6 +110,8 @@ def ingress_request(
                 device_type = "TABLET"
             elif ua.is_pc:
                 device_type = "DESKTOP"
+            if device_type == "ROBOT" and service.ignore_robots:
+                return
             session = Session.objects.create(
                 service=service,
                 ip=ip if service.collect_ips else None,

shynet/core/migrations/0007_service_ignore_robots.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0.6 on 2020-06-15 16:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0006_service_hide_referrer_regex'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='service',
+            name='ignore_robots',
+            field=models.BooleanField(default=False),
+        )
+    ]

shynet/core/models.py

@@ -65,6 +65,7 @@ class Service(models.Model):
         max_length=2, choices=SERVICE_STATUSES, default=ACTIVE, db_index=True
     )
     respect_dnt = models.BooleanField(default=True)
+    ignore_robots = models.BooleanField(default=False)
     collect_ips = models.BooleanField(default=True)
     ignored_ips = models.TextField(
         default="", blank=True, validators=[_validate_network_list]

shynet/dashboard/forms.py

@@ -14,6 +14,7 @@ class ServiceForm(forms.ModelForm):
             "respect_dnt",
             "collect_ips",
             "ignored_ips",
+            "ignore_robots",
             "hide_referrer_regex",
             "origins",
             "collaborators",
@@ -24,6 +25,7 @@ class ServiceForm(forms.ModelForm):
             "ignored_ips": forms.TextInput(),
             "respect_dnt": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
             "collect_ips": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
+            "ignore_robots": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
             "hide_referrer_regex": forms.TextInput(),
         }
         labels = {
@@ -31,6 +33,7 @@ class ServiceForm(forms.ModelForm):
             "respect_dnt": "Respect DNT",
             "collect_ips": "Collect IP addresses",
             "ignored_ips": "Ignored IP addresses",
+            "ignore_robots": "Ignore robots",
             "hide_referrer_regex": "Hide specific referrers",
         }
         help_texts = {
@@ -42,6 +45,7 @@ class ServiceForm(forms.ModelForm):
             "respect_dnt": "Should visitors who have enabled <a href='https://en.wikipedia.org/wiki/Do_Not_Track'>Do Not Track</a> be excluded from all data?",
             "collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected.",
             "ignored_ips": "A comma-separated list of IP addresses or IP ranges (IPv4 and IPv6) to exclude from tracking (e.g., '192.168.0.2, 127.0.0.1/32').",
+            "ignore_robots": "Should sessions generated by bots be excluded from tracking?",
             "hide_referrer_regex": "Any referrers that match this <a href='https://regexr.com/'>RegEx</a> will not be listed in the referrer summary. Sessions will still be tracked normally. No effect if left blank.",
         }
 

shynet/dashboard/templates/dashboard/includes/service_form.html

@@ -10,6 +10,7 @@
     {{form.respect_dnt|a17t}}
     {{form.collect_ips|a17t}}
     {{form.ignored_ips|a17t}}
+    {{form.ignore_robots|a17t}}
     {{form.hide_referrer_regex|a17t}}
     {{form.origins|a17t}}
 </details>