Bump version, cleanup

Co-authored-by: Anthony Abeo <anthonyabeo@gmail.com>

.gitignore

@@ -109,6 +109,9 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+Vagrantfile
+.vagrant 
+ubuntu-xenial-16.04-cloudimg-console.log
 
 # Spyder project settings
 .spyderproject

GUIDE.md

@@ -114,8 +114,6 @@ A reverse proxy has many benefits. It can be used for DDoS protection, caching f
 
 Nginx is a self hosted, highly configurable webserver. Nginx can be configured to run as a reverse proxy on either the same machine or a remote machine.
 
-##### Set up
-
 > **These commands assume Ubuntu.** If you're installing Nginx on a different platform, the process will be different.
 
 0. Before starting, shut down your Docker containers (if any are running)
@@ -181,6 +179,7 @@ Here are solutions for some common issues. If your situation isn't described her
 #### Shynet isn't linking different pageviews from the same visitor into a single session!
 
 * Verify that your cache is properly configured. (See #2 above.) In multi-instance deployments, it's critical that all webservers are using the _same_ cache—so make sure you configure a Redis cache if you're using a non-default installation.
+* This can happen between Shynet restarts if you're not using an external cache provider (like Redis).
 
 #### I changed the `SHYNET_WHITELABEL`/`SHYNET_HOST` environment variable, but nothing happened!
 

Pipfile

@@ -23,6 +23,7 @@ psycopg2-binary = "*"
 redis = "*"
 django-redis-cache = "*"
 pycountry = "*"
+ipaddress = "*"
 
 [pipenv]
 allow_prereleases = true

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "2cd3e33ea333a40476d2030b6be66826e93c3a4de67032655061725835c92f09"
+            "sha256": "75fe7f0efbc05d6bc32c5ccaa08d3d619bf925682ca5eaffa728e74d0e8e5f66"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -59,18 +59,18 @@
         },
         "defusedxml": {
             "hashes": [
-                "sha256:6687150770438374ab581bb7a1b327a847dd9c5749e396102de3fad4e8a3ef93",
-                "sha256:f684034d135af4c6cbb949b8a4d2ed61634515257a67299e5f940fbaa34377f5"
+                "sha256:8ede8ba04cf5bf7999e1492fa77df545db83717f52c5eab625f97228ebd539bf",
+                "sha256:aa621655d72cdd30f57073893b96cd0c3831a85b08b8e4954531bdac47e3e8c8"
             ],
-            "version": "==0.6.0"
+            "version": "==0.7.0rc1"
         },
         "django": {
             "hashes": [
-                "sha256:642d8eceab321ca743ae71e0f985ff8fdca59f07aab3a9fb362c617d23e33a76",
-                "sha256:d4666c2edefa38c5ede0ec1655424c56dc47ceb04b6d8d62a7eac09db89545c1"
+                "sha256:051ba55d42daa3eeda3944a8e4df2bc96d4c62f94316dea217248a22563c3621",
+                "sha256:9aaa6a09678e1b8f0d98a948c56482eac3e3dd2ddbfb8de70a868135ef3b5e01"
             ],
             "index": "pypi",
-            "version": "==3.0.5"
+            "version": "==3.0.6"
         },
         "django-allauth": {
             "hashes": [
@@ -125,6 +125,14 @@
             ],
             "version": "==2.9"
         },
+        "ipaddress": {
+            "hashes": [
+                "sha256:6e0f4a39e66cb5bb9a137b00276a2eff74f93b71dcbdad6f10ff7df9d3557fcc",
+                "sha256:b7f8e0369580bb4a24d5ba1d7cc29660a4a6987763faf1d8a8046830e020e7e2"
+            ],
+            "index": "pypi",
+            "version": "==1.0.23"
+        },
         "kombu": {
             "hashes": [
                 "sha256:2d1cda774126a044d91a7ff5fa6d09edf99f46924ab332a810760fe6740e9b76",
@@ -134,9 +142,9 @@
         },
         "maxminddb": {
             "hashes": [
-                "sha256:d0ce131d901eb11669996b49a59f410efd3da2c6dbe2c0094fe2fef8d85b6336"
+                "sha256:f4d28823d9ca23323d113dc7af8db2087aa4f657fafc64ff8f7a8afda871425b"
             ],
-            "version": "==1.5.2"
+            "version": "==1.5.4"
         },
         "oauthlib": {
             "hashes": [
@@ -197,10 +205,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
-                "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
+                "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed",
+                "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048"
             ],
-            "version": "==2019.3"
+            "version": "==2020.1"
         },
         "pyyaml": {
             "hashes": [
@@ -221,11 +229,11 @@
         },
         "redis": {
             "hashes": [
-                "sha256:0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f",
-                "sha256:b205cffd05ebfd0a468db74f0eedbff8df1a7bfc47521516ade4692991bb0833"
+                "sha256:174101a3ce04560d716616290bb40e0a2af45d5844c8bd474c23fc5c52e7a46a",
+                "sha256:7378105cd8ea20c4edc49f028581e830c01ad5f00be851def0f4bc616a83cd89"
             ],
             "index": "pypi",
-            "version": "==3.4.1"
+            "version": "==3.5.0"
         },
         "requests": {
             "hashes": [
@@ -326,10 +334,10 @@
         },
         "click": {
             "hashes": [
-                "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc",
-                "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a"
+                "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a",
+                "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"
             ],
-            "version": "==7.1.1"
+            "version": "==7.1.2"
         },
         "pathspec": {
             "hashes": [
@@ -340,29 +348,29 @@
         },
         "regex": {
             "hashes": [
-                "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b",
-                "sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8",
-                "sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3",
-                "sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e",
-                "sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683",
-                "sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1",
-                "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142",
-                "sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3",
-                "sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468",
-                "sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e",
-                "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3",
-                "sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a",
-                "sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f",
-                "sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6",
-                "sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156",
-                "sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b",
-                "sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db",
-                "sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd",
-                "sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a",
-                "sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948",
-                "sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89"
+                "sha256:021a0ae4d2baeeb60a3014805a2096cb329bd6d9f30669b7ad0da51a9cb73349",
+                "sha256:04d6e948ef34d3eac133bedc0098364a9e635a7914f050edb61272d2ddae3608",
+                "sha256:099568b372bda492be09c4f291b398475587d49937c659824f891182df728cdf",
+                "sha256:0ff50843535593ee93acab662663cb2f52af8e31c3f525f630f1dc6156247938",
+                "sha256:1b17bf37c2aefc4cac8436971fe6ee52542ae4225cfc7762017f7e97a63ca998",
+                "sha256:1e2255ae938a36e9bd7db3b93618796d90c07e5f64dd6a6750c55f51f8b76918",
+                "sha256:2bc6a17a7fa8afd33c02d51b6f417fc271538990297167f68a98cae1c9e5c945",
+                "sha256:3ab5e41c4ed7cd4fa426c50add2892eb0f04ae4e73162155cd668257d02259dd",
+                "sha256:3b059e2476b327b9794c792c855aa05531a3f3044737e455d283c7539bd7534d",
+                "sha256:4df91094ced6f53e71f695c909d9bad1cca8761d96fd9f23db12245b5521136e",
+                "sha256:5493a02c1882d2acaaf17be81a3b65408ff541c922bfd002535c5f148aa29f74",
+                "sha256:5b741ecc3ad3e463d2ba32dce512b412c319993c1bb3d999be49e6092a769fb2",
+                "sha256:652ab4836cd5531d64a34403c00ada4077bb91112e8bcdae933e2eae232cf4a8",
+                "sha256:669a8d46764a09f198f2e91fc0d5acdac8e6b620376757a04682846ae28879c4",
+                "sha256:73a10404867b835f1b8a64253e4621908f0d71150eb4e97ab2e7e441b53e9451",
+                "sha256:7ce4a213a96d6c25eeae2f7d60d4dad89ac2b8134ec3e69db9bc522e2c0f9388",
+                "sha256:8127ca2bf9539d6a64d03686fd9e789e8c194fc19af49b69b081f8c7e6ecb1bc",
+                "sha256:b5b5b2e95f761a88d4c93691716ce01dc55f288a153face1654f868a8034f494",
+                "sha256:b7c9f65524ff06bf70c945cd8d8d1fd90853e27ccf86026af2afb4d9a63d06b1",
+                "sha256:f7f2f4226db6acd1da228adf433c5c3792858474e49d80668ea82ac87cf74a03",
+                "sha256:fa09da4af4e5b15c0e8b4986a083f3fd159302ea115a6cc0649cd163435538b8"
             ],
-            "version": "==2020.4.4"
+            "version": "==2020.5.7"
         },
         "toml": {
             "hashes": [

TEMPLATE.env

@@ -57,8 +57,10 @@ SHYNET_HOST=shynet.example.com
 # What you'd like to call your Shynet instance.
 SHYNET_WHITELABEL=My Shynet Instance
 
-# Redis and queue settings; not necessary for single-instance deployments.
+# Redis, queue, and parellization settings; not necessary for single-instance deployments.
 # Don't uncomment these unless you know what you are doing!
+# NUM_WORKERS=1
+# Make sure you set a REDIS_CACHE_LOCATION if you have more than one frontend worker/instance.
 # REDIS_CACHE_LOCATION=redis://redis.default.svc.cluster.local/0 
 # If CELERY_BROKER_URL is set, make sure CELERY_TASK_ALWAYS_EAGER is False and
 # that you have a separate queue consumer running somewhere via `celeryworker.sh`.

kubernetes/deployments.yml

@@ -16,12 +16,12 @@ spec:
         app: "shynet-webserver"
     spec:
       containers:
-      - name: "covideo-webserver"
+      - name: "shynet-webserver"
         image: "milesmcc/shynet:latest"
         imagePullPolicy: Always
         envFrom:
         - secretRef:
-            name: django-settings
+            name: shynet-settings
 ---
 apiVersion: "apps/v1"
 kind: "Deployment"
@@ -41,43 +41,43 @@ spec:
         app: "shynet-celeryworker"
     spec:
       containers:
-      - name: "covideo-celeryworker"
+      - name: "shynet-celeryworker"
         image: "milesmcc/shynet:latest"
         command: ["./celeryworker.sh"]
         imagePullPolicy: Always
         envFrom:
         - secretRef:
-            name: django-settings
+            name: shynet-settings
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: redis
+  name: shynet-redis
 spec:
   ports:
     - port: 6379
       name: redis
   clusterIP: None
   selector:
-    app: redis
+    app: shynet-redis
 ---
 apiVersion: apps/v1beta2
 kind: StatefulSet
 metadata:
-  name: redis
+  name: shynet-redis
 spec:
   selector:
     matchLabels:
-      app: redis
-  serviceName: redis
+      app: shynet-redis
+  serviceName: shynet-redis
   replicas: 1
   template:
     metadata:
       labels:
-        app: redis
+        app: shynet-redis
     spec:
       containers:
-        - name: redis
+        - name: shynet-redis
           image: redis:latest
           imagePullPolicy: Always
           ports:

kubernetes/secrets_template.yml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Secret
 metadata:
-    name: django-settings
+    name: shynet-settings
 type: Opaque
 stringData:
     # Django settings
@@ -12,8 +12,8 @@ stringData:
     TIME_ZONE: "America/New_York"
 
     # Redis configuration (if you use the default Kubernetes config, this will work)
-    REDIS_CACHE_LOCATION: "redis://redis.default.svc.cluster.local/0"
-    CELERY_BROKER_URL: "redis://redis.default.svc.cluster.local/1"
+    REDIS_CACHE_LOCATION: "redis://shynet-redis.default.svc.cluster.local/0"
+    CELERY_BROKER_URL: "redis://shynet-redis.default.svc.cluster.local/1"
 
     # PostgreSQL settings
     DB_NAME: ""

shynet/analytics/migrations/0003_auto_20200502_1227.py

@@ -6,13 +6,13 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('analytics', '0002_auto_20200415_1742'),
+        ("analytics", "0002_auto_20200415_1742"),
     ]
 
     operations = [
         migrations.AlterField(
-            model_name='session',
-            name='ip',
+            model_name="session",
+            name="ip",
             field=models.GenericIPAddressField(db_index=True, null=True),
         ),
     ]

shynet/analytics/tasks.py

@@ -1,9 +1,10 @@
+import ipaddress
 import json
 import logging
+from hashlib import sha1
 
 import geoip2.database
 import user_agents
-from hashlib import sha1
 from celery import shared_task
 from django.conf import settings
 from django.core.cache import cache
@@ -60,6 +61,14 @@ def ingress_request(
         if dnt and service.respect_dnt:
             return
 
+        try:
+            remote_ip = ipaddress.ip_network(ip)
+            for ignored_network in service.get_ignored_networks():
+                if ignored_network.supernet_of(remote_ip):
+                    return
+        except ValueError as e:
+            log.exception(e)
+
         # Validate payload
         if payload.get("loadTime", 1) <= 0:
             payload["loadTime"] = None

shynet/analytics/views/ingress.py

@@ -4,7 +4,7 @@ import json
 from django.conf import settings
 from django.core.cache import cache
 from django.core.exceptions import ValidationError
-from django.http import HttpResponse, Http404, HttpResponseBadRequest
+from django.http import Http404, HttpResponse, HttpResponseBadRequest
 from django.shortcuts import render, reverse
 from django.utils import timezone
 from django.utils.decorators import method_decorator
@@ -36,6 +36,7 @@ def ingress(request, service_uuid, identifier, tracker, payload):
         identifier=identifier,
     )
 
+
 class ValidateServiceOriginsMixin:
     def dispatch(self, request, *args, **kwargs):
         try:

shynet/core/management/commands/startup_checks.py

@@ -3,12 +3,11 @@ import uuid
 
 from django.conf import settings
 from django.contrib.sites.models import Site
-from django.core.management.base import BaseCommand, CommandError
-from django.utils.crypto import get_random_string
-
-from django.db import DEFAULT_DB_ALIAS, connections
-from django.db.utils import OperationalError, ConnectionHandler
 from django.core.exceptions import ImproperlyConfigured
+from django.core.management.base import BaseCommand, CommandError
+from django.db import DEFAULT_DB_ALIAS, connections
+from django.db.utils import ConnectionHandler, OperationalError
+from django.utils.crypto import get_random_string
 
 from core.models import User
 
@@ -18,6 +17,7 @@ class Command(BaseCommand):
 
     def check_migrations(self):
         from django.db.migrations.executor import MigrationExecutor
+
         try:
             executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS])
         except OperationalError:
@@ -26,24 +26,31 @@ class Command(BaseCommand):
         except ImproperlyConfigured:
             # No databases are configured (or the dummy one)
             return True
-        
+
         if executor.migration_plan(executor.loader.graph.leaf_nodes()):
             return True
-        
+
         return False
-    
-    
+
     def handle(self, *args, **options):
         migration = self.check_migrations()
 
         admin, hostname, whitelabel = [True] * 3
         if not migration:
             admin = not User.objects.all().exists()
-            hostname = not Site.objects.filter(domain__isnull=False).exclude(domain__exact="").exclude(domain__exact="example.com").exists()
-            whitelabel = not Site.objects.filter(name__isnull=False).exclude(name__exact="").exclude(name__exact="example.com").exists()
+            hostname = (
+                not Site.objects.filter(domain__isnull=False)
+                .exclude(domain__exact="")
+                .exclude(domain__exact="example.com")
+                .exists()
+            )
+            whitelabel = (
+                not Site.objects.filter(name__isnull=False)
+                .exclude(name__exact="")
+                .exclude(name__exact="example.com")
+                .exists()
+            )
 
         self.stdout.write(
-            self.style.SUCCESS(
-                f"{migration} {admin} {hostname} {whitelabel}"
-            )
+            self.style.SUCCESS(f"{migration} {admin} {hostname} {whitelabel}")
         )

shynet/core/migrations/0004_service_collect_ips.py

@@ -6,13 +6,13 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('core', '0003_service_respect_dnt'),
+        ("core", "0003_service_respect_dnt"),
     ]
 
     operations = [
         migrations.AddField(
-            model_name='service',
-            name='collect_ips',
+            model_name="service",
+            name="collect_ips",
             field=models.BooleanField(default=True),
         ),
     ]

shynet/core/migrations/0005_service_ignored_ips.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.0.6 on 2020-05-07 20:28
+
+from django.db import migrations, models
+
+import core.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0004_service_collect_ips"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="service",
+            name="ignored_ips",
+            field=models.TextField(
+                blank=True, default="", validators=[core.models._validate_network_list]
+            ),
+        ),
+    ]

shynet/core/migrations/0006_service_hide_referrer_regex.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.0.6 on 2020-05-07 21:23
+
+from django.db import migrations, models
+
+import core.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0005_service_ignored_ips"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="service",
+            name="hide_referrer_regex",
+            field=models.TextField(
+                blank=True, default="", validators=[core.models._validate_regex]
+            ),
+        ),
+    ]

shynet/core/models.py

@@ -1,8 +1,11 @@
+import ipaddress
 import json
+import re
 import uuid
 
 from django.apps import apps
 from django.contrib.auth.models import AbstractUser
+from django.core.exceptions import ValidationError
 from django.db import models
 from django.db.models.functions import TruncDate
 from django.db.utils import NotSupportedError
@@ -14,6 +17,26 @@ def _default_uuid():
     return str(uuid.uuid4())
 
 
+def _validate_network_list(networks: str):
+    try:
+        _parse_network_list(networks)
+    except ValueError as e:
+        raise ValidationError(str(e))
+
+
+def _validate_regex(regex: str):
+    try:
+        re.compile(regex)
+    except re.error:
+        raise ValidationError(f"'{regex}' is not valid RegEx")
+
+
+def _parse_network_list(networks: str):
+    if len(networks.strip()) == 0:
+        return []
+    return [ipaddress.ip_network(network.strip()) for network in networks.split(",")]
+
+
 class User(AbstractUser):
     username = models.TextField(default=_default_uuid, unique=True)
     email = models.EmailField(unique=True)
@@ -43,6 +66,12 @@ class Service(models.Model):
     )
     respect_dnt = models.BooleanField(default=True)
     collect_ips = models.BooleanField(default=True)
+    ignored_ips = models.TextField(
+        default="", blank=True, validators=[_validate_network_list]
+    )
+    hide_referrer_regex = models.TextField(
+        default="", blank=True, validators=[_validate_regex]
+    )
 
     class Meta:
         ordering = ["name", "uuid"]
@@ -50,6 +79,21 @@ class Service(models.Model):
     def __str__(self):
         return self.name
 
+    def get_ignored_networks(self):
+        return _parse_network_list(self.ignored_ips)
+
+    def get_ignored_referrer_regex(self):
+        if len(self.hide_referrer_regex.strip()) == 0:
+            return re.compile(r".^")  # matches nothing
+        else:
+            try:
+                return re.compile(self.hide_referrer_regex)
+            except re.error:
+                # Regexes are validated in the form, but this is an important
+                # fallback to prevent form validation and malformed source
+                # data from causing all service pages to error
+                return re.compile(r".^")
+
     def get_daily_stats(self):
         return self.get_core_stats(
             start_time=timezone.now() - timezone.timedelta(days=1)
@@ -96,12 +140,17 @@ class Service(models.Model):
             .order_by("-count")
         )
 
-        referrers = (
-            hits.filter(initial=True)
-            .values("referrer")
-            .annotate(count=models.Count("referrer"))
-            .order_by("-count")
-        )
+        referrer_ignore = self.get_ignored_referrer_regex()
+        referrers = [
+            referrer
+            for referrer in (
+                hits.filter(initial=True)
+                .values("referrer")
+                .annotate(count=models.Count("referrer"))
+                .order_by("-count")
+            )
+            if not referrer_ignore.match(referrer["referrer"])
+        ]
 
         countries = (
             sessions.values("country")
@@ -131,12 +180,6 @@ class Service(models.Model):
             .order_by("-count")
         )
 
-        device_types = (
-            sessions.values("device_type")
-            .annotate(count=models.Count("device_type"))
-            .order_by("-count")
-        )
-
         avg_load_time = hits.aggregate(load_time__avg=models.Avg("load_time"))[
             "load_time__avg"
         ]

shynet/dashboard/forms.py

@@ -8,17 +8,30 @@ from core.models import Service, User
 class ServiceForm(forms.ModelForm):
     class Meta:
         model = Service
-        fields = ["name", "link", "respect_dnt", "collect_ips", "origins", "collaborators"]
+        fields = [
+            "name",
+            "link",
+            "respect_dnt",
+            "collect_ips",
+            "ignored_ips",
+            "hide_referrer_regex",
+            "origins",
+            "collaborators",
+        ]
         widgets = {
             "name": forms.TextInput(),
             "origins": forms.TextInput(),
+            "ignored_ips": forms.TextInput(),
             "respect_dnt": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
             "collect_ips": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
+            "hide_referrer_regex": forms.TextInput(),
         }
         labels = {
             "origins": "Allowed Hostnames",
             "respect_dnt": "Respect DNT",
-            "collect_ips": "Collect IP addresses"
+            "collect_ips": "Collect IP addresses",
+            "ignored_ips": "Ignored IP addresses",
+            "hide_referrer_regex": "Hide specific referrers",
         }
         help_texts = {
             "name": _("What should the service be called?"),
@@ -27,7 +40,9 @@ class ServiceForm(forms.ModelForm):
                 "At what hostnames does the service operate? This sets CORS headers, so use '*' if you're not sure (or don't care)."
             ),
             "respect_dnt": "Should visitors who have enabled <a href='https://en.wikipedia.org/wiki/Do_Not_Track'>Do Not Track</a> be excluded from all data?",
-            "collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected."
+            "collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected.",
+            "ignored_ips": "A comma-separated list of IP addresses or IP ranges (IPv4 and IPv6) to exclude from tracking (e.g., '192.168.0.2, 127.0.0.1/32').",
+            "hide_referrer_regex": "Any referrers that match this <a href='https://regexr.com/'>RegEx</a> will not be listed in the referrer summary. Sessions will still be tracked normally. No effect if left blank.",
         }
 
     collaborators = forms.CharField(

shynet/dashboard/templates/dashboard/includes/service_form.html

@@ -4,10 +4,12 @@
 {{form.link|a17t}}
 {{form.collaborators|a17t}}
 
-<details class="p-4 border rounded">
+<details {% if form.errors %}open{% endif %}>
     <summary class="cursor-pointer text-sm">Advanced settings</summary>
     <hr class="sep h-4">
     {{form.respect_dnt|a17t}}
     {{form.collect_ips|a17t}}
+    {{form.ignored_ips|a17t}}
+    {{form.hide_referrer_regex|a17t}}
     {{form.origins|a17t}}
 </details>

shynet/dashboard/templates/dashboard/pages/service_update.html

@@ -14,8 +14,8 @@
     <p>Place the following snippet at the end of the <code>&lt;body&gt;</code> tag on any page you'd like to track.</p>
     <div class="card ~neutral !high font-mono text-sm">
         {% filter force_escape %}<noscript><img
-                src="//{{request.site.domain}}{% url 'ingress:endpoint_pixel' object.uuid %}"></noscript>
-        <script src="//{{request.site.domain}}{% url 'ingress:endpoint_script' object.uuid %}"></script>
+                src="{{script_protocol}}{{request.site.domain}}{% url 'ingress:endpoint_pixel' object.uuid %}"></noscript>
+        <script src="{{script_protocol}}{{request.site.domain}}{% url 'ingress:endpoint_script' object.uuid %}"></script>
         {% endfilter %}
     </div>
     <hr class="sep h-4">

shynet/dashboard/templatetags/helpers.py

@@ -81,11 +81,11 @@ def percent_change_display(start, end):
 
     return SafeString(direction + pct_change)
 
+
 @register.inclusion_tag("dashboard/includes/sidebar_footer.html")
 def sidebar_footer():
-    return {
-        "version": settings.VERSION
-    }
+    return {"version": settings.VERSION}
+
 
 @register.inclusion_tag("dashboard/includes/stat_comparison.html")
 def compare(

shynet/dashboard/views.py

@@ -1,3 +1,4 @@
+from django.conf import settings
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.contrib.messages.views import SuccessMessageMixin
 from django.core.cache import cache
@@ -85,6 +86,11 @@ class ServiceUpdateView(
         )
         return resp
 
+    def get_context_data(self, *args, **kwargs):
+        data = super().get_context_data(*args, **kwargs)
+        data["script_protocol"] = "https://" if settings.SCRIPT_USE_HTTPS else "http://"
+        return data
+
 
 class ServiceDeleteView(
     LoginRequiredMixin, PermissionRequiredMixin, SuccessMessageMixin, DeleteView

shynet/shynet/settings.py

@@ -14,7 +14,7 @@ import os
 from django.contrib.messages import constants as messages
 
 # Increment on new releases
-VERSION = "v0.3.1"
+VERSION = "v0.4.0"
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

shynet/ssl.webserver.sh

@@ -4,7 +4,7 @@
 echo Launching Shynet web server...
 exec gunicorn shynet.wsgi:application \
     --bind 0.0.0.0:8080 \
-    --workers 3 \
+    --workers ${NUM_WORKERS:-1} \
     --timeout 100 \
     --certfile=cert.pem \
     --keyfile=privkey.pem

shynet/webserver.sh

@@ -3,5 +3,5 @@
 echo Launching Shynet web server...
 exec gunicorn shynet.wsgi:application \
     --bind 0.0.0.0:8080 \
-    --workers 3 \
+    --workers ${NUM_WORKERS:-1} \
     --timeout 100