From a766c1eaa261abf0c2e203cf455d798348035446 Mon Sep 17 00:00:00 2001 From: "R. Miles McCain" Date: Thu, 7 May 2020 16:53:03 -0400 Subject: [PATCH] Add ip address exclusion support (closes #22) Co-authored-by: Anthony Abeo --- Pipfile | 1 + Pipfile.lock | 88 ++++++++++--------- .../migrations/0003_auto_20200502_1227.py | 6 +- shynet/analytics/tasks.py | 11 ++- shynet/analytics/views/ingress.py | 3 +- .../management/commands/startup_checks.py | 35 +++++--- .../migrations/0004_service_collect_ips.py | 6 +- .../migrations/0005_service_ignored_ips.py | 22 +++++ shynet/core/models.py | 21 +++++ shynet/dashboard/forms.py | 17 +++- .../dashboard/includes/service_form.html | 3 +- shynet/dashboard/templatetags/helpers.py | 6 +- 12 files changed, 150 insertions(+), 69 deletions(-) create mode 100644 shynet/core/migrations/0005_service_ignored_ips.py diff --git a/Pipfile b/Pipfile index ddfa874..56e2d3d 100644 --- a/Pipfile +++ b/Pipfile @@ -23,6 +23,7 @@ psycopg2-binary = "*" redis = "*" django-redis-cache = "*" pycountry = "*" +ipaddress = "*" [pipenv] allow_prereleases = true diff --git a/Pipfile.lock b/Pipfile.lock index c913d2f..132cc59 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "2cd3e33ea333a40476d2030b6be66826e93c3a4de67032655061725835c92f09" + "sha256": "75fe7f0efbc05d6bc32c5ccaa08d3d619bf925682ca5eaffa728e74d0e8e5f66" }, "pipfile-spec": 6, "requires": {}, @@ -59,18 +59,18 @@ }, "defusedxml": { "hashes": [ - "sha256:6687150770438374ab581bb7a1b327a847dd9c5749e396102de3fad4e8a3ef93", - "sha256:f684034d135af4c6cbb949b8a4d2ed61634515257a67299e5f940fbaa34377f5" + "sha256:8ede8ba04cf5bf7999e1492fa77df545db83717f52c5eab625f97228ebd539bf", + "sha256:aa621655d72cdd30f57073893b96cd0c3831a85b08b8e4954531bdac47e3e8c8" ], - "version": "==0.6.0" + "version": "==0.7.0rc1" }, "django": { "hashes": [ - "sha256:642d8eceab321ca743ae71e0f985ff8fdca59f07aab3a9fb362c617d23e33a76", - "sha256:d4666c2edefa38c5ede0ec1655424c56dc47ceb04b6d8d62a7eac09db89545c1" + "sha256:051ba55d42daa3eeda3944a8e4df2bc96d4c62f94316dea217248a22563c3621", + "sha256:9aaa6a09678e1b8f0d98a948c56482eac3e3dd2ddbfb8de70a868135ef3b5e01" ], "index": "pypi", - "version": "==3.0.5" + "version": "==3.0.6" }, "django-allauth": { "hashes": [ @@ -125,6 +125,14 @@ ], "version": "==2.9" }, + "ipaddress": { + "hashes": [ + "sha256:6e0f4a39e66cb5bb9a137b00276a2eff74f93b71dcbdad6f10ff7df9d3557fcc", + "sha256:b7f8e0369580bb4a24d5ba1d7cc29660a4a6987763faf1d8a8046830e020e7e2" + ], + "index": "pypi", + "version": "==1.0.23" + }, "kombu": { "hashes": [ "sha256:2d1cda774126a044d91a7ff5fa6d09edf99f46924ab332a810760fe6740e9b76", @@ -134,9 +142,9 @@ }, "maxminddb": { "hashes": [ - "sha256:d0ce131d901eb11669996b49a59f410efd3da2c6dbe2c0094fe2fef8d85b6336" + "sha256:f4d28823d9ca23323d113dc7af8db2087aa4f657fafc64ff8f7a8afda871425b" ], - "version": "==1.5.2" + "version": "==1.5.4" }, "oauthlib": { "hashes": [ @@ -197,10 +205,10 @@ }, "pytz": { "hashes": [ - "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", - "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" + "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", + "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" ], - "version": "==2019.3" + "version": "==2020.1" }, "pyyaml": { "hashes": [ @@ -221,11 +229,11 @@ }, "redis": { "hashes": [ - "sha256:0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f", - "sha256:b205cffd05ebfd0a468db74f0eedbff8df1a7bfc47521516ade4692991bb0833" + "sha256:174101a3ce04560d716616290bb40e0a2af45d5844c8bd474c23fc5c52e7a46a", + "sha256:7378105cd8ea20c4edc49f028581e830c01ad5f00be851def0f4bc616a83cd89" ], "index": "pypi", - "version": "==3.4.1" + "version": "==3.5.0" }, "requests": { "hashes": [ @@ -326,10 +334,10 @@ }, "click": { "hashes": [ - "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc", - "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a" + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" ], - "version": "==7.1.1" + "version": "==7.1.2" }, "pathspec": { "hashes": [ @@ -340,29 +348,29 @@ }, "regex": { "hashes": [ - "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b", - "sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8", - "sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3", - "sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e", - "sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683", - "sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1", - "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142", - "sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3", - "sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468", - "sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e", - "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3", - "sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a", - "sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f", - "sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6", - "sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156", - "sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b", - "sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db", - "sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd", - "sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a", - "sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948", - "sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89" + "sha256:021a0ae4d2baeeb60a3014805a2096cb329bd6d9f30669b7ad0da51a9cb73349", + "sha256:04d6e948ef34d3eac133bedc0098364a9e635a7914f050edb61272d2ddae3608", + "sha256:099568b372bda492be09c4f291b398475587d49937c659824f891182df728cdf", + "sha256:0ff50843535593ee93acab662663cb2f52af8e31c3f525f630f1dc6156247938", + "sha256:1b17bf37c2aefc4cac8436971fe6ee52542ae4225cfc7762017f7e97a63ca998", + "sha256:1e2255ae938a36e9bd7db3b93618796d90c07e5f64dd6a6750c55f51f8b76918", + "sha256:2bc6a17a7fa8afd33c02d51b6f417fc271538990297167f68a98cae1c9e5c945", + "sha256:3ab5e41c4ed7cd4fa426c50add2892eb0f04ae4e73162155cd668257d02259dd", + "sha256:3b059e2476b327b9794c792c855aa05531a3f3044737e455d283c7539bd7534d", + "sha256:4df91094ced6f53e71f695c909d9bad1cca8761d96fd9f23db12245b5521136e", + "sha256:5493a02c1882d2acaaf17be81a3b65408ff541c922bfd002535c5f148aa29f74", + "sha256:5b741ecc3ad3e463d2ba32dce512b412c319993c1bb3d999be49e6092a769fb2", + "sha256:652ab4836cd5531d64a34403c00ada4077bb91112e8bcdae933e2eae232cf4a8", + "sha256:669a8d46764a09f198f2e91fc0d5acdac8e6b620376757a04682846ae28879c4", + "sha256:73a10404867b835f1b8a64253e4621908f0d71150eb4e97ab2e7e441b53e9451", + "sha256:7ce4a213a96d6c25eeae2f7d60d4dad89ac2b8134ec3e69db9bc522e2c0f9388", + "sha256:8127ca2bf9539d6a64d03686fd9e789e8c194fc19af49b69b081f8c7e6ecb1bc", + "sha256:b5b5b2e95f761a88d4c93691716ce01dc55f288a153face1654f868a8034f494", + "sha256:b7c9f65524ff06bf70c945cd8d8d1fd90853e27ccf86026af2afb4d9a63d06b1", + "sha256:f7f2f4226db6acd1da228adf433c5c3792858474e49d80668ea82ac87cf74a03", + "sha256:fa09da4af4e5b15c0e8b4986a083f3fd159302ea115a6cc0649cd163435538b8" ], - "version": "==2020.4.4" + "version": "==2020.5.7" }, "toml": { "hashes": [ diff --git a/shynet/analytics/migrations/0003_auto_20200502_1227.py b/shynet/analytics/migrations/0003_auto_20200502_1227.py index 7c71b33..1fe6a65 100644 --- a/shynet/analytics/migrations/0003_auto_20200502_1227.py +++ b/shynet/analytics/migrations/0003_auto_20200502_1227.py @@ -6,13 +6,13 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('analytics', '0002_auto_20200415_1742'), + ("analytics", "0002_auto_20200415_1742"), ] operations = [ migrations.AlterField( - model_name='session', - name='ip', + model_name="session", + name="ip", field=models.GenericIPAddressField(db_index=True, null=True), ), ] diff --git a/shynet/analytics/tasks.py b/shynet/analytics/tasks.py index 7da239b..bd2c6a0 100644 --- a/shynet/analytics/tasks.py +++ b/shynet/analytics/tasks.py @@ -1,9 +1,10 @@ +import ipaddress import json import logging +from hashlib import sha1 import geoip2.database import user_agents -from hashlib import sha1 from celery import shared_task from django.conf import settings from django.core.cache import cache @@ -60,6 +61,14 @@ def ingress_request( if dnt and service.respect_dnt: return + try: + remote_ip = ipaddress.ip_network(ip) + for ignored_network in service.get_ignored_networks(): + if ignored_network.supernet_of(remote_ip): + return + except ValueError as e: + log.exception(e) + # Validate payload if payload.get("loadTime", 1) <= 0: payload["loadTime"] = None diff --git a/shynet/analytics/views/ingress.py b/shynet/analytics/views/ingress.py index eb09cc0..ef39b01 100644 --- a/shynet/analytics/views/ingress.py +++ b/shynet/analytics/views/ingress.py @@ -4,7 +4,7 @@ import json from django.conf import settings from django.core.cache import cache from django.core.exceptions import ValidationError -from django.http import HttpResponse, Http404, HttpResponseBadRequest +from django.http import Http404, HttpResponse, HttpResponseBadRequest from django.shortcuts import render, reverse from django.utils import timezone from django.utils.decorators import method_decorator @@ -36,6 +36,7 @@ def ingress(request, service_uuid, identifier, tracker, payload): identifier=identifier, ) + class ValidateServiceOriginsMixin: def dispatch(self, request, *args, **kwargs): try: diff --git a/shynet/core/management/commands/startup_checks.py b/shynet/core/management/commands/startup_checks.py index 15da2d3..b4cc9d8 100644 --- a/shynet/core/management/commands/startup_checks.py +++ b/shynet/core/management/commands/startup_checks.py @@ -3,12 +3,11 @@ import uuid from django.conf import settings from django.contrib.sites.models import Site -from django.core.management.base import BaseCommand, CommandError -from django.utils.crypto import get_random_string - -from django.db import DEFAULT_DB_ALIAS, connections -from django.db.utils import OperationalError, ConnectionHandler from django.core.exceptions import ImproperlyConfigured +from django.core.management.base import BaseCommand, CommandError +from django.db import DEFAULT_DB_ALIAS, connections +from django.db.utils import ConnectionHandler, OperationalError +from django.utils.crypto import get_random_string from core.models import User @@ -18,6 +17,7 @@ class Command(BaseCommand): def check_migrations(self): from django.db.migrations.executor import MigrationExecutor + try: executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS]) except OperationalError: @@ -26,24 +26,31 @@ class Command(BaseCommand): except ImproperlyConfigured: # No databases are configured (or the dummy one) return True - + if executor.migration_plan(executor.loader.graph.leaf_nodes()): return True - + return False - - + def handle(self, *args, **options): migration = self.check_migrations() admin, hostname, whitelabel = [True] * 3 if not migration: admin = not User.objects.all().exists() - hostname = not Site.objects.filter(domain__isnull=False).exclude(domain__exact="").exclude(domain__exact="example.com").exists() - whitelabel = not Site.objects.filter(name__isnull=False).exclude(name__exact="").exclude(name__exact="example.com").exists() + hostname = ( + not Site.objects.filter(domain__isnull=False) + .exclude(domain__exact="") + .exclude(domain__exact="example.com") + .exists() + ) + whitelabel = ( + not Site.objects.filter(name__isnull=False) + .exclude(name__exact="") + .exclude(name__exact="example.com") + .exists() + ) self.stdout.write( - self.style.SUCCESS( - f"{migration} {admin} {hostname} {whitelabel}" - ) + self.style.SUCCESS(f"{migration} {admin} {hostname} {whitelabel}") ) diff --git a/shynet/core/migrations/0004_service_collect_ips.py b/shynet/core/migrations/0004_service_collect_ips.py index f5f48fb..c38beb3 100644 --- a/shynet/core/migrations/0004_service_collect_ips.py +++ b/shynet/core/migrations/0004_service_collect_ips.py @@ -6,13 +6,13 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('core', '0003_service_respect_dnt'), + ("core", "0003_service_respect_dnt"), ] operations = [ migrations.AddField( - model_name='service', - name='collect_ips', + model_name="service", + name="collect_ips", field=models.BooleanField(default=True), ), ] diff --git a/shynet/core/migrations/0005_service_ignored_ips.py b/shynet/core/migrations/0005_service_ignored_ips.py new file mode 100644 index 0000000..9cb5440 --- /dev/null +++ b/shynet/core/migrations/0005_service_ignored_ips.py @@ -0,0 +1,22 @@ +# Generated by Django 3.0.6 on 2020-05-07 20:28 + +from django.db import migrations, models + +import core.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0004_service_collect_ips"), + ] + + operations = [ + migrations.AddField( + model_name="service", + name="ignored_ips", + field=models.TextField( + blank=True, default="", validators=[core.models._validate_network_list] + ), + ), + ] diff --git a/shynet/core/models.py b/shynet/core/models.py index 82380fc..93a5cce 100644 --- a/shynet/core/models.py +++ b/shynet/core/models.py @@ -1,8 +1,10 @@ +import ipaddress import json import uuid from django.apps import apps from django.contrib.auth.models import AbstractUser +from django.core.exceptions import ValidationError from django.db import models from django.db.models.functions import TruncDate from django.db.utils import NotSupportedError @@ -14,6 +16,19 @@ def _default_uuid(): return str(uuid.uuid4()) +def _validate_network_list(networks: str): + try: + _parse_network_list(networks) + except ValueError as e: + raise ValidationError(str(e)) + + +def _parse_network_list(networks: str): + if len(networks.strip()) == 0: + return [] + return [ipaddress.ip_network(network.strip()) for network in networks.split(",")] + + class User(AbstractUser): username = models.TextField(default=_default_uuid, unique=True) email = models.EmailField(unique=True) @@ -43,6 +58,9 @@ class Service(models.Model): ) respect_dnt = models.BooleanField(default=True) collect_ips = models.BooleanField(default=True) + ignored_ips = models.TextField( + default="", blank=True, validators=[_validate_network_list] + ) class Meta: ordering = ["name", "uuid"] @@ -50,6 +68,9 @@ class Service(models.Model): def __str__(self): return self.name + def get_ignored_networks(self): + return _parse_network_list(self.ignored_ips) + def get_daily_stats(self): return self.get_core_stats( start_time=timezone.now() - timezone.timedelta(days=1) diff --git a/shynet/dashboard/forms.py b/shynet/dashboard/forms.py index 54cb31f..d0f4163 100644 --- a/shynet/dashboard/forms.py +++ b/shynet/dashboard/forms.py @@ -8,17 +8,27 @@ from core.models import Service, User class ServiceForm(forms.ModelForm): class Meta: model = Service - fields = ["name", "link", "respect_dnt", "collect_ips", "origins", "collaborators"] + fields = [ + "name", + "link", + "respect_dnt", + "collect_ips", + "ignored_ips", + "origins", + "collaborators", + ] widgets = { "name": forms.TextInput(), "origins": forms.TextInput(), + "ignored_ips": forms.TextInput(), "respect_dnt": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]), "collect_ips": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]), } labels = { "origins": "Allowed Hostnames", "respect_dnt": "Respect DNT", - "collect_ips": "Collect IP addresses" + "collect_ips": "Collect IP addresses", + "ignored_ips": "Ignored IP addresses", } help_texts = { "name": _("What should the service be called?"), @@ -27,7 +37,8 @@ class ServiceForm(forms.ModelForm): "At what hostnames does the service operate? This sets CORS headers, so use '*' if you're not sure (or don't care)." ), "respect_dnt": "Should visitors who have enabled Do Not Track be excluded from all data?", - "collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected." + "collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected.", + "ignored_ips": "A comma-separated list of IP addresses or IP ranges (IPv4 and IPv6) to exclude from tracking (e.g., '192.168.0.2, 127.0.0.1/32').", } collaborators = forms.CharField( diff --git a/shynet/dashboard/templates/dashboard/includes/service_form.html b/shynet/dashboard/templates/dashboard/includes/service_form.html index c92e653..2413dd7 100644 --- a/shynet/dashboard/templates/dashboard/includes/service_form.html +++ b/shynet/dashboard/templates/dashboard/includes/service_form.html @@ -4,10 +4,11 @@ {{form.link|a17t}} {{form.collaborators|a17t}} -
+
Advanced settings
{{form.respect_dnt|a17t}} {{form.collect_ips|a17t}} + {{form.ignored_ips|a17t}} {{form.origins|a17t}}
\ No newline at end of file diff --git a/shynet/dashboard/templatetags/helpers.py b/shynet/dashboard/templatetags/helpers.py index 33efeaa..b4f4da6 100644 --- a/shynet/dashboard/templatetags/helpers.py +++ b/shynet/dashboard/templatetags/helpers.py @@ -81,11 +81,11 @@ def percent_change_display(start, end): return SafeString(direction + pct_change) + @register.inclusion_tag("dashboard/includes/sidebar_footer.html") def sidebar_footer(): - return { - "version": settings.VERSION - } + return {"version": settings.VERSION} + @register.inclusion_tag("dashboard/includes/stat_comparison.html") def compare(