diff --git a/Pipfile b/Pipfile
index ddfa874..56e2d3d 100644
--- a/Pipfile
+++ b/Pipfile
@@ -23,6 +23,7 @@ psycopg2-binary = "*"
redis = "*"
django-redis-cache = "*"
pycountry = "*"
+ipaddress = "*"
[pipenv]
allow_prereleases = true
diff --git a/Pipfile.lock b/Pipfile.lock
index c913d2f..132cc59 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "2cd3e33ea333a40476d2030b6be66826e93c3a4de67032655061725835c92f09"
+ "sha256": "75fe7f0efbc05d6bc32c5ccaa08d3d619bf925682ca5eaffa728e74d0e8e5f66"
},
"pipfile-spec": 6,
"requires": {},
@@ -59,18 +59,18 @@
},
"defusedxml": {
"hashes": [
- "sha256:6687150770438374ab581bb7a1b327a847dd9c5749e396102de3fad4e8a3ef93",
- "sha256:f684034d135af4c6cbb949b8a4d2ed61634515257a67299e5f940fbaa34377f5"
+ "sha256:8ede8ba04cf5bf7999e1492fa77df545db83717f52c5eab625f97228ebd539bf",
+ "sha256:aa621655d72cdd30f57073893b96cd0c3831a85b08b8e4954531bdac47e3e8c8"
],
- "version": "==0.6.0"
+ "version": "==0.7.0rc1"
},
"django": {
"hashes": [
- "sha256:642d8eceab321ca743ae71e0f985ff8fdca59f07aab3a9fb362c617d23e33a76",
- "sha256:d4666c2edefa38c5ede0ec1655424c56dc47ceb04b6d8d62a7eac09db89545c1"
+ "sha256:051ba55d42daa3eeda3944a8e4df2bc96d4c62f94316dea217248a22563c3621",
+ "sha256:9aaa6a09678e1b8f0d98a948c56482eac3e3dd2ddbfb8de70a868135ef3b5e01"
],
"index": "pypi",
- "version": "==3.0.5"
+ "version": "==3.0.6"
},
"django-allauth": {
"hashes": [
@@ -125,6 +125,14 @@
],
"version": "==2.9"
},
+ "ipaddress": {
+ "hashes": [
+ "sha256:6e0f4a39e66cb5bb9a137b00276a2eff74f93b71dcbdad6f10ff7df9d3557fcc",
+ "sha256:b7f8e0369580bb4a24d5ba1d7cc29660a4a6987763faf1d8a8046830e020e7e2"
+ ],
+ "index": "pypi",
+ "version": "==1.0.23"
+ },
"kombu": {
"hashes": [
"sha256:2d1cda774126a044d91a7ff5fa6d09edf99f46924ab332a810760fe6740e9b76",
@@ -134,9 +142,9 @@
},
"maxminddb": {
"hashes": [
- "sha256:d0ce131d901eb11669996b49a59f410efd3da2c6dbe2c0094fe2fef8d85b6336"
+ "sha256:f4d28823d9ca23323d113dc7af8db2087aa4f657fafc64ff8f7a8afda871425b"
],
- "version": "==1.5.2"
+ "version": "==1.5.4"
},
"oauthlib": {
"hashes": [
@@ -197,10 +205,10 @@
},
"pytz": {
"hashes": [
- "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
- "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
+ "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed",
+ "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048"
],
- "version": "==2019.3"
+ "version": "==2020.1"
},
"pyyaml": {
"hashes": [
@@ -221,11 +229,11 @@
},
"redis": {
"hashes": [
- "sha256:0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f",
- "sha256:b205cffd05ebfd0a468db74f0eedbff8df1a7bfc47521516ade4692991bb0833"
+ "sha256:174101a3ce04560d716616290bb40e0a2af45d5844c8bd474c23fc5c52e7a46a",
+ "sha256:7378105cd8ea20c4edc49f028581e830c01ad5f00be851def0f4bc616a83cd89"
],
"index": "pypi",
- "version": "==3.4.1"
+ "version": "==3.5.0"
},
"requests": {
"hashes": [
@@ -326,10 +334,10 @@
},
"click": {
"hashes": [
- "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc",
- "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a"
+ "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a",
+ "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"
],
- "version": "==7.1.1"
+ "version": "==7.1.2"
},
"pathspec": {
"hashes": [
@@ -340,29 +348,29 @@
},
"regex": {
"hashes": [
- "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b",
- "sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8",
- "sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3",
- "sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e",
- "sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683",
- "sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1",
- "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142",
- "sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3",
- "sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468",
- "sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e",
- "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3",
- "sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a",
- "sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f",
- "sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6",
- "sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156",
- "sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b",
- "sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db",
- "sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd",
- "sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a",
- "sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948",
- "sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89"
+ "sha256:021a0ae4d2baeeb60a3014805a2096cb329bd6d9f30669b7ad0da51a9cb73349",
+ "sha256:04d6e948ef34d3eac133bedc0098364a9e635a7914f050edb61272d2ddae3608",
+ "sha256:099568b372bda492be09c4f291b398475587d49937c659824f891182df728cdf",
+ "sha256:0ff50843535593ee93acab662663cb2f52af8e31c3f525f630f1dc6156247938",
+ "sha256:1b17bf37c2aefc4cac8436971fe6ee52542ae4225cfc7762017f7e97a63ca998",
+ "sha256:1e2255ae938a36e9bd7db3b93618796d90c07e5f64dd6a6750c55f51f8b76918",
+ "sha256:2bc6a17a7fa8afd33c02d51b6f417fc271538990297167f68a98cae1c9e5c945",
+ "sha256:3ab5e41c4ed7cd4fa426c50add2892eb0f04ae4e73162155cd668257d02259dd",
+ "sha256:3b059e2476b327b9794c792c855aa05531a3f3044737e455d283c7539bd7534d",
+ "sha256:4df91094ced6f53e71f695c909d9bad1cca8761d96fd9f23db12245b5521136e",
+ "sha256:5493a02c1882d2acaaf17be81a3b65408ff541c922bfd002535c5f148aa29f74",
+ "sha256:5b741ecc3ad3e463d2ba32dce512b412c319993c1bb3d999be49e6092a769fb2",
+ "sha256:652ab4836cd5531d64a34403c00ada4077bb91112e8bcdae933e2eae232cf4a8",
+ "sha256:669a8d46764a09f198f2e91fc0d5acdac8e6b620376757a04682846ae28879c4",
+ "sha256:73a10404867b835f1b8a64253e4621908f0d71150eb4e97ab2e7e441b53e9451",
+ "sha256:7ce4a213a96d6c25eeae2f7d60d4dad89ac2b8134ec3e69db9bc522e2c0f9388",
+ "sha256:8127ca2bf9539d6a64d03686fd9e789e8c194fc19af49b69b081f8c7e6ecb1bc",
+ "sha256:b5b5b2e95f761a88d4c93691716ce01dc55f288a153face1654f868a8034f494",
+ "sha256:b7c9f65524ff06bf70c945cd8d8d1fd90853e27ccf86026af2afb4d9a63d06b1",
+ "sha256:f7f2f4226db6acd1da228adf433c5c3792858474e49d80668ea82ac87cf74a03",
+ "sha256:fa09da4af4e5b15c0e8b4986a083f3fd159302ea115a6cc0649cd163435538b8"
],
- "version": "==2020.4.4"
+ "version": "==2020.5.7"
},
"toml": {
"hashes": [
diff --git a/shynet/analytics/migrations/0003_auto_20200502_1227.py b/shynet/analytics/migrations/0003_auto_20200502_1227.py
index 7c71b33..1fe6a65 100644
--- a/shynet/analytics/migrations/0003_auto_20200502_1227.py
+++ b/shynet/analytics/migrations/0003_auto_20200502_1227.py
@@ -6,13 +6,13 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
- ('analytics', '0002_auto_20200415_1742'),
+ ("analytics", "0002_auto_20200415_1742"),
]
operations = [
migrations.AlterField(
- model_name='session',
- name='ip',
+ model_name="session",
+ name="ip",
field=models.GenericIPAddressField(db_index=True, null=True),
),
]
diff --git a/shynet/analytics/tasks.py b/shynet/analytics/tasks.py
index 7da239b..bd2c6a0 100644
--- a/shynet/analytics/tasks.py
+++ b/shynet/analytics/tasks.py
@@ -1,9 +1,10 @@
+import ipaddress
import json
import logging
+from hashlib import sha1
import geoip2.database
import user_agents
-from hashlib import sha1
from celery import shared_task
from django.conf import settings
from django.core.cache import cache
@@ -60,6 +61,14 @@ def ingress_request(
if dnt and service.respect_dnt:
return
+ try:
+ remote_ip = ipaddress.ip_network(ip)
+ for ignored_network in service.get_ignored_networks():
+ if ignored_network.supernet_of(remote_ip):
+ return
+ except ValueError as e:
+ log.exception(e)
+
# Validate payload
if payload.get("loadTime", 1) <= 0:
payload["loadTime"] = None
diff --git a/shynet/analytics/views/ingress.py b/shynet/analytics/views/ingress.py
index eb09cc0..ef39b01 100644
--- a/shynet/analytics/views/ingress.py
+++ b/shynet/analytics/views/ingress.py
@@ -4,7 +4,7 @@ import json
from django.conf import settings
from django.core.cache import cache
from django.core.exceptions import ValidationError
-from django.http import HttpResponse, Http404, HttpResponseBadRequest
+from django.http import Http404, HttpResponse, HttpResponseBadRequest
from django.shortcuts import render, reverse
from django.utils import timezone
from django.utils.decorators import method_decorator
@@ -36,6 +36,7 @@ def ingress(request, service_uuid, identifier, tracker, payload):
identifier=identifier,
)
+
class ValidateServiceOriginsMixin:
def dispatch(self, request, *args, **kwargs):
try:
diff --git a/shynet/core/management/commands/startup_checks.py b/shynet/core/management/commands/startup_checks.py
index 15da2d3..b4cc9d8 100644
--- a/shynet/core/management/commands/startup_checks.py
+++ b/shynet/core/management/commands/startup_checks.py
@@ -3,12 +3,11 @@ import uuid
from django.conf import settings
from django.contrib.sites.models import Site
-from django.core.management.base import BaseCommand, CommandError
-from django.utils.crypto import get_random_string
-
-from django.db import DEFAULT_DB_ALIAS, connections
-from django.db.utils import OperationalError, ConnectionHandler
from django.core.exceptions import ImproperlyConfigured
+from django.core.management.base import BaseCommand, CommandError
+from django.db import DEFAULT_DB_ALIAS, connections
+from django.db.utils import ConnectionHandler, OperationalError
+from django.utils.crypto import get_random_string
from core.models import User
@@ -18,6 +17,7 @@ class Command(BaseCommand):
def check_migrations(self):
from django.db.migrations.executor import MigrationExecutor
+
try:
executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS])
except OperationalError:
@@ -26,24 +26,31 @@ class Command(BaseCommand):
except ImproperlyConfigured:
# No databases are configured (or the dummy one)
return True
-
+
if executor.migration_plan(executor.loader.graph.leaf_nodes()):
return True
-
+
return False
-
-
+
def handle(self, *args, **options):
migration = self.check_migrations()
admin, hostname, whitelabel = [True] * 3
if not migration:
admin = not User.objects.all().exists()
- hostname = not Site.objects.filter(domain__isnull=False).exclude(domain__exact="").exclude(domain__exact="example.com").exists()
- whitelabel = not Site.objects.filter(name__isnull=False).exclude(name__exact="").exclude(name__exact="example.com").exists()
+ hostname = (
+ not Site.objects.filter(domain__isnull=False)
+ .exclude(domain__exact="")
+ .exclude(domain__exact="example.com")
+ .exists()
+ )
+ whitelabel = (
+ not Site.objects.filter(name__isnull=False)
+ .exclude(name__exact="")
+ .exclude(name__exact="example.com")
+ .exists()
+ )
self.stdout.write(
- self.style.SUCCESS(
- f"{migration} {admin} {hostname} {whitelabel}"
- )
+ self.style.SUCCESS(f"{migration} {admin} {hostname} {whitelabel}")
)
diff --git a/shynet/core/migrations/0004_service_collect_ips.py b/shynet/core/migrations/0004_service_collect_ips.py
index f5f48fb..c38beb3 100644
--- a/shynet/core/migrations/0004_service_collect_ips.py
+++ b/shynet/core/migrations/0004_service_collect_ips.py
@@ -6,13 +6,13 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
- ('core', '0003_service_respect_dnt'),
+ ("core", "0003_service_respect_dnt"),
]
operations = [
migrations.AddField(
- model_name='service',
- name='collect_ips',
+ model_name="service",
+ name="collect_ips",
field=models.BooleanField(default=True),
),
]
diff --git a/shynet/core/migrations/0005_service_ignored_ips.py b/shynet/core/migrations/0005_service_ignored_ips.py
new file mode 100644
index 0000000..9cb5440
--- /dev/null
+++ b/shynet/core/migrations/0005_service_ignored_ips.py
@@ -0,0 +1,22 @@
+# Generated by Django 3.0.6 on 2020-05-07 20:28
+
+from django.db import migrations, models
+
+import core.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("core", "0004_service_collect_ips"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="service",
+ name="ignored_ips",
+ field=models.TextField(
+ blank=True, default="", validators=[core.models._validate_network_list]
+ ),
+ ),
+ ]
diff --git a/shynet/core/models.py b/shynet/core/models.py
index 82380fc..93a5cce 100644
--- a/shynet/core/models.py
+++ b/shynet/core/models.py
@@ -1,8 +1,10 @@
+import ipaddress
import json
import uuid
from django.apps import apps
from django.contrib.auth.models import AbstractUser
+from django.core.exceptions import ValidationError
from django.db import models
from django.db.models.functions import TruncDate
from django.db.utils import NotSupportedError
@@ -14,6 +16,19 @@ def _default_uuid():
return str(uuid.uuid4())
+def _validate_network_list(networks: str):
+ try:
+ _parse_network_list(networks)
+ except ValueError as e:
+ raise ValidationError(str(e))
+
+
+def _parse_network_list(networks: str):
+ if len(networks.strip()) == 0:
+ return []
+ return [ipaddress.ip_network(network.strip()) for network in networks.split(",")]
+
+
class User(AbstractUser):
username = models.TextField(default=_default_uuid, unique=True)
email = models.EmailField(unique=True)
@@ -43,6 +58,9 @@ class Service(models.Model):
)
respect_dnt = models.BooleanField(default=True)
collect_ips = models.BooleanField(default=True)
+ ignored_ips = models.TextField(
+ default="", blank=True, validators=[_validate_network_list]
+ )
class Meta:
ordering = ["name", "uuid"]
@@ -50,6 +68,9 @@ class Service(models.Model):
def __str__(self):
return self.name
+ def get_ignored_networks(self):
+ return _parse_network_list(self.ignored_ips)
+
def get_daily_stats(self):
return self.get_core_stats(
start_time=timezone.now() - timezone.timedelta(days=1)
diff --git a/shynet/dashboard/forms.py b/shynet/dashboard/forms.py
index 54cb31f..d0f4163 100644
--- a/shynet/dashboard/forms.py
+++ b/shynet/dashboard/forms.py
@@ -8,17 +8,27 @@ from core.models import Service, User
class ServiceForm(forms.ModelForm):
class Meta:
model = Service
- fields = ["name", "link", "respect_dnt", "collect_ips", "origins", "collaborators"]
+ fields = [
+ "name",
+ "link",
+ "respect_dnt",
+ "collect_ips",
+ "ignored_ips",
+ "origins",
+ "collaborators",
+ ]
widgets = {
"name": forms.TextInput(),
"origins": forms.TextInput(),
+ "ignored_ips": forms.TextInput(),
"respect_dnt": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
"collect_ips": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
}
labels = {
"origins": "Allowed Hostnames",
"respect_dnt": "Respect DNT",
- "collect_ips": "Collect IP addresses"
+ "collect_ips": "Collect IP addresses",
+ "ignored_ips": "Ignored IP addresses",
}
help_texts = {
"name": _("What should the service be called?"),
@@ -27,7 +37,8 @@ class ServiceForm(forms.ModelForm):
"At what hostnames does the service operate? This sets CORS headers, so use '*' if you're not sure (or don't care)."
),
"respect_dnt": "Should visitors who have enabled Do Not Track be excluded from all data?",
- "collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected."
+ "collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected.",
+ "ignored_ips": "A comma-separated list of IP addresses or IP ranges (IPv4 and IPv6) to exclude from tracking (e.g., '192.168.0.2, 127.0.0.1/32').",
}
collaborators = forms.CharField(
diff --git a/shynet/dashboard/templates/dashboard/includes/service_form.html b/shynet/dashboard/templates/dashboard/includes/service_form.html
index c92e653..2413dd7 100644
--- a/shynet/dashboard/templates/dashboard/includes/service_form.html
+++ b/shynet/dashboard/templates/dashboard/includes/service_form.html
@@ -4,10 +4,11 @@
{{form.link|a17t}}
{{form.collaborators|a17t}}
-
+
Advanced settings
{{form.respect_dnt|a17t}}
{{form.collect_ips|a17t}}
+ {{form.ignored_ips|a17t}}
{{form.origins|a17t}}
\ No newline at end of file
diff --git a/shynet/dashboard/templatetags/helpers.py b/shynet/dashboard/templatetags/helpers.py
index 33efeaa..b4f4da6 100644
--- a/shynet/dashboard/templatetags/helpers.py
+++ b/shynet/dashboard/templatetags/helpers.py
@@ -81,11 +81,11 @@ def percent_change_display(start, end):
return SafeString(direction + pct_change)
+
@register.inclusion_tag("dashboard/includes/sidebar_footer.html")
def sidebar_footer():
- return {
- "version": settings.VERSION
- }
+ return {"version": settings.VERSION}
+
@register.inclusion_tag("dashboard/includes/stat_comparison.html")
def compare(