theluyuan/shynet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add ip address exclusion support (closes #22)

Browse Source 
Co-authored-by: Anthony Abeo <anthonyabeo@gmail.com>
This commit is contained in:
R. Miles McCain 2020-05-07 16:53:03 -04:00
parent bd88617dc5
commit a766c1eaa2
No known key found for this signature in database
GPG Key ID: 24F9B6A2588C5408
12 changed files with 150 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Pipfile
View File

@ -23,6 +23,7 @@ psycopg2-binary = "*"
redis = "*" redis = "*"
django-redis-cache = "*" django-redis-cache = "*"
pycountry = "*" pycountry = "*"
ipaddress = "*"
[pipenv] [pipenv]
allow_prereleases = true allow_prereleases = true

88
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "2cd3e33ea333a40476d2030b6be66826e93c3a4de67032655061725835c92f09" "sha256": "75fe7f0efbc05d6bc32c5ccaa08d3d619bf925682ca5eaffa728e74d0e8e5f66"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": {}, "requires": {},
@ -59,18 +59,18 @@
}, },
"defusedxml": { "defusedxml": {
"hashes": [ "hashes": [
"sha256:6687150770438374ab581bb7a1b327a847dd9c5749e396102de3fad4e8a3ef93", "sha256:8ede8ba04cf5bf7999e1492fa77df545db83717f52c5eab625f97228ebd539bf",
"sha256:f684034d135af4c6cbb949b8a4d2ed61634515257a67299e5f940fbaa34377f5" "sha256:aa621655d72cdd30f57073893b96cd0c3831a85b08b8e4954531bdac47e3e8c8"
], ],
"version": "==0.6.0" "version": "==0.7.0rc1"
}, },
"django": { "django": {
"hashes": [ "hashes": [
"sha256:642d8eceab321ca743ae71e0f985ff8fdca59f07aab3a9fb362c617d23e33a76", "sha256:051ba55d42daa3eeda3944a8e4df2bc96d4c62f94316dea217248a22563c3621",
"sha256:d4666c2edefa38c5ede0ec1655424c56dc47ceb04b6d8d62a7eac09db89545c1" "sha256:9aaa6a09678e1b8f0d98a948c56482eac3e3dd2ddbfb8de70a868135ef3b5e01"
], ],
"index": "pypi", "index": "pypi",
"version": "==3.0.5" "version": "==3.0.6"
}, },
"django-allauth": { "django-allauth": {
"hashes": [ "hashes": [
@ -125,6 +125,14 @@
], ],
"version": "==2.9" "version": "==2.9"
}, },
"ipaddress": {
"hashes": [
"sha256:6e0f4a39e66cb5bb9a137b00276a2eff74f93b71dcbdad6f10ff7df9d3557fcc",
"sha256:b7f8e0369580bb4a24d5ba1d7cc29660a4a6987763faf1d8a8046830e020e7e2"
],
"index": "pypi",
"version": "==1.0.23"
},
"kombu": { "kombu": {
"hashes": [ "hashes": [
"sha256:2d1cda774126a044d91a7ff5fa6d09edf99f46924ab332a810760fe6740e9b76", "sha256:2d1cda774126a044d91a7ff5fa6d09edf99f46924ab332a810760fe6740e9b76",
@ -134,9 +142,9 @@
}, },
"maxminddb": { "maxminddb": {
"hashes": [ "hashes": [
"sha256:d0ce131d901eb11669996b49a59f410efd3da2c6dbe2c0094fe2fef8d85b6336" "sha256:f4d28823d9ca23323d113dc7af8db2087aa4f657fafc64ff8f7a8afda871425b"
], ],
"version": "==1.5.2" "version": "==1.5.4"
}, },
"oauthlib": { "oauthlib": {
"hashes": [ "hashes": [
@ -197,10 +205,10 @@
}, },
"pytz": { "pytz": {
"hashes": [ "hashes": [
"sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed",
"sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048"
], ],
"version": "==2019.3" "version": "==2020.1"
}, },
"pyyaml": { "pyyaml": {
"hashes": [ "hashes": [
@ -221,11 +229,11 @@
}, },
"redis": { "redis": {
"hashes": [ "hashes": [
"sha256:0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f", "sha256:174101a3ce04560d716616290bb40e0a2af45d5844c8bd474c23fc5c52e7a46a",
"sha256:b205cffd05ebfd0a468db74f0eedbff8df1a7bfc47521516ade4692991bb0833" "sha256:7378105cd8ea20c4edc49f028581e830c01ad5f00be851def0f4bc616a83cd89"
], ],
"index": "pypi", "index": "pypi",
"version": "==3.4.1" "version": "==3.5.0"
}, },
"requests": { "requests": {
"hashes": [ "hashes": [
@ -326,10 +334,10 @@
}, },
"click": { "click": {
"hashes": [ "hashes": [
"sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc", "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a",
"sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a" "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"
], ],
"version": "==7.1.1" "version": "==7.1.2"
}, },
"pathspec": { "pathspec": {
"hashes": [ "hashes": [
@ -340,29 +348,29 @@
}, },
"regex": { "regex": {
"hashes": [ "hashes": [
"sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b", "sha256:021a0ae4d2baeeb60a3014805a2096cb329bd6d9f30669b7ad0da51a9cb73349",
"sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8", "sha256:04d6e948ef34d3eac133bedc0098364a9e635a7914f050edb61272d2ddae3608",
"sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3", "sha256:099568b372bda492be09c4f291b398475587d49937c659824f891182df728cdf",
"sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e", "sha256:0ff50843535593ee93acab662663cb2f52af8e31c3f525f630f1dc6156247938",
"sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683", "sha256:1b17bf37c2aefc4cac8436971fe6ee52542ae4225cfc7762017f7e97a63ca998",
"sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1", "sha256:1e2255ae938a36e9bd7db3b93618796d90c07e5f64dd6a6750c55f51f8b76918",
"sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142", "sha256:2bc6a17a7fa8afd33c02d51b6f417fc271538990297167f68a98cae1c9e5c945",
"sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3", "sha256:3ab5e41c4ed7cd4fa426c50add2892eb0f04ae4e73162155cd668257d02259dd",
"sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468", "sha256:3b059e2476b327b9794c792c855aa05531a3f3044737e455d283c7539bd7534d",
"sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e", "sha256:4df91094ced6f53e71f695c909d9bad1cca8761d96fd9f23db12245b5521136e",
"sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3", "sha256:5493a02c1882d2acaaf17be81a3b65408ff541c922bfd002535c5f148aa29f74",
"sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a", "sha256:5b741ecc3ad3e463d2ba32dce512b412c319993c1bb3d999be49e6092a769fb2",
"sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f", "sha256:652ab4836cd5531d64a34403c00ada4077bb91112e8bcdae933e2eae232cf4a8",
"sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6", "sha256:669a8d46764a09f198f2e91fc0d5acdac8e6b620376757a04682846ae28879c4",
"sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156", "sha256:73a10404867b835f1b8a64253e4621908f0d71150eb4e97ab2e7e441b53e9451",
"sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b", "sha256:7ce4a213a96d6c25eeae2f7d60d4dad89ac2b8134ec3e69db9bc522e2c0f9388",
"sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db", "sha256:8127ca2bf9539d6a64d03686fd9e789e8c194fc19af49b69b081f8c7e6ecb1bc",
"sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd", "sha256:b5b5b2e95f761a88d4c93691716ce01dc55f288a153face1654f868a8034f494",
"sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a", "sha256:b7c9f65524ff06bf70c945cd8d8d1fd90853e27ccf86026af2afb4d9a63d06b1",
"sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948", "sha256:f7f2f4226db6acd1da228adf433c5c3792858474e49d80668ea82ac87cf74a03",
"sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89" "sha256:fa09da4af4e5b15c0e8b4986a083f3fd159302ea115a6cc0649cd163435538b8"
], ],
"version": "==2020.4.4" "version": "==2020.5.7"
}, },
"toml": { "toml": {
"hashes": [ "hashes": [

6
shynet/analytics/migrations/0003_auto_20200502_1227.py
View File

@ -6,13 +6,13 @@ from django.db import migrations, models
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('analytics', '0002_auto_20200415_1742'), ("analytics", "0002_auto_20200415_1742"),
] ]
operations = [ operations = [
migrations.AlterField( migrations.AlterField(
model_name='session', model_name="session",
name='ip', name="ip",
field=models.GenericIPAddressField(db_index=True, null=True), field=models.GenericIPAddressField(db_index=True, null=True),
), ),
] ]

11
shynet/analytics/tasks.py
View File

@ -1,9 +1,10 @@
import ipaddress
import json import json
import logging import logging
from hashlib import sha1
import geoip2.database import geoip2.database
import user_agents import user_agents
from hashlib import sha1
from celery import shared_task from celery import shared_task
from django.conf import settings from django.conf import settings
from django.core.cache import cache from django.core.cache import cache
@ -60,6 +61,14 @@ def ingress_request(
if dnt and service.respect_dnt: if dnt and service.respect_dnt:
return return
try:
remote_ip = ipaddress.ip_network(ip)
for ignored_network in service.get_ignored_networks():
if ignored_network.supernet_of(remote_ip):
return
except ValueError as e:
log.exception(e)
# Validate payload # Validate payload
if payload.get("loadTime", 1) <= 0: if payload.get("loadTime", 1) <= 0:
payload["loadTime"] = None payload["loadTime"] = None

3
shynet/analytics/views/ingress.py
View File

@ -4,7 +4,7 @@ import json
from django.conf import settings from django.conf import settings
from django.core.cache import cache from django.core.cache import cache
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from django.http import HttpResponse, Http404, HttpResponseBadRequest from django.http import Http404, HttpResponse, HttpResponseBadRequest
from django.shortcuts import render, reverse from django.shortcuts import render, reverse
from django.utils import timezone from django.utils import timezone
from django.utils.decorators import method_decorator from django.utils.decorators import method_decorator
@ -36,6 +36,7 @@ def ingress(request, service_uuid, identifier, tracker, payload):
identifier=identifier, identifier=identifier,
) )
class ValidateServiceOriginsMixin: class ValidateServiceOriginsMixin:
def dispatch(self, request, *args, **kwargs): def dispatch(self, request, *args, **kwargs):
try: try:

35
shynet/core/management/commands/startup_checks.py
View File

@ -3,12 +3,11 @@ import uuid
from django.conf import settings from django.conf import settings
from django.contrib.sites.models import Site from django.contrib.sites.models import Site
from django.core.management.base import BaseCommand, CommandError
from django.utils.crypto import get_random_string
from django.db import DEFAULT_DB_ALIAS, connections
from django.db.utils import OperationalError, ConnectionHandler
from django.core.exceptions import ImproperlyConfigured from django.core.exceptions import ImproperlyConfigured
from django.core.management.base import BaseCommand, CommandError
from django.db import DEFAULT_DB_ALIAS, connections
from django.db.utils import ConnectionHandler, OperationalError
from django.utils.crypto import get_random_string
from core.models import User from core.models import User
@ -18,6 +17,7 @@ class Command(BaseCommand):
def check_migrations(self): def check_migrations(self):
from django.db.migrations.executor import MigrationExecutor from django.db.migrations.executor import MigrationExecutor
try: try:
executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS]) executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS])
except OperationalError: except OperationalError:
@ -26,24 +26,31 @@ class Command(BaseCommand):
except ImproperlyConfigured: except ImproperlyConfigured:
# No databases are configured (or the dummy one) # No databases are configured (or the dummy one)
return True return True
if executor.migration_plan(executor.loader.graph.leaf_nodes()): if executor.migration_plan(executor.loader.graph.leaf_nodes()):
return True return True
return False return False
def handle(self, *args, **options): def handle(self, *args, **options):
migration = self.check_migrations() migration = self.check_migrations()
admin, hostname, whitelabel = [True] * 3 admin, hostname, whitelabel = [True] * 3
if not migration: if not migration:
admin = not User.objects.all().exists() admin = not User.objects.all().exists()
hostname = not Site.objects.filter(domain__isnull=False).exclude(domain__exact="").exclude(domain__exact="example.com").exists() hostname = (
whitelabel = not Site.objects.filter(name__isnull=False).exclude(name__exact="").exclude(name__exact="example.com").exists() not Site.objects.filter(domain__isnull=False)
.exclude(domain__exact="")
.exclude(domain__exact="example.com")
.exists()
)
whitelabel = (
not Site.objects.filter(name__isnull=False)
.exclude(name__exact="")
.exclude(name__exact="example.com")
.exists()
)
self.stdout.write( self.stdout.write(
self.style.SUCCESS( self.style.SUCCESS(f"{migration} {admin} {hostname} {whitelabel}")
f"{migration} {admin} {hostname} {whitelabel}"
)
) )

6
shynet/core/migrations/0004_service_collect_ips.py
View File

@ -6,13 +6,13 @@ from django.db import migrations, models
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('core', '0003_service_respect_dnt'), ("core", "0003_service_respect_dnt"),
] ]
operations = [ operations = [
migrations.AddField( migrations.AddField(
model_name='service', model_name="service",
name='collect_ips', name="collect_ips",
field=models.BooleanField(default=True), field=models.BooleanField(default=True),
), ),
] ]

22
shynet/core/migrations/0005_service_ignored_ips.py Normal file
View File

@ -0,0 +1,22 @@
# Generated by Django 3.0.6 on 2020-05-07 20:28
from django.db import migrations, models
import core.models
class Migration(migrations.Migration):
dependencies = [
("core", "0004_service_collect_ips"),
]
operations = [
migrations.AddField(
model_name="service",
name="ignored_ips",
field=models.TextField(
blank=True, default="", validators=[core.models._validate_network_list]
),
),
]

21
shynet/core/models.py
View File

@ -1,8 +1,10 @@
import ipaddress
import json import json
import uuid import uuid
from django.apps import apps from django.apps import apps
from django.contrib.auth.models import AbstractUser from django.contrib.auth.models import AbstractUser
from django.core.exceptions import ValidationError
from django.db import models from django.db import models
from django.db.models.functions import TruncDate from django.db.models.functions import TruncDate
from django.db.utils import NotSupportedError from django.db.utils import NotSupportedError
@ -14,6 +16,19 @@ def _default_uuid():
return str(uuid.uuid4()) return str(uuid.uuid4())
def _validate_network_list(networks: str):
try:
_parse_network_list(networks)
except ValueError as e:
raise ValidationError(str(e))
def _parse_network_list(networks: str):
if len(networks.strip()) == 0:
return []
return [ipaddress.ip_network(network.strip()) for network in networks.split(",")]
class User(AbstractUser): class User(AbstractUser):
username = models.TextField(default=_default_uuid, unique=True) username = models.TextField(default=_default_uuid, unique=True)
email = models.EmailField(unique=True) email = models.EmailField(unique=True)
@ -43,6 +58,9 @@ class Service(models.Model):
) )
respect_dnt = models.BooleanField(default=True) respect_dnt = models.BooleanField(default=True)
collect_ips = models.BooleanField(default=True) collect_ips = models.BooleanField(default=True)
ignored_ips = models.TextField(
default="", blank=True, validators=[_validate_network_list]
)
class Meta: class Meta:
ordering = ["name", "uuid"] ordering = ["name", "uuid"]
@ -50,6 +68,9 @@ class Service(models.Model):
def __str__(self): def __str__(self):
return self.name return self.name
def get_ignored_networks(self):
return _parse_network_list(self.ignored_ips)
def get_daily_stats(self): def get_daily_stats(self):
return self.get_core_stats( return self.get_core_stats(
start_time=timezone.now() - timezone.timedelta(days=1) start_time=timezone.now() - timezone.timedelta(days=1)

17
shynet/dashboard/forms.py
View File

@ -8,17 +8,27 @@ from core.models import Service, User
class ServiceForm(forms.ModelForm): class ServiceForm(forms.ModelForm):
class Meta: class Meta:
model = Service model = Service
fields = ["name", "link", "respect_dnt", "collect_ips", "origins", "collaborators"] fields = [
"name",
"link",
"respect_dnt",
"collect_ips",
"ignored_ips",
"origins",
"collaborators",
]
widgets = { widgets = {
"name": forms.TextInput(), "name": forms.TextInput(),
"origins": forms.TextInput(), "origins": forms.TextInput(),
"ignored_ips": forms.TextInput(),
"respect_dnt": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]), "respect_dnt": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
"collect_ips": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]), "collect_ips": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
} }
labels = { labels = {
"origins": "Allowed Hostnames", "origins": "Allowed Hostnames",
"respect_dnt": "Respect DNT", "respect_dnt": "Respect DNT",
"collect_ips": "Collect IP addresses" "collect_ips": "Collect IP addresses",
"ignored_ips": "Ignored IP addresses",
} }
help_texts = { help_texts = {
"name": _("What should the service be called?"), "name": _("What should the service be called?"),
@ -27,7 +37,8 @@ class ServiceForm(forms.ModelForm):
"At what hostnames does the service operate? This sets CORS headers, so use '*' if you're not sure (or don't care)." "At what hostnames does the service operate? This sets CORS headers, so use '*' if you're not sure (or don't care)."
), ),
"respect_dnt": "Should visitors who have enabled <a href='https://en.wikipedia.org/wiki/Do_Not_Track'>Do Not Track</a> be excluded from all data?", "respect_dnt": "Should visitors who have enabled <a href='https://en.wikipedia.org/wiki/Do_Not_Track'>Do Not Track</a> be excluded from all data?",
"collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected." "collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected.",
"ignored_ips": "A comma-separated list of IP addresses or IP ranges (IPv4 and IPv6) to exclude from tracking (e.g., '192.168.0.2, 127.0.0.1/32').",
} }
collaborators = forms.CharField( collaborators = forms.CharField(

3
shynet/dashboard/templates/dashboard/includes/service_form.html
View File

@ -4,10 +4,11 @@
{{form.link|a17t}} {{form.link|a17t}}
{{form.collaborators|a17t}} {{form.collaborators|a17t}}
<details class="p-4 border rounded"> <details class="p-4 border rounded" {% if form.errors %}open{% endif %}>
<summary class="cursor-pointer text-sm">Advanced settings</summary> <summary class="cursor-pointer text-sm">Advanced settings</summary>
<hr class="sep h-4"> <hr class="sep h-4">
{{form.respect_dnt|a17t}} {{form.respect_dnt|a17t}}
{{form.collect_ips|a17t}} {{form.collect_ips|a17t}}
{{form.ignored_ips|a17t}}
{{form.origins|a17t}} {{form.origins|a17t}}
</details> </details>

6
shynet/dashboard/templatetags/helpers.py
View File

@ -81,11 +81,11 @@ def percent_change_display(start, end):
return SafeString(direction + pct_change) return SafeString(direction + pct_change)
@register.inclusion_tag("dashboard/includes/sidebar_footer.html") @register.inclusion_tag("dashboard/includes/sidebar_footer.html")
def sidebar_footer(): def sidebar_footer():
return { return {"version": settings.VERSION}
"version": settings.VERSION
}
@register.inclusion_tag("dashboard/includes/stat_comparison.html") @register.inclusion_tag("dashboard/includes/stat_comparison.html")
def compare( def compare(