diff --git a/Pipfile b/Pipfile index f18f0da..9d35a69 100644 --- a/Pipfile +++ b/Pipfile @@ -9,6 +9,12 @@ black = "*" [packages] django = "*" django-allauth = "*" +geoip2 = "*" +celery = "*" +django-ipware = "*" +pyyaml = "*" +ua-parser = "*" +user-agents = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index 45ef0b9..bac52fe 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "75719a3500a5a6da268121b31defc32235442e00bca1872232a93b5db41b61c7" + "sha256": "03f23e0c7409df9eb5a85b3df03d1975a1fe5563496602492cdcff46b8c5c829" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,13 @@ ] }, "default": { + "amqp": { + "hashes": [ + "sha256:6e649ca13a7df3faacdc8bbb280aa9a6602d22fd9d545336077e573a1f4ff3b8", + "sha256:77f1aef9410698d20eaeac5b73a87817365f457a507d82edf292e12cbb83b08d" + ], + "version": "==2.5.2" + }, "asgiref": { "hashes": [ "sha256:8036f90603c54e93521e5777b2b9a39ba1bad05773fcf2d208f0299d1df58ce5", @@ -23,6 +30,21 @@ ], "version": "==3.2.7" }, + "billiard": { + "hashes": [ + "sha256:bff575450859a6e0fbc2f9877d9b715b0bbc07c3565bb7ed2280526a0cdf5ede", + "sha256:d91725ce6425f33a97dfa72fb6bfef0e47d4652acd98a032bd1a7fbf06d5fa6a" + ], + "version": "==3.6.3.0" + }, + "celery": { + "hashes": [ + "sha256:108a0bf9018a871620936c33a3ee9f6336a89f8ef0a0f567a9001f4aa361415f", + "sha256:5b4b37e276033fe47575107a2775469f0b721646a08c96ec2c61531e4fe45f2a" + ], + "index": "pypi", + "version": "==4.4.2" + }, "certifi": { "hashes": [ "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304", @@ -59,6 +81,21 @@ "index": "pypi", "version": "==0.41.0" }, + "django-ipware": { + "hashes": [ + "sha256:a7c7a8fd019dbdc9c357e6e582f65034e897572fc79a7e467674efa8aef9d00b" + ], + "index": "pypi", + "version": "==2.1.0" + }, + "geoip2": { + "hashes": [ + "sha256:5869e987bc54c0d707264fec4710661332cc38d2dca5a7f9bb5362d0308e2ce0", + "sha256:99ec12d2f1271a73a0a4a2b663fe6ce25fd02289c0a6bef05c0a1c3b30ee95a4" + ], + "index": "pypi", + "version": "==3.0.0" + }, "idna": { "hashes": [ "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", @@ -66,6 +103,27 @@ ], "version": "==2.9" }, + "importlib-metadata": { + "hashes": [ + "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f", + "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e" + ], + "markers": "python_version < '3.8'", + "version": "==1.6.0" + }, + "kombu": { + "hashes": [ + "sha256:2d1cda774126a044d91a7ff5fa6d09edf99f46924ab332a810760fe6740e9b76", + "sha256:598e7e749d6ab54f646b74b2d2df67755dee13894f73ab02a2a9feb8870c7cb2" + ], + "version": "==4.6.8" + }, + "maxminddb": { + "hashes": [ + "sha256:d0ce131d901eb11669996b49a59f410efd3da2c6dbe2c0094fe2fef8d85b6336" + ], + "version": "==1.5.2" + }, "oauthlib": { "hashes": [ "sha256:bee41cc35fcca6e988463cacc3bcb8a96224f470ca547e697b604cc697b2f889", @@ -87,6 +145,23 @@ ], "version": "==2019.3" }, + "pyyaml": { + "hashes": [ + "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", + "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", + "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", + "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", + "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", + "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", + "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", + "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", + "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", + "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", + "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" + ], + "index": "pypi", + "version": "==5.3.1" + }, "requests": { "hashes": [ "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", @@ -108,12 +183,42 @@ ], "version": "==0.3.1" }, + "ua-parser": { + "hashes": [ + "sha256:46ab2e383c01dbd2ab284991b87d624a26a08f72da4d7d413f5bfab8b9036f8a", + "sha256:47b1782ed130d890018d983fac37c2a80799d9e0b9c532e734c67cf70f185033" + ], + "index": "pypi", + "version": "==0.10.0" + }, "urllib3": { "hashes": [ "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" ], "version": "==1.25.8" + }, + "user-agents": { + "hashes": [ + "sha256:da54371d856c35d8ead0622da24ad5ef6d667eda3629a750e3373a3e847a054b", + "sha256:e727ab6f169e829bc25d41dbd25b9ff679b4631bd81959bcf7de1e246da67194" + ], + "index": "pypi", + "version": "==2.1" + }, + "vine": { + "hashes": [ + "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87", + "sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af" + ], + "version": "==1.3.0" + }, + "zipp": { + "hashes": [ + "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b", + "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96" + ], + "version": "==3.1.0" } }, "develop": { diff --git a/shynet/analytics/__init__.py b/shynet/analytics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shynet/analytics/admin.py b/shynet/analytics/admin.py new file mode 100644 index 0000000..309f2f0 --- /dev/null +++ b/shynet/analytics/admin.py @@ -0,0 +1,9 @@ +from django.contrib import admin + +from .models import Hit, Session + +admin.site.register(Session) + +admin.site.register(Hit) + +# Register your models here. diff --git a/shynet/analytics/apps.py b/shynet/analytics/apps.py new file mode 100644 index 0000000..67851df --- /dev/null +++ b/shynet/analytics/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class AnalyticsConfig(AppConfig): + name = "analytics" diff --git a/shynet/analytics/ingress_urls.py b/shynet/analytics/ingress_urls.py new file mode 100644 index 0000000..378e997 --- /dev/null +++ b/shynet/analytics/ingress_urls.py @@ -0,0 +1,23 @@ +from django.contrib import admin +from django.urls import include, path + +from .views import ingress + +urlpatterns = [ + path( + "/pixel.gif", ingress.PixelView.as_view(), name="endpoint_pixel" + ), + path( + "/script.js", ingress.ScriptView.as_view(), name="endpoint_script" + ), + path( + "//pixel.gif", + ingress.PixelView.as_view(), + name="endpoint_pixel_id", + ), + path( + "//script.js", + ingress.ScriptView.as_view(), + name="endpoint_pixel_id", + ), +] diff --git a/shynet/analytics/migrations/0001_initial.py b/shynet/analytics/migrations/0001_initial.py new file mode 100644 index 0000000..0cc8d69 --- /dev/null +++ b/shynet/analytics/migrations/0001_initial.py @@ -0,0 +1,65 @@ +# Generated by Django 3.0.5 on 2020-04-10 06:58 + +from django.db import migrations, models + +import analytics.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Hit", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("start", models.DateTimeField(auto_now_add=True)), + ("duration", models.FloatField(default=0.0)), + ("heartbeats", models.IntegerField(default=0)), + ("tracker", models.TextField()), + ("location", models.TextField(blank=True)), + ("referrer", models.TextField(blank=True)), + ("loadTime", models.FloatField(null=True)), + ("httpStatus", models.IntegerField(null=True)), + ("metadata_raw", models.TextField()), + ], + ), + migrations.CreateModel( + name="Session", + fields=[ + ( + "uuid", + models.UUIDField( + default=analytics.models._default_uuid, + primary_key=True, + serialize=False, + ), + ), + ("identifier", models.TextField(blank=True)), + ("first_seen", models.DateTimeField(auto_now_add=True)), + ("last_seen", models.DateTimeField(auto_now_add=True)), + ("user_agent", models.TextField()), + ("browser", models.TextField()), + ("device", models.TextField()), + ("os", models.TextField()), + ("ip", models.GenericIPAddressField()), + ("asn", models.TextField(blank=True)), + ("country", models.TextField(blank=True)), + ("longitude", models.FloatField(null=True)), + ("latitude", models.FloatField(null=True)), + ("time_zone", models.TextField(blank=True)), + ("metadata_raw", models.TextField()), + ], + ), + ] diff --git a/shynet/analytics/migrations/0002_auto_20200410_0258.py b/shynet/analytics/migrations/0002_auto_20200410_0258.py new file mode 100644 index 0000000..db60bdd --- /dev/null +++ b/shynet/analytics/migrations/0002_auto_20200410_0258.py @@ -0,0 +1,31 @@ +# Generated by Django 3.0.5 on 2020-04-10 06:58 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("core", "0001_initial"), + ("analytics", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="session", + name="service", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="core.Service" + ), + ), + migrations.AddField( + model_name="hit", + name="session", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="analytics.Session" + ), + ), + ] diff --git a/shynet/analytics/migrations/__init__.py b/shynet/analytics/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shynet/analytics/models.py b/shynet/analytics/models.py new file mode 100644 index 0000000..c81b1f6 --- /dev/null +++ b/shynet/analytics/models.py @@ -0,0 +1,72 @@ +import json +import uuid + +from django.db import models + +from core.models import Service + + +def _default_uuid(): + return str(uuid.uuid4()) + + +class Session(models.Model): + uuid = models.UUIDField(default=_default_uuid, primary_key=True) + service = models.ForeignKey(Service, on_delete=models.CASCADE) + + # Cross-session identification; optional, and provided by the service + identifier = models.TextField(blank=True) + + # Time + first_seen = models.DateTimeField(auto_now_add=True) + last_seen = models.DateTimeField(auto_now_add=True) + + # Core request information + user_agent = models.TextField() + browser = models.TextField() + device = models.TextField() + os = models.TextField() + ip = models.GenericIPAddressField() + + # GeoIP data + asn = models.TextField(blank=True) + country = models.TextField(blank=True) + longitude = models.FloatField(null=True) + latitude = models.FloatField(null=True) + time_zone = models.TextField(blank=True) + + # Additional metadata, stored as JSON string + metadata_raw = models.TextField() + + @property + def metadata(self): + try: + return json.loads(self.metadata_raw) + except: # Metadata is not crucial; in the case of a read error, just ignore it + return {} + + +class Hit(models.Model): + session = models.ForeignKey(Session, on_delete=models.CASCADE) + + # Base request information + start = models.DateTimeField(auto_now_add=True) + duration = models.FloatField(default=0.0) # Seconds spent on page + heartbeats = models.IntegerField(default=0) + tracker = models.TextField() # Tracking pixel or JS + + # Advanced page information + location = models.TextField(blank=True) + referrer = models.TextField(blank=True) + loadTime = models.FloatField(null=True) + httpStatus = models.IntegerField(null=True) + + # Additional metadata, stored as JSON string + metadata_raw = models.TextField() + + @property + def metadata(self): + try: + return json.loads(self.metadata_raw) + except: # Metadata is not crucial; in the case of a read error, just ignore it + return {} diff --git a/shynet/analytics/tasks.py b/shynet/analytics/tasks.py new file mode 100644 index 0000000..5848a57 --- /dev/null +++ b/shynet/analytics/tasks.py @@ -0,0 +1,125 @@ +import json +import logging + +import geoip2.database +import user_agents +from celery import shared_task +from django.conf import settings +from django.core.cache import cache +from django.utils import timezone + +from core.models import Service + +from .models import Hit, Session + +log = logging.getLogger(__name__) + +_geoip2_city_reader = None +_geoip2_asn_reader = None + + +def _geoip2_lookup(ip): + global _geoip2_city_reader, _geoip2_asn_reader # TODO: is there a better way to do global Django vars? Is this thread safe? + try: + if settings.MAXMIND_CITY_DB == None or settings.MAXMIND_ASN_DB == None: + return None + if _geoip2_city_reader == None or _geoip2_asn_reader == None: + _geoip2_city_reader = geoip2.database.Reader(settings.MAXMIND_CITY_DB) + _geoip2_asn_reader = geoip2.database.Reader(settings.MAXMIND_ASN_DB) + city_results = _geoip2_city_reader.city(ip) + asn_results = _geoip2_asn_reader.asn(ip) + return { + "asn": asn_results.autonomous_system_organization, + "country": city_results.country.iso_code, + "longitude": city_results.location.longitude, + "latitude": city_results.location.latitude, + "time_zone": city_results.location.time_zone, + } + except geoip2.errors.AddressNotFoundError: + return {} + + +@shared_task +def ingress_request( + service_uuid, tracker, time, payload, ip, location, user_agent, identifier="" +): + try: + ip_data = _geoip2_lookup(ip) + + service = Service.objects.get(uuid=service_uuid) + log.debug(f"Linked to service {service}") + + # Create or update session + session_metadata = payload.get("sessionMetadata", {}) + session = Session.objects.filter( + service=service, + last_seen__gt=timezone.now() - timezone.timedelta(minutes=30), + ip=ip, + user_agent=user_agent, + identifier=identifier, + ).first() + if session is None: + log.debug("Cannot link to existing session; creating a new one...") + ua = user_agents.parse(user_agent) + + session = Session.objects.create( + service=service, + ip=ip, + user_agent=user_agent, + identifier=identifier, + browser=f"{ua.browser.family or ''} {ua.browser.version_string or ''}".strip(), + device=f"{ua.device.model or ''}", + os=f"{ua.os.family or ''} {ua.os.version_string or ''}".strip(), + metadata_raw=json.dumps(session_metadata), + asn=ip_data.get("asn", ""), + country=ip_data.get("country", ""), + longitude=ip_data.get("longitude"), + latitude=ip_data.get("latitude"), + time_zone=ip_data.get("time_zone", ""), + ) + else: + log.debug("Updating old session with new data...") + # Update old metadata with new metadata + new_metadata = session.metadata + new_metadata.update(session_metadata) + session.metadata_raw = json.dumps(new_metadata) + # Update last seen time + session.last_seen = timezone.now() + session.save() + + # Create or update hit + hit_metadata = payload.get("hitMetadata", {}) + idempotency = payload.get("idempotency") + idempotency_path = f"hit_idempotency_{idempotency}" + hit = None + if idempotency is not None: + if cache.get(idempotency_path) is not None: + hit = Hit.objects.filter( + pk=cache.get(idempotency_path), session=session + ).first() + if hit is not None: + # There is an existing hit with an identical idempotency key. That means + # this is a heartbeat. + log.debug("Hit is a heartbeat; updating old hit with new data...") + hit.heartbeats += 1 + hit.duration = (timezone.now() - hit.start).total_seconds() + new_metadata = hit.metadata + new_metadata.update(hit_metadata) + hit.metadata_raw = json.dumps(new_metadata) + hit.save() + if hit is None: + log.debug("Hit is a page load; creating new hit...") + # There is no existing hit; create a new one + hit = Hit.objects.create( + session=session, + tracker=tracker, + location=location, + referrer=payload.get("referrer", ""), + loadTime=payload.get("loadTime"), + metadata_raw=json.dumps(hit_metadata), + ) + # Set idempotency (if applicable) + if idempotency is not None: + cache.set(idempotency_path, hit.pk, timeout=30 * 60) + except Exception as e: + log.error(e) diff --git a/shynet/analytics/templates/analytics/scripts/page.js b/shynet/analytics/templates/analytics/scripts/page.js new file mode 100644 index 0000000..fa54c98 --- /dev/null +++ b/shynet/analytics/templates/analytics/scripts/page.js @@ -0,0 +1,27 @@ +window.onload = function () { + var idempotency = + Math.random().toString(36).substring(2, 15) + + Math.random().toString(36).substring(2, 15); + function sendUpdate() { + var xhr = new XMLHttpRequest(); + xhr.open("POST", "{{endpoint}}", true); + xhr.setRequestHeader("Content-Type", "application/json"); + xhr.send( + JSON.stringify({ + idempotency: idempotency, + referrer: document.referrer, + loadTime: + window.performance.timing.domContentLoadedEventEnd - + window.performance.timing.navigationStart, + hitMetadata: + typeof shynetHitMetadata !== "undefined" ? shynetHitMetadata : {}, + sessionMetadata: + typeof shynetSessionMetadata !== "undefined" + ? shynetSessionMetadata + : {}, + }) + ); + } + setInterval(sendUpdate, 5000); + sendUpdate(); +}; diff --git a/shynet/analytics/tests.py b/shynet/analytics/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/shynet/analytics/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/shynet/analytics/views/__init__.py b/shynet/analytics/views/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shynet/analytics/views/ingress.py b/shynet/analytics/views/ingress.py new file mode 100644 index 0000000..40868fa --- /dev/null +++ b/shynet/analytics/views/ingress.py @@ -0,0 +1,85 @@ +import base64 +import json + +from django.http import HttpResponse +from django.shortcuts import render +from django.utils import timezone +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from django.views.generic import TemplateView, View +from ipware import get_client_ip + +from ..tasks import ingress_request + + +def ingress(request, service_uuid, identifier, tracker, payload): + time = timezone.now() + client_ip, is_routable = get_client_ip(request) + location = request.META.get("HTTP_REFERER") + user_agent = request.META.get("HTTP_USER_AGENT") + + ingress_request.delay( + service_uuid, + tracker, + time, + payload, + client_ip, + location, + user_agent, + identifier, + ) + + +class PixelView(View): + # Fallback view to serve an unobtrusive 1x1 transparent tracking pixel for browsers with + # JavaScript disabled. + def dispatch(self, request, *args, **kwargs): + # Extract primary data + ingress( + request, + self.kwargs.get("service_uuid"), + self.kwargs.get("identifier", ""), + "PIXEL", + {}, + ) + + data = base64.b64decode( + "R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==" + ) + resp = HttpResponse(data, content_type="image/gif") + resp["Cache-Control"] = "no-cache" + resp["Access-Control-Allow-Origin"] = "*" + return resp + + +@method_decorator(csrf_exempt, name="dispatch") +class ScriptView(View): + def dispatch(self, request, *args, **kwargs): + resp = super().dispatch(request, *args, **kwargs) + resp["Access-Control-Allow-Origin"] = "*" + resp["Access-Control-Allow-Methods"] = "GET,HEAD,OPTIONS,POST" + resp[ + "Access-Control-Allow-Headers" + ] = "Origin, X-Requested-With, Content-Type, Accept, Authorization, Referer" + return resp + + def get(self, *args, **kwargs): + return render( + self.request, + "analytics/scripts/page.js", + context={"endpoint": self.request.build_absolute_uri()}, + content_type="application/javascript", + ) + + def post(self, *args, **kwargs): + payload = json.loads(self.request.body) + ingress( + self.request, + self.kwargs.get("service_uuid"), + self.kwargs.get("identifier", ""), + "JS", + payload, + ) + return HttpResponse( + json.dumps({"status": "OK"}), content_type="application/json" + ) diff --git a/shynet/core/admin.py b/shynet/core/admin.py index dc50dc4..168fe7b 100644 --- a/shynet/core/admin.py +++ b/shynet/core/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin from django.contrib.auth.admin import UserAdmin -from .models import User, Service + +from .models import Service, User admin.site.register(User, UserAdmin) -admin.site.register(Service) \ No newline at end of file +admin.site.register(Service) diff --git a/shynet/core/apps.py b/shynet/core/apps.py index 26f78a8..5ef1d60 100644 --- a/shynet/core/apps.py +++ b/shynet/core/apps.py @@ -2,4 +2,4 @@ from django.apps import AppConfig class CoreConfig(AppConfig): - name = 'core' + name = "core" diff --git a/shynet/core/migrations/0001_initial.py b/shynet/core/migrations/0001_initial.py index ee4d8c3..d058e5a 100644 --- a/shynet/core/migrations/0001_initial.py +++ b/shynet/core/migrations/0001_initial.py @@ -1,11 +1,12 @@ -# Generated by Django 3.0.5 on 2020-04-09 19:01 +# Generated by Django 3.0.5 on 2020-04-10 06:58 -from django.conf import settings import django.contrib.auth.models -from django.db import migrations, models import django.db.models.deletion import django.utils.timezone -import uuid +from django.conf import settings +from django.db import migrations, models + +import core.models class Migration(migrations.Migration): @@ -13,47 +14,146 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('auth', '0011_update_proxy_permissions'), + ("auth", "0011_update_proxy_permissions"), ] operations = [ migrations.CreateModel( - name='User', + name="User", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('password', models.CharField(max_length=128, verbose_name='password')), - ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), - ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), - ('first_name', models.CharField(blank=True, max_length=30, verbose_name='first name')), - ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')), - ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), - ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), - ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), - ('username', models.TextField(default=uuid.uuid4, unique=True)), - ('email', models.EmailField(max_length=254, unique=True)), - ('verified', models.BooleanField(default=False)), - ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.Group', verbose_name='groups')), - ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("password", models.CharField(max_length=128, verbose_name="password")), + ( + "last_login", + models.DateTimeField( + blank=True, null=True, verbose_name="last login" + ), + ), + ( + "is_superuser", + models.BooleanField( + default=False, + help_text="Designates that this user has all permissions without explicitly assigning them.", + verbose_name="superuser status", + ), + ), + ( + "first_name", + models.CharField( + blank=True, max_length=30, verbose_name="first name" + ), + ), + ( + "last_name", + models.CharField( + blank=True, max_length=150, verbose_name="last name" + ), + ), + ( + "is_staff", + models.BooleanField( + default=False, + help_text="Designates whether the user can log into this admin site.", + verbose_name="staff status", + ), + ), + ( + "is_active", + models.BooleanField( + default=True, + help_text="Designates whether this user should be treated as active. Unselect this instead of deleting accounts.", + verbose_name="active", + ), + ), + ( + "date_joined", + models.DateTimeField( + default=django.utils.timezone.now, verbose_name="date joined" + ), + ), + ( + "username", + models.TextField(default=core.models._default_uuid, unique=True), + ), + ("email", models.EmailField(max_length=254, unique=True)), + ( + "groups", + models.ManyToManyField( + blank=True, + help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.", + related_name="user_set", + related_query_name="user", + to="auth.Group", + verbose_name="groups", + ), + ), + ( + "user_permissions", + models.ManyToManyField( + blank=True, + help_text="Specific permissions for this user.", + related_name="user_set", + related_query_name="user", + to="auth.Permission", + verbose_name="user permissions", + ), + ), ], options={ - 'verbose_name': 'user', - 'verbose_name_plural': 'users', - 'abstract': False, + "verbose_name": "user", + "verbose_name_plural": "users", + "abstract": False, }, - managers=[ - ('objects', django.contrib.auth.models.UserManager()), - ], + managers=[("objects", django.contrib.auth.models.UserManager()),], ), migrations.CreateModel( - name='Service', + name="Service", fields=[ - ('uuid', models.UUIDField(primary_key=True, serialize=False)), - ('name', models.TextField(max_length=64)), - ('created', models.DateTimeField(auto_now_add=True)), - ('link', models.URLField(blank=True)), - ('status', models.CharField(choices=[('AC', 'Active'), ('AR', 'Archived')], default='AC', max_length=2)), - ('collaborators', models.ManyToManyField(related_name='collaborating_services', to=settings.AUTH_USER_MODEL)), - ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='owning_services', to=settings.AUTH_USER_MODEL)), + ( + "uuid", + models.UUIDField( + default=core.models._default_uuid, + primary_key=True, + serialize=False, + ), + ), + ("name", models.TextField(max_length=64)), + ("created", models.DateTimeField(auto_now_add=True)), + ("link", models.URLField(blank=True)), + ("origins", models.TextField(default="*")), + ( + "status", + models.CharField( + choices=[("AC", "Active"), ("AR", "Archived")], + db_index=True, + default="AC", + max_length=2, + ), + ), + ( + "collaborators", + models.ManyToManyField( + blank=True, + related_name="collaborating_services", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "owner", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="owning_services", + to=settings.AUTH_USER_MODEL, + ), + ), ], ), ] diff --git a/shynet/core/models.py b/shynet/core/models.py index 9f15f3f..7faa66b 100644 --- a/shynet/core/models.py +++ b/shynet/core/models.py @@ -1,23 +1,37 @@ -from django.db import models import uuid + from django.contrib.auth.models import AbstractUser +from django.db import models + + +def _default_uuid(): + return str(uuid.uuid4()) + class User(AbstractUser): - username = models.TextField(default=lambda: str(uuid.uuid4()), unique=True) + username = models.TextField(default=_default_uuid, unique=True) email = models.EmailField(unique=True) def __str__(self): return self.email + class Service(models.Model): ACTIVE = "AC" ARCHIVED = "AR" SERVICE_STATUSES = [(ACTIVE, "Active"), (ARCHIVED, "Archived")] - uuid = models.UUIDField(primary_key=True) + uuid = models.UUIDField(default=_default_uuid, primary_key=True) name = models.TextField(max_length=64) - owner = models.ForeignKey(User, on_delete=models.CASCADE, related_name="owning_services") - collaborators = models.ManyToManyField(User, related_name="collaborating_services") + owner = models.ForeignKey( + User, on_delete=models.CASCADE, related_name="owning_services" + ) + collaborators = models.ManyToManyField( + User, related_name="collaborating_services", blank=True + ) created = models.DateTimeField(auto_now_add=True) link = models.URLField(blank=True) - status = models.CharField(max_length=2, choices=SERVICE_STATUSES, default=ACTIVE) + origins = models.TextField(default="*") + status = models.CharField( + max_length=2, choices=SERVICE_STATUSES, default=ACTIVE, db_index=True + ) diff --git a/shynet/core/templates/base.html b/shynet/core/templates/base.html new file mode 100644 index 0000000..b551b37 --- /dev/null +++ b/shynet/core/templates/base.html @@ -0,0 +1,50 @@ + + + + + + {% block head_title %}Privacy-oriented analytics{% endblock %} | Shynet + + + + + {% block extra_head %} + {% endblock %} + + + + {% block body %} + +
+ {% if messages %} +
+ {% for message in messages %} +
{{message}}
+ {% endfor %} + +
+ {% endif %} + +
+ Menu: + +
+
+ {% block content %} + {% endblock %} +
+
+ {% endblock %} + {% block extra_body %} + {% endblock %} + + + \ No newline at end of file diff --git a/shynet/core/templates/core/base.html b/shynet/core/templates/core/base.html deleted file mode 100644 index a74f3ab..0000000 --- a/shynet/core/templates/core/base.html +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - {{title|default:"Privacy-oriented analytics"}} | Shynet - - - - - - - -
- {% block body %} - {% endblock %} -
- - - \ No newline at end of file diff --git a/shynet/core/templates/core/pages/index.html b/shynet/core/templates/core/pages/index.html index 4975a75..dd0d456 100644 --- a/shynet/core/templates/core/pages/index.html +++ b/shynet/core/templates/core/pages/index.html @@ -1,9 +1,9 @@ -{% extends "core/base.html" %} +{% extends "base.html" %} -{% block body %} -
+{% block content %} +

Shynet Analytics

Eventually, more information about Shynet will be available here.

Log In -
+ {% endblock %} \ No newline at end of file diff --git a/shynet/core/urls.py b/shynet/core/urls.py index 7e8740d..9891109 100644 --- a/shynet/core/urls.py +++ b/shynet/core/urls.py @@ -1,8 +1,8 @@ from django.contrib import admin -from django.urls import path, include +from django.urls import include, path from . import views urlpatterns = [ path("", views.IndexView.as_view(), name="index"), -] \ No newline at end of file +] diff --git a/shynet/core/views.py b/shynet/core/views.py index a4a7e51..dc27844 100644 --- a/shynet/core/views.py +++ b/shynet/core/views.py @@ -1,4 +1,5 @@ from django.views.generic import TemplateView + class IndexView(TemplateView): - template_name = "core/pages/index.html" \ No newline at end of file + template_name = "core/pages/index.html" diff --git a/shynet/manage.py b/shynet/manage.py index 3e6373c..1f0fb41 100755 --- a/shynet/manage.py +++ b/shynet/manage.py @@ -5,7 +5,7 @@ import sys def main(): - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'shynet.settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "shynet.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: @@ -17,5 +17,5 @@ def main(): execute_from_command_line(sys.argv) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/shynet/shynet/__init__.py b/shynet/shynet/__init__.py index e69de29..53f4ccb 100644 --- a/shynet/shynet/__init__.py +++ b/shynet/shynet/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/shynet/shynet/celery.py b/shynet/shynet/celery.py new file mode 100644 index 0000000..f2427b4 --- /dev/null +++ b/shynet/shynet/celery.py @@ -0,0 +1,11 @@ +import os + +from celery import Celery + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "shynet.settings") + +app = Celery("shynet") + +app.config_from_object("django.conf:settings", namespace="CELERY") + +app.autodiscover_tasks() diff --git a/shynet/shynet/settings.py b/shynet/shynet/settings.py index c05f41d..371ec7b 100644 --- a/shynet/shynet/settings.py +++ b/shynet/shynet/settings.py @@ -38,10 +38,11 @@ INSTALLED_APPS = [ "django.contrib.messages", "django.contrib.staticfiles", "django.contrib.sites", + "core", + "analytics", "allauth", "allauth.account", "allauth.socialaccount", - "core", ] MIDDLEWARE = [ @@ -134,3 +135,16 @@ ACCOUNT_USER_MODEL_USERNAME_FIELD = None ACCOUNT_USERNAME_REQUIRED = False SITE_ID = 1 + +# Celery + +if DEBUG: + CELERY_TASK_ALWAYS_EAGER = True + +CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL") +CELERY_REDIS_SOCKET_TIMEOUT = 15 + +# GeoIP + +MAXMIND_CITY_DB = os.getenv("MAXMIND_CITY_DB") +MAXMIND_ASN_DB = os.getenv("MAXMIND_ASN_DB") diff --git a/shynet/shynet/urls.py b/shynet/shynet/urls.py index 39aaecc..1a3dba7 100644 --- a/shynet/shynet/urls.py +++ b/shynet/shynet/urls.py @@ -14,10 +14,11 @@ Including another URLconf 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin -from django.urls import path, include +from django.urls import include, path urlpatterns = [ path("admin/", admin.site.urls), path("accounts/", include("allauth.urls")), + path("ingress/", include("analytics.ingress_urls"), name="ingress"), path("", include(("core.urls", "core"), namespace="core")), ] diff --git a/shynet/shynet/wsgi.py b/shynet/shynet/wsgi.py index be31a76..ebe74ed 100644 --- a/shynet/shynet/wsgi.py +++ b/shynet/shynet/wsgi.py @@ -11,6 +11,6 @@ import os from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'shynet.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "shynet.settings") application = get_wsgi_application() diff --git a/tests/pixel.html b/tests/pixel.html new file mode 100644 index 0000000..c6749f2 --- /dev/null +++ b/tests/pixel.html @@ -0,0 +1,24 @@ + + + + + + Pixel test + + + + + + + + + \ No newline at end of file