Preliminary monitoring

Pipfile

@@ -9,6 +9,12 @@ black = "*"
 [packages]
 django = "*"
 django-allauth = "*"
+geoip2 = "*"
+celery = "*"
+django-ipware = "*"
+pyyaml = "*"
+ua-parser = "*"
+user-agents = "*"
 
 [requires]
 python_version = "3.6"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "75719a3500a5a6da268121b31defc32235442e00bca1872232a93b5db41b61c7"
+            "sha256": "03f23e0c7409df9eb5a85b3df03d1975a1fe5563496602492cdcff46b8c5c829"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -16,6 +16,13 @@
         ]
     },
     "default": {
+        "amqp": {
+            "hashes": [
+                "sha256:6e649ca13a7df3faacdc8bbb280aa9a6602d22fd9d545336077e573a1f4ff3b8",
+                "sha256:77f1aef9410698d20eaeac5b73a87817365f457a507d82edf292e12cbb83b08d"
+            ],
+            "version": "==2.5.2"
+        },
         "asgiref": {
             "hashes": [
                 "sha256:8036f90603c54e93521e5777b2b9a39ba1bad05773fcf2d208f0299d1df58ce5",
@@ -23,6 +30,21 @@
             ],
             "version": "==3.2.7"
         },
+        "billiard": {
+            "hashes": [
+                "sha256:bff575450859a6e0fbc2f9877d9b715b0bbc07c3565bb7ed2280526a0cdf5ede",
+                "sha256:d91725ce6425f33a97dfa72fb6bfef0e47d4652acd98a032bd1a7fbf06d5fa6a"
+            ],
+            "version": "==3.6.3.0"
+        },
+        "celery": {
+            "hashes": [
+                "sha256:108a0bf9018a871620936c33a3ee9f6336a89f8ef0a0f567a9001f4aa361415f",
+                "sha256:5b4b37e276033fe47575107a2775469f0b721646a08c96ec2c61531e4fe45f2a"
+            ],
+            "index": "pypi",
+            "version": "==4.4.2"
+        },
         "certifi": {
             "hashes": [
                 "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304",
@@ -59,6 +81,21 @@
             "index": "pypi",
             "version": "==0.41.0"
         },
+        "django-ipware": {
+            "hashes": [
+                "sha256:a7c7a8fd019dbdc9c357e6e582f65034e897572fc79a7e467674efa8aef9d00b"
+            ],
+            "index": "pypi",
+            "version": "==2.1.0"
+        },
+        "geoip2": {
+            "hashes": [
+                "sha256:5869e987bc54c0d707264fec4710661332cc38d2dca5a7f9bb5362d0308e2ce0",
+                "sha256:99ec12d2f1271a73a0a4a2b663fe6ce25fd02289c0a6bef05c0a1c3b30ee95a4"
+            ],
+            "index": "pypi",
+            "version": "==3.0.0"
+        },
         "idna": {
             "hashes": [
                 "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb",
@@ -66,6 +103,27 @@
             ],
             "version": "==2.9"
         },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f",
+                "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==1.6.0"
+        },
+        "kombu": {
+            "hashes": [
+                "sha256:2d1cda774126a044d91a7ff5fa6d09edf99f46924ab332a810760fe6740e9b76",
+                "sha256:598e7e749d6ab54f646b74b2d2df67755dee13894f73ab02a2a9feb8870c7cb2"
+            ],
+            "version": "==4.6.8"
+        },
+        "maxminddb": {
+            "hashes": [
+                "sha256:d0ce131d901eb11669996b49a59f410efd3da2c6dbe2c0094fe2fef8d85b6336"
+            ],
+            "version": "==1.5.2"
+        },
         "oauthlib": {
             "hashes": [
                 "sha256:bee41cc35fcca6e988463cacc3bcb8a96224f470ca547e697b604cc697b2f889",
@@ -87,6 +145,23 @@
             ],
             "version": "==2019.3"
         },
+        "pyyaml": {
+            "hashes": [
+                "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97",
+                "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76",
+                "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2",
+                "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648",
+                "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf",
+                "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f",
+                "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2",
+                "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee",
+                "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d",
+                "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c",
+                "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"
+            ],
+            "index": "pypi",
+            "version": "==5.3.1"
+        },
         "requests": {
             "hashes": [
                 "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee",
@@ -108,12 +183,42 @@
             ],
             "version": "==0.3.1"
         },
+        "ua-parser": {
+            "hashes": [
+                "sha256:46ab2e383c01dbd2ab284991b87d624a26a08f72da4d7d413f5bfab8b9036f8a",
+                "sha256:47b1782ed130d890018d983fac37c2a80799d9e0b9c532e734c67cf70f185033"
+            ],
+            "index": "pypi",
+            "version": "==0.10.0"
+        },
         "urllib3": {
             "hashes": [
                 "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc",
                 "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"
             ],
             "version": "==1.25.8"
+        },
+        "user-agents": {
+            "hashes": [
+                "sha256:da54371d856c35d8ead0622da24ad5ef6d667eda3629a750e3373a3e847a054b",
+                "sha256:e727ab6f169e829bc25d41dbd25b9ff679b4631bd81959bcf7de1e246da67194"
+            ],
+            "index": "pypi",
+            "version": "==2.1"
+        },
+        "vine": {
+            "hashes": [
+                "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87",
+                "sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af"
+            ],
+            "version": "==1.3.0"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b",
+                "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96"
+            ],
+            "version": "==3.1.0"
         }
     },
     "develop": {

shynet/analytics/admin.py

@@ -0,0 +1,9 @@
+from django.contrib import admin
+
+from .models import Hit, Session
+
+admin.site.register(Session)
+
+admin.site.register(Hit)
+
+# Register your models here.

shynet/analytics/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class AnalyticsConfig(AppConfig):
+    name = "analytics"

shynet/analytics/ingress_urls.py

@@ -0,0 +1,23 @@
+from django.contrib import admin
+from django.urls import include, path
+
+from .views import ingress
+
+urlpatterns = [
+    path(
+        "<service_uuid>/pixel.gif", ingress.PixelView.as_view(), name="endpoint_pixel"
+    ),
+    path(
+        "<service_uuid>/script.js", ingress.ScriptView.as_view(), name="endpoint_script"
+    ),
+    path(
+        "<service_uuid>/<identifier>/pixel.gif",
+        ingress.PixelView.as_view(),
+        name="endpoint_pixel_id",
+    ),
+    path(
+        "<service_uuid>/<identifier>/script.js",
+        ingress.ScriptView.as_view(),
+        name="endpoint_pixel_id",
+    ),
+]

shynet/analytics/migrations/0001_initial.py

@@ -0,0 +1,65 @@
+# Generated by Django 3.0.5 on 2020-04-10 06:58
+
+from django.db import migrations, models
+
+import analytics.models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="Hit",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("start", models.DateTimeField(auto_now_add=True)),
+                ("duration", models.FloatField(default=0.0)),
+                ("heartbeats", models.IntegerField(default=0)),
+                ("tracker", models.TextField()),
+                ("location", models.TextField(blank=True)),
+                ("referrer", models.TextField(blank=True)),
+                ("loadTime", models.FloatField(null=True)),
+                ("httpStatus", models.IntegerField(null=True)),
+                ("metadata_raw", models.TextField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name="Session",
+            fields=[
+                (
+                    "uuid",
+                    models.UUIDField(
+                        default=analytics.models._default_uuid,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                ("identifier", models.TextField(blank=True)),
+                ("first_seen", models.DateTimeField(auto_now_add=True)),
+                ("last_seen", models.DateTimeField(auto_now_add=True)),
+                ("user_agent", models.TextField()),
+                ("browser", models.TextField()),
+                ("device", models.TextField()),
+                ("os", models.TextField()),
+                ("ip", models.GenericIPAddressField()),
+                ("asn", models.TextField(blank=True)),
+                ("country", models.TextField(blank=True)),
+                ("longitude", models.FloatField(null=True)),
+                ("latitude", models.FloatField(null=True)),
+                ("time_zone", models.TextField(blank=True)),
+                ("metadata_raw", models.TextField()),
+            ],
+        ),
+    ]

shynet/analytics/migrations/0002_auto_20200410_0258.py

@@ -0,0 +1,31 @@
+# Generated by Django 3.0.5 on 2020-04-10 06:58
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ("core", "0001_initial"),
+        ("analytics", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="session",
+            name="service",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE, to="core.Service"
+            ),
+        ),
+        migrations.AddField(
+            model_name="hit",
+            name="session",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE, to="analytics.Session"
+            ),
+        ),
+    ]

shynet/analytics/models.py

@@ -0,0 +1,72 @@
+import json
+import uuid
+
+from django.db import models
+
+from core.models import Service
+
+
+def _default_uuid():
+    return str(uuid.uuid4())
+
+
+class Session(models.Model):
+    uuid = models.UUIDField(default=_default_uuid, primary_key=True)
+    service = models.ForeignKey(Service, on_delete=models.CASCADE)
+
+    # Cross-session identification; optional, and provided by the service
+    identifier = models.TextField(blank=True)
+
+    # Time
+    first_seen = models.DateTimeField(auto_now_add=True)
+    last_seen = models.DateTimeField(auto_now_add=True)
+
+    # Core request information
+    user_agent = models.TextField()
+    browser = models.TextField()
+    device = models.TextField()
+    os = models.TextField()
+    ip = models.GenericIPAddressField()
+
+    # GeoIP data
+    asn = models.TextField(blank=True)
+    country = models.TextField(blank=True)
+    longitude = models.FloatField(null=True)
+    latitude = models.FloatField(null=True)
+    time_zone = models.TextField(blank=True)
+
+    # Additional metadata, stored as JSON string
+    metadata_raw = models.TextField()
+
+    @property
+    def metadata(self):
+        try:
+            return json.loads(self.metadata_raw)
+        except:  # Metadata is not crucial; in the case of a read error, just ignore it
+            return {}
+
+
+class Hit(models.Model):
+    session = models.ForeignKey(Session, on_delete=models.CASCADE)
+
+    # Base request information
+    start = models.DateTimeField(auto_now_add=True)
+    duration = models.FloatField(default=0.0)  # Seconds spent on page
+    heartbeats = models.IntegerField(default=0)
+    tracker = models.TextField()  # Tracking pixel or JS
+
+    # Advanced page information
+    location = models.TextField(blank=True)
+    referrer = models.TextField(blank=True)
+    loadTime = models.FloatField(null=True)
+    httpStatus = models.IntegerField(null=True)
+
+    # Additional metadata, stored as JSON string
+    metadata_raw = models.TextField()
+
+    @property
+    def metadata(self):
+        try:
+            return json.loads(self.metadata_raw)
+        except:  # Metadata is not crucial; in the case of a read error, just ignore it
+            return {}

shynet/analytics/tasks.py

@@ -0,0 +1,125 @@
+import json
+import logging
+
+import geoip2.database
+import user_agents
+from celery import shared_task
+from django.conf import settings
+from django.core.cache import cache
+from django.utils import timezone
+
+from core.models import Service
+
+from .models import Hit, Session
+
+log = logging.getLogger(__name__)
+
+_geoip2_city_reader = None
+_geoip2_asn_reader = None
+
+
+def _geoip2_lookup(ip):
+    global _geoip2_city_reader, _geoip2_asn_reader  # TODO: is there a better way to do global Django vars? Is this thread safe?
+    try:
+        if settings.MAXMIND_CITY_DB == None or settings.MAXMIND_ASN_DB == None:
+            return None
+        if _geoip2_city_reader == None or _geoip2_asn_reader == None:
+            _geoip2_city_reader = geoip2.database.Reader(settings.MAXMIND_CITY_DB)
+            _geoip2_asn_reader = geoip2.database.Reader(settings.MAXMIND_ASN_DB)
+        city_results = _geoip2_city_reader.city(ip)
+        asn_results = _geoip2_asn_reader.asn(ip)
+        return {
+            "asn": asn_results.autonomous_system_organization,
+            "country": city_results.country.iso_code,
+            "longitude": city_results.location.longitude,
+            "latitude": city_results.location.latitude,
+            "time_zone": city_results.location.time_zone,
+        }
+    except geoip2.errors.AddressNotFoundError:
+        return {}
+
+
+@shared_task
+def ingress_request(
+    service_uuid, tracker, time, payload, ip, location, user_agent, identifier=""
+):
+    try:
+        ip_data = _geoip2_lookup(ip)
+
+        service = Service.objects.get(uuid=service_uuid)
+        log.debug(f"Linked to service {service}")
+
+        # Create or update session
+        session_metadata = payload.get("sessionMetadata", {})
+        session = Session.objects.filter(
+            service=service,
+            last_seen__gt=timezone.now() - timezone.timedelta(minutes=30),
+            ip=ip,
+            user_agent=user_agent,
+            identifier=identifier,
+        ).first()
+        if session is None:
+            log.debug("Cannot link to existing session; creating a new one...")
+            ua = user_agents.parse(user_agent)
+
+            session = Session.objects.create(
+                service=service,
+                ip=ip,
+                user_agent=user_agent,
+                identifier=identifier,
+                browser=f"{ua.browser.family or ''} {ua.browser.version_string or ''}".strip(),
+                device=f"{ua.device.model or ''}",
+                os=f"{ua.os.family or ''} {ua.os.version_string or ''}".strip(),
+                metadata_raw=json.dumps(session_metadata),
+                asn=ip_data.get("asn", ""),
+                country=ip_data.get("country", ""),
+                longitude=ip_data.get("longitude"),
+                latitude=ip_data.get("latitude"),
+                time_zone=ip_data.get("time_zone", ""),
+            )
+        else:
+            log.debug("Updating old session with new data...")
+            # Update old metadata with new metadata
+            new_metadata = session.metadata
+            new_metadata.update(session_metadata)
+            session.metadata_raw = json.dumps(new_metadata)
+            # Update last seen time
+            session.last_seen = timezone.now()
+            session.save()
+
+        # Create or update hit
+        hit_metadata = payload.get("hitMetadata", {})
+        idempotency = payload.get("idempotency")
+        idempotency_path = f"hit_idempotency_{idempotency}"
+        hit = None
+        if idempotency is not None:
+            if cache.get(idempotency_path) is not None:
+                hit = Hit.objects.filter(
+                    pk=cache.get(idempotency_path), session=session
+                ).first()
+                if hit is not None:
+                    # There is an existing hit with an identical idempotency key. That means
+                    # this is a heartbeat.
+                    log.debug("Hit is a heartbeat; updating old hit with new data...")
+                    hit.heartbeats += 1
+                    hit.duration = (timezone.now() - hit.start).total_seconds()
+                    new_metadata = hit.metadata
+                    new_metadata.update(hit_metadata)
+                    hit.metadata_raw = json.dumps(new_metadata)
+                    hit.save()
+        if hit is None:
+            log.debug("Hit is a page load; creating new hit...")
+            # There is no existing hit; create a new one
+            hit = Hit.objects.create(
+                session=session,
+                tracker=tracker,
+                location=location,
+                referrer=payload.get("referrer", ""),
+                loadTime=payload.get("loadTime"),
+                metadata_raw=json.dumps(hit_metadata),
+            )
+            # Set idempotency (if applicable)
+            if idempotency is not None:
+                cache.set(idempotency_path, hit.pk, timeout=30 * 60)
+    except Exception as e:
+        log.error(e)

shynet/analytics/templates/analytics/scripts/page.js

@@ -0,0 +1,27 @@
+window.onload = function () {
+  var idempotency =
+    Math.random().toString(36).substring(2, 15) +
+    Math.random().toString(36).substring(2, 15);
+  function sendUpdate() {
+    var xhr = new XMLHttpRequest();
+    xhr.open("POST", "{{endpoint}}", true);
+    xhr.setRequestHeader("Content-Type", "application/json");
+    xhr.send(
+      JSON.stringify({
+        idempotency: idempotency,
+        referrer: document.referrer,
+        loadTime:
+          window.performance.timing.domContentLoadedEventEnd -
+          window.performance.timing.navigationStart,
+        hitMetadata:
+          typeof shynetHitMetadata !== "undefined" ? shynetHitMetadata : {},
+        sessionMetadata:
+          typeof shynetSessionMetadata !== "undefined"
+            ? shynetSessionMetadata
+            : {},
+      })
+    );
+  }
+  setInterval(sendUpdate, 5000);
+  sendUpdate();
+};

shynet/analytics/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

shynet/analytics/views/ingress.py

@@ -0,0 +1,85 @@
+import base64
+import json
+
+from django.http import HttpResponse
+from django.shortcuts import render
+from django.utils import timezone
+from django.utils.decorators import method_decorator
+from django.views.decorators.csrf import csrf_exempt
+from django.views.generic import TemplateView, View
+from ipware import get_client_ip
+
+from ..tasks import ingress_request
+
+
+def ingress(request, service_uuid, identifier, tracker, payload):
+    time = timezone.now()
+    client_ip, is_routable = get_client_ip(request)
+    location = request.META.get("HTTP_REFERER")
+    user_agent = request.META.get("HTTP_USER_AGENT")
+
+    ingress_request.delay(
+        service_uuid,
+        tracker,
+        time,
+        payload,
+        client_ip,
+        location,
+        user_agent,
+        identifier,
+    )
+
+
+class PixelView(View):
+    # Fallback view to serve an unobtrusive 1x1 transparent tracking pixel for browsers with
+    # JavaScript disabled.
+    def dispatch(self, request, *args, **kwargs):
+        # Extract primary data
+        ingress(
+            request,
+            self.kwargs.get("service_uuid"),
+            self.kwargs.get("identifier", ""),
+            "PIXEL",
+            {},
+        )
+
+        data = base64.b64decode(
+            "R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="
+        )
+        resp = HttpResponse(data, content_type="image/gif")
+        resp["Cache-Control"] = "no-cache"
+        resp["Access-Control-Allow-Origin"] = "*"
+        return resp
+
+
+@method_decorator(csrf_exempt, name="dispatch")
+class ScriptView(View):
+    def dispatch(self, request, *args, **kwargs):
+        resp = super().dispatch(request, *args, **kwargs)
+        resp["Access-Control-Allow-Origin"] = "*"
+        resp["Access-Control-Allow-Methods"] = "GET,HEAD,OPTIONS,POST"
+        resp[
+            "Access-Control-Allow-Headers"
+        ] = "Origin, X-Requested-With, Content-Type, Accept, Authorization, Referer"
+        return resp
+
+    def get(self, *args, **kwargs):
+        return render(
+            self.request,
+            "analytics/scripts/page.js",
+            context={"endpoint": self.request.build_absolute_uri()},
+            content_type="application/javascript",
+        )
+
+    def post(self, *args, **kwargs):
+        payload = json.loads(self.request.body)
+        ingress(
+            self.request,
+            self.kwargs.get("service_uuid"),
+            self.kwargs.get("identifier", ""),
+            "JS",
+            payload,
+        )
+        return HttpResponse(
+            json.dumps({"status": "OK"}), content_type="application/json"
+        )

shynet/core/admin.py

@@ -1,6 +1,7 @@
 from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin
-from .models import User, Service
+
+from .models import Service, User
 
 admin.site.register(User, UserAdmin)
 admin.site.register(Service)

shynet/core/apps.py

@@ -2,4 +2,4 @@ from django.apps import AppConfig
 
 
 class CoreConfig(AppConfig):
-    name = 'core'
+    name = "core"

shynet/core/migrations/0001_initial.py

@@ -1,11 +1,12 @@
-# Generated by Django 3.0.5 on 2020-04-09 19:01
+# Generated by Django 3.0.5 on 2020-04-10 06:58
 
-from django.conf import settings
 import django.contrib.auth.models
-from django.db import migrations, models
 import django.db.models.deletion
 import django.utils.timezone
-import uuid
+from django.conf import settings
+from django.db import migrations, models
+
+import core.models
 
 
 class Migration(migrations.Migration):
@@ -13,47 +14,146 @@ class Migration(migrations.Migration):
     initial = True
 
     dependencies = [
-        ('auth', '0011_update_proxy_permissions'),
+        ("auth", "0011_update_proxy_permissions"),
     ]
 
     operations = [
         migrations.CreateModel(
-            name='User',
+            name="User",
             fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                (
-                ('password', models.CharField(max_length=128, verbose_name='password')),
+                    "id",
-                ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')),
+                    models.AutoField(
-                ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')),
+                        auto_created=True,
-                ('first_name', models.CharField(blank=True, max_length=30, verbose_name='first name')),
+                        primary_key=True,
-                ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')),
+                        serialize=False,
-                ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')),
+                        verbose_name="ID",
-                ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')),
+                    ),
-                ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')),
+                ),
-                ('username', models.TextField(default=uuid.uuid4, unique=True)),
+                ("password", models.CharField(max_length=128, verbose_name="password")),
-                ('email', models.EmailField(max_length=254, unique=True)),
+                (
-                ('verified', models.BooleanField(default=False)),
+                    "last_login",
-                ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.Group', verbose_name='groups')),
+                    models.DateTimeField(
-                ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions')),
+                        blank=True, null=True, verbose_name="last login"
+                    ),
+                ),
+                (
+                    "is_superuser",
+                    models.BooleanField(
+                        default=False,
+                        help_text="Designates that this user has all permissions without explicitly assigning them.",
+                        verbose_name="superuser status",
+                    ),
+                ),
+                (
+                    "first_name",
+                    models.CharField(
+                        blank=True, max_length=30, verbose_name="first name"
+                    ),
+                ),
+                (
+                    "last_name",
+                    models.CharField(
+                        blank=True, max_length=150, verbose_name="last name"
+                    ),
+                ),
+                (
+                    "is_staff",
+                    models.BooleanField(
+                        default=False,
+                        help_text="Designates whether the user can log into this admin site.",
+                        verbose_name="staff status",
+                    ),
+                ),
+                (
+                    "is_active",
+                    models.BooleanField(
+                        default=True,
+                        help_text="Designates whether this user should be treated as active. Unselect this instead of deleting accounts.",
+                        verbose_name="active",
+                    ),
+                ),
+                (
+                    "date_joined",
+                    models.DateTimeField(
+                        default=django.utils.timezone.now, verbose_name="date joined"
+                    ),
+                ),
+                (
+                    "username",
+                    models.TextField(default=core.models._default_uuid, unique=True),
+                ),
+                ("email", models.EmailField(max_length=254, unique=True)),
+                (
+                    "groups",
+                    models.ManyToManyField(
+                        blank=True,
+                        help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.",
+                        related_name="user_set",
+                        related_query_name="user",
+                        to="auth.Group",
+                        verbose_name="groups",
+                    ),
+                ),
+                (
+                    "user_permissions",
+                    models.ManyToManyField(
+                        blank=True,
+                        help_text="Specific permissions for this user.",
+                        related_name="user_set",
+                        related_query_name="user",
+                        to="auth.Permission",
+                        verbose_name="user permissions",
+                    ),
+                ),
             ],
             options={
-                'verbose_name': 'user',
+                "verbose_name": "user",
-                'verbose_name_plural': 'users',
+                "verbose_name_plural": "users",
-                'abstract': False,
+                "abstract": False,
             },
-            managers=[
+            managers=[("objects", django.contrib.auth.models.UserManager()),],
-                ('objects', django.contrib.auth.models.UserManager()),
-            ],
         ),
         migrations.CreateModel(
-            name='Service',
+            name="Service",
             fields=[
-                ('uuid', models.UUIDField(primary_key=True, serialize=False)),
+                (
-                ('name', models.TextField(max_length=64)),
+                    "uuid",
-                ('created', models.DateTimeField(auto_now_add=True)),
+                    models.UUIDField(
-                ('link', models.URLField(blank=True)),
+                        default=core.models._default_uuid,
-                ('status', models.CharField(choices=[('AC', 'Active'), ('AR', 'Archived')], default='AC', max_length=2)),
+                        primary_key=True,
-                ('collaborators', models.ManyToManyField(related_name='collaborating_services', to=settings.AUTH_USER_MODEL)),
+                        serialize=False,
-                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='owning_services', to=settings.AUTH_USER_MODEL)),
+                    ),
+                ),
+                ("name", models.TextField(max_length=64)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("link", models.URLField(blank=True)),
+                ("origins", models.TextField(default="*")),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[("AC", "Active"), ("AR", "Archived")],
+                        db_index=True,
+                        default="AC",
+                        max_length=2,
+                    ),
+                ),
+                (
+                    "collaborators",
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name="collaborating_services",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+                (
+                    "owner",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="owning_services",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
             ],
         ),
     ]

shynet/core/models.py

@@ -1,23 +1,37 @@
-from django.db import models
 import uuid
+
 from django.contrib.auth.models import AbstractUser
+from django.db import models
+
+
+def _default_uuid():
+    return str(uuid.uuid4())
+
 
 class User(AbstractUser):
-    username = models.TextField(default=lambda: str(uuid.uuid4()), unique=True)
+    username = models.TextField(default=_default_uuid, unique=True)
     email = models.EmailField(unique=True)
 
     def __str__(self):
         return self.email
 
+
 class Service(models.Model):
     ACTIVE = "AC"
     ARCHIVED = "AR"
     SERVICE_STATUSES = [(ACTIVE, "Active"), (ARCHIVED, "Archived")]
 
-    uuid = models.UUIDField(primary_key=True)
+    uuid = models.UUIDField(default=_default_uuid, primary_key=True)
     name = models.TextField(max_length=64)
-    owner = models.ForeignKey(User, on_delete=models.CASCADE, related_name="owning_services")
+    owner = models.ForeignKey(
-    collaborators = models.ManyToManyField(User, related_name="collaborating_services")
+        User, on_delete=models.CASCADE, related_name="owning_services"
+    )
+    collaborators = models.ManyToManyField(
+        User, related_name="collaborating_services", blank=True
+    )
     created = models.DateTimeField(auto_now_add=True)
     link = models.URLField(blank=True)
-    status = models.CharField(max_length=2, choices=SERVICE_STATUSES, default=ACTIVE)
+    origins = models.TextField(default="*")
+    status = models.CharField(
+        max_length=2, choices=SERVICE_STATUSES, default=ACTIVE, db_index=True
+    )

shynet/core/templates/base.html

@@ -0,0 +1,50 @@
+<!DOCTYPE html>
+
+<html>
+
+<head>
+  <title>{% block head_title %}Privacy-oriented analytics{% endblock %} | Shynet</title>
+  <meta name="robots" content="noindex">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <link href="https://unpkg.com/tailwindcss@^1.0/dist/tailwind.min.css" rel="stylesheet">
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/a17t@latest/dist/a17t.css">
+  {% block extra_head %}
+  {% endblock %}
+</head>
+
+<body class="bg-gray-100 min-h-full">
+  {% block body %}
+
+  <section class="max-w-4xl mx-auto px-6 md:py-12">
+    {% if messages %}
+    <div>
+      {% for message in messages %}
+      <article class="card {{message.tags}} mb-2 w-full">{{message}}</article>
+      {% endfor %}
+      </ul>
+    </div>
+    {% endif %}
+
+    <div>
+      <strong>Menu:</strong>
+      <ul>
+        {% if user.is_authenticated %}
+        <li><a href="{% url 'account_email' %}">Change E-mail</a></li>
+        <li><a href="{% url 'account_logout' %}">Sign Out</a></li>
+        {% else %}
+        <li><a href="{% url 'account_login' %}">Sign In</a></li>
+        <li><a href="{% url 'account_signup' %}">Sign Up</a></li>
+        {% endif %}
+      </ul>
+    </div>
+    <main>
+      {% block content %}
+      {% endblock %}
+    </main>
+  </section>
+  {% endblock %}
+  {% block extra_body %}
+  {% endblock %}
+</body>
+
+</html>

shynet/core/templates/core/base.html

@@ -1,20 +0,0 @@
-<!DOCTYPE html>
-
-<html>
-
-<head>
-    <title>{{title|default:"Privacy-oriented analytics"}} | Shynet</title>
-    <meta name="robots" content="noindex">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <link href="https://unpkg.com/tailwindcss@^1.0/dist/tailwind.min.css" rel="stylesheet">
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/a17t@latest/dist/a17t.css">
-</head>
-
-<body class="bg-gray-100 min-h-full">
-    <section class="max-w-4xl mx-auto px-6 md:py-12">
-        {% block body %}
-        {% endblock %}
-    </section>
-</body>
-
-</html>

shynet/core/templates/core/pages/index.html

@@ -1,9 +1,9 @@
-{% extends "core/base.html" %}
+{% extends "base.html" %}
 
-{% block body %}
+{% block content %}
-<main class="content">
+<section class="content">
     <h2>Shynet Analytics</h2>
     <p>Eventually, more information about Shynet will be available here.</p>
     <a href="{% url 'account_login' %}" class="button ~urge !high">Log In</a>
-</main>
+</section>
 {% endblock %}

shynet/core/urls.py

@@ -1,5 +1,5 @@
 from django.contrib import admin
-from django.urls import path, include
+from django.urls import include, path
 
 from . import views
 

shynet/core/views.py

@@ -1,4 +1,5 @@
 from django.views.generic import TemplateView
 
+
 class IndexView(TemplateView):
     template_name = "core/pages/index.html"

shynet/manage.py

@@ -5,7 +5,7 @@ import sys
 
 
 def main():
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'shynet.settings')
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "shynet.settings")
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:
@@ -17,5 +17,5 @@ def main():
     execute_from_command_line(sys.argv)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

shynet/shynet/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ("celery_app",)

shynet/shynet/celery.py

@@ -0,0 +1,11 @@
+import os
+
+from celery import Celery
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "shynet.settings")
+
+app = Celery("shynet")
+
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+app.autodiscover_tasks()

shynet/shynet/settings.py

@@ -38,10 +38,11 @@ INSTALLED_APPS = [
     "django.contrib.messages",
     "django.contrib.staticfiles",
     "django.contrib.sites",
+    "core",
+    "analytics",
     "allauth",
     "allauth.account",
     "allauth.socialaccount",
-    "core",
 ]
 
 MIDDLEWARE = [
@@ -134,3 +135,16 @@ ACCOUNT_USER_MODEL_USERNAME_FIELD = None
 ACCOUNT_USERNAME_REQUIRED = False
 
 SITE_ID = 1
+
+# Celery
+
+if DEBUG:
+    CELERY_TASK_ALWAYS_EAGER = True
+
+CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL")
+CELERY_REDIS_SOCKET_TIMEOUT = 15
+
+# GeoIP
+
+MAXMIND_CITY_DB = os.getenv("MAXMIND_CITY_DB")
+MAXMIND_ASN_DB = os.getenv("MAXMIND_ASN_DB")

shynet/shynet/urls.py

@@ -14,10 +14,11 @@ Including another URLconf
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 from django.contrib import admin
-from django.urls import path, include
+from django.urls import include, path
 
 urlpatterns = [
     path("admin/", admin.site.urls),
     path("accounts/", include("allauth.urls")),
+    path("ingress/", include("analytics.ingress_urls"), name="ingress"),
     path("", include(("core.urls", "core"), namespace="core")),
 ]

shynet/shynet/wsgi.py

@@ -11,6 +11,6 @@ import os
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'shynet.settings')
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "shynet.settings")
 
 application = get_wsgi_application()

tests/pixel.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+
+<html>
+
+<head>
+    <title>Pixel test</title>
+</head>
+
+<body>
+    <noscript>
+        <img src="http://localhost:8000/ingress/fc4008d3-f2fa-4500-9968-d96719e3819c/pixel.gif">
+    </noscript>
+    <script>
+        var shynetSessionMetadata = {
+            session: "this is some session metadata",
+        };
+        var shynetHitMetadata = {
+            hit: "this is some hit metadata",
+        };
+    </script>
+    <script src="http://localhost:8000/ingress/fc4008d3-f2fa-4500-9968-d96719e3819c/identifier/script.js"></script>
+</body>
+
+</html>