theluyuan/shynet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Preliminary monitoring

Browse Source
This commit is contained in:
R. Miles McCain 2020-04-10 03:25:32 -04:00
parent aab7fbb86a
commit 844c44ae6c
No known key found for this signature in database
GPG Key ID: 91CB47BDDF2671A5
31 changed files with 831 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Pipfile
View File

@ -9,6 +9,12 @@ black = "*"
[packages]
django = "*"
django-allauth = "*"
geoip2 = "*"
celery = "*"
django-ipware = "*"
pyyaml = "*"
ua-parser = "*"
user-agents = "*"
[requires]
python_version = "3.6"

107
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "75719a3500a5a6da268121b31defc32235442e00bca1872232a93b5db41b61c7"
"sha256": "03f23e0c7409df9eb5a85b3df03d1975a1fe5563496602492cdcff46b8c5c829"
},
"pipfile-spec": 6,
"requires": {
@ -16,6 +16,13 @@
]
},
"default": {
"amqp": {
"hashes": [
"sha256:6e649ca13a7df3faacdc8bbb280aa9a6602d22fd9d545336077e573a1f4ff3b8",
"sha256:77f1aef9410698d20eaeac5b73a87817365f457a507d82edf292e12cbb83b08d"
],
"version": "==2.5.2"
},
"asgiref": {
"hashes": [
"sha256:8036f90603c54e93521e5777b2b9a39ba1bad05773fcf2d208f0299d1df58ce5",
@ -23,6 +30,21 @@
],
"version": "==3.2.7"
},
"billiard": {
"hashes": [
"sha256:bff575450859a6e0fbc2f9877d9b715b0bbc07c3565bb7ed2280526a0cdf5ede",
"sha256:d91725ce6425f33a97dfa72fb6bfef0e47d4652acd98a032bd1a7fbf06d5fa6a"
],
"version": "==3.6.3.0"
},
"celery": {
"hashes": [
"sha256:108a0bf9018a871620936c33a3ee9f6336a89f8ef0a0f567a9001f4aa361415f",
"sha256:5b4b37e276033fe47575107a2775469f0b721646a08c96ec2c61531e4fe45f2a"
],
"index": "pypi",
"version": "==4.4.2"
},
"certifi": {
"hashes": [
"sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304",
@ -59,6 +81,21 @@
"index": "pypi",
"version": "==0.41.0"
},
"django-ipware": {
"hashes": [
"sha256:a7c7a8fd019dbdc9c357e6e582f65034e897572fc79a7e467674efa8aef9d00b"
],
"index": "pypi",
"version": "==2.1.0"
},
"geoip2": {
"hashes": [
"sha256:5869e987bc54c0d707264fec4710661332cc38d2dca5a7f9bb5362d0308e2ce0",
"sha256:99ec12d2f1271a73a0a4a2b663fe6ce25fd02289c0a6bef05c0a1c3b30ee95a4"
],
"index": "pypi",
"version": "==3.0.0"
},
"idna": {
"hashes": [
"sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb",
@ -66,6 +103,27 @@
],
"version": "==2.9"
},
"importlib-metadata": {
"hashes": [
"sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f",
"sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e"
],
"markers": "python_version < '3.8'",
"version": "==1.6.0"
},
"kombu": {
"hashes": [
"sha256:2d1cda774126a044d91a7ff5fa6d09edf99f46924ab332a810760fe6740e9b76",
"sha256:598e7e749d6ab54f646b74b2d2df67755dee13894f73ab02a2a9feb8870c7cb2"
],
"version": "==4.6.8"
},
"maxminddb": {
"hashes": [
"sha256:d0ce131d901eb11669996b49a59f410efd3da2c6dbe2c0094fe2fef8d85b6336"
],
"version": "==1.5.2"
},
"oauthlib": {
"hashes": [
"sha256:bee41cc35fcca6e988463cacc3bcb8a96224f470ca547e697b604cc697b2f889",
@ -87,6 +145,23 @@
],
"version": "==2019.3"
},
"pyyaml": {
"hashes": [
"sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97",
"sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76",
"sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2",
"sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648",
"sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf",
"sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f",
"sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2",
"sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee",
"sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d",
"sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c",
"sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"
],
"index": "pypi",
"version": "==5.3.1"
},
"requests": {
"hashes": [
"sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee",
@ -108,12 +183,42 @@
],
"version": "==0.3.1"
},
"ua-parser": {
"hashes": [
"sha256:46ab2e383c01dbd2ab284991b87d624a26a08f72da4d7d413f5bfab8b9036f8a",
"sha256:47b1782ed130d890018d983fac37c2a80799d9e0b9c532e734c67cf70f185033"
],
"index": "pypi",
"version": "==0.10.0"
},
"urllib3": {
"hashes": [
"sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc",
"sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"
],
"version": "==1.25.8"
},
"user-agents": {
"hashes": [
"sha256:da54371d856c35d8ead0622da24ad5ef6d667eda3629a750e3373a3e847a054b",
"sha256:e727ab6f169e829bc25d41dbd25b9ff679b4631bd81959bcf7de1e246da67194"
],
"index": "pypi",
"version": "==2.1"
},
"vine": {
"hashes": [
"sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87",
"sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af"
],
"version": "==1.3.0"
},
"zipp": {
"hashes": [
"sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b",
"sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96"
],
"version": "==3.1.0"
}
},
"develop": {

0
shynet/analytics/__init__.py Normal file
View File

9
shynet/analytics/admin.py Normal file
View File

@ -0,0 +1,9 @@
from django.contrib import admin
from .models import Hit, Session
admin.site.register(Session)
admin.site.register(Hit)
# Register your models here.

5
shynet/analytics/apps.py Normal file
View File

@ -0,0 +1,5 @@
from django.apps import AppConfig
class AnalyticsConfig(AppConfig):
name = "analytics"

23
shynet/analytics/ingress_urls.py Normal file
View File

@ -0,0 +1,23 @@
from django.contrib import admin
from django.urls import include, path
from .views import ingress
urlpatterns = [
path(
"<service_uuid>/pixel.gif", ingress.PixelView.as_view(), name="endpoint_pixel"
),
path(
"<service_uuid>/script.js", ingress.ScriptView.as_view(), name="endpoint_script"
),
path(
"<service_uuid>/<identifier>/pixel.gif",
ingress.PixelView.as_view(),
name="endpoint_pixel_id",
),
path(
"<service_uuid>/<identifier>/script.js",
ingress.ScriptView.as_view(),
name="endpoint_pixel_id",
),
]

65
shynet/analytics/migrations/0001_initial.py Normal file
View File

@ -0,0 +1,65 @@
# Generated by Django 3.0.5 on 2020-04-10 06:58
from django.db import migrations, models
import analytics.models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="Hit",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("start", models.DateTimeField(auto_now_add=True)),
("duration", models.FloatField(default=0.0)),
("heartbeats", models.IntegerField(default=0)),
("tracker", models.TextField()),
("location", models.TextField(blank=True)),
("referrer", models.TextField(blank=True)),
("loadTime", models.FloatField(null=True)),
("httpStatus", models.IntegerField(null=True)),
("metadata_raw", models.TextField()),
],
),
migrations.CreateModel(
name="Session",
fields=[
(
"uuid",
models.UUIDField(
default=analytics.models._default_uuid,
primary_key=True,
serialize=False,
),
),
("identifier", models.TextField(blank=True)),
("first_seen", models.DateTimeField(auto_now_add=True)),
("last_seen", models.DateTimeField(auto_now_add=True)),
("user_agent", models.TextField()),
("browser", models.TextField()),
("device", models.TextField()),
("os", models.TextField()),
("ip", models.GenericIPAddressField()),
("asn", models.TextField(blank=True)),
("country", models.TextField(blank=True)),
("longitude", models.FloatField(null=True)),
("latitude", models.FloatField(null=True)),
("time_zone", models.TextField(blank=True)),
("metadata_raw", models.TextField()),
],
),
]

31
shynet/analytics/migrations/0002_auto_20200410_0258.py Normal file
View File

@ -0,0 +1,31 @@
# Generated by Django 3.0.5 on 2020-04-10 06:58
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
("core", "0001_initial"),
("analytics", "0001_initial"),
]
operations = [
migrations.AddField(
model_name="session",
name="service",
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="core.Service"
),
),
migrations.AddField(
model_name="hit",
name="session",
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="analytics.Session"
),
),
]

0
shynet/analytics/migrations/__init__.py Normal file
View File

72
shynet/analytics/models.py Normal file
View File

@ -0,0 +1,72 @@
import json
import uuid
from django.db import models
from core.models import Service
def _default_uuid():
return str(uuid.uuid4())
class Session(models.Model):
uuid = models.UUIDField(default=_default_uuid, primary_key=True)
service = models.ForeignKey(Service, on_delete=models.CASCADE)
# Cross-session identification; optional, and provided by the service
identifier = models.TextField(blank=True)
# Time
first_seen = models.DateTimeField(auto_now_add=True)
last_seen = models.DateTimeField(auto_now_add=True)
# Core request information
user_agent = models.TextField()
browser = models.TextField()
device = models.TextField()
os = models.TextField()
ip = models.GenericIPAddressField()
# GeoIP data
asn = models.TextField(blank=True)
country = models.TextField(blank=True)
longitude = models.FloatField(null=True)
latitude = models.FloatField(null=True)
time_zone = models.TextField(blank=True)
# Additional metadata, stored as JSON string
metadata_raw = models.TextField()
@property
def metadata(self):
try:
return json.loads(self.metadata_raw)
except: # Metadata is not crucial; in the case of a read error, just ignore it
return {}
class Hit(models.Model):
session = models.ForeignKey(Session, on_delete=models.CASCADE)
# Base request information
start = models.DateTimeField(auto_now_add=True)
duration = models.FloatField(default=0.0) # Seconds spent on page
heartbeats = models.IntegerField(default=0)
tracker = models.TextField() # Tracking pixel or JS
# Advanced page information
location = models.TextField(blank=True)
referrer = models.TextField(blank=True)
loadTime = models.FloatField(null=True)
httpStatus = models.IntegerField(null=True)
# Additional metadata, stored as JSON string
metadata_raw = models.TextField()
@property
def metadata(self):
try:
return json.loads(self.metadata_raw)
except: # Metadata is not crucial; in the case of a read error, just ignore it
return {}

125
shynet/analytics/tasks.py Normal file
View File

@ -0,0 +1,125 @@
import json
import logging
import geoip2.database
import user_agents
from celery import shared_task
from django.conf import settings
from django.core.cache import cache
from django.utils import timezone
from core.models import Service
from .models import Hit, Session
log = logging.getLogger(__name__)
_geoip2_city_reader = None
_geoip2_asn_reader = None
def _geoip2_lookup(ip):
global _geoip2_city_reader, _geoip2_asn_reader # TODO: is there a better way to do global Django vars? Is this thread safe?
try:
if settings.MAXMIND_CITY_DB == None or settings.MAXMIND_ASN_DB == None:
return None
if _geoip2_city_reader == None or _geoip2_asn_reader == None:
_geoip2_city_reader = geoip2.database.Reader(settings.MAXMIND_CITY_DB)
_geoip2_asn_reader = geoip2.database.Reader(settings.MAXMIND_ASN_DB)
city_results = _geoip2_city_reader.city(ip)
asn_results = _geoip2_asn_reader.asn(ip)
return {
"asn": asn_results.autonomous_system_organization,
"country": city_results.country.iso_code,
"longitude": city_results.location.longitude,
"latitude": city_results.location.latitude,
"time_zone": city_results.location.time_zone,
}
except geoip2.errors.AddressNotFoundError:
return {}
@shared_task
def ingress_request(
service_uuid, tracker, time, payload, ip, location, user_agent, identifier=""
):
try:
ip_data = _geoip2_lookup(ip)
service = Service.objects.get(uuid=service_uuid)
log.debug(f"Linked to service {service}")
# Create or update session
session_metadata = payload.get("sessionMetadata", {})
session = Session.objects.filter(
service=service,
last_seen__gt=timezone.now() - timezone.timedelta(minutes=30),
ip=ip,
user_agent=user_agent,
identifier=identifier,
).first()
if session is None:
log.debug("Cannot link to existing session; creating a new one...")
ua = user_agents.parse(user_agent)
session = Session.objects.create(
service=service,
ip=ip,
user_agent=user_agent,
identifier=identifier,
browser=f"{ua.browser.family or ''} {ua.browser.version_string or ''}".strip(),
device=f"{ua.device.model or ''}",
os=f"{ua.os.family or ''} {ua.os.version_string or ''}".strip(),
metadata_raw=json.dumps(session_metadata),
asn=ip_data.get("asn", ""),
country=ip_data.get("country", ""),
longitude=ip_data.get("longitude"),
latitude=ip_data.get("latitude"),
time_zone=ip_data.get("time_zone", ""),
)
else:
log.debug("Updating old session with new data...")
# Update old metadata with new metadata
new_metadata = session.metadata
new_metadata.update(session_metadata)
session.metadata_raw = json.dumps(new_metadata)
# Update last seen time
session.last_seen = timezone.now()
session.save()
# Create or update hit
hit_metadata = payload.get("hitMetadata", {})
idempotency = payload.get("idempotency")
idempotency_path = f"hit_idempotency_{idempotency}"
hit = None
if idempotency is not None:
if cache.get(idempotency_path) is not None:
hit = Hit.objects.filter(
pk=cache.get(idempotency_path), session=session
).first()
if hit is not None:
# There is an existing hit with an identical idempotency key. That means
# this is a heartbeat.
log.debug("Hit is a heartbeat; updating old hit with new data...")
hit.heartbeats += 1
hit.duration = (timezone.now() - hit.start).total_seconds()
new_metadata = hit.metadata
new_metadata.update(hit_metadata)
hit.metadata_raw = json.dumps(new_metadata)
hit.save()
if hit is None:
log.debug("Hit is a page load; creating new hit...")
# There is no existing hit; create a new one
hit = Hit.objects.create(
session=session,
tracker=tracker,
location=location,
referrer=payload.get("referrer", ""),
loadTime=payload.get("loadTime"),
metadata_raw=json.dumps(hit_metadata),
)
# Set idempotency (if applicable)
if idempotency is not None:
cache.set(idempotency_path, hit.pk, timeout=30 * 60)
except Exception as e:
log.error(e)

27
shynet/analytics/templates/analytics/scripts/page.js Normal file
View File

@ -0,0 +1,27 @@
window.onload = function () {
var idempotency =
Math.random().toString(36).substring(2, 15) +
Math.random().toString(36).substring(2, 15);
function sendUpdate() {
var xhr = new XMLHttpRequest();
xhr.open("POST", "{{endpoint}}", true);
xhr.setRequestHeader("Content-Type", "application/json");
xhr.send(
JSON.stringify({
idempotency: idempotency,
referrer: document.referrer,
loadTime:
window.performance.timing.domContentLoadedEventEnd -
window.performance.timing.navigationStart,
hitMetadata:
typeof shynetHitMetadata !== "undefined" ? shynetHitMetadata : {},
sessionMetadata:
typeof shynetSessionMetadata !== "undefined"
? shynetSessionMetadata
: {},
})
);
}
setInterval(sendUpdate, 5000);
sendUpdate();
};

3
shynet/analytics/tests.py Normal file
View File

@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

0
shynet/analytics/views/__init__.py Normal file
View File

85
shynet/analytics/views/ingress.py Normal file
View File

@ -0,0 +1,85 @@
import base64
import json
from django.http import HttpResponse
from django.shortcuts import render
from django.utils import timezone
from django.utils.decorators import method_decorator
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import TemplateView, View
from ipware import get_client_ip
from ..tasks import ingress_request
def ingress(request, service_uuid, identifier, tracker, payload):
time = timezone.now()
client_ip, is_routable = get_client_ip(request)
location = request.META.get("HTTP_REFERER")
user_agent = request.META.get("HTTP_USER_AGENT")
ingress_request.delay(
service_uuid,
tracker,
time,
payload,
client_ip,
location,
user_agent,
identifier,
)
class PixelView(View):
# Fallback view to serve an unobtrusive 1x1 transparent tracking pixel for browsers with
# JavaScript disabled.
def dispatch(self, request, *args, **kwargs):
# Extract primary data
ingress(
request,
self.kwargs.get("service_uuid"),
self.kwargs.get("identifier", ""),
"PIXEL",
{},
)
data = base64.b64decode(
"R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="
)
resp = HttpResponse(data, content_type="image/gif")
resp["Cache-Control"] = "no-cache"
resp["Access-Control-Allow-Origin"] = "*"
return resp
@method_decorator(csrf_exempt, name="dispatch")
class ScriptView(View):
def dispatch(self, request, *args, **kwargs):
resp = super().dispatch(request, *args, **kwargs)
resp["Access-Control-Allow-Origin"] = "*"
resp["Access-Control-Allow-Methods"] = "GET,HEAD,OPTIONS,POST"
resp[
"Access-Control-Allow-Headers"
] = "Origin, X-Requested-With, Content-Type, Accept, Authorization, Referer"
return resp
def get(self, *args, **kwargs):
return render(
self.request,
"analytics/scripts/page.js",
context={"endpoint": self.request.build_absolute_uri()},
content_type="application/javascript",
)
def post(self, *args, **kwargs):
payload = json.loads(self.request.body)
ingress(
self.request,
self.kwargs.get("service_uuid"),
self.kwargs.get("identifier", ""),
"JS",
payload,
)
return HttpResponse(
json.dumps({"status": "OK"}), content_type="application/json"
)

5
shynet/core/admin.py
View File

@ -1,6 +1,7 @@
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from .models import User, Service
from .models import Service, User
admin.site.register(User, UserAdmin)
admin.site.register(Service)
admin.site.register(Service)

2
shynet/core/apps.py
View File

@ -2,4 +2,4 @@ from django.apps import AppConfig
class CoreConfig(AppConfig):
name = 'core'
name = "core"

168
shynet/core/migrations/0001_initial.py
View File

@ -1,11 +1,12 @@
# Generated by Django 3.0.5 on 2020-04-09 19:01
# Generated by Django 3.0.5 on 2020-04-10 06:58
from django.conf import settings
import django.contrib.auth.models
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
import uuid
from django.conf import settings
from django.db import migrations, models
import core.models
class Migration(migrations.Migration):
@ -13,47 +14,146 @@ class Migration(migrations.Migration):
initial = True
dependencies = [
('auth', '0011_update_proxy_permissions'),
("auth", "0011_update_proxy_permissions"),
]
operations = [
migrations.CreateModel(
name='User',
name="User",
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('password', models.CharField(max_length=128, verbose_name='password')),
('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')),
('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')),
('first_name', models.CharField(blank=True, max_length=30, verbose_name='first name')),
('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')),
('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')),
('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')),
('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')),
('username', models.TextField(default=uuid.uuid4, unique=True)),
('email', models.EmailField(max_length=254, unique=True)),
('verified', models.BooleanField(default=False)),
('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.Group', verbose_name='groups')),
('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions')),
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("password", models.CharField(max_length=128, verbose_name="password")),
(
"last_login",
models.DateTimeField(
blank=True, null=True, verbose_name="last login"
),
),
(
"is_superuser",
models.BooleanField(
default=False,
help_text="Designates that this user has all permissions without explicitly assigning them.",
verbose_name="superuser status",
),
),
(
"first_name",
models.CharField(
blank=True, max_length=30, verbose_name="first name"
),
),
(
"last_name",
models.CharField(
blank=True, max_length=150, verbose_name="last name"
),
),
(
"is_staff",
models.BooleanField(
default=False,
help_text="Designates whether the user can log into this admin site.",
verbose_name="staff status",
),
),
(
"is_active",
models.BooleanField(
default=True,
help_text="Designates whether this user should be treated as active. Unselect this instead of deleting accounts.",
verbose_name="active",
),
),
(
"date_joined",
models.DateTimeField(
default=django.utils.timezone.now, verbose_name="date joined"
),
),
(
"username",
models.TextField(default=core.models._default_uuid, unique=True),
),
("email", models.EmailField(max_length=254, unique=True)),
(
"groups",
models.ManyToManyField(
blank=True,
help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.",
related_name="user_set",
related_query_name="user",
to="auth.Group",
verbose_name="groups",
),
),
(
"user_permissions",
models.ManyToManyField(
blank=True,
help_text="Specific permissions for this user.",
related_name="user_set",
related_query_name="user",
to="auth.Permission",
verbose_name="user permissions",
),
),
],
options={
'verbose_name': 'user',
'verbose_name_plural': 'users',
'abstract': False,
"verbose_name": "user",
"verbose_name_plural": "users",
"abstract": False,
},
managers=[
('objects', django.contrib.auth.models.UserManager()),
],
managers=[("objects", django.contrib.auth.models.UserManager()),],
),
migrations.CreateModel(
name='Service',
name="Service",
fields=[
('uuid', models.UUIDField(primary_key=True, serialize=False)),
('name', models.TextField(max_length=64)),
('created', models.DateTimeField(auto_now_add=True)),
('link', models.URLField(blank=True)),
('status', models.CharField(choices=[('AC', 'Active'), ('AR', 'Archived')], default='AC', max_length=2)),
('collaborators', models.ManyToManyField(related_name='collaborating_services', to=settings.AUTH_USER_MODEL)),
('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='owning_services', to=settings.AUTH_USER_MODEL)),
(
"uuid",
models.UUIDField(
default=core.models._default_uuid,
primary_key=True,
serialize=False,
),
),
("name", models.TextField(max_length=64)),
("created", models.DateTimeField(auto_now_add=True)),
("link", models.URLField(blank=True)),
("origins", models.TextField(default="*")),
(
"status",
models.CharField(
choices=[("AC", "Active"), ("AR", "Archived")],
db_index=True,
default="AC",
max_length=2,
),
),
(
"collaborators",
models.ManyToManyField(
blank=True,
related_name="collaborating_services",
to=settings.AUTH_USER_MODEL,
),
),
(
"owner",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="owning_services",
to=settings.AUTH_USER_MODEL,
),
),
],
),
]

26
shynet/core/models.py
View File

@ -1,23 +1,37 @@
from django.db import models
import uuid
from django.contrib.auth.models import AbstractUser
from django.db import models
def _default_uuid():
return str(uuid.uuid4())
class User(AbstractUser):
username = models.TextField(default=lambda: str(uuid.uuid4()), unique=True)
username = models.TextField(default=_default_uuid, unique=True)
email = models.EmailField(unique=True)
def __str__(self):
return self.email
class Service(models.Model):
ACTIVE = "AC"
ARCHIVED = "AR"
SERVICE_STATUSES = [(ACTIVE, "Active"), (ARCHIVED, "Archived")]
uuid = models.UUIDField(primary_key=True)
uuid = models.UUIDField(default=_default_uuid, primary_key=True)
name = models.TextField(max_length=64)
owner = models.ForeignKey(User, on_delete=models.CASCADE, related_name="owning_services")
collaborators = models.ManyToManyField(User, related_name="collaborating_services")
owner = models.ForeignKey(
User, on_delete=models.CASCADE, related_name="owning_services"
)
collaborators = models.ManyToManyField(
User, related_name="collaborating_services", blank=True
)
created = models.DateTimeField(auto_now_add=True)
link = models.URLField(blank=True)
status = models.CharField(max_length=2, choices=SERVICE_STATUSES, default=ACTIVE)
origins = models.TextField(default="*")
status = models.CharField(
max_length=2, choices=SERVICE_STATUSES, default=ACTIVE, db_index=True
)

50
shynet/core/templates/base.html Normal file
View File

@ -0,0 +1,50 @@
<!DOCTYPE html>
<html>
<head>
<title>{% block head_title %}Privacy-oriented analytics{% endblock %} | Shynet</title>
<meta name="robots" content="noindex">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link href="https://unpkg.com/tailwindcss@^1.0/dist/tailwind.min.css" rel="stylesheet">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/a17t@latest/dist/a17t.css">
{% block extra_head %}
{% endblock %}
</head>
<body class="bg-gray-100 min-h-full">
{% block body %}
<section class="max-w-4xl mx-auto px-6 md:py-12">
{% if messages %}
<div>
{% for message in messages %}
<article class="card {{message.tags}} mb-2 w-full">{{message}}</article>
{% endfor %}
</ul>
</div>
{% endif %}
<div>
<strong>Menu:</strong>
<ul>
{% if user.is_authenticated %}
<li><a href="{% url 'account_email' %}">Change E-mail</a></li>
<li><a href="{% url 'account_logout' %}">Sign Out</a></li>
{% else %}
<li><a href="{% url 'account_login' %}">Sign In</a></li>
<li><a href="{% url 'account_signup' %}">Sign Up</a></li>
{% endif %}
</ul>
</div>
<main>
{% block content %}
{% endblock %}
</main>
</section>
{% endblock %}
{% block extra_body %}
{% endblock %}
</body>
</html>

20
shynet/core/templates/core/base.html
View File

@ -1,20 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>{{title|default:"Privacy-oriented analytics"}} | Shynet</title>
<meta name="robots" content="noindex">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link href="https://unpkg.com/tailwindcss@^1.0/dist/tailwind.min.css" rel="stylesheet">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/a17t@latest/dist/a17t.css">
</head>
<body class="bg-gray-100 min-h-full">
<section class="max-w-4xl mx-auto px-6 md:py-12">
{% block body %}
{% endblock %}
</section>
</body>
</html>

8
shynet/core/templates/core/pages/index.html
View File

@ -1,9 +1,9 @@
{% extends "core/base.html" %}
{% extends "base.html" %}
{% block body %}
<main class="content">
{% block content %}
<section class="content">
<h2>Shynet Analytics</h2>
<p>Eventually, more information about Shynet will be available here.</p>
<a href="{% url 'account_login' %}" class="button ~urge !high">Log In</a>
</main>
</section>
{% endblock %}

4
shynet/core/urls.py
View File

@ -1,8 +1,8 @@
from django.contrib import admin
from django.urls import path, include
from django.urls import include, path
from . import views
urlpatterns = [
path("", views.IndexView.as_view(), name="index"),
]
]

3
shynet/core/views.py
View File

@ -1,4 +1,5 @@
from django.views.generic import TemplateView
class IndexView(TemplateView):
template_name = "core/pages/index.html"
template_name = "core/pages/index.html"

4
shynet/manage.py
View File

@ -5,7 +5,7 @@ import sys
def main():
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'shynet.settings')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "shynet.settings")
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
@ -17,5 +17,5 @@ def main():
execute_from_command_line(sys.argv)
if __name__ == '__main__':
if __name__ == "__main__":
main()

3
shynet/shynet/__init__.py
View File

@ -0,0 +1,3 @@
from .celery import app as celery_app
__all__ = ("celery_app",)

11
shynet/shynet/celery.py Normal file
View File

@ -0,0 +1,11 @@
import os
from celery import Celery
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "shynet.settings")
app = Celery("shynet")
app.config_from_object("django.conf:settings", namespace="CELERY")
app.autodiscover_tasks()

16
shynet/shynet/settings.py
View File

@ -38,10 +38,11 @@ INSTALLED_APPS = [
"django.contrib.messages",
"django.contrib.staticfiles",
"django.contrib.sites",
"core",
"analytics",
"allauth",
"allauth.account",
"allauth.socialaccount",
"core",
]
MIDDLEWARE = [
@ -134,3 +135,16 @@ ACCOUNT_USER_MODEL_USERNAME_FIELD = None
ACCOUNT_USERNAME_REQUIRED = False
SITE_ID = 1
# Celery
if DEBUG:
CELERY_TASK_ALWAYS_EAGER = True
CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL")
CELERY_REDIS_SOCKET_TIMEOUT = 15
# GeoIP
MAXMIND_CITY_DB = os.getenv("MAXMIND_CITY_DB")
MAXMIND_ASN_DB = os.getenv("MAXMIND_ASN_DB")

3
shynet/shynet/urls.py
View File

@ -14,10 +14,11 @@ Including another URLconf
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include
from django.urls import include, path
urlpatterns = [
path("admin/", admin.site.urls),
path("accounts/", include("allauth.urls")),
path("ingress/", include("analytics.ingress_urls"), name="ingress"),
path("", include(("core.urls", "core"), namespace="core")),
]

2
shynet/shynet/wsgi.py
View File

@ -11,6 +11,6 @@ import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'shynet.settings')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "shynet.settings")
application = get_wsgi_application()

24
tests/pixel.html Normal file
View File

@ -0,0 +1,24 @@
<!DOCTYPE html>
<html>
<head>
<title>Pixel test</title>
</head>
<body>
<noscript>
<img src="http://localhost:8000/ingress/fc4008d3-f2fa-4500-9968-d96719e3819c/pixel.gif">
</noscript>
<script>
var shynetSessionMetadata = {
session: "this is some session metadata",
};
var shynetHitMetadata = {
hit: "this is some hit metadata",
};
</script>
<script src="http://localhost:8000/ingress/fc4008d3-f2fa-4500-9968-d96719e3819c/identifier/script.js"></script>
</body>
</html>