theluyuan/shynet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add base dashboard view

Browse Source
This commit is contained in:
R. Miles McCain
2020-04-12 01:46:28 -04:00
parent 170a8ec00b
commit 565cba18e2
34 changed files with 762 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
shynet/analytics/migrations/0003_auto_20200410_1325.py
View File

@@ -6,16 +6,10 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('analytics', '0002_auto_20200410_0258'),
("analytics", "0002_auto_20200410_0258"),
]
operations = [
migrations.RemoveField(
model_name='hit',
name='metadata_raw',
),
migrations.RemoveField(
model_name='session',
name='metadata_raw',
),
migrations.RemoveField(model_name="hit", name="metadata_raw",),
migrations.RemoveField(model_name="session", name="metadata_raw",),
]

25
shynet/analytics/migrations/0004_auto_20200411_1541.py
View File

@@ -1,34 +1,29 @@
# Generated by Django 3.0.5 on 2020-04-11 19:41
from django.db import migrations, models
import django.utils.timezone
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('analytics', '0003_auto_20200410_1325'),
("analytics", "0003_auto_20200410_1325"),
]
operations = [
migrations.RenameField(
model_name='hit',
old_name='start',
new_name='start_time',
model_name="hit", old_name="start", new_name="start_time",
),
migrations.RenameField(
model_name='session',
old_name='first_seen',
new_name='start_time',
),
migrations.RemoveField(
model_name='hit',
name='duration',
model_name="session", old_name="first_seen", new_name="start_time",
),
migrations.RemoveField(model_name="hit", name="duration",),
migrations.AddField(
model_name='hit',
name='last_seen',
field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
model_name="hit",
name="last_seen",
field=models.DateTimeField(
auto_now_add=True, default=django.utils.timezone.now
),
preserve_default=False,
),
]

16
shynet/analytics/migrations/0005_hit_initial.py Normal file
View File

@@ -0,0 +1,16 @@
# Generated by Django 3.0.5 on 2020-04-12 03:47
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("analytics", "0004_auto_20200411_1541"),
]
operations = [
migrations.AddField(
model_name="hit", name="initial", field=models.BooleanField(default=True),
),
]

29
shynet/analytics/migrations/0006_auto_20200412_0003.py Normal file
View File

@@ -0,0 +1,29 @@
# Generated by Django 3.0.5 on 2020-04-12 04:03
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("analytics", "0005_hit_initial"),
]
operations = [
migrations.RemoveField(model_name="hit", name="httpStatus",),
migrations.AddField(
model_name="session",
name="device_type",
field=models.CharField(
choices=[
("PHONE", "Phone"),
("TABLET", "Tablet"),
("DESK", "Desktop / Laptop"),
("ROBOT", "Robot"),
("OTHER", "Other"),
],
default="OTHER",
max_length=6,
),
),
]

28
shynet/analytics/migrations/0007_auto_20200412_0010.py Normal file
View File

@@ -0,0 +1,28 @@
# Generated by Django 3.0.5 on 2020-04-12 04:10
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("analytics", "0006_auto_20200412_0003"),
]
operations = [
migrations.AlterField(
model_name="session",
name="device_type",
field=models.CharField(
choices=[
("PHONE", "Phone"),
("TABLET", "Tablet"),
("DESKTOP", "Desktop"),
("ROBOT", "Robot"),
("OTHER", "Other"),
],
default="OTHER",
max_length=7,
),
),
]

16
shynet/analytics/migrations/0008_auto_20200412_0015.py Normal file
View File

@@ -0,0 +1,16 @@
# Generated by Django 3.0.5 on 2020-04-12 04:15
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("analytics", "0007_auto_20200412_0010"),
]
operations = [
migrations.RenameField(
model_name="hit", old_name="loadTime", new_name="load_time",
),
]

25
shynet/analytics/models.py
View File

@@ -2,6 +2,7 @@ import json
import uuid
from django.db import models
from django.utils import timezone
from core.models import Service
@@ -25,6 +26,17 @@ class Session(models.Model):
user_agent = models.TextField()
browser = models.TextField()
device = models.TextField()
device_type = models.CharField(
max_length=7,
choices=[
("PHONE", "Phone"),
("TABLET", "Tablet"),
("DESKTOP", "Desktop"),
("ROBOT", "Robot"),
("OTHER", "Other"),
],
default="OTHER",
)
os = models.TextField()
ip = models.GenericIPAddressField()
@@ -35,9 +47,18 @@ class Session(models.Model):
latitude = models.FloatField(null=True)
time_zone = models.TextField(blank=True)
@property
def is_currently_active(self):
return timezone.now() - self.last_seen < timezone.timedelta(seconds=10)
@property
def duration(self):
return self.last_seen - self.start_time
class Hit(models.Model):
session = models.ForeignKey(Session, on_delete=models.CASCADE)
initial = models.BooleanField(default=True)
# Base request information
start_time = models.DateTimeField(auto_now_add=True)
@@ -48,6 +69,4 @@ class Hit(models.Model):
# Advanced page information
location = models.TextField(blank=True)
referrer = models.TextField(blank=True)
loadTime = models.FloatField(null=True)
httpStatus = models.IntegerField(null=True)
load_time = models.FloatField(null=True)

31
shynet/analytics/tasks.py
View File

@@ -7,7 +7,6 @@ from celery import shared_task
from django.conf import settings
from django.core.cache import cache
from django.utils import timezone
from anonymizeip import anonymize_ip
from core.models import Service
@@ -51,13 +50,10 @@ def ingress_request(
ip_data = _geoip2_lookup(ip)
log.debug(f"Found geoip2 data")
if service.anonymize_ips:
ip = anonymize_ip(ip)
# Create or update session
session = Session.objects.filter(
service=service,
last_seen__gt=timezone.now() - timezone.timedelta(minutes=30),
last_seen__gt=timezone.now() - timezone.timedelta(minutes=10),
ip=ip,
user_agent=user_agent,
identifier=identifier,
@@ -65,15 +61,25 @@ def ingress_request(
if session is None:
log.debug("Cannot link to existing session; creating a new one...")
ua = user_agents.parse(user_agent)
initial = True
device_type = "OTHER"
if ua.is_mobile:
device_type = "PHONE"
elif ua.is_tablet:
device_type = "TABLET"
elif ua.is_pc:
device_type = "DESKTOP"
elif ua.is_bot:
device_type = "ROBOT"
session = Session.objects.create(
service=service,
ip=ip,
user_agent=user_agent,
identifier=identifier,
browser=f"{ua.browser.family or ''} {ua.browser.version_string or ''}".strip(),
device=f"{ua.device.model or ''}",
os=f"{ua.os.family or ''} {ua.os.version_string or ''}".strip(),
browser=ua.browser.family or "",
device=ua.device.model or "",
device_type=device_type,
os=ua.os.family or "",
asn=ip_data.get("asn", ""),
country=ip_data.get("country", ""),
longitude=ip_data.get("longitude"),
@@ -82,6 +88,7 @@ def ingress_request(
)
else:
log.debug("Updating old session with new data...")
initial = False
# Update last seen time
session.last_seen = timezone.now()
session.save()
@@ -92,6 +99,7 @@ def ingress_request(
hit = None
if idempotency is not None:
if cache.get(idempotency_path) is not None:
cache.touch(idempotency_path, 10 * 60)
hit = Hit.objects.filter(
pk=cache.get(idempotency_path), session=session
).first()
@@ -107,14 +115,15 @@ def ingress_request(
# There is no existing hit; create a new one
hit = Hit.objects.create(
session=session,
initial=initial,
tracker=tracker,
location=location,
referrer=payload.get("referrer", ""),
loadTime=payload.get("loadTime"),
load_time=payload.get("loadTime"),
)
# Set idempotency (if applicable)
if idempotency is not None:
cache.set(idempotency_path, hit.pk, timeout=30 * 60)
cache.set(idempotency_path, hit.pk, timeout=10 * 60)
except Exception as e:
log.exception(e)
raise e