Bump version

* First working version of world map chart

* Cleanup code, fix aspect ratio, add GeoIP Map header

* Remove limited-height on session list with already limited content

* Update package lock

* Integrate map into service page

* Adjust map colors

* Adjust colors further

Co-authored-by: R. Miles McCain <github@sendmiles.email>

.github/workflows/build-docker-edge.yml

@@ -0,0 +1,43 @@
+name: Build edge Docker images
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  publish_to_docker_hub:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      
+      - name: Prepare tags
+        id: prep
+        run: |
+          DOCKER_IMAGE=milesmcc/shynet
+          TAGS="${DOCKER_IMAGE}:edge"
+          echo ::set-output name=tags::${TAGS}
+          
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Build and push advanced image
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          push: true
+          tags: ${{ steps.prep.outputs.tags }}

.github/workflows/build-docker-release.yml

@@ -0,0 +1,44 @@
+name: Build release Docker images
+
+on:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  publish_to_docker_hub:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      
+      - name: Prepare tags
+        id: prep
+        run: |
+          DOCKER_IMAGE=milesmcc/shynet
+          VERSION=${GITHUB_REF#refs/tags/}
+          TAGS="${DOCKER_IMAGE}:${VERSION},${DOCKER_IMAGE}:latest"
+          echo ::set-output name=tags::${TAGS}
+          
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Build and push advanced image
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          push: true
+          tags: ${{ steps.prep.outputs.tags }}

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3-alpine
+FROM python:alpine3.12
 
 # Getting things ready
 WORKDIR /usr/src/shynet

Pipfile.lock

@@ -19,6 +19,7 @@
                 "sha256:70cdb10628468ff14e57ec2f751c7aa9e48e7e3651cfd62d431213c0c4e58f21",
                 "sha256:aa7f313fb887c91f15474c1229907a04dac0b8135822d6603437803424c0aa59"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==2.6.1"
         },
         "asgiref": {
@@ -26,6 +27,7 @@
                 "sha256:92906c611ce6c967347bbfea733f13d6313901d54dcca88195eaeb52b2a8e8ee",
                 "sha256:d1216dfbdfb63826470995d31caed36225dcaf34f182e0fa257a4dd9e86f1b78"
             ],
+            "markers": "python_version >= '3.6'",
             "version": "==3.3.4"
         },
         "billiard": {
@@ -45,16 +47,17 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
-                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
+                "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
+                "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"
             ],
-            "version": "==2020.12.5"
+            "version": "==2021.5.30"
         },
         "chardet": {
             "hashes": [
                 "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
                 "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==4.0.0"
         },
         "defusedxml": {
@@ -62,15 +65,16 @@
                 "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69",
                 "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==0.7.1"
         },
         "django": {
             "hashes": [
-                "sha256:c348b3ddc452bf4b62361f0752f71a339140c777ebea3cdaaaa8fdb7f417a862",
-                "sha256:f8393103e15ec2d2d313ccbb95a3f1da092f9f58d74ac1c61ca2ac0436ae1eac"
+                "sha256:a523d62b7ab2908f551dabc32b99017a86aa7784e32b761708e52be3dce6d35d",
+                "sha256:dc41bf07357f1f4810c1c555b685cb51f780b41e37892d6cc92b89789f2847e1"
             ],
             "index": "pypi",
-            "version": "==3.1.8"
+            "version": "==3.1.12"
         },
         "django-allauth": {
             "hashes": [
@@ -145,6 +149,7 @@
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                 "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.10"
         },
         "kombu": {
@@ -152,20 +157,23 @@
                 "sha256:be48cdffb54a2194d93ad6533d73f69408486483d189fe9f5990ee24255b0e0a",
                 "sha256:ca1b45faac8c0b18493d02a8571792f3c40291cf2bcf1f55afed3d8f3aa7ba74"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==4.6.11"
         },
         "maxminddb": {
             "hashes": [
                 "sha256:47e86a084dd814fac88c99ea34ba3278a74bc9de5a25f4b815b608798747c7dc"
             ],
+            "markers": "python_version >= '3.6'",
             "version": "==2.0.3"
         },
         "oauthlib": {
             "hashes": [
-                "sha256:bee41cc35fcca6e988463cacc3bcb8a96224f470ca547e697b604cc697b2f889",
-                "sha256:df884cd6cbe20e32633f1db1072e9356f53638e4361bef4e8b03c9127c9328ea"
+                "sha256:42bf6354c2ed8c6acb54d971fce6f88193d97297e18602a3a886603f9d7730cc",
+                "sha256:8f0215fcc533dd8dd1bee6f4c412d4f0cd7297307d43ac61666389e3bc3198a3"
             ],
-            "version": "==3.1.0"
+            "markers": "python_version >= '3.6'",
+            "version": "==3.1.1"
         },
         "psycopg2-binary": {
             "hashes": [
@@ -277,12 +285,14 @@
                 "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
                 "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==2.25.1"
         },
         "requests-oauthlib": {
             "hashes": [
                 "sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d",
-                "sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a"
+                "sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a",
+                "sha256:fa6c47b933f01060936d87ae9327fead68768b69c6c9ea2109c48be30f2d4dbc"
             ],
             "version": "==1.3.0"
         },
@@ -298,6 +308,7 @@
                 "sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0",
                 "sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8"
             ],
+            "markers": "python_version >= '3.5'",
             "version": "==0.4.1"
         },
         "ua-parser": {
@@ -310,10 +321,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
-                "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
+                "sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c",
+                "sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098"
             ],
-            "version": "==1.26.4"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.5"
         },
         "user-agents": {
             "hashes": [
@@ -328,6 +340,7 @@
                 "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87",
                 "sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.3.0"
         },
         "whitenoise": {

TEMPLATE.env

@@ -33,6 +33,8 @@ ACCOUNT_SIGNUPS_ENABLED=False
 ACCOUNT_EMAIL_VERIFICATION=none
 
 # The timezone of the admin panel. Affects how dates are displayed.
+# This must match a value from the IANA's tz database.
+# Wikipedia has a list of valid strings: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
 TIME_ZONE=America/New_York
 
 # Set to "False" if you will not be serving content over HTTPS
@@ -92,3 +94,10 @@ AGGRESSIVE_HASH_SALTING=True
 #  - https://www.google.com/maps/search/?api=1&query=$LATITUDE,$LONGITUDE
 #  - https://www.mapquest.com/near-$LATITUDE,$LONGITUDE
 LOCATION_URL=https://www.openstreetmap.org/?mlat=$LATITUDE&mlon=$LONGITUDE
+
+# How many services should be displayed on dashboard page?
+# Set to big number if you don't want pagination at all.
+DASHBOARD_PAGE_SIZE=5
+
+# Should background bars be scaled to full width?
+USE_RELATIVE_MAX_IN_BAR_VISUALIZATION=True

app.json

@@ -127,6 +127,16 @@
       "description": "Custom location url to link to in frontend.",
       "value": "https://www.openstreetmap.org/?mlat=$LATITUDE&mlon=$LONGITUDE",
       "required": false
+    },
+    "DASHBOARD_PAGE_SIZE": {
+      "description": "How many services should be displayed on dashboard page?",
+      "value": "5",
+      "required": false
+    },
+    "USE_RELATIVE_MAX_IN_BAR_VISUALIZATION": {
+      "description": "Should background bars be scaled to full width?",
+      "value": "True",
+      "required": false
     }
   }
 }

kubernetes/deployments.yml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
       - name: "shynet-webserver"
-        image: "milesmcc/shynet:dev"
+        image: "milesmcc/shynet:latest"
         imagePullPolicy: Always
         envFrom:
         - secretRef:
@@ -42,7 +42,7 @@ spec:
     spec:
       containers:
       - name: "shynet-celeryworker"
-        image: "milesmcc/shynet:dev"
+        image: "milesmcc/shynet:latest"
         command: ["./celeryworker.sh"]
         imagePullPolicy: Always
         envFrom:

package.json

@@ -20,6 +20,7 @@
     "@fortawesome/fontawesome-free": "^5.15.1",
     "a17t": "^0.5.1",
     "apexcharts": "^3.24.0",
+    "datamaps": "^0.5.9",
     "flag-icon-css": "^3.5.0",
     "inter-ui": "^3.15.0",
     "litepicker": "^2.0.11"

shynet/analytics/templates/analytics/scripts/page.js

@@ -8,11 +8,14 @@
 var Shynet = {
   idempotency: null,
   heartbeatTaskId: null,
+  skipHeartbeat: false,
   sendHeartbeat: function () {
     try {
-      if (document.hidden) {
+      if (document.hidden || Shynet.skipHeartbeat) {
         return;
       }
+
+      Shynet.skipHeartbeat = true;
       var xhr = new XMLHttpRequest();
       xhr.open(
         "POST",
@@ -20,6 +23,12 @@ var Shynet = {
         true
       );
       xhr.setRequestHeader("Content-Type", "application/json");
+      xhr.onload = function () { 
+        Shynet.skipHeartbeat = false;
+      };
+      xhr.onerror = function () { 
+        Shynet.skipHeartbeat = false;
+      };
       xhr.send(
         JSON.stringify({
           idempotency: Shynet.idempotency,
@@ -30,13 +39,14 @@ var Shynet = {
             window.performance.timing.navigationStart,
         })
       );
-    } catch (e) { }
+    } catch (e) {}
   },
   newPageLoad: function () {
     if (Shynet.heartbeatTaskId != null) {
       clearInterval(Shynet.heartbeatTaskId);
     }
     Shynet.idempotency = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
+    Shynet.skipHeartbeat = false;
     Shynet.heartbeatTaskId = setInterval(Shynet.sendHeartbeat, parseInt("{{heartbeat_frequency}}"));
     Shynet.sendHeartbeat();
   }

shynet/core/models.py

@@ -8,12 +8,12 @@ from django.conf import settings
 from django.contrib.auth.models import AbstractUser
 from django.core.exceptions import ValidationError
 from django.db import models
-from django.db.models.functions import TruncDate
+from django.db.models.functions import TruncDate, TruncHour
 from django.db.utils import NotSupportedError
 from django.shortcuts import reverse
 from django.utils import timezone
 
-
+# How long a session a needs to go without an update to no longer be considered 'active' (i.e., currently online)
 ACTIVE_USER_TIMEDELTA = timezone.timedelta(
     milliseconds=settings.SCRIPT_HEARTBEAT_FREQUENCY * 2
 )
@@ -125,8 +125,10 @@ class Service(models.Model):
         Session = apps.get_model("analytics", "Session")
         Hit = apps.get_model("analytics", "Hit")
 
+        tz_now = timezone.now()
+
         currently_online = Session.objects.filter(
-            service=self, last_seen__gt=timezone.now() - ACTIVE_USER_TIMEDELTA
+            service=self, last_seen__gt=tz_now - ACTIVE_USER_TIMEDELTA
         ).count()
 
         sessions = Session.objects.filter(
@@ -196,6 +198,36 @@ class Service(models.Model):
 
         avg_hits_per_session = hit_count / session_count if session_count > 0 else None
 
+        avg_session_duration = self._get_avg_session_duration(sessions, session_count)
+
+        chart_data, chart_tooltip_format, chart_granularity = self._get_chart_data(
+            sessions, hits, start_time, end_time, tz_now
+        )
+        return {
+            "currently_online": currently_online,
+            "session_count": session_count,
+            "hit_count": hit_count,
+            "has_hits": has_hits,
+            "bounce_rate_pct": bounce_count * 100 / session_count
+            if session_count > 0
+            else None,
+            "avg_session_duration": avg_session_duration,
+            "avg_load_time": avg_load_time,
+            "avg_hits_per_session": avg_hits_per_session,
+            "locations": locations,
+            "referrers": referrers,
+            "countries": countries,
+            "operating_systems": operating_systems,
+            "browsers": browsers,
+            "devices": devices,
+            "device_types": device_types,
+            "chart_data": chart_data,
+            "chart_tooltip_format": chart_tooltip_format,
+            "chart_granularity": chart_granularity,
+            "online": True,
+        }
+
+    def _get_avg_session_duration(self, sessions, session_count):
         try:
             avg_session_duration = sessions.annotate(
                 duration=models.F("last_seen") - models.F("start_time")
@@ -210,48 +242,72 @@ class Service(models.Model):
         if session_count == 0:
             avg_session_duration = None
 
-        session_chart_data = {
-            k["date"]: k["count"]
-            for k in sessions.annotate(date=TruncDate("start_time"))
+        return avg_session_duration
+
+    def _get_chart_data(self, sessions, hits, start_time, end_time, tz_now):
+        # Show hourly chart for date ranges of 3 days or less, otherwise daily chart
+        if (end_time - start_time).days < 3:
+            chart_tooltip_format = "MM/dd HH:mm"
+            chart_granularity = "hourly"
+            sessions_per_hour = (
+                sessions.annotate(hour=TruncHour("start_time"))
+                .values("hour")
+                .annotate(count=models.Count("uuid"))
+                .order_by("hour")
+            )
+            chart_data = {
+                k["hour"]: {"sessions": k["count"]} for k in sessions_per_hour
+            }
+            hits_per_hour = (
+                hits.annotate(hour=TruncHour("start_time"))
+                .values("hour")
+                .annotate(count=models.Count("id"))
+                .order_by("hour")
+            )
+            for k in hits_per_hour:
+                if k["hour"] not in chart_data:
+                    chart_data[k["hour"]] = {"hits": k["count"], "sessions": 0}
+                else:
+                    chart_data[k["hour"]]["hits"] = k["count"]
+
+            hours_range = range(int((end_time - start_time).total_seconds() / 3600) + 1)
+            for hour_offset in hours_range:
+                hour = start_time + timezone.timedelta(hours=hour_offset)
+                if hour not in chart_data and hour <= tz_now:
+                    chart_data[hour] = {"sessions": 0, "hits": 0}
+        else:
+            chart_tooltip_format = "MMM d"
+            chart_granularity = "daily"
+            sessions_per_day = (
+                sessions.annotate(date=TruncDate("start_time"))
                 .values("date")
                 .annotate(count=models.Count("uuid"))
                 .order_by("date")
-        }
+            )
+            chart_data = {k["date"]: {"sessions": k["count"]} for k in sessions_per_day}
+            hits_per_day = (
+                hits.annotate(date=TruncDate("start_time"))
+                .values("date")
+                .annotate(count=models.Count("id"))
+                .order_by("date")
+            )
+            for k in hits_per_day:
+                chart_data[k["date"]]["hits"] = k["count"]
+
             for day_offset in range((end_time - start_time).days + 1):
                 day = (start_time + timezone.timedelta(days=day_offset)).date()
-            if day not in session_chart_data:
-                session_chart_data[day] = 0
+                if day not in chart_data and day <= tz_now.date():
+                    chart_data[day] = {"sessions": 0, "hits": 0}
 
-        return {
-            "currently_online": currently_online,
-            "session_count": session_count,
-            "hit_count": hit_count,
-            "has_hits": has_hits,
-            "avg_hits_per_session": hit_count / (max(session_count, 1)),
-            "bounce_rate_pct": bounce_count * 100 / session_count
-            if session_count > 0
-            else None,
-            "avg_session_duration": avg_session_duration,
-            "avg_load_time": avg_load_time,
-            "avg_hits_per_session": avg_hits_per_session,
-            "locations": locations,
-            "referrers": referrers,
-            "countries": countries,
-            "operating_systems": operating_systems,
-            "browsers": browsers,
-            "devices": devices,
-            "device_types": device_types,
-            "session_chart_data": json.dumps(
-                [
-                    {"x": str(key), "y": value}
-                    for key, value in sorted(
-                        session_chart_data.items(), key=lambda k: k[0]
-                    )
-                ]
-            ),
-            "online": True,
+        chart_data = sorted(chart_data.items(), key=lambda k: k[0])
+        chart_data = {
+            'sessions': [v['sessions'] for k, v in chart_data],
+            'hits': [v['hits'] for k, v in chart_data],
+            'labels': [str(k) for k, v in chart_data],
         }
 
+        return chart_data, chart_tooltip_format, chart_granularity
+
     def get_absolute_url(self):
         return reverse(
             "dashboard:service",

shynet/dashboard/static/dashboard/css/global.css

@@ -11,6 +11,11 @@
     max-height: 400px;
 }
 
+.force-limited-height {
+    max-height: 400px;
+    overflow: hidden;
+}
+
 .rf {
     text-align: right !important;
 }

shynet/dashboard/templates/base.html

@@ -12,11 +12,13 @@
   <script src="{% static 'apexcharts/dist/apexcharts.min.js'%}"></script>
   <script src="{% static 'litepicker/dist/nocss/litepicker.js' %}"></script>
   <script src="{% static 'litepicker/dist/plugins/ranges.js' %}"></script>
-  <link rel="stylesheet" href="{% static 'litepicker/dist/css/litepicker.css' %}">
-
+  <script src="{% static 'd3/d3.min.js' %}"></script>
+  <script src="{% static 'topojson/build/topojson.min.js' %}"></script>
+  <script src="{% static 'datamaps/dist/datamaps.world.min.js' %}"></script>
   <script src="{% static 'dashboard/js/base.js' %}"></script>
   <link rel="stylesheet" href="{% static 'dashboard/css/global.css' %}">
   <link rel="stylesheet" href="{% static 'flag-icon-css/css/flag-icon.min.css' %}">
+  <link rel="stylesheet" href="{% static 'litepicker/dist/css/litepicker.css' %}">
   {% block extra_head %}
   {% endblock %}
 </head>

shynet/dashboard/templates/dashboard/includes/bar.html

@@ -0,0 +1,6 @@
+{% load helpers %}
+<div
+    class="absolute h-6 rounded-md"
+    style="width: {% bar_width count max total %}; top: 6px; left: 0px; height: calc(100% - 12px); background-color: var(--color-urge-100-fallback)"
+>
+</div>

shynet/dashboard/templates/dashboard/includes/date_range.html

@@ -2,7 +2,7 @@
     <input type="hidden" name="startDate" value="{{start_date.isoformat}}" id="startDate">
     <input type="hidden" name="endDate" value="{{end_date.isoformat}}" id="endDate">
 </form>
-<input type="input" id="rangePicker" placeholder="Date range" class="input ~neutral bg-neutral-000 cursor-pointer w-auto" readonly>
+<input type="input" id="rangePicker" placeholder="Date range" class="input ~neutral bg-neutral-000 cursor-pointer" style="max-width: 200px;" readonly>
 <style>
     :root {
         --litepicker-button-prev-month-color-hover: var(--color-urge);

shynet/dashboard/templates/dashboard/includes/map_chart.html

@@ -0,0 +1,59 @@
+{% load helpers %}
+
+<div id="map-chart" class="relative"></div>
+<script>
+    // Colors
+    const lightBlue = "#C4B5FD";
+    const highlightBlue = "#8B5CF6";
+    const white = "#ffffff";
+    
+    // Data maps
+    const countryMapData = {};
+    const countryMapColors = {};
+    const countryMap = {
+        {% for country in countries %}"{{country.country|safe|datamap_id}}": {{country.count}},
+        {% endfor %}
+    };
+
+    // Max session count will be full opacity
+    const maxSessionCount = Math.max(...Object.values(countryMap));
+
+    // Color scale starts from opacity 0.1 - 1.0, 0 sessions gets opacity 0
+    const minPercentage = 0.1
+
+    // Loop over country map and transform data for Datamaps use
+    const keys = Object.keys(countryMap);
+    const length = keys.length;
+    for (let i = 0; i < length; i++) {
+        countryMapData[keys[i]] = {
+            sessionCount: countryMap[keys[i]],
+            color: `rgba(124, 58, 237, ${countryMap[keys[i]] === 0 ? 0 : minPercentage + (countryMap[keys[i]] / maxSessionCount * (1 - minPercentage))})`
+        };
+        countryMapColors[keys[i]] = countryMapData[keys[i]].color;
+    }
+
+    // Create datamap
+    const map = new Datamap({
+        element: document.getElementById('map-chart'),
+        projection: 'mercator',
+        responsive: true,
+        geographyConfig: {
+            borderColor: lightBlue,
+            highlightBorderColor: highlightBlue,
+            highlightBorderWidth: 1.5,
+            highlightFillColor: (geography) => geography.color || white,
+            highlightFillOpacity: 0.9,
+            popupTemplate: (geography, data) => '<div class="hoverinfo"><strong>' + geography.properties.name + '</strong>: ' +  data.sessionCount + ' sessions</div>'
+    
+        },
+        fills: {
+            defaultFill: white
+        },
+        data: countryMapData,
+        aspectRatio: 0.68
+    });
+    map.updateChoropleth(countryMapColors);
+    
+    // Handle resize. TODO: debounce?
+    window.onresize = () => map.resize();
+</script>

shynet/dashboard/templates/dashboard/includes/service_overview.html

@@ -51,7 +51,7 @@
     </div>
     <hr class="sep h-4">
     <div style="bottom: -1px;">
-        {% include 'dashboard/includes/time_chart.html' with data=stats.session_chart_data sparkline=True height=100 name=object.uuid %}
+        {% include 'dashboard/includes/time_chart.html' with data=stats.chart_data sparkline=True height=100 name=object.uuid tooltip_format=stats.chart_tooltip_format granularity=stats.chart_granularity %}
     </div>
     {% endwith %}
 </a>

shynet/dashboard/templates/dashboard/includes/time_chart.html

@@ -5,9 +5,14 @@
             enabled: false
         },
         tooltip: {
-            shared: false,
+            shared: true,
+            x: {
+                format: '{{tooltip_format|default:"MMM d"}}',
+            },
+        },
+        legend: {
+            show: false,
         },
-        colors: ["#805AD5"],
         chart: {
             zoom: {
                 enabled: false,
@@ -15,7 +20,7 @@
             toolbar: {
                 show: false,
             },
-            type: 'area',
+            type: 'line',
             height: {{height|default:"200"}},
             offsetY: -1,
             animations: {
@@ -24,16 +29,14 @@
             sparkline: {
                 enabled: {% if sparkline %}true{% else %}false{% endif %},
             },
-            fill: {
-                type: 'gradient',
-                gradient: {
-                    shadeIntensity: 1,
-                    inverseColors: false,
-                    opacityFrom: 0.8,
-                    opacityTo: 0,
-                    stops: [0, 75, 100]
+            {% if granularity == "daily" and click_zoom %}
+            events: {
+                markerClick: function(event, chartContext, { seriesIndex, dataPointIndex, w: {config}}) {
+                    const day = config.labels[dataPointIndex]
+                    window.location.href = `?startDate=${day}&endDate=${day}`
                 },
             },
+            {% endif %}
         },
         grid: {
             padding: {
@@ -63,14 +66,26 @@
         },
         xaxis: {
             type: "datetime",
+            labels: {
+                datetimeUTC: false
+            },
         },
         stroke: {
-            width: 1.5,
+            width: 2,
+            curve: 'smooth',
         },
         series: [{
-            name: "{{unit|default:'Sessions'}}",
-            data: {{data|safe}}
-        }]
+            name: "Hits",
+            type: 'area',
+            color: "#ddd6fe",
+            data: {{data.hits|safe}}
+        }, {
+            name: "Sessions",
+            type: 'line',
+            color: "#805AD5",
+            data: {{data.sessions|safe}}
+        }],
+        labels: {{data.labels|safe}}
     };
     var triggerMatchesChart = new ApexCharts(document.querySelector("#chart{{name|default:'Main'}}"), triggerMatchesChartOptions);
     triggerMatchesChart.render();

shynet/dashboard/templates/dashboard/pages/service.html

@@ -94,7 +94,7 @@
     {% endwith %}
 </div>
 <div class="card overflow-visible ~neutral !low py-0 mb-6">
-    {% include 'dashboard/includes/time_chart.html' with data=stats.session_chart_data %}
+    {% include 'dashboard/includes/time_chart.html' with data=stats.chart_data tooltip_format=stats.chart_tooltip_format granularity=stats.chart_granularity click_zoom=True %}
 </div>
 {% endif %}
 <div class="grid grid-cols-1 md:grid-cols-2 gap-6 mb-6">
@@ -109,8 +109,20 @@
             <tbody>
                 {% for location in stats.locations %}
                 <tr>
-                    <td class="truncate w-full max-w-0">{{location.location|default:"Unknown"|urldisplay}}</td>
-                    <td class="rf">{{location.count|intcomma}}</td>
+                    <td class="truncate w-full max-w-0 relative">
+                        {% include 'dashboard/includes/bar.html' with count=location.count max=stats.locations.0.count total=stats.hit_count %}
+                        <div class="relative flex items-center">
+                            {{location.location|default:"Unknown"|urldisplay}}
+                        </div>
+                    </td>
+                    <td>
+                        <div class="flex justify-end items-center">
+                            {{location.count|intcomma}}
+                            <span class="text-xs rf" style="min-width: 48px">
+                                ({{location.count|percent:stats.hit_count}})
+                            </span>
+                        </div>
+                    </td>
                 </tr>
                 {% empty %}
                 <tr>
@@ -120,6 +132,10 @@
             </tbody>
         </table>
     </div>
+    <div class="card ~neutral !low force-limited-height py-2 overflow-y-hidden">
+        <p class="text-sm font-semibold mx-2 p-2 border-b mb-2">Sessions by Geography</p>
+        {% include 'dashboard/includes/map_chart.html' with countries=stats.countries %}
+    </div>
     <div class="card ~neutral !low limited-height py-2">
         <table class="table">
             <thead class="text-sm">
@@ -131,32 +147,20 @@
             <tbody>
                 {% for referrer in stats.referrers %}
                 <tr>
-                    <td class="truncate w-full max-w-0">{{referrer.referrer|default:"Direct"|urldisplay}}</td>
-                    <td class="rf">{{referrer.count|intcomma}}</td>
-                </tr>
-                {% empty %}
-                <tr>
-                    <td><span class="text-gray-600">No data yet...</span></td>
-                </tr>
-                {% endfor %}
-            </tbody>
-        </table>
+                    <td class="truncate w-full max-w-0 relative">
+                        {% include 'dashboard/includes/bar.html' with count=referrer.count max=stats.referrers.0.count total=stats.session_count %}
+                        <div class="relative flex items-center">
+                            {{referrer.referrer|default:"Direct"|urldisplay}}
+                        </div>
+                    </td>
+                    <td>
+                        <div class="flex justify-end items-center">
+                            {{referrer.count|intcomma}}
+                            <span class="text-xs rf" style="min-width: 48px">
+                                ({{referrer.count|percent:stats.session_count}})
+                            </span>
                         </div>
-    <div class="card ~neutral !low limited-height py-2">
-        <table class="table">
-            <thead class="text-sm">
-                <tr>
-                    <th>Country</th>
-                    <th class="rf">Sessions</th>
-                </tr>
-            </thead>
-            <tbody>
-                {% for country in stats.countries %}
-                <tr>
-                    <td class="truncate w-full max-w-0" title="{{country.country|country_name}}">
-                        <span class="{{country.country|flag_class}}"></span> {{country.country|country_name}}
                     </td>
-                    <td class="rf">{{country.count|intcomma}}</td>
                 </tr>
                 {% empty %}
                 <tr>
@@ -177,10 +181,20 @@
             <tbody>
                 {% for os in stats.operating_systems %}
                 <tr>
-                    <td class="flex items-center truncate w-full max-w-0" title="{{os.os|default:'Unknown'}}">
+                    <td class="flex items-center truncate w-full max-w-0 relative" title="{{os.os|default:'Unknown'}}">
+                        {% include 'dashboard/includes/bar.html' with count=os.count max=stats.operating_systems.0.count total=stats.session_count %}
+                        <div class="relative flex items-center">
                             {{os.os|iconify}}<span>{{os.os|default:"Unknown"}}</span>
+                        </div>
+                    </td>
+                    <td>
+                        <div class="flex justify-end items-center">
+                            {{os.count|intcomma}}
+                            <span class="text-xs rf" style="min-width: 48px">
+                                ({{os.count|percent:stats.session_count}})
+                            </span>
+                        </div>
                     </td>
-                    <td class="rf">{{os.count|intcomma}}</td>
                 </tr>
                 {% empty %}
                 <tr>
@@ -201,9 +215,21 @@
             <tbody>
                 {% for browser in stats.browsers %}
                 <tr>
-                    <td class="flex items-center truncate w-full max-w-0" title="{{browser.browser|default:'Unknown'}}">
-                        {{browser.browser|iconify}}<span>{{browser.browser|default:"Unknown"}}</span></td>
-                    <td class="rf">{{browser.count|intcomma}}</td>
+                    <td class="flex items-center truncate w-full max-w-0 relative" title="{{browser.browser|default:'Unknown'}}">
+                        {% include 'dashboard/includes/bar.html' with count=browser.count max=stats.browsers.0.count total=stats.session_count %}
+                        </div>
+                        <div class="relative flex items-center">
+                            {{browser.browser|iconify}}<span>{{browser.browser|default:"Unknown"}}</span>
+                        </div>
+                    </td>
+                    <td>
+                        <div class="flex justify-end items-center">
+                            {{browser.count|intcomma}}
+                            <span class="text-xs rf" style="min-width: 48px">
+                                ({{browser.count|percent:stats.session_count}})
+                            </span>
+                        </div>
+                    </td>
                 </tr>
                 {% empty %}
                 <tr>
@@ -224,8 +250,20 @@
             <tbody>
                 {% for device_type in stats.device_types %}
                 <tr>
-                    <td class="truncate w-full max-w-0">{{device_type.device_type|default:"Unknown"|title}}</td>
-                    <td class="rf">{{device_type.count|intcomma}}</td>
+                    <td class="truncate w-full max-w-0 relative">
+                        {% include 'dashboard/includes/bar.html' with count=device_type.count max=stats.device_types.0.count total=stats.session_count %}
+                        <div class="relative flex items-center">
+                            {{device_type.device_type|default:"Unknown"|title}}
+                        </div>
+                    </td>
+                    <td>
+                        <div class="flex justify-end items-center">
+                            {{device_type.count|intcomma}}
+                            <span class="text-xs rf" style="min-width: 48px">
+                                ({{device_type.count|percent:stats.session_count}})
+                            </span>
+                        </div>
+                    </td>
                 </tr>
                 {% empty %}
                 <tr>
@@ -236,7 +274,7 @@
         </table>
     </div>
 </div>
-<div class="card ~neutral !low limited-height py-2">
+<div class="card ~neutral !low py-2">
     {% include 'dashboard/includes/session_list.html' %}
     <hr class="sep h-8 md:h-12">
     <a href="{% contextual_url 'dashboard:service_session_list' service.uuid %}" class="button ~neutral w-auto mb-2">View more

shynet/dashboard/templatetags/helpers.py

@@ -43,6 +43,14 @@ def country_name(isocode):
         return "Unknown"
 
 
+@register.filter
+def datamap_id(isocode):
+    try:
+        return pycountry.countries.get(alpha_2=isocode).alpha_3
+    except:
+        return "UNKNOWN"
+
+
 @register.simple_tag
 def relative_stat_tone(
     start,
@@ -186,6 +194,7 @@ def urldisplay(url):
     else:
         return url
 
+
 class ContextualURLNode(template.Node):
     """Extension of the Django URLNode to support including contextual parameters in URL outputs. In other words, URLs generated will keep the start and end date parameters."""
 
@@ -205,9 +214,13 @@ class ContextualURLNode(template.Node):
         url_parts = list(urlparse(url))
         query = dict(urllib.parse.parse_qsl(url_parts[4]))
 
-        query.update({
-            param: context.request.GET.get(param) for param in self.CONTEXT_PARAMS if param in context.request.GET and param not in query
-        })
+        query.update(
+            {
+                param: context.request.GET.get(param)
+                for param in self.CONTEXT_PARAMS
+                if param in context.request.GET and param not in query
+            }
+        )
 
         url_parts[4] = urllib.parse.urlencode(query)
 
@@ -225,6 +238,38 @@ def contextual_url(*args, **kwargs):
     urlnode = url_tag(*args, **kwargs)
     return ContextualURLNode(urlnode)
 
+
 @register.filter
 def location_url(session):
-    return settings.LOCATION_URL.replace("$LATITUDE", str(session.latitude)).replace("$LONGITUDE", str(session.longitude))
+    return settings.LOCATION_URL.replace("$LATITUDE", str(session.latitude)).replace(
+        "$LONGITUDE", str(session.longitude)
+    )
+
+
+@register.filter
+def percent(value, total):
+    if total == 0:
+        return "N/A"
+
+    percent = value / total
+
+    if percent < 0.001:
+        return "<0.1%"
+
+    return f'{percent:.1%}'
+
+
+@register.simple_tag
+def bar_width(count, max, total):
+    if total == 0 or max == 0:
+        return "0"
+
+    if settings.USE_RELATIVE_MAX_IN_BAR_VISUALIZATION:
+        percent = count / max
+    else:
+        percent = count / total
+
+    if percent < 0.001:
+        return "0"
+
+    return f'{percent:.1%}'

shynet/dashboard/views.py

@@ -25,7 +25,7 @@ from .mixins import DateRangeMixin
 class DashboardView(LoginRequiredMixin, DateRangeMixin, ListView):
     model = Service
     template_name = "dashboard/pages/dashboard.html"
-    paginate_by = 5
+    paginate_by = settings.DASHBOARD_PAGE_SIZE
 
     def get_queryset(self):
         return Service.objects.filter(

shynet/shynet/settings.py

@@ -18,7 +18,7 @@ import urllib.parse as urlparse
 from django.contrib.messages import constants as messages
 
 # Increment on new releases
-VERSION = "v0.8.2"
+VERSION = "v0.10.0"
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -310,7 +310,13 @@ NPM_FILE_PATTERNS = {
     "stimulus": [os.path.join("dist", "stimulus.umd.js")],
     "inter-ui": [os.path.join("Inter (web)", "*")],
     "@fortawesome": [os.path.join("fontawesome-free", "js", "all.min.js")],
-    "flag-icon-css": [os.path.join("css", "flag-icon.min.css"), os.path.join("flags", "*")],
+    "datamaps": [os.path.join("dist", "datamaps.world.min.js")],
+    "d3": ["d3.min.js"],
+    "topojson": [os.path.join("build", "topojson.min.js")],
+    "flag-icon-css": [
+        os.path.join("css", "flag-icon.min.css"),
+        os.path.join("flags", "*"),
+    ],
 }
 
 # Shynet
@@ -346,4 +352,14 @@ BLOCK_ALL_IPS = os.getenv("BLOCK_ALL_IPS", "False") == "True"
 AGGRESSIVE_HASH_SALTING = os.getenv("AGGRESSIVE_HASH_SALTING", "False") == "True"
 
 # What location url should be linked to in the frontend?
-LOCATION_URL = os.getenv("LOCATION_URL", "https://www.openstreetmap.org/?mlat=$LATITUDE&mlon=$LONGITUDE")
+LOCATION_URL = os.getenv(
+    "LOCATION_URL", "https://www.openstreetmap.org/?mlat=$LATITUDE&mlon=$LONGITUDE"
+)
+
+# How many services should be displayed on dashboard page?
+DASHBOARD_PAGE_SIZE = int(os.getenv("DASHBOARD_PAGE_SIZE", "5"))
+
+# Should background bars be scaled to full width?
+USE_RELATIVE_MAX_IN_BAR_VISUALIZATION = (
+    os.getenv("USE_RELATIVE_MAX_IN_BAR_VISUALIZATION", "True") == "True"
+)