Format

Bumps [pyyaml](https://github.com/yaml/pyyaml) from 5.3.1 to 5.4.
- [Release notes](https://github.com/yaml/pyyaml/releases)
- [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES)
- [Commits](https://github.com/yaml/pyyaml/compare/5.3.1...5.4)

Signed-off-by: dependabot[bot] <support@github.com>

.gitignore

@@ -138,4 +138,7 @@ dmypy.json
 secrets.yml
 .vscode
 .DS_Store
-compiledstatic/
+compiledstatic/
+
+# Pycharm
+.idea

Pipfile

@@ -4,13 +4,13 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [packages]
-django = "~=3.0"
+django = "~=3.1"
 django-allauth = "~=0.42.0"
 geoip2 = "~=3.0.0"
 whitenoise = "~=5.1.0"
 celery = "~=4.4.6"
 django-ipware = "~=2.1.0"
-pyyaml = "~=5.3.1"
+pyyaml = "~=5.4"
 ua-parser = "~=0.10.0"
 user-agents = "~=2.1"
 emoji-country-flag = "~=1.2.1"
@@ -22,4 +22,5 @@ django-redis-cache = "~=3.0.0"
 pycountry = "~=19.8.18"
 html2text = "~=2020.1.16"
 django-health-check = "~=3.12.1"
-django-npm = "~=1.0.0"
+django-npm = "~=1.0.0"
+django-debug-toolbar = "*"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "73fbc4a0251ccae805550aa48b70ec9f668bee6fa4ff9503e023b3854a06bce8"
+            "sha256": "f8c76565a776f1bd36364077a86d6c16fccc522d9d2024bb9b51be5cb9f8b4b5"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -19,6 +19,7 @@
                 "sha256:70cdb10628468ff14e57ec2f751c7aa9e48e7e3651cfd62d431213c0c4e58f21",
                 "sha256:aa7f313fb887c91f15474c1229907a04dac0b8135822d6603437803424c0aa59"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==2.6.1"
         },
         "asgiref": {
@@ -26,6 +27,7 @@
                 "sha256:5ee950735509d04eb673bd7f7120f8fa1c9e2df495394992c73234d526907e17",
                 "sha256:7162a3cb30ab0609f1a4c95938fd73e8604f63bdba516a7f7d64b83ff09478f0"
             ],
+            "markers": "python_version >= '3.5'",
             "version": "==3.3.1"
         },
         "billiard": {
@@ -45,32 +47,34 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd",
+                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
-                "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4"
+                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
             ],
-            "version": "==2020.11.8"
+            "version": "==2020.12.5"
         },
         "chardet": {
             "hashes": [
-                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
-                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+                "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
             ],
-            "version": "==3.0.4"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==4.0.0"
         },
         "defusedxml": {
             "hashes": [
-                "sha256:6687150770438374ab581bb7a1b327a847dd9c5749e396102de3fad4e8a3ef93",
+                "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69",
-                "sha256:f684034d135af4c6cbb949b8a4d2ed61634515257a67299e5f940fbaa34377f5"
+                "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"
             ],
-            "version": "==0.6.0"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==0.7.1"
         },
         "django": {
             "hashes": [
-                "sha256:14a4b7cd77297fba516fc0d92444cc2e2e388aa9de32d7a68d4a83d58f5a4927",
+                "sha256:32ce792ee9b6a0cbbec340123e229ac9f765dff8c2a4ae9247a14b2ba3a365a7",
-                "sha256:14b87775ffedab2ef6299b73343d1b4b41e5d4e2aa58c6581f114dbec01e3f8f"
+                "sha256:baf099db36ad31f970775d0be5587cc58a6256a6771a44eb795b554d45f211b8"
             ],
             "index": "pypi",
-            "version": "==3.1.3"
+            "version": "==3.1.7"
         },
         "django-allauth": {
             "hashes": [
@@ -79,6 +83,14 @@
             "index": "pypi",
             "version": "==0.42.0"
         },
+        "django-debug-toolbar": {
+            "hashes": [
+                "sha256:84e2607d900dbd571df0a2acf380b47c088efb787dce9805aefeb407341961d2",
+                "sha256:9e5a25d0c965f7e686f6a8ba23613ca9ca30184daa26487706d4829f5cfb697a"
+            ],
+            "index": "pypi",
+            "version": "==3.2"
+        },
         "django-health-check": {
             "hashes": [
                 "sha256:2667b89b8f85ad9b2a24c90581b376016d22ea912fedf37f9866413a3c2e0a5d",
@@ -110,11 +122,11 @@
         },
         "emoji-country-flag": {
             "hashes": [
-                "sha256:256e47d30fb43bf154f370cc3c9e767f003aeb8653e31ba7b87151669c608e19",
+                "sha256:338f5e374119dcde093cfeaa8ca3af372d4b8d984d89a7fb2fb0db0011662560",
-                "sha256:3f6c32699c19489383497865b208260b1d55b8424d66e08049187b13db2f0b8a"
+                "sha256:a3a068191294294143d8ef294fdfe9792c5c243753eac130798bf2fa5de38185"
             ],
             "index": "pypi",
-            "version": "==1.2.3"
+            "version": "==1.2.4"
         },
         "geoip2": {
             "hashes": [
@@ -145,6 +157,7 @@
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                 "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.10"
         },
         "kombu": {
@@ -152,12 +165,14 @@
                 "sha256:be48cdffb54a2194d93ad6533d73f69408486483d189fe9f5990ee24255b0e0a",
                 "sha256:ca1b45faac8c0b18493d02a8571792f3c40291cf2bcf1f55afed3d8f3aa7ba74"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==4.6.11"
         },
         "maxminddb": {
             "hashes": [
                 "sha256:47e86a084dd814fac88c99ea34ba3278a74bc9de5a25f4b815b608798747c7dc"
             ],
+            "markers": "python_version >= '3.6'",
             "version": "==2.0.3"
         },
         "oauthlib": {
@@ -165,6 +180,7 @@
                 "sha256:bee41cc35fcca6e988463cacc3bcb8a96224f470ca547e697b604cc697b2f889",
                 "sha256:df884cd6cbe20e32633f1db1072e9356f53638e4361bef4e8b03c9127c9328ea"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==3.1.0"
         },
         "psycopg2-binary": {
@@ -224,29 +240,45 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268",
+                "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
-                "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"
+                "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
             ],
-            "version": "==2020.4"
+            "version": "==2021.1"
         },
         "pyyaml": {
             "hashes": [
-                "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97",
+                "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf",
-                "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76",
+                "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696",
-                "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2",
+                "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393",
-                "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e",
+                "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77",
-                "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648",
+                "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922",
-                "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf",
+                "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5",
-                "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f",
+                "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8",
-                "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2",
+                "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10",
-                "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee",
+                "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc",
-                "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a",
+                "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018",
-                "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d",
+                "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e",
-                "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c",
+                "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253",
-                "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"
+                "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347",
+                "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183",
+                "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541",
+                "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb",
+                "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185",
+                "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc",
+                "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db",
+                "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa",
+                "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46",
+                "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122",
+                "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b",
+                "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63",
+                "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df",
+                "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc",
+                "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247",
+                "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6",
+                "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"
             ],
             "index": "pypi",
-            "version": "==5.3.1"
+            "version": "==5.4.1"
         },
         "redis": {
             "hashes": [
@@ -258,15 +290,17 @@
         },
         "requests": {
             "hashes": [
-                "sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8",
+                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
-                "sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998"
+                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
             ],
-            "version": "==2.25.0"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==2.25.1"
         },
         "requests-oauthlib": {
             "hashes": [
                 "sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d",
-                "sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a"
+                "sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a",
+                "sha256:fa6c47b933f01060936d87ae9327fead68768b69c6c9ea2109c48be30f2d4dbc"
             ],
             "version": "==1.3.0"
         },
@@ -282,6 +316,7 @@
                 "sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0",
                 "sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8"
             ],
+            "markers": "python_version >= '3.5'",
             "version": "==0.4.1"
         },
         "ua-parser": {
@@ -294,10 +329,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
+                "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
-                "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
+                "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
             ],
-            "version": "==1.26.2"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.4"
         },
         "user-agents": {
             "hashes": [
@@ -312,6 +348,7 @@
                 "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87",
                 "sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.3.0"
         },
         "whitenoise": {

TEMPLATE.env

@@ -72,4 +72,14 @@ SHOW_SHYNET_VERSION=True
 # CELERY_BROKER_URL=redis://redis.default.svc.cluster.local/1
 
 # Should Shynet show third-party icons in the dashboard?
-SHOW_THIRD_PARTY_ICONS=True
+SHOW_THIRD_PARTY_ICONS=True
+
+# Should Shynet block collection of IP addresses globally?
+BLOCK_ALL_IPS=False
+
+# Should Shynet include the date and site ID when hashing users?
+# This will prevent any possibility of cross-site tracking provided
+# that IP collection is also disabled, and external keys (primary
+# keys) aren't supplied. It will also prevent sessions from spanning
+# one day to another.
+AGGRESSIVE_HASH_SALTING=True

kubernetes/deployments.yml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
       - name: "shynet-webserver"
-        image: "milesmcc/shynet:latest"
+        image: "milesmcc/shynet:dev"
         imagePullPolicy: Always
         envFrom:
         - secretRef:
@@ -42,7 +42,7 @@ spec:
     spec:
       containers:
       - name: "shynet-celeryworker"
-        image: "milesmcc/shynet:latest"
+        image: "milesmcc/shynet:dev"
         command: ["./celeryworker.sh"]
         imagePullPolicy: Always
         envFrom:
@@ -61,7 +61,7 @@ spec:
   selector:
     app: shynet-redis
 ---
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: shynet-redis
@@ -83,3 +83,37 @@ spec:
           ports:
             - containerPort: 6379
               name: redis
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: shynet-webserver-service  
+spec:
+  type: ClusterIP
+  ports:
+  - port: 8080
+  selector:
+    app: shynet-webserver
+---
+apiVersion: networking.k8s.io/v1beta1
+kind: Ingress
+metadata:
+  name: shynet-webserver-ingress
+  annotations:
+    kubernetes.io/ingress.class: addon-http-application-routing
+spec:
+  rules:
+  - host: shynet.rmrm.io
+    http:
+      paths:
+      - backend:
+          serviceName: shynet-webserver-service
+          servicePort: 8080
+        path: /
+  - host: shynet-beta.rmrm.io
+    http:
+      paths:
+      - backend:
+          serviceName: shynet-webserver-service
+          servicePort: 8080
+        path: /

package-lock.json

@@ -14,9 +14,9 @@
       "integrity": "sha512-peIPrH9eDiu49LLzLlSTFFrXj6WLlEX3TRsUkqyyOHi/i58ilJ/eERnu7AcswXhuCBx+/2W9EUuHM+8iAq4ipg=="
     },
     "apexcharts": {
-      "version": "3.23.1",
+      "version": "3.24.0",
-      "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.23.1.tgz",
+      "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.24.0.tgz",
-      "integrity": "sha512-7fRpquXp725BUew5OO1mJWk16/IJPCUl0l8SjhISnAhAtbTaM9PnXPSmN2BvKO4RcT457CzMM7MCG5UokiTwcA==",
+      "integrity": "sha512-iT6czJCIVrmAtrcO90MZTQCvC+xi6R6Acf0jNH/d40FVTtCfcqECuKIh5iAMyOTtgUb7+fQ8rbadH2bm1kbL9Q==",
       "requires": {
         "svg.draggable.js": "^2.2.2",
         "svg.easing.js": "^2.0.0",
@@ -99,11 +99,6 @@
       "requires": {
         "svg.js": "^2.6.5"
       }
-    },
-    "turbolinks": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/turbolinks/-/turbolinks-5.2.0.tgz",
-      "integrity": "sha512-pMiez3tyBo6uRHFNNZoYMmrES/IaGgMhQQM+VFF36keryjb5ms0XkVpmKHkfW/4Vy96qiGW3K9bz0tF5sK9bBw=="
     }
   }
 }

package.json

@@ -19,9 +19,8 @@
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.15.1",
     "a17t": "^0.5.1",
-    "apexcharts": "^3.23.1",
+    "apexcharts": "^3.24.0",
     "inter-ui": "^3.15.0",
-    "litepicker": "^1.5.7",
+    "litepicker": "^1.5.7"
-    "turbolinks": "^5.2.0"
   }
 }

shynet/a17t/templates/a17t/includes/pagination.html

@@ -1,12 +1,12 @@
 <nav class="flex w-full flex-wrap items-center justify-between" role="navigation" aria-label="pagination">
     <div class="w-full md:w-auto mb-2">
         {% if page.has_previous %}
-        <a href="?page={{ page.previous_page_number }}{{url_parameters}}" class="button field bg-neutral-000 w-auto mr-1">Previous</a>
+        <a href="?page={{ page.previous_page_number }}&{{url_parameters}}" class="button field bg-neutral-000 w-auto mr-1">Previous</a>
         {% else %}
         <a class="button field bg-neutral-000 w-auto mr-1" disabled>Previous</a>
         {% endif %}
         {% if page.has_next %}
-        <a href="?page={{ page.next_page_number }}{{url_parameters}}" class="button field bg-neutral-000 w-auto">Next</a>
+        <a href="?page={{ page.next_page_number }}&{{url_parameters}}" class="button field bg-neutral-000 w-auto">Next</a>
         {% else %}
         <a class="button field bg-neutral-000 w-auto" disabled>Next</a>
         {% endif %}
@@ -17,7 +17,7 @@
         {% ifequal page.number pnum %}
         <li><a class="button field w-auto mx-1 text-white bg-neutral-700">{{ pnum }}</a></li>
         {% else %}
-        <li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}{{url_parameters}}">{{ pnum }}</a></li>
+        <li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}&{{url_parameters}}">{{ pnum }}</a></li>
         {% endifequal %}
         {% endfor %}
 
@@ -27,7 +27,7 @@
         {% ifequal page.number pnum %}
         <li><a class="button field w-auto mx-1 text-white bg-neutral-700">{{ pnum }}</a></li>
         {% else %}
-        <li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}{{url_parameters}}">{{ pnum }}</a></li>
+        <li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}&{{url_parameters}}">{{ pnum }}</a></li>
         {% endifequal %}
         {% endfor %}
         {% endif %}
@@ -38,7 +38,7 @@
         {% ifequal page.number pnum %}
         <li><a class="button field w-auto mx-1 text-white bg-neutral-700">{{ pnum }}</a></li>
         {% else %}
-        <li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}{{url_parameters}}">{{ pnum }}</a></li>
+        <li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}&{{url_parameters}}">{{ pnum }}</a></li>
         {% endifequal %}
         {% endfor %}
         {% endif %}

shynet/a17t/templatetags/pagination.py

@@ -15,12 +15,8 @@ def pagination(
     before_current_pages=4,
     after_current_pages=4,
 ):
-    url_parameters = "".join(
+    url_parameters = urlencode(
-        [
+        [(key, value) for key, value in request.GET.items() if key != "page"]
-            f"&{urlencode(key)}={urlencode(value)}"
-            for key, value in request.GET.items()
-            if key != "page"
-        ]
     )
 
     before = max(page.number - before_current_pages - 1, 0)

shynet/analytics/migrations/0001_initial.py

@@ -60,7 +60,9 @@ class Migration(migrations.Migration):
                     ),
                 ),
             ],
-            options={"ordering": ["-start_time"],},
+            options={
+                "ordering": ["-start_time"],
+            },
         ),
         migrations.CreateModel(
             name="Hit",
@@ -90,7 +92,9 @@ class Migration(migrations.Migration):
                     ),
                 ),
             ],
-            options={"ordering": ["-start_time"],},
+            options={
+                "ordering": ["-start_time"],
+            },
         ),
         migrations.AddIndex(
             model_name="session",

shynet/analytics/migrations/0004_auto_20210328_1514.py

@@ -0,0 +1,46 @@
+# Generated by Django 3.1.7 on 2021-03-28 19:14
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analytics", "0003_auto_20200502_1227"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="hit",
+            name="last_seen",
+            field=models.DateTimeField(default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name="hit",
+            name="start_time",
+            field=models.DateTimeField(
+                db_index=True, default=django.utils.timezone.now
+            ),
+        ),
+        migrations.AlterField(
+            model_name="session",
+            name="last_seen",
+            field=models.DateTimeField(
+                db_index=True, default=django.utils.timezone.now
+            ),
+        ),
+        migrations.AlterField(
+            model_name="session",
+            name="start_time",
+            field=models.DateTimeField(
+                db_index=True, default=django.utils.timezone.now
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="session",
+            index=models.Index(
+                fields=["service", "-last_seen"], name="analytics_s_service_10bb96_idx"
+            ),
+        ),
+    ]

shynet/analytics/migrations/0005_auto_20210328_1518.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.1.7 on 2021-03-28 19:18
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analytics", "0004_auto_20210328_1514"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="hit",
+            name="last_seen",
+            field=models.DateTimeField(
+                db_index=True, default=django.utils.timezone.now
+            ),
+        ),
+        migrations.AlterField(
+            model_name="hit",
+            name="load_time",
+            field=models.FloatField(db_index=True, null=True),
+        ),
+    ]

shynet/analytics/migrations/0006_hit_service.py

@@ -0,0 +1,40 @@
+# Generated by Django 3.1.7 on 2021-03-28 19:36
+
+from ..models import Hit, Session
+from django.db import migrations, models
+import django.db.models.deletion
+from django.db.models import Subquery, OuterRef
+
+
+def add_service_to_hits(_a, _b):
+    service = Session.objects.filter(pk=OuterRef("session")).values_list("service")[:1]
+
+    Hit.objects.update(service=Subquery(service))
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0008_auto_20200628_1403"),
+        ("analytics", "0005_auto_20210328_1518"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="hit",
+            name="service",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="core.service",
+            ),
+        ),
+        migrations.RunPython(add_service_to_hits, lambda: ()),
+        migrations.AlterField(
+            model_name="hit",
+            name="service",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE, to="core.service"
+            ),
+        ),
+    ]

shynet/analytics/migrations/0007_auto_20210328_1634.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.1.7 on 2021-03-28 20:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analytics", "0006_hit_service"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="hit",
+            index=models.Index(
+                fields=["service", "-start_time"], name="analytics_h_service_f4f41e_idx"
+            ),
+        ),
+    ]

shynet/analytics/migrations/0008_session_is_bounce.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.1.7 on 2021-03-28 21:38
+
+from django.db.models.expressions import F
+from ..models import Session, Hit
+from django.db import migrations, models
+from django.db.models import Subquery, OuterRef
+
+
+def update_bounce_stats(_a, _b):
+    Session.objects.all().annotate(hit_count=models.Count("hit")).filter(
+        hit_count__gt=1
+    ).update(is_bounce=False)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analytics", "0007_auto_20210328_1634"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="session",
+            name="is_bounce",
+            field=models.BooleanField(db_index=True, default=True),
+        ),
+        migrations.RunPython(update_bounce_stats, lambda: ()),
+    ]

shynet/analytics/models.py

@@ -1,6 +1,7 @@
 import json
 import uuid
 
+from django.conf import settings
 from django.db import models
 from django.shortcuts import reverse
 from django.utils import timezone
@@ -21,7 +22,7 @@ class Session(models.Model):
 
     # Time
     start_time = models.DateTimeField(default=timezone.now, db_index=True)
-    last_seen = models.DateTimeField(default=timezone.now)
+    last_seen = models.DateTimeField(default=timezone.now, db_index=True)
 
     # Core request information
     user_agent = models.TextField()
@@ -48,16 +49,21 @@ class Session(models.Model):
     latitude = models.FloatField(null=True)
     time_zone = models.TextField(blank=True)
 
+    is_bounce = models.BooleanField(default=True, db_index=True)
+
     class Meta:
         ordering = ["-start_time"]
         indexes = [
             models.Index(fields=["service", "-start_time"]),
+            models.Index(fields=["service", "-last_seen"]),
             models.Index(fields=["service", "identifier"]),
         ]
 
     @property
     def is_currently_active(self):
-        return timezone.now() - self.last_seen < timezone.timedelta(seconds=10)
+        return timezone.now() - self.last_seen < timezone.timedelta(
+            milliseconds=settings.SCRIPT_HEARTBEAT_FREQUENCY * 2
+        )
 
     @property
     def duration(self):
@@ -72,6 +78,12 @@ class Session(models.Model):
             kwargs={"pk": self.service.pk, "session_pk": self.uuid},
         )
 
+    def recalculate_bounce(self):
+        bounce = self.hit_set.count() == 1
+        if bounce != self.is_bounce:
+            self.is_bounce = bounce
+            self.save()
+
 
 class Hit(models.Model):
     session = models.ForeignKey(Session, on_delete=models.CASCADE, db_index=True)
@@ -79,7 +91,7 @@ class Hit(models.Model):
 
     # Base request information
     start_time = models.DateTimeField(default=timezone.now, db_index=True)
-    last_seen = models.DateTimeField(default=timezone.now)
+    last_seen = models.DateTimeField(default=timezone.now, db_index=True)
     heartbeats = models.IntegerField(default=0)
     tracker = models.TextField(
         choices=[("JS", "JavaScript"), ("PIXEL", "Pixel (noscript)")]
@@ -88,12 +100,17 @@ class Hit(models.Model):
     # Advanced page information
     location = models.TextField(blank=True, db_index=True)
     referrer = models.TextField(blank=True, db_index=True)
-    load_time = models.FloatField(null=True)
+    load_time = models.FloatField(null=True, db_index=True)
+
+    # While not necessary, we store the root service directly for performance.
+    # It makes querying much easier; no need for inner joins.
+    service = models.ForeignKey(Service, on_delete=models.CASCADE, db_index=True)
 
     class Meta:
         ordering = ["-start_time"]
         indexes = [
             models.Index(fields=["session", "-start_time"]),
+            models.Index(fields=["service", "-start_time"]),
             models.Index(fields=["session", "location"]),
             models.Index(fields=["session", "referrer"]),
         ]
@@ -105,5 +122,5 @@ class Hit(models.Model):
     def get_absolute_url(self):
         return reverse(
             "dashboard:service_session",
-            kwargs={"pk": self.session.service.pk, "session_pk": self.session.pk},
+            kwargs={"pk": self.service.pk, "session_pk": self.session.pk},
         )

shynet/analytics/tasks.py

@@ -1,5 +1,4 @@
 import ipaddress
-import json
 import logging
 from hashlib import sha256
 
@@ -9,6 +8,7 @@ from celery import shared_task
 from django.conf import settings
 from django.core.cache import cache
 from django.db.models import Q
+from django.utils import timezone
 
 from core.models import Service
 
@@ -78,6 +78,11 @@ def ingress_request(
         association_id_hash = sha256()
         association_id_hash.update(str(ip).encode("utf-8"))
         association_id_hash.update(str(user_agent).encode("utf-8"))
+        if settings.AGGRESSIVE_HASH_SALTING:
+            association_id_hash.update(str(service.pk).encode("utf-8"))
+            association_id_hash.update(
+                str(timezone.now().date().isoformat()).encode("utf-8")
+            )
         session_cache_path = (
             f"session_association_{service.pk}_{association_id_hash.hexdigest()}"
         )
@@ -116,7 +121,7 @@ def ingress_request(
                 return
             session = Session.objects.create(
                 service=service,
-                ip=ip if service.collect_ips else None,
+                ip=ip if service.collect_ips and not settings.BLOCK_ALL_IPS else None,
                 user_agent=user_agent,
                 identifier=identifier.strip(),
                 browser=ua.browser.family or "",
@@ -179,7 +184,12 @@ def ingress_request(
                 load_time=payload.get("loadTime"),
                 start_time=time,
                 last_seen=time,
+                service=service,
             )
+
+            # Recalculate whether the session is a bounce
+            session.recalculate_bounce()
+
             # Set idempotency (if applicable)
             if idempotency is not None:
                 cache.set(

shynet/analytics/views/ingress.py

@@ -112,7 +112,9 @@ class ScriptView(ValidateServiceOriginsMixin, View):
         endpoint = (
             reverse(
                 "ingress:endpoint_script",
-                kwargs={"service_uuid": self.kwargs.get("service_uuid"),},
+                kwargs={
+                    "service_uuid": self.kwargs.get("service_uuid"),
+                },
             )
             if self.kwargs.get("identifier") == None
             else reverse(

shynet/core/management/commands/demo.py

@@ -54,15 +54,18 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument(
-            "name", type=str,
+            "name",
+            type=str,
         )
         parser.add_argument("owner_email", type=str)
         parser.add_argument(
-            "avg", type=int,
+            "avg",
+            type=int,
         )
         parser.add_argument("deviation", type=float, default=0.4)
         parser.add_argument(
-            "days", type=int,
+            "days",
+            type=int,
         )
         parser.add_argument("load_time", type=float, default=1000)
 

shynet/core/management/commands/hostname.py

@@ -14,7 +14,8 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument(
-            "hostname", type=str,
+            "hostname",
+            type=str,
         )
 
     def handle(self, *args, **options):

shynet/core/management/commands/registeradmin.py

@@ -14,7 +14,8 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument(
-            "email", type=str,
+            "email",
+            type=str,
         )
 
     def handle(self, *args, **options):

shynet/core/management/commands/whitelabel.py

@@ -14,7 +14,8 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument(
-            "name", type=str,
+            "name",
+            type=str,
         )
 
     def handle(self, *args, **options):

shynet/core/migrations/0001_initial.py

@@ -112,7 +112,9 @@ class Migration(migrations.Migration):
                 "verbose_name_plural": "users",
                 "abstract": False,
             },
-            managers=[("objects", django.contrib.auth.models.UserManager()),],
+            managers=[
+                ("objects", django.contrib.auth.models.UserManager()),
+            ],
         ),
         migrations.CreateModel(
             name="Service",

shynet/core/migrations/0002_auto_20200415_1742.py

@@ -11,6 +11,7 @@ class Migration(migrations.Migration):
 
     operations = [
         migrations.AlterModelOptions(
-            name="service", options={"ordering": ["name", "uuid"]},
+            name="service",
+            options={"ordering": ["name", "uuid"]},
         ),
     ]

shynet/core/models.py

@@ -129,11 +129,11 @@ class Service(models.Model):
         session_count = sessions.count()
 
         hits = Hit.objects.filter(
-            session__service=self, start_time__lt=end_time, start_time__gt=start_time
+            service=self, start_time__lt=end_time, start_time__gt=start_time
         )
         hit_count = hits.count()
 
-        bounces = sessions.annotate(hit_count=models.Count("hit")).filter(hit_count=1)
+        bounces = sessions.filter(is_bounce=True)
         bounce_count = bounces.count()
 
         locations = (
@@ -244,4 +244,7 @@ class Service(models.Model):
         }
 
     def get_absolute_url(self):
-        return reverse("dashboard:service", kwargs={"pk": self.pk},)
+        return reverse(
+            "dashboard:service",
+            kwargs={"pk": self.pk},
+        )

shynet/dashboard/forms.py

@@ -1,5 +1,6 @@
 from allauth.account.admin import EmailAddress
 from django import forms
+from django.conf import settings
 from django.utils.translation import gettext_lazy as _
 
 from core.models import Service, User
@@ -33,7 +34,6 @@ class ServiceForm(forms.ModelForm):
         labels = {
             "origins": "Allowed origins",
             "respect_dnt": "Respect DNT",
-            "collect_ips": "Collect IP addresses",
             "ignored_ips": "Ignored IP addresses",
             "ignore_robots": "Ignore robots",
             "hide_referrer_regex": "Hide specific referrers",
@@ -46,13 +46,27 @@ class ServiceForm(forms.ModelForm):
                 "At what origins does the service operate? Use commas to separate multiple values. This sets CORS headers, so use '*' if you're not sure (or don't care)."
             ),
             "respect_dnt": "Should visitors who have enabled <a href='https://en.wikipedia.org/wiki/Do_Not_Track'>Do Not Track</a> be excluded from all data?",
-            "collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected.",
             "ignored_ips": "A comma-separated list of IP addresses or IP ranges (IPv4 and IPv6) to exclude from tracking (e.g., '192.168.0.2, 127.0.0.1/32').",
             "ignore_robots": "Should sessions generated by bots be excluded from tracking?",
             "hide_referrer_regex": "Any referrers that match this <a href='https://regexr.com/'>RegEx</a> will not be listed in the referrer summary. Sessions will still be tracked normally. No effect if left blank.",
             "script_inject": "Optional additional JavaScript to inject at the end of the Shynet script. This code will be injected on every page where this service is installed.",
         }
 
+    collect_ips = forms.BooleanField(
+        help_text="IP address collection is disabled globally by your administrator."
+        if settings.BLOCK_ALL_IPS
+        else "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected.",
+        widget=forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
+        initial=False if settings.BLOCK_ALL_IPS else True,
+        required=False,
+        disabled=settings.BLOCK_ALL_IPS,
+    )
+
+    def clean_collect_ips(self):
+        collect_ips = self.cleaned_data["collect_ips"]
+        # Forces collect IPs to be false if it is disabled globally
+        return False if settings.BLOCK_ALL_IPS else collect_ips
+
     collaborators = forms.CharField(
         help_text="Which users on this Shynet instance should have read-only access to this service? (Comma separated list of emails.)",
         required=False,

shynet/dashboard/templates/base.html

@@ -11,7 +11,6 @@
   <link rel="icon" type="image/png" href="{% static 'dashboard/images/icon.png' %}">
   <script src="{% static 'apexcharts/dist/apexcharts.min.js'%}"></script>
   <script src="{% static 'litepicker/dist/js/main.js' %}"></script>
-  <script src="{% static 'turbolinks/dist/turbolinks.js' %}"></script>
   <script src="{% static 'dashboard/js/base.js' %}"></script>
   <link rel="stylesheet" href="{% static 'dashboard/css/global.css' %}">
   {% block extra_head %}

shynet/dashboard/templates/dashboard/pages/dashboard.html

@@ -1,6 +1,6 @@
 {% extends "base.html" %}
 
-{% load rules %}
+{% load rules pagination %}
 
 {% block content %}
 <div class="md:flex justify-between items-center">
@@ -18,9 +18,12 @@
     </div>
 </div>
 <hr class="sep">
-{% for object in services|dictsortreversed:"stats.session_count" %}
+{% for object in object_list|dictsortreversed:"stats.session_count" %}
 {% include 'dashboard/includes/service_overview.html' %}
 {% empty %}
 <p>You don't have any services on {{request.site.name|default:"Shynet"}} yet.</p>
 {% endfor %}
+
+{% pagination page_obj request %}
+
 {% endblock %}

shynet/dashboard/templates/dashboard/pages/service_update.html

@@ -15,7 +15,7 @@
     <div class="card ~neutral !high font-mono text-sm">
         {% filter force_escape %}<noscript><img
                 src="{{script_protocol}}{{request.site.domain}}{% url 'ingress:endpoint_pixel' object.uuid %}"></noscript>
-        <script src="{{script_protocol}}{{request.site.domain}}{% url 'ingress:endpoint_script' object.uuid %}"></script>
+        <script defer src="{{script_protocol}}{{request.site.domain}}{% url 'ingress:endpoint_script' object.uuid %}"></script>
         {% endfilter %}
     </div>
     <hr class="sep h-4">
@@ -36,4 +36,4 @@
         </div>
     </form>
 </div>
-{% endblock %}
+{% endblock %}

shynet/dashboard/templatetags/helpers.py

@@ -1,3 +1,4 @@
+from datetime import timedelta
 from urllib.parse import urlparse
 
 import flag
@@ -43,7 +44,12 @@ def country_name(isocode):
 
 @register.simple_tag
 def relative_stat_tone(
-    start, end, good="UP", good_classes=None, bad_classes=None, neutral_classes=None,
+    start,
+    end,
+    good="UP",
+    good_classes=None,
+    bad_classes=None,
+    neutral_classes=None,
 ):
     good_classes = good_classes or "~positive"
     bad_classes = bad_classes or "~critical"
@@ -97,6 +103,12 @@ def compare(
     bad_classes=None,
     neutral_classes=None,
 ):
+    if isinstance(start, timedelta):
+        start = start.seconds
+
+    if isinstance(end, timedelta):
+        end = end.seconds
+
     return {
         "start": start,
         "end": end,

shynet/dashboard/views.py

@@ -22,16 +22,24 @@ from .forms import ServiceForm
 from .mixins import DateRangeMixin
 
 
-class DashboardView(LoginRequiredMixin, DateRangeMixin, TemplateView):
+class DashboardView(LoginRequiredMixin, DateRangeMixin, ListView):
+    model = Service
     template_name = "dashboard/pages/dashboard.html"
+    paginate_by = 5
+
+    def get_queryset(self):
+        return Service.objects.filter(
+            Q(owner=self.request.user) | Q(collaborators__in=[self.request.user])
+        ).distinct()
 
     def get_context_data(self, **kwargs):
         data = super().get_context_data(**kwargs)
-        data["services"] = Service.objects.filter(
+
-            Q(owner=self.request.user) | Q(collaborators__in=[self.request.user])
+        for service in data["object_list"]:
-        ).distinct()
+            service.stats = service.get_core_stats(
-        for service in data["services"]:
+                self.get_start_date(), self.get_end_date()
-            service.stats = service.get_core_stats(data["start_date"], data["end_date"])
+            )
+
         return data
 
 

shynet/shynet/settings.py

@@ -18,7 +18,7 @@ import urllib.parse as urlparse
 from django.contrib.messages import constants as messages
 
 # Increment on new releases
-VERSION = "v0.7.2"
+VERSION = "v0.8.1"
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -58,6 +58,7 @@ INSTALLED_APPS = [
     "allauth",
     "allauth.account",
     "allauth.socialaccount",
+    "debug_toolbar",
 ]
 
 MIDDLEWARE = [
@@ -70,6 +71,7 @@ MIDDLEWARE = [
     "django.contrib.sites.middleware.CurrentSiteMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "debug_toolbar.middleware.DebugToolbarMiddleware",
 ]
 
 ROOT_URLCONF = "shynet.urls"
@@ -145,9 +147,15 @@ AUTH_PASSWORD_VALIDATORS = [
     {
         "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
     },
-    {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",},
+    {
-    {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",},
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
-    {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",},
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
+    },
 ]
 
 # Logging
@@ -247,6 +255,10 @@ LOGIN_REDIRECT_URL = "/"
 
 SITE_ID = 1
 
+INTERNAL_IPS = [
+    "127.0.0.1",
+]
+
 # Celery
 
 CELERY_TASK_ALWAYS_EAGER = os.getenv("CELERY_TASK_ALWAYS_EAGER", "True") == "True"
@@ -321,3 +333,9 @@ SHOW_SHYNET_VERSION = os.getenv("SHOW_SHYNET_VERSION", "True") == "True"
 
 # Should Shynet show third-party icons in the dashboard?
 SHOW_THIRD_PARTY_ICONS = os.getenv("SHOW_THIRD_PARTY_ICONS", "True") == "True"
+
+# Should Shynet never collect any IP?
+BLOCK_ALL_IPS = os.getenv("BLOCK_ALL_IPS", "False") == "True"
+
+# Include date and service ID in salt?
+AGGRESSIVE_HASH_SALTING = os.getenv("AGGRESSIVE_HASH_SALTING", "False") == "True"

shynet/shynet/urls.py

@@ -15,8 +15,10 @@ Including another URLconf
 """
 from django.contrib import admin
 from django.urls import include, path
+import debug_toolbar
 
 urlpatterns = [
+    path("__debug__/", include(debug_toolbar.urls)),
     path("admin/", admin.site.urls),
     path("accounts/", include("allauth.urls")),
     path("ingress/", include(("analytics.ingress_urls", "ingress")), name="ingress"),