Compare commits

...

37 Commits

Author SHA1 Message Date
R. Miles McCain
fe8e766670 Format 2021-03-29 14:37:59 +00:00
R. Miles McCain
b63863e283 Bump version 2021-03-29 14:37:21 +00:00
R. Miles McCain
516f9fb951 Fix aggressive hash salting 2021-03-29 14:37:08 +00:00
R. Miles McCain
c2234ec647 Bump version 2021-03-28 22:15:30 +00:00
R. Miles McCain
02cbee5c8c Cache bounce 2021-03-28 21:55:38 +00:00
R. Miles McCain
518436ffd2 Relock npm packages 2021-03-28 21:37:09 +00:00
R. Miles McCain
311aa2b1ac Drop Turbolinks 2021-03-28 21:36:53 +00:00
R. Miles McCain
8ad44ddc23 Add pagination to dashboard 2021-03-28 21:29:54 +00:00
R. Miles McCain
874aad87a8 Store service directly in Hit 2021-03-28 20:54:19 +00:00
R. Miles McCain
f2e875d03d Add indexes to key Hit fields 2021-03-28 19:18:57 +00:00
R. Miles McCain
45fd32c8ca Index last_seen 2021-03-28 19:15:03 +00:00
R. Miles McCain
08b36ba69f Integrate debug toolbar 2021-03-28 19:14:56 +00:00
R. Miles McCain
d5cfe577a0 Add debug toolbar 2021-03-28 19:14:33 +00:00
R. Miles McCain
c131cfef27 Merge branch 'patch-1' into dev 2021-03-28 18:54:24 +00:00
R. Miles McCain
526d4cd133 Relock Pipfile 2021-03-28 18:53:49 +00:00
R. Miles McCain
8e09871b44 Merge branch 'dependabot/pip/django-3.1.6' into dev 2021-03-28 18:51:33 +00:00
R. Miles McCain
6aa3ce0b32 Merge branch 'dependabot/pip/pyyaml-5.4' into dev 2021-03-28 18:49:49 +00:00
dependabot[bot]
23ea8e493e Bump pyyaml from 5.3.1 to 5.4
Bumps [pyyaml](https://github.com/yaml/pyyaml) from 5.3.1 to 5.4.
- [Release notes](https://github.com/yaml/pyyaml/releases)
- [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES)
- [Commits](https://github.com/yaml/pyyaml/compare/5.3.1...5.4)

Signed-off-by: dependabot[bot] <support@github.com>
2021-03-25 23:41:11 +00:00
dependabot[bot]
22d996bed7 Bump django from 3.1.3 to 3.1.6
Bumps [django](https://github.com/django/django) from 3.1.3 to 3.1.6.
- [Release notes](https://github.com/django/django/releases)
- [Commits](https://github.com/django/django/compare/3.1.3...3.1.6)

Signed-off-by: dependabot[bot] <support@github.com>
2021-03-19 02:04:53 +00:00
Kasper Seweryn
9df864787c Fix #99 2021-02-17 02:24:35 +00:00
dependabot[bot]
b7a6ac9ec0 Bump apexcharts from 3.23.1 to 3.24.0 (#97)
Bumps [apexcharts](https://github.com/apexcharts/apexcharts.js) from 3.23.1 to 3.24.0.
- [Release notes](https://github.com/apexcharts/apexcharts.js/releases)
- [Commits](https://github.com/apexcharts/apexcharts.js/commits)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2021-02-15 18:56:29 -05:00
R. Miles McCain
38d8d416e1 Update kubernetes deployments 2021-01-25 13:39:53 -05:00
R. Miles McCain
592613a99a Use dev shynet in kubernetes deployments 2021-01-23 23:28:24 -05:00
R. Miles McCain
e9f43c6a53 Bump version 2021-01-23 23:23:19 -05:00
R. Miles McCain
89c6800913 Fix formatting 2021-01-23 23:16:33 -05:00
R. Miles McCain
db9c807289 Add optional more aggressive salting (fixes #95) 2021-01-23 23:13:44 -05:00
R. Miles McCain
6e48a3eac7 Merge branch 'global-ip-block' into dev 2021-01-23 23:01:53 -05:00
R. Miles McCain
ba9a716913 Merge branch 'heartbeat-frequency' into dev 2021-01-23 22:41:25 -05:00
R. Miles McCain
6d7292a60a Fix duration change being unknown (fixes #89) 2021-01-23 22:40:19 -05:00
Oliver Kamer
c0d02732e7 Add additional env variable to template 2021-01-19 22:05:33 +01:00
Oliver Kamer
d071a91917 Block Collect IP option if disabled globally 2021-01-19 22:02:57 +01:00
Oliver Kamer
d67e14b08f Block IP collection from settings 2021-01-19 21:41:54 +01:00
Oliver Kamer
174a386f54 Add block all ips to settings 2021-01-19 21:31:02 +01:00
Oliver Kamer
ce23cfc5b5 Add pycharm gitignore stuff 2021-01-19 21:20:30 +01:00
Oliver Kamer
8be690c417 Use heartbeat frequency for currently active
If the heartbeat frequency is more than 10 seconds, shynet will display as not active, even though it still is.

Using 2x the heartbeat frequency should give better results.
2021-01-19 11:32:36 +01:00
R. Miles McCain
2f778dc4b4 Bump version to v0.7.3 2021-01-11 12:12:15 -05:00
R. Miles McCain
e0c165313b Add fallback to percent_change_display (fixes #89) 2021-01-11 12:11:27 -05:00
33 changed files with 444 additions and 109 deletions

3
.gitignore vendored
View File

@@ -139,3 +139,6 @@ secrets.yml
.vscode
.DS_Store
compiledstatic/
# Pycharm
.idea

View File

@@ -4,13 +4,13 @@ url = "https://pypi.org/simple"
verify_ssl = true
[packages]
django = "~=3.0"
django = "~=3.1"
django-allauth = "~=0.42.0"
geoip2 = "~=3.0.0"
whitenoise = "~=5.1.0"
celery = "~=4.4.6"
django-ipware = "~=2.1.0"
pyyaml = "~=5.3.1"
pyyaml = "~=5.4"
ua-parser = "~=0.10.0"
user-agents = "~=2.1"
emoji-country-flag = "~=1.2.1"
@@ -23,3 +23,4 @@ pycountry = "~=19.8.18"
html2text = "~=2020.1.16"
django-health-check = "~=3.12.1"
django-npm = "~=1.0.0"
django-debug-toolbar = "*"

117
Pipfile.lock generated
View File

@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "73fbc4a0251ccae805550aa48b70ec9f668bee6fa4ff9503e023b3854a06bce8"
"sha256": "f8c76565a776f1bd36364077a86d6c16fccc522d9d2024bb9b51be5cb9f8b4b5"
},
"pipfile-spec": 6,
"requires": {},
@@ -19,6 +19,7 @@
"sha256:70cdb10628468ff14e57ec2f751c7aa9e48e7e3651cfd62d431213c0c4e58f21",
"sha256:aa7f313fb887c91f15474c1229907a04dac0b8135822d6603437803424c0aa59"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==2.6.1"
},
"asgiref": {
@@ -26,6 +27,7 @@
"sha256:5ee950735509d04eb673bd7f7120f8fa1c9e2df495394992c73234d526907e17",
"sha256:7162a3cb30ab0609f1a4c95938fd73e8604f63bdba516a7f7d64b83ff09478f0"
],
"markers": "python_version >= '3.5'",
"version": "==3.3.1"
},
"billiard": {
@@ -45,32 +47,34 @@
},
"certifi": {
"hashes": [
"sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd",
"sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4"
"sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
"sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
],
"version": "==2020.11.8"
"version": "==2020.12.5"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
"sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
"sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
],
"version": "==3.0.4"
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==4.0.0"
},
"defusedxml": {
"hashes": [
"sha256:6687150770438374ab581bb7a1b327a847dd9c5749e396102de3fad4e8a3ef93",
"sha256:f684034d135af4c6cbb949b8a4d2ed61634515257a67299e5f940fbaa34377f5"
"sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69",
"sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"
],
"version": "==0.6.0"
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==0.7.1"
},
"django": {
"hashes": [
"sha256:14a4b7cd77297fba516fc0d92444cc2e2e388aa9de32d7a68d4a83d58f5a4927",
"sha256:14b87775ffedab2ef6299b73343d1b4b41e5d4e2aa58c6581f114dbec01e3f8f"
"sha256:32ce792ee9b6a0cbbec340123e229ac9f765dff8c2a4ae9247a14b2ba3a365a7",
"sha256:baf099db36ad31f970775d0be5587cc58a6256a6771a44eb795b554d45f211b8"
],
"index": "pypi",
"version": "==3.1.3"
"version": "==3.1.7"
},
"django-allauth": {
"hashes": [
@@ -79,6 +83,14 @@
"index": "pypi",
"version": "==0.42.0"
},
"django-debug-toolbar": {
"hashes": [
"sha256:84e2607d900dbd571df0a2acf380b47c088efb787dce9805aefeb407341961d2",
"sha256:9e5a25d0c965f7e686f6a8ba23613ca9ca30184daa26487706d4829f5cfb697a"
],
"index": "pypi",
"version": "==3.2"
},
"django-health-check": {
"hashes": [
"sha256:2667b89b8f85ad9b2a24c90581b376016d22ea912fedf37f9866413a3c2e0a5d",
@@ -110,11 +122,11 @@
},
"emoji-country-flag": {
"hashes": [
"sha256:256e47d30fb43bf154f370cc3c9e767f003aeb8653e31ba7b87151669c608e19",
"sha256:3f6c32699c19489383497865b208260b1d55b8424d66e08049187b13db2f0b8a"
"sha256:338f5e374119dcde093cfeaa8ca3af372d4b8d984d89a7fb2fb0db0011662560",
"sha256:a3a068191294294143d8ef294fdfe9792c5c243753eac130798bf2fa5de38185"
],
"index": "pypi",
"version": "==1.2.3"
"version": "==1.2.4"
},
"geoip2": {
"hashes": [
@@ -145,6 +157,7 @@
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==2.10"
},
"kombu": {
@@ -152,12 +165,14 @@
"sha256:be48cdffb54a2194d93ad6533d73f69408486483d189fe9f5990ee24255b0e0a",
"sha256:ca1b45faac8c0b18493d02a8571792f3c40291cf2bcf1f55afed3d8f3aa7ba74"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==4.6.11"
},
"maxminddb": {
"hashes": [
"sha256:47e86a084dd814fac88c99ea34ba3278a74bc9de5a25f4b815b608798747c7dc"
],
"markers": "python_version >= '3.6'",
"version": "==2.0.3"
},
"oauthlib": {
@@ -165,6 +180,7 @@
"sha256:bee41cc35fcca6e988463cacc3bcb8a96224f470ca547e697b604cc697b2f889",
"sha256:df884cd6cbe20e32633f1db1072e9356f53638e4361bef4e8b03c9127c9328ea"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==3.1.0"
},
"psycopg2-binary": {
@@ -224,29 +240,45 @@
},
"pytz": {
"hashes": [
"sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268",
"sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"
"sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
"sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
],
"version": "==2020.4"
"version": "==2021.1"
},
"pyyaml": {
"hashes": [
"sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97",
"sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76",
"sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2",
"sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e",
"sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648",
"sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf",
"sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f",
"sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2",
"sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee",
"sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a",
"sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d",
"sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c",
"sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"
"sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf",
"sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696",
"sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393",
"sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77",
"sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922",
"sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5",
"sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8",
"sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10",
"sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc",
"sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018",
"sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e",
"sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253",
"sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347",
"sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183",
"sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541",
"sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb",
"sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185",
"sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc",
"sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db",
"sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa",
"sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46",
"sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122",
"sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b",
"sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63",
"sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df",
"sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc",
"sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247",
"sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6",
"sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"
],
"index": "pypi",
"version": "==5.3.1"
"version": "==5.4.1"
},
"redis": {
"hashes": [
@@ -258,15 +290,17 @@
},
"requests": {
"hashes": [
"sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8",
"sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998"
"sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
"sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
],
"version": "==2.25.0"
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==2.25.1"
},
"requests-oauthlib": {
"hashes": [
"sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d",
"sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a"
"sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a",
"sha256:fa6c47b933f01060936d87ae9327fead68768b69c6c9ea2109c48be30f2d4dbc"
],
"version": "==1.3.0"
},
@@ -282,6 +316,7 @@
"sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0",
"sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8"
],
"markers": "python_version >= '3.5'",
"version": "==0.4.1"
},
"ua-parser": {
@@ -294,10 +329,11 @@
},
"urllib3": {
"hashes": [
"sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
"sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
"sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
"sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
],
"version": "==1.26.2"
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
"version": "==1.26.4"
},
"user-agents": {
"hashes": [
@@ -312,6 +348,7 @@
"sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87",
"sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==1.3.0"
},
"whitenoise": {

View File

@@ -73,3 +73,13 @@ SHOW_SHYNET_VERSION=True
# Should Shynet show third-party icons in the dashboard?
SHOW_THIRD_PARTY_ICONS=True
# Should Shynet block collection of IP addresses globally?
BLOCK_ALL_IPS=False
# Should Shynet include the date and site ID when hashing users?
# This will prevent any possibility of cross-site tracking provided
# that IP collection is also disabled, and external keys (primary
# keys) aren't supplied. It will also prevent sessions from spanning
# one day to another.
AGGRESSIVE_HASH_SALTING=True

View File

@@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: "shynet-webserver"
image: "milesmcc/shynet:latest"
image: "milesmcc/shynet:dev"
imagePullPolicy: Always
envFrom:
- secretRef:
@@ -42,7 +42,7 @@ spec:
spec:
containers:
- name: "shynet-celeryworker"
image: "milesmcc/shynet:latest"
image: "milesmcc/shynet:dev"
command: ["./celeryworker.sh"]
imagePullPolicy: Always
envFrom:
@@ -61,7 +61,7 @@ spec:
selector:
app: shynet-redis
---
apiVersion: apps/v1beta2
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: shynet-redis
@@ -83,3 +83,37 @@ spec:
ports:
- containerPort: 6379
name: redis
---
apiVersion: v1
kind: Service
metadata:
name: shynet-webserver-service
spec:
type: ClusterIP
ports:
- port: 8080
selector:
app: shynet-webserver
---
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: shynet-webserver-ingress
annotations:
kubernetes.io/ingress.class: addon-http-application-routing
spec:
rules:
- host: shynet.rmrm.io
http:
paths:
- backend:
serviceName: shynet-webserver-service
servicePort: 8080
path: /
- host: shynet-beta.rmrm.io
http:
paths:
- backend:
serviceName: shynet-webserver-service
servicePort: 8080
path: /

11
package-lock.json generated
View File

@@ -14,9 +14,9 @@
"integrity": "sha512-peIPrH9eDiu49LLzLlSTFFrXj6WLlEX3TRsUkqyyOHi/i58ilJ/eERnu7AcswXhuCBx+/2W9EUuHM+8iAq4ipg=="
},
"apexcharts": {
"version": "3.23.1",
"resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.23.1.tgz",
"integrity": "sha512-7fRpquXp725BUew5OO1mJWk16/IJPCUl0l8SjhISnAhAtbTaM9PnXPSmN2BvKO4RcT457CzMM7MCG5UokiTwcA==",
"version": "3.24.0",
"resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.24.0.tgz",
"integrity": "sha512-iT6czJCIVrmAtrcO90MZTQCvC+xi6R6Acf0jNH/d40FVTtCfcqECuKIh5iAMyOTtgUb7+fQ8rbadH2bm1kbL9Q==",
"requires": {
"svg.draggable.js": "^2.2.2",
"svg.easing.js": "^2.0.0",
@@ -99,11 +99,6 @@
"requires": {
"svg.js": "^2.6.5"
}
},
"turbolinks": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/turbolinks/-/turbolinks-5.2.0.tgz",
"integrity": "sha512-pMiez3tyBo6uRHFNNZoYMmrES/IaGgMhQQM+VFF36keryjb5ms0XkVpmKHkfW/4Vy96qiGW3K9bz0tF5sK9bBw=="
}
}
}

View File

@@ -19,9 +19,8 @@
"dependencies": {
"@fortawesome/fontawesome-free": "^5.15.1",
"a17t": "^0.5.1",
"apexcharts": "^3.23.1",
"apexcharts": "^3.24.0",
"inter-ui": "^3.15.0",
"litepicker": "^1.5.7",
"turbolinks": "^5.2.0"
"litepicker": "^1.5.7"
}
}

View File

@@ -1,12 +1,12 @@
<nav class="flex w-full flex-wrap items-center justify-between" role="navigation" aria-label="pagination">
<div class="w-full md:w-auto mb-2">
{% if page.has_previous %}
<a href="?page={{ page.previous_page_number }}{{url_parameters}}" class="button field bg-neutral-000 w-auto mr-1">Previous</a>
<a href="?page={{ page.previous_page_number }}&{{url_parameters}}" class="button field bg-neutral-000 w-auto mr-1">Previous</a>
{% else %}
<a class="button field bg-neutral-000 w-auto mr-1" disabled>Previous</a>
{% endif %}
{% if page.has_next %}
<a href="?page={{ page.next_page_number }}{{url_parameters}}" class="button field bg-neutral-000 w-auto">Next</a>
<a href="?page={{ page.next_page_number }}&{{url_parameters}}" class="button field bg-neutral-000 w-auto">Next</a>
{% else %}
<a class="button field bg-neutral-000 w-auto" disabled>Next</a>
{% endif %}
@@ -17,7 +17,7 @@
{% ifequal page.number pnum %}
<li><a class="button field w-auto mx-1 text-white bg-neutral-700">{{ pnum }}</a></li>
{% else %}
<li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}{{url_parameters}}">{{ pnum }}</a></li>
<li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}&{{url_parameters}}">{{ pnum }}</a></li>
{% endifequal %}
{% endfor %}
@@ -27,7 +27,7 @@
{% ifequal page.number pnum %}
<li><a class="button field w-auto mx-1 text-white bg-neutral-700">{{ pnum }}</a></li>
{% else %}
<li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}{{url_parameters}}">{{ pnum }}</a></li>
<li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}&{{url_parameters}}">{{ pnum }}</a></li>
{% endifequal %}
{% endfor %}
{% endif %}
@@ -38,7 +38,7 @@
{% ifequal page.number pnum %}
<li><a class="button field w-auto mx-1 text-white bg-neutral-700">{{ pnum }}</a></li>
{% else %}
<li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}{{url_parameters}}">{{ pnum }}</a></li>
<li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}&{{url_parameters}}">{{ pnum }}</a></li>
{% endifequal %}
{% endfor %}
{% endif %}

View File

@@ -15,12 +15,8 @@ def pagination(
before_current_pages=4,
after_current_pages=4,
):
url_parameters = "".join(
[
f"&{urlencode(key)}={urlencode(value)}"
for key, value in request.GET.items()
if key != "page"
]
url_parameters = urlencode(
[(key, value) for key, value in request.GET.items() if key != "page"]
)
before = max(page.number - before_current_pages - 1, 0)

View File

@@ -60,7 +60,9 @@ class Migration(migrations.Migration):
),
),
],
options={"ordering": ["-start_time"],},
options={
"ordering": ["-start_time"],
},
),
migrations.CreateModel(
name="Hit",
@@ -90,7 +92,9 @@ class Migration(migrations.Migration):
),
),
],
options={"ordering": ["-start_time"],},
options={
"ordering": ["-start_time"],
},
),
migrations.AddIndex(
model_name="session",

View File

@@ -0,0 +1,46 @@
# Generated by Django 3.1.7 on 2021-03-28 19:14
from django.db import migrations, models
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [
("analytics", "0003_auto_20200502_1227"),
]
operations = [
migrations.AlterField(
model_name="hit",
name="last_seen",
field=models.DateTimeField(default=django.utils.timezone.now),
),
migrations.AlterField(
model_name="hit",
name="start_time",
field=models.DateTimeField(
db_index=True, default=django.utils.timezone.now
),
),
migrations.AlterField(
model_name="session",
name="last_seen",
field=models.DateTimeField(
db_index=True, default=django.utils.timezone.now
),
),
migrations.AlterField(
model_name="session",
name="start_time",
field=models.DateTimeField(
db_index=True, default=django.utils.timezone.now
),
),
migrations.AddIndex(
model_name="session",
index=models.Index(
fields=["service", "-last_seen"], name="analytics_s_service_10bb96_idx"
),
),
]

View File

@@ -0,0 +1,26 @@
# Generated by Django 3.1.7 on 2021-03-28 19:18
from django.db import migrations, models
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [
("analytics", "0004_auto_20210328_1514"),
]
operations = [
migrations.AlterField(
model_name="hit",
name="last_seen",
field=models.DateTimeField(
db_index=True, default=django.utils.timezone.now
),
),
migrations.AlterField(
model_name="hit",
name="load_time",
field=models.FloatField(db_index=True, null=True),
),
]

View File

@@ -0,0 +1,40 @@
# Generated by Django 3.1.7 on 2021-03-28 19:36
from ..models import Hit, Session
from django.db import migrations, models
import django.db.models.deletion
from django.db.models import Subquery, OuterRef
def add_service_to_hits(_a, _b):
service = Session.objects.filter(pk=OuterRef("session")).values_list("service")[:1]
Hit.objects.update(service=Subquery(service))
class Migration(migrations.Migration):
dependencies = [
("core", "0008_auto_20200628_1403"),
("analytics", "0005_auto_20210328_1518"),
]
operations = [
migrations.AddField(
model_name="hit",
name="service",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
to="core.service",
),
),
migrations.RunPython(add_service_to_hits, lambda: ()),
migrations.AlterField(
model_name="hit",
name="service",
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="core.service"
),
),
]

View File

@@ -0,0 +1,19 @@
# Generated by Django 3.1.7 on 2021-03-28 20:34
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("analytics", "0006_hit_service"),
]
operations = [
migrations.AddIndex(
model_name="hit",
index=models.Index(
fields=["service", "-start_time"], name="analytics_h_service_f4f41e_idx"
),
),
]

View File

@@ -0,0 +1,28 @@
# Generated by Django 3.1.7 on 2021-03-28 21:38
from django.db.models.expressions import F
from ..models import Session, Hit
from django.db import migrations, models
from django.db.models import Subquery, OuterRef
def update_bounce_stats(_a, _b):
Session.objects.all().annotate(hit_count=models.Count("hit")).filter(
hit_count__gt=1
).update(is_bounce=False)
class Migration(migrations.Migration):
dependencies = [
("analytics", "0007_auto_20210328_1634"),
]
operations = [
migrations.AddField(
model_name="session",
name="is_bounce",
field=models.BooleanField(db_index=True, default=True),
),
migrations.RunPython(update_bounce_stats, lambda: ()),
]

View File

@@ -1,6 +1,7 @@
import json
import uuid
from django.conf import settings
from django.db import models
from django.shortcuts import reverse
from django.utils import timezone
@@ -21,7 +22,7 @@ class Session(models.Model):
# Time
start_time = models.DateTimeField(default=timezone.now, db_index=True)
last_seen = models.DateTimeField(default=timezone.now)
last_seen = models.DateTimeField(default=timezone.now, db_index=True)
# Core request information
user_agent = models.TextField()
@@ -48,16 +49,21 @@ class Session(models.Model):
latitude = models.FloatField(null=True)
time_zone = models.TextField(blank=True)
is_bounce = models.BooleanField(default=True, db_index=True)
class Meta:
ordering = ["-start_time"]
indexes = [
models.Index(fields=["service", "-start_time"]),
models.Index(fields=["service", "-last_seen"]),
models.Index(fields=["service", "identifier"]),
]
@property
def is_currently_active(self):
return timezone.now() - self.last_seen < timezone.timedelta(seconds=10)
return timezone.now() - self.last_seen < timezone.timedelta(
milliseconds=settings.SCRIPT_HEARTBEAT_FREQUENCY * 2
)
@property
def duration(self):
@@ -72,6 +78,12 @@ class Session(models.Model):
kwargs={"pk": self.service.pk, "session_pk": self.uuid},
)
def recalculate_bounce(self):
bounce = self.hit_set.count() == 1
if bounce != self.is_bounce:
self.is_bounce = bounce
self.save()
class Hit(models.Model):
session = models.ForeignKey(Session, on_delete=models.CASCADE, db_index=True)
@@ -79,7 +91,7 @@ class Hit(models.Model):
# Base request information
start_time = models.DateTimeField(default=timezone.now, db_index=True)
last_seen = models.DateTimeField(default=timezone.now)
last_seen = models.DateTimeField(default=timezone.now, db_index=True)
heartbeats = models.IntegerField(default=0)
tracker = models.TextField(
choices=[("JS", "JavaScript"), ("PIXEL", "Pixel (noscript)")]
@@ -88,12 +100,17 @@ class Hit(models.Model):
# Advanced page information
location = models.TextField(blank=True, db_index=True)
referrer = models.TextField(blank=True, db_index=True)
load_time = models.FloatField(null=True)
load_time = models.FloatField(null=True, db_index=True)
# While not necessary, we store the root service directly for performance.
# It makes querying much easier; no need for inner joins.
service = models.ForeignKey(Service, on_delete=models.CASCADE, db_index=True)
class Meta:
ordering = ["-start_time"]
indexes = [
models.Index(fields=["session", "-start_time"]),
models.Index(fields=["service", "-start_time"]),
models.Index(fields=["session", "location"]),
models.Index(fields=["session", "referrer"]),
]
@@ -105,5 +122,5 @@ class Hit(models.Model):
def get_absolute_url(self):
return reverse(
"dashboard:service_session",
kwargs={"pk": self.session.service.pk, "session_pk": self.session.pk},
kwargs={"pk": self.service.pk, "session_pk": self.session.pk},
)

View File

@@ -1,5 +1,4 @@
import ipaddress
import json
import logging
from hashlib import sha256
@@ -9,6 +8,7 @@ from celery import shared_task
from django.conf import settings
from django.core.cache import cache
from django.db.models import Q
from django.utils import timezone
from core.models import Service
@@ -78,6 +78,11 @@ def ingress_request(
association_id_hash = sha256()
association_id_hash.update(str(ip).encode("utf-8"))
association_id_hash.update(str(user_agent).encode("utf-8"))
if settings.AGGRESSIVE_HASH_SALTING:
association_id_hash.update(str(service.pk).encode("utf-8"))
association_id_hash.update(
str(timezone.now().date().isoformat()).encode("utf-8")
)
session_cache_path = (
f"session_association_{service.pk}_{association_id_hash.hexdigest()}"
)
@@ -116,7 +121,7 @@ def ingress_request(
return
session = Session.objects.create(
service=service,
ip=ip if service.collect_ips else None,
ip=ip if service.collect_ips and not settings.BLOCK_ALL_IPS else None,
user_agent=user_agent,
identifier=identifier.strip(),
browser=ua.browser.family or "",
@@ -179,7 +184,12 @@ def ingress_request(
load_time=payload.get("loadTime"),
start_time=time,
last_seen=time,
service=service,
)
# Recalculate whether the session is a bounce
session.recalculate_bounce()
# Set idempotency (if applicable)
if idempotency is not None:
cache.set(

View File

@@ -112,7 +112,9 @@ class ScriptView(ValidateServiceOriginsMixin, View):
endpoint = (
reverse(
"ingress:endpoint_script",
kwargs={"service_uuid": self.kwargs.get("service_uuid"),},
kwargs={
"service_uuid": self.kwargs.get("service_uuid"),
},
)
if self.kwargs.get("identifier") == None
else reverse(

View File

@@ -54,15 +54,18 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
"name", type=str,
"name",
type=str,
)
parser.add_argument("owner_email", type=str)
parser.add_argument(
"avg", type=int,
"avg",
type=int,
)
parser.add_argument("deviation", type=float, default=0.4)
parser.add_argument(
"days", type=int,
"days",
type=int,
)
parser.add_argument("load_time", type=float, default=1000)

View File

@@ -14,7 +14,8 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
"hostname", type=str,
"hostname",
type=str,
)
def handle(self, *args, **options):

View File

@@ -14,7 +14,8 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
"email", type=str,
"email",
type=str,
)
def handle(self, *args, **options):

View File

@@ -14,7 +14,8 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
"name", type=str,
"name",
type=str,
)
def handle(self, *args, **options):

View File

@@ -112,7 +112,9 @@ class Migration(migrations.Migration):
"verbose_name_plural": "users",
"abstract": False,
},
managers=[("objects", django.contrib.auth.models.UserManager()),],
managers=[
("objects", django.contrib.auth.models.UserManager()),
],
),
migrations.CreateModel(
name="Service",

View File

@@ -11,6 +11,7 @@ class Migration(migrations.Migration):
operations = [
migrations.AlterModelOptions(
name="service", options={"ordering": ["name", "uuid"]},
name="service",
options={"ordering": ["name", "uuid"]},
),
]

View File

@@ -129,11 +129,11 @@ class Service(models.Model):
session_count = sessions.count()
hits = Hit.objects.filter(
session__service=self, start_time__lt=end_time, start_time__gt=start_time
service=self, start_time__lt=end_time, start_time__gt=start_time
)
hit_count = hits.count()
bounces = sessions.annotate(hit_count=models.Count("hit")).filter(hit_count=1)
bounces = sessions.filter(is_bounce=True)
bounce_count = bounces.count()
locations = (
@@ -244,4 +244,7 @@ class Service(models.Model):
}
def get_absolute_url(self):
return reverse("dashboard:service", kwargs={"pk": self.pk},)
return reverse(
"dashboard:service",
kwargs={"pk": self.pk},
)

View File

@@ -1,5 +1,6 @@
from allauth.account.admin import EmailAddress
from django import forms
from django.conf import settings
from django.utils.translation import gettext_lazy as _
from core.models import Service, User
@@ -33,7 +34,6 @@ class ServiceForm(forms.ModelForm):
labels = {
"origins": "Allowed origins",
"respect_dnt": "Respect DNT",
"collect_ips": "Collect IP addresses",
"ignored_ips": "Ignored IP addresses",
"ignore_robots": "Ignore robots",
"hide_referrer_regex": "Hide specific referrers",
@@ -46,13 +46,27 @@ class ServiceForm(forms.ModelForm):
"At what origins does the service operate? Use commas to separate multiple values. This sets CORS headers, so use '*' if you're not sure (or don't care)."
),
"respect_dnt": "Should visitors who have enabled <a href='https://en.wikipedia.org/wiki/Do_Not_Track'>Do Not Track</a> be excluded from all data?",
"collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected.",
"ignored_ips": "A comma-separated list of IP addresses or IP ranges (IPv4 and IPv6) to exclude from tracking (e.g., '192.168.0.2, 127.0.0.1/32').",
"ignore_robots": "Should sessions generated by bots be excluded from tracking?",
"hide_referrer_regex": "Any referrers that match this <a href='https://regexr.com/'>RegEx</a> will not be listed in the referrer summary. Sessions will still be tracked normally. No effect if left blank.",
"script_inject": "Optional additional JavaScript to inject at the end of the Shynet script. This code will be injected on every page where this service is installed.",
}
collect_ips = forms.BooleanField(
help_text="IP address collection is disabled globally by your administrator."
if settings.BLOCK_ALL_IPS
else "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected.",
widget=forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
initial=False if settings.BLOCK_ALL_IPS else True,
required=False,
disabled=settings.BLOCK_ALL_IPS,
)
def clean_collect_ips(self):
collect_ips = self.cleaned_data["collect_ips"]
# Forces collect IPs to be false if it is disabled globally
return False if settings.BLOCK_ALL_IPS else collect_ips
collaborators = forms.CharField(
help_text="Which users on this Shynet instance should have read-only access to this service? (Comma separated list of emails.)",
required=False,

View File

@@ -11,7 +11,6 @@
<link rel="icon" type="image/png" href="{% static 'dashboard/images/icon.png' %}">
<script src="{% static 'apexcharts/dist/apexcharts.min.js'%}"></script>
<script src="{% static 'litepicker/dist/js/main.js' %}"></script>
<script src="{% static 'turbolinks/dist/turbolinks.js' %}"></script>
<script src="{% static 'dashboard/js/base.js' %}"></script>
<link rel="stylesheet" href="{% static 'dashboard/css/global.css' %}">
{% block extra_head %}

View File

@@ -1,6 +1,6 @@
{% extends "base.html" %}
{% load rules %}
{% load rules pagination %}
{% block content %}
<div class="md:flex justify-between items-center">
@@ -18,9 +18,12 @@
</div>
</div>
<hr class="sep">
{% for object in services|dictsortreversed:"stats.session_count" %}
{% for object in object_list|dictsortreversed:"stats.session_count" %}
{% include 'dashboard/includes/service_overview.html' %}
{% empty %}
<p>You don't have any services on {{request.site.name|default:"Shynet"}} yet.</p>
{% endfor %}
{% pagination page_obj request %}
{% endblock %}

View File

@@ -15,7 +15,7 @@
<div class="card ~neutral !high font-mono text-sm">
{% filter force_escape %}<noscript><img
src="{{script_protocol}}{{request.site.domain}}{% url 'ingress:endpoint_pixel' object.uuid %}"></noscript>
<script src="{{script_protocol}}{{request.site.domain}}{% url 'ingress:endpoint_script' object.uuid %}"></script>
<script defer src="{{script_protocol}}{{request.site.domain}}{% url 'ingress:endpoint_script' object.uuid %}"></script>
{% endfilter %}
</div>
<hr class="sep h-4">

View File

@@ -1,3 +1,4 @@
from datetime import timedelta
from urllib.parse import urlparse
import flag
@@ -43,7 +44,12 @@ def country_name(isocode):
@register.simple_tag
def relative_stat_tone(
start, end, good="UP", good_classes=None, bad_classes=None, neutral_classes=None,
start,
end,
good="UP",
good_classes=None,
bad_classes=None,
neutral_classes=None,
):
good_classes = good_classes or "~positive"
bad_classes = bad_classes or "~critical"
@@ -97,6 +103,12 @@ def compare(
bad_classes=None,
neutral_classes=None,
):
if isinstance(start, timedelta):
start = start.seconds
if isinstance(end, timedelta):
end = end.seconds
return {
"start": start,
"end": end,

View File

@@ -22,16 +22,24 @@ from .forms import ServiceForm
from .mixins import DateRangeMixin
class DashboardView(LoginRequiredMixin, DateRangeMixin, TemplateView):
class DashboardView(LoginRequiredMixin, DateRangeMixin, ListView):
model = Service
template_name = "dashboard/pages/dashboard.html"
paginate_by = 5
def get_queryset(self):
return Service.objects.filter(
Q(owner=self.request.user) | Q(collaborators__in=[self.request.user])
).distinct()
def get_context_data(self, **kwargs):
data = super().get_context_data(**kwargs)
data["services"] = Service.objects.filter(
Q(owner=self.request.user) | Q(collaborators__in=[self.request.user])
).distinct()
for service in data["services"]:
service.stats = service.get_core_stats(data["start_date"], data["end_date"])
for service in data["object_list"]:
service.stats = service.get_core_stats(
self.get_start_date(), self.get_end_date()
)
return data

View File

@@ -18,7 +18,7 @@ import urllib.parse as urlparse
from django.contrib.messages import constants as messages
# Increment on new releases
VERSION = "v0.7.2"
VERSION = "v0.8.1"
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -58,6 +58,7 @@ INSTALLED_APPS = [
"allauth",
"allauth.account",
"allauth.socialaccount",
"debug_toolbar",
]
MIDDLEWARE = [
@@ -70,6 +71,7 @@ MIDDLEWARE = [
"django.contrib.sites.middleware.CurrentSiteMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
"debug_toolbar.middleware.DebugToolbarMiddleware",
]
ROOT_URLCONF = "shynet.urls"
@@ -145,9 +147,15 @@ AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
},
{"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",},
{"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",},
{"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",},
{
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
},
{
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
},
{
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
},
]
# Logging
@@ -247,6 +255,10 @@ LOGIN_REDIRECT_URL = "/"
SITE_ID = 1
INTERNAL_IPS = [
"127.0.0.1",
]
# Celery
CELERY_TASK_ALWAYS_EAGER = os.getenv("CELERY_TASK_ALWAYS_EAGER", "True") == "True"
@@ -321,3 +333,9 @@ SHOW_SHYNET_VERSION = os.getenv("SHOW_SHYNET_VERSION", "True") == "True"
# Should Shynet show third-party icons in the dashboard?
SHOW_THIRD_PARTY_ICONS = os.getenv("SHOW_THIRD_PARTY_ICONS", "True") == "True"
# Should Shynet never collect any IP?
BLOCK_ALL_IPS = os.getenv("BLOCK_ALL_IPS", "False") == "True"
# Include date and service ID in salt?
AGGRESSIVE_HASH_SALTING = os.getenv("AGGRESSIVE_HASH_SALTING", "False") == "True"

View File

@@ -15,8 +15,10 @@ Including another URLconf
"""
from django.contrib import admin
from django.urls import include, path
import debug_toolbar
urlpatterns = [
path("__debug__/", include(debug_toolbar.urls)),
path("admin/", admin.site.urls),
path("accounts/", include("allauth.urls")),
path("ingress/", include(("analytics.ingress_urls", "ingress")), name="ingress"),