Bump version to v0.7.3

*Total -- 765.10kb -> 502.59kb (34.31%)

/images/service.png -- 366.95kb -> 239.64kb (34.69%)
/images/homepage.png -- 398.15kb -> 262.95kb (33.96%)

Signed-off-by: ImgBotApp <ImgBotHelp@gmail.com>

Co-authored-by: ImgBotApp <ImgBotHelp@gmail.com>

.github/release-drafter.yml

@@ -0,0 +1,4 @@
+template: |
+  ## What’s Changed
+
+  $CHANGES

.github/workflows/draft.yml

@@ -0,0 +1,14 @@
+name: Release Drafter
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  update_release_draft:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: release-drafter/release-drafter@v5.11.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -3,6 +3,9 @@ __pycache__/
 *.py[cod]
 *$py.class
 
+# JavaScript packages
+node_modules/
+
 # C extensions
 *.so
 
@@ -109,6 +112,9 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+Vagrantfile
+.vagrant 
+ubuntu-xenial-16.04-cloudimg-console.log
 
 # Spyder project settings
 .spyderproject
@@ -131,3 +137,5 @@ dmypy.json
 # Secrets & env
 secrets.yml
 .vscode
+.DS_Store
+compiledstatic/

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at shynet@sendmiles.email. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

Dockerfile

@@ -1,33 +1,37 @@
-FROM python:3
+FROM python:3-alpine
 
+# Getting things ready
 WORKDIR /usr/src/shynet
+COPY Pipfile.lock Pipfile ./
+COPY package.json package-lock.json ../
+# Django expects node_modules to be in its parent directory.
 
-RUN apt update
+# Install dependencies & configure machine
-RUN apt install -y gettext
-
-# URL from https://github.com/shlinkio/shlink/issues/596 :)
-RUN curl "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN&license_key=G4Lm0C60yJsnkdPi&suffix=tar.gz" | tar -xvz -C /tmp
-RUN curl "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=G4Lm0C60yJsnkdPi&suffix=tar.gz" | tar -xvz -C /tmp
-RUN mv /tmp/GeoLite2*/*.mmdb /etc
-
-RUN pip install pipenv
-COPY Pipfile.lock ./
-COPY Pipfile ./
-RUN pipenv install --system --deploy
-
-COPY shynet .
-RUN python manage.py collectstatic --noinput
-RUN python manage.py compilemessages
-
 ARG GF_UID="500"
 ARG GF_GID="500"
+RUN apk update && \
+	apk add gettext curl bash npm && \
+	curl -m 180 "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN&license_key=kKG1ebhL3iWVd0iv&suffix=tar.gz" | tar -xvz -C /tmp && \
+	curl -m 180 "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=kKG1ebhL3iWVd0iv&suffix=tar.gz" | tar -xvz -C /tmp && \
+	mv /tmp/GeoLite2*/*.mmdb /etc && \
+	apk del curl && \
+	apk add --no-cache postgresql-libs && \
+	apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev && \
+	npm i -P --prefix .. && \
+	pip install pipenv~=2020.6.2 && \
+	pipenv install --system --deploy && \
+	apk --purge del .build-deps && \
+	rm -rf /var/lib/apt/lists/* && \
+	rm /var/cache/apk/* && \
+	addgroup --system -g $GF_GID appgroup && \
+	adduser appuser --system --uid $GF_UID -G appgroup
 
-# add group & user
+# Install Shynet
-RUN groupadd -r -g $GF_GID appgroup && \
+COPY shynet .
-   useradd appuser -r -u $GF_UID -g appgroup
+RUN python manage.py collectstatic --noinput && \
+	python manage.py compilemessages
 
+# Launch
 USER appuser
-
 EXPOSE 8080
-
+CMD [ "./entrypoint.sh" ]
-CMD [ "./webserver.sh" ]

GUIDE.md

@@ -0,0 +1,275 @@
+# Usage Guide
+
+## Table of Contents
+
+- [Installation](#installation)
+- [Heroku](#heroku)
+- [Render](#render)
+- [Updating Your Configuration](#updating-your-configuration)
+- [Advanced Usage](#advanced-usage)
+  * [Installation with SSL](#installation-with-ssl)
+  * [Configuring a Reverse Proxy](#configuring-a-reverse-proxy)
+    + [Cloudflare](#cloudflare)
+    + [Nginx](#nginx)
+  * [Health Checks](#health-checks)
+  * [Primary Key Integration](#primary-key-integration)
+  * [Usage with Single-Page Applications](#usage-with-single-page-applications)
++ [Troubleshooting](#troubleshooting)
+---
+
+## Staying Updated
+
+**If you install Shynet, you should strongly consider enabling notifications when new versions are released.** You can do this under the "Watch" tab on GitHub (above). This will ensure that you are notified when new versions are available, some of which may be security updates. (Shynet will never automatically update itself.)
+
+## Installation
+
+Installation of Shynet is easy! Follow the [Basic Installation](#basic-installation) guide or the [Basic Installation with Docker Compose](#basic-installation-with-docker-compose) below for a minimal installation, or if you are going to be running Shynet over HTTPS through a reverse proxy. If you'd like to run Shynet over HTTPS without a reverse proxy, skip ahead to [Installation with SSL](#installation-with-ssl) instead.
+
+> **These commands assume Ubuntu.** If you're installing Shynet on a different platform, the process will be different.
+
+Before continuing, please be sure to have the latest version of Docker installed.
+
+### Basic Installation
+
+1. Pull the latest version of Shynet using `docker pull milesmcc/shynet:latest`. If you don't have Docker installed, [install it](https://docs.docker.com/get-docker/).
+
+2. Have a PostgreSQL server ready to go. This can be on the same machine as the deployment, or elsewhere. You'll just need a username, password, host, and port. (For info on how to setup a PostgreSQL server on Ubuntu, follow [this guide](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-18-04)).
+
+3. Configure an environment file for Shynet, using [this file](/TEMPLATE.env) as a template. (This file is typically named `.env`.) Make sure you set the database settings, or Shynet won't be able to run.
+
+4. Launch the Shynet server for the first time by running `docker run --env-file=<your env file> milesmcc/shynet:latest`. Provided you're using the default environment information (i.e., `PERFORM_CHECKS_AND_SETUP` is `True`), you'll see a few warnings about not having an admin user or host setup; these are normal. Don't worry — we'll do this in the next step. You only need to stop if you see a stacktrace about being unable to connect to the database.
+
+5. Create an admin user by running `docker run --env-file=<your env file> milesmcc/shynet:latest ./manage.py registeradmin <your email>`. A temporary password will be printed to the console.
+
+6. Set the hostname of your Shynet instance by running `docker run --env-file=<your env file> milesmcc/shynet:latest ./manage.py hostname <your public hostname>`, where `<your public hostname>` is the _publicly accessible hostname_ of your instance, including port. This setting affects the URL that the tracking script sends its results to, so make sure it's correct. (Example hostnames: `shynet.example.com` or `example.com:8000`.)
+
+7. Set the whitelabel of your Shynet instance by running `docker run --env-file=<your env file> milesmcc/shynet:latest ./manage.py whitelabel <whitelabel>`. While this setting doesn't affect any core operations of Shynet, it lets you rename Shynet to whatever you want. (Example whitelabels: `"My Shynet Instance"` or `"Acme Analytics"`.)
+
+8. Launch your webserver by running `docker run --env-file=<your env file> milesmcc/shynet:latest`. You may need to bind Docker's port 8080 (where Shynet runs) to your local port 80 (http); this can be done using the flag `-p 80:8080` after `run`. Visit your service's homepage, and verify everything looks right! You should see a login prompt. Log in with the credentials from step 5. You'll probably be prompted to "confirm your email"—if you haven't set up an email server, the confirmation email will be printed to the console instead.
+
+9. Create a service by clicking "+ Create Service" in the top right hand corner. Fill out the options as appropriate. Once you're done, press "create" and you'll be redirected to your new service's analytics page.
+
+10. Finally, click on "Manage" in the top right of the service's page to get the tracking script code. Inject this script on all pages you'd like the service to track.
+
+
+### Basic Installation with Docker Compose
+
+> Make sure you have `docker-compose` installed. If not, [install it](https://docs.docker.com/compose/install/) 
+
+1. Clone the repository.
+
+2. Using [TEMPLATE.env](/TEMPLATE.env) as a template, confiure the environment for your Shynet instance and place the modified config in a file called `.env` in the root of the repository. Do _not_ change the port number at the end; you can set the public facing port in the next step. 
+
+3. On line 2 of the `nginx.conf` file located in the root of the repository, replace `example.com` with your hostname. Then, in the `docker-compose.yml` file, set the port number by replacing `8080` in line 38 ( `- 8080:80` ) with whatever local port you want to bind it to. For example, set the port number to `- 80:80` if you want your site will be available via HTTP (port 80) at `http://<your hostname>`.
+
+4. Launch the Shynet server for the first time by running `docker-compose up -d`. If you get an error like "permission denied" or "Couldn't connect to Docker daemon", either prefix the command with `sudo` or add your user to the `docker` group.
+
+5. Create an admin user by running `docker exec -it shynet_main ./manage.py registeradmin <your email>`. A temporary password will be printed to the console.
+
+6. Set the hostname of your Shynet instance by running `docker exec -it shynet_main ./manage.py hostname <your public hostname>`, where `<your public hostname>` is the same as the hostname you set in step 3. This setting affects the URL that the tracking script sends its results to, so make sure it's correct. (Example hostnames: shynet.example.com or example.com:8000.)
+
+7. Set the whitelabel of your Shynet instance by running `docker exec -it shynet_main ./manage.py whitelabel <whitelabel>`. While this setting doesn't affect any core operations of Shynet, it lets you rename Shynet to whatever you want. (Example whitelabels: "My Shynet Instance" or "Acme Analytics".)
+
+Your site should now be accessible at `http://hostname:port`. Now you can follow steps 9-10 of the [Basic Installation](#basic-installation) guide above to get Shynet integrated on your sites.
+
+## Heroku
+
+You may wish to deploy Shynet on Heroku. Note that Heroku's free offerings (namely the free Postgres addon) are unlikely to support running any Shynet instance that records more than a few hundred requests per day &mdash; the database will quickly fill up. In most cases, the more cost-effective option for running Shynet is renting a VPS from a full cloud service provider. However, if you're sure Heroku is the right option for you, or you just want to try Shynet out, you can use the Quick Deploy button then follow the steps below. 
+
+[![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy?template=https://github.com/milesmcc/shynet/tree/master)
+
+Once you deploy, you'll need to setup an admin user, whitelabel, and hostname before you can use Shynet. Do that with the following commands:
+
+1. `heroku run --app=<your app> ./manage.py registeradmin <your email>`
+2. `heroku run --app=<your app> ./manage.py hostname <the hostname where you will run Shynet>`
+3. `heroku run --app=<your app> ./manage.py whitelabel "<your Shynet instance's name>"`
+
+## Render
+
+[Render](https://render.com) is a modern cloud platform to build and run all your apps and websites with free SSL, a global CDN, private networks and auto deploys from Git. To deploy Shynet, click the `Deploy to Render` button and follow the steps below.
+
+[![Deploy to Render](https://render.com/images/deploy-to-render-button.svg)](https://render.com/deploy?repo=https://github.com/render-examples/shynet)
+
+Once your deploy has completed, use the **Render Shell** to configure your app:
+
+1. Set your email: `./manage.py registeradmin your-email@example.com`
+1. Add your onrender.com domain: `./manage.py hostname your-shynet-domain.onrender.com`
+1. Set your whitelabel: `./manage.py whitelabel "Your Shynet Instance Name"`
+
+See the [Render docs](https://render.com/docs/deploy-shynet) for more information on deploying your application on Render.
+
+---
+
+## Advanced Usage
+
+### Installation with SSL
+
+If you are going to be running Shynet through a reverse proxy, please see [Configuring a Reverse Proxy](#configuring-a-reverse-proxy) instead.
+
+0. We'll be cloning this into the home directory to make this installation easier, so run `cd ~/` if you need to.
+
+1. Instead of pulling from Docker, we will be pulling from GitHub and building using Docker in order to easily add SSL certificates. You will want to run `git clone https://github.com/milesmcc/shynet.git` to clone the GitHub repo to your current working directory.
+
+2. To install `certbot` follow [the guide here](https://certbot.eff.org/instructions) or follow along below
+   * Ubuntu 18.04
+     * `sudo apt-get update`
+     * `sudo apt-get install software-properties-common`
+     * `sudo add-apt-repository universe`
+     * `sudo add-apt-repository ppa:certbot/certbot`
+     * `sudo apt-get update`
+     * `sudo apt-get install certbot`
+
+3. Run `sudo certbot certonly --standalone` and follow the instructions to generate your SSL certificate.
+   * If you registering the certificate to a domain name like `example.com`, please be sure to point your DNS records to your current server before running `certbot`.
+
+4. We are going to move the SSL certificates to the Shynet repo with with command below. Replace `<domain>` with the domain name you used in step 3.
+   * `cp /etc/letsencrypt/live/<domain>/{cert,privkey}.pem ~/shynet/shynet/`
+
+5. With that, we are going to replace the `webserver.sh` with `ssl.webserver.sh` to enable the use of SSL certificates. The original `webserver.sh` will be backed up to `backup.webserver.sh`
+   * `mv ~/shynet/shynet/webserver.sh ~/shynet/shynet/backup.webserver.sh`
+   * `mv ~/shynet/shynet/ssl.webserver.sh ~/shynet/shynet/webserver.sh`
+
+6. Now we build the image!
+   * `docker image build shynet -t shynet-ssl:latest`
+
+7. Have a PostgreSQL server ready to go. This can be on the same machine as the deployment, or elsewhere. You'll just need a username, password, host, and port (default is `5432`). (For info on how to setup a PostgreSQL server on Ubuntu, follow [this guide](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-18-04)).
+
+8. Follow the [Basic Installation](#basic-installation) guide with just one modification: in step #4, change the local bind port from `80` to `443`, and use `shynet-ssl:latest` as your Docker image instead of `milesmcc/shynet:latest`.
+
+### Configuring a Reverse Proxy
+
+A reverse proxy has many benefits. It can be used for DDoS protection, caching files to reduce server load, routing HTTPS and/or HTTP connections, hosting multiple services on a single server, [and more](https://www.cloudflare.com/learning/cdn/glossary/reverse-proxy/)!
+
+#### Cloudflare
+
+[Cloudflare](https://www.cloudflare.com/) is a great reverse proxy option. It's free, automatically configures HTTPs, offers out-of-the-box security features, provides DNS, and requires minimal setup.
+
+1. Follow Cloudflare's [getting started guide](https://support.cloudflare.com/hc/en-us/articles/201720164-Creating-a-Cloudflare-account-and-adding-a-website).
+
+2. After setting up Cloudflare, here are a few things you should consider doing:
+   * Under the `SSL` Tab > `Overview` > Change your `SSL/TLS Encryption Mode` to `Flexible`
+   * The following will block your admin panel from anyone who isn't on your IP address. This is optional, but great for security.
+     * Under the `Firewall` tab > `Overview` > `+ Create Firewall Rule`:
+     * Name: `Admin Panel Restriction`
+     * Field: `URI Path`
+     * Operator: `equals`
+     * Value: `/admin`
+     * Click `AND`
+     * Field: `IP Address`
+     * Operator: `does not equal`
+     * Value: `<your public IP address>`
+     * Then: `Block`
+
+#### Nginx
+
+Nginx is a self hosted, highly configurable webserver. Nginx can be configured to run as a reverse proxy on either the same machine or a remote machine.
+
+> **These commands assume Ubuntu.** If you're installing Nginx on a different platform, the process will be different.
+
+0. Before starting, shut down your Docker containers (if any are running)
+   * Run `docker container ls` to find the container ID
+   * Run `docker stop <container id from the last step>`
+
+1. Update your packages and install Nginx
+   * `sudo apt-get update`
+   * `sudo apt-get install nginx`
+
+2. Disable the default Nginx placeholder
+   * `sudo unlink /etc/nginx/sites-enabled/default`
+
+3. Create the Nginx reverse proxy config file
+   * `cd /etc/nginx/sites-available/`
+   * `vi reverse-proxy.conf` or `nano reverse-proxy.conf`
+   * Paste the following configuration into that file:
+
+   ```nginx
+   # Know what you're pasting! Read the Reference!
+   # Reference: https://nginx.org/en/docs/
+   server {
+       listen 80;
+       location / {
+           proxy_set_header X-Real-IP $remote_addr;
+           proxy_pass http://127.0.0.1:8080;
+       }
+   }
+   ```
+
+   * Save and exit the text editor
+     * `:wq` for vi
+     * `ctrl+x` then `y` for nano
+   * Link Nginx's `sites-enabled` to read the new config
+     * `sudo ln -s /etc/nginx/sites-available/reverse-proxy.conf /etc/nginx/sites-enabled/reverse-proxy.conf`
+   * Make sure the config is working
+     * `service nginx configtest`
+     * `service nginx restart`
+
+4. Restart your Docker image, but this time use `8080` as the local bind port, as that's where we configured Nginx to look
+   * `cd ~/`
+   * `docker run -p 8080:8080 --env-file=<your env file> milesmcc/shynet:latest`
+
+5. Finally, time to test!
+   * Go to `http://<your site>/admin`
+
+6. If everything is working as expected, please read through some of the following links below to customize Nginx
+   * [How to add SSL/HTTPS to Nginx (Ubuntu 18.04)](https://www.digitalocean.com/community/tutorials/how-to-secure-nginx-with-let-s-encrypt-on-ubuntu-18-04)
+   * [How to add SSL/HTTPS to Nginx (Ubuntu 16.04)](https://www.digitalocean.com/community/tutorials/how-to-secure-nginx-with-let-s-encrypt-on-ubuntu-16-04)
+   * [Nginx Documentation](https://nginx.org/en/docs/)
+
+### Health Checks
+
+By default, Shynet includes a default health check endpoint at `/healthz/`. If the instance is running normally, this endpoint will return an HTTP status code of 200; if something is wrong, it will have a non-200 status code. To view the health data as JSON, send your request to `/healthz/?format=json`.
+
+This feature is helpful when running Shynet with Kubernetes, as it allows you to setup [startup readiness probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) that prevent traffic from being sent to your Shynet instances before they are ready.
+
+### Primary-Key Integration
+
+In some cases, it is useful to associate particular users on your platform with their sessions in Shynet. In Shynet, this is called _primary key integration_, and is done by adding an additional element to the Shynet script url for each particular user.
+
+If the Shynet script location (for either the pixel or the script) is, for example, `//shynet.example.com/ingress/your_service_uuid/pixel.gif` and `//shynet.example.com/ingress/your_service_uuid/script.js`, the URLs for primary-key enabled users would be `//shynet.example.com/ingress/your_service_uuid/USER_PRIMARY_KEY/pixel.gif` and `//shynet.example.com/ingress/your_service_uuid/USER_PRIMARY_KEY/script.js`.
+
+Adding this path can be done easily using server-side rendering. For example, here is a Django template that adds users' primary keys to the Shynet tracking script:
+
+```html
+{% if request.user.is_authenticated %}
+<noscript>
+   <img src="//shynet.example.com/ingress/service-uuid/{{request.user.email|urlencode:""}}/pixel.gif">
+</noscript>
+<script src="//shynet.example.com/ingress/service-uuid/{{request.user.email|urlencode:""}}/script.js"></script>
+{% else %}
+<noscript>
+   <img src="//shynet.example.com/ingress/service-uuid/pixel.gif">
+</noscript>
+<script src="//shynet.example.com/ingress/service-uuid/script.js"></script>
+{% endif %}
+```
+
+### Usage with Single-Page Applications
+
+In a single-page application, the page never reloads. (That's the entire point of single-page applications, after all!) Unfortunately, this also means that Shynet will not automatically recognize and track when the user navigates between pages _within_ your application.
+
+Fortunately, Shynet offers a simple method you can call from anywhere within your JavaScript to indicate that a new page has been loaded: `Shynet.newPageLoad()`. Add this method call to the code that handles routing in your app, and you'll be ready to go.
+
+---
+
+## Troubleshooting
+
+Here are solutions for some common issues. If your situation isn't described here or the solution didn't work, feel free to [create an issue](https://github.com/milesmcc/shynet/issues/new) (but be sure to check for duplicate issues first).
+
+#### The admin panel works, but no page views are showing up!
+
+* If you are running a single Shynet webserver instance (i.e., you followed the default installation instructions), verify that you haven't set `CELERY_TASK_ALWAYS_EAGER` to `False` in your environment file.
+* Verify that your cache is properly configured. In single-instance deployments, this means making sure that you haven't set any `REDIS_*` or `CELERY_*` environment variables (those are for more advanced deployments; you'll just want the defaults).
+* If your service is configured to respect Do Not Track (under "Advanced Settings"), verify that your browser isn't sending the `DNT=1` header with your requests (or temporarily disable DNT support in Shynet while testing). Sometimes, an adblocker or privacy browser extension will add this header to requests unexpectedly.
+
+#### Shynet isn't linking different pageviews from the same visitor into a single session!
+
+* Verify that your cache is properly configured. (See #2 above.) In multi-instance deployments, it's critical that all webservers are using the _same_ cache—so make sure you configure a Redis cache if you're using a non-default installation.
+* This can happen between Shynet restarts if you're not using an external cache provider (like Redis).
+
+#### I changed the `SHYNET_WHITELABEL`/`SHYNET_HOST` environment variable, but nothing happened!
+
+* Those values only affect how your Shynet instance is setup on first run; once it's configured, they have no effect. See [updating your configuration](#updating-your-configuration) for help on how to update your configuration. (Note: these environment variables are not present in newer Shynet versions; they have been removed from the guide.)
+
+#### Shynet can't connect to my database running on `localhost`/`127.0.0.1`
+
+* The problem is likely that to Shynet, `localhost` points to the local network in the container itself, not on the host machine. Try adding the `--network='host'` option when you run Docker.

Pipfile

@@ -3,26 +3,23 @@ name = "pypi"
 url = "https://pypi.org/simple"
 verify_ssl = true
 
-[dev-packages]
-black = "*"
-
 [packages]
-django = "*"
+django = "~=3.0"
-django-allauth = "*"
+django-allauth = "~=0.42.0"
-geoip2 = "*"
+geoip2 = "~=3.0.0"
-whitenoise = "*"
+whitenoise = "~=5.1.0"
-celery = "*"
+celery = "~=4.4.6"
-django-ipware = "*"
+django-ipware = "~=2.1.0"
-pyyaml = "*"
+pyyaml = "~=5.3.1"
-ua-parser = "*"
+ua-parser = "~=0.10.0"
-user-agents = "*"
+user-agents = "~=2.1"
-emoji-country-flag = "*"
+emoji-country-flag = "~=1.2.1"
-rules = "*"
+rules = "~=2.2"
-gunicorn = "*"
+gunicorn = "~=20.0.4"
-psycopg2-binary = "*"
+psycopg2-binary = "~=2.8.5"
-redis = "*"
+redis = "~=3.5.3"
-django-redis-cache = "*"
+django-redis-cache = "~=3.0.0"
-pycountry = "*"
+pycountry = "~=19.8.18"
-
+html2text = "~=2020.1.16"
-[pipenv]
+django-health-check = "~=3.12.1"
-allow_prereleases = true
+django-npm = "~=1.0.0"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "2cd3e33ea333a40476d2030b6be66826e93c3a4de67032655061725835c92f09"
+            "sha256": "73fbc4a0251ccae805550aa48b70ec9f668bee6fa4ff9503e023b3854a06bce8"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -16,17 +16,17 @@
     "default": {
         "amqp": {
             "hashes": [
-                "sha256:6e649ca13a7df3faacdc8bbb280aa9a6602d22fd9d545336077e573a1f4ff3b8",
+                "sha256:70cdb10628468ff14e57ec2f751c7aa9e48e7e3651cfd62d431213c0c4e58f21",
-                "sha256:77f1aef9410698d20eaeac5b73a87817365f457a507d82edf292e12cbb83b08d"
+                "sha256:aa7f313fb887c91f15474c1229907a04dac0b8135822d6603437803424c0aa59"
             ],
-            "version": "==2.5.2"
+            "version": "==2.6.1"
         },
         "asgiref": {
             "hashes": [
-                "sha256:8036f90603c54e93521e5777b2b9a39ba1bad05773fcf2d208f0299d1df58ce5",
+                "sha256:5ee950735509d04eb673bd7f7120f8fa1c9e2df495394992c73234d526907e17",
-                "sha256:9ca8b952a0a9afa61d30aa6d3d9b570bb3fd6bafcf7ec9e6bed43b936133db1c"
+                "sha256:7162a3cb30ab0609f1a4c95938fd73e8604f63bdba516a7f7d64b83ff09478f0"
             ],
-            "version": "==3.2.7"
+            "version": "==3.3.1"
         },
         "billiard": {
             "hashes": [
@@ -37,18 +37,18 @@
         },
         "celery": {
             "hashes": [
-                "sha256:108a0bf9018a871620936c33a3ee9f6336a89f8ef0a0f567a9001f4aa361415f",
+                "sha256:a92e1d56e650781fb747032a3997d16236d037c8199eacd5217d1a72893bca45",
-                "sha256:5b4b37e276033fe47575107a2775469f0b721646a08c96ec2c61531e4fe45f2a"
+                "sha256:d220b13a8ed57c78149acf82c006785356071844afe0b27012a4991d44026f9f"
             ],
             "index": "pypi",
-            "version": "==4.4.2"
+            "version": "==4.4.7"
         },
         "certifi": {
             "hashes": [
-                "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304",
+                "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd",
-                "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"
+                "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4"
             ],
-            "version": "==2020.4.5.1"
+            "version": "==2020.11.8"
         },
         "chardet": {
             "hashes": [
@@ -66,18 +66,26 @@
         },
         "django": {
             "hashes": [
-                "sha256:642d8eceab321ca743ae71e0f985ff8fdca59f07aab3a9fb362c617d23e33a76",
+                "sha256:14a4b7cd77297fba516fc0d92444cc2e2e388aa9de32d7a68d4a83d58f5a4927",
-                "sha256:d4666c2edefa38c5ede0ec1655424c56dc47ceb04b6d8d62a7eac09db89545c1"
+                "sha256:14b87775ffedab2ef6299b73343d1b4b41e5d4e2aa58c6581f114dbec01e3f8f"
             ],
             "index": "pypi",
-            "version": "==3.0.5"
+            "version": "==3.1.3"
         },
         "django-allauth": {
             "hashes": [
-                "sha256:7ab91485b80d231da191d5c7999ba93170ef1bf14ab6487d886598a1ad03e1d8"
+                "sha256:f17209410b7f87da0a84639fd79d3771b596a6d3fc1a8e48ce50dabc7f441d30"
             ],
             "index": "pypi",
-            "version": "==0.41.0"
+            "version": "==0.42.0"
+        },
+        "django-health-check": {
+            "hashes": [
+                "sha256:2667b89b8f85ad9b2a24c90581b376016d22ea912fedf37f9866413a3c2e0a5d",
+                "sha256:894738bd7e461b2405c005927403ad5ee8048bbaf5934cf30b2c81a4e047d4b0"
+            ],
+            "index": "pypi",
+            "version": "==3.12.3"
         },
         "django-ipware": {
             "hashes": [
@@ -86,21 +94,27 @@
             "index": "pypi",
             "version": "==2.1.0"
         },
-        "django-redis-cache": {
+        "django-npm": {
             "hashes": [
-                "sha256:06d4e48545243883f88dc9263dda6c8a0012cb7d0cee2d8758d8917eca92cece",
+                "sha256:2e6bba65e728fa18b9db3c8dc0d4490b70cb7f43bacf60eb3654d7dcb6424272"
-                "sha256:b19ee6654cc2f2c89078c99255e07e19dc2dba8792351d76ba7ea899d465fbb0"
             ],
             "index": "pypi",
-            "version": "==2.1.1"
+            "version": "==1.0.0"
+        },
+        "django-redis-cache": {
+            "hashes": [
+                "sha256:9a2eebef421d996a82098a19d17ff6b321265cd73178fa398913019764e8394a"
+            ],
+            "index": "pypi",
+            "version": "==3.0.0"
         },
         "emoji-country-flag": {
             "hashes": [
-                "sha256:67c0cb6a3765fb53f31b34160d6b1c8a5f44b297bc278d1835c6f2e5b0a9a592",
+                "sha256:256e47d30fb43bf154f370cc3c9e767f003aeb8653e31ba7b87151669c608e19",
-                "sha256:ae7edb38077b0840210fa9e37673f481f2b9c032446e13ad6dab2b1108cd7ad6"
+                "sha256:3f6c32699c19489383497865b208260b1d55b8424d66e08049187b13db2f0b8a"
             ],
             "index": "pypi",
-            "version": "==1.2.1"
+            "version": "==1.2.3"
         },
         "geoip2": {
             "hashes": [
@@ -118,25 +132,33 @@
             "index": "pypi",
             "version": "==20.0.4"
         },
+        "html2text": {
+            "hashes": [
+                "sha256:c7c629882da0cf377d66f073329ccf34a12ed2adf0169b9285ae4e63ef54c82b",
+                "sha256:e296318e16b059ddb97f7a8a1d6a5c1d7af4544049a01e261731d2d5cc277bbb"
+            ],
+            "index": "pypi",
+            "version": "==2020.1.16"
+        },
         "idna": {
             "hashes": [
-                "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb",
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
-                "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
-            "version": "==2.9"
+            "version": "==2.10"
         },
         "kombu": {
             "hashes": [
-                "sha256:2d1cda774126a044d91a7ff5fa6d09edf99f46924ab332a810760fe6740e9b76",
+                "sha256:be48cdffb54a2194d93ad6533d73f69408486483d189fe9f5990ee24255b0e0a",
-                "sha256:598e7e749d6ab54f646b74b2d2df67755dee13894f73ab02a2a9feb8870c7cb2"
+                "sha256:ca1b45faac8c0b18493d02a8571792f3c40291cf2bcf1f55afed3d8f3aa7ba74"
             ],
-            "version": "==4.6.8"
+            "version": "==4.6.11"
         },
         "maxminddb": {
             "hashes": [
-                "sha256:d0ce131d901eb11669996b49a59f410efd3da2c6dbe2c0094fe2fef8d85b6336"
+                "sha256:47e86a084dd814fac88c99ea34ba3278a74bc9de5a25f4b815b608798747c7dc"
             ],
-            "version": "==1.5.2"
+            "version": "==2.0.3"
         },
         "oauthlib": {
             "hashes": [
@@ -147,39 +169,44 @@
         },
         "psycopg2-binary": {
             "hashes": [
-                "sha256:008da3ab51adc70a5f1cfbbe5db3a22607ab030eb44bcecf517ad11a0c2b3cac",
+                "sha256:0deac2af1a587ae12836aa07970f5cb91964f05a7c6cdb69d8425ff4c15d4e2c",
-                "sha256:07cf82c870ec2d2ce94d18e70c13323c89f2f2a2628cbf1feee700630be2519a",
+                "sha256:0e4dc3d5996760104746e6cfcdb519d9d2cd27c738296525d5867ea695774e67",
-                "sha256:08507efbe532029adee21b8d4c999170a83760d38249936038bd0602327029b5",
+                "sha256:11b9c0ebce097180129e422379b824ae21c8f2a6596b159c7659e2e5a00e1aa0",
-                "sha256:107d9be3b614e52a192719c6bf32e8813030020ea1d1215daa86ded9a24d8b04",
+                "sha256:15978a1fbd225583dd8cdaf37e67ccc278b5abecb4caf6b2d6b8e2b948e953f6",
-                "sha256:17a0ea0b0eabf07035e5e0d520dabc7950aeb15a17c6d36128ba99b2721b25b1",
+                "sha256:1fabed9ea2acc4efe4671b92c669a213db744d2af8a9fc5d69a8e9bc14b7a9db",
-                "sha256:3286541b9d85a340ee4ed42732d15fc1bb441dc500c97243a768154ab8505bb5",
+                "sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94",
-                "sha256:3939cf75fc89c5e9ed836e228c4a63604dff95ad19aed2bbf71d5d04c15ed5ce",
+                "sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52",
-                "sha256:40abc319f7f26c042a11658bf3dd3b0b3bceccf883ec1c565d5c909a90204434",
+                "sha256:6422f2ff0919fd720195f64ffd8f924c1395d30f9a495f31e2392c2efafb5056",
-                "sha256:51f7823f1b087d2020d8e8c9e6687473d3d239ba9afc162d9b2ab6e80b53f9f9",
+                "sha256:6a32f3a4cb2f6e1a0b15215f448e8ce2da192fd4ff35084d80d5e39da683e79b",
-                "sha256:6bb2dd006a46a4a4ce95201f836194eb6a1e863f69ee5bab506673e0ca767057",
+                "sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd",
-                "sha256:702f09d8f77dc4794651f650828791af82f7c2efd8c91ae79e3d9fe4bb7d4c98",
+                "sha256:7d92a09b788cbb1aec325af5fcba9fed7203897bbd9269d5691bb1e3bce29550",
-                "sha256:7036ccf715925251fac969f4da9ad37e4b7e211b1e920860148a10c0de963522",
+                "sha256:833709a5c66ca52f1d21d41865a637223b368c0ee76ea54ca5bad6f2526c7679",
-                "sha256:7b832d76cc65c092abd9505cc670c4e3421fd136fb6ea5b94efbe4c146572505",
+                "sha256:89705f45ce07b2dfa806ee84439ec67c5d9a0ef20154e0e475e2b2ed392a5b83",
-                "sha256:8f74e631b67482d504d7e9cf364071fc5d54c28e79a093ff402d5f8f81e23bfa",
+                "sha256:8cd0fb36c7412996859cb4606a35969dd01f4ea34d9812a141cd920c3b18be77",
-                "sha256:930315ac53dc65cbf52ab6b6d27422611f5fb461d763c531db229c7e1af6c0b3",
+                "sha256:950bc22bb56ee6ff142a2cb9ee980b571dd0912b0334aa3fe0fe3788d860bea2",
-                "sha256:96d3038f5bd061401996614f65d27a4ecb62d843eb4f48e212e6d129171a721f",
+                "sha256:a0c50db33c32594305b0ef9abc0cb7db13de7621d2cadf8392a1d9b3c437ef77",
-                "sha256:a20299ee0ea2f9cca494396ac472d6e636745652a64a418b39522c120fd0a0a4",
+                "sha256:a0eb43a07386c3f1f1ebb4dc7aafb13f67188eab896e7397aa1ee95a9c884eb2",
-                "sha256:a34826d6465c2e2bbe9d0605f944f19d2480589f89863ed5f091943be27c9de4",
+                "sha256:aaa4213c862f0ef00022751161df35804127b78adf4a2755b9f991a507e425fd",
-                "sha256:a69970ee896e21db4c57e398646af9edc71c003bc52a3cc77fb150240fefd266",
+                "sha256:ac0c682111fbf404525dfc0f18a8b5f11be52657d4f96e9fcb75daf4f3984859",
-                "sha256:b9a8b391c2b0321e0cd7ec6b4cfcc3dd6349347bd1207d48bcb752aa6c553a66",
+                "sha256:ad20d2eb875aaa1ea6d0f2916949f5c08a19c74d05b16ce6ebf6d24f2c9f75d1",
-                "sha256:ba13346ff6d3eb2dca0b6fa0d8a9d999eff3dcd9b55f3a890f12b0b6362b2b38",
+                "sha256:b4afc542c0ac0db720cf516dd20c0846f71c248d2b3d21013aa0d4ef9c71ca25",
-                "sha256:bb0608694a91db1e230b4a314e8ed00ad07ed0c518f9a69b83af2717e31291a3",
+                "sha256:b8a3715b3c4e604bcc94c90a825cd7f5635417453b253499664f784fc4da0152",
-                "sha256:c8830b7d5f16fd79d39b21e3d94f247219036b29b30c8270314c46bf8b732389",
+                "sha256:ba28584e6bca48c59eecbf7efb1576ca214b47f05194646b081717fa628dfddf",
-                "sha256:cac918cd7c4c498a60f5d2a61d4f0a6091c2c9490d81bc805c963444032d0dab",
+                "sha256:ba381aec3a5dc29634f20692349d73f2d21f17653bda1decf0b52b11d694541f",
-                "sha256:cc30cb900f42c8a246e2cb76539d9726f407330bc244ca7729c41a44e8d807fb",
+                "sha256:bd1be66dde2b82f80afb9459fc618216753f67109b859a361cf7def5c7968729",
-                "sha256:ccdc6a87f32b491129ada4b87a43b1895cf2c20fdb7f98ad979647506ffc41b6",
+                "sha256:c2507d796fca339c8fb03216364cca68d87e037c1f774977c8fc377627d01c71",
-                "sha256:d1a8b01f6a964fec702d6b6dac1f91f2b9f9fe41b310cbb16c7ef1fac82df06d",
+                "sha256:cec7e622ebc545dbb4564e483dd20e4e404da17ae07e06f3e780b2dacd5cee66",
-                "sha256:e004db88e5a75e5fdab1620fb9f90c9598c2a195a594225ac4ed2a6f1c23e162",
+                "sha256:d14b140a4439d816e3b1229a4a525df917d6ea22a0771a2a78332273fd9528a4",
-                "sha256:eb2f43ae3037f1ef5e19339c41cf56947021ac892f668765cd65f8ab9814192e",
+                "sha256:d1b4ab59e02d9008efe10ceabd0b31e79519da6fb67f7d8e8977118832d0f449",
-                "sha256:fa466306fcf6b39b8a61d003123d442b23707d635a5cb05ac4e1b62cc79105cd"
+                "sha256:d5227b229005a696cc67676e24c214740efd90b148de5733419ac9aaba3773da",
+                "sha256:e1f57aa70d3f7cc6947fd88636a481638263ba04a742b4a37dd25c373e41491a",
+                "sha256:e74a55f6bad0e7d3968399deb50f61f4db1926acf4a6d83beaaa7df986f48b1c",
+                "sha256:e82aba2188b9ba309fd8e271702bd0d0fc9148ae3150532bbb474f4590039ffb",
+                "sha256:ee69dad2c7155756ad114c02db06002f4cded41132cc51378e57aad79cc8e4f4",
+                "sha256:f5ab93a2cb2d8338b1674be43b442a7f544a0971da062a5da774ed40587f18f5"
             ],
             "index": "pypi",
-            "version": "==2.8.5"
+            "version": "==2.8.6"
         },
         "pycountry": {
             "hashes": [
@@ -190,28 +217,30 @@
         },
         "python3-openid": {
             "hashes": [
-                "sha256:0086da6b6ef3161cfe50fb1ee5cceaf2cda1700019fda03c2c5c440ca6abe4fa",
+                "sha256:33fbf6928f401e0b790151ed2b5290b02545e8775f982485205a066f874aaeaf",
-                "sha256:628d365d687e12da12d02c6691170f4451db28d6d68d050007e4a40065868502"
+                "sha256:6626f771e0417486701e0b4daff762e7212e820ca5b29fcc0d05f6f8736dfa6b"
             ],
-            "version": "==3.1.0"
+            "version": "==3.2.0"
         },
         "pytz": {
             "hashes": [
-                "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
+                "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268",
-                "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
+                "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"
             ],
-            "version": "==2019.3"
+            "version": "==2020.4"
         },
         "pyyaml": {
             "hashes": [
                 "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97",
                 "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76",
                 "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2",
+                "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e",
                 "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648",
                 "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf",
                 "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f",
                 "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2",
                 "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee",
+                "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a",
                 "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d",
                 "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c",
                 "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"
@@ -221,18 +250,18 @@
         },
         "redis": {
             "hashes": [
-                "sha256:0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f",
+                "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2",
-                "sha256:b205cffd05ebfd0a468db74f0eedbff8df1a7bfc47521516ade4692991bb0833"
+                "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24"
             ],
             "index": "pypi",
-            "version": "==3.4.1"
+            "version": "==3.5.3"
         },
         "requests": {
             "hashes": [
-                "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee",
+                "sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8",
-                "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"
+                "sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998"
             ],
-            "version": "==2.23.0"
+            "version": "==2.25.0"
         },
         "requests-oauthlib": {
             "hashes": [
@@ -248,19 +277,12 @@
             "index": "pypi",
             "version": "==2.2"
         },
-        "six": {
-            "hashes": [
-                "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",
-                "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"
-            ],
-            "version": "==1.14.0"
-        },
         "sqlparse": {
             "hashes": [
-                "sha256:022fb9c87b524d1f7862b3037e541f68597a730a8843245c349fc93e1643dc4e",
+                "sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0",
-                "sha256:e162203737712307dfe78860cc56c8da8a852ab2ee33750e33aeadf38d12c548"
+                "sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8"
             ],
-            "version": "==0.3.1"
+            "version": "==0.4.1"
         },
         "ua-parser": {
             "hashes": [
@@ -272,18 +294,18 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527",
+                "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
-                "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"
+                "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
             ],
-            "version": "==1.25.9"
+            "version": "==1.26.2"
         },
         "user-agents": {
             "hashes": [
-                "sha256:da54371d856c35d8ead0622da24ad5ef6d667eda3629a750e3373a3e847a054b",
+                "sha256:a98c4dc72ecbc64812c4534108806fb0a0b3a11ec3fd1eafe807cee5b0a942e7",
-                "sha256:e727ab6f169e829bc25d41dbd25b9ff679b4631bd81959bcf7de1e246da67194"
+                "sha256:d36d25178db65308d1458c5fa4ab39c9b2619377010130329f3955e7626ead26"
             ],
             "index": "pypi",
-            "version": "==2.1"
+            "version": "==2.2.0"
         },
         "vine": {
             "hashes": [
@@ -294,108 +316,12 @@
         },
         "whitenoise": {
             "hashes": [
-                "sha256:0f9137f74bd95fa54329ace88d8dc695fbe895369a632e35f7a136e003e41d73",
+                "sha256:60154b976a13901414a25b0273a841145f77eb34a141f9ae032a0ace3e4d5b27",
-                "sha256:62556265ec1011bd87113fb81b7516f52688887b7a010ee899ff1fd18fd22700"
+                "sha256:6dd26bfda3af29177d8ab7333a0c7b7642eb615ce83764f4d15a9aecda3201c4"
             ],
             "index": "pypi",
-            "version": "==5.0.1"
+            "version": "==5.1.0"
         }
     },
-    "develop": {
+    "develop": {}
-        "appdirs": {
-            "hashes": [
-                "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92",
-                "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e"
-            ],
-            "version": "==1.4.3"
-        },
-        "attrs": {
-            "hashes": [
-                "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
-                "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
-            ],
-            "version": "==19.3.0"
-        },
-        "black": {
-            "hashes": [
-                "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b",
-                "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"
-            ],
-            "index": "pypi",
-            "version": "==19.10b0"
-        },
-        "click": {
-            "hashes": [
-                "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc",
-                "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a"
-            ],
-            "version": "==7.1.1"
-        },
-        "pathspec": {
-            "hashes": [
-                "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0",
-                "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061"
-            ],
-            "version": "==0.8.0"
-        },
-        "regex": {
-            "hashes": [
-                "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b",
-                "sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8",
-                "sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3",
-                "sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e",
-                "sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683",
-                "sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1",
-                "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142",
-                "sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3",
-                "sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468",
-                "sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e",
-                "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3",
-                "sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a",
-                "sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f",
-                "sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6",
-                "sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156",
-                "sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b",
-                "sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db",
-                "sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd",
-                "sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a",
-                "sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948",
-                "sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89"
-            ],
-            "version": "==2020.4.4"
-        },
-        "toml": {
-            "hashes": [
-                "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c",
-                "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"
-            ],
-            "version": "==0.10.0"
-        },
-        "typed-ast": {
-            "hashes": [
-                "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355",
-                "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919",
-                "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa",
-                "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652",
-                "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75",
-                "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01",
-                "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d",
-                "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1",
-                "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907",
-                "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c",
-                "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3",
-                "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b",
-                "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614",
-                "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb",
-                "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b",
-                "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41",
-                "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6",
-                "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34",
-                "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe",
-                "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4",
-                "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"
-            ],
-            "version": "==1.4.1"
-        }
-    }
 }

README.md

@@ -1,15 +1,18 @@
+
 <p align="center">
-  <h3 align="center">🔭 Shynet 🔭</h3>
+  <img align="center" src="images/logo.png" height="50" alt="Shynet logo">
+  <br>
 
   <p align="center">
-     Web analytics that's self hosted, cookie free, privacy friendly, and useful(?) 
+    Modern, privacy-friendly, and cookie-free web analytics.
-    <br>
-    <br>
-    <a href="#installation"><strong>Getting started »</strong></a>
     <br>
+    <strong><a href="#installation">Getting started »</a></strong>
   </p>
+  <p align="center"><a href="#screenshots">Screenshots</a> &bull; <a href="#features">Features</a> &bull; <a href="https://github.com/milesmcc/a17t">Design</a></p>
 </p>
 
+<br>
+
 ## Motivation
 
 There are a _lot_ of web analytics tools. Unfortunately, most of them come with the following caveats:
@@ -31,16 +34,21 @@ _Note: These screenshots have been edited to hide sensitive data. The "real" Shy
 ![Shynet's homepage](images/homepage.png)
 _Shynet's homepage, where you can see all of your services at a glance._
 
-Not shown: service view, management view, session view, full service view. (You'll need to install Shynet to see those!)
+![A service page](images/service.png)
+_A real service page, where you can see higher-level details about a site._
+
+Not shown: management view, session view, full service view. (You'll need to install Shynet for yourself to see those!)
+
+> **Shynet is built using [a17t](https://github.com/milesmcc/a17t),** an atomic design library. Customization and extension is simple; [learn more about a17t](https://github.com/milesmcc/a17t).
 
 ## Features
 
 #### Architecture
 
-* **Runs on a single machine** &mdash; Because it's so small, Shynet can easily run as a single docker container on a single small VPS.
+* **Runs on a single machine** &mdash; Because it's so small, Shynet can easily run as a single docker container on a single small VPS
-* **...or across a giant Kubernetes cluster** &mdash; For higher traffic installations, Shynet can be deployed with as many parallelized ingress nodes as needed, with Redis caching and separate backend workers for database IO.
+* **...or across a giant Kubernetes cluster** &mdash; For higher traffic installations, Shynet can be deployed with as many parallelized ingress nodes as needed, with Redis caching and separate backend workers for database IO
-* **Built using Django** &mdash; Shynet is built using Django, so deploying, updating, and migrating can be done without headaches.
+* **Built using Django** &mdash; Shynet is built using Django, so deploying, updating, and migrating can be done without headaches
-* **Multiple users and sites** &mdash; A single Shynet instance can support multiple users, each tracking multiple different sites.
+* **Multiple users and sites** &mdash; A single Shynet instance can support multiple users, each tracking multiple different sites
 
 #### Tracking
 
@@ -67,11 +75,11 @@ Here's the information Shynet can give you about your visitors:
 
 #### Workflow
 * **Collaboration built-in** &mdash; Administrators can easily share services with other users, as well
-* **Accounts (or not)** &mdash; Shynet has a fully featured account management workflow (powered by [Django Allauth](https://github.com/pennersr/django-allauth/)).
+* **Accounts (or not)** &mdash; Shynet has a fully featured account management workflow (powered by [Django Allauth](https://github.com/pennersr/django-allauth/))
 
 ## Recommendations
 
-Shynet isn't for everyone. It's great for personal projects and small to medium size websites, but hasn't been tested with ultra-high traffic sites. It's also requires a fair amount of technical know-how to deploy and maintain, so if you need a one-click solution, you're best served with other tools. 
+Shynet isn't for everyone. It's great for personal projects and small to medium size websites, but hasn't been tested with ultra-high traffic sites. It also requires a fair amount of technical know-how to deploy and maintain, so if you need a one-click solution, you're best served with other tools. 
 
 ## Concepts
 
@@ -85,88 +93,21 @@ Shynet is pretty simple, but there are a few key terms you need to know in order
 
 ## Installation
 
-To install Shynet using the simplest possible setup, follow these instructions. Instructions for multi-machine deployments will be available soon.
+You can find intructions on getting started and usage in the [Usage Guide](GUIDE.md#installation). Out of the box, we support deploying via a simple Docker container, docker-compose, Heroku, or Kubernetes (see [kubernetes](/kubernetes)).
-
-> **These commands assume Ubuntu.** If you're installing Shynet on a different platform, the process will be different.
-
-1. Pull the latest version of Shynet using `docker pull milesmcc/shynet:latest`. If you don't have Docker installed, [install it](https://docs.docker.com/get-docker/).
-
-2. Have a PostgreSQL server ready to go. This can be on the same machine as the deployment, or elsewhere. You'll just need a username, password, and host. (For info on how to setup a PostgreSQL server on Ubuntu, follow [this guide](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-18-04)).
-
-3. Configure an environment file for Shynet. (For example, create a file called `.env`.) Be sure to swap out the variables below with the correct values for your setup. (The comments refer to the lines that follow. Note that Docker is weird with quotes, so it tends to be better to omit them from your env file.)
-
-```
-# Database
-DB_NAME=<your db name>
-DB_USER=<your db user>
-DB_PASSWORD=<your db user password>
-DB_HOST=<your db host>
-
-# General Django settings
-DJANGO_SECRET_KEY=<your Django secret key; just a random string>
-# Don't leak error details to visitors, very important
-DEBUG=False
-CELERY_TASK_ALWAYS_EAGER=True 
-# For better security, set this to your deployment's domain. Comma separated.
-ALLOWED_HOSTS=*
-# Set to True (capitalized) if you want people to be able to sign up for your Shynet instance (not recommended)
-SIGNUPS_ENABLED=False
-# Change as required
-TIME_ZONE=America/New_York
-# Set to "False" if you will not be serving content over HTTPS
-SCRIPT_USE_HTTPS=True
-```
-
-For more advanced deployments, you may consider adding the following settings to your environment file. **The following settings are optional, and not required for simple deployments.**
-
-```env
-# Email settings
-EMAIL_HOST_USER=<your SMTP email user>
-EMAIL_HOST_PASSWORD=<your SMTP email password>
-EMAIL_HOST=<your SMTP email hostname>
-SERVER_EMAIL=Shynet <noreply@shynet.example.com>
-
-# Redis and queue settings; not necessary for single-instance deployments
-REDIS_CACHE_LOCATION=redis://redis.default.svc.cluster.local/0 
-# If set, make sure CELERY_TASK_ALWAYS_EAGER is False
-CELERY_BROKER_URL=redis://redis.default.svc.cluster.local/1
-```
-
-4. Setup the Shynet database by running `docker run --env-file=<your env file> milesmcc/shynet:latest python manage.py migrate`.
-
-5. Create your admin account by running `docker run --env-file=<your env file> milesmcc/shynet:latest python manage.py registeradmin <your email>`. The command will print a temporary password that you'll be able to use to log in.
-
-6. Configure Shynet's hostname (e.g. `shynet.example.com` or `localhost:8000`) by running `docker run --env-file=<your env file> milesmcc/shynet:latest python manage.py hostname "<your hostname>"`. This doesn't affect Shynet's bind port; instead, it determines what hostname to inject into the tracking script. (So you'll want to use the "user-facing" hostname here.)
-
-7. Name your Shynet instance by running `docker run --env-file=<your env file> milesmcc/shynet:latest python manage.py whitelabel "<your instance name>"`. This could be something like "My Shynet Server" or "Acme Analytics"—whatever suits you.
-
-8. Launch the Shynet server by running `docker run --env-file=<your env file> milesmcc/shynet:latest`. You may need to bind Docker's port 8000 (where Shynet runs) to your local port 8000; this can be done using the flag `-p 8080:8080`.
-
-9. Visit your service's homepage, and verify everything looks right! You should see a login prompt. Log in with the credentials from step 5. You'll probably be prompted to "confirm your email"—if you haven't set up an email server, the confirmation email will be printed to the console instead.
-
-10. Create a service by clicking "+ Create Service" in the top right hand corner. Fill out the options as appropriate. Once you're done, press "create" and you'll be redirected to your new service's analytics page.
-
-11. Finally, click on "Manage" in the top right of the service's page to get the tracking script code. Inject this script on all pages you'd like the service to track.
-
-**Next steps:** while out of the scope of this short guide, next steps include setting up Shynet behind a reverse proxy (be it your own [Nginx server](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/) or [Cloudflare](https://cloudflare.com)), making it run in the background, and integrating it on your sites. Integration instructions are available on each service's management page.
-
 
 ## FAQ
 
 **Does Shynet respond to Do Not Track (DNT) signals?** Yes. While there isn't any standardized way to handle DNT requests, Shynet allows you to specify whether you want to collect any data from users with DNT enabled on a per-service basis. (By default, Shynet will _not_ collect any data from users who specify DNT.)
 
-**Is this GDPR compliant?** I think so, but it also depends on how you use it. If you're worried about GDPR, you should talk to a lawyer about your particular data collection practices. I'm not a lawyer. (And this isn't legal advice.)
+**Is this GDPR compliant?** It depends on how you use it. If you're worried about GDPR, you should talk to a lawyer about your particular data collection practices. I'm not a lawyer. (And this isn't legal advice.)
+
+## Troubleshooting
+
+Having trouble with Shynet? Check out the [troubleshooting guide](GUIDE.md#troubleshooting), or [create an issue](https://github.com/milesmcc/shynet/issues/new) if you think you found a bug in Shynet itself (or have a feature suggestion).
 
 ## Roadmap
 
-The following features are planned:
+To see the upcoming planned features, check out the repository's [roadmap project](https://github.com/milesmcc/shynet/projects/1). Upcoming features include data aggregation through rollups, anomaly detection, detailed data exports, two-factor authentication, and a data deletion tool.
-
-* **Rollups** (aggregate old data to save space)
-* **Anomaly detection** (get email alerts when you get a traffic spike or dip)
-* **Interactive traffic heatmap** (see where in the world your visitors are coming from)
-* **Better collaboration interface** (the current interface is... a draft)
-* **Data deletion tool** (easily prune user data by specifying an ID or IP)
-* **Differential privacy** (explore and share your data without revealing any personal information)
 
 ## In the Wild
 
@@ -174,7 +115,7 @@ These sites use Shynet to monitor usage without violating visitors' privacy: [Po
 
 ## Contributing
 
-Are you interested in contributing to Shynet? Just send a pull request! Maybe once the project matures there will be more detailed contribution guidelines, but for now just send the code this way. Just know that by contributing, you agree to share all of your contributions under the same license as the project (see [LICENSE](LICENSE)).
+Are you interested in contributing to Shynet? Just send a pull request! Maybe once the project matures there will be more detailed contribution guidelines, but for now just send the code this way and we'll make sure it meets our standards together. Just know that by contributing, you agree to share all of your contributions under the same license as the project (see [LICENSE](LICENSE)). And always be sure to follow the [Code of Conduct](https://github.com/milesmcc/shynet/blob/master/CODE_OF_CONDUCT.md).
 
 ## License
 
@@ -182,4 +123,4 @@ Shynet is made available under the [Apache License, version 2.0](LICENSE).
 
 ---
 
-a17t was created by [Miles McCain](https://miles.land) at the [Recurse Center](https://recurse.com) using [a17t](https://a17t.miles.land).
+Shynet was created by [Miles McCain](https://miles.land) ([@MilesMcCain](https://twitter.com/MilesMcCain)) at the [Recurse Center](https://recurse.com) using [a17t](https://a17t.miles.land).

TEMPLATE.env

@@ -0,0 +1,75 @@
+# This file shows all of the environment variables you can
+# set to configure Shynet, as well as information about their
+# effects. Make a copy of this file to configure your deployment.
+
+# Database settings (PostgreSQL)
+DB_NAME=shynet_db
+DB_USER=shynet_db_user
+DB_PASSWORD=shynet_db_user_password
+DB_HOST=db
+DB_PORT=5432
+
+# Email settings (optional)
+EMAIL_HOST_USER=example
+EMAIL_HOST_PASSWORD=example_password
+EMAIL_HOST=smtp.example.com
+EMAIL_PORT=465
+EMAIL_USE_SSL=True
+# Comment out EMAIL_USE_SSL & uncomment EMAIL_USE_TLS if your SMTP server uses TLS.
+# EMAIL_USE_TLS=True
+SERVER_EMAIL=Shynet <noreply@shynet.example.com>
+
+# General Django settings
+DJANGO_SECRET_KEY=random_string
+
+# For better security, set this to your deployment's domain. Comma separated.
+ALLOWED_HOSTS=*
+
+# Set to True (capitalized) if you want people to be able to sign up for your Shynet instance (not recommended)
+ACCOUNT_SIGNUPS_ENABLED=False
+
+# Should user email addresses be verified? Only set this to `required` if you've setup the email settings and allow
+# public sign-ups; otherwise, it's unnecessary.
+ACCOUNT_EMAIL_VERIFICATION=none
+
+# The timezone of the admin panel. Affects how dates are displayed.
+TIME_ZONE=America/New_York
+
+# Set to "False" if you will not be serving content over HTTPS
+SCRIPT_USE_HTTPS=True
+
+# How frequently should the monitoring script "phone home" (in ms)?
+SCRIPT_HEARTBEAT_FREQUENCY=5000
+
+# How much time can elapse between requests from the same user before a new
+# session is created, in seconds?
+SESSION_MEMORY_TIMEOUT=1800
+
+# Should only superusers (admins) be able to create services? This is helpful
+# when you'd like to invite others to your Shynet instance but don't want
+# them to be able to create services of their own.
+ONLY_SUPERUSERS_CREATE=True
+
+# Whether to perform checks and setup at startup, including applying unapplied
+# migrations. For most setups, the recommended value is True. Defaults to True.
+# Will skip only if value is False.
+PERFORM_CHECKS_AND_SETUP=True
+
+# The port that Shynet should bind to. Don't set this if you're deploying on Heroku.
+PORT=8080
+
+# Set to "False" if you do not want the version to be displayed on the frontend.
+SHOW_SHYNET_VERSION=True
+
+# Redis, queue, and parellization settings; not necessary for single-instance deployments.
+# Don't uncomment these unless you know what you are doing!
+# NUM_WORKERS=1
+# Make sure you set a REDIS_CACHE_LOCATION if you have more than one frontend worker/instance.
+# REDIS_CACHE_LOCATION=redis://redis.default.svc.cluster.local/0
+# If CELERY_BROKER_URL is set, make sure CELERY_TASK_ALWAYS_EAGER is False and
+# that you have a separate queue consumer running somewhere via `celeryworker.sh`.
+# CELERY_TASK_ALWAYS_EAGER=False
+# CELERY_BROKER_URL=redis://redis.default.svc.cluster.local/1
+
+# Should Shynet show third-party icons in the dashboard?
+SHOW_THIRD_PARTY_ICONS=True

app.json

@@ -0,0 +1,127 @@
+{
+  "name": "Shynet",
+  "description": "Modern, privacy-friendly, and detailed web analytics that works without cookies or JS.",
+  "keywords": [
+    "app.json",
+    "shynet",
+    "heroku",
+    "analytics",
+    "privacy",
+    "friendly"
+  ],
+  "website": "https://github.com/milesmcc/shynet",
+  "repository": "https://github.com/milesmcc/shynet",
+  "logo": "https://github.com/milesmcc/shynet/raw/master/images/slogo.png",
+  "success_url": "/",
+  "stack": "container",
+  "addons": [
+    "heroku-postgresql:hobby-dev"
+  ],
+  "formation": {
+    "web": {
+      "quantity": 1,
+      "size": "free"
+    }
+  },
+  "env": {
+    "DB_NAME": {
+      "description": "Postgres database name (not required if using Postgres addon)",
+      "value": "shynet",
+      "required": false
+    },
+    "DB_USER": {
+      "description": "Postgres database username (not required if using Postgres addon)",
+      "value": "",
+      "required": false
+    },
+    "DB_PASSWORD": {
+      "description": "Postgres database password (not required if using Postgres addon)",
+      "value": "",
+      "required": false
+    },
+    "DB_HOST": {
+      "description": "Postgres database hostname (not required if using Postgres addon)",
+      "value": "",
+      "required": false
+    },
+    "DB_PORT": {
+      "description": "Postgres database port (not required if using Postgres addon)",
+      "value": "5432",
+      "required": false
+    },
+    "EMAIL_HOST": {
+      "description": "SMTP server hostname (for sending emails)",
+      "value": "smtp.gmail.com",
+      "required": false
+    },
+    "EMAIL_PORT": {
+      "description": "SMTP server port (for sending emails)",
+      "value": "465",
+      "required": false
+    },
+    "EMAIL_HOST_USER": {
+      "description": "SMTP server username (for sending emails)",
+      "value": "",
+      "required": false
+    },
+    "EMAIL_HOST_PASSWORD": {
+      "description": "SMTP server password (for sending emails)",
+      "value": "",
+      "required": false
+    },
+    "SERVER_EMAIL": {
+      "description": "Email address (for sending emails)",
+      "value": "<Shynet> noreply@shynet.example.com",
+      "required": false
+    },
+    "DJANGO_SECRET_KEY": {
+      "description": "Django secret key",
+      "generator": "secret"
+    },
+    "ALLOWED_HOSTS": {
+      "description": "For better security, set this to your deployment's domain. (Where you will actually host, not embed, Shynet.) Set to '*' to allow serving all domains.",
+      "value": "*",
+      "required": false
+    },
+    "ACCOUNT_SIGNUPS_ENABLED": {
+      "description": "Set to True (capitalized) if you want people to be able to sign up for your Shynet instance (not recommended).",
+      "value": "False",
+      "required": false
+    },
+    "TIME_ZONE": {
+      "description": "The timezone of the admin panel. Affects how dates are displayed.",
+      "value": "America/New_York",
+      "required": false
+    },
+    "SCRIPT_USE_HTTPS": {
+      "description": "Set to 'False' if you will not be serving Shynet over HTTPS.",
+      "value": "True",
+      "required": false
+    },
+    "SCRIPT_HEARTBEAT_FREQUENCY": {
+      "description": "How frequently should the monitoring script 'phone home' (in ms)?",
+      "value": "5000",
+      "required": false
+    },
+    "SESSION_MEMORY_TIMEOUT": {
+      "description": "How much time can elapse between requests from the same user before a new session is created, in seconds?",
+      "value": "1800",
+      "required": false
+    },
+    "ONLY_SUPERUSERS_CREATE": {
+      "description": "Should only superusers (admins) be able to create tracked services?",
+      "value": "True",
+      "required": false
+    },
+    "PERFORM_CHECKS_AND_SETUP": {
+      "description": "Whether to perform checks and setup at startup. Recommended value is 'True' for Heroku users.",
+      "value": "True",
+      "required": false
+    },
+    "SHOW_SHYNET_VERSION": {
+      "description": "Set to 'False' if you do not want the version to be displayed on the frontend.",
+      "value": "True",
+      "required": false
+    }
+  }
+}

docker-compose.yml

@@ -0,0 +1,46 @@
+version: '3'
+services:
+  shynet:
+    container_name: shynet_main
+    image: milesmcc/shynet:latest
+    restart: unless-stopped
+    expose:
+      - 8080
+    env_file:
+      # Create a file called '.env' if it doesn't already exist.
+      # You can use `TEMPLATE.env` as a guide.
+      - .env
+    environment:
+      - DB_HOST=db
+    networks:
+      - internal
+    depends_on:
+      - db
+  db:
+    container_name: shynet_database
+    image: postgres
+    restart: always
+    environment:
+      - "POSTGRES_USER=${DB_USER}"
+      - "POSTGRES_PASSWORD=${DB_PASSWORD}"
+      - "POSTGRES_DB=${DB_NAME}"
+    volumes:
+      - shynet_db:/var/lib/postgresql/data
+    networks:
+      - internal
+  webserver:
+    container_name: shynet_webserver
+    image: nginx
+    restart: always
+    volumes:
+      - ./nginx.conf:/etc/nginx/conf.d/default.conf    
+    ports:
+      - 8080:80
+    depends_on:
+      - shynet
+    networks:
+      - internal
+volumes:
+  shynet_db:
+networks:
+  internal:

heroku.yml

@@ -0,0 +1,3 @@
+build:
+  docker:
+    web: Dockerfile

kubernetes/deployments.yml

@@ -16,13 +16,12 @@ spec:
         app: "shynet-webserver"
     spec:
       containers:
-      - name: "covideo-webserver"
+      - name: "shynet-webserver"
         image: "milesmcc/shynet:latest"
-        command: ["./webserver.sh"]
         imagePullPolicy: Always
         envFrom:
         - secretRef:
-            name: django-settings
+            name: shynet-settings
 ---
 apiVersion: "apps/v1"
 kind: "Deployment"
@@ -42,43 +41,43 @@ spec:
         app: "shynet-celeryworker"
     spec:
       containers:
-      - name: "covideo-celeryworker"
+      - name: "shynet-celeryworker"
         image: "milesmcc/shynet:latest"
         command: ["./celeryworker.sh"]
         imagePullPolicy: Always
         envFrom:
         - secretRef:
-            name: django-settings
+            name: shynet-settings
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: redis
+  name: shynet-redis
 spec:
   ports:
     - port: 6379
       name: redis
   clusterIP: None
   selector:
-    app: redis
+    app: shynet-redis
 ---
 apiVersion: apps/v1beta2
 kind: StatefulSet
 metadata:
-  name: redis
+  name: shynet-redis
 spec:
   selector:
     matchLabels:
-      app: redis
+      app: shynet-redis
-  serviceName: redis
+  serviceName: shynet-redis
   replicas: 1
   template:
     metadata:
       labels:
-        app: redis
+        app: shynet-redis
     spec:
       containers:
-        - name: redis
+        - name: shynet-redis
           image: redis:latest
           imagePullPolicy: Always
           ports:

kubernetes/secrets_template.yml

@@ -1,19 +1,19 @@
 apiVersion: v1
 kind: Secret
 metadata:
-    name: django-settings
+    name: shynet-settings
 type: Opaque
 stringData:
     # Django settings
     DEBUG: "False"
     ALLOWED_HOSTS: "*" # For better security, set this to your deployment's domain. Comma separated.
     DJANGO_SECRET_KEY: ""
-    SIGNUPS_ENABLED: "False"
+    ACCOUNT_SIGNUPS_ENABLED: "False"
     TIME_ZONE: "America/New_York"
 
     # Redis configuration (if you use the default Kubernetes config, this will work)
-    REDIS_CACHE_LOCATION: "redis://redis.default.svc.cluster.local/0"
+    REDIS_CACHE_LOCATION: "redis://shynet-redis.default.svc.cluster.local/0"
-    CELERY_BROKER_URL: "redis://redis.default.svc.cluster.local/1"
+    CELERY_BROKER_URL: "redis://shynet-redis.default.svc.cluster.local/1"
 
     # PostgreSQL settings
     DB_NAME: ""

nginx.conf

@@ -0,0 +1,19 @@
+server {
+    server_name example.com;
+    access_log /var/log/nginx/bin.access.log;
+    error_log /var/log/nginx/bin.error.log error;
+
+
+    location / {
+        proxy_pass http://shynet:8080;
+        proxy_redirect off;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Protocol $scheme;
+        proxy_set_header X-Url-Scheme $scheme;
+    }
+    listen 80; 
+
+}

package-lock.json

@@ -0,0 +1,109 @@
+{
+  "name": "shynet",
+  "requires": true,
+  "lockfileVersion": 1,
+  "dependencies": {
+    "@fortawesome/fontawesome-free": {
+      "version": "5.15.1",
+      "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/fontawesome-free-5.15.1.tgz",
+      "integrity": "sha512-OEdH7SyC1suTdhBGW91/zBfR6qaIhThbcN8PUXtXilY4GYnSBbVqOntdHbC1vXwsDnX0Qix2m2+DSU1J51ybOQ=="
+    },
+    "a17t": {
+      "version": "0.5.1",
+      "resolved": "https://registry.npmjs.org/a17t/-/a17t-0.5.1.tgz",
+      "integrity": "sha512-peIPrH9eDiu49LLzLlSTFFrXj6WLlEX3TRsUkqyyOHi/i58ilJ/eERnu7AcswXhuCBx+/2W9EUuHM+8iAq4ipg=="
+    },
+    "apexcharts": {
+      "version": "3.23.1",
+      "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.23.1.tgz",
+      "integrity": "sha512-7fRpquXp725BUew5OO1mJWk16/IJPCUl0l8SjhISnAhAtbTaM9PnXPSmN2BvKO4RcT457CzMM7MCG5UokiTwcA==",
+      "requires": {
+        "svg.draggable.js": "^2.2.2",
+        "svg.easing.js": "^2.0.0",
+        "svg.filter.js": "^2.0.2",
+        "svg.pathmorphing.js": "^0.1.3",
+        "svg.resize.js": "^1.4.3",
+        "svg.select.js": "^3.0.1"
+      }
+    },
+    "inter-ui": {
+      "version": "3.15.0",
+      "resolved": "https://registry.npmjs.org/inter-ui/-/inter-ui-3.15.0.tgz",
+      "integrity": "sha512-6v0WK8FHkVYbNQZ7L9O5tP8280pgTBR9ydxqYwssMuUH6SZO70ZFK/NQ1Ob8nNmOOzpUJAzT0WE73ty96z1tAQ=="
+    },
+    "litepicker": {
+      "version": "1.5.7",
+      "resolved": "https://registry.npmjs.org/litepicker/-/litepicker-1.5.7.tgz",
+      "integrity": "sha512-4L2ZcF8iqCE4A/qGWS3PbdFplZR1g751x5SsZ87zCRZ4LQN1Fgezarnvqi0eHk/kDWK7Qx0HZ9Y4bNznJMF1xA=="
+    },
+    "svg.draggable.js": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/svg.draggable.js/-/svg.draggable.js-2.2.2.tgz",
+      "integrity": "sha512-JzNHBc2fLQMzYCZ90KZHN2ohXL0BQJGQimK1kGk6AvSeibuKcIdDX9Kr0dT9+UJ5O8nYA0RB839Lhvk4CY4MZw==",
+      "requires": {
+        "svg.js": "^2.0.1"
+      }
+    },
+    "svg.easing.js": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/svg.easing.js/-/svg.easing.js-2.0.0.tgz",
+      "integrity": "sha1-iqmUawqOJ4V6XEChDrpAkeVpHxI=",
+      "requires": {
+        "svg.js": ">=2.3.x"
+      }
+    },
+    "svg.filter.js": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/svg.filter.js/-/svg.filter.js-2.0.2.tgz",
+      "integrity": "sha1-kQCOFROJ3ZIwd5/L5uLJo2LRwgM=",
+      "requires": {
+        "svg.js": "^2.2.5"
+      }
+    },
+    "svg.js": {
+      "version": "2.7.1",
+      "resolved": "https://registry.npmjs.org/svg.js/-/svg.js-2.7.1.tgz",
+      "integrity": "sha512-ycbxpizEQktk3FYvn/8BH+6/EuWXg7ZpQREJvgacqn46gIddG24tNNe4Son6omdXCnSOaApnpZw6MPCBA1dODA=="
+    },
+    "svg.pathmorphing.js": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/svg.pathmorphing.js/-/svg.pathmorphing.js-0.1.3.tgz",
+      "integrity": "sha512-49HWI9X4XQR/JG1qXkSDV8xViuTLIWm/B/7YuQELV5KMOPtXjiwH4XPJvr/ghEDibmLQ9Oc22dpWpG0vUDDNww==",
+      "requires": {
+        "svg.js": "^2.4.0"
+      }
+    },
+    "svg.resize.js": {
+      "version": "1.4.3",
+      "resolved": "https://registry.npmjs.org/svg.resize.js/-/svg.resize.js-1.4.3.tgz",
+      "integrity": "sha512-9k5sXJuPKp+mVzXNvxz7U0uC9oVMQrrf7cFsETznzUDDm0x8+77dtZkWdMfRlmbkEEYvUn9btKuZ3n41oNA+uw==",
+      "requires": {
+        "svg.js": "^2.6.5",
+        "svg.select.js": "^2.1.2"
+      },
+      "dependencies": {
+        "svg.select.js": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/svg.select.js/-/svg.select.js-2.1.2.tgz",
+          "integrity": "sha512-tH6ABEyJsAOVAhwcCjF8mw4crjXSI1aa7j2VQR8ZuJ37H2MBUbyeqYr5nEO7sSN3cy9AR9DUwNg0t/962HlDbQ==",
+          "requires": {
+            "svg.js": "^2.2.5"
+          }
+        }
+      }
+    },
+    "svg.select.js": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/svg.select.js/-/svg.select.js-3.0.1.tgz",
+      "integrity": "sha512-h5IS/hKkuVCbKSieR9uQCj9w+zLHoPh+ce19bBYyqF53g6mnPB8sAtIbe1s9dh2S2fCmYX2xel1Ln3PJBbK4kw==",
+      "requires": {
+        "svg.js": "^2.6.5"
+      }
+    },
+    "turbolinks": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/turbolinks/-/turbolinks-5.2.0.tgz",
+      "integrity": "sha512-pMiez3tyBo6uRHFNNZoYMmrES/IaGgMhQQM+VFF36keryjb5ms0XkVpmKHkfW/4Vy96qiGW3K9bz0tF5sK9bBw=="
+    }
+  }
+}

package.json

@@ -0,0 +1,27 @@
+{
+  "name": "shynet",
+  "description": "Modern, privacy-friendly, and cookie-free web analytics.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/milesmcc/shynet.git"
+  },
+  "keywords": [
+    "privacy",
+    "analytics",
+    "self-host"
+  ],
+  "author": "R. Miles McCain <shynet@sendmiles.email>",
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://github.com/milesmcc/shynet/issues"
+  },
+  "homepage": "https://github.com/milesmcc/shynet#readme",
+  "dependencies": {
+    "@fortawesome/fontawesome-free": "^5.15.1",
+    "a17t": "^0.5.1",
+    "apexcharts": "^3.23.1",
+    "inter-ui": "^3.15.0",
+    "litepicker": "^1.5.7",
+    "turbolinks": "^5.2.0"
+  }
+}

shynet/a17t/templates/a17t/head.html

@@ -1,3 +0,0 @@
-<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/a17t@0.1.3/dist/a17t.css">
-<script async src="https://use.fontawesome.com/releases/v5.3.1/js/all.js"></script>
-<link href="https://cdn.jsdelivr.net/npm/tailwindcss@^1.0/dist/tailwind.min.css" rel="stylesheet">

shynet/a17t/templates/a17t/includes/head.html

@@ -0,0 +1,12 @@
+{% load static %}
+
+<link rel="stylesheet" href="{% static 'a17t/dist/a17t.css' %}">
+<script async src="{% static '@fortawesome/fontawesome-free/js/all.min.js' %}" data-mutate-approach="sync"></script>
+<link href="{% static 'a17t/dist/tailwind.css' %}" rel="stylesheet">
+<link href="{% static 'inter-ui/Inter (web)/inter.css' %}" rel="stylesheet">
+<style>
+    :root {
+        --family-primary: "Inter", system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
+        --family-secondary: var(--family-primary);
+    }
+</style>

shynet/a17t/templates/a17t/includes/pagination.html

@@ -1,23 +1,23 @@
 <nav class="flex w-full flex-wrap items-center justify-between" role="navigation" aria-label="pagination">
     <div class="w-full md:w-auto mb-2">
         {% if page.has_previous %}
-        <a href="?page={{ page.previous_page_number }}{{url_parameters}}" class="button field w-auto mr-1">Previous</a>
+        <a href="?page={{ page.previous_page_number }}{{url_parameters}}" class="button field bg-neutral-000 w-auto mr-1">Previous</a>
         {% else %}
-        <a class="button field w-auto mr-1" disabled>Previous</a>
+        <a class="button field bg-neutral-000 w-auto mr-1" disabled>Previous</a>
         {% endif %}
         {% if page.has_next %}
-        <a href="?page={{ page.next_page_number }}{{url_parameters}}" class="button field w-auto">Next</a>
+        <a href="?page={{ page.next_page_number }}{{url_parameters}}" class="button field bg-neutral-000 w-auto">Next</a>
         {% else %}
-        <a class="button field w-auto" disabled>Next</a>
+        <a class="button field bg-neutral-000 w-auto" disabled>Next</a>
         {% endif %}
     </div>
 
     <ul class="pagination-list w-full md:w-auto mb-2 flex">
         {% for pnum in begin %}
         {% ifequal page.number pnum %}
-        <li><a class="button field w-auto mx-1 text-white bg-gray-700">{{ pnum }}</a></li>
+        <li><a class="button field w-auto mx-1 text-white bg-neutral-700">{{ pnum }}</a></li>
         {% else %}
-        <li><a class="button field w-auto mx-1" href="?page={{ pnum }}{{url_parameters}}">{{ pnum }}</a></li>
+        <li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}{{url_parameters}}">{{ pnum }}</a></li>
         {% endifequal %}
         {% endfor %}
 
@@ -25,9 +25,9 @@
         <li><span class="pagination-ellipsis">&hellip;</span></li>
         {% for pnum in middle %}
         {% ifequal page.number pnum %}
-        <li><a class="button field w-auto mx-1 text-white bg-gray-700">{{ pnum }}</a></li>
+        <li><a class="button field w-auto mx-1 text-white bg-neutral-700">{{ pnum }}</a></li>
         {% else %}
-        <li><a class="button field w-auto mx-1" href="?page={{ pnum }}{{url_parameters}}">{{ pnum }}</a></li>
+        <li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}{{url_parameters}}">{{ pnum }}</a></li>
         {% endifequal %}
         {% endfor %}
         {% endif %}
@@ -36,9 +36,9 @@
         <li><span class="pagination-ellipsis">&hellip;</span></li>
         {% for pnum in end %}
         {% ifequal page.number pnum %}
-        <li><a class="button field w-auto mx-1 text-white bg-gray-700">{{ pnum }}</a></li>
+        <li><a class="button field w-auto mx-1 text-white bg-neutral-700">{{ pnum }}</a></li>
         {% else %}
-        <li><a class="button field w-auto mx-1" href="?page={{ pnum }}{{url_parameters}}">{{ pnum }}</a></li>
+        <li><a class="button field bg-neutral-000 w-auto mx-1" href="?page={{ pnum }}{{url_parameters}}">{{ pnum }}</a></li>
         {% endifequal %}
         {% endfor %}
         {% endif %}

shynet/a17t/templatetags/a17t_tags.py

@@ -92,4 +92,6 @@ def is_file(field):
 def add_class(field, css_class):
     if len(field.errors) > 0:
         css_class += " ~critical"
+    if field.field.widget.attrs.get("class") != None:
+        css_class += " " + field.field.widget.attrs["class"]
     return field.as_widget(attrs={"class": field.css_classes(extra_classes=css_class)})

shynet/analytics/migrations/0003_auto_20200502_1227.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0.5 on 2020-05-02 16:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analytics", "0002_auto_20200415_1742"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="session",
+            name="ip",
+            field=models.GenericIPAddressField(db_index=True, null=True),
+        ),
+    ]

shynet/analytics/models.py

@@ -20,8 +20,8 @@ class Session(models.Model):
     identifier = models.TextField(blank=True, db_index=True)
 
     # Time
-    start_time = models.DateTimeField(auto_now_add=True, db_index=True)
+    start_time = models.DateTimeField(default=timezone.now, db_index=True)
-    last_seen = models.DateTimeField(auto_now_add=True)
+    last_seen = models.DateTimeField(default=timezone.now)
 
     # Core request information
     user_agent = models.TextField()
@@ -39,7 +39,7 @@ class Session(models.Model):
         default="OTHER",
     )
     os = models.TextField()
-    ip = models.GenericIPAddressField(db_index=True)
+    ip = models.GenericIPAddressField(db_index=True, null=True)
 
     # GeoIP data
     asn = models.TextField(blank=True)
@@ -78,8 +78,8 @@ class Hit(models.Model):
     initial = models.BooleanField(default=True, db_index=True)
 
     # Base request information
-    start_time = models.DateTimeField(auto_now_add=True, db_index=True)
+    start_time = models.DateTimeField(default=timezone.now, db_index=True)
-    last_seen = models.DateTimeField(auto_now_add=True)
+    last_seen = models.DateTimeField(default=timezone.now)
     heartbeats = models.IntegerField(default=0)
     tracker = models.TextField(
         choices=[("JS", "JavaScript"), ("PIXEL", "Pixel (noscript)")]

shynet/analytics/tasks.py

@@ -1,5 +1,7 @@
+import ipaddress
 import json
 import logging
+from hashlib import sha256
 
 import geoip2.database
 import user_agents
@@ -7,7 +9,6 @@ from celery import shared_task
 from django.conf import settings
 from django.core.cache import cache
 from django.db.models import Q
-from django.utils import timezone
 
 from core.models import Service
 
@@ -42,7 +43,15 @@ def _geoip2_lookup(ip):
 
 @shared_task
 def ingress_request(
-    service_uuid, tracker, time, payload, ip, location, user_agent, dnt=False, identifier=""
+    service_uuid,
+    tracker,
+    time,
+    payload,
+    ip,
+    location,
+    user_agent,
+    dnt=False,
+    identifier="",
 ):
     try:
         service = Service.objects.get(pk=service_uuid, status=Service.ACTIVE)
@@ -51,34 +60,50 @@ def ingress_request(
         if dnt and service.respect_dnt:
             return
 
-        ip_data = _geoip2_lookup(ip)
+        try:
-        log.debug(f"Found geoip2 data")
+            remote_ip = ipaddress.ip_network(ip)
+            for ignored_network in service.get_ignored_networks():
+                if (
+                    ignored_network.version == remote_ip.version
+                    and ignored_network.supernet_of(remote_ip)
+                ):
+                    return
+        except ValueError as e:
+            log.exception(e)
 
         # Validate payload
         if payload.get("loadTime", 1) <= 0:
             payload["loadTime"] = None
 
-        # Create or update session
+        association_id_hash = sha256()
-        session = (
+        association_id_hash.update(str(ip).encode("utf-8"))
-            Session.objects.filter(
+        association_id_hash.update(str(user_agent).encode("utf-8"))
-                service=service,
+        session_cache_path = (
-                last_seen__gt=timezone.now() - timezone.timedelta(minutes=10),
+            f"session_association_{service.pk}_{association_id_hash.hexdigest()}"
-                ip=ip,
-                user_agent=user_agent,
-            ).first()
-            # We used to check for identifiers, but that can cause issues when people
-            # re-open the page in a new tab, for example. It's better to match sessions
-            # solely based on IP and user agent.
         )
+
+        # Create or update session
+        session = None
+        if cache.get(session_cache_path) is not None:
+            cache.touch(session_cache_path, settings.SESSION_MEMORY_TIMEOUT)
+            session = Session.objects.filter(
+                pk=cache.get(session_cache_path), service=service
+            ).first()
         if session is None:
-            log.debug("Cannot link to existing session; creating a new one...")
-            ua = user_agents.parse(user_agent)
             initial = True
+
+            log.debug("Cannot link to existing session; creating a new one...")
+
+            ip_data = _geoip2_lookup(ip)
+            log.debug(f"Found geoip2 data...")
+
+            ua = user_agents.parse(user_agent)
             device_type = "OTHER"
             if (
                 ua.is_bot
                 or (ua.browser.family or "").strip().lower() == "googlebot"
-                or (ua.device.family or ua.device.model or "").strip().lower() == "spider"
+                or (ua.device.family or ua.device.model or "").strip().lower()
+                == "spider"
             ):
                 device_type = "ROBOT"
             elif ua.is_mobile:
@@ -87,26 +112,35 @@ def ingress_request(
                 device_type = "TABLET"
             elif ua.is_pc:
                 device_type = "DESKTOP"
+            if device_type == "ROBOT" and service.ignore_robots:
+                return
             session = Session.objects.create(
                 service=service,
-                ip=ip,
+                ip=ip if service.collect_ips else None,
                 user_agent=user_agent,
                 identifier=identifier.strip(),
                 browser=ua.browser.family or "",
                 device=ua.device.family or ua.device.model or "",
                 device_type=device_type,
+                start_time=time,
+                last_seen=time,
                 os=ua.os.family or "",
-                asn=ip_data.get("asn", ""),
+                asn=ip_data.get("asn") or "",
-                country=ip_data.get("country", ""),
+                country=ip_data.get("country") or "",
                 longitude=ip_data.get("longitude"),
                 latitude=ip_data.get("latitude"),
-                time_zone=ip_data.get("time_zone", ""),
+                time_zone=ip_data.get("time_zone") or "",
+            )
+            cache.set(
+                session_cache_path, session.pk, timeout=settings.SESSION_MEMORY_TIMEOUT
             )
         else:
-            log.debug("Updating old session with new data...")
             initial = False
+
+            log.debug("Updating old session with new data...")
+
             # Update last seen time
-            session.last_seen = timezone.now()
+            session.last_seen = time
             if session.identifier == "" and identifier.strip() != "":
                 session.identifier = identifier.strip()
             session.save()
@@ -115,9 +149,10 @@ def ingress_request(
         idempotency = payload.get("idempotency")
         idempotency_path = f"hit_idempotency_{idempotency}"
         hit = None
+
         if idempotency is not None:
             if cache.get(idempotency_path) is not None:
-                cache.touch(idempotency_path, 10 * 60)
+                cache.touch(idempotency_path, settings.SESSION_MEMORY_TIMEOUT)
                 hit = Hit.objects.filter(
                     pk=cache.get(idempotency_path), session=session
                 ).first()
@@ -126,8 +161,9 @@ def ingress_request(
                     # this is a heartbeat.
                     log.debug("Hit is a heartbeat; updating old hit with new data...")
                     hit.heartbeats += 1
-                    hit.last_seen = timezone.now()
+                    hit.last_seen = time
                     hit.save()
+
         if hit is None:
             log.debug("Hit is a page load; creating new hit...")
             # There is no existing hit; create a new one
@@ -141,10 +177,14 @@ def ingress_request(
                 location=payload.get("location", location),
                 referrer=payload.get("referrer", ""),
                 load_time=payload.get("loadTime"),
+                start_time=time,
+                last_seen=time,
             )
             # Set idempotency (if applicable)
             if idempotency is not None:
-                cache.set(idempotency_path, hit.pk, timeout=10 * 60)
+                cache.set(
+                    idempotency_path, hit.pk, timeout=settings.SESSION_MEMORY_TIMEOUT
+                )
     except Exception as e:
         log.exception(e)
         raise e

shynet/analytics/templates/analytics/scripts/page.js

@@ -1,9 +1,18 @@
-window.onload = function () {
+// This is a lightweight and privacy-friendly analytics script from Shynet, a self-hosted
-  var idempotency =
+// analytics tool. To give you full visibility into how your data is being monitored, this
-    Math.random().toString(36).substring(2, 15) +
+// file is intentionally not minified or obfuscated. To learn more about Shynet (and to view
-    Math.random().toString(36).substring(2, 15);
+// its source code), visit <https://github.com/milesmcc/shynet>.
-  function sendUpdate() {
+//
+// This script only sends the current URL, the referrer URL, and the page load time. That's it!
+
+var Shynet = {
+  idempotency: null,
+  heartbeatTaskId: null,
+  sendHeartbeat: function () {
     try {
+      if (document.hidden) {
+        return;
+      }
       var xhr = new XMLHttpRequest();
       xhr.open(
         "POST",
@@ -13,7 +22,7 @@ window.onload = function () {
       xhr.setRequestHeader("Content-Type", "application/json");
       xhr.send(
         JSON.stringify({
-          idempotency: idempotency,
+          idempotency: Shynet.idempotency,
           referrer: document.referrer,
           location: window.location.href,
           loadTime:
@@ -21,8 +30,25 @@ window.onload = function () {
             window.performance.timing.navigationStart,
         })
       );
-    } catch {}
+    } catch (e) { }
+  },
+  newPageLoad: function () {
+    if (Shynet.heartbeatTaskId != null) {
+      clearInterval(Shynet.heartbeatTaskId);
+    }
+    Shynet.idempotency = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
+    Shynet.heartbeatTaskId = setInterval(Shynet.sendHeartbeat, parseInt("{{heartbeat_frequency}}"));
+    Shynet.sendHeartbeat();
   }
-  setInterval(sendUpdate, 5000);
-  sendUpdate();
 };
+
+window.addEventListener("load", Shynet.newPageLoad);
+
+{% if script_inject %}
+// The following is script is not part of Shynet, and was instead
+// provided by this site's administrator.
+// 
+// -- START --
+{{script_inject|safe}}
+// -- END --
+{% endif %}

shynet/analytics/views/ingress.py

@@ -1,8 +1,16 @@
 import base64
 import json
+from urllib.parse import urlparse
 
 from django.conf import settings
-from django.http import HttpResponse
+from django.core.cache import cache
+from django.core.exceptions import ValidationError
+from django.http import (
+    Http404,
+    HttpResponse,
+    HttpResponseBadRequest,
+    HttpResponseForbidden,
+)
 from django.shortcuts import render, reverse
 from django.utils import timezone
 from django.utils.decorators import method_decorator
@@ -10,6 +18,8 @@ from django.views.decorators.csrf import csrf_exempt
 from django.views.generic import TemplateView, View
 from ipware import get_client_ip
 
+from core.models import Service
+
 from ..tasks import ingress_request
 
 
@@ -33,13 +43,53 @@ def ingress(request, service_uuid, identifier, tracker, payload):
     )
 
 
-class PixelView(View):
+class ValidateServiceOriginsMixin:
+    def dispatch(self, request, *args, **kwargs):
+        try:
+            service_uuid = self.kwargs.get("service_uuid")
+            origins = cache.get(f"service_origins_{service_uuid}")
+
+            if origins is None:
+                service = Service.objects.get(uuid=service_uuid)
+                origins = service.origins
+                cache.set(f"service_origins_{service_uuid}", origins, timeout=3600)
+
+            resp = super().dispatch(request, *args, **kwargs)
+
+            if origins != "*":
+                remote_origin = request.META.get("HTTP_ORIGIN")
+                if (
+                    remote_origin is None
+                    and request.META.get("HTTP_REFERER") is not None
+                ):
+                    parsed = urlparse(request.META.get("HTTP_REFERER"))
+                    remote_origin = f"{parsed.scheme}://{parsed.netloc}".lower()
+                origins = [origin.strip().lower() for origin in origins.split(",")]
+                if remote_origin in origins:
+                    resp["Access-Control-Allow-Origin"] = remote_origin
+                else:
+                    return HttpResponseForbidden()
+            else:
+                resp["Access-Control-Allow-Origin"] = "*"
+
+            resp["Access-Control-Allow-Methods"] = "GET,HEAD,OPTIONS,POST"
+            resp[
+                "Access-Control-Allow-Headers"
+            ] = "Origin, X-Requested-With, Content-Type, Accept, Authorization, Referer"
+            return resp
+        except Service.DoesNotExist:
+            raise Http404()
+        except ValidationError:
+            return HttpResponseBadRequest()
+
+
+class PixelView(ValidateServiceOriginsMixin, View):
     # Fallback view to serve an unobtrusive 1x1 transparent tracking pixel for browsers with
     # JavaScript disabled.
-    def dispatch(self, request, *args, **kwargs):
+    def get(self, *args, **kwargs):
         # Extract primary data
         ingress(
-            request,
+            self.request,
             self.kwargs.get("service_uuid"),
             self.kwargs.get("identifier", ""),
             "PIXEL",
@@ -50,22 +100,13 @@ class PixelView(View):
             "R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="
         )
         resp = HttpResponse(data, content_type="image/gif")
-        resp["Cache-Control"] = "no-cache"
+        resp["Cache-Control"] = "no-cache, no-store, must-revalidate"
         resp["Access-Control-Allow-Origin"] = "*"
         return resp
 
 
 @method_decorator(csrf_exempt, name="dispatch")
-class ScriptView(View):
+class ScriptView(ValidateServiceOriginsMixin, View):
-    def dispatch(self, request, *args, **kwargs):
-        resp = super().dispatch(request, *args, **kwargs)
-        resp["Access-Control-Allow-Origin"] = "*"
-        resp["Access-Control-Allow-Methods"] = "GET,HEAD,OPTIONS,POST"
-        resp[
-            "Access-Control-Allow-Headers"
-        ] = "Origin, X-Requested-With, Content-Type, Accept, Authorization, Referer"
-        return resp
-
     def get(self, *args, **kwargs):
         protocol = "https" if settings.SCRIPT_USE_HTTPS else "http"
         endpoint = (
@@ -82,10 +123,18 @@ class ScriptView(View):
                 },
             )
         )
+        heartbeat_frequency = settings.SCRIPT_HEARTBEAT_FREQUENCY
         return render(
             self.request,
             "analytics/scripts/page.js",
-            context={"endpoint": endpoint, "protocol": protocol},
+            context=dict(
+                {
+                    "endpoint": endpoint,
+                    "protocol": protocol,
+                    "heartbeat_frequency": heartbeat_frequency,
+                    "script_inject": self.get_script_inject(),
+                }
+            ),
             content_type="application/javascript",
         )
 
@@ -101,3 +150,12 @@ class ScriptView(View):
         return HttpResponse(
             json.dumps({"status": "OK"}), content_type="application/json"
         )
+
+    def get_script_inject(self):
+        service_uuid = self.kwargs.get("service_uuid")
+        script_inject = cache.get(f"script_inject_{service_uuid}")
+        if script_inject == None:
+            service = Service.objects.get(uuid=service_uuid)
+            script_inject = service.script_inject
+            cache.set(f"script_inject_{service_uuid}", script_inject, timeout=3600)
+        return script_inject

shynet/core/management/commands/demo.py

@@ -0,0 +1,114 @@
+import traceback
+from django.utils.timezone import now
+from django.utils.timezone import timedelta
+import random
+import uuid
+
+from django.conf import settings
+from django.contrib.sites.models import Site
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.crypto import get_random_string
+import user_agents
+from logging import info
+
+from core.models import User, Service
+from analytics.models import Session, Hit
+from analytics.tasks import ingress_request
+
+LOCATIONS = [
+    "/",
+    "/post/{rand}",
+    "/login",
+    "/me",
+]
+
+REFERRERS = [
+    "https://news.ycombinator.com/item?id=11116274",
+    "https://news.ycombinator.com/item?id=24872911",
+    "https://reddit.com",
+    "https://facebook.com",
+    "https://twitter.com/milesmccain",
+    "https://twitter.com",
+    "https://stanford.edu/~mccain/",
+    "https://tiktok.com",
+    "https://io.stanford.edu",
+    "https://en.wikipedia.org",
+    "https://stackoverflow.com",
+    "",
+    "",
+    "",
+    "",
+]
+
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/43.4",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 11_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko)",
+    "Version/10.0 Mobile/14E304 Safari/602.1",
+]
+
+
+class Command(BaseCommand):
+    help = "Configures a Shynet demo service"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "name", type=str,
+        )
+        parser.add_argument("owner_email", type=str)
+        parser.add_argument(
+            "avg", type=int,
+        )
+        parser.add_argument("deviation", type=float, default=0.4)
+        parser.add_argument(
+            "days", type=int,
+        )
+        parser.add_argument("load_time", type=float, default=1000)
+
+    def handle(self, *args, **options):
+        owner = User.objects.get(email=options.get("owner_email"))
+        service = Service.objects.create(name=options.get("name"), owner=owner)
+
+        print(
+            f"Created demo service `{service.name}` (uuid: `{service.uuid}`, owner: {owner})"
+        )
+
+        # Go through each day requested, creating sessions and hits
+        for days in range(options.get("days")):
+            day = (now() - timedelta(days=days)).replace(hour=0, minute=0, second=0)
+            print(f"Populating info for {day}...")
+            avg = options.get("avg")
+            deviation = options.get("deviation")
+            ips = [
+                ".".join(map(str, (random.randint(0, 255) for _ in range(4))))
+                for _ in range(avg)
+            ]
+
+            n = avg + random.randrange(-1 * deviation * avg, deviation * avg)
+            for _ in range(n):
+                time = day + timedelta(
+                    hours=random.randrange(0, 23),
+                    minutes=random.randrange(0, 59),
+                    seconds=random.randrange(0, 59),
+                )
+                ip = random.choice(ips)
+                load_time = random.normalvariate(options.get("load_time"), 500)
+                referrer = random.choice(REFERRERS)
+                location = "https://example.com" + random.choice(LOCATIONS).replace(
+                    "{rand}", str(random.randint(0, n))
+                )
+                user_agent = random.choice(USER_AGENTS)
+                ingress_request(
+                    service.uuid,
+                    "JS",
+                    time,
+                    {"loadTime": load_time, "referrer": referrer},
+                    ip,
+                    location,
+                    user_agent,
+                )
+
+            print(f"Created {n} demo hits on {day}!")
+
+        self.stdout.write(self.style.SUCCESS(f"Successfully created demo data!"))

shynet/core/management/commands/hostname.py

@@ -1,10 +1,12 @@
-from django.core.management.base import BaseCommand, CommandError
+import traceback
+import uuid
+
 from django.conf import settings
 from django.contrib.sites.models import Site
-from core.models import User
+from django.core.management.base import BaseCommand, CommandError
 from django.utils.crypto import get_random_string
-import uuid
+
-import traceback
+from core.models import User
 
 
 class Command(BaseCommand):
@@ -12,13 +14,18 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument(
-            "hostname",
+            "hostname", type=str,
-            type=str,
         )
 
     def handle(self, *args, **options):
         site = Site.objects.get(pk=settings.SITE_ID)
         site.domain = options.get("hostname")
+        if options.get("hostname").lower().startswith("http"):
+            self.stdout.write(
+                self.style.WARNING(
+                    f"Warning: the hostname '{options.get('hostname')}' starts with `http`. You almost certainly don't want this. The hostname is supposed to be the raw domain name of your Shynet instance, without `http://` or `https://`. For example, if your Shynet instance will eventually be hosted at `https://analytics.example.com`, the hostname should be `analytics.example.com`."
+                )
+            )
         site.save()
         self.stdout.write(
             self.style.SUCCESS(

shynet/core/management/commands/registeradmin.py

@@ -1,10 +1,12 @@
-from django.core.management.base import BaseCommand, CommandError
+import traceback
+import uuid
+
 from django.conf import settings
 from django.contrib.sites.models import Site
-from core.models import User
+from django.core.management.base import BaseCommand, CommandError
 from django.utils.crypto import get_random_string
-import uuid
+
-import traceback
+from core.models import User
 
 
 class Command(BaseCommand):
@@ -12,18 +14,13 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument(
-            "email",
+            "email", type=str,
-            type=str,
         )
 
     def handle(self, *args, **options):
         email = options.get("email")
         password = get_random_string()
-        User.objects.create_superuser(
+        User.objects.create_superuser(str(uuid.uuid4()), email=email, password=password)
-            str(uuid.uuid4()), email=email, password=password
+        self.stdout.write(self.style.SUCCESS("Successfully created a Shynet superuser"))
-        )
-        self.stdout.write(
-            self.style.SUCCESS("Successfully created a Shynet superuser")
-        )
         self.stdout.write(f"Email address: {email}")
         self.stdout.write(f"Password: {password}")

shynet/core/management/commands/startup_checks.py

@@ -0,0 +1,56 @@
+import traceback
+import uuid
+
+from django.conf import settings
+from django.contrib.sites.models import Site
+from django.core.exceptions import ImproperlyConfigured
+from django.core.management.base import BaseCommand, CommandError
+from django.db import DEFAULT_DB_ALIAS, connections
+from django.db.utils import ConnectionHandler, OperationalError
+from django.utils.crypto import get_random_string
+
+from core.models import User
+
+
+class Command(BaseCommand):
+    help = "Internal command to perform startup checks."
+
+    def check_migrations(self):
+        from django.db.migrations.executor import MigrationExecutor
+
+        try:
+            executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS])
+        except OperationalError:
+            # DB_NAME database not found?
+            return True
+        except ImproperlyConfigured:
+            # No databases are configured (or the dummy one)
+            return True
+
+        if executor.migration_plan(executor.loader.graph.leaf_nodes()):
+            return True
+
+        return False
+
+    def handle(self, *args, **options):
+        migration = self.check_migrations()
+
+        admin, hostname, whitelabel = [True] * 3
+        if not migration:
+            admin = not User.objects.all().exists()
+            hostname = (
+                not Site.objects.filter(domain__isnull=False)
+                .exclude(domain__exact="")
+                .exclude(domain__exact="example.com")
+                .exists()
+            )
+            whitelabel = (
+                not Site.objects.filter(name__isnull=False)
+                .exclude(name__exact="")
+                .exclude(name__exact="example.com")
+                .exists()
+            )
+
+        self.stdout.write(
+            self.style.SUCCESS(f"{migration} {admin} {hostname} {whitelabel}")
+        )

shynet/core/management/commands/whitelabel.py

@@ -1,10 +1,12 @@
-from django.core.management.base import BaseCommand, CommandError
+import traceback
+import uuid
+
 from django.conf import settings
 from django.contrib.sites.models import Site
-from core.models import User
+from django.core.management.base import BaseCommand, CommandError
 from django.utils.crypto import get_random_string
-import uuid
+
-import traceback
+from core.models import User
 
 
 class Command(BaseCommand):
@@ -12,8 +14,7 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument(
-            "name",
+            "name", type=str,
-            type=str,
         )
 
     def handle(self, *args, **options):

shynet/core/migrations/0003_service_respect_dnt.py

@@ -6,13 +6,13 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('core', '0002_auto_20200415_1742'),
+        ("core", "0002_auto_20200415_1742"),
     ]
 
     operations = [
         migrations.AddField(
-            model_name='service',
+            model_name="service",
-            name='respect_dnt',
+            name="respect_dnt",
             field=models.BooleanField(default=True),
         ),
     ]

shynet/core/migrations/0004_service_collect_ips.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0.5 on 2020-05-02 16:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0003_service_respect_dnt"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="service",
+            name="collect_ips",
+            field=models.BooleanField(default=True),
+        ),
+    ]

shynet/core/migrations/0005_service_ignored_ips.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.0.6 on 2020-05-07 20:28
+
+from django.db import migrations, models
+
+import core.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0004_service_collect_ips"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="service",
+            name="ignored_ips",
+            field=models.TextField(
+                blank=True, default="", validators=[core.models._validate_network_list]
+            ),
+        ),
+    ]

shynet/core/migrations/0006_service_hide_referrer_regex.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.0.6 on 2020-05-07 21:23
+
+from django.db import migrations, models
+
+import core.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0005_service_ignored_ips"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="service",
+            name="hide_referrer_regex",
+            field=models.TextField(
+                blank=True, default="", validators=[core.models._validate_regex]
+            ),
+        ),
+    ]

shynet/core/migrations/0007_service_ignore_robots.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0.6 on 2020-06-15 16:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0006_service_hide_referrer_regex"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="service",
+            name="ignore_robots",
+            field=models.BooleanField(default=False),
+        )
+    ]

shynet/core/migrations/0008_auto_20200628_1403.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.1b1 on 2020-06-28 18:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0007_service_ignore_robots"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="service",
+            name="script_inject",
+            field=models.TextField(blank=True, default=""),
+        ),
+        migrations.AlterField(
+            model_name="user",
+            name="first_name",
+            field=models.CharField(
+                blank=True, max_length=150, verbose_name="first name"
+            ),
+        ),
+    ]

shynet/core/models.py

@@ -1,8 +1,11 @@
+import ipaddress
 import json
+import re
 import uuid
 
 from django.apps import apps
 from django.contrib.auth.models import AbstractUser
+from django.core.exceptions import ValidationError
 from django.db import models
 from django.db.models.functions import TruncDate
 from django.db.utils import NotSupportedError
@@ -14,6 +17,26 @@ def _default_uuid():
     return str(uuid.uuid4())
 
 
+def _validate_network_list(networks: str):
+    try:
+        _parse_network_list(networks)
+    except ValueError as e:
+        raise ValidationError(str(e))
+
+
+def _validate_regex(regex: str):
+    try:
+        re.compile(regex)
+    except re.error:
+        raise ValidationError(f"'{regex}' is not valid RegEx")
+
+
+def _parse_network_list(networks: str):
+    if len(networks.strip()) == 0:
+        return []
+    return [ipaddress.ip_network(network.strip()) for network in networks.split(",")]
+
+
 class User(AbstractUser):
     username = models.TextField(default=_default_uuid, unique=True)
     email = models.EmailField(unique=True)
@@ -42,6 +65,15 @@ class Service(models.Model):
         max_length=2, choices=SERVICE_STATUSES, default=ACTIVE, db_index=True
     )
     respect_dnt = models.BooleanField(default=True)
+    ignore_robots = models.BooleanField(default=False)
+    collect_ips = models.BooleanField(default=True)
+    ignored_ips = models.TextField(
+        default="", blank=True, validators=[_validate_network_list]
+    )
+    hide_referrer_regex = models.TextField(
+        default="", blank=True, validators=[_validate_regex]
+    )
+    script_inject = models.TextField(default="", blank=True)
 
     class Meta:
         ordering = ["name", "uuid"]
@@ -49,6 +81,21 @@ class Service(models.Model):
     def __str__(self):
         return self.name
 
+    def get_ignored_networks(self):
+        return _parse_network_list(self.ignored_ips)
+
+    def get_ignored_referrer_regex(self):
+        if len(self.hide_referrer_regex.strip()) == 0:
+            return re.compile(r".^")  # matches nothing
+        else:
+            try:
+                return re.compile(self.hide_referrer_regex)
+            except re.error:
+                # Regexes are validated in the form, but this is an important
+                # fallback to prevent form validation and malformed source
+                # data from causing all service pages to error
+                return re.compile(r".^")
+
     def get_daily_stats(self):
         return self.get_core_stats(
             start_time=timezone.now() - timezone.timedelta(days=1)
@@ -95,12 +142,17 @@ class Service(models.Model):
             .order_by("-count")
         )
 
-        referrers = (
+        referrer_ignore = self.get_ignored_referrer_regex()
-            hits.filter(initial=True)
+        referrers = [
-            .values("referrer")
+            referrer
-            .annotate(count=models.Count("referrer"))
+            for referrer in (
-            .order_by("-count")
+                hits.filter(initial=True)
-        )
+                .values("referrer")
+                .annotate(count=models.Count("referrer"))
+                .order_by("-count")
+            )
+            if not referrer_ignore.match(referrer["referrer"])
+        ]
 
         countries = (
             sessions.values("country")
@@ -130,12 +182,6 @@ class Service(models.Model):
             .order_by("-count")
         )
 
-        device_types = (
-            sessions.values("device_type")
-            .annotate(count=models.Count("device_type"))
-            .order_by("-count")
-        )
-
         avg_load_time = hits.aggregate(load_time__avg=models.Avg("load_time"))[
             "load_time__avg"
         ]
@@ -189,7 +235,9 @@ class Service(models.Model):
             "session_chart_data": json.dumps(
                 [
                     {"x": str(key), "y": value}
-                    for key, value in session_chart_data.items()
+                    for key, value in sorted(
+                        session_chart_data.items(), key=lambda k: k[0]
+                    )
                 ]
             ),
             "online": True,

shynet/dashboard/forms.py

@@ -1,44 +1,83 @@
+from allauth.account.admin import EmailAddress
 from django import forms
 from django.utils.translation import gettext_lazy as _
 
 from core.models import Service, User
-from allauth.account.admin import EmailAddress
 
 
 class ServiceForm(forms.ModelForm):
     class Meta:
         model = Service
-        fields = ["name", "link", "respect_dnt", "collaborators"]
+        fields = [
+            "name",
+            "link",
+            "respect_dnt",
+            "collect_ips",
+            "ignored_ips",
+            "ignore_robots",
+            "hide_referrer_regex",
+            "origins",
+            "collaborators",
+            "script_inject",
+        ]
         widgets = {
             "name": forms.TextInput(),
             "origins": forms.TextInput(),
+            "ignored_ips": forms.TextInput(),
             "respect_dnt": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
+            "collect_ips": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
+            "ignore_robots": forms.RadioSelect(choices=[(True, "Yes"), (False, "No")]),
+            "hide_referrer_regex": forms.TextInput(),
+            "script_inject": forms.Textarea(attrs={"class": "font-mono", "rows": 5}),
         }
         labels = {
-            "origins": "Allowed Hostnames",
+            "origins": "Allowed origins",
             "respect_dnt": "Respect DNT",
+            "collect_ips": "Collect IP addresses",
+            "ignored_ips": "Ignored IP addresses",
+            "ignore_robots": "Ignore robots",
+            "hide_referrer_regex": "Hide specific referrers",
+            "script_inject": "Additional injected JS",
         }
         help_texts = {
             "name": _("What should the service be called?"),
             "link": _("What's the service's primary URL?"),
             "origins": _(
-                "At what hostnames does the service operate? This sets CORS headers, so use '*' if you're not sure (or don't care)."
+                "At what origins does the service operate? Use commas to separate multiple values. This sets CORS headers, so use '*' if you're not sure (or don't care)."
             ),
             "respect_dnt": "Should visitors who have enabled <a href='https://en.wikipedia.org/wiki/Do_Not_Track'>Do Not Track</a> be excluded from all data?",
+            "collect_ips": "Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected.",
+            "ignored_ips": "A comma-separated list of IP addresses or IP ranges (IPv4 and IPv6) to exclude from tracking (e.g., '192.168.0.2, 127.0.0.1/32').",
+            "ignore_robots": "Should sessions generated by bots be excluded from tracking?",
+            "hide_referrer_regex": "Any referrers that match this <a href='https://regexr.com/'>RegEx</a> will not be listed in the referrer summary. Sessions will still be tracked normally. No effect if left blank.",
+            "script_inject": "Optional additional JavaScript to inject at the end of the Shynet script. This code will be injected on every page where this service is installed.",
         }
 
-    collaborators = forms.CharField(help_text="Which users should have read-only access to this service? (Comma separated list of emails.)", required=False)
+    collaborators = forms.CharField(
-
+        help_text="Which users on this Shynet instance should have read-only access to this service? (Comma separated list of emails.)",
+        required=False,
+    )
 
     def clean_collaborators(self):
         collaborators = []
+        users_to_emails = (
+            {}
+        )  # maps users to the email they are listed under as a collaborator
         for collaborator_email in self.cleaned_data["collaborators"].split(","):
             email = collaborator_email.strip()
             if email == "":
                 continue
-            collaborator_email_linked = EmailAddress.objects.filter(email__iexact=email).first()
+            collaborator_email_linked = EmailAddress.objects.filter(
+                email__iexact=email
+            ).first()
             if collaborator_email_linked is None:
                 raise forms.ValidationError(f"Email '{email}' is not registered")
+            user = collaborator_email_linked.user
+            if user in collaborators:
+                raise forms.ValidationError(
+                    f"The emails '{email}' and '{users_to_emails[user]}' both correspond to the same user"
+                )
+            users_to_emails[user] = email
             collaborators.append(collaborator_email_linked.user)
         return collaborators
 

shynet/dashboard/mixins.py

@@ -1,6 +1,6 @@
+from datetime import datetime, time
 from urllib.parse import urlparse
 
-from datetime import time, datetime
 from django.utils import timezone
 
 

shynet/dashboard/static/dashboard/css/global.css

@@ -14,3 +14,17 @@
 .rf {
     text-align: right !important;
 }
+
+:root {
+    --color-neutral-000: white;
+    --color-neutral-50: #F8FAFC;
+    --color-neutral-100: #F1F5F9;
+    --color-neutral-200: #E2E8F0;
+    --color-neutral-300: #CBD5E1;
+    --color-neutral-400: #94A3B8;
+    --color-neutral-500: #64748B;
+    --color-neutral-600: #475569;
+    --color-neutral-700: #334155;
+    --color-neutral-800: #1E293B;
+    --color-neutral-900: #0F172A;
+}

shynet/dashboard/tasks.py

@@ -0,0 +1,16 @@
+from celery import shared_task
+from django.core import mail
+from django.conf import settings
+import html2text
+
+
+@shared_task
+def send_email(to: [str], subject: str, content: str, from_email: str = None):
+    text_content = html2text.html2text(content)
+    mail.send_mail(
+        subject,
+        text_content,
+        from_email or settings.DEFAULT_FROM_EMAIL,
+        to,
+        html_message=content,
+    )

shynet/dashboard/templates/base.html

@@ -1,4 +1,4 @@
-{% load static rules %}
+{% load static rules helpers %}
 <!DOCTYPE html>
 
 <html>
@@ -7,24 +7,26 @@
   <title>{% block head_title %}Privacy-oriented analytics{% endblock %} | {{request.site.name}}</title>
   <meta name="robots" content="noindex">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  {% include 'a17t/head.html' %}
+  {% include 'a17t/includes/head.html' %}
-  <script src="https://cdn.jsdelivr.net/npm/litepicker@1.2.0/dist/js/main.js"
+  <link rel="icon" type="image/png" href="{% static 'dashboard/images/icon.png' %}">
-    integrity="sha256-mOlCEHUNWZPYIrc5OFL4Ab2rsJGzIPld3cy1ok7Cfx0=" crossorigin="anonymous"></script>
+  <script src="{% static 'apexcharts/dist/apexcharts.min.js'%}"></script>
-  <script src="https://cdn.jsdelivr.net/npm/apexcharts@3.18.1/dist/apexcharts.min.js"
+  <script src="{% static 'litepicker/dist/js/main.js' %}"></script>
-    integrity="sha256-RalQXBZdisB04aaBsm+6YZ0b/iRYjX1MZn90m19AnCY=" crossorigin="anonymous"></script>
+  <script src="{% static 'turbolinks/dist/turbolinks.js' %}"></script>
+  <script src="{% static 'dashboard/js/base.js' %}"></script>
   <link rel="stylesheet" href="{% static 'dashboard/css/global.css' %}">
   {% block extra_head %}
   {% endblock %}
 </head>
 
-<body class="bg-gray-200 min-h-full">
+<body class="bg-neutral-100 min-h-full">
   {% block body %}
 
-  <section class="max-w-screen-xl mx-auto px-4 md:px-0 py-4 md:py-12 md:flex">
+  <section class="max-w-screen-xl mx-auto px-4 py-4 md:py-12 md:flex">
-    <aside class="mb-8 md:w-2/12 md:pr-6 relative flex flex-wrap md:block justify-between items-center">
+    <aside
+      class="mb-8 md:w-2/12 md:pr-6 relative flex flex-wrap md:block justify-between items-center overflow-x-hidden">
       <a class="icon ~urge ml-2 md:ml-6 md:mb-8 md:mt-3" href="{% url 'dashboard:dashboard' %}">
-        <i class="fas fa-binoculars fa-3x text-purple-600 hidden md:block"></i>
+        <i class="fas fa-binoculars fa-3x text-urge-600 hidden md:block"></i>
-        <i class="fas fa-binoculars fa-2x text-purple-600 md:hidden"></i>
+        <i class="fas fa-binoculars fa-2x text-urge-600 md:hidden"></i>
       </a>
 
       <button class="button ~neutral !low md:hidden"
@@ -35,13 +37,13 @@
       </button>
       <hr class="sep h-4 md:h-8 w-full">
       <div id="navMenuExpanded"
-        class="bg-white shadow-lg md:shadow-none p-4 hidden rounded-lg md:block md:bg-transparent md:border-none md:p-0 w-full">
+        class="bg-neutral-000 shadow-lg md:shadow-none p-4 hidden rounded-lg md:block md:bg-transparent md:border-none md:p-0 w-full">
         {% if user.owning_services.all %}
         <p class="ml-2 mb-1 supra font-medium text-gray-500 pointer-events-none">Services</p>
 
         {% for service in user.owning_services.all %}
         {% url 'dashboard:service' service.uuid as url %}
-        {% include 'dashboard/includes/sidebar_portal.html' with label=service.name|truncatechars:20 url=url %}
+        {% include 'dashboard/includes/sidebar_portal.html' with label=service.name|truncatechars:16 url=url icon=service.link|iconify %}
         {% endfor %}
 
         {% endif %}
@@ -72,7 +74,7 @@
 
         {% if user.is_superuser %}
         {% url 'admin:index' as url %}
-        {% include 'dashboard/includes/sidebar_portal.html' with label="Admin" url=url %}
+        {% include 'dashboard/includes/sidebar_portal.html' with label="Admin" disable_turbolinks=True url=url %}
         {% endif %}
 
         {% url 'account_email' as url %}
@@ -93,6 +95,10 @@
         {% include 'dashboard/includes/sidebar_portal.html' with label="Sign Up" url=url %}
 
         {% endif %}
+
+        <hr class="sep h-8">
+
+        {% sidebar_footer %}
       </div>
     </aside>
     <div class="md:w-10/12">

shynet/dashboard/templates/dashboard/includes/date_range.html

@@ -2,7 +2,7 @@
     <input type="hidden" name="startDate" value="{{start_date.isoformat}}" id="startDate">
     <input type="hidden" name="endDate" value="{{end_date.isoformat}}" id="endDate">
 </form>
-<input type="input" id="rangePicker" placeholder="Date range" class="input ~neutral cursor-pointer" readonly>
+<input type="input" id="rangePicker" placeholder="Date range" class="input ~neutral bg-neutral-000 cursor-pointer" readonly>
 <style>
     :root {
         --litepickerMonthButtonHover: var(--color-urge);

shynet/dashboard/templates/dashboard/includes/service_form.html

@@ -0,0 +1,17 @@
+{% load a17t_tags %}
+
+{{form.name|a17t}}
+{{form.link|a17t}}
+{{form.collaborators|a17t}}
+
+<details {% if form.errors %}open{% endif %}>
+    <summary class="cursor-pointer text-sm">Advanced settings</summary>
+    <hr class="sep h-4">
+    {{form.respect_dnt|a17t}}
+    {{form.collect_ips|a17t}}
+    {{form.ignored_ips|a17t}}
+    {{form.ignore_robots|a17t}}
+    {{form.hide_referrer_regex|a17t}}
+    {{form.origins|a17t}}
+    {{form.script_inject|a17t}}
+</details>

shynet/dashboard/templates/dashboard/includes/service_overview.html

@@ -4,8 +4,9 @@
     {% with stats=object.stats %}
     <div class="p-4 md:flex justify-between">
         <div class="flex items-center mb-4 md:mb-0">
-            <h3 class="heading text-xl md:text-2xl mr-2 mb-1 text-purple-600">
+            <h3 class="heading text-xl md:text-2xl mr-2 mb-1 text-urge-600 flex items-center">
-                {{object.name}}
+                {{object.link|iconify}}
+                <span>{{object.name}}</span>
             </h3>
             {% include 'dashboard/includes/stats_status_chip.html' %}
         </div>

shynet/dashboard/templates/dashboard/includes/session_list.html

@@ -13,7 +13,7 @@
         <tr>
             <td>
                 <a href="{% url 'dashboard:service_session' object.pk session.pk %}"
-                    class="font-medium text-purple-700">
+                    class="font-medium text-urge-700">
                     {{session.start_time|date:"M j Y, g:i a"|capfirst}}
                     {% if session.is_currently_active %}
                     <span class="badge ~positive">Online</span>

shynet/dashboard/templates/dashboard/includes/sidebar_footer.html

@@ -0,0 +1,5 @@
+<div class="pl-2">
+  <p class="support text-gray-600">
+    <a href="https://github.com/milesmcc/shynet">Shynet {{version}}</a>
+  </p>
+</div>

shynet/dashboard/templates/dashboard/includes/sidebar_portal.html

@@ -1,6 +1,6 @@
 {% load helpers %}
 
 <div>
-    <a class="portal !low {% if request.get_full_path|startswith:url %}~urge active bg-gray-100{% endif %}"
+    <a class="portal !low {% if request.get_full_path|startswith:url %}~urge active bg-neutral-100{% endif %} flex items-center"
-        href="{{url}}">{{label}}</a>
+        {% if disable_turbolinks %}data-turbolinks="false"{% endif %} href="{{url}}">{{icon}} {{label}}</a>
 </div>

shynet/dashboard/templates/dashboard/pages/dashboard.html

@@ -13,7 +13,7 @@
         </div>
         {% has_perm "core.create_service" user as can_create %}
         {% if can_create %}
-        <a href="{% url 'dashboard:service_create' %}" class="button field w-auto">+ New Service</a>
+        <a href="{% url 'dashboard:service_create' %}" class="button field bg-neutral-000 w-auto">+ New Service</a>
         {% endif %}
     </div>
 </div>

shynet/dashboard/templates/dashboard/pages/service.html

@@ -6,29 +6,29 @@
 <div class="mr-2">{% include 'dashboard/includes/date_range.html' %}</div>
 {% has_perm 'core.change_service' user object as can_update %}
 {% if can_update %}
-<a href="{% url 'dashboard:service_update' service.uuid %}" class="button field ~neutral w-auto">Manage &rarr;</a>
+<a href="{% url 'dashboard:service_update' service.uuid %}" class="button field bg-neutral-000 w-auto">Manage &rarr;</a>
 {% endif %}
 {% endblock %}
 
 {% block service_content %}
 <div class="grid grid-cols-2 gap-6 md:flex justify-between mb-6 card ~neutral !high px-6" id="stats">
-    {% with classes="text-sm font-semibold" good_classes="text-green-400" bad_classes="text-red-400" neutral_classes="text-gray-400" %}
+    {% with classes="text-sm font-semibold" good_classes="text-positive-400" bad_classes="text-critical-400" neutral_classes="text-gray-400" %}
     <article class="">
         <p class="label text-gray-400">Sessions</p>
         <p class="heading">
             {{stats.session_count|intcomma}}
-            <div>
+        <div>
-                {% compare stats.compare.session_count stats.session_count "UP" classes=classes good_classes=good_classes bad_classes=bad_classes neutral_classes=neutral_classes %}
+            {% compare stats.compare.session_count stats.session_count "UP" classes=classes good_classes=good_classes bad_classes=bad_classes neutral_classes=neutral_classes %}
-            </div>
+        </div>
         </p>
     </article>
     <article class="">
         <p class="label text-gray-400">Hits</p>
         <p class="heading">
             {{stats.hit_count|intcomma}}
-            <div>
+        <div>
-                {% compare stats.compare.hit_count stats.hit_count "UP" classes=classes good_classes=good_classes bad_classes=bad_classes neutral_classes=neutral_classes %}
+            {% compare stats.compare.hit_count stats.hit_count "UP" classes=classes good_classes=good_classes bad_classes=bad_classes neutral_classes=neutral_classes %}
-            </div>
+        </div>
         </p>
     </article>
     <article class="">
@@ -39,9 +39,9 @@
             {% else %}
             ?
             {% endif %}
-            <div>
+        <div>
-                {% compare stats.compare.avg_load_time stats.avg_load_time "DOWN" classes=classes good_classes=good_classes bad_classes=bad_classes neutral_classes=neutral_classes %}
+            {% compare stats.compare.avg_load_time stats.avg_load_time "DOWN" classes=classes good_classes=good_classes bad_classes=bad_classes neutral_classes=neutral_classes %}
-            </div>
+        </div>
         </p>
     </article>
     <article class="">
@@ -52,9 +52,9 @@
             {% else %}
             ?
             {% endif %}
-            <div>
+        <div>
-                {% compare stats.compare.bounce_rate_pct stats.bounce_rate_pct "DOWN" classes=classes good_classes=good_classes bad_classes=bad_classes neutral_classes=neutral_classes %}
+            {% compare stats.compare.bounce_rate_pct stats.bounce_rate_pct "DOWN" classes=classes good_classes=good_classes bad_classes=bad_classes neutral_classes=neutral_classes %}
-            </div>
+        </div>
         </p>
     </article>
     <article class="">
@@ -65,9 +65,9 @@
             {% else %}
             ?
             {% endif %}
-            <div>
+        <div>
-                {% compare stats.compare.avg_session_duration stats.avg_session_duration "UP" classes=classes good_classes=good_classes bad_classes=bad_classes neutral_classes=neutral_classes %}
+            {% compare stats.compare.avg_session_duration stats.avg_session_duration "UP" classes=classes good_classes=good_classes bad_classes=bad_classes neutral_classes=neutral_classes %}
-            </div>
+        </div>
         </p>
     </article>
     <article class="">
@@ -78,9 +78,9 @@
             {% else %}
             ?
             {% endif %}
-            <div>
+        <div>
-                {% compare stats.compare.avg_hits_per_session stats.avg_hits_per_session "UP" classes=classes good_classes=good_classes bad_classes=bad_classes neutral_classes=neutral_classes %}
+            {% compare stats.compare.avg_hits_per_session stats.avg_hits_per_session "UP" classes=classes good_classes=good_classes bad_classes=bad_classes neutral_classes=neutral_classes %}
-            </div>
+        </div>
         </p>
     </article>
     {% endwith %}
@@ -166,7 +166,7 @@
             <tbody>
                 {% for os in stats.operating_systems %}
                 <tr>
-                    <td>{{os.os|default:"Unknown"}}</td>
+                    <td class="flex items-center">{{os.os|iconify}}<span>{{os.os|default:"Unknown"}}</span></td>
                     <td class="rf">{{os.count|intcomma}}</td>
                 </tr>
                 {% empty %}
@@ -188,7 +188,8 @@
             <tbody>
                 {% for browser in stats.browsers %}
                 <tr>
-                    <td>{{browser.browser|default:"Unknown"}}</td>
+                    <td class="flex items-center">
+                        {{browser.browser|iconify}}<span>{{browser.browser|default:"Unknown"}}</span></td>
                     <td class="rf">{{browser.count|intcomma}}</td>
                 </tr>
                 {% empty %}
@@ -225,7 +226,8 @@
 <div class="card ~neutral !low limited-height py-2">
     {% include 'dashboard/includes/session_list.html' %}
     <hr class="sep h-8">
-    <a href="{% url 'dashboard:service_session_list' service.uuid %}" class="button ~neutral w-auto mb-2">View more sessions
+    <a href="{% url 'dashboard:service_session_list' service.uuid %}" class="button ~neutral w-auto mb-2">View more
+        sessions
         &rarr;</a>
 </div>
 {% endblock %}

shynet/dashboard/templates/dashboard/pages/service_create.html

@@ -10,7 +10,7 @@
 <form class="card ~neutral !low p-0 max-w-xl" method="POST">
     {% csrf_token %}
     <div class="p-4">
-        {{form|a17t}}
+        {% include 'dashboard/includes/service_form.html' %}
     </div>
     <div class="section ~urge !normal p-4">
         <button type="submit" class="button ~urge !high">Create</button>

shynet/dashboard/templates/dashboard/pages/service_session.html

@@ -5,7 +5,7 @@
 {% block head_title %}{{object.name}} Session{% endblock %}
 
 {% block service_actions %}
-<a href="{% url 'dashboard:service' object.uuid %}" class="button field ~neutral w-auto">Analytics &rarr;</a>
+<a href="{% url 'dashboard:service' object.uuid %}" class="button field bg-neutral-000 w-auto">Analytics &rarr;</a>
 {% endblock %}
 
 {% block service_content %}
@@ -59,8 +59,8 @@
             </p>
         </div>
         <div>
-            <p>Time Zone</p>
+            <p>IP</p>
-            <p class="label">{{session.time_zone|default:"Unknown"}}</p>
+            <p class="label" title="{{session.ip}}">{{session.ip|default:"Not Collected"|truncatechars:"16"}}</p>
         </div>
     </div>
 </article>

shynet/dashboard/templates/dashboard/pages/service_session_list.html

@@ -6,7 +6,7 @@
 
 {% block service_actions %}
 <div class="mr-2">{% include 'dashboard/includes/date_range.html' %}</div>
-<a href="{% url 'dashboard:service' object.uuid %}" class="button field ~neutral w-auto">Analytics &rarr;</a>
+<a href="{% url 'dashboard:service' object.uuid %}" class="button field ~neutral bg-neutral-000 w-auto">Analytics &rarr;</a>
 {% endblock %}
 
 {% block service_content %}

shynet/dashboard/templates/dashboard/pages/service_update.html

@@ -5,7 +5,7 @@
 {% block head_title %}{{object.name}} Management{% endblock %}
 
 {% block service_actions %}
-<a href="{% url 'dashboard:service' object.uuid %}" class="button field ~neutral w-auto">View &rarr;</a>
+<a href="{% url 'dashboard:service' object.uuid %}" class="button field bg-neutral-000 w-auto">View &rarr;</a>
 {% endblock %}
 
 {% block service_content %}
@@ -14,8 +14,8 @@
     <p>Place the following snippet at the end of the <code>&lt;body&gt;</code> tag on any page you'd like to track.</p>
     <div class="card ~neutral !high font-mono text-sm">
         {% filter force_escape %}<noscript><img
-                src="//{{request.site.domain}}{% url 'ingress:endpoint_pixel' object.uuid %}"></noscript>
+                src="{{script_protocol}}{{request.site.domain}}{% url 'ingress:endpoint_pixel' object.uuid %}"></noscript>
-        <script src="//{{request.site.domain}}{% url 'ingress:endpoint_script' object.uuid %}"></script>
+        <script src="{{script_protocol}}{{request.site.domain}}{% url 'ingress:endpoint_script' object.uuid %}"></script>
         {% endfilter %}
     </div>
     <hr class="sep h-4">
@@ -23,7 +23,7 @@
     <form class="card ~neutral !low p-0" method="POST">
         {% csrf_token %}
         <div class="p-4">
-            {{form|a17t}}
+            {% include 'dashboard/includes/service_form.html' %}
         </div>
         <div class="section ~neutral !normal p-4 flex justify-between">
             <div>

shynet/dashboard/templates/dashboard/service_base.html

@@ -8,6 +8,7 @@
 <div class="md:flex justify-between items-center" id="heading">
     <a class="flex items-center mb-4 md:mb-0" href="{% url 'dashboard:service' object.uuid %}">
         <h3 class="heading leading-none mr-4">
+            {{object.link|iconify}}
             {{object.name}}
         </h3>
         <div class='text-3xl'>

shynet/dashboard/templatetags/helpers.py

@@ -1,7 +1,9 @@
 from urllib.parse import urlparse
-import pycountry
+
 import flag
+import pycountry
 from django import template
+from django.conf import settings
 from django.utils import timezone
 from django.utils.html import escape
 from django.utils.safestring import SafeString
@@ -41,12 +43,7 @@ def country_name(isocode):
 
 @register.simple_tag
 def relative_stat_tone(
-    start,
+    start, end, good="UP", good_classes=None, bad_classes=None, neutral_classes=None,
-    end,
-    good="UP",
-    good_classes=None,
-    bad_classes=None,
-    neutral_classes=None,
 ):
     good_classes = good_classes or "~positive"
     bad_classes = bad_classes or "~critical"
@@ -64,25 +61,33 @@ def relative_stat_tone(
 
 @register.simple_tag
 def percent_change_display(start, end):
-    if start == None or end == None:
+    try:
-        return SafeString("Δ n/a")
+        if start == None or end == None:
-    if start == end:
+            return SafeString("Δ n/a")
-        direction = "Δ "
+        if start == end:
-    else:
+            direction = "Δ "
-        direction = "↑ " if end > start else "↓ "
-
-    if start == 0 and end != 0:
-        pct_change = "100%"
-    elif start == 0:
-        pct_change = "0%"
-    else:
-        change = int(round(100 * abs(end - start) / start))
-        if change > 999:
-            return "> 999%"
         else:
-            pct_change = str(change) + "%"
+            direction = "↑ " if end > start else "↓ "
 
-    return SafeString(direction + pct_change)
+        if start == 0 and end != 0:
+            pct_change = "100%"
+        elif start == 0:
+            pct_change = "0%"
+        else:
+            change = int(round(100 * abs(end - start) / max(start, 1)))
+            if change > 999:
+                return "> 999%"
+            else:
+                pct_change = str(change) + "%"
+
+        return SafeString(direction + pct_change)
+    except: # TODO: filter for specific issues
+        return SafeString("Δ ?")
+
+
+@register.inclusion_tag("dashboard/includes/sidebar_footer.html")
+def sidebar_footer():
+    return {"version": settings.VERSION if settings.SHOW_SHYNET_VERSION else ""}
 
 
 @register.inclusion_tag("dashboard/includes/stat_comparison.html")
@@ -113,12 +118,60 @@ def startswith(text, starts):
     return False
 
 
+@register.filter
+def iconify(text):
+    if not settings.SHOW_THIRD_PARTY_ICONS:
+        return ""
+
+    text = text.lower()
+    icons = {
+        "chrome": "chrome.com",
+        "safari": "www.apple.com",
+        "windows": "windows.com",
+        "edge": "microsoft.com",
+        "firefox": "firefox.com",
+        "opera": "opera.com",
+        "unknown": "example.com",
+        "linux": "kernel.org",
+        "ios": "www.apple.com",
+        "mac": "www.apple.com",
+        "macos": "www.apple.com",
+        "mac os x": "www.apple.com",
+        "android": "android.com",
+        "chrome os": "chrome.com",
+        "ubuntu": "ubuntu.com",
+        "fedora": "getfedora.org",
+        "mobile safari": "www.apple.com",
+        "chrome mobile ios": "chrome.com",
+        "chrome mobile": "chrome.com",
+        "samsung internet": "samsung.com",
+        "google": "google.com",
+        "chrome mobile webview": "chrome.com",
+        "firefox mobile": "firefox.com",
+        "edge mobile": "microsoft.com",
+        "chromium": "chromium.org",
+    }
+
+    domain = None
+    if text.startswith("http"):
+        domain = urlparse(text).netloc
+    elif text in icons:
+        domain = icons[text]
+    else:
+        # This fallback works better than you'd think!
+        domain = text + ".com"
+
+    return SafeString(
+        f'<span class="icon mr-1"><img src="https://icons.duckduckgo.com/ip3/{domain}.ico"></span>'
+    )
+
+
 @register.filter
 def urldisplay(url):
     if url.startswith("http"):
         display_url = url.replace("http://", "").replace("https://", "")
         return SafeString(
-            f"<a href='{url}' title='{url}' rel='nofollow'>{escape(display_url if len(display_url) < 40 else display_url[:40] + '...')}</a>"
+            f"<a href='{url}' title='{url}' rel='nofollow' class='flex items-center'>{iconify(url)} {escape(display_url if len(display_url) < 40 else display_url[:40] + '...')}</a>"
         )
     else:
         return url

shynet/dashboard/views.py

@@ -1,5 +1,7 @@
+from django.conf import settings
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.contrib.messages.views import SuccessMessageMixin
+from django.core.cache import cache
 from django.db.models import Q
 from django.shortcuts import get_object_or_404, reverse
 from django.utils import timezone
@@ -27,7 +29,7 @@ class DashboardView(LoginRequiredMixin, DateRangeMixin, TemplateView):
         data = super().get_context_data(**kwargs)
         data["services"] = Service.objects.filter(
             Q(owner=self.request.user) | Q(collaborators__in=[self.request.user])
-        )
+        ).distinct()
         for service in data["services"]:
             service.stats = service.get_core_stats(data["start_date"], data["end_date"])
         return data
@@ -77,6 +79,21 @@ class ServiceUpdateView(
     def get_success_url(self):
         return reverse("dashboard:service", kwargs={"pk": self.object.uuid})
 
+    def form_valid(self, *args, **kwargs):
+        resp = super().form_valid(*args, **kwargs)
+        cache.set(
+            f"service_origins_{self.object.uuid}", self.object.origins, timeout=3600
+        )
+        cache.set(
+            f"script_inject_{self.object.uuid}", self.object.script_inject, timeout=3600
+        )
+        return resp
+
+    def get_context_data(self, *args, **kwargs):
+        data = super().get_context_data(*args, **kwargs)
+        data["script_protocol"] = "https://" if settings.SCRIPT_USE_HTTPS else "http://"
+        return data
+
 
 class ServiceDeleteView(
     LoginRequiredMixin, PermissionRequiredMixin, SuccessMessageMixin, DeleteView
@@ -122,6 +139,9 @@ class ServiceSessionView(LoginRequiredMixin, PermissionRequiredMixin, DetailView
     context_object_name = "session"
     permission_required = "core.view_service"
 
+    def get_permission_object(self, **kwargs):
+        return self.get_object().service
+
     def get_context_data(self, **kwargs):
         data = super().get_context_data(**kwargs)
         data["object"] = get_object_or_404(Service, pk=self.kwargs.get("pk"))

shynet/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ ! $PERFORM_CHECKS_AND_SETUP == False ]]; then
+  ./startup_checks.sh && exec ./webserver.sh
+  else
+  exec ./webserver.sh
+fi

shynet/manage.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """Django's command-line utility for administrative tasks."""
 import os
 import sys

shynet/shynet/settings.py

@@ -1,7 +1,5 @@
 """
-Django settings for shynet project.
+Django settings for Shynet.
-
-Generated by 'django-admin startproject' using Django 2.2.5.
 
 For more information on this file, see
 https://docs.djangoproject.com/en/2.2/topics/settings/
@@ -12,9 +10,16 @@ https://docs.djangoproject.com/en/2.2/ref/settings/
 
 import os
 
+# import module sys to get the type of exception
+import sys
+import urllib.parse as urlparse
+
 # Messages
 from django.contrib.messages import constants as messages
 
+# Increment on new releases
+VERSION = "v0.7.3"
+
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -26,7 +31,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", "onlyusethisindev")
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = os.getenv("DEBUG", "True") == "True"
+DEBUG = os.getenv("DEBUG", "False") == "True"
 
 ALLOWED_HOSTS = os.getenv("ALLOWED_HOSTS", "*").split(",")
 
@@ -42,6 +47,9 @@ INSTALLED_APPS = [
     "django.contrib.staticfiles",
     "django.contrib.sites",
     "django.contrib.humanize",
+    "health_check",
+    "health_check.db",
+    "health_check.cache",
     "rules.apps.AutodiscoverRulesConfig",
     "a17t",
     "core",
@@ -104,9 +112,31 @@ else:
             "PASSWORD": os.environ.get("DB_PASSWORD"),
             "HOST": os.environ.get("DB_HOST"),
             "PORT": os.environ.get("DB_PORT"),
+            "OPTIONS": {"connect_timeout": 5},
         }
     }
 
+# Solution to removal of Heroku DB Injection
+if "DATABASE_URL" in os.environ:
+    if "DATABASES" not in locals():
+        DATABASES = {}
+    url = urlparse.urlparse(os.environ["DATABASE_URL"])
+
+    # Ensure default database exists.
+    DATABASES["default"] = DATABASES.get("default", {})
+
+    # Update with environment configuration.
+    DATABASES["default"].update(
+        {
+            "NAME": url.path[1:],
+            "USER": url.username,
+            "PASSWORD": url.password,
+            "HOST": url.hostname,
+            "PORT": url.port,
+        }
+    )
+    if url.scheme == "postgres":
+        DATABASES["default"]["ENGINE"] = "django.db.backends.postgresql_psycopg2"
 
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
@@ -175,6 +205,11 @@ USE_TZ = True
 STATIC_URL = "/static/"
 STATIC_ROOT = "compiledstatic/"
 STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
+STATICFILES_FINDERS = [
+    "npm.finders.NpmFinder",
+    "django.contrib.staticfiles.finders.FileSystemFinder",
+    "django.contrib.staticfiles.finders.AppDirectoriesFinder",
+]
 
 # Redis
 if not DEBUG and os.getenv("REDIS_CACHE_LOCATION") is not None:
@@ -206,6 +241,7 @@ ACCOUNT_EMAIL_VERIFICATION = "mandatory"
 ACCOUNT_EMAIL_SUBJECT_PREFIX = ""
 ACCOUNT_USER_DISPLAY = lambda k: k.email
 ACCOUNT_SIGNUPS_ENABLED = os.getenv("ACCOUNT_SIGNUPS_ENABLED", "False") == "True"
+ACCOUNT_EMAIL_VERIFICATION = os.getenv("ACCOUNT_EMAIL_VERIFICATION", "none")
 
 LOGIN_REDIRECT_URL = "/"
 
@@ -213,7 +249,7 @@ SITE_ID = 1
 
 # Celery
 
-CELERY_TASK_ALWAYS_EAGER = os.getenv("CELERY_TASK_ALWAYS_EAGER", "False") == "True"
+CELERY_TASK_ALWAYS_EAGER = os.getenv("CELERY_TASK_ALWAYS_EAGER", "True") == "True"
 
 CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL")
 CELERY_REDIS_SOCKET_TIMEOUT = 15
@@ -243,7 +279,22 @@ else:
     EMAIL_PORT = int(os.environ.get("EMAIL_PORT", 465))
     EMAIL_HOST_USER = os.environ.get("EMAIL_HOST_USER")
     EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_HOST_PASSWORD")
-    EMAIL_USE_SSL = True
+    EMAIL_USE_SSL = os.environ.get("EMAIL_USE_SSL")
+    EMAIL_USE_TLS = os.environ.get("EMAIL_USE_TLS")
+
+# NPM
+
+NPM_ROOT_PATH = "../"
+
+NPM_FILE_PATTERNS = {
+    "a17t": [os.path.join("dist", "a17t.css"), os.path.join("dist", "tailwind.css")],
+    "apexcharts": [os.path.join("dist", "apexcharts.min.js")],
+    "litepicker": [os.path.join("dist", "js", "main.js")],
+    "turbolinks": [os.path.join("dist", "turbolinks.js")],
+    "stimulus": [os.path.join("dist", "stimulus.umd.js")],
+    "inter-ui": [os.path.join("Inter (web)", "*")],
+    "@fortawesome": [os.path.join("fontawesome-free", "js", "all.min.js")],
+}
 
 # Shynet
 
@@ -256,3 +307,17 @@ ONLY_SUPERUSERS_CREATE = os.getenv("ONLY_SUPERUSERS_CREATE", "True") == "True"
 # Should the script use HTTPS to send the POST requests? The hostname is from
 # the django SITE default. (Edit it using the admin panel.)
 SCRIPT_USE_HTTPS = os.getenv("SCRIPT_USE_HTTPS", "True") == "True"
+
+# How frequently should the tracking script "phone home" with a heartbeat, in
+# milliseconds?
+SCRIPT_HEARTBEAT_FREQUENCY = int(os.getenv("SCRIPT_HEARTBEAT_FREQUENCY", "5000"))
+
+# How much time can elapse between requests from the same user before a new
+# session is created, in seconds?
+SESSION_MEMORY_TIMEOUT = int(os.getenv("SESSION_MEMORY_TIMEOUT", "1800"))
+
+# Should the Shynet version information be displayed?
+SHOW_SHYNET_VERSION = os.getenv("SHOW_SHYNET_VERSION", "True") == "True"
+
+# Should Shynet show third-party icons in the dashboard?
+SHOW_THIRD_PARTY_ICONS = os.getenv("SHOW_THIRD_PARTY_ICONS", "True") == "True"

shynet/shynet/urls.py

@@ -21,5 +21,6 @@ urlpatterns = [
     path("accounts/", include("allauth.urls")),
     path("ingress/", include(("analytics.ingress_urls", "ingress")), name="ingress"),
     path("dashboard/", include(("dashboard.urls", "dashboard"), namespace="dashboard")),
+    path("healthz/", include("health_check.urls")),
     path("", include(("core.urls", "core"), namespace="core")),
 ]

shynet/ssl.webserver.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Start Gunicorn processes
+echo Launching Shynet web server...
+exec gunicorn shynet.wsgi:application \
+    --bind 0.0.0.0:${PORT:-8080} \
+    --workers ${NUM_WORKERS:-1} \
+    --timeout 100 \
+    --certfile=cert.pem \
+    --keyfile=privkey.pem

shynet/startup_checks.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Check if setup is necessary, do setup as needed
+echo "Performing startup checks..."
+startup_results=( $(./manage.py startup_checks) )
+if [[ ${startup_results[0]} == True ]]; then
+  echo "Running migrations (setting up DB)..."
+  {
+    ./manage.py migrate && echo "Migrations complete!"
+  } || {
+    echo "Migrations failed, exiting" && exit 1
+  }
+  else
+  echo "Database is ready to go."
+fi
+if [[ ${startup_results[1]} == True ]]; then
+  echo "Warning: no admin user available. Consult docs for instructions."
+fi
+if [[ ${startup_results[2]} == True ]]; then
+  echo "Warning: Shynet's hostname is not set. The script won't work correctly. Consult docs for instructions."
+fi
+if [[ ${startup_results[3]} == True ]]; then
+  echo "Warning: Shynet's whitelabel is not set. Consult docs for instructions."
+fi
+echo "Startup checks complete!"

shynet/webserver.sh

@@ -1,8 +1,7 @@
 #!/bin/bash
-
 # Start Gunicorn processes
 echo Launching Shynet web server...
 exec gunicorn shynet.wsgi:application \
-    --bind 0.0.0.0:8080 \
+    --bind 0.0.0.0:${PORT:-8080} \
-    --workers 3 \
+    --workers ${NUM_WORKERS:-1} \
     --timeout 100

tests/js.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+
+<html>
+
+<head>
+    <title>Pixel test</title>
+</head>
+
+<body>
+    <script src="http://localhost:8000/ingress/9b2c4e2f-8d29-4418-82d4-b68e06795025/script.js"></script>
+</body>
+
+</html>

tests/pixel.html

@@ -7,8 +7,7 @@
 </head>
 
 <body>
-    <noscript><img src="//localhost:8000/ingress/211410e6-b401-4fe8-8740-7926368590be/pixel.gif"></noscript>
+    <img src="http://localhost:8000/ingress/9b2c4e2f-8d29-4418-82d4-b68e06795025/pixel.gif">
-    <script src="//localhost:8000/ingress/211410e6-b401-4fe8-8740-7926368590be/script.js"></script>
 </body>
 
 </html>