Merge branch 'dev' into identex/master

Dockerfile

@@ -30,4 +30,4 @@ USER appuser
 
 EXPOSE 8080
 
-CMD [ "./webserver.sh" ]
+ENTRYPOINT [ "./entrypoint.sh" ]

README.md

@@ -93,7 +93,8 @@ Shynet is pretty simple, but there are a few key terms you need to know in order
 
 ## Installation
 
-You can find installation instructions in the [Getting Started Guide](GUIDE.md#installation).
+You can find installation instructions in the [Getting Started Guide](GUIDE.md#installation). Out of the box, we support deploying via a simple
+Docker container, docker-compose, or Kubernetes (see [kubernetes](/kubernetes)).
 
 ## FAQ
 

TEMPLATE.env

@@ -0,0 +1,59 @@
+# This file shows all of the environment variables you can
+# set to configure Shynet, as well as information about their
+# effects. Make a copy of this file to configure your deployment.
+
+
+# Whether to perform checks and setup at startup. For most setups,
+# the recommended value is True.
+PERFORM_CHECKS_AND_SETUP=True
+
+
+# Database settings (PostgreSQL)
+DB_NAME=shynet_db
+DB_USER=shynet_db_user
+DB_PASSWORD=shynet_db_user_password
+DB_HOST=db
+DB_PORT=5432
+
+# Email settings (optional)
+EMAIL_HOST_USER=example
+EMAIL_HOST_PASSWORD=example_password
+EMAIL_HOST=smtp.example.com
+SERVER_EMAIL=<Shynet> noreply@shynet.example.com
+
+# General Django settings
+DJANGO_SECRET_KEY=random_string
+
+# For better security, set this to your deployment's domain. Comma separated.
+ALLOWED_HOSTS=*
+
+# Set to True (capitalized) if you want people to be able to sign up for your Shynet instance (not recommended)
+SIGNUPS_ENABLED=False
+
+# The timezone of the admin panel. Affects how dates are displayed.
+TIME_ZONE=America/New_York
+
+# Set to "False" if you will not be serving content over HTTPS
+SCRIPT_USE_HTTPS=True
+
+# How frequently should the monitoring script "phone home" (in ms)?
+SCRIPT_HEARTBEAT_FREQUENCY=5000
+
+# Should only superusers (admins) be able to create services? This is helpful
+# when you'd like to invite others to your Shynet instance but don't want
+# them to be able to create services of their own.
+ONLY_SUPERUSERS_CREATE=True
+
+# If PERFORM_CHECKS_AND_SETUP is True, the following values will be set on
+# first run. After they are set once, they won't have any effect.
+# (Changing these values WILL NOT affect your Shynet instance.)
+# 
+# Your admin user's email. A temporary password will be printed
+# to the console on first run.
+SHYNET_ADMIN_EMAIL=you@example.com
+
+# The domain on which you'll be hosting Shynet. 
+SHYNET_HOST=shynet.example.com
+
+# What you'd like to call your Shynet instance.
+SHYNET_WHITELABEL=My Shynet Instance

docker-compose.yml

@@ -0,0 +1,32 @@
+version: '3'
+services:
+  shynet:
+    image: milesmcc/shynet:latest
+    restart: unless-stopped
+    expose:
+      - 8080
+    env_file:
+      # Create a file called '.env' if it doesn't already exist.
+      # You can use `TEMPLATE.env` as a guide.
+      - .env
+    environment:
+      - DB_HOST=db
+    networks:
+      - internal
+    depends_on:
+      - db
+  db:
+    image: postgres
+    restart: always
+    environment:
+      - "POSTGRES_USER=${DB_USER}"
+      - "POSTGRES_PASSWORD=${DB_PASSWORD}"
+      - "POSTGRES_DB=${DB_NAME}"
+    volumes:
+      - shynet_db:/var/lib/postgresql/data
+    networks:
+      - internal
+volumes:
+  shynet_db:
+networks:
+  internal:

shynet/analytics/views/ingress.py

@@ -3,7 +3,8 @@ import json
 
 from django.conf import settings
 from django.core.cache import cache
-from django.http import HttpResponse
+from django.core.exceptions import ValidationError
+from django.http import HttpResponse, Http404, HttpResponseBadRequest
 from django.shortcuts import render, reverse
 from django.utils import timezone
 from django.utils.decorators import method_decorator
@@ -35,14 +36,37 @@ def ingress(request, service_uuid, identifier, tracker, payload):
         identifier=identifier,
     )
 
+class ValidateServiceOriginsMixin:
+    def dispatch(self, request, *args, **kwargs):
+        try:
+            service_uuid = self.kwargs.get("service_uuid")
+            origins = cache.get(f"service_origins_{service_uuid}")
 
-class PixelView(View):
+            if origins is None:
+                service = Service.objects.get(uuid=service_uuid)
+                origins = service.origins
+                cache.set(f"service_origins_{service_uuid}", origins, timeout=3600)
+
+            resp = super().dispatch(request, *args, **kwargs)
+            resp["Access-Control-Allow-Origin"] = origins
+            resp["Access-Control-Allow-Methods"] = "GET,HEAD,OPTIONS,POST"
+            resp[
+                "Access-Control-Allow-Headers"
+            ] = "Origin, X-Requested-With, Content-Type, Accept, Authorization, Referer"
+            return resp
+        except Service.DoesNotExist:
+            raise Http404()
+        except ValidationError:
+            return HttpResponseBadRequest()
+
+
+class PixelView(ValidateServiceOriginsMixin, View):
     # Fallback view to serve an unobtrusive 1x1 transparent tracking pixel for browsers with
     # JavaScript disabled.
-    def dispatch(self, request, *args, **kwargs):
+    def get(self, *args, **kwargs):
         # Extract primary data
         ingress(
-            request,
+            self.request,
             self.kwargs.get("service_uuid"),
             self.kwargs.get("identifier", ""),
             "PIXEL",
@@ -59,23 +83,7 @@ class PixelView(View):
 
 
 @method_decorator(csrf_exempt, name="dispatch")
-class ScriptView(View):
+class ScriptView(ValidateServiceOriginsMixin, View):
-    def dispatch(self, request, *args, **kwargs):
-        service_uuid = self.kwargs.get("service_uuid")
-        origins = cache.get(f"service_origins_{service_uuid}")
-        if origins is None:
-            service = Service.objects.get(uuid=service_uuid)
-            origins = service.origins
-            cache.set(f"service_origins_{service_uuid}", origins, timeout=3600)
-
-        resp = super().dispatch(request, *args, **kwargs)
-        resp["Access-Control-Allow-Origin"] = origins
-        resp["Access-Control-Allow-Methods"] = "GET,HEAD,OPTIONS,POST"
-        resp[
-            "Access-Control-Allow-Headers"
-        ] = "Origin, X-Requested-With, Content-Type, Accept, Authorization, Referer"
-        return resp
-
     def get(self, *args, **kwargs):
         protocol = "https" if settings.SCRIPT_USE_HTTPS else "http"
         endpoint = (

shynet/core/management/commands/startup_checks.py

@@ -0,0 +1,49 @@
+import traceback
+import uuid
+
+from django.conf import settings
+from django.contrib.sites.models import Site
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.crypto import get_random_string
+
+from django.db import DEFAULT_DB_ALIAS, connections
+from django.db.utils import OperationalError, ConnectionHandler
+from django.core.exceptions import ImproperlyConfigured
+
+from core.models import User
+
+
+class Command(BaseCommand):
+    help = "Internal command to perform startup sanity checks."
+
+    def check_migrations(self):
+        from django.db.migrations.executor import MigrationExecutor
+        try:
+            executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS])
+        except OperationalError:
+            # DB_NAME database not found?
+            return True
+        except ImproperlyConfigured:
+            # No databases are configured (or the dummy one)
+            return True
+        
+        if executor.migration_plan(executor.loader.graph.leaf_nodes()):
+            return True
+        
+        return False
+    
+    
+    def handle(self, *args, **options):
+        migration = self.check_migrations()
+
+        admin, hostname, whitelabel = [True] * 3
+        if not migration:
+            admin = not User.objects.all().exists()
+            hostname = not Site.objects.filter(domain__isnull=False).exclude(domain__exact="").exclude(domain__exact="example.com").exists()
+            whitelabel = not Site.objects.filter(name__isnull=False).exclude(name__exact="").exclude(name__exact="example.com").exists()
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"{migration} {admin} {hostname} {whitelabel}"
+            )
+        )

shynet/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ $PERFORM_CHECKS_AND_SETUP == True ]]; then
+  ./startup_checks.sh
+fi
+
+./webserver.sh

shynet/shynet/settings.py

@@ -27,7 +27,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", "onlyusethisindev")
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = os.getenv("DEBUG", "True") == "True"
+DEBUG = os.getenv("DEBUG", "False") == "True"
 
 ALLOWED_HOSTS = os.getenv("ALLOWED_HOSTS", "*").split(",")
 
@@ -214,7 +214,7 @@ SITE_ID = 1
 
 # Celery
 
-CELERY_TASK_ALWAYS_EAGER = os.getenv("CELERY_TASK_ALWAYS_EAGER", "False") == "True"
+CELERY_TASK_ALWAYS_EAGER = os.getenv("CELERY_TASK_ALWAYS_EAGER", "True") == "True"
 
 CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL")
 CELERY_REDIS_SOCKET_TIMEOUT = 15

shynet/startup_checks.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Check if setup is necessary, do setup as needed
+echo "Performing startup checks..."
+sanity_results=( $(./manage.py startup_checks) )
+if [[ ${sanity_results[0]} == True ]]; then
+  echo "Running migrations (setting up DB)..."
+  {
+    ./manage.py migrate && echo "Migrations complete!"
+  } || {
+    echo "Migrations failed, exiting" && exit 1
+  }
+  else
+  echo "Database is ready to go."
+fi
+if [[ -n $SHYNET_ADMIN_EMAIL && ${sanity_results[1]} == True ]]; then
+  echo "Creating an admin user..."
+  {
+    temppwd=$( ./manage.py registeradmin $SHYNET_ADMIN_EMAIL ) && echo "Admin user ($SHYNET_ADMIN_EMAIL) created! Password: $temppwd"
+  } || {
+    echo "Failed to create admin, exiting" & exit 1
+  }
+  else
+  echo "Making no changes to admin user."
+fi
+if [[ -n $SHYNET_HOST && ${sanity_results[2]} == True ]]; then
+  echo "Setting hostname..."
+  {
+    ./manage.py hostname $SHYNET_HOST && echo "Hostname set to $SHYNET_HOST!"
+  } || {
+    echo "Failed setting hostname, exiting" & exit 1
+  }
+  else
+  echo "Making no changes to hostname."
+fi
+if [[ -n $SHYNET_WHITELABEL && ${sanity_results[3]} == True ]]; then
+  echo "Setting whitelabel..."
+  {
+    ./manage.py whitelabel $SHYNET_WHITELABEL && echo "Whitelabel set! Whitelabel: $SHYNET_WHITELABEL"
+  } || {
+    echo "Failed to set whitelabel, exiting" & exit 1
+  }
+  else
+  echo "Making no changes to whitelabel."
+fi
+echo "Startup checks complete!"

shynet/webserver.sh

@@ -1,8 +1,7 @@
 #!/bin/bash
-
 # Start Gunicorn processes
 echo Launching Shynet web server...
 exec gunicorn shynet.wsgi:application \
     --bind 0.0.0.0:8080 \
     --workers 3 \
-    --timeout 100
+    --timeout 100