From 6fa67f0531c4962498c5b048404e891830158978 Mon Sep 17 00:00:00 2001 From: Windyo Date: Thu, 30 Apr 2020 22:59:04 +0200 Subject: [PATCH 1/2] Docker-Compose Single-Run Add firstrun checks Avoids running commands on firstrun Add Docker-compose file allows setup in a single docker-compose up command Updated Readme re: docker-compose Added stub detailing how to use the compose file. Update README.md Changed Entrypoint Moved sanity checks to their own script, changed entrypoint logic updated entrypoint --- Dockerfile | 2 +- README.md | 6 +- docker-compose.yml | 64 +++++++++++++++++++ .../core/management/commands/sanity_checks.py | 49 ++++++++++++++ shynet/entrypoint.sh | 44 +++++++++++++ shynet/webserver.sh | 3 +- 6 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 docker-compose.yml create mode 100644 shynet/core/management/commands/sanity_checks.py create mode 100755 shynet/entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 690bb2b..58e61a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,4 +30,4 @@ USER appuser EXPOSE 8080 -CMD [ "./webserver.sh" ] \ No newline at end of file +ENTRYPOINT [ "./entrypoint.sh" ] \ No newline at end of file diff --git a/README.md b/README.md index df2c8b5..6aae24b 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,11 @@ Shynet is pretty simple, but there are a few key terms you need to know in order ## Installation -You can find installation instructions in the [Getting Started Guide](GUIDE.md#installation). +If you use docker-compose, you can use the docker-compose.yml file to set up shynet and launch it by doing docker-compose up. +Note that the application will generate a password for you on initial run, which will be output to standard docker logs. +Also note that this compose file assumes you use a reverse-proxy in front of it, whether Traefik v2, Nginx-Proxy, or a solution on your host. + +You can find further installation instructions in the [Getting Started Guide](GUIDE.md#installation). ## FAQ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7d81743 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,64 @@ +version: '3' +services: + shynet: + image: milesmcc/shynet:latest + restart: unless-stopped + expose: + - 8080 + environment: + # Database Settings + - DB_NAME=shynet_DBname + - DB_USER=shynet_User + - DB_PASSWORD=shynet_RandomPassword + # Database connection, relies on DB service + - DB_HOST=db + - DB_PORT=5432 + # General Django settings + - DJANGO_SECRET_KEY=shynet_OtherRandomPassword + # Don't leak error details to visitors, very important + - DEBUG=False + # Unless you are using an external Celery task queue, make sure this + # is set to True. + - CELERY_TASK_ALWAYS_EAGER=True + # For better security, set this to your deployment's domain. Comma separated. + - ALLOWED_HOSTS=* + # Set to True (capitalized) if you want people to be able to sign up for your Shynet instance (not recommended) + - SIGNUPS_ENABLED=False + # Change as required + - TIME_ZONE=America/New_York + # Set to "False" if you will not be serving content over HTTPS + - SCRIPT_USE_HTTPS=True + # Email settings + - EMAIL_HOST_USER=youruser@yourhost.example.com + - EMAIL_HOST_PASSWORD=youruser_password + - EMAIL_HOST=host.email.provider + - SERVER_EMAIL=noreply.analytics@shynet.example.com + # Redis and queue settings; not necessary for single-instance deployments + #- REDIS_CACHE_LOCATION=redis://redis.default.svc.cluster.local/0 + # If set, make sure CELERY_TASK_ALWAYS_EAGER is False + #- CELERY_BROKER_URL=redis://redis.default.svc.cluster.local/1 + # How frequently should the monitoring script "phone home" (in ms)? + - SCRIPT_HEARTBEAT_FREQUENCY=5000 + # Should only superusers (admins) be able to create services? This is helpful + # when you'd like to invite others to your Shynet instance but don't want + # them to be able to create services of their own. + - ONLY_SUPERUSERS_CREATE=True + networks: + - internal + depends_on: + - db + db: + image: postgres + restart: always + environment: + - POSTGRES_USER=shynet_User + - POSTGRES_PASSWORD=shynet_RandomPassword + - POSTGRES_DB=shynet_DBname + volumes: + - shynet_db:/var/lib/postgresql/data + networks: + - internal +volumes: + shynet_db: +networks: + internal: diff --git a/shynet/core/management/commands/sanity_checks.py b/shynet/core/management/commands/sanity_checks.py new file mode 100644 index 0000000..c959297 --- /dev/null +++ b/shynet/core/management/commands/sanity_checks.py @@ -0,0 +1,49 @@ +import traceback +import uuid + +from django.conf import settings +from django.contrib.sites.models import Site +from django.core.management.base import BaseCommand, CommandError +from django.utils.crypto import get_random_string + +from django.db import DEFAULT_DB_ALIAS, connections +from django.db.utils import OperationalError, ConnectionHandler +from django.core.exceptions import ImproperlyConfigured + +from core.models import User + + +class Command(BaseCommand): + help = "Performs sanity checks on the Shynet setup" + + def check_migrations(self): + from django.db.migrations.executor import MigrationExecutor + try: + executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS]) + except OperationalError: + # DB_NAME database not found? + return True + except ImproperlyConfigured: + # No databases are configured (or the dummy one) + return True + + if executor.migration_plan(executor.loader.graph.leaf_nodes()): + return True + + return False + + + def handle(self, *args, **options): + migration = self.check_migrations() + + admin, hostname, whitelabel = [True] * 3 + if not migration: + admin = not User.objects.all().exists() + hostname = not Site.objects.filter(domain__isnull=False).exclude(domain__exact="").exclude(domain__exact="example.com").exists() + whitelabel = not Site.objects.filter(name__isnull=False).exclude(name__exact="").exclude(name__exact="example.com").exists() + + self.stdout.write( + self.style.SUCCESS( + f"{migration} {admin} {hostname} {whitelabel}" + ) + ) diff --git a/shynet/entrypoint.sh b/shynet/entrypoint.sh new file mode 100755 index 0000000..096437a --- /dev/null +++ b/shynet/entrypoint.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Check if Setup is necessary, do setup as needed +sanity_results=( $(python manage.py sanity_checks) ) +if [[ ${sanity_results[0]} == True ]]; then + echo "Running Migrations..." + { + python manage.py migrate && echo "Migrations Done" + } || { + echo "Failed Migrations, exiting" && exit 1 + } + else + echo "Migrations Unecessary, skipping" +fi +if [[ ${sanity_results[1]} == True ]]; then + echo "Running CreateAdmin..." + { + temppwd=$( python manage.py registeradmin $SHYNET_EMAIL ) && echo "Admin Created, password $temppwd" + } || { + echo "Failed CreateAdmin, exiting" & exit 1 + } + else + echo "CreateAdmin Unecessary, skipping" +fi +if [[ ${sanity_results[2]} == True ]]; then + echo "Setting Hostname..." + { + python manage.py hostname $SHYNET_HOST && echo "Host Set" + } || { + echo "Failed setting Hostname, exiting" & exit 1 + } + else + echo "Hostname Unecessary, skipping" +fi +if [[ ${sanity_results[3]} == True ]]; then + echo "Setting Hostname..." + { + python manage.py whitelabel $SHYNET_NAME && echo "WhiteLabel Set" + } || { + echo "Failed Migrations, exiting" & exit 1 + } + else + echo "WhiteLabel Unecessary, skipping" +fi +./webserver.sh diff --git a/shynet/webserver.sh b/shynet/webserver.sh index a8478ac..5e092a7 100755 --- a/shynet/webserver.sh +++ b/shynet/webserver.sh @@ -1,8 +1,7 @@ #!/bin/bash - # Start Gunicorn processes echo Launching Shynet web server... exec gunicorn shynet.wsgi:application \ --bind 0.0.0.0:8080 \ --workers 3 \ - --timeout 100 \ No newline at end of file + --timeout 100 From 5d26ab292b055c0516c4850f26d08bc3d870bd0c Mon Sep 17 00:00:00 2001 From: "R. Miles McCain" Date: Sat, 2 May 2020 09:25:37 -0400 Subject: [PATCH 2/2] Refactoring & consistency changes Make all scripts executable Disable debug mode by default Use eager tasks by default Fix typo in settings Refactoring --- README.md | 7 +-- TEMPLATE.env | 59 +++++++++++++++++++ docker-compose.yml | 46 +++------------ .../{sanity_checks.py => startup_checks.py} | 2 +- shynet/entrypoint.sh | 45 ++------------ shynet/shynet/settings.py | 4 +- shynet/ssl.webserver.sh | 0 shynet/startup_checks.sh | 45 ++++++++++++++ 8 files changed, 120 insertions(+), 88 deletions(-) create mode 100644 TEMPLATE.env rename shynet/core/management/commands/{sanity_checks.py => startup_checks.py} (96%) mode change 100644 => 100755 shynet/ssl.webserver.sh create mode 100755 shynet/startup_checks.sh diff --git a/README.md b/README.md index 6aae24b..578576f 100644 --- a/README.md +++ b/README.md @@ -93,11 +93,8 @@ Shynet is pretty simple, but there are a few key terms you need to know in order ## Installation -If you use docker-compose, you can use the docker-compose.yml file to set up shynet and launch it by doing docker-compose up. -Note that the application will generate a password for you on initial run, which will be output to standard docker logs. -Also note that this compose file assumes you use a reverse-proxy in front of it, whether Traefik v2, Nginx-Proxy, or a solution on your host. - -You can find further installation instructions in the [Getting Started Guide](GUIDE.md#installation). +You can find installation instructions in the [Getting Started Guide](GUIDE.md#installation). Out of the box, we support deploying via a simple +Docker container, docker-compose, or Kubernetes (see [kubernetes](/kubernetes)). ## FAQ diff --git a/TEMPLATE.env b/TEMPLATE.env new file mode 100644 index 0000000..5dcaa1b --- /dev/null +++ b/TEMPLATE.env @@ -0,0 +1,59 @@ +# This file shows all of the environment variables you can +# set to configure Shynet, as well as information about their +# effects. Make a copy of this file to configure your deployment. + + +# Whether to perform checks and setup at startup. For most setups, +# the recommended value is True. +PERFORM_CHECKS_AND_SETUP=True + + +# Database settings (PostgreSQL) +DB_NAME=shynet_db +DB_USER=shynet_db_user +DB_PASSWORD=shynet_db_user_password +DB_HOST=db +DB_PORT=5432 + +# Email settings (optional) +EMAIL_HOST_USER=example +EMAIL_HOST_PASSWORD=example_password +EMAIL_HOST=smtp.example.com +SERVER_EMAIL= noreply@shynet.example.com + +# General Django settings +DJANGO_SECRET_KEY=random_string + +# For better security, set this to your deployment's domain. Comma separated. +ALLOWED_HOSTS=* + +# Set to True (capitalized) if you want people to be able to sign up for your Shynet instance (not recommended) +SIGNUPS_ENABLED=False + +# The timezone of the admin panel. Affects how dates are displayed. +TIME_ZONE=America/New_York + +# Set to "False" if you will not be serving content over HTTPS +SCRIPT_USE_HTTPS=True + +# How frequently should the monitoring script "phone home" (in ms)? +SCRIPT_HEARTBEAT_FREQUENCY=5000 + +# Should only superusers (admins) be able to create services? This is helpful +# when you'd like to invite others to your Shynet instance but don't want +# them to be able to create services of their own. +ONLY_SUPERUSERS_CREATE=True + +# If PERFORM_CHECKS_AND_SETUP is True, the following values will be set on +# first run. After they are set once, they won't have any effect. +# (Changing these values WILL NOT affect your Shynet instance.) +# +# Your admin user's email. A temporary password will be printed +# to the console on first run. +SHYNET_ADMIN_EMAIL=you@example.com + +# The domain on which you'll be hosting Shynet. +SHYNET_HOST=shynet.example.com + +# What you'd like to call your Shynet instance. +SHYNET_WHITELABEL=My Shynet Instance \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 7d81743..80fd9a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,44 +5,12 @@ services: restart: unless-stopped expose: - 8080 + env_file: + # Create a file called '.env' if it doesn't already exist. + # You can use `TEMPLATE.env` as a guide. + - .env environment: - # Database Settings - - DB_NAME=shynet_DBname - - DB_USER=shynet_User - - DB_PASSWORD=shynet_RandomPassword - # Database connection, relies on DB service - DB_HOST=db - - DB_PORT=5432 - # General Django settings - - DJANGO_SECRET_KEY=shynet_OtherRandomPassword - # Don't leak error details to visitors, very important - - DEBUG=False - # Unless you are using an external Celery task queue, make sure this - # is set to True. - - CELERY_TASK_ALWAYS_EAGER=True - # For better security, set this to your deployment's domain. Comma separated. - - ALLOWED_HOSTS=* - # Set to True (capitalized) if you want people to be able to sign up for your Shynet instance (not recommended) - - SIGNUPS_ENABLED=False - # Change as required - - TIME_ZONE=America/New_York - # Set to "False" if you will not be serving content over HTTPS - - SCRIPT_USE_HTTPS=True - # Email settings - - EMAIL_HOST_USER=youruser@yourhost.example.com - - EMAIL_HOST_PASSWORD=youruser_password - - EMAIL_HOST=host.email.provider - - SERVER_EMAIL=noreply.analytics@shynet.example.com - # Redis and queue settings; not necessary for single-instance deployments - #- REDIS_CACHE_LOCATION=redis://redis.default.svc.cluster.local/0 - # If set, make sure CELERY_TASK_ALWAYS_EAGER is False - #- CELERY_BROKER_URL=redis://redis.default.svc.cluster.local/1 - # How frequently should the monitoring script "phone home" (in ms)? - - SCRIPT_HEARTBEAT_FREQUENCY=5000 - # Should only superusers (admins) be able to create services? This is helpful - # when you'd like to invite others to your Shynet instance but don't want - # them to be able to create services of their own. - - ONLY_SUPERUSERS_CREATE=True networks: - internal depends_on: @@ -51,9 +19,9 @@ services: image: postgres restart: always environment: - - POSTGRES_USER=shynet_User - - POSTGRES_PASSWORD=shynet_RandomPassword - - POSTGRES_DB=shynet_DBname + - "POSTGRES_USER=${DB_USER}" + - "POSTGRES_PASSWORD=${DB_PASSWORD}" + - "POSTGRES_DB=${DB_NAME}" volumes: - shynet_db:/var/lib/postgresql/data networks: diff --git a/shynet/core/management/commands/sanity_checks.py b/shynet/core/management/commands/startup_checks.py similarity index 96% rename from shynet/core/management/commands/sanity_checks.py rename to shynet/core/management/commands/startup_checks.py index c959297..15da2d3 100644 --- a/shynet/core/management/commands/sanity_checks.py +++ b/shynet/core/management/commands/startup_checks.py @@ -14,7 +14,7 @@ from core.models import User class Command(BaseCommand): - help = "Performs sanity checks on the Shynet setup" + help = "Internal command to perform startup sanity checks." def check_migrations(self): from django.db.migrations.executor import MigrationExecutor diff --git a/shynet/entrypoint.sh b/shynet/entrypoint.sh index 096437a..a97bb97 100755 --- a/shynet/entrypoint.sh +++ b/shynet/entrypoint.sh @@ -1,44 +1,7 @@ #!/bin/bash -# Check if Setup is necessary, do setup as needed -sanity_results=( $(python manage.py sanity_checks) ) -if [[ ${sanity_results[0]} == True ]]; then - echo "Running Migrations..." - { - python manage.py migrate && echo "Migrations Done" - } || { - echo "Failed Migrations, exiting" && exit 1 - } - else - echo "Migrations Unecessary, skipping" -fi -if [[ ${sanity_results[1]} == True ]]; then - echo "Running CreateAdmin..." - { - temppwd=$( python manage.py registeradmin $SHYNET_EMAIL ) && echo "Admin Created, password $temppwd" - } || { - echo "Failed CreateAdmin, exiting" & exit 1 - } - else - echo "CreateAdmin Unecessary, skipping" -fi -if [[ ${sanity_results[2]} == True ]]; then - echo "Setting Hostname..." - { - python manage.py hostname $SHYNET_HOST && echo "Host Set" - } || { - echo "Failed setting Hostname, exiting" & exit 1 - } - else - echo "Hostname Unecessary, skipping" -fi -if [[ ${sanity_results[3]} == True ]]; then - echo "Setting Hostname..." - { - python manage.py whitelabel $SHYNET_NAME && echo "WhiteLabel Set" - } || { - echo "Failed Migrations, exiting" & exit 1 - } - else - echo "WhiteLabel Unecessary, skipping" + +if [[ $PERFORM_CHECKS_AND_SETUP == True ]]; then + ./startup_checks.sh fi + ./webserver.sh diff --git a/shynet/shynet/settings.py b/shynet/shynet/settings.py index 531a40c..c90d633 100644 --- a/shynet/shynet/settings.py +++ b/shynet/shynet/settings.py @@ -27,7 +27,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", "onlyusethisindev") # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = os.getenv("DEBUG", "True") == "True" +DEBUG = os.getenv("DEBUG", "False") == "True" ALLOWED_HOSTS = os.getenv("ALLOWED_HOSTS", "*").split(",") @@ -214,7 +214,7 @@ SITE_ID = 1 # Celery -CELERY_TASK_ALWAYS_EAGER = os.getenv("CELERY_TASK_ALWAYS_EAGER", "False") == "True" +CELERY_TASK_ALWAYS_EAGER = os.getenv("CELERY_TASK_ALWAYS_EAGER", "True") == "True" CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL") CELERY_REDIS_SOCKET_TIMEOUT = 15 diff --git a/shynet/ssl.webserver.sh b/shynet/ssl.webserver.sh old mode 100644 new mode 100755 diff --git a/shynet/startup_checks.sh b/shynet/startup_checks.sh new file mode 100755 index 0000000..0363144 --- /dev/null +++ b/shynet/startup_checks.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Check if setup is necessary, do setup as needed +echo "Performing startup checks..." +sanity_results=( $(./manage.py startup_checks) ) +if [[ ${sanity_results[0]} == True ]]; then + echo "Running migrations (setting up DB)..." + { + ./manage.py migrate && echo "Migrations complete!" + } || { + echo "Migrations failed, exiting" && exit 1 + } + else + echo "Database is ready to go." +fi +if [[ -n $SHYNET_ADMIN_EMAIL && ${sanity_results[1]} == True ]]; then + echo "Creating an admin user..." + { + temppwd=$( ./manage.py registeradmin $SHYNET_ADMIN_EMAIL ) && echo "Admin user ($SHYNET_ADMIN_EMAIL) created! Password: $temppwd" + } || { + echo "Failed to create admin, exiting" & exit 1 + } + else + echo "Making no changes to admin user." +fi +if [[ -n $SHYNET_HOST && ${sanity_results[2]} == True ]]; then + echo "Setting hostname..." + { + ./manage.py hostname $SHYNET_HOST && echo "Hostname set to $SHYNET_HOST!" + } || { + echo "Failed setting hostname, exiting" & exit 1 + } + else + echo "Making no changes to hostname." +fi +if [[ -n $SHYNET_WHITELABEL && ${sanity_results[3]} == True ]]; then + echo "Setting whitelabel..." + { + ./manage.py whitelabel $SHYNET_WHITELABEL && echo "Whitelabel set! Whitelabel: $SHYNET_WHITELABEL" + } || { + echo "Failed to set whitelabel, exiting" & exit 1 + } + else + echo "Making no changes to whitelabel." +fi +echo "Startup checks complete!" \ No newline at end of file