From e34e135a844260717509afdb8ccebb36a7a36528 Mon Sep 17 00:00:00 2001 From: The Dod Date: Thu, 14 Jan 2021 15:53:58 +0200 Subject: [PATCH] Use cloud db (and optional S3) Also make settings more env-configurable, based on https://www.digitalocean.com/community/tutorials/how-to-build-a-django-and-gunicorn-application-with-docker --- .gitignore | 1 + dev.env | 7 +- djangoproject/Dockerfile | 4 +- djangoproject/djangoproject/settings.py | 99 +++++++++++++++++-------- djangoproject/entrypoint.sh | 18 +++-- djangoproject/requirements.txt | 6 ++ docker-compose.yml | 13 ---- project.env.example | 25 +++++-- 8 files changed, 110 insertions(+), 63 deletions(-) diff --git a/.gitignore b/.gitignore index f03cb44..1cb414d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ project.env venv staticfiles +*.backup __pycache__ *.pyc *.sqlite3 diff --git a/dev.env b/dev.env index 8a7e282..293b0ae 100644 --- a/dev.env +++ b/dev.env @@ -1,2 +1,7 @@ export ENVIRONMENT=dev -export SECRET_KEY=nevermindthisisdev +export DJANGO_SECRET_KEY='nevermindthisisdev' +export DJANGO_DEBUG=True +export DJANGO_ALLOWED_HOSTS='localhost,127.0.0.1' +export DATABASE_ENGINE=sqlite3 +export DATABASE_NAME=db.sqlite3 +export DJANGO_LOGLEVEL=info diff --git a/djangoproject/Dockerfile b/djangoproject/Dockerfile index a22e29e..5f842a3 100644 --- a/djangoproject/Dockerfile +++ b/djangoproject/Dockerfile @@ -14,9 +14,9 @@ RUN useradd --user-group --create-home --no-log-init --shell /bin/bash app ENV APP_HOME=/home/app/web -# Create the staticfiles directory. This avoids permission errors. +# Create the staticfiles directory. RUN mkdir -p $APP_HOME/staticfiles -RUN chown app:app $APP_HOME/staticfiles +RUN chown -R app:app $APP_HOME/staticfiles # Change the workdir. WORKDIR $APP_HOME diff --git a/djangoproject/djangoproject/settings.py b/djangoproject/djangoproject/settings.py index 2fdd8bb..ba0ccb0 100644 --- a/djangoproject/djangoproject/settings.py +++ b/djangoproject/djangoproject/settings.py @@ -11,6 +11,8 @@ https://docs.djangoproject.com/en/3.1/ref/settings/ """ import os +import json +import logging.config from pathlib import Path # Build paths inside the project like this: BASE_DIR / 'subdir'. @@ -18,20 +20,16 @@ BASE_DIR = Path(__file__).resolve().parent.parent ENVIRONMENT = os.getenv('ENVIRONMENT', 'production') - -from djangoproject.local_settings import Database, Secrets - - # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = Secrets.SECRET_KEY +SECRET_KEY = os.getenv('DJANGO_SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = ENVIRONMENT.lower()=='dev' +DEBUG = os.getenv('DJANGO_DEBUG', False) -ALLOWED_HOSTS = ["localhost", "127.0.0.1", "0.0.0.0"] +ALLOWED_HOSTS = os.getenv('DJANGO_ALLOWED_HOSTS', '127.0.0.1,localhost').split(',') # Application definition @@ -76,28 +74,21 @@ TEMPLATES = [ WSGI_APPLICATION = 'djangoproject.wsgi.application' -# Database -# https://docs.djangoproject.com/en/3.1/ref/settings/#databases -if DEBUG: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'db.sqlite3', - } - } -else: - - DATABASES = { - "default": { - "ENGINE": "django.db.backends.postgresql", - "NAME": Database.NAME, - "USER": Database.USER, - "PASSWORD": Database.PASSWORD, - "HOST": Database.HOST, - "PORT": Database.PORT, - } -} - +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.{}'.format( + os.getenv('DATABASE_ENGINE', 'sqlite3') + ), + 'NAME': os.getenv('DATABASE_NAME'), + 'USER': os.getenv('DATABASE_USERNAME'), + 'PASSWORD': os.getenv('DATABASE_PASSWORD'), + 'HOST': os.getenv('DATABASE_HOST'), + 'PORT': os.getenv('DATABASE_PORT'), + 'OPTIONS': json.loads( + os.getenv('DATABASE_OPTIONS', '{}') + ), + } + } # Password validation # https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators @@ -133,7 +124,51 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.1/howto/static-files/ +# Either S3, or local files (served via nginx) +if os.getenv('STATIC_ACCESS_KEY_ID'): + INSTALLED_APPS.append('storages') + STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + AWS_ACCESS_KEY_ID = os.getenv('STATIC_ACCESS_KEY_ID') + AWS_SECRET_ACCESS_KEY = os.getenv('STATIC_SECRET_KEY') + AWS_STORAGE_BUCKET_NAME = os.getenv('STATIC_BUCKET_NAME') + AWS_S3_ENDPOINT_URL = os.getenv('STATIC_ENDPOINT_URL') + AWS_S3_OBJECT_PARAMETERS = { + 'CacheControl': 'max-age=86400', + } + AWS_LOCATION = 'billboard_static' + AWS_DEFAULT_ACL = 'public-read' + STATIC_URL = '{}/{}/'.format(AWS_S3_ENDPOINT_URL, AWS_LOCATION) +else: + STATIC_URL = '/static/' + STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles") -STATIC_URL = '/static/' -STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles") +# Logging Configuration +# (make sure we always log to console) + +# Clear prev config +LOGGING_CONFIG = None + +# Get loglevel from env +LOGLEVEL = os.getenv('DJANGO_LOGLEVEL', 'info').upper() + +logging.config.dictConfig({ + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'console': { + 'format': '%(asctime)s %(levelname)s [%(name)s:%(lineno)s] %(module)s %(process)d %(thread)d %(message)s', + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'console', + }, + }, + 'loggers': { + '': { + 'level': LOGLEVEL, + 'handlers': ['console',], + }, + }, +}) diff --git a/djangoproject/entrypoint.sh b/djangoproject/entrypoint.sh index 13e596e..ba26b27 100755 --- a/djangoproject/entrypoint.sh +++ b/djangoproject/entrypoint.sh @@ -9,11 +9,15 @@ if [ "$DATABASE" = "postgres" ]; then echo "PostgreSQL started" fi -# Make migrations and migrate the database. -echo "Making migrations and migrating the database" -python manage.py makemigrations --noinput -python manage.py migrate --noinput -echo "collectiong static files" -python manage.py collectstatic --noinput +if false; then + # Make migrations and migrate the database. + echo "Making migrations and migrating the database" + python manage.py makemigrations --noinput + python manage.py migrate --noinput + echo "collecting static files" + python manage.py collectstatic --noinput +fi -exec "$@" +#exec "$@" + +gunicorn djangoproject.wsgi:application --bind 0.0.0.0:8000 --workers=4 diff --git a/djangoproject/requirements.txt b/djangoproject/requirements.txt index 4f9431b..492dba8 100644 --- a/djangoproject/requirements.txt +++ b/djangoproject/requirements.txt @@ -1,5 +1,7 @@ appdirs==1.4.3 asgiref==3.3.1 +boto3==1.16.54 +botocore==1.19.54 CacheControl==0.12.6 certifi==2019.11.28 chardet==3.0.4 @@ -8,10 +10,12 @@ contextlib2==0.6.0 distlib==0.3.0 distro==1.4.0 Django==3.1.5 +django-storages==1.11.1 gunicorn==20.0.4 html5lib==1.0.1 idna==2.8 ipaddr==2.2.0 +jmespath==0.10.0 lockfile==0.12.2 msgpack==0.6.2 packaging==20.3 @@ -19,10 +23,12 @@ pep517==0.8.2 progress==1.5 psycopg2-binary==2.8.6 pyparsing==2.4.6 +python-dateutil==2.8.1 pytoml==0.1.21 pytz==2020.5 requests==2.22.0 retrying==1.3.3 +s3transfer==0.3.4 six==1.14.0 sqlparse==0.4.1 urllib3==1.25.8 diff --git a/docker-compose.yml b/docker-compose.yml index 1c7c719..4afd85f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,26 +1,13 @@ version: '3' services: - db: - container_name: postgresdb - image: postgres:latest - restart: always - env_file: - - project.env - ports: - - 5432:5432 - volumes: - - postgres-data:/var/lib/postgresql/data web: container_name: django build: djangoproject/ - command: gunicorn djangoproject.wsgi:application --bind 0.0.0.0:8000 --workers=4 env_file: - project.env expose: - 8000 - depends_on: - - db volumes: - staticfiles:/home/app/web/staticfiles nginx: diff --git a/project.env.example b/project.env.example index 2da5444..79303cf 100644 --- a/project.env.example +++ b/project.env.example @@ -1,8 +1,17 @@ -ENVIRONMENT=production -SECRET_KEY=verystrongsecretkey -POSTGRES_USER=dbadmin -POSTGRES_PASSWORD=verysecretdbpassword -POSTGRES_DB=project_db -DATABASE=postgres -DATABASE_HOST=postgresdb -DATABASE_PORT=5432 +# To generate a strong secret key: +# python3 -c "import secrets; print(secrets.token_urlsafe(64))" +DJANGO_SECRET_KEY= +DEBUG= +DJANGO_LOGLEVEL=info +DJANGO_ALLOWED_HOSTS= +DATABASE_ENGINE=postgresql_psycopg2 +DATABASE_NAME=billboard +DATABASE_USERNAME=billboard +DATABASE_PASSWORD= +DATABASE_HOST= +DATABASE_PORT= +# If you don't want S3 storage, ommit STATIC_* lines +STATIC_ACCESS_KEY_ID= +STATIC_SECRET_KEY= +STATIC_BUCKET_NAME= +STATIC_ENDPOINT_URL=