Docker deployment

2022-03-29 10:54:07 +02:00
parent a674d85afd
commit 5ee4a7b1a2
11 changed files with 180 additions and 90 deletions
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -0,0 +1,39 @@
+name: Deploy master
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build:
+    name: Build image
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - run: docker login ghcr.io -u $GITHUB_ACTOR -p ${{ secrets.GHCR_TOKEN }}
+      - run: docker build -t ghcr.io/$GITHUB_ACTOR/infomate:latest -t ghcr.io/$GITHUB_ACTOR/infomate:$GITHUB_SHA .
+      - run: docker image push ghcr.io/$GITHUB_ACTOR/infomate:$GITHUB_SHA
+      - run: docker image push ghcr.io/$GITHUB_ACTOR/infomate:latest
+
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      SSH_KEY_PATH: /tmp/ssh_key
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Make envfile
+        run: export | grep "secret_" | sed "s/declare -x secret_//" > .env
+        env:
+          secret_SECRET_KEY: ${{ secrets.SECRET_KEY }}
+          secret_APP_HOST: ${{ secrets.APP_HOST }}
+          secret_MEDIA_UPLOAD_CODE: ${{ secrets.MEDIA_UPLOAD_CODE }}
+          secret_SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+      - run: echo "GITHUB_SHA=$GITHUB_SHA" >> .env
+      - run: echo "${{ secrets.PRODUCTION_SSH_KEY }}" > ${{ env.SSH_KEY_PATH }} && chmod 600 ${{ env.SSH_KEY_PATH }}
+      - run: scp -o StrictHostKeyChecking=no -i ${{ env.SSH_KEY_PATH }} .env ${{ secrets.PRODUCTION_SSH_USERNAME }}@${{ secrets.PRODUCTION_SSH_HOST }}:/home/vas3k/infomate.club/.env
+      - run: scp -o StrictHostKeyChecking=no -i ${{ env.SSH_KEY_PATH }} docker-compose.production.yml ${{ secrets.PRODUCTION_SSH_USERNAME }}@${{ secrets.PRODUCTION_SSH_HOST }}:/home/vas3k/infomate.club/docker-compose.production.yml
+      - run: ssh -i ${{ env.SSH_KEY_PATH }} ${{ secrets.PRODUCTION_SSH_USERNAME }}@${{ secrets.PRODUCTION_SSH_HOST }} "cd /home/vas3k/infomate.club && docker login ghcr.io -u $GITHUB_ACTOR -p ${{ secrets.GHCR_TOKEN }} && docker pull ghcr.io/$GITHUB_ACTOR/infomate:$GITHUB_SHA && docker-compose -f docker-compose.production.yml --env-file=.env up -d && docker system prune --all --force"
--- a/36
+++ b/36
@@ -5,22 +5,30 @@

 PROJECT_NAME=infomate

-dev-requirements:  ## Install dev requirements
-	@pip3 install -r requirements.txt
+run:  ## Run dev server
+	python3 manage.py migrate
+	python3 manage.py runserver 0.0.0.0:8000

-docker_run:  ## Run dev server in docker
-	@python3 ./utils/wait_for_postgres.py
-	@python3 manage.py migrate
-	@python3 manage.py runserver 0.0.0.0:8000
+dev-requirements:  ## Install dev requirements
+	pip3 install -r requirements.txt
+
+docker-run-app:  ## Run production setup in docker
+	python3 ./utils/wait_for_postgres.py
+	python3 manage.py migrate
+	gunicorn infomate.asgi:application -w 3 -k uvicorn.workers.UvicornWorker --bind=0.0.0.0:8816 --capture-output --log-level debug --access-logfile - --error-logfile -
+
+docker-run-cron:  ## Run production cron container
+	env >> /etc/environment
+	cron -f -l 2

 feed_cleanup:  ## Cleanup RSS feeds
-	@python3 ./scripts/cleanup.py
+	python3 ./scripts/cleanup.py

 feed_init:  ## Initialize feeds from boards.yml
-	@python3 ./scripts/initialize.py --config boards.yml --no-upload-favicons -y
+	python3 ./scripts/initialize.py --config boards.yml --no-upload-favicons -y

 feed_refresh:  ## Refresh RSS feeds
-	@python3 ./scripts/update.py
+	python3 ./scripts/update.py

 help:  ## Display this help
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \
@@ -31,16 +39,16 @@ lint:  ## Lint code with flake8
 	flake8 $(PROJECT_NAME)

 migrate:  ## Migrate database to the latest version
-	@python3 manage.py migrate
+	python3 manage.py migrate

 mypy:  ## Check types with mypy
 	mypy $(PROJECT_NAME)

 run:  ## Runs dev server
-	@python3 manage.py runserver
+	python3 manage.py runserver

 telegram:
-	@python3 setup_telegram.py
+	python3 setup_telegram.py

 test-ci: test-requirements lint mypy  ## Run tests (intended for CI usage)

@@ -48,8 +56,10 @@ test-requirements:  ## Install requirements to run tests
 	@pip3 install -r ./requirements-test.txt

 .PHONY: \
+  run \
  dev-requirements \
-  docker_run \
+  docker-run-app \
+  docker-run-cron \
  feed_cleanup \
  feed_init \
  feed_refresh \
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Infomate.club

-[![Build Status](https://travis-ci.org/vas3k/infomate.club.svg?branch=master)](https://travis-ci.org/vas3k/infomate.club) [![GitHub license](https://img.shields.io/github/license/vas3k/infomate.club)](https://github.com/vas3k/infomate.club/blob/master/LICENSE) [![GitHub contributors](https://img.shields.io/github/contributors/vas3k/infomate.club)](https://GitHub.com/vas3k/infomate.club/graphs/contributors/)
+[![Build Status](https://travis-ci.org/vas3k/infomate.club.svg?branch=master)](https://travis-ci.org/vas3k/infomate.club)

 Infomate is a small web service that shows multiple RSS sources on one page and performs tricky parsing and summarizing articles using TextRank algorithm. 

@@ -128,6 +128,27 @@ boards:
            word: Trump   # exclude articles with a word "Trump" in title
 ```

+## Running in production
+
+Deployment is done using a simple Github Action which builds a docker container, puts it into Github Registry, logs into your server via SSH and pulls it. 
+The pipeline is triggered on every push to master branch. If you want to set up your own fork, please add these constants to your repo SECRETS:
+
+```
+APP_HOST — e.g. "https://your.host.com"
+GHCR_TOKEN — your personal guthib access token with permissions to read/write into Github Registry
+SECRET_KEY — random string for django stuff (not really used)
+SENTRY_DSN — if you want to use Sentry
+PRODUCTION_SSH_HOST — hostname or IP of your server
+PRODUCTION_SSH_USERNAME — user which can deploy to your server
+PRODUCTION_SSH_KEY — private key for this user
+```
+
+After you install them all and commit something to the master, the action should run and deploy it to your server on port **8816**. 
+
+Don't forget to set up nginx as a proxy for that app (add SSL and everything else in there). Here's example config for that: [etc/nginx/infomate.club.conf](etc/nginx/infomate.club.conf)
+
+If something doesn't work, check the action itself: [.github/workflows/deploy.yml](.github/workflows/deploy.yml)
+
 ## Contributing

 Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change.
--- a/boards.yml
+++ b/boards.yml
@@ -639,12 +639,6 @@ boards:
      - name: "Наблюдения, события, места"
        slug: events
        feeds:
-          - name: "Kiez in Berlin"
-            url: http://kiezinberlin.com/
-            rss: http://kiezinberlin.com/feed/
-          - name: "Канал Глазами Богдана"
-            url: https://t.me/bogdandevisu
-            rss: https://infomate.club/parsing/telegram/bogdandevisu?only=text
          - name: "Канал Travelclever"
            url: https://t.me/travelclever
            rss: https://infomate.club/parsing/telegram/travelclever?only=text
--- a/docker-compose.production.yml
+++ b/docker-compose.production.yml
@@ -0,0 +1,50 @@
+version: "3"
+services:
+  app: &app
+    image: ghcr.io/vas3k/infomate:${GITHUB_SHA:-latest}
+    command: make docker-run-production
+    container_name: infomate_app
+    environment:
+      - MODE=production
+      - PYTHONUNBUFFERED=1
+      - DEBUG=false
+      - APP_HOST=https://infomate.club
+      - POSTGRES_DB=infomate
+      - POSTGRES_USER=vas3k
+      - POSTGRES_PASSWORD=vas3k
+      - POSTGRES_HOST=host.docker.internal
+    env_file:
+      - .env
+    restart: always
+    depends_on:
+      - postgres
+    ports:
+      - "127.0.0.1:8816:8816"
+
+  cron:
+    <<: *app
+    command: make docker-run-cron
+    container_name: infomate_cron
+    depends_on:
+      - app
+      - postgres
+    ports: []
+
+#  postgres:
+#    image: postgres:12
+#    container_name: infomate_postgres
+#    restart: always
+#    environment:
+#      POSTGRES_USER: vas3k
+#      POSTGRES_PASSWORD: vas3k
+#      POSTGRES_DB: vas3k_club
+#    volumes:
+#      - /home/vas3k/pgdata:/var/lib/postgresql/data:rw
+
+  migrate_and_init:
+    <<: *app
+    container_name: infomate_init_feeds
+    restart: "no"
+    ports: []
+    command: make feed_init feed_refresh
+
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -6,7 +6,7 @@ services:
      context: .
      args:
        requirements: requirements.txt
-    command: make docker_run
+    command: make docker-run-app
    container_name: infomate_app
    environment:
      - DEBUG=True
--- a/etc/crontab
+++ b/etc/crontab
@@ -1,2 +1,2 @@
-0 * * * * cd /home/vas3k/infomate.club/scripts && python3 update.py >/dev/null 2>&1
-0 4 * * * cd /home/vas3k/infomate.club/scripts && python3 cleanup.py >/dev/null 2>&1
+0 * * * * cd /home/vas3k/infomate.club/scripts && python3 update.py >/proc/1/fd/1 2>/proc/1/fd/2
+0 4 * * * cd /home/vas3k/infomate.club/scripts && python3 cleanup.py >/proc/1/fd/1 2>/proc/1/fd/2
--- a/etc/nginx/infomate.club.conf
+++ b/etc/nginx/infomate.club.conf
@@ -1,7 +1,3 @@
-upstream infomate_club_uwsgi {
-    server unix:/home/vas3k/infomate.club.sock weight=1 max_fails=5 fail_timeout=30s;
-}
-
 server {
    listen 80;
    listen 443 ssl http2;
@@ -49,14 +45,21 @@ server {
 		add_header Cache-Control "public";
 	}

-    location ^~ /.well-known/acme-challenge/ {
-        default_type "text/plain";
-        root /var/www/letsencrypt;
-    }
-
 	location / {
-		uwsgi_pass infomate_club_uwsgi;
-        uwsgi_ignore_client_abort on;
-		include uwsgi_params;
+        add_header "Access-Control-Allow-Origin" "*";
+        add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS";
+        add_header "Access-Control-Allow-Headers" "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range";
+        add_header "Access-Control-Expose-Headers" "Content-Length,Content-Range";
+        add_header "Strict-Transport-Security" "max-age=31536000;includeSubDomains";
+        add_header "X-Content-Type-Options" "nosniff";
+        add_header "Referrer-Policy" "strict-origin-when-cross-origin";
+
+        proxy_set_header "Host" $http_host;
+        proxy_set_header "X-Forwarded-For" $proxy_add_x_forwarded_for;
+        proxy_set_header "X-Forwarded-Proto" $scheme;
+        proxy_redirect off;
+        proxy_buffering off;
+
+        proxy_pass http://127.0.0.1:8816;
 	}
 }
--- a/etc/nginx/ssl.conf
+++ b/etc/nginx/ssl.conf
@@ -1,15 +0,0 @@
-ssl_session_timeout 1d;
-ssl_session_cache shared:SSL:50m;
-ssl_session_tickets off;
-
-ssl_protocols TLSv1.2;
-ssl_ciphers EECDH+AESGCM:EECDH+AES;
-ssl_ecdh_curve secp384r1;
-ssl_prefer_server_ciphers on;
-
-ssl_stapling on;
-ssl_stapling_verify on;
-
-add_header Strict-Transport-Security "max-age=15768000; includeSubdomains; preload";
-add_header X-Frame-Options SAMEORIGIN;
-add_header X-Content-Type-Options nosniff;
--- a/infomate/settings.py
+++ b/infomate/settings.py
@@ -6,11 +6,10 @@ from random import random
 import sentry_sdk
 from sentry_sdk.integrations.django import DjangoIntegration

-DEBUG = os.getenv("DEBUG", True)
-
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-SECRET_KEY = "wow so secret"
-ALLOWED_HOSTS = ["127.0.0.1", "localhost", "0.0.0.0", "vas3k.ru", "infomate.club"]
+DEBUG = (os.getenv("DEBUG") != "false")  # SECURITY WARNING: don't run with debug turned on in production!
+SECRET_KEY = os.getenv("SECRET_KEY") or "wow so secret"
+ALLOWED_HOSTS = ["127.0.0.1", "localhost", "0.0.0.0", "infomate.club"]

 INSTALLED_APPS = [
    "django.contrib.staticfiles",
@@ -48,11 +47,11 @@ WSGI_APPLICATION = "infomate.wsgi.application"
 DATABASES = {
    "default": {
        "ENGINE": "django.db.backends.postgresql_psycopg2",
-        "NAME": "infomate",
-        "USER": "postgres",  # redefined in private_settings.py
-        "PASSWORD": "postgres",
-        "HOST": "postgres",
-        "PORT": "5432",
+        "NAME": os.getenv("POSTGRES_DB") or "infomate",
+        "USER": os.getenv("POSTGRES_USER") or "postgres",
+        "PASSWORD": os.getenv("POSTGRES_PASSWORD") or "",
+        "HOST": os.getenv("POSTGRES_HOST") or "localhost",
+        "PORT": os.getenv("POSTGRES_PORT") or 5432,
    }
 }

@@ -86,11 +85,20 @@ CSS_HASH = str(random())

 # Cache

-CACHES = {
-    "default": {
-        "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+if DEBUG:
+    CACHES = {
+        "default": {
+            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+        }
    }
-}
+else:
+    CACHES = {
+        "default": {
+            "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
+            "LOCATION": "/tmp/infomate_cache"
+        }
+    }
+
 STATIC_PAGE_CACHE_SECONDS = 5 * 60  # 5 min
 BOARD_CACHE_SECONDS = 10 * 60  # 10 min

@@ -99,39 +107,17 @@ BOARD_CACHE_SECONDS = 10 * 60  # 10 min
 APP_NAME = "Infomate"
 APP_TITLE = "Агрегатор инфополя"
 APP_DESCRIPTION = APP_TITLE
-APP_HOST = "https://infomate.club"
+APP_HOST = os.getenv("APP_HOST") or "http://127.0.0.1:8000"

-JWT_SECRET = "wow so secret"  # should be the same as on vas3k.ru
-JWT_ALGORITHM = "HS256"
-JWT_EXP_TIMEDELTA = timedelta(days=120)
-
-AUTH_COOKIE_NAME = "jwt"
-AUTH_COOKIE_MAX_AGE = 300 * 24 * 60 * 60  # 300 days
-AUTH_REDIRECT_URL = "https://vas3k.ru/auth/external/"
-AUTH_FAILED_REDIRECT_URL = "https://vas3k.ru/auth/login/"
-
-SENTRY_DSN = None
+SENTRY_DSN = os.getenv("SENTRY_DSN")

 MEDIA_UPLOAD_URL = "https://i.vas3k.ru/upload/"
-MEDIA_UPLOAD_CODE = None  # should be set in private_settings.py
+MEDIA_UPLOAD_CODE = os.getenv("MEDIA_UPLOAD_CODE")

-TELEGRAM_APP_ID = None  # should set in private_settings.py
-TELEGRAM_APP_HASH = None  # should set in private_settings.py
-TELEGRAM_SESSION_FILE = None  # should set in private settings.py
 TELEGRAM_CACHE_SECONDS = 10 * 60  # 10 min

 BLEACH_STRIP_TAGS = True

-try:
-    # poor mans' private settings
-    # As due to obvious reasons this file is missing in the repository, suppress the following 'pyflakes' error codes:
-    # - F401 'infomate.private_settings.*' imported but unused
-    # - F403 'from infomate.private_settings import *' used; unable to detect undefined names
-    from infomate.private_settings import *  # noqa: F401 F403
-except ImportError:
-    pass
-
-
 if SENTRY_DSN and not DEBUG:
    sentry_sdk.init(
        dsn=SENTRY_DSN,
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,6 @@
-Django==2.2.13
+Django==3.2.12
+gunicorn==20.1.0
+uvicorn==0.17.6
 psycopg2-binary==2.8.6
 click==7.0
 pillow==8.2.0
@@ -10,4 +12,4 @@ feedparser==6
 sentry-sdk==0.14.1
 nltk==3.4.5
 newspaper3k>=0.2.8
-django-bleach==0.6.1
+django-bleach==0.6.1