From 5ee4a7b1a2c9262f90501532d3a16ab4c46e1695 Mon Sep 17 00:00:00 2001 From: vas3k Date: Tue, 29 Mar 2022 10:54:07 +0200 Subject: [PATCH] Docker deployment --- .github/workflows/deploy.yml | 39 ++++++++++++++++++++++ Makefile | 36 ++++++++++++-------- README.md | 23 ++++++++++++- boards.yml | 6 ---- docker-compose.production.yml | 50 ++++++++++++++++++++++++++++ docker-compose.yml | 2 +- etc/crontab | 4 +-- etc/nginx/infomate.club.conf | 27 ++++++++------- etc/nginx/ssl.conf | 15 --------- infomate/settings.py | 62 ++++++++++++++--------------------- requirements.txt | 6 ++-- 11 files changed, 180 insertions(+), 90 deletions(-) create mode 100644 .github/workflows/deploy.yml create mode 100644 docker-compose.production.yml delete mode 100644 etc/nginx/ssl.conf diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..fb1f418 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,39 @@ +name: Deploy master + +on: + push: + branches: + - master + +jobs: + build: + name: Build image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - run: docker login ghcr.io -u $GITHUB_ACTOR -p ${{ secrets.GHCR_TOKEN }} + - run: docker build -t ghcr.io/$GITHUB_ACTOR/infomate:latest -t ghcr.io/$GITHUB_ACTOR/infomate:$GITHUB_SHA . + - run: docker image push ghcr.io/$GITHUB_ACTOR/infomate:$GITHUB_SHA + - run: docker image push ghcr.io/$GITHUB_ACTOR/infomate:latest + + deploy: + name: Deploy + runs-on: ubuntu-latest + needs: build + env: + SSH_KEY_PATH: /tmp/ssh_key + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Make envfile + run: export | grep "secret_" | sed "s/declare -x secret_//" > .env + env: + secret_SECRET_KEY: ${{ secrets.SECRET_KEY }} + secret_APP_HOST: ${{ secrets.APP_HOST }} + secret_MEDIA_UPLOAD_CODE: ${{ secrets.MEDIA_UPLOAD_CODE }} + secret_SENTRY_DSN: ${{ secrets.SENTRY_DSN }} + - run: echo "GITHUB_SHA=$GITHUB_SHA" >> .env + - run: echo "${{ secrets.PRODUCTION_SSH_KEY }}" > ${{ env.SSH_KEY_PATH }} && chmod 600 ${{ env.SSH_KEY_PATH }} + - run: scp -o StrictHostKeyChecking=no -i ${{ env.SSH_KEY_PATH }} .env ${{ secrets.PRODUCTION_SSH_USERNAME }}@${{ secrets.PRODUCTION_SSH_HOST }}:/home/vas3k/infomate.club/.env + - run: scp -o StrictHostKeyChecking=no -i ${{ env.SSH_KEY_PATH }} docker-compose.production.yml ${{ secrets.PRODUCTION_SSH_USERNAME }}@${{ secrets.PRODUCTION_SSH_HOST }}:/home/vas3k/infomate.club/docker-compose.production.yml + - run: ssh -i ${{ env.SSH_KEY_PATH }} ${{ secrets.PRODUCTION_SSH_USERNAME }}@${{ secrets.PRODUCTION_SSH_HOST }} "cd /home/vas3k/infomate.club && docker login ghcr.io -u $GITHUB_ACTOR -p ${{ secrets.GHCR_TOKEN }} && docker pull ghcr.io/$GITHUB_ACTOR/infomate:$GITHUB_SHA && docker-compose -f docker-compose.production.yml --env-file=.env up -d && docker system prune --all --force" diff --git a/Makefile b/Makefile index e7e5c02..7206dfe 100644 --- a/Makefile +++ b/Makefile @@ -5,22 +5,30 @@ PROJECT_NAME=infomate -dev-requirements: ## Install dev requirements - @pip3 install -r requirements.txt +run: ## Run dev server + python3 manage.py migrate + python3 manage.py runserver 0.0.0.0:8000 -docker_run: ## Run dev server in docker - @python3 ./utils/wait_for_postgres.py - @python3 manage.py migrate - @python3 manage.py runserver 0.0.0.0:8000 +dev-requirements: ## Install dev requirements + pip3 install -r requirements.txt + +docker-run-app: ## Run production setup in docker + python3 ./utils/wait_for_postgres.py + python3 manage.py migrate + gunicorn infomate.asgi:application -w 3 -k uvicorn.workers.UvicornWorker --bind=0.0.0.0:8816 --capture-output --log-level debug --access-logfile - --error-logfile - + +docker-run-cron: ## Run production cron container + env >> /etc/environment + cron -f -l 2 feed_cleanup: ## Cleanup RSS feeds - @python3 ./scripts/cleanup.py + python3 ./scripts/cleanup.py feed_init: ## Initialize feeds from boards.yml - @python3 ./scripts/initialize.py --config boards.yml --no-upload-favicons -y + python3 ./scripts/initialize.py --config boards.yml --no-upload-favicons -y feed_refresh: ## Refresh RSS feeds - @python3 ./scripts/update.py + python3 ./scripts/update.py help: ## Display this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ @@ -31,16 +39,16 @@ lint: ## Lint code with flake8 flake8 $(PROJECT_NAME) migrate: ## Migrate database to the latest version - @python3 manage.py migrate + python3 manage.py migrate mypy: ## Check types with mypy mypy $(PROJECT_NAME) run: ## Runs dev server - @python3 manage.py runserver + python3 manage.py runserver telegram: - @python3 setup_telegram.py + python3 setup_telegram.py test-ci: test-requirements lint mypy ## Run tests (intended for CI usage) @@ -48,8 +56,10 @@ test-requirements: ## Install requirements to run tests @pip3 install -r ./requirements-test.txt .PHONY: \ + run \ dev-requirements \ - docker_run \ + docker-run-app \ + docker-run-cron \ feed_cleanup \ feed_init \ feed_refresh \ diff --git a/README.md b/README.md index 3dfc11c..5fafe03 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Infomate.club -[![Build Status](https://travis-ci.org/vas3k/infomate.club.svg?branch=master)](https://travis-ci.org/vas3k/infomate.club) [![GitHub license](https://img.shields.io/github/license/vas3k/infomate.club)](https://github.com/vas3k/infomate.club/blob/master/LICENSE) [![GitHub contributors](https://img.shields.io/github/contributors/vas3k/infomate.club)](https://GitHub.com/vas3k/infomate.club/graphs/contributors/) +[![Build Status](https://travis-ci.org/vas3k/infomate.club.svg?branch=master)](https://travis-ci.org/vas3k/infomate.club) Infomate is a small web service that shows multiple RSS sources on one page and performs tricky parsing and summarizing articles using TextRank algorithm. @@ -128,6 +128,27 @@ boards: word: Trump # exclude articles with a word "Trump" in title ``` +## Running in production + +Deployment is done using a simple Github Action which builds a docker container, puts it into Github Registry, logs into your server via SSH and pulls it. +The pipeline is triggered on every push to master branch. If you want to set up your own fork, please add these constants to your repo SECRETS: + +``` +APP_HOST — e.g. "https://your.host.com" +GHCR_TOKEN — your personal guthib access token with permissions to read/write into Github Registry +SECRET_KEY — random string for django stuff (not really used) +SENTRY_DSN — if you want to use Sentry +PRODUCTION_SSH_HOST — hostname or IP of your server +PRODUCTION_SSH_USERNAME — user which can deploy to your server +PRODUCTION_SSH_KEY — private key for this user +``` + +After you install them all and commit something to the master, the action should run and deploy it to your server on port **8816**. + +Don't forget to set up nginx as a proxy for that app (add SSL and everything else in there). Here's example config for that: [etc/nginx/infomate.club.conf](etc/nginx/infomate.club.conf) + +If something doesn't work, check the action itself: [.github/workflows/deploy.yml](.github/workflows/deploy.yml) + ## Contributing Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change. diff --git a/boards.yml b/boards.yml index 3ac6706..720b7b1 100644 --- a/boards.yml +++ b/boards.yml @@ -639,12 +639,6 @@ boards: - name: "Наблюдения, события, места" slug: events feeds: - - name: "Kiez in Berlin" - url: http://kiezinberlin.com/ - rss: http://kiezinberlin.com/feed/ - - name: "Канал Глазами Богдана" - url: https://t.me/bogdandevisu - rss: https://infomate.club/parsing/telegram/bogdandevisu?only=text - name: "Канал Travelclever" url: https://t.me/travelclever rss: https://infomate.club/parsing/telegram/travelclever?only=text diff --git a/docker-compose.production.yml b/docker-compose.production.yml new file mode 100644 index 0000000..6ae17ee --- /dev/null +++ b/docker-compose.production.yml @@ -0,0 +1,50 @@ +version: "3" +services: + app: &app + image: ghcr.io/vas3k/infomate:${GITHUB_SHA:-latest} + command: make docker-run-production + container_name: infomate_app + environment: + - MODE=production + - PYTHONUNBUFFERED=1 + - DEBUG=false + - APP_HOST=https://infomate.club + - POSTGRES_DB=infomate + - POSTGRES_USER=vas3k + - POSTGRES_PASSWORD=vas3k + - POSTGRES_HOST=host.docker.internal + env_file: + - .env + restart: always + depends_on: + - postgres + ports: + - "127.0.0.1:8816:8816" + + cron: + <<: *app + command: make docker-run-cron + container_name: infomate_cron + depends_on: + - app + - postgres + ports: [] + +# postgres: +# image: postgres:12 +# container_name: infomate_postgres +# restart: always +# environment: +# POSTGRES_USER: vas3k +# POSTGRES_PASSWORD: vas3k +# POSTGRES_DB: vas3k_club +# volumes: +# - /home/vas3k/pgdata:/var/lib/postgresql/data:rw + + migrate_and_init: + <<: *app + container_name: infomate_init_feeds + restart: "no" + ports: [] + command: make feed_init feed_refresh + diff --git a/docker-compose.yml b/docker-compose.yml index 216760c..3dc5e8b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: context: . args: requirements: requirements.txt - command: make docker_run + command: make docker-run-app container_name: infomate_app environment: - DEBUG=True diff --git a/etc/crontab b/etc/crontab index 14cf59f..757418e 100644 --- a/etc/crontab +++ b/etc/crontab @@ -1,2 +1,2 @@ -0 * * * * cd /home/vas3k/infomate.club/scripts && python3 update.py >/dev/null 2>&1 -0 4 * * * cd /home/vas3k/infomate.club/scripts && python3 cleanup.py >/dev/null 2>&1 +0 * * * * cd /home/vas3k/infomate.club/scripts && python3 update.py >/proc/1/fd/1 2>/proc/1/fd/2 +0 4 * * * cd /home/vas3k/infomate.club/scripts && python3 cleanup.py >/proc/1/fd/1 2>/proc/1/fd/2 diff --git a/etc/nginx/infomate.club.conf b/etc/nginx/infomate.club.conf index f167a57..4cfac0b 100644 --- a/etc/nginx/infomate.club.conf +++ b/etc/nginx/infomate.club.conf @@ -1,7 +1,3 @@ -upstream infomate_club_uwsgi { - server unix:/home/vas3k/infomate.club.sock weight=1 max_fails=5 fail_timeout=30s; -} - server { listen 80; listen 443 ssl http2; @@ -49,14 +45,21 @@ server { add_header Cache-Control "public"; } - location ^~ /.well-known/acme-challenge/ { - default_type "text/plain"; - root /var/www/letsencrypt; - } - location / { - uwsgi_pass infomate_club_uwsgi; - uwsgi_ignore_client_abort on; - include uwsgi_params; + add_header "Access-Control-Allow-Origin" "*"; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS"; + add_header "Access-Control-Allow-Headers" "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range"; + add_header "Access-Control-Expose-Headers" "Content-Length,Content-Range"; + add_header "Strict-Transport-Security" "max-age=31536000;includeSubDomains"; + add_header "X-Content-Type-Options" "nosniff"; + add_header "Referrer-Policy" "strict-origin-when-cross-origin"; + + proxy_set_header "Host" $http_host; + proxy_set_header "X-Forwarded-For" $proxy_add_x_forwarded_for; + proxy_set_header "X-Forwarded-Proto" $scheme; + proxy_redirect off; + proxy_buffering off; + + proxy_pass http://127.0.0.1:8816; } } diff --git a/etc/nginx/ssl.conf b/etc/nginx/ssl.conf deleted file mode 100644 index 6fa59b0..0000000 --- a/etc/nginx/ssl.conf +++ /dev/null @@ -1,15 +0,0 @@ -ssl_session_timeout 1d; -ssl_session_cache shared:SSL:50m; -ssl_session_tickets off; - -ssl_protocols TLSv1.2; -ssl_ciphers EECDH+AESGCM:EECDH+AES; -ssl_ecdh_curve secp384r1; -ssl_prefer_server_ciphers on; - -ssl_stapling on; -ssl_stapling_verify on; - -add_header Strict-Transport-Security "max-age=15768000; includeSubdomains; preload"; -add_header X-Frame-Options SAMEORIGIN; -add_header X-Content-Type-Options nosniff; diff --git a/infomate/settings.py b/infomate/settings.py index 4141578..86b8c49 100644 --- a/infomate/settings.py +++ b/infomate/settings.py @@ -6,11 +6,10 @@ from random import random import sentry_sdk from sentry_sdk.integrations.django import DjangoIntegration -DEBUG = os.getenv("DEBUG", True) - BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SECRET_KEY = "wow so secret" -ALLOWED_HOSTS = ["127.0.0.1", "localhost", "0.0.0.0", "vas3k.ru", "infomate.club"] +DEBUG = (os.getenv("DEBUG") != "false") # SECURITY WARNING: don't run with debug turned on in production! +SECRET_KEY = os.getenv("SECRET_KEY") or "wow so secret" +ALLOWED_HOSTS = ["127.0.0.1", "localhost", "0.0.0.0", "infomate.club"] INSTALLED_APPS = [ "django.contrib.staticfiles", @@ -48,11 +47,11 @@ WSGI_APPLICATION = "infomate.wsgi.application" DATABASES = { "default": { "ENGINE": "django.db.backends.postgresql_psycopg2", - "NAME": "infomate", - "USER": "postgres", # redefined in private_settings.py - "PASSWORD": "postgres", - "HOST": "postgres", - "PORT": "5432", + "NAME": os.getenv("POSTGRES_DB") or "infomate", + "USER": os.getenv("POSTGRES_USER") or "postgres", + "PASSWORD": os.getenv("POSTGRES_PASSWORD") or "", + "HOST": os.getenv("POSTGRES_HOST") or "localhost", + "PORT": os.getenv("POSTGRES_PORT") or 5432, } } @@ -86,11 +85,20 @@ CSS_HASH = str(random()) # Cache -CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", +if DEBUG: + CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + } } -} +else: + CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/tmp/infomate_cache" + } + } + STATIC_PAGE_CACHE_SECONDS = 5 * 60 # 5 min BOARD_CACHE_SECONDS = 10 * 60 # 10 min @@ -99,39 +107,17 @@ BOARD_CACHE_SECONDS = 10 * 60 # 10 min APP_NAME = "Infomate" APP_TITLE = "Агрегатор инфополя" APP_DESCRIPTION = APP_TITLE -APP_HOST = "https://infomate.club" +APP_HOST = os.getenv("APP_HOST") or "http://127.0.0.1:8000" -JWT_SECRET = "wow so secret" # should be the same as on vas3k.ru -JWT_ALGORITHM = "HS256" -JWT_EXP_TIMEDELTA = timedelta(days=120) - -AUTH_COOKIE_NAME = "jwt" -AUTH_COOKIE_MAX_AGE = 300 * 24 * 60 * 60 # 300 days -AUTH_REDIRECT_URL = "https://vas3k.ru/auth/external/" -AUTH_FAILED_REDIRECT_URL = "https://vas3k.ru/auth/login/" - -SENTRY_DSN = None +SENTRY_DSN = os.getenv("SENTRY_DSN") MEDIA_UPLOAD_URL = "https://i.vas3k.ru/upload/" -MEDIA_UPLOAD_CODE = None # should be set in private_settings.py +MEDIA_UPLOAD_CODE = os.getenv("MEDIA_UPLOAD_CODE") -TELEGRAM_APP_ID = None # should set in private_settings.py -TELEGRAM_APP_HASH = None # should set in private_settings.py -TELEGRAM_SESSION_FILE = None # should set in private settings.py TELEGRAM_CACHE_SECONDS = 10 * 60 # 10 min BLEACH_STRIP_TAGS = True -try: - # poor mans' private settings - # As due to obvious reasons this file is missing in the repository, suppress the following 'pyflakes' error codes: - # - F401 'infomate.private_settings.*' imported but unused - # - F403 'from infomate.private_settings import *' used; unable to detect undefined names - from infomate.private_settings import * # noqa: F401 F403 -except ImportError: - pass - - if SENTRY_DSN and not DEBUG: sentry_sdk.init( dsn=SENTRY_DSN, diff --git a/requirements.txt b/requirements.txt index 7f623ff..9c1a599 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ -Django==2.2.13 +Django==3.2.12 +gunicorn==20.1.0 +uvicorn==0.17.6 psycopg2-binary==2.8.6 click==7.0 pillow==8.2.0 @@ -10,4 +12,4 @@ feedparser==6 sentry-sdk==0.14.1 nltk==3.4.5 newspaper3k>=0.2.8 -django-bleach==0.6.1 \ No newline at end of file +django-bleach==0.6.1