From 8dc8313c98090ffc3d3a4498b2550d4613233dbc Mon Sep 17 00:00:00 2001 From: vas3k Date: Sat, 5 Sep 2020 16:39:57 +0200 Subject: [PATCH] Infomate 1.2 release --- Dockerfile | 2 + README.md | 4 + auth/__init__.py | 0 auth/admin.py | 3 - auth/apps.py | 5 - auth/context_processors.py | 7 - auth/helpers.py | 20 -- auth/migrations/0001_initial.py | 29 --- auth/migrations/0002_auto_20200107_2024.py | 18 -- auth/migrations/__init__.py | 0 auth/models.py | 15 -- auth/views.py | 55 ----- boards.yml | 244 +++++++++++-------- boards/icons.py | 22 ++ boards/migrations/0009_auto_20200905_1246.py | 34 +++ boards/models.py | 32 ++- boards/views.py | 17 -- docker-compose.yml | 2 +- infomate/settings.py | 2 - infomate/urls.py | 8 +- requirements.txt | 1 - scripts/common.py | 93 +++++++ scripts/initialize.py | 124 +++++----- scripts/update.py | 142 +++-------- static/css/components.css | 11 +- static/css/layout.css | 45 +++- templates/blocks/three.html | 13 + templates/blocks/two.html | 13 + templates/board.html | 57 +---- templates/board_no_access.html | 230 ----------------- templates/export.html | 21 -- templates/feeds/favicons.html | 27 ++ templates/feeds/simple.html | 27 ++ templates/layout.html | 7 +- templates/tooltips/simple.html | 21 ++ 35 files changed, 582 insertions(+), 769 deletions(-) delete mode 100644 auth/__init__.py delete mode 100644 auth/admin.py delete mode 100644 auth/apps.py delete mode 100644 auth/context_processors.py delete mode 100644 auth/helpers.py delete mode 100644 auth/migrations/0001_initial.py delete mode 100644 auth/migrations/0002_auto_20200107_2024.py delete mode 100644 auth/migrations/__init__.py delete mode 100644 auth/models.py delete mode 100644 auth/views.py create mode 100644 boards/migrations/0009_auto_20200905_1246.py create mode 100644 templates/blocks/three.html create mode 100644 templates/blocks/two.html delete mode 100644 templates/board_no_access.html delete mode 100644 templates/export.html create mode 100644 templates/feeds/favicons.html create mode 100644 templates/feeds/simple.html create mode 100644 templates/tooltips/simple.html diff --git a/Dockerfile b/Dockerfile index a5765c1..181b7f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,3 +19,5 @@ ADD . /app RUN pip install --no-cache-dir -e . \ && pip install --no-cache-dir -r $requirements + +RUN python -c "import nltk; nltk.download('punkt')" diff --git a/README.md b/README.md index 2dba3e3..3dfc11c 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,10 @@ boards: rss: http://feeds.feedburner.com/TechCrunch/ url: https://techcrunch.com is_parsable: false # do not try to parse pages, show RSS content only + conditions: + - type: not_in + field: title + word: Trump # exclude articles with a word "Trump" in title ``` ## Contributing diff --git a/auth/__init__.py b/auth/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/auth/admin.py b/auth/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/auth/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/auth/apps.py b/auth/apps.py deleted file mode 100644 index bdf15e1..0000000 --- a/auth/apps.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.apps import AppConfig - - -class AuthConfig(AppConfig): - name = 'auth' diff --git a/auth/context_processors.py b/auth/context_processors.py deleted file mode 100644 index d3233a0..0000000 --- a/auth/context_processors.py +++ /dev/null @@ -1,7 +0,0 @@ -from auth.helpers import authorized_user - - -def me(request): - return { - "me": authorized_user(request) - } diff --git a/auth/helpers.py b/auth/helpers.py deleted file mode 100644 index 71e2856..0000000 --- a/auth/helpers.py +++ /dev/null @@ -1,20 +0,0 @@ -from datetime import datetime - -import jwt -from django.conf import settings - - -def authorized_user(request): - token = request.COOKIES.get(settings.AUTH_COOKIE_NAME) - if not token: - return None - - try: - payload = jwt.decode(token, settings.JWT_SECRET, algorithms=[settings.JWT_ALGORITHM]) - except (jwt.DecodeError, jwt.ExpiredSignatureError): - return None - - if datetime.utcfromtimestamp(payload["exp"]) < datetime.utcnow(): - return None - - return payload diff --git a/auth/migrations/0001_initial.py b/auth/migrations/0001_initial.py deleted file mode 100644 index e776fc2..0000000 --- a/auth/migrations/0001_initial.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 2.2.8 on 2020-01-05 13:40 - -from django.db import migrations, models -import uuid - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Session', - fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('token', models.CharField(max_length=256, unique=True)), - ('user_id', models.IntegerField()), - ('user_name', models.CharField(max_length=32, null=True)), - ('created_at', models.DateTimeField(auto_now_add=True)), - ('expires_at', models.DateTimeField()), - ], - options={ - 'db_table': 'sessions', - }, - ), - ] diff --git a/auth/migrations/0002_auto_20200107_2024.py b/auth/migrations/0002_auto_20200107_2024.py deleted file mode 100644 index 1836112..0000000 --- a/auth/migrations/0002_auto_20200107_2024.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.8 on 2020-01-07 20:24 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('auth', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='session', - name='token', - field=models.CharField(max_length=1024, unique=True), - ), - ] diff --git a/auth/migrations/__init__.py b/auth/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/auth/models.py b/auth/models.py deleted file mode 100644 index 76063d5..0000000 --- a/auth/models.py +++ /dev/null @@ -1,15 +0,0 @@ -import uuid - -from django.db import models - - -class Session(models.Model): - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - token = models.CharField(max_length=1024, unique=True) - user_id = models.IntegerField() # original id of a club user (we don't store profiles) - user_name = models.CharField(max_length=32, null=True) - created_at = models.DateTimeField(auto_now_add=True) - expires_at = models.DateTimeField() - - class Meta: - db_table = "sessions" diff --git a/auth/views.py b/auth/views.py deleted file mode 100644 index 12047eb..0000000 --- a/auth/views.py +++ /dev/null @@ -1,55 +0,0 @@ -import logging -from datetime import datetime - -import jwt -from django.conf import settings -from django.shortcuts import redirect, render - -from auth.models import Session - -log = logging.getLogger() - - -def login(request): - return redirect(f"{settings.AUTH_REDIRECT_URL}?redirect={settings.APP_HOST}/auth/club_callback/") - - -def club_callback(request): - token = request.GET.get("jwt") - if not token: - return render(request, "message.html", { - "title": "Что-то пошло не так", - "message": "При авторизации потерялся токен. Попробуйте войти еще раз." - }) - - try: - payload = jwt.decode(token, settings.JWT_SECRET, algorithms=[settings.JWT_ALGORITHM]) - except (jwt.DecodeError, jwt.ExpiredSignatureError) as ex: - log.error(f"JWT token error: {ex}") - return render(request, "message.html", { - "title": "Что-то сломалось", - "message": "Неправильный ключ. Наверное, что-то сломалось. Либо ты ХАКИР!!11" - }) - - Session.objects.get_or_create( - token=token[:1024], - defaults=dict( - user_id=payload["user_id"], - user_name=str(payload.get("user_name") or "")[:32], - expires_at=datetime.utcfromtimestamp(payload["exp"]) - ) - ) - - response = redirect("index") - response.set_cookie(settings.AUTH_COOKIE_NAME, token, max_age=settings.AUTH_COOKIE_MAX_AGE) - return response - - -def logout(request): - token = request.COOKIES.get(settings.AUTH_COOKIE_NAME) - if token: - Session.objects.filter(token=token).delete() - - response = redirect("index") - response.delete_cookie(settings.AUTH_COOKIE_NAME) - return response diff --git a/boards.yml b/boards.yml index 7b4be9a..beed44c 100644 --- a/boards.yml +++ b/boards.yml @@ -4,7 +4,7 @@ boards: is_visible: true is_private: false curator: - name: Главные новости + name: Новости title: События в мире avatar: https://i.vas3k.ru/63cd2ebddba4422aa684b2bd754c636eb061ef0555f542042c31850525d2f5bb.png bio: Подборка основных новостных изданий, чтобы следить за событиями в России и в мире. Пока бета-версия. Предложения новых источников присылайте в личку @@ -19,6 +19,7 @@ boards: url: https://tass.ru/ rss: https://tass.ru/rss/v2.xml icon: https://i.vas3k.ru/aca2f29518b01b25b3a40d63109d45dde74be15e47877aaa89553ff567b05151.png + is_parsable: false - name: TJ url: https://tjournal.ru rss: https://tjournal.ru/rss/all @@ -36,13 +37,14 @@ boards: - name: Дождь url: https://tvrain.ru/news/ rss: https://tvrain.ru/export/rss/all.xml + is_parsable: false - name: Lenta.ru url: https://lenta.ru/ rss: https://lenta.ru/rss - name: BBC Россия url: https://www.bbc.com/russian rss: https://feeds.bbci.co.uk/russian/rss.xml - - name: Телеграм-каналы + - name: Телеграм slug: tg feeds: - name: Varlamov News @@ -64,34 +66,60 @@ boards: url: https://t.me/rtvimain rss: https://infomate.club/parsing/telegram/rtvimain?only=text icon: https://i.vas3k.ru/de4d266b4744ca0cb8bc9ded3842989593fdd54ec78484c7c982b8951db08617.jpg - - name: Expresso - url: https://t.me/expressotoday - rss: https://infomate.club/parsing/telegram/expressotoday?only=text - icon: https://i.vas3k.ru/8ecf3ff7c82f89b0bfdde441e32082b063d7bf33e7b3b4b13a77dd35b38aa744.jpg +# - name: Expresso +# url: https://t.me/expressotoday +# rss: https://infomate.club/parsing/telegram/expressotoday?only=text +# icon: https://i.vas3k.ru/8ecf3ff7c82f89b0bfdde441e32082b063d7bf33e7b3b4b13a77dd35b38aa744.jpg - name: США slug: us feeds: - - name: Reddit News - url: https://www.reddit.com/r/news/ - rss: https://www.reddit.com/r/news.rss - - name: CNN - url: https://cnn.com/ - rss: http://rss.cnn.com/rss/edition.rss - icon: https://i.vas3k.ru/d2d88f4a1d1646bf3a70f76d6b585c472ee1735abd73dd30719d7f2f42f5743a.png - - name: NYT - url: https://www.nytimes.com/ - rss: https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml + - name: Axios + url: https://www.axios.com/ + rss: https://api.axios.com/feed/ + icon: https://i.vas3k.ru/17f55ad102b80a85c618d5e56c61f24c17c20d12f8c960a87902845154a5bdfc.jpg + - name: Reuters + url: https://www.reuters.com/news/world + rss: https://news.google.com/rss/search?q=when:24h+allinurl:reuters.com&ceid=US:en&hl=en-US&gl=US - name: Bloomberg url: https://www.bloomberg.com/ rss: http://www.bloomberg.com/politics/feeds/site.xml icon: https://i.vas3k.ru/35c6ae6df0fe47166ed5c656bde6faa974ae1beca949c89443f0aed0b86e0806.png - - name: Reuters - url: https://www.reuters.com/news/world - rss: https://news.google.com/rss/search?q=when:24h+allinurl:reuters.com&ceid=US:en&hl=en-US&gl=US + - name: Reddit News + url: https://www.reddit.com/r/news/ + rss: https://www.reddit.com/r/news.rss + is_parsable: false + - name: NYT + url: https://www.nytimes.com/ + rss: https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml + is_parsable: false - name: POLITICO url: https://www.politico.com/ rss: https://www.politico.com/rss/politicopicks.xml icon: https://i.vas3k.ru/0281ddd9b3bd890e1476666d5ea74688bc5fcf313500a5fc166127bc433b1287.jpg + is_parsable: false + - name: 🏳️‍🌈 Левые + slug: us_left + view: "blocks/two.html" + feeds: + - view: "feeds/favicons.html" + is_parsable: false + mix: + - http://rss.cnn.com/rss/edition.rss + - https://www.buzzfeed.com/index.xml + - https://www.huffpost.com/section/front-page/feed + - https://www.newyorker.com/feed/news + - https://www.msnbc.com/feed + - name: 💰 Правые + slug: us_right + view: "blocks/two.html" + feeds: + - view: "feeds/favicons.html" + is_parsable: false + mix: + - http://feeds.feedburner.com/foxnews/latest + - https://spectator.org/feed + - https://www.washingtontimes.com/rss/headlines/news/ + - https://www.dailymail.co.uk/ushome/index.rss - name: Европа slug: eu feeds: @@ -127,7 +155,7 @@ boards: is_private: false curator: name: Технологии - title: Основные СМИ + title: Главные новости avatar: https://i.vas3k.ru/229b722cc79faca1f148c66b1e7240488e7405704f94b8d7f0fadddcf66212f0.jpg bio: Подборка мейнстримовых новостей о технологиях на русском и английском языках blocks: @@ -208,12 +236,11 @@ boards: - name: addmeto url: https://t.me/addmeto rss: https://infomate.club/parsing/telegram/addmeto + icon: https://i.vas3k.ru/cb1fe74c1a42fbe9d145c8538ed9230b7512633d06f680b96464fc4b355b23ef.jpg - name: Rozetked url: https://t.me/rozetked rss: https://infomate.club/parsing/telegram/rozetked - - name: Rozetked - url: https://t.me/rozetked - rss: https://infomate.club/parsing/telegram/rozetked + icon: https://i.vas3k.ru/abe55f96279f22704cd1cc5009be2f6527c8d205f289dc1c7c328a03314f3d5d.jpg - name: Мейнстрим slug: mainstream feeds: @@ -274,6 +301,14 @@ boards: url: https://www.reddit.com/r/technology rss: https://www.reddit.com/r/technology.rss is_parsable: false + - name: "Pinboard: Popular" + url: https://pinboard.in/popular/ + rss: http://feeds.pinboard.in/rss/popular/ + icon: https://i.vas3k.ru/adfe6b6f09b2be1398df020bdd8d1b8dade25b139c88c68c0177d26e5ae0bce0.jpg + conditions: + - type: not_in + field: title + word: Trump - name: ZDNet rss: https://www.zdnet.com/news/rss.xml url: https://www.zdnet.com @@ -284,100 +319,21 @@ boards: - name: Slashdot rss: http://rss.slashdot.org/Slashdot/slashdotMain url: https://slashdot.org + icon: https://i.vas3k.ru/2da938e66d63ca719a8854ead38a09d17d6ab17725aaf15fa68f401aa937340e.png - name: 'Medium: Technology' icon: https://i.vas3k.ru/fhb.png url: https://medium.com/topic/technology rss: https://medium.com/feed/topic/technology is_parsable: false - - name: Hacker Noon - url: https://hackernoon.com/ - rss: https://hackernoon.com/feed - - name: Европейское айти - slug: eu - feeds: - - name: "EU-startups" - url: https://www.eu-startups.com/ - rss: https://www.eu-startups.com/feed/ - icon: https://i.vas3k.ru/fkp.jpg - - name: Tech.eu - url: https://tech.eu/ - rss: https://tech.eu/feed/ - icon: https://i.vas3k.ru/fl9.jpg - - name: "TechCrunch: Europe" - url: https://techcrunch.com/europe/ - rss: http://feeds.feedburner.com/Techcrunch/europe - is_parsable: false - - name: Инди-разработка - slug: make - feeds: - - name: Show HN - url: https://news.ycombinator.com/show - rss: https://hnrss.org/show - - name: Starter Story - url: https://www.starterstory.com - rss: https://www.starterstory.com/feed?format=rss - - name: 'Reddit: /r/SideProject' - url: https://www.reddit.com/r/SideProject/ - rss: https://www.reddit.com/r/SideProject.rss - is_parsable: false - - name: Путешествия - slug: travel - feeds: - - name: PeritoBurrito - url: https://perito-burrito.com - rss: http://perito-burrito.com/feed - - name: Vandrouki - url: https://vandrouki.ru - rss: https://feeds.feedburner.com/vandroukiru - icon: https://i.vas3k.ru/fer.jpg - - name: Secret Flying - url: https://www.secretflying.com - rss: https://www.secretflying.com/feed/ - - name: 'Atlas Obscura: Stories' - url: https://www.atlasobscura.com/articles - rss: https://www.atlasobscura.com/feeds/latest - - name: "T—Ж" - url: https://journal.tinkoff.ru/chemodan/ - rss: https://journal.tinkoff.ru/feed/ - - name: Geeky Explorer - url: https://www.geekyexplorer.com - rss: https://www.geekyexplorer.com/feed/ - - name: Микс из блогов крупных компаний - slug: engineering - feeds: - - url: http://www.rssmix.com/1/ - rss: http://www.rssmix.com/u/10536474/rss.xml - columns: 3 - mix: - - http://blog.stackoverflow.com/feed/ - - https://machinelearning.apple.com/feed.xml - - http://nerds.airbnb.com/feed/ - - http://www.ebaytechblog.com/feed/ - - http://blog.agilebits.com/feed/ - - http://research.microsoft.com/rss/news.xml - - http://blog.keras.io/feeds/all.atom.xml - - http://techblog.netflix.com/feeds/posts/default - - http://nickcraver.com/blog/feed.xml - - http://googlerussiablog.blogspot.com/atom.xml - - https://eng.uber.com/feed/ - - http://web.mit.edu/newsoffice/topic/mitcomputers-rss.xml - - http://feeds.feedburner.com/corner-squareup-com - - http://googleresearch.blogspot.com/atom.xml - - http://githubengineering.com/atom.xml - - http://labs.spotify.com/feed/ - - http://blog.kaggle.com/feed/ - - http://clubs.ya.ru/company/rss/posts.xml - name: Блоги людей slug: people feeds: - - url: http://www.rssmix.com/2/ - rss: http://www.rssmix.com/u/10538572/rss.xml - columns: 3 + - columns: 3 + view: "feeds/favicons.html" mix: - http://nedbatchelder.com/blog/rss.xml - http://rasskazov.pro/blog/?go=rss/ - http://vas3k.ru/rss/ - - http://www.dmitriysivak.com/feed/ - http://nl.livejournal.com/data/rss - http://sashavolkova.ru/rss/ - http://alexmak.net/blog/feed/ @@ -410,6 +366,81 @@ boards: - http://feeds.feedburner.com/codinghorror/ - http://theoatmeal.com/feed/rss - https://waitbutwhy.com/feed + - name: Инди-разработка + slug: make + feeds: + - name: Show HN + url: https://news.ycombinator.com/show + rss: https://hnrss.org/show + - name: Starter Story + url: https://www.starterstory.com + rss: https://www.starterstory.com/feed?format=rss + - name: 'Reddit: /r/SideProject' + url: https://www.reddit.com/r/SideProject/ + rss: https://www.reddit.com/r/SideProject.rss + is_parsable: false + - name: Путешествия + slug: travel + feeds: + - name: PeritoBurrito + url: https://perito-burrito.com + rss: http://perito-burrito.com/feed + - name: Vandrouki + url: https://vandrouki.ru + rss: https://feeds.feedburner.com/vandroukiru + icon: https://i.vas3k.ru/fer.jpg + - name: Secret Flying + url: https://www.secretflying.com + rss: https://www.secretflying.com/feed/ + - name: 'Atlas Obscura: Stories' + url: https://www.atlasobscura.com/articles + rss: https://www.atlasobscura.com/feeds/latest + icon: https://i.vas3k.ru/345139fb86cb52076134880d1b4ef700d6354c4cf4639ebdfaf1f9891115f7ad.jpg + - name: "T—Ж" + url: https://journal.tinkoff.ru/chemodan/ + rss: https://journal.tinkoff.ru/feed/ + - name: Geeky Explorer + url: https://www.geekyexplorer.com + rss: https://www.geekyexplorer.com/feed/ + - name: Европейское айти + slug: eu + feeds: + - name: "EU-startups" + url: https://www.eu-startups.com/ + rss: https://www.eu-startups.com/feed/ + icon: https://i.vas3k.ru/fkp.jpg + - name: Tech.eu + url: https://tech.eu/ + rss: https://tech.eu/feed/ + icon: https://i.vas3k.ru/fl9.jpg + - name: "TechCrunch: Europe" + url: https://techcrunch.com/europe/ + rss: http://feeds.feedburner.com/Techcrunch/europe + is_parsable: false + - name: Микс из блогов крупных компаний + slug: engineering + feeds: + - columns: 3 + view: "feeds/favicons.html" + mix: + - http://blog.stackoverflow.com/feed/ + - https://machinelearning.apple.com/feed.xml + - http://nerds.airbnb.com/feed/ + - http://www.ebaytechblog.com/feed/ + - http://blog.agilebits.com/feed/ + - http://research.microsoft.com/rss/news.xml + - http://blog.keras.io/feeds/all.atom.xml + - http://techblog.netflix.com/feeds/posts/default + - http://nickcraver.com/blog/feed.xml + - http://googlerussiablog.blogspot.com/atom.xml + - https://eng.uber.com/feed/ + - http://web.mit.edu/newsoffice/topic/mitcomputers-rss.xml + - http://feeds.feedburner.com/corner-squareup-com + - http://googleresearch.blogspot.com/atom.xml + - http://githubengineering.com/atom.xml + - http://labs.spotify.com/feed/ + - http://blog.kaggle.com/feed/ + - http://clubs.ya.ru/company/rss/posts.xml - name: Фотография slug: photo feeds: @@ -629,6 +660,7 @@ boards: - name: "DeepMind" url: https://www.deepmind.com/blog rss: https://www.deepmind.com/blog/feed/basic/ + icon: https://i.vas3k.ru/aff485d139d37ac7236f0bdf831812a2ef2419972ca3996c66885f229dccf7e2.jpg - name: "Google" url: https://ai.googleblog.com/ rss: http://rssmix.com/u/10966870/rss.xml @@ -721,6 +753,7 @@ boards: - name: "DeepMind: The Podcast" url: https://deepmind.com/blog/article/welcome-to-the-deepmind-podcast rss: https://feeds.simplecast.com/JT6pbPkg + icon: https://i.vas3k.ru/aff485d139d37ac7236f0bdf831812a2ef2419972ca3996c66885f229dccf7e2.jpg - name: "Microsoft Research Podcast" icon: https://i.vas3k.ru/i0t.png url: https://www.microsoft.com/en-us/research/blog/category/podcast/ @@ -2140,4 +2173,3 @@ boards: - name: "QA Mania (Украиноязычный канал)" url: https://t.me/qamania rss: https://infomate.club/parsing/telegram/qamania - diff --git a/boards/icons.py b/boards/icons.py index d8676f4..39099d8 100644 --- a/boards/icons.py +++ b/boards/icons.py @@ -4,3 +4,25 @@ DOMAIN_ICONS = { "reddit.com": "fa:fab fa-reddit-alien", "github.com": "fa:fab fa-github", } + +DOMAIN_FAVICONS = { + "bbc.com": "https://i.vas3k.ru/635c5e5828a4868b73bdb777611084a3459873b628f3f7f9752a34e1516fc505.png", + "bbc.co.uk": "https://i.vas3k.ru/635c5e5828a4868b73bdb777611084a3459873b628f3f7f9752a34e1516fc505.png", + "cnn.com": "https://i.vas3k.ru/d2d88f4a1d1646bf3a70f76d6b585c472ee1735abd73dd30719d7f2f42f5743a.png", + "cnn.it": "https://i.vas3k.ru/d2d88f4a1d1646bf3a70f76d6b585c472ee1735abd73dd30719d7f2f42f5743a.png", + "foxnews.com": "https://i.vas3k.ru/46b5aabb4269c34ff22c90afeae1cf4b2fc64078efb0611dbddd53d336395ea4.png", + "vox.com": "https://i.vas3k.ru/1a7ab1f00077e12d840d236e4438db9cc47a21b2b25c9cedea68ea9b167bb7d3.png", + "buzzfeednews.com": "https://i.vas3k.ru/ebe634861320fccd85c2fe96a5c93efd2b6643c2e191e3c8ee4b2ac0fb5b5e1c.png", + "buzzfeed.com": "https://i.vas3k.ru/ebe634861320fccd85c2fe96a5c93efd2b6643c2e191e3c8ee4b2ac0fb5b5e1c.png", + "huffpost.com": "https://i.vas3k.ru/a357095a972b2846476d95fb045de8880fe6e16f77fb34f7e07cbb13f196cb0b.png", + "newyorker.com": "https://i.vas3k.ru/b807c76f8514e36216d3593d9585bfe0335ff1514a5913e475a97fcc52e1fe21.png", + "dailymail.co.uk": "https://i.vas3k.ru/16371b2d1711d17cae30aa943e337b1486442d8ea471b08e2f3bd29f14239a81.png", + "dailymail.com": "https://i.vas3k.ru/16371b2d1711d17cae30aa943e337b1486442d8ea471b08e2f3bd29f14239a81.png", + "washingtontimes.com": "https://i.vas3k.ru/6da045463763191ed01c6bced5321f47d10d6804d244baee335b7702efbd8bc0.png", + "msnbc.com": "https://i.vas3k.ru/1b87330e504abdd0ffa6767827f9a2a3475c72ecc1eb4c80e3cd73e99b5d348a.png", + "vas3k.ru": "https://i.vas3k.ru/90e7cad4728678bf11dd2379dbcbda94f485b92ccd229d492a1bf2da31441c7c.png", + "vas3k.com": "https://i.vas3k.ru/90e7cad4728678bf11dd2379dbcbda94f485b92ccd229d492a1bf2da31441c7c.png", + "maximilyahov.ru": "https://i.vas3k.ru/5a40caa8b7c7ebeab10516ebc7402151d4c98cd96704b9149d4869f6028bf642.jpg", + "skaplichniy.ru": "https://i.vas3k.ru/c0c5e651be84c1a84b6b819448605def65db8373607fea958bc53a27ea7d3902.jpg", + "mikeozornin.ru": "https://i.vas3k.ru/31344c5d6928534842caf0b0f173205556baaea370ee0966853260c4713004b1.png", +} diff --git a/boards/migrations/0009_auto_20200905_1246.py b/boards/migrations/0009_auto_20200905_1246.py new file mode 100644 index 0000000..92fce97 --- /dev/null +++ b/boards/migrations/0009_auto_20200905_1246.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.13 on 2020-09-05 12:46 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('boards', '0008_boardfeed_is_parsable'), + ] + + operations = [ + migrations.AddField( + model_name='boardblock', + name='view', + field=models.CharField(default='blocks/three.html', max_length=256), + ), + migrations.AddField( + model_name='boardfeed', + name='mix', + field=django.contrib.postgres.fields.jsonb.JSONField(null=True), + ), + migrations.AddField( + model_name='boardfeed', + name='view', + field=models.CharField(default='feeds/simple.html', max_length=256), + ), + migrations.AlterField( + model_name='boardfeed', + name='url', + field=models.TextField(), + ), + ] diff --git a/boards/models.py b/boards/models.py index 9380c82..f335cba 100644 --- a/boards/models.py +++ b/boards/models.py @@ -6,7 +6,7 @@ from django.db import models from django.contrib.postgres.fields import JSONField from slugify import slugify -from boards.icons import DOMAIN_ICONS +from boards.icons import DOMAIN_ICONS, DOMAIN_FAVICONS class Board(models.Model): @@ -57,11 +57,15 @@ class Board(models.Model): class BoardBlock(models.Model): + DEFAULT_VIEW = "blocks/three.html" + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) board = models.ForeignKey(Board, related_name="blocks", on_delete=models.CASCADE, db_index=True) name = models.CharField(max_length=512, null=True) slug = models.SlugField() + view = models.CharField(max_length=256, default=DEFAULT_VIEW, null=False) + created_at = models.DateTimeField(db_index=True) updated_at = models.DateTimeField() @@ -71,6 +75,12 @@ class BoardBlock(models.Model): db_table = "board_blocks" ordering = ["index"] + @property + def template(self): + if self.view and self.view.endswith(".html"): + return self.view + return self.DEFAULT_VIEW + def save(self, *args, **kwargs): if not self.created_at: self.created_at = datetime.utcnow() @@ -84,14 +94,19 @@ class BoardBlock(models.Model): class BoardFeed(models.Model): + DEFAULT_VIEW = "feeds/simple.html" + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) board = models.ForeignKey(Board, related_name="feeds", on_delete=models.CASCADE, db_index=True) block = models.ForeignKey(BoardBlock, related_name="feeds", on_delete=models.CASCADE, db_index=True) name = models.CharField(max_length=512, null=True) comment = models.TextField(null=True) - url = models.URLField(max_length=512) + url = models.TextField() icon = models.URLField(max_length=512, null=True) rss = models.URLField(max_length=512, null=True) + mix = JSONField(null=True) + + view = models.CharField(max_length=256, default=DEFAULT_VIEW, null=False) created_at = models.DateTimeField(db_index=True) last_article_at = models.DateTimeField(null=True) @@ -109,6 +124,12 @@ class BoardFeed(models.Model): db_table = "board_feeds" ordering = ["index"] + @property + def template(self): + if self.view and self.view.endswith(".html"): + return self.view + return self.DEFAULT_VIEW + def save(self, *args, **kwargs): if not self.created_at: self.created_at = datetime.utcnow() @@ -167,6 +188,13 @@ class Article(models.Model): return f"""{self.domain} """ + def favicon(self): + if not self.domain: + return None + + favicon_domain = self.domain.lower().replace("www.", "") + return DOMAIN_FAVICONS.get(favicon_domain) + def natural_created_at(self): if not self.created_at: return None diff --git a/boards/views.py b/boards/views.py index 3a0aefe..b6b385b 100644 --- a/boards/views.py +++ b/boards/views.py @@ -6,7 +6,6 @@ from django.shortcuts import render, get_object_or_404 from django.views.decorators.cache import cache_page from django.views.decorators.http import last_modified -from auth.helpers import authorized_user from boards.cache import board_last_modified_at from boards.models import Board, BoardBlock, BoardFeed @@ -23,13 +22,6 @@ def index(request): def board(request, board_slug): board = get_object_or_404(Board, slug=board_slug) - if board.is_private: - me = authorized_user(request) - if not me: - return render(request, "board_no_access.html", { - "board": board - }, status=401) - cached_page = cache.get(f"board_{board.slug}") if cached_page and board.refreshed_at and board.refreshed_at <= \ datetime.utcnow() - timedelta(seconds=settings.BOARD_CACHE_SECONDS): @@ -46,15 +38,6 @@ def board(request, board_slug): return result -@cache_page(settings.STATIC_PAGE_CACHE_SECONDS) -def export(request, board_slug): - board = get_object_or_404(Board, slug=board_slug) - - return render(request, "export.html", { - "board": board, - }) - - @cache_page(settings.STATIC_PAGE_CACHE_SECONDS) def what(request): return render(request, "what.html") diff --git a/docker-compose.yml b/docker-compose.yml index eebc38e..216760c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,7 +27,7 @@ services: - POSTGRES_PASSWORD=postgres - POSTGRES_DB=infomate ports: - - 5432 + - "54321:5432" migrate_and_init: <<: *app diff --git a/infomate/settings.py b/infomate/settings.py index fe01599..200bdf8 100644 --- a/infomate/settings.py +++ b/infomate/settings.py @@ -16,7 +16,6 @@ INSTALLED_APPS = [ "django.contrib.staticfiles", "django.contrib.humanize", "django_bleach", - "auth", "boards", "parsing" ] @@ -39,7 +38,6 @@ TEMPLATES = [ "django.template.context_processors.debug", "django.template.context_processors.request", "boards.context_processors.settings_processor", - "auth.context_processors.me", ], }, }, diff --git a/infomate/urls.py b/infomate/urls.py index 6183d0e..6c4c0db 100644 --- a/infomate/urls.py +++ b/infomate/urls.py @@ -1,8 +1,7 @@ from django.urls import path from django.views.decorators.cache import cache_page -from auth.views import login, logout, club_callback -from boards.views import index, board, privacy_policy, what, export +from boards.views import index, board, privacy_policy, what from infomate import settings from parsing.views import TelegramChannelFeed @@ -12,12 +11,7 @@ urlpatterns = [ path("docs/privacy_policy/", privacy_policy, name="privacy_policy"), - path("auth/login/", login, name="login"), - path("auth/club_callback/", club_callback, name="club_callback"), - path("auth/logout/", logout, name="logout"), - path("/", board, name="board"), - path("/export/", export, name="export"), path("parsing/telegram//", cache_page(settings.TELEGRAM_CACHE_SECONDS)(TelegramChannelFeed()), diff --git a/requirements.txt b/requirements.txt index ae14306..27c9b4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ beautifulsoup4==4.6.2 pyyaml==5.2 feedparser==5.2.1 sentry-sdk==0.14.1 -pyjwt>=1.7.1 nltk==3.4.5 newspaper3k>=0.2.8 telethon==1.10.10 diff --git a/scripts/common.py b/scripts/common.py index 6ee9fd7..044b978 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -1,8 +1,18 @@ +import logging +import re import socket +from datetime import datetime +from time import mktime +from urllib.parse import urlparse + +from bs4 import BeautifulSoup +from requests import RequestException from urllib3.exceptions import InsecureRequestWarning import requests +log = logging.getLogger(__name__) + DEFAULT_REQUEST_HEADERS = { "User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) " "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 " @@ -13,3 +23,86 @@ MAX_PARSABLE_CONTENT_LENGTH = 15 * 1024 * 1024 # 15Mb socket.setdefaulttimeout(DEFAULT_REQUEST_TIMEOUT) requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) + + +def resolve_url(entry_link): + url = str(entry_link) + content_type = None + content_length = MAX_PARSABLE_CONTENT_LENGTH + 1 # don't parse null content-types + depth = 10 + while depth > 0: + depth -= 1 + + try: + response = requests.head(url, timeout=DEFAULT_REQUEST_TIMEOUT, verify=False, stream=True) + except RequestException: + log.warning(f"Failed to resolve URL: {url}") + return None, content_type, content_length + + if 300 < response.status_code < 400: + url = response.headers["location"] # follow redirect + else: + content_type = response.headers.get("content-type") + content_length = int(response.headers.get("content-length") or 0) + break + + return url, content_type, content_length + + +def parse_domain(url): + domain = urlparse(url).netloc + if domain.startswith("www."): + domain = domain[4:] + return domain + + +def parse_datetime(entry): + published_time = entry.get("published_parsed") or entry.get("updated_parsed") + if published_time: + return datetime.fromtimestamp(mktime(published_time)) + return datetime.utcnow() + + +def parse_title(entry): + title = entry.get("title") or entry.get("description") or entry.get("summary") + return re.sub("<[^<]+?>", "", title).strip() + + +def parse_link(entry): + if entry.get("link"): + return entry["link"] + + if entry.get("links"): + return entry["links"][0]["href"] + + return None + + +def parse_rss_image(entry): + if entry.get("media_content"): + images = [m["url"] for m in entry["media_content"] if m.get("medium") == "image" and m.get("url")] + if images: + return images[0] + + if entry.get("image"): + if isinstance(entry["image"], dict): + return entry["image"].get("href") + return entry["image"] + + return None + + +def parse_rss_text_and_image(entry): + if not entry.get("summary"): + return "", "" + + bs = BeautifulSoup(entry["summary"], features="lxml") + text = re.sub(r"\s\s+", " ", bs.text or "").strip() + + img_tags = bs.findAll("img") + for img_tag in img_tags: + src = img_tag.get("src", None) + if src: + return text, src + + return text, "" \ No newline at end of file diff --git a/scripts/initialize.py b/scripts/initialize.py index eac3874..7f9e615 100644 --- a/scripts/initialize.py +++ b/scripts/initialize.py @@ -11,12 +11,12 @@ from urllib.parse import urljoin import click import requests import yaml -import feedparser from bs4 import BeautifulSoup from boards.models import Board, BoardFeed, BoardBlock +from boards.icons import DOMAIN_FAVICONS from utils.images import upload_image_from_url -from scripts.common import DEFAULT_REQUEST_HEADERS +from scripts.common import DEFAULT_REQUEST_HEADERS, parse_domain @click.command() @@ -79,76 +79,84 @@ def initialize(config, board_slug, upload_favicons, always_yes): slug=block_config["slug"], defaults=dict( name=block_name, - index=block_index + index=block_index, + view=block_config.get("view") or BoardBlock.DEFAULT_VIEW, ) ) if not is_created: block.index = block_index block.name = block_name + block.view = block_config.get("view") or BoardBlock.DEFAULT_VIEW block.save() if not block_config.get("feeds"): continue + updated_feed_urls = set() + for feed_index, feed_config in enumerate(block_config.get("feeds") or []): feed_name = feed_config.get("name") - feed_url = feed_config["url"] + feed_mix = feed_config.get("mix") + if feed_mix: + feed_url = feed_config.get("url") or f"mix:{'|'.join(feed_mix)}" + feed_rss = None + else: + feed_url = feed_config["url"] + feed_rss = feed_config["rss"] + + updated_feed_urls.add(feed_url) - print(f"Creating or updating feed: {feed_name}...") + print(f"Creating or updating feed {feed_name} ({feed_url})...") feed, is_created = BoardFeed.objects.get_or_create( board=board, block=block, - url=feed_config["url"], + url=feed_url, defaults=dict( - rss=feed_config.get("rss"), + rss=feed_rss, + mix=feed_mix, name=feed_name, comment=feed_config.get("comment"), icon=feed_config.get("icon"), index=feed_index, columns=feed_config.get("columns") or 1, conditions=feed_config.get("conditions"), - is_parsable=feed_config.get("is_parsable", True) + is_parsable=feed_config.get("is_parsable", True), + view=feed_config.get("view") or BoardFeed.DEFAULT_VIEW, ) ) if not is_created: - feed.rss = feed_config.get("rss") + feed.rss = feed_rss + feed.mix = feed_mix feed.name = feed_name feed.comment = feed_config.get("comment") feed.index = feed_index + feed.icon = feed.icon or feed_config.get("icon") feed.columns = feed_config.get("columns") or 1 feed.conditions = feed_config.get("conditions") feed.is_parsable = feed_config.get("is_parsable", True) + feed.view = feed_config.get("view") or BoardFeed.DEFAULT_VIEW html = None - if not feed.rss: - html = html or load_page_html(feed_url) - rss_url = feed_config.get("rss") - if not rss_url: - rss_url = find_rss_feed(feed_url, html) - if not rss_url: - print(f"RSS feed for '{feed_name}' not found. " - f"Please specify 'rss' key.") - exit(1) - print(f"- found RSS: {rss_url}") + if not feed.mix: + if not feed.icon: + feed.icon = DOMAIN_FAVICONS.get(parse_domain(feed_url)) - feed.rss = rss_url + if not feed.icon: + html = html or load_page_html(feed_url) + icon = feed_config.get("icon") + if not icon: + icon = find_favicon(feed_url, html) + print(f"- found favicon: {icon}") - if not feed.icon: - html = html or load_page_html(feed_url) - icon = feed_config.get("icon") - if not icon: - icon = find_favicon(feed_url, html) - print(f"- found favicon: {icon}") + if upload_favicons: + icon = upload_image_from_url(icon) + print(f"- uploaded favicon: {icon}") - if upload_favicons: - icon = upload_image_from_url(icon) - print(f"- uploaded favicon: {icon}") - - feed.icon = icon + feed.icon = icon feed.save() @@ -157,7 +165,7 @@ def initialize(config, board_slug, upload_favicons, always_yes): board=board, block=block ).exclude( - url__in={feed["url"] for feed in block_config.get("feeds") or []} + url__in=updated_feed_urls ).delete() # delete unused blocks @@ -180,32 +188,32 @@ def load_page_html(url): ).text -def find_rss_feed(url, html): - bs = BeautifulSoup(html, features="lxml") - possible_feeds = set() - - feed_urls = bs.findAll("link", rel="alternate") - for feed_url in feed_urls: - t = feed_url.get("type", None) - if t: - if "rss" in t or "xml" in t: - href = feed_url.get("href", None) - if href: - possible_feeds.add(urljoin(url, href)) - - a_tags = bs.findAll("a") - for a in a_tags: - href = a.get("href", None) - if href: - if "xml" in href or "rss" in href or "feed" in href: - possible_feeds.add(urljoin(url, href)) - - for feed_url in possible_feeds: - feed = feedparser.parse(feed_url) - if feed.entries: - return feed_url - - return None +# def find_rss_feed(url, html): +# bs = BeautifulSoup(html, features="lxml") +# possible_feeds = set() +# +# feed_urls = bs.findAll("link", rel="alternate") +# for feed_url in feed_urls: +# t = feed_url.get("type", None) +# if t: +# if "rss" in t or "xml" in t: +# href = feed_url.get("href", None) +# if href: +# possible_feeds.add(urljoin(url, href)) +# +# a_tags = bs.findAll("a") +# for a in a_tags: +# href = a.get("href", None) +# if href: +# if "xml" in href or "rss" in href or "feed" in href: +# possible_feeds.add(urljoin(url, href)) +# +# for feed_url in possible_feeds: +# feed = feedparser.parse(feed_url) +# if feed.entries: +# return feed_url +# +# return None def find_favicon(url, html): diff --git a/scripts/update.py b/scripts/update.py index f7b293f..99717cc 100644 --- a/scripts/update.py +++ b/scripts/update.py @@ -6,27 +6,25 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "..")) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "infomate.settings") django.setup() -import re import logging from datetime import timedelta, datetime -from urllib.parse import urlparse -from time import mktime import threading import queue import requests import click import feedparser -from bs4 import BeautifulSoup from requests import RequestException from newspaper import Article as NewspaperArticle, ArticleException from boards.models import BoardFeed, Article, Board -from scripts.common import DEFAULT_REQUEST_HEADERS, DEFAULT_REQUEST_TIMEOUT, MAX_PARSABLE_CONTENT_LENGTH +from scripts.common import DEFAULT_REQUEST_HEADERS, DEFAULT_REQUEST_TIMEOUT, MAX_PARSABLE_CONTENT_LENGTH, resolve_url, \ + parse_domain, parse_datetime, parse_title, parse_link, parse_rss_image, parse_rss_text_and_image DEFAULT_NUM_WORKER_THREADS = 5 -DEFAULT_ENTRIES_LIMIT = 100 +DEFAULT_ENTRIES_LIMIT = 30 MIN_REFRESH_DELTA = timedelta(minutes=30) +DELETE_OLD_ARTICLES_DELTA = timedelta(days=300) log = logging.getLogger() queue = queue.Queue() @@ -57,6 +55,7 @@ def update(num_workers, force, feed): "board_id": feed.board_id, "name": feed.name, "rss": feed.rss, + "mix": feed.mix, "conditions": feed.conditions, "is_parsable": feed.is_parsable, }) @@ -78,6 +77,9 @@ def update(num_workers, force, feed): updated_boards = {feed.board_id for feed in need_to_update_feeds} Board.objects.filter(id__in=updated_boards).update(refreshed_at=datetime.utcnow()) + # remove old data + Article.objects.filter(created_at__lte=datetime.now() - DELETE_OLD_ARTICLES_DELTA).delete() + # stop workers for i in range(num_workers): queue.put(None) @@ -103,7 +105,28 @@ def worker(): def refresh_feed(item): print(f"Updating feed {item['name']}...") - feed = feedparser.parse(item['rss']) + if item["mix"]: + for rss in item["mix"]: + fetch_rss(item, rss) + else: + fetch_rss(item, item["rss"]) + + week_ago = datetime.utcnow() - timedelta(days=7) + frequency = Article.objects.filter(feed_id=item["id"], created_at__gte=week_ago).count() + last_article = Article.objects.filter(feed_id=item["id"]).order_by("-created_at").first() + + BoardFeed.objects.filter(id=item["id"]).update( + refreshed_at=datetime.utcnow(), + last_article_at=last_article.created_at if last_article else None, + frequency=frequency or 0 + ) + + +def fetch_rss(item, rss): + print(f"Parsing RSS: {rss}") + + feed = feedparser.parse(rss) + print(f"Entries found: {len(feed.entries)}") for entry in feed.entries[:DEFAULT_ENTRIES_LIMIT]: entry_title = parse_title(entry) @@ -113,14 +136,14 @@ def refresh_feed(item): continue print(f"- article: '{entry_title}' {entry_link}") - - conditions = item.get("conditions") + + conditions = item.get("conditions") if conditions: is_valid = check_conditions(conditions, entry) if not is_valid: print(f"Condition {conditions} does not match. Skipped") continue - + article, is_created = Article.objects.get_or_create( board_id=item["board_id"], feed_id=item["id"], @@ -171,16 +194,6 @@ def refresh_feed(item): article.save() - week_ago = datetime.utcnow() - timedelta(days=7) - frequency = Article.objects.filter(feed_id=item["id"], created_at__gte=week_ago).count() - last_article = Article.objects.filter(feed_id=item["id"]).order_by("-created_at").first() - - BoardFeed.objects.filter(id=item["id"]).update( - refreshed_at=datetime.utcnow(), - last_article_at=last_article.created_at if last_article else None, - frequency=frequency or 0 - ) - def check_conditions(conditions, entry): if not conditions: @@ -188,95 +201,16 @@ def check_conditions(conditions, entry): for condition in conditions: if condition["type"] == "in": - if condition["in"] not in entry[condition["field"]]: + if condition["word"] not in entry[condition["field"]]: + return False + + if condition["type"] == "not_in": + if condition["word"] in entry[condition["field"]]: return False return True -def resolve_url(entry_link): - url = str(entry_link) - content_type = None - content_length = MAX_PARSABLE_CONTENT_LENGTH + 1 # don't parse null content-types - depth = 10 - while depth > 0: - depth -= 1 - - try: - response = requests.head(url, timeout=DEFAULT_REQUEST_TIMEOUT, verify=False, stream=True) - except RequestException: - log.warning(f"Failed to resolve URL: {url}") - return None, content_type, content_length - - if 300 < response.status_code < 400: - url = response.headers["location"] # follow redirect - else: - content_type = response.headers.get("content-type") - content_length = int(response.headers.get("content-length") or 0) - break - - return url, content_type, content_length - - -def parse_domain(url): - domain = urlparse(url).netloc - if domain.startswith("www."): - domain = domain[4:] - return domain - - -def parse_datetime(entry): - published_time = entry.get("published_parsed") or entry.get("updated_parsed") - if published_time: - return datetime.fromtimestamp(mktime(published_time)) - return datetime.utcnow() - - -def parse_title(entry): - title = entry.get("title") or entry.get("description") or entry.get("summary") - return re.sub("<[^<]+?>", "", title).strip() - - -def parse_link(entry): - if entry.get("link"): - return entry["link"] - - if entry.get("links"): - return entry["links"][0]["href"] - - return None - - -def parse_rss_image(entry): - if entry.get("media_content"): - images = [m["url"] for m in entry["media_content"] if m.get("medium") == "image" and m.get("url")] - if images: - return images[0] - - if entry.get("image"): - if isinstance(entry["image"], dict): - return entry["image"].get("href") - return entry["image"] - - return None - - -def parse_rss_text_and_image(entry): - if not entry.get("summary"): - return "", "" - - bs = BeautifulSoup(entry["summary"], features="lxml") - text = re.sub(r"\s\s+", " ", bs.text or "").strip() - - img_tags = bs.findAll("img") - for img_tag in img_tags: - src = img_tag.get("src", None) - if src: - return text, src - - return text, "" - - def load_page_safe(url): try: response = requests.get( diff --git a/static/css/components.css b/static/css/components.css index e3af3b7..aec3e0e 100644 --- a/static/css/components.css +++ b/static/css/components.css @@ -120,7 +120,7 @@ .curator-bio { font-size: 120%; - padding-top: 20px; + padding-top: 10px; } .curator-info a { @@ -256,10 +256,19 @@ text-decoration: none; } + .article-favicon { + max-width: 20px; + max-height: 20px; + padding-right: 4px; + display: inline-block; + vertical-align: middle; + } + .article-tooltip { visibility: hidden; transition: visibility 0.1s; width: 100%; + max-width: 400px; position: absolute; bottom: 25px; left: 0; diff --git a/static/css/layout.css b/static/css/layout.css index 3e60e97..02e2e32 100644 --- a/static/css/layout.css +++ b/static/css/layout.css @@ -46,7 +46,10 @@ } .board { - display: block; + display: flex; + flex-wrap: wrap; + justify-content: space-between; + align-items: flex-start; position: relative; max-width: var(--max-content-width); margin: 0 auto; @@ -56,21 +59,51 @@ display: block; } + .block-100 { + width: 100%; + } + + .block-50 { + width: 49%; + } + + @media only screen and (max-width : 570px) { + .block-100, .block-50 { + width: auto; + } + } + + .feeds { - display: grid; - grid-template-columns: 33% 33% 33%; - grid-template-rows: auto auto; + display: flex; + flex-wrap: wrap; + justify-content: flex-start; + align-items: flex-start; } + .feed { + width: 100%; + box-sizing: border-box; + } + + .feeds-3 .feed { + width: 33%; + } + + .feeds-2 .feed { + width: 49%; + } + .feed .article-tooltip { /* Shift tooltip to right */ - left: 85%; + left: 250px; top: -100px; right: auto; bottom: auto; } - .feed:nth-child(3n) .article-tooltip { + .feeds-3 .feed:nth-child(3n) .article-tooltip, + .feeds-2 .feed:nth-child(2n) .article-tooltip { /* Shift tooltip to left */ left: -105%; top: -100px; diff --git a/templates/blocks/three.html b/templates/blocks/three.html new file mode 100644 index 0000000..1c9c33b --- /dev/null +++ b/templates/blocks/three.html @@ -0,0 +1,13 @@ +
+ {% if block.name %} +
{{ block.name }}
+ {% endif %} + +
+ {% for feed in feeds %} + {% if feed.block == block %} + {% include feed.template %} + {% endif %} + {% endfor %} +
+
diff --git a/templates/blocks/two.html b/templates/blocks/two.html new file mode 100644 index 0000000..616f1ec --- /dev/null +++ b/templates/blocks/two.html @@ -0,0 +1,13 @@ +
+ {% if block.name %} +
{{ block.name }}
+ {% endif %} + +
+ {% for feed in feeds %} + {% if feed.block == block %} + {% include feed.template %} + {% endif %} + {% endfor %} +
+
diff --git a/templates/board.html b/templates/board.html index 2919e5f..e5b3677 100644 --- a/templates/board.html +++ b/templates/board.html @@ -1,7 +1,6 @@ {% extends "layout.html" %} {% load text_filters %} {% load static %} -{% load bleach_tags %} {% block title %}{{ board.curator_name }}{% if board.curator_title %} | {{ board.curator_title }}{% endif %} | {{ block.super }}{% endblock %} @@ -46,57 +45,7 @@ {% endif %} {% for block in blocks %} -
- {% if block.name %} -
{{ block.name }}
- {% endif %} - -
- {% for feed in feeds %} - {% if feed.block == block %} - {% for column, articles in feed.articles_by_column %} -
- {% if feed.name %} -
- {% if feed.icon %} - {{ feed.name }} - {% endif %} - {{ feed.name }}
- последний пост {{ feed.natural_last_article_at }} -
- {% endif %} -
- {% for article in articles %} -
-
{{ article.icon|safe }}{{ article.title|bleach }}
- - {% if article.image %} - {{ article.title|striptags }} - {% endif %} - - {{ article.title|striptags|truncatechars:300 }} - - {% if article.description or article.summary %} - - {% if feed.is_parsable and article.summary %} - {{ article.summary|striptags|truncatechars:700|escape|nl2p|safe }} - {% else %} - {{ article.description|striptags|truncatechars:700|escape|nl2p|safe }} - {% endif %} - - {% endif %} - - {{ article.natural_created_at }} @ {{ article.domain }} - -
- {% endfor %} -
-
- {% endfor %} - {% endif %} - {% endfor %} -
-
+ {% include block.template %} {% endfor %} {% endblock %} @@ -110,10 +59,6 @@ {% if board.natural_refreshed_at %}
Обновлено {{ board.natural_refreshed_at }} {% endif %} - -
- Экспортировать подборку себе -
{% endif %} {% endblock %} diff --git a/templates/board_no_access.html b/templates/board_no_access.html deleted file mode 100644 index 1084c4f..0000000 --- a/templates/board_no_access.html +++ /dev/null @@ -1,230 +0,0 @@ -{% extends "board.html" %} - -{% block board %} -
-
-
Привет, друг! Тоже думал, что самый умный? Штош...
-
-
-
- twitter - Twitter
- последний недушный пост: никогда -
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
-
-
- twitter - Facebook
- верни мои данные! -
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
-
-
- twitter - Одноклассники
- все скорее туда -
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
-
-
- twitter - Телеграм
- чо, многому научились из каналов? -
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
-
-
- twitter - Вконтакте
- верни стену -
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
Возьми с полки пирожок, хакир. Ты заслужил!
-
-
-
-
-
- -{% endblock %} diff --git a/templates/export.html b/templates/export.html deleted file mode 100644 index c2db5f6..0000000 --- a/templates/export.html +++ /dev/null @@ -1,21 +0,0 @@ -{% extends "layout.html" %} -{% load text_filters %} -{% load static %} - -{% block title %}Экспорт | {{ board.curator_name }} | {{ board.curator_title }} | {{ block.super }}{% endblock %} - -{% block content %} -
-
-

Экспорт скоро будет!

- -

- Эта идея появилась совсем недавно и мы пока думаем как её сделать удобнее всего. -

- -

- Подпишитесь на мой канал ⭐️ Вастрик.Пынь чтобы узнать сразу когда он появится. -

-
-
-{% endblock %} \ No newline at end of file diff --git a/templates/feeds/favicons.html b/templates/feeds/favicons.html new file mode 100644 index 0000000..565eae0 --- /dev/null +++ b/templates/feeds/favicons.html @@ -0,0 +1,27 @@ +{% load static %} +{% load text_filters %} +{% load bleach_tags %} + +{% for column, articles in feed.articles_by_column %} +
+ {% if feed.name %} +
+ {% if feed.icon %} + {{ feed.name }} + {% endif %} + {{ feed.name }}
+ последний пост {{ feed.natural_last_article_at }} +
+ {% endif %} +
+ {% for article in articles %} +
+
+ {% if article.favicon %}>{% endif %}{{ article.title|bleach }} +
+ {% include "tooltips/simple.html" %} +
+ {% endfor %} +
+
+{% endfor %} diff --git a/templates/feeds/simple.html b/templates/feeds/simple.html new file mode 100644 index 0000000..3134a5e --- /dev/null +++ b/templates/feeds/simple.html @@ -0,0 +1,27 @@ +{% load static %} +{% load text_filters %} +{% load bleach_tags %} + +{% for column, articles in feed.articles_by_column %} +
+ {% if feed.name %} +
+ {% if feed.icon %} + {{ feed.name }} + {% endif %} + {{ feed.name }}
+ последний пост {{ feed.natural_last_article_at }} +
+ {% endif %} +
+ {% for article in articles %} +
+
+ {{ article.icon|safe }}{{ article.title|bleach }} +
+ {% include "tooltips/simple.html" %} +
+ {% endfor %} +
+
+{% endfor %} diff --git a/templates/layout.html b/templates/layout.html index dd19f04..4a060ec 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -41,11 +41,8 @@ {% block footer %}
- Сделал Вастрик. Код проекта открыт.

- Сайт использует куки для авторизации
и собирает анонимные данные для статистики. - {% if me %} -
Выйти - {% endif %} + Проект Вастрик.Клуба. Код открыт.
+ Идеи и предложения новых источников присылайте на гитхаб.

{% endblock %} diff --git a/templates/tooltips/simple.html b/templates/tooltips/simple.html new file mode 100644 index 0000000..238247b --- /dev/null +++ b/templates/tooltips/simple.html @@ -0,0 +1,21 @@ +{% load text_filters %} + + + {% if article.image %} + {{ article.title|striptags }} + {% endif %} + + {{ article.title|striptags|truncatechars:300 }} + + {% if article.description or article.summary %} + + {% if feed.is_parsable and article.summary %} + {{ article.summary|striptags|truncatechars:700|escape|nl2p|safe }} + {% else %} + {{ article.description|striptags|truncatechars:700|escape|nl2p|safe }} + {% endif %} + + {% endif %} + + {{ article.natural_created_at }} @ {{ article.domain }} +