Add "Events" page

README.md

@@ -11,9 +11,13 @@ Czech Python community homepage.
 The code is **Python 3** (the production Python version is 3.6).
 
 ```sh
-$ pip install .
+$ git clone git@github.com:pyvec/python.cz.git ./python.cz --depth=10
+$ cd ./python.cz
+$ pip install -e .
 ```
 
+It is recommended to start with a [shallow clone](https://git-scm.com/docs/git-clone#git-clone---depthltdepthgt) as historically, this repo has contained a lot of rather large photos.
+
 ### Development
 
 The site uses GitHub API. For certian pages to work correctly, you need to set the `GITHUB_TOKEN` environment variable to a [GitHub Personal Access Token](https://github.com/settings/tokens) (no scopes needed).

pythoncz/__init__.py

@@ -1,5 +1,6 @@
 import os
 
+import yaml
 from flask import Flask
 from flask_frozen import Freezer
 
@@ -12,6 +13,7 @@
 TEMPLATES_DIR_URL = GITHUB_EDIT_URL + '/templates/'
 BUSINESS_LIST_URL = GITHUB_EDIT_URL + '/static/data/business.geojson'
 BEGINNERS_DATA_URL = GITHUB_EDIT_URL + '/static/data/beginners.yml'
+EVENTS_DATA_URL = GITHUB_EDIT_URL + '/static/data/events_feeds.yml'
 JOBS_DATA_URL = GITHUB_EDIT_URL + '/static/data/jobs.yml'
 
 PYVEC_ACCOUNT_URL = (
@@ -21,6 +23,7 @@
 GET_INVOLVED_URL = (
     'https://github.com/pulls?utf8=%E2%9C%93&q=is%3Aopen+org%3Apyvec'
 )
+
 GITHUB_ORGANIZATIONS = ('pyvec', 'pyladiescz')
 GOOGLE_ANALYTICS_CODE = 'UA-1316071-13'
 GITHUB_TOKEN = os.getenv('GITHUB_TOKEN') or os.getenv('GH_TOKEN')
@@ -29,6 +32,14 @@
 
 app = Flask('pythoncz')
 
+
+with open(os.path.join(app.static_folder, 'data', 'events_feeds.yml')) as f:
+    app.config['CZECH_PYTHON_EVENTS_CALENDAR_URL'] = next(filter(
+        lambda feed: feed['name'] == 'Czech Python Events',
+        yaml.safe_load(f.read())['feeds']
+    ))['url']
+
+
 freezer = Freezer(app)
 
 app.config.from_object(__name__)

pythoncz/models/beginners.py

@@ -11,4 +11,4 @@
 # Data loaded on import time so file system is not read and YAML parsed
 # on every request.
 with open(os.path.join(app.static_folder, 'data', 'beginners.yml')) as f:
-    data = yaml.load(f.read())
+    data = yaml.safe_load(f.read())

pythoncz/models/events.py

@@ -0,0 +1,75 @@
+from pathlib import Path
+from typing import List
+
+import yaml
+import requests
+import ics
+import arrow
+
+from pythoncz import app
+
+feeds_datafile = Path(app.root_path) / "static/data/events_feeds.yml"
+
+data = []
+
+
+def preprocess_ical(text: str) -> List[str]:
+    """ Removes VALARM ACTION:NONE from the iCal file (included in some events from Google Calendar)
+    """
+    lines = text.splitlines()
+    new_lines = []
+
+    i = 0
+
+    try:
+        while i < len(lines):
+            if lines[i] != "BEGIN:VALARM" or lines[i + 1].replace("ACTION:", "") in {"DISPLAY", "AUDIO"}:
+                new_lines.append(lines[i])
+                i += 1
+                continue
+
+            while lines[i] != "END:VALARM":
+                i += 1
+
+            i += 1
+    except IndexError:
+        raise ValueError("Can't preprocess iCal file to remove ACTION:NONE")
+
+    return new_lines
+
+
+def load_events(ical_url):
+    try:
+        response = requests.get(ical_url)
+
+        response.raise_for_status()
+    except Exception as e:
+        raise ValueError(f"Could not load iCal feed from {ical_url}") from e
+
+    ical = preprocess_ical(response.text)
+
+    calendar = ics.Calendar(ical)
+
+    return calendar.events
+
+
+@app.before_first_request
+def load_data():
+    feeds = yaml.safe_load(feeds_datafile.read_text())["feeds"]
+
+    now = arrow.now("Europe/Prague")
+
+    for feed in feeds:
+        data.extend(
+            {"feed": feed, "event": event} for event in load_events(feed["ical"]) if event.begin > now
+        )
+
+    data.sort(key=lambda e: e["event"].begin)
+
+
+def get_calendar():
+    events = [e["event"] for e in data]
+
+    calendar = ics.Calendar(events=events)
+
+    return calendar

pythoncz/models/jobs.py

@@ -54,7 +54,7 @@ def item_key(item):
 # jobs.yml
 _path = os.path.join(app.static_folder, 'data', 'jobs.yml')
 with open(os.path.join(_path)) as f:
-    data = yaml.load(f.read())
+    data = yaml.safe_load(f.read())
 
 data['job_boards'] = _sort_cs(data['job_boards'])
 data['knowledge_tests'] = _sort_cs(data['knowledge_tests'])

pythoncz/models/meetups.py

@@ -0,0 +1,69 @@
+import requests
+from lxml import html
+from slugify import slugify
+
+
+__all__ = ('get_meetups',)
+
+
+WIKI_URL = ('https://cs.wikipedia.org/wiki/'
+            'Seznam_m%C4%9Bst_v_%C4%8Cesku_podle_po%C4%8Dtu_obyvatel')
+
+
+def get_meetups(lang='cs'):
+    return sort_by_city_size(scrape_meetups(lang))
+
+
+def scrape_meetups(lang='cs'):
+    """
+    Ideally, pyvo.cz would have an API where we get all this info. Let's assume
+    HTML API is good enough API for us now.
+    """
+    url = 'https://pyvo.cz/en/' if lang == 'en' else 'https://pyvo.cz/'
+    res = requests.get(url, headers={'Accept-Charset': 'utf-8'})
+    res.raise_for_status()
+
+    root = html.fromstring(res.content.decode('utf-8'))
+    root.make_links_absolute(res.url)
+
+    for event in root.cssselect('#events .event'):
+        try:
+            yield {
+                'name': event.cssselect('h3')[0].text_content().strip(),
+                'url': event.cssselect('h3 a')[0].get('href'),
+            }
+        except IndexError:
+            continue
+
+
+def sort_by_city_size(meetups):
+    """
+    Sorts given iterable of meetups by the size of the city. While pyvo.cz
+    lists the meetups according to when the closest event happens or happened,
+    this doesn't make sense for python.cz where the meetups are listed just
+    as a general overview. Also alphabetical sorting is pretty much just
+    confusing for the visitor. It only makes sense to sort the meetups by the
+    size of the city. The most populated cities have a larger probability
+    that the visitor of the page is close to them, thus they deserve to be
+    higher in the list.
+    """
+    res = requests.get(WIKI_URL)
+    res.raise_for_status()
+
+    root = html.fromstring(res.content.decode('utf-8'))
+    rows = root.cssselect('.wikitable tbody tr')
+    rows.pop(0)
+
+    city_slugs = [
+        slugify(row.cssselect('td')[0].text_content().strip()) + '-pyvo'
+        for row in rows
+    ]
+
+    # fix discrepancies
+    city_slugs[city_slugs.index('hradec-kralove-pyvo')] = 'hradec-pyvo'
+
+    def key_func(meetup):
+        slug = meetup['url'].rstrip('/').split('/')[-1]
+        return city_slugs.index(slug)
+
+    return sorted(meetups, key=key_func)

pythoncz/static/data/events_feeds.yml

@@ -0,0 +1,13 @@
+feeds:
+  - name: Pyvo
+    url: https://pyvo.cz/
+    ical: https://pyvo.cz/api/pyvo.ics
+  - name: PyWorking
+    url: https://pyworking.cz/
+    ical: https://pyworking.cz/workshops.ics
+  - name: PyWorking Sessions
+    url: https://pyworking.cz/
+    ical: http://api.meetup.com/pyworking/upcoming.ical
+  - name: Czech Python Events
+    url: https://calendar.google.com/calendar/embed?src=kfdeelic1a13jsp7jvai861vfs%40group.calendar.google.com&ctz=Europe%2FPrague
+    ical: https://calendar.google.com/calendar/ical/kfdeelic1a13jsp7jvai861vfs%40group.calendar.google.com/public/basic.ics