diff options
Diffstat (limited to 'management/parsers')
| -rw-r--r-- | management/parsers/abstractparser.py | 52 | ||||
| -rw-r--r-- | management/parsers/ups2017.py | 162 | ||||
| -rw-r--r-- | management/parsers/ups2018.py | 213 |
3 files changed, 427 insertions, 0 deletions
diff --git a/management/parsers/abstractparser.py b/management/parsers/abstractparser.py new file mode 100644 index 0000000..8d55b6d --- /dev/null +++ b/management/parsers/abstractparser.py @@ -0,0 +1,52 @@ +# Copyright (C) 2018 Alban Gruin +# +# celcatsanitizer is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# celcatsanitizer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with celcatsanitizer. If not, see <http://www.gnu.org/licenses/>. + +import abc +import requests + +import edt + + +class AbstractParser(metaclass=abc.ABCMeta): + def __init__(self, source): + self.source = source + self.user_agent = "celcatsanitizer/" + edt.VERSION + + def _make_request(self, url, user_agent=None, encoding="utf8", **kwargs): + user_agent = user_agent if user_agent is not None else self.user_agent + + params = kwargs["params"] if "params" in kwargs else {} + headers = kwargs["headers"] if "headers" in kwargs else {} + headers["User-Agent"] = user_agent + + req = requests.get(url, headers=headers, params=params) + req.encoding = encoding + + return req + + @abc.abstractmethod + def get_events(self): + pass + + @abc.abstractmethod + def get_update_date(self): + pass + + @abc.abstractmethod + def get_weeks(self): + pass + + def get_source(self): + return self._make_request(self.source.url) diff --git a/management/parsers/ups2017.py b/management/parsers/ups2017.py new file mode 100644 index 0000000..99ce34d --- /dev/null +++ b/management/parsers/ups2017.py @@ -0,0 +1,162 @@ +# Copyright (C) 2017-2018 Alban Gruin +# +# celcatsanitizer is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# celcatsanitizer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with celcatsanitizer. If not, see <http://www.gnu.org/licenses/>. + +import datetime +import re + +from bs4 import BeautifulSoup +from django.utils import timezone + +from ...models import Course, Group, Room +from .abstractparser import AbstractParser + + +def add_time(date, time): + ptime = datetime.datetime.strptime(time, "%H:%M") + delta = datetime.timedelta(hours=ptime.hour, minutes=ptime.minute) + return date + delta + + +class Parser(AbstractParser): + def __get_event(self, event, event_week, today): + """Renvoie une classe Course à partir d’un événement lu par BS4""" + # On récupère la date de l’évènement à partir de la semaine + # et de la semaine référencée, puis l’heure de début et de fin + date = event_week + datetime.timedelta(int(event.day.text)) + begin = add_time(date, event.starttime.text) + end = add_time(date, event.endtime.text) + + # On ne traite pas le cours si il commence après le moment du + # traitement + if today is not None and begin < today: + return + + # Création de l’objet cours + course = Course.objects.create(source=self.source, begin=begin, + end=end) + + # On récupère les groupes concernés par les cours + groups = [ + Group.objects.get_or_create( + source=self.source, celcat_name=item.text + )[0] + for item in event.resources.group.find_all("item") + ] + course.groups.add(*groups) + + # On récupère le champ « remarque » + if event.notes is not None: + course.notes = "\n".join(event.notes.find_all(text=True)) + + # On récupère le champ « nom » + if event.resources.module is not None: + course.name = event.resources.module.item.text + elif event.category is not None: + # Il est possible qu’un cours n’ait pas de nom. Dans ce + # cas, si le cours a un type, il devient son nom. + course.type = event.category.text + # Si il n’a pas de type, il obtiendra une valeur par + # défaut définie à l’avance. + + # Récupération du type de cours + if event.category is not None: + course.type = event.category.text + + # Si un cours a une salle attribuée, on les insère dans la + # base de données, et on les ajoute dans l’objet cours + if event.resources.room is not None: + rooms = [ + Room.objects.get_or_create(name=item.text)[0] + for item in event.resources.room.find_all("item") + ] + course.rooms.add(*rooms) + + return course + + def get_events(self, today, year=None, week=None): + """Récupère tous les cours disponibles dans l’emploi du temps Celcat. + Le traîtement se limitera à la semaine indiquée si il y en a une.""" + for event in self.soup.find_all("event"): + event_week = self.weeks[event.rawweeks.text] + event_week_num = event_week.isocalendar()[1] # Numéro de semaine + + # On passe le traitement si la semaine de l’événement ne + # correspond pas à la semaine passée, ou qu’il ne contient + # pas de groupe ou n’a pas de date de début ou de fin. + if ( + ( + event_week_num == week + and event_week.year == year + or year is None + or week is None + ) + and event.resources.group is not None + and event.starttime is not None + and event.endtime is not None + ): + course = self.__get_event(event, event_week, today) + + # On renvoie le cours si il n’est pas nul + if course is not None: + yield course + + def get_update_date(self): + # Explication de la regex + # + # (\d+)/(\d+)/(\d+)\s+(\d+):(\d+):(\d+) + # (\d+) au moins un nombre + # / un slash + # (\d+) au moins un nombre + # / un slash + # (\d+) au moins un nombre + # \s+ au moins un espace + # (\d+) au moins un nombre + # : un deux-points + # (\d+) au moins un nombre + # : un deux-points + # (\d+) au moins un nombre + datetime_regex = re.compile(r"(\d+)/(\d+)/(\d+)\s+(\d+):(\d+):(\d+)") + search = datetime_regex.search(self.soup.footer.text) + if search is None: + return None + + day, month, year, hour, minute, second = [ + int(v) for v in search.groups() + ] + date = datetime.datetime(year, month, day, hour, minute, second) + return timezone.make_aware(date) + + def get_weeks(self): + # Les semaines présentes dans l’emploi du temps sont toutes + # stockées dans un élément span. Il contient une chaîne de + # caractère qui correspond à une forme d’ID, et un champ date, + # qui correspond au lundi de cette semaine. Un cours contient + # un ID correspondant à une semaine, puis le nombre de jours + # après le début de cette semaine. + self.weeks = {} + + # Liste de toutes les semaines définies + for span in self.soup.find_all("span"): + # On parse la date et on la fait correspondre à l’ID + self.weeks[span.alleventweeks.text] = timezone.make_aware( + datetime.datetime.strptime(span["date"], "%d/%m/%Y") + ) + + return self.weeks + + def get_source(self): + req = super(Parser, self).get_source() + self.soup = BeautifulSoup(req.content, "html.parser") + return self.soup diff --git a/management/parsers/ups2018.py b/management/parsers/ups2018.py new file mode 100644 index 0000000..8d97517 --- /dev/null +++ b/management/parsers/ups2018.py @@ -0,0 +1,213 @@ +# Copyright (C) 2018 Alban Gruin +# +# celcatsanitizer is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# celcatsanitizer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with celcatsanitizer. If not, see <http://www.gnu.org/licenses/>. + +from datetime import datetime, timedelta + +import asyncio +import calendar +import json + +from django.utils import timezone + +import lxml.html +import requests + +from ...models import Course, Group, Room +from ...utils import get_current_week, get_week +from .abstractparser import AbstractParser + +VARNAME = "v.events.list = " + + +def find_events_list(soup): + res = [] + for script in soup.xpath("//script/text()"): + if VARNAME in script: + for var in script.split('\n'): + if var.startswith(VARNAME): + res = json.loads(var[len(VARNAME):-2]) + + return res + + +def get_next_month(dt): + n = dt.replace(day=1) + timedelta(days=32) + return n.replace(day=1) + + +class Parser(AbstractParser): + def __init__(self, source): + super(Parser, self).__init__(source) + + # En-tête tiré de mon Firefox… + base_req = self._make_request( + source.url, headers={"Accept-Language": "en-US,en;q=0.5"} + ) + + parser = lxml.html.HTMLParser(encoding="utf-8") + self.soup = lxml.html.document_fromstring( + base_req.content, parser=parser + ) + + self.months = [] + for option in self.soup.xpath("//option"): + if option.get("selected") is not None or len(self.months) > 0: + self.months.append(option.text) + + def __get_event(self, event, today, + beginning_of_month, end_of_month, + year, week): + begin = timezone.make_aware( + datetime.strptime(event["start"], "%Y-%m-%dT%H:%M:%S") + ) + end = timezone.make_aware( + datetime.strptime(event["end"], "%Y-%m-%dT%H:%M:%S") + ) + + if begin < beginning_of_month or begin >= end_of_month or \ + (today is not None and begin < today): + return + + if year is not None and week is not None: + event_year, event_week, _ = begin.isocalendar() + if event_year != year or event_week != week: + return + + course = Course.objects.create( + source=self.source, begin=begin, end=end + ) + + data = event["text"].split("<br>") + rooms = None + if data[0] == "Global Event": + return + + i = 1 + while i < len(data) and not data[i].startswith( + ("L1 ", "L2 ", "L3 ", "L3P ", "M1 ", "M2 ", "DEUST ", "MAG1 ", + "1ERE ANNEE ", "2EME ANNEE ", "3EME ANNEE ", + "MAT-Agreg Interne ") + ): + i += 1 + + groups = data[i] + if i - 1 > 0: + course.name = ", ".join(set(data[i - 1].split(';'))) + else: + course.name = "Sans nom" + if i - 2 > 0: + course.type = data[i - 2] + if len(data) >= i + 2: + rooms = data[i + 1] + if len(data) >= i + 3: + course.notes = data[i + 2] + + groups = [ + Group.objects.get_or_create( + source=self.source, celcat_name=name + )[0] + for name in groups.split(';') + ] + course.groups.add(*groups) + + if rooms is not None: + rooms_objs = Room.objects.filter(name__in=rooms.split(';')) + if rooms_objs.count() > 0: + course.rooms.add(*rooms_objs) + elif course.notes: + course.notes = "{0}\n{1}".format(rooms, course.notes) + else: + course.notes = rooms + + if course.notes is not None: + course.notes = course.notes.strip() + + return course + + def get_events(self, today, year=None, week=None): + for i, month in enumerate(self.events): + beginning_of_month = timezone.make_aware( + datetime.strptime(self.months[i], "%B, %Y") + ) + end_of_month = get_next_month(beginning_of_month) + + for event in month: + course = self.__get_event(event, today, + beginning_of_month, end_of_month, + year, week) + if course is not None: + yield course + + def get_update_date(self): + return None # Pas de date de mise à jour dans ce format + + def get_weeks(self): + # FIXME: détection automatique à partir des événements présents + beginning, _ = get_week(*get_current_week()) + self.weeks = {"1": beginning} + + return self.weeks + + def ajax_req(self, month): + month = datetime.strptime(month, "%B, %Y") + first_monday = min( + week[calendar.MONDAY] + for week in calendar.monthcalendar(month.year, month.month) + if week[calendar.MONDAY] > 0 + ) + month_str = month.replace(day=first_monday).strftime("%Y%m%d") + + req = self._make_request( + self.source.url, + headers={ + "Accept-Language": "en-US,en;q=0.5", + }, + params={"Date": month_str}, + ) + req.raise_for_status() + + parser = lxml.html.HTMLParser(encoding="utf8") + soup = lxml.html.document_fromstring(req.content, parser=parser) + + return find_events_list(soup) + + @asyncio.coroutine + def get_months_async(self): + loop = asyncio.get_event_loop() + futures = [] + + for month in self.months[1:]: + futures.append(loop.run_in_executor(None, self.ajax_req, month)) + + responses = yield from asyncio.gather(*futures) + return responses + + def get_source_from_months(self, async=True): + events = [] + + if async: + loop = asyncio.get_event_loop() + events = loop.run_until_complete(self.get_months_async()) + else: + for month in self.months[1:]: + events.append(self.ajax_req(month)) + + return events + + def get_source(self): + self.events = [ + find_events_list(self.soup) + ] + self.get_source_from_months() + return self.events |