From 6fbcee6ddfdfb7d2674d3296d7a20f59905db7f8 Mon Sep 17 00:00:00 2001 From: Alban Gruin Date: Wed, 28 Aug 2019 15:42:58 +0200 Subject: models & admin: ajout des champs nécessaires pour le parseur UPS2019 Le nouveau format utilisé requiert d’effectuer des requêtes POST et non plus GET, une URL n’est donc plus suffisante pour indiquer la source. Un champ `metadata' est rajouté, contenant une métadonnée au format texte. Il serait plus judicieux d’utiliser un champ JSONField, mais ce type est restreint à PostgreSQL (mon environnement de développement utilise toujours SQLite). Les ID des cours dans celcat ne sont plus de simples nombres mais des chaînes de caractères. Ce changement est donc reflété dans le modèle des cours. Dans le nouveau format, si un cours a plus de 3 groupes, seul les 3 premiers groupes sont listés, les autres sont marqués avec un texte du genre « 2 autres… ». (Ça fait 2 ans et demi que je travaille sur ce projet, et plus le temps passe, plus j’ai l’impression que la drogue est un outil de travail comme un autre chez celcat.) Le champ `buggy' est rajouté pour indiquer si c’est le cas de ce cours ou non, ce qui permettra d’ajouter une remarque sur les pages ouèbe ou les ICS. Les interfaces d’administration sont modifiées pour intégrer ces changements. Signed-off-by: Alban Gruin --- admin.py | 4 ++-- models.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/admin.py b/admin.py index 5350aad..527b1eb 100644 --- a/admin.py +++ b/admin.py @@ -38,7 +38,7 @@ class YearAdmin(admin.ModelAdmin): @admin.register(Source) class SourceAdmin(admin.ModelAdmin): - list_display = ("url", "last_update_date",) + list_display = ("url", "metadata", "last_update_date",) @admin.register(Timetable) @@ -75,7 +75,7 @@ class CourseAdmin(admin.ModelAdmin): (None, {"fields": ("name", "type", "source", "groups", "rooms",)}), ("Horaires", {"fields": ("begin", "end",)}), ("Remarques", {"fields": ("notes",)}), - ("Avancé", {"fields": ("celcat_id", "last_update",), + ("Avancé", {"fields": ("celcat_id", "last_update", "buggy",), "classes": ("collapse",)}),) list_display = ("name", "type", "source", "begin", "end",) list_filter = ("type", "source__timetables", "groups",) diff --git a/models.py b/models.py index b59f2d8..7fd5b3e 100644 --- a/models.py +++ b/models.py @@ -48,13 +48,15 @@ class Year(SlugModel): class Source(models.Model): - url = models.URLField(max_length=255, verbose_name="URL", unique=True) + url = models.URLField(max_length=255, verbose_name="URL") + metadata = models.CharField(max_length=256, verbose_name="Métadonnée", + blank=True, null=True) last_update_date = models.DateTimeField(null=True, blank=True, verbose_name="dernière mise à jour" " Celcat") def __str__(self): - return self.url + return "{}, {}".format(self.url, self.metadata) @property def formatted_timetables(self): @@ -62,6 +64,7 @@ class Source(models.Model): self.timetables.all()]) class Meta: + unique_together = (("url", "metadata",),) verbose_name = "source d’emploi du temps" verbose_name_plural = "sources d’emploi du temps" @@ -258,9 +261,12 @@ class Course(models.Model): last_update = models.DateTimeField(verbose_name="dernière mise à jour", default=timezone.now) - celcat_id = models.IntegerField(verbose_name="ID Celcat", null=True) + celcat_id = models.CharField(max_length=64, verbose_name="ID Celcat", + null=True) module = models.ForeignKey(Module, on_delete=models.SET_NULL, null=True) + buggy = models.BooleanField(verbose_name="Bogué", default=False) + def __str__(self): return self.name -- cgit v1.2.1 From 797a8a7b57b91823ee4b306ca91256dad4e3f504 Mon Sep 17 00:00:00 2001 From: Alban Gruin Date: Thu, 29 Aug 2019 12:52:38 +0200 Subject: models: remplacement de RENCONTRE par rencontre dans le type d’un cours Signed-off-by: Alban Gruin --- models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/models.py b/models.py index 7fd5b3e..448b996 100644 --- a/models.py +++ b/models.py @@ -274,6 +274,7 @@ class Course(models.Model): if self.type is not None: self.type = self.type.replace("COURS", "cours") self.type = self.type.replace("REUNION", "réunion") + self.type = self.type.replace("RENCONTRE", "rencontre") if self.name is not None: self.name = self.name.split("(")[0].strip() -- cgit v1.2.1 From a235752368c6eff21400f6f8089ee3bf781cf36e Mon Sep 17 00:00:00 2001 From: Alban Gruin Date: Fri, 30 Aug 2019 12:27:00 +0200 Subject: ups2018: déplacement des préfixes de cours dans une constante Le parseur UPS2018 va se servir de cette liste aussi, elle est donc déplacée dans sa propre constante. Signed-off-by: Alban Gruin --- management/parsers/ups2018.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/management/parsers/ups2018.py b/management/parsers/ups2018.py index afbfc4b..0d6d798 100644 --- a/management/parsers/ups2018.py +++ b/management/parsers/ups2018.py @@ -32,6 +32,10 @@ from .abstractparser import AbstractParser, ParserError VARNAME = "v.events.list = " +GROUP_PREFIXES = ("L1 ", "L2 ", "L3 ", "L3P ", "M1 ", "M2 ", "DEUST ", "MAG1 ", + "1ERE ANNEE ", "2EME ANNEE ", "3EME ANNEE ", + "MAT-Agreg Interne ") + def find_events_list(soup): res = [] @@ -123,11 +127,7 @@ class Parser(AbstractParser): min_i = 1 i = min_i - while i < len(data) and not data[i].startswith( - ("L1 ", "L2 ", "L3 ", "L3P ", "M1 ", "M2 ", "DEUST ", "MAG1 ", - "1ERE ANNEE ", "2EME ANNEE ", "3EME ANNEE ", - "MAT-Agreg Interne ") - ): + while i < len(data) and not data[i].startswith(GROUP_PREFIXES): i += 1 groups = data[i] -- cgit v1.2.1 From 0717c8ccd6ac10989d86593ff73a86a0c4398408 Mon Sep 17 00:00:00 2001 From: Alban Gruin Date: Thu, 29 Aug 2019 12:53:05 +0200 Subject: ups2019: nouveau parseur pour le format UPS2019 Signed-off-by: Alban Gruin --- management/parsers/ups2019.py | 128 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 management/parsers/ups2019.py diff --git a/management/parsers/ups2019.py b/management/parsers/ups2019.py new file mode 100644 index 0000000..c7ab7c9 --- /dev/null +++ b/management/parsers/ups2019.py @@ -0,0 +1,128 @@ +# Copyright (C) 2019 Alban Gruin +# +# celcatsanitizer is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# celcatsanitizer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with celcatsanitizer. If not, see . + +from datetime import date, datetime, timedelta +from html import unescape + +from django.utils import timezone + +import requests + +from ...models import Course, Group, Module, Room +from ...utils import get_current_week, get_week +from .abstractparser import AbstractParser +from .ups2018 import GROUP_PREFIXES + + +class Parser(AbstractParser): + def __get_name(self, raw_name): + return raw_name.split('[')[1][:-1] + + def __get_event(self, event, year, week): + if event["allDay"]: + return + + begin = timezone.make_aware( + datetime.strptime(event["start"], "%Y-%m-%dT%H:%M:%S") + ) + end = timezone.make_aware( + datetime.strptime(event["end"], "%Y-%m-%dT%H:%M:%S") + ) + + if year is not None and week is not None: + event_year, event_week, _ = begin.isocalendar() + if event_year != year or event_week != week: + return + + data = [unescape(st.strip()) + for st in event["description"].split("
")] + groups = [] + rooms = [] + + course = Course.objects.create( + source=self.source, begin=begin, end=end, + celcat_id=event["id"] + ) + + i = 0 + if event.get("eventCategory") is not None and \ + len(event.get("eventCategory", "")) > 0: + course.type = event["eventCategory"] + i = 1 + + if event.get("module", "") is not None and \ + len(event.get("module", "")) > 0: + module, _ = Module.objects.get_or_create(name=event["module"]) + course.module = module + + if '[' in data[i]: + course.name = self.__get_name(data[i]) + i += 1 + + while '[' in data[i]: + course.name += ", " + self.__get_name(data[i]) + i += 1 + + while i < len(data) and not data[i].startswith(GROUP_PREFIXES): + rooms.append(data[i]) + i += 1 + course.rooms.add(*Room.objects.filter(name__in=rooms)) + + while i < len(data) and data[i].startswith(GROUP_PREFIXES): + groups.append(Group.objects.get_or_create( + source=self.source, celcat_name=data[i] + )[0]) + i += 1 + course.groups.add(*groups) + + if i < len(data): + course.notes = "\n".join(data[i:]).strip() + if "other" in course.notes: + print("Warning: 'other' in course.notes") + + return course + + def get_events(self, today, year=None, week=None): + for event in self.events: + course = self.__get_event(event, year, week) + if course is not None: + yield course + + def get_update_date(self): + return + + def get_weeks(self): + # FIXME: détection automatique à partir des événements présents + beginning, _ = get_week(*get_current_week()) + self.weeks = {"1": beginning} + + return self.weeks + + def get_source(self): + start = date.today() + end = start + timedelta(days=365) + + req = requests.post(self.source.url, + headers={"User-Agent": self.user_agent}, + data={"calView": "month", + "resType": 103, + "federationIds[]": self.source.metadata, + "start": start.strftime("%Y-%m-%d"), + "end": end.strftime("%Y-%m-%d")}) + req.encoding = "uft8" + req.raise_for_status() + + self.events = req.json() + return self.events -- cgit v1.2.1 From 5045af42503158db4155988f00ee0db0a6a67ff7 Mon Sep 17 00:00:00 2001 From: Alban Gruin Date: Thu, 29 Aug 2019 17:19:12 +0200 Subject: templatetags/rooms: correction du nom brut des salles Les salles et amphis commencent maintenant par leur département, de cette manière : FSI / Amphi FERMAT (bat.1A) Au lieu de : Amphi FERMAT (bat.1A) Cela ne fait que rajouter du bruit sur la page. Ceci modifie le filtre de formatage des salles pour retirer le département si il est présent. Signed-off-by: Alban Gruin --- templatetags/rooms.py | 14 ++++++++++---- tests.py | 34 +++++++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/templatetags/rooms.py b/templatetags/rooms.py index f0e1b2e..d8b0e23 100644 --- a/templatetags/rooms.py +++ b/templatetags/rooms.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 Alban Gruin +# Copyright (C) 2017, 2019 Alban Gruin # # celcatsanitizer is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published @@ -18,11 +18,17 @@ from django import template register = template.Library() +def __filter_room_name(name): + if '/' in name: + return name.split('/')[1].strip() + return name + + @register.filter def format_rooms(rooms): - amphi_list = [room.name for room in rooms if room.name.startswith("Amphi")] - room_list = [room.name for room in rooms - if not room.name.startswith("Amphi")] + names = [__filter_room_name(room.name) for room in rooms] + amphi_list = [name for name in names if name.startswith("Amphi")] + room_list = [name for name in names if not name.startswith("Amphi")] amphis = ", ".join(amphi_list) joined = ", ".join(room_list) diff --git a/tests.py b/tests.py index b411db3..1e80b48 100644 --- a/tests.py +++ b/tests.py @@ -341,6 +341,16 @@ class RoomTestCase(TestCase): for i in range(5, 7) ] + # On n’insère pas ces salles dans la base de données, elles ne + # servent que pour le test de formatage. + self.formatted_rooms = [ + Room(name="FSI / {}".format(str(i))) + for i in range(5) + ] + [ + Room(name="FSI / Amphi {}".format(str(i))) + for i in range(5, 7) + ] + hours = [({"begin": datetime.time(hour=14, minute=0)},), ({"begin": datetime.time(hour=16, minute=0)},), ({"begin": datetime.time(hour=13, minute=30)}, @@ -364,23 +374,29 @@ class RoomTestCase(TestCase): course.groups.add(group) course.rooms.add(room) - def test_format(self): - amphis = self.rooms[-2:] - + def __test_format(self, rooms, amphis): self.assertEqual(format_rooms([]), "") - self.assertEqual(format_rooms(self.rooms[:1]), "Salle 0") - self.assertEqual(format_rooms(self.rooms[:2]), "Salles 0, 1") + self.assertEqual(format_rooms(rooms[:1]), "Salle 0") + self.assertEqual(format_rooms(rooms[:2]), "Salles 0, 1") self.assertEqual(format_rooms([amphis[0]]), "Amphi 5") self.assertEqual(format_rooms(amphis), "Amphi 5, Amphi 6") - self.assertEqual(format_rooms([amphis[0]] + self.rooms[:1]), + self.assertEqual(format_rooms([amphis[0]] + rooms[:1]), "Amphi 5, salle 0") - self.assertEqual(format_rooms([amphis[0]] + self.rooms[:2]), + self.assertEqual(format_rooms([amphis[0]] + rooms[:2]), "Amphi 5, salles 0, 1") - self.assertEqual(format_rooms(amphis + self.rooms[:1]), + self.assertEqual(format_rooms(amphis + rooms[:1]), "Amphi 5, Amphi 6, salle 0") - self.assertEqual(format_rooms(amphis + self.rooms[:2]), + self.assertEqual(format_rooms(amphis + rooms[:2]), "Amphi 5, Amphi 6, salles 0, 1") + def test_format(self): + amphis = self.rooms[-2:] + self.__test_format(self.rooms, amphis) + + def test_reformat(self): + amphis = self.formatted_rooms[-2:] + self.__test_format(self.formatted_rooms, amphis) + def test_qsjps(self): begin = timezone.make_aware(datetime.datetime.combine( self.day, datetime.time(hour=15, minute=0))) -- cgit v1.2.1 From a85f2fb91d1a4a0e31c41c392e404d7ddbf21109 Mon Sep 17 00:00:00 2001 From: Alban Gruin Date: Sun, 1 Sep 2019 12:41:19 +0200 Subject: ups2019: réadaptation du parseur L’emplacement des différents éléments a changé, mdr. Signed-off-by: Alban Gruin --- management/parsers/ups2019.py | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/management/parsers/ups2019.py b/management/parsers/ups2019.py index c7ab7c9..c6bd7e3 100644 --- a/management/parsers/ups2019.py +++ b/management/parsers/ups2019.py @@ -27,9 +27,6 @@ from .ups2018 import GROUP_PREFIXES class Parser(AbstractParser): - def __get_name(self, raw_name): - return raw_name.split('[')[1][:-1] - def __get_event(self, event, year, week): if event["allDay"]: return @@ -56,41 +53,42 @@ class Parser(AbstractParser): celcat_id=event["id"] ) - i = 0 + max_i = len(data) + if event.get("eventCategory") is not None and \ len(event.get("eventCategory", "")) > 0: course.type = event["eventCategory"] - i = 1 + max_i -= 1 if event.get("module", "") is not None and \ len(event.get("module", "")) > 0: module, _ = Module.objects.get_or_create(name=event["module"]) course.module = module - if '[' in data[i]: - course.name = self.__get_name(data[i]) - i += 1 - - while '[' in data[i]: - course.name += ", " + self.__get_name(data[i]) - i += 1 - - while i < len(data) and not data[i].startswith(GROUP_PREFIXES): + i = 0 + while i < max_i and not data[i].startswith(GROUP_PREFIXES): rooms.append(data[i]) i += 1 course.rooms.add(*Room.objects.filter(name__in=rooms)) - while i < len(data) and data[i].startswith(GROUP_PREFIXES): - groups.append(Group.objects.get_or_create( - source=self.source, celcat_name=data[i] - )[0]) + if len(rooms) != course.rooms.count(): + print(rooms, course.rooms) + + while i < max_i and data[i].startswith(GROUP_PREFIXES): + group, _ = Group.objects.get_or_create(source=self.source, + celcat_name=data[i]) + groups.append(group) i += 1 course.groups.add(*groups) - if i < len(data): - course.notes = "\n".join(data[i:]).strip() - if "other" in course.notes: - print("Warning: 'other' in course.notes") + if i < max_i and course.module is not None and \ + data[i].startswith(course.module.name): + course.name = data[i] + i += 1 + + course.notes = "\n".join(data[i:max_i]).strip() + if "other" in data[i]: + print("Warning: \"other\" in notes") return course -- cgit v1.2.1