diff options
| -rw-r--r-- | Documentation/index.rst | 1 | ||||
| -rw-r--r-- | Documentation/usage/commands/printvalues.rst | 48 | ||||
| -rw-r--r-- | admin.py | 14 | ||||
| -rw-r--r-- | management/commands/__parsercommand.py | 26 | ||||
| -rw-r--r-- | management/commands/printvalues.py | 45 | ||||
| -rw-r--r-- | management/commands/timetables.py | 16 | ||||
| -rw-r--r-- | management/parsers/ups2018.py | 14 | ||||
| -rw-r--r-- | models.py | 17 | ||||
| -rw-r--r-- | tests.py | 75 | ||||
| -rw-r--r-- | tests/data/2018/october.html | 2 | ||||
| -rw-r--r-- | tests/data/2018/september.html | 2 |
11 files changed, 217 insertions, 43 deletions
diff --git a/Documentation/index.rst b/Documentation/index.rst index e793dd2..7051b93 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -61,6 +61,7 @@ Utilisation de celcatsanitizer usage/installation usage/commands/cleancourses usage/commands/listtimetables + usage/commands/printvalues usage/commands/reparse usage/commands/timetables usage/versions diff --git a/Documentation/usage/commands/printvalues.rst b/Documentation/usage/commands/printvalues.rst new file mode 100644 index 0000000..7d53d44 --- /dev/null +++ b/Documentation/usage/commands/printvalues.rst @@ -0,0 +1,48 @@ +=============== +``printvalues`` +=============== + +``printvalues`` affiche le contenu brut d’un ou plusieurs cours, sans +chercher à interpréter son contenu ou à l’enregistrer dans la base de +données. + +Utilisation +=========== +.. code:: shell + + $ ./manage.py printvalues --source id [--limit nb] + +``--source`` permet de spécifier la source depuis laquelle les cours +doivent être récupérés. ``id`` correspond à l’ID de la source, +trouvable à l’aide de la commande :doc:`listtimetables`. + +``--limit`` permet de limiter le nombre de cours affichés. ``nb`` +correspond au nombre maximum de cours affichés. + +Format de sortie +================ +:: + + { + "backColor": "#7D4F72", + "clickDisabled": true, + "doubleClickDisabled": true, + "end": "2019-01-28T09:45:00", + "html": "<div style=\"color:White \">(07:45-09:45)<br>COURS/TD<br>ELINF6Q1 - BIOLOGIE<br>L3 INFO s2 CMA<br>U3-307</div>", + "id": "76330023", + "moveDisabled": true, + "resizeDisabled": true, + "sort": [], + "start": "2019-01-28T07:45:00", + "tag": [ + "celcat", + "sat_notvalid", + "1", + "reg_notmark", + "ELINF6Q1", + "7491453" + ], + "text": "(07:45-09:45)<br>COURS/TD<br>ELINF6Q1 - BIOLOGIE<br>L3 INFO s2 CMA<br>U3-307", + "toolTip": "(07:45-09:45)<br>COURS/TD<br>ELINF6Q1 - BIOLOGIE<br>L3 INFO s2 CMA<br>U3-307" + } + Done. @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 Alban Gruin +# Copyright (C) 2017-2019 Alban Gruin # # celcatsanitizer is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published @@ -14,7 +14,7 @@ # along with celcatsanitizer. If not, see <http://www.gnu.org/licenses/>. from django.contrib import admin -from .models import Course, Group, Room, Source, Timetable, Year +from .models import Course, Group, Module, Room, Source, Timetable, Year def make_hidden(modeladmin, request, queryset): @@ -62,7 +62,7 @@ class GroupAdmin(admin.ModelAdmin): actions = (make_hidden, make_visible,) -@admin.register(Room) +@admin.register(Room, Module) class RoomAdmin(admin.ModelAdmin): prepopulated_fields = {"slug": ("name",)} list_display = ("name",) @@ -72,10 +72,12 @@ class RoomAdmin(admin.ModelAdmin): @admin.register(Course) class CourseAdmin(admin.ModelAdmin): fieldsets = ( - (None, {"fields": ("name", "type", "source", "groups", "rooms", - "last_update",)}), + (None, {"fields": ("name", "type", "source", "groups", "rooms",)}), ("Horaires", {"fields": ("begin", "end",)}), - ("Remarques", {"fields": ("notes",)}),) + ("Remarques", {"fields": ("notes",)}), + ("Avancé", {"fields": ("celcat_id", "last_update",), + "classes": ("collapse",)}),) list_display = ("name", "type", "source", "begin", "end",) list_filter = ("type", "source__timetables", "groups",) ordering = ("begin",) + readonly_fields = ("celcat_id", "last_update",) diff --git a/management/commands/__parsercommand.py b/management/commands/__parsercommand.py new file mode 100644 index 0000000..99480cc --- /dev/null +++ b/management/commands/__parsercommand.py @@ -0,0 +1,26 @@ +# Copyright (C) 2019 Alban Gruin +# +# celcatsanitizer is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# celcatsanitizer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with celcatsanitizer. If not, see <http://www.gnu.org/licenses/>. + +from importlib import import_module + +from django.conf import settings + +DEFAULT_PARSER = "edt.management.parsers.ups2017" + + +class ParserCommand: + def get_parser(self): + parser_module = getattr(settings, "CS_PARSER", DEFAULT_PARSER) + return getattr(import_module(parser_module), "Parser") diff --git a/management/commands/printvalues.py b/management/commands/printvalues.py new file mode 100644 index 0000000..91dd18b --- /dev/null +++ b/management/commands/printvalues.py @@ -0,0 +1,45 @@ +# Copyright (C) 2019 Alban Gruin +# +# celcatsanitizer is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# celcatsanitizer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with celcatsanitizer. If not, see <http://www.gnu.org/licenses/>. + +from django.core.management.base import BaseCommand + +from ...models import Source +from .__parsercommand import ParserCommand + +import json + + +class Command(BaseCommand, ParserCommand): + help = "List values from courses from a source" + + def add_arguments(self, parser): + parser.add_argument("--source", type=int, nargs=1, required=True) + parser.add_argument("--limit", type=int, nargs=1) + + def handle(self, *args, **options): + source = Source.objects.get(pk=options["source"][0]) + parser = self.get_parser()(source) + events = [event for month in parser.get_source() for event in month] + + i = 0 + limit = len(events) + if options["limit"] is not None: + limit = min(options["limit"][0], limit) + + while i < limit: + self.stdout.write(json.dumps(events[i], indent=4, sort_keys=True)) + i += 1 + + self.stdout.write(self.style.SUCCESS("Done.")) diff --git a/management/commands/timetables.py b/management/commands/timetables.py index ee33f7e..f71accf 100644 --- a/management/commands/timetables.py +++ b/management/commands/timetables.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 Alban Gruin +# Copyright (C) 2017-2019 Alban Gruin # # celcatsanitizer is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published @@ -13,20 +13,16 @@ # You should have received a copy of the GNU Affero General Public License # along with celcatsanitizer. If not, see <http://www.gnu.org/licenses/>. -from importlib import import_module - import datetime import traceback -from django.conf import settings from django.core.management.base import BaseCommand from django.db import transaction from django.db.models import Min from ...models import Course, Source from ...utils import get_week, tz_now - -DEFAULT_PARSER = "edt.management.parsers.ups2017" +from .__parsercommand import ParserCommand def delete_courses_in_week(source, year, week, today): @@ -122,7 +118,7 @@ def process_timetable(source, force, parser_cls, year=None, weeks=None): process_timetable_week(source, force, parser) -class Command(BaseCommand): +class Command(BaseCommand, ParserCommand): help = "Fetches registered celcat timetables" def add_arguments(self, parser): @@ -134,14 +130,10 @@ class Command(BaseCommand): nargs="+") parser.add_argument("--year", type=int, nargs=1) - def __get_parser(self): - parser_module = getattr(settings, "CS_PARSER", DEFAULT_PARSER) - return getattr(import_module(parser_module), "Parser") - def handle(self, *args, **options): year = None errcount = 0 - parser = self.__get_parser() + parser = self.get_parser() if options["all"]: weeks = None diff --git a/management/parsers/ups2018.py b/management/parsers/ups2018.py index f1da5bf..afbfc4b 100644 --- a/management/parsers/ups2018.py +++ b/management/parsers/ups2018.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018 Alban Gruin +# Copyright (C) 2018-2019 Alban Gruin # # celcatsanitizer is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published @@ -26,7 +26,7 @@ from django.utils import timezone import lxml.html import requests -from ...models import Course, Group, Room +from ...models import Course, Group, Module, Room from ...utils import get_current_week, get_week from .abstractparser import AbstractParser, ParserError @@ -114,7 +114,8 @@ class Parser(AbstractParser): return course = Course.objects.create( - source=self.source, begin=begin, end=end + source=self.source, begin=begin, end=end, + celcat_id=int(event["id"]) ) min_i = 0 @@ -136,6 +137,13 @@ class Parser(AbstractParser): # par un dictionnaire classique. names = OrderedDict.fromkeys(data[i - 1].split(';')) course.name = ", ".join(names.keys()) + + module_names = [t for t in event["tag"] + if len(t) > 0 and + any(n.startswith(t) for n in names.keys())] + if len(module_names) > 0: + module, _ = Module.objects.get_or_create(name=module_names[0]) + course.module = module else: course.name = "Sans nom" if i - 2 >= min_i: @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 Alban Gruin +# Copyright (C) 2017-2019 Alban Gruin # # celcatsanitizer is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published @@ -200,6 +200,18 @@ class Room(SlugModel): verbose_name_plural = "salles" +class Module(SlugModel): + name = models.CharField(max_length=255, unique=True, verbose_name="nom") + slug = models.SlugField(max_length=64, default="", unique=True) + + def __str__(self): + return self.name + + class Meta: + verbose_name = "module" + verbose_name_plural = "modules" + + class CourseManager(Manager): def get_courses(self, obj, **criteria): qs = self.get_queryset() @@ -246,6 +258,9 @@ class Course(models.Model): last_update = models.DateTimeField(verbose_name="dernière mise à jour", default=timezone.now) + celcat_id = models.IntegerField(verbose_name="ID Celcat", null=True) + module = models.ForeignKey(Module, on_delete=models.SET_NULL, null=True) + def __str__(self): return self.name @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 Alban Gruin +# Copyright (C) 2017-2019 Alban Gruin # # celcatsanitizer is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published @@ -20,7 +20,7 @@ from django.utils import timezone from .management.parsers.abstractparser import ParserError from .management.parsers.ups2018 import Parser as UPS2018Parser -from .models import Course, Group, Room, Source, Timetable, Year +from .models import Course, Group, Module, Room, Source, Timetable, Year from .templatetags.rooms import format_rooms from .utils import tz_now @@ -84,7 +84,7 @@ class CourseTestCase(TestCase): for group in (cma, tda2, self.tpa21, cmb, tdb2, self.tpb21,): course = Course.objects.create( name="{0} course".format(group.name), type="cours", - source=source, begin=dt, end=dt) + source=source, begin=dt, end=dt, celcat_id=0) course.groups.add(group) def test_get_courses_for_group(self): @@ -359,7 +359,8 @@ class RoomTestCase(TestCase): end = begin + datetime.timedelta(hours=rn.get("duration", 2)) course = Course.objects.create(source=self.source, - begin=begin, end=end) + begin=begin, end=end, + celcat_id=0) course.groups.add(group) course.rooms.add(room) @@ -413,12 +414,14 @@ class UPS2018ParserTestCase(TestCase): def test_get_event(self): get_event = self.parser._Parser__get_event count = Course.objects.count() + module_count = Module.objects.count() event = get_event( {"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque;AAA" "<br>L3 Info s1 CMA;L3 Info s1 TDA2<br>" - "Salle quelconque;Salle quelconque 2<br>Commentaire"}, + "Salle quelconque;Salle quelconque 2<br>Commentaire", "id": "0", + "tag": ["abc", "def", "AAA"]}, timezone.make_aware(datetime.datetime(2018, 9, 21)), timezone.make_aware(datetime.datetime(2018, 9, 1)), timezone.make_aware(datetime.datetime(2018, 10, 1)), @@ -436,13 +439,18 @@ class UPS2018ParserTestCase(TestCase): self.assertIn(self.room2, event.rooms.all()) self.assertEqual(event.rooms.count(), 2) self.assertEqual(event.notes, "Commentaire") + self.assertEqual(event.celcat_id, 0) self.assertEqual(event.begin, timezone.make_aware( datetime.datetime(2018, 9, 21, 10, 0, 0))) self.assertEqual(event.end, timezone.make_aware( datetime.datetime(2018, 9, 21, 12, 0, 0))) + self.assertEqual(event.module.name, "AAA") - self.assertEqual(count, Course.objects.count() - 1) count += 1 + module_count += 1 + + self.assertEqual(count, Course.objects.count()) + self.assertEqual(module_count, Module.objects.count()) events = [ { @@ -451,6 +459,9 @@ class UPS2018ParserTestCase(TestCase): "name": "Cours quelconque", "type": "COURS/TD", "group": self.group, "room": self.room, + "id": "1", + "tag": ["aaa", "Cours"], + "meta": 1 }, { "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque" @@ -458,7 +469,8 @@ class UPS2018ParserTestCase(TestCase): "name": "Cours quelconque", "type": "COURS/TD", "group": ngroup, - "notes": "Salle quelconque 3" + "notes": "Salle quelconque 3", + "id": "2" }, { "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque" @@ -467,44 +479,50 @@ class UPS2018ParserTestCase(TestCase): "type": "COURS/TD", "group": self.group, "notes": "Salle quelconque 3\nCommentaire", + "id": "3" }, { "text": "(10:00-12:00)<br>COURS/TD" "<br>L3 Info s1 CMA<br>Salle quelconque 3", "name": "COURS/TD", "group": self.group, - "notes": "Salle quelconque 3" + "notes": "Salle quelconque 3", + "id": "4" }, { "text": "COURS/TD<br>L3 Info s1 CMA<br>Salle quelconque 3", "name": "COURS/TD", "group": self.group, - "notes": "Salle quelconque 3" + "notes": "Salle quelconque 3", + "id": "5" }, { "text": "L3 Info s1 CMA<br>Salle quelconque", "name": "Sans nom", "group": self.group, - "room": self.room + "room": self.room, + "id": "6" }, { "text": "L3 Info s1 CMA<br>Salle quelconque 3", "name": "Sans nom", "group": self.group, - "notes": "Salle quelconque 3" + "notes": "Salle quelconque 3", + "id": "7" }, { "text": "(10:00-12:00)<br>L3 Info s1 CMA<br>Salle quelconque", "name": "Sans nom", "group": self.group, - "room": self.room + "room": self.room, + "id": "8" } ] for e in events: event = get_event( {"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00", - "text": e["text"]}, + "text": e["text"], "id": e["id"], "tag": e.get("tag", [])}, timezone.make_aware(datetime.datetime(2018, 9, 21)), timezone.make_aware(datetime.datetime(2018, 9, 1)), timezone.make_aware(datetime.datetime(2018, 10, 1)), @@ -513,6 +531,7 @@ class UPS2018ParserTestCase(TestCase): self.assertEqual(event.name, e["name"]) self.assertIn(e["group"], event.groups.all()) self.assertEqual(event.groups.count(), 1) + self.assertEqual(str(event.celcat_id), e["id"]) if "type" in e: self.assertEqual(event.type, e["type"]) @@ -530,9 +549,17 @@ class UPS2018ParserTestCase(TestCase): else: self.assertIsNone(event.notes) - self.assertEqual(count, Course.objects.count() - 1) + if "tag" in e: + self.assertEqual(event.module.name, e["tag"][e["meta"]]) + module_count += 1 + else: + self.assertIsNone(event.module) + count += 1 + self.assertEqual(count, Course.objects.count()) + self.assertEqual(module_count, Module.objects.count()) + event = get_event( {"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00", "text": "Global Event"}, @@ -560,11 +587,13 @@ class UPS2018ParserTestCase(TestCase): {"begin": timezone.make_aware(datetime.datetime(2018, 9, 21, 10, 00, 00)), "end": - timezone.make_aware(datetime.datetime(2018, 9, 21, 12, 00, 00))}, + timezone.make_aware(datetime.datetime(2018, 9, 21, 12, 00, 00)), + "id": 0}, {"begin": timezone.make_aware(datetime.datetime(2018, 10, 22, 10, 00, 00)), "end": - timezone.make_aware(datetime.datetime(2018, 10, 22, 12, 00, 00))} + timezone.make_aware(datetime.datetime(2018, 10, 22, 12, 00, 00)), + "id": 2} ] for i, course in enumerate(self.parser.get_events( @@ -578,6 +607,8 @@ class UPS2018ParserTestCase(TestCase): self.assertIsNone(course.notes) self.assertEqual(course.begin, courses[i]["begin"]) self.assertEqual(course.end, courses[i]["end"]) + self.assertEqual(course.celcat_id, courses[i]["id"]) + self.assertEqual(course.module, None) self.assertEqual(i, len(courses) - 1) @@ -588,15 +619,21 @@ class UPS2018ParserTestCase(TestCase): [{ "start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque" - "<br>L3 Info s1 CMA<br>Salle quelconque" + "<br>L3 Info s1 CMA<br>Salle quelconque", + "id": "0", + "tag": [], }], [{ "start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque" - "<br>L3 Info s1 CMA<br>Salle quelconque" + "<br>L3 Info s1 CMA<br>Salle quelconque", + "id": "1", + "tag": [], }, { "start": "2018-10-22T10:00:00", "end": "2018-10-22T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque" - "<br>L3 Info s1 CMA<br>Salle quelconque" + "<br>L3 Info s1 CMA<br>Salle quelconque", + "id": "2", + "tag": [], }], [], [], [], [], [], [], [], [], []]) def test_get_update_date(self): diff --git a/tests/data/2018/october.html b/tests/data/2018/october.html index 6f326f9..ab3da0b 100644 --- a/tests/data/2018/october.html +++ b/tests/data/2018/october.html @@ -40,7 +40,7 @@ do_something(); <script> function courses() { var v = {}; -v.events.list = [{"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque<br>L3 Info s1 CMA<br>Salle quelconque"}, {"start": "2018-10-22T10:00:00", "end": "2018-10-22T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque<br>L3 Info s1 CMA<br>Salle quelconque"}];; +v.events.list = [{"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque<br>L3 Info s1 CMA<br>Salle quelconque", "id": "1", "tag": []}, {"start": "2018-10-22T10:00:00", "end": "2018-10-22T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque<br>L3 Info s1 CMA<br>Salle quelconque", "id": "2", "tag": []}];; } </script> diff --git a/tests/data/2018/september.html b/tests/data/2018/september.html index 3db6cbc..c81fc3b 100644 --- a/tests/data/2018/september.html +++ b/tests/data/2018/september.html @@ -41,7 +41,7 @@ alert("something"); <script> function courses() { var v = {}; -v.events.list = [{"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque<br>L3 Info s1 CMA<br>Salle quelconque"}];; +v.events.list = [{"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque<br>L3 Info s1 CMA<br>Salle quelconque", "id": "0", "tag": []}];; } </script> |