Merge branch 'ag/parser' into futur

11 files changed, 217 insertions, 43 deletions

diff --git a/Documentation/index.rst b/Documentation/index.rst
index e793dd2..7051b93 100644
--- a/Documentation/index.rst
+++ b/Documentation/index.rst
@@ -61,6 +61,7 @@ Utilisation de celcatsanitizer
    usage/installation
    usage/commands/cleancourses
    usage/commands/listtimetables
+   usage/commands/printvalues
    usage/commands/reparse
    usage/commands/timetables
    usage/versions
diff --git a/Documentation/usage/commands/printvalues.rst b/Documentation/usage/commands/printvalues.rst
new file mode 100644
index 0000000..7d53d44
--- /dev/null
+++ b/Documentation/usage/commands/printvalues.rst
@@ -0,0 +1,48 @@
+===============
+``printvalues``
+===============
+
+``printvalues`` affiche le contenu brut d’un ou plusieurs cours, sans
+chercher à interpréter son contenu ou à l’enregistrer dans la base de
+données.
+
+Utilisation
+===========
+.. code:: shell
+
+    $ ./manage.py printvalues --source id [--limit nb]
+
+``--source`` permet de spécifier la source depuis laquelle les cours
+doivent être récupérés.  ``id`` correspond à l’ID de la source,
+trouvable à l’aide de la commande :doc:`listtimetables`.
+
+``--limit`` permet de limiter le nombre de cours affichés.  ``nb``
+correspond au nombre maximum de cours affichés.
+
+Format de sortie
+================
+::
+
+   {
+       "backColor": "#7D4F72",
+       "clickDisabled": true,
+       "doubleClickDisabled": true,
+       "end": "2019-01-28T09:45:00",
+       "html": "<div style=\"color:White \">(07:45-09:45)<br>COURS/TD<br>ELINF6Q1 - BIOLOGIE<br>L3 INFO s2 CMA<br>U3-307</div>",
+       "id": "76330023",
+       "moveDisabled": true,
+       "resizeDisabled": true,
+       "sort": [],
+       "start": "2019-01-28T07:45:00",
+       "tag": [
+           "celcat",
+           "sat_notvalid",
+           "1",
+           "reg_notmark",
+           "ELINF6Q1",
+           "7491453"
+       ],
+       "text": "(07:45-09:45)<br>COURS/TD<br>ELINF6Q1 - BIOLOGIE<br>L3 INFO s2 CMA<br>U3-307",
+       "toolTip": "(07:45-09:45)<br>COURS/TD<br>ELINF6Q1 - BIOLOGIE<br>L3 INFO s2 CMA<br>U3-307"
+   }
+   Done.
diff --git a/admin.py b/admin.py
index 0dc7987..5350aad 100644
--- a/admin.py
+++ b/admin.py
@@ -1,4 +1,4 @@
-#    Copyright (C) 2017-2018  Alban Gruin
+#    Copyright (C) 2017-2019  Alban Gruin
 #
 #    celcatsanitizer is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU Affero General Public License as published
@@ -14,7 +14,7 @@
 #    along with celcatsanitizer.  If not, see <http://www.gnu.org/licenses/>.
 
 from django.contrib import admin
-from .models import Course, Group, Room, Source, Timetable, Year
+from .models import Course, Group, Module, Room, Source, Timetable, Year
 
 
 def make_hidden(modeladmin, request, queryset):
@@ -62,7 +62,7 @@ class GroupAdmin(admin.ModelAdmin):
     actions = (make_hidden, make_visible,)
 
 
-@admin.register(Room)
+@admin.register(Room, Module)
 class RoomAdmin(admin.ModelAdmin):
     prepopulated_fields = {"slug": ("name",)}
     list_display = ("name",)
@@ -72,10 +72,12 @@ class RoomAdmin(admin.ModelAdmin):
 @admin.register(Course)
 class CourseAdmin(admin.ModelAdmin):
     fieldsets = (
-        (None, {"fields": ("name", "type", "source", "groups", "rooms",
-                           "last_update",)}),
+        (None, {"fields": ("name", "type", "source", "groups", "rooms",)}),
         ("Horaires", {"fields": ("begin", "end",)}),
-        ("Remarques", {"fields": ("notes",)}),)
+        ("Remarques", {"fields": ("notes",)}),
+        ("Avancé", {"fields": ("celcat_id", "last_update",),
+                    "classes": ("collapse",)}),)
     list_display = ("name", "type", "source", "begin", "end",)
     list_filter = ("type", "source__timetables", "groups",)
     ordering = ("begin",)
+    readonly_fields = ("celcat_id", "last_update",)
diff --git a/management/commands/__parsercommand.py b/management/commands/__parsercommand.py
new file mode 100644
index 0000000..99480cc
--- /dev/null
+++ b/management/commands/__parsercommand.py
@@ -0,0 +1,26 @@
+#    Copyright (C) 2019  Alban Gruin
+#
+#    celcatsanitizer is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    celcatsanitizer is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with celcatsanitizer.  If not, see <http://www.gnu.org/licenses/>.
+
+from importlib import import_module
+
+from django.conf import settings
+
+DEFAULT_PARSER = "edt.management.parsers.ups2017"
+
+
+class ParserCommand:
+    def get_parser(self):
+        parser_module = getattr(settings, "CS_PARSER", DEFAULT_PARSER)
+        return getattr(import_module(parser_module), "Parser")
diff --git a/management/commands/printvalues.py b/management/commands/printvalues.py
new file mode 100644
index 0000000..91dd18b
--- /dev/null
+++ b/management/commands/printvalues.py
@@ -0,0 +1,45 @@
+#    Copyright (C) 2019  Alban Gruin
+#
+#    celcatsanitizer is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    celcatsanitizer is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with celcatsanitizer.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.core.management.base import BaseCommand
+
+from ...models import Source
+from .__parsercommand import ParserCommand
+
+import json
+
+
+class Command(BaseCommand, ParserCommand):
+    help = "List values from courses from a source"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--source", type=int, nargs=1, required=True)
+        parser.add_argument("--limit", type=int, nargs=1)
+
+    def handle(self, *args, **options):
+        source = Source.objects.get(pk=options["source"][0])
+        parser = self.get_parser()(source)
+        events = [event for month in parser.get_source() for event in month]
+
+        i = 0
+        limit = len(events)
+        if options["limit"] is not None:
+            limit = min(options["limit"][0], limit)
+
+        while i < limit:
+            self.stdout.write(json.dumps(events[i], indent=4, sort_keys=True))
+            i += 1
+
+        self.stdout.write(self.style.SUCCESS("Done."))
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index ee33f7e..f71accf 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -1,4 +1,4 @@
-#    Copyright (C) 2017-2018  Alban Gruin
+#    Copyright (C) 2017-2019  Alban Gruin
 #
 #    celcatsanitizer is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU Affero General Public License as published
@@ -13,20 +13,16 @@
 #    You should have received a copy of the GNU Affero General Public License
 #    along with celcatsanitizer.  If not, see <http://www.gnu.org/licenses/>.
 
-from importlib import import_module
-
 import datetime
 import traceback
 
-from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.db import transaction
 from django.db.models import Min
 
 from ...models import Course, Source
 from ...utils import get_week, tz_now
-
-DEFAULT_PARSER = "edt.management.parsers.ups2017"
+from .__parsercommand import ParserCommand
 
 
 def delete_courses_in_week(source, year, week, today):
@@ -122,7 +118,7 @@ def process_timetable(source, force, parser_cls, year=None, weeks=None):
         process_timetable_week(source, force, parser)
 
 
-class Command(BaseCommand):
+class Command(BaseCommand, ParserCommand):
     help = "Fetches registered celcat timetables"
 
     def add_arguments(self, parser):
@@ -134,14 +130,10 @@ class Command(BaseCommand):
                             nargs="+")
         parser.add_argument("--year", type=int, nargs=1)
 
-    def __get_parser(self):
-        parser_module = getattr(settings, "CS_PARSER", DEFAULT_PARSER)
-        return getattr(import_module(parser_module), "Parser")
-
     def handle(self, *args, **options):
         year = None
         errcount = 0
-        parser = self.__get_parser()
+        parser = self.get_parser()
 
         if options["all"]:
             weeks = None
diff --git a/management/parsers/ups2018.py b/management/parsers/ups2018.py
index f1da5bf..afbfc4b 100644
--- a/management/parsers/ups2018.py
+++ b/management/parsers/ups2018.py
@@ -1,4 +1,4 @@
-#    Copyright (C) 2018  Alban Gruin
+#    Copyright (C) 2018-2019  Alban Gruin
 #
 #    celcatsanitizer is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU Affero General Public License as published
@@ -26,7 +26,7 @@ from django.utils import timezone
 import lxml.html
 import requests
 
-from ...models import Course, Group, Room
+from ...models import Course, Group, Module, Room
 from ...utils import get_current_week, get_week
 from .abstractparser import AbstractParser, ParserError
 
@@ -114,7 +114,8 @@ class Parser(AbstractParser):
             return
 
         course = Course.objects.create(
-            source=self.source, begin=begin, end=end
+            source=self.source, begin=begin, end=end,
+            celcat_id=int(event["id"])
         )
 
         min_i = 0
@@ -136,6 +137,13 @@ class Parser(AbstractParser):
             # par un dictionnaire classique.
             names = OrderedDict.fromkeys(data[i - 1].split(';'))
             course.name = ", ".join(names.keys())
+
+            module_names = [t for t in event["tag"]
+                            if len(t) > 0 and
+                            any(n.startswith(t) for n in names.keys())]
+            if len(module_names) > 0:
+                module, _ = Module.objects.get_or_create(name=module_names[0])
+                course.module = module
         else:
             course.name = "Sans nom"
         if i - 2 >= min_i:
diff --git a/models.py b/models.py
index c8e7b3d..b59f2d8 100644
--- a/models.py
+++ b/models.py
@@ -1,4 +1,4 @@
-#    Copyright (C) 2017-2018  Alban Gruin
+#    Copyright (C) 2017-2019  Alban Gruin
 #
 #    celcatsanitizer is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU Affero General Public License as published
@@ -200,6 +200,18 @@ class Room(SlugModel):
         verbose_name_plural = "salles"
 
 
+class Module(SlugModel):
+    name = models.CharField(max_length=255, unique=True, verbose_name="nom")
+    slug = models.SlugField(max_length=64, default="", unique=True)
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        verbose_name = "module"
+        verbose_name_plural = "modules"
+
+
 class CourseManager(Manager):
     def get_courses(self, obj, **criteria):
         qs = self.get_queryset()
@@ -246,6 +258,9 @@ class Course(models.Model):
     last_update = models.DateTimeField(verbose_name="dernière mise à jour",
                                        default=timezone.now)
 
+    celcat_id = models.IntegerField(verbose_name="ID Celcat", null=True)
+    module = models.ForeignKey(Module, on_delete=models.SET_NULL, null=True)
+
     def __str__(self):
         return self.name
 
diff --git a/tests.py b/tests.py
index 627602c..b411db3 100644
--- a/tests.py
+++ b/tests.py
@@ -1,4 +1,4 @@
-#    Copyright (C) 2017-2018  Alban Gruin
+#    Copyright (C) 2017-2019  Alban Gruin
 #
 #    celcatsanitizer is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU Affero General Public License as published
@@ -20,7 +20,7 @@ from django.utils import timezone
 
 from .management.parsers.abstractparser import ParserError
 from .management.parsers.ups2018 import Parser as UPS2018Parser
-from .models import Course, Group, Room, Source, Timetable, Year
+from .models import Course, Group, Module, Room, Source, Timetable, Year
 from .templatetags.rooms import format_rooms
 from .utils import tz_now
 
@@ -84,7 +84,7 @@ class CourseTestCase(TestCase):
         for group in (cma, tda2, self.tpa21, cmb, tdb2, self.tpb21,):
             course = Course.objects.create(
                 name="{0} course".format(group.name), type="cours",
-                source=source, begin=dt, end=dt)
+                source=source, begin=dt, end=dt, celcat_id=0)
             course.groups.add(group)
 
     def test_get_courses_for_group(self):
@@ -359,7 +359,8 @@ class RoomTestCase(TestCase):
                 end = begin + datetime.timedelta(hours=rn.get("duration", 2))
 
                 course = Course.objects.create(source=self.source,
-                                               begin=begin, end=end)
+                                               begin=begin, end=end,
+                                               celcat_id=0)
                 course.groups.add(group)
                 course.rooms.add(room)
 
@@ -413,12 +414,14 @@ class UPS2018ParserTestCase(TestCase):
     def test_get_event(self):
         get_event = self.parser._Parser__get_event
         count = Course.objects.count()
+        module_count = Module.objects.count()
 
         event = get_event(
             {"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00",
              "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque;AAA"
              "<br>L3 Info s1 CMA;L3 Info s1 TDA2<br>"
-             "Salle quelconque;Salle quelconque 2<br>Commentaire"},
+             "Salle quelconque;Salle quelconque 2<br>Commentaire", "id": "0",
+             "tag": ["abc", "def", "AAA"]},
             timezone.make_aware(datetime.datetime(2018, 9, 21)),
             timezone.make_aware(datetime.datetime(2018, 9, 1)),
             timezone.make_aware(datetime.datetime(2018, 10, 1)),
@@ -436,13 +439,18 @@ class UPS2018ParserTestCase(TestCase):
         self.assertIn(self.room2, event.rooms.all())
         self.assertEqual(event.rooms.count(), 2)
         self.assertEqual(event.notes, "Commentaire")
+        self.assertEqual(event.celcat_id, 0)
         self.assertEqual(event.begin, timezone.make_aware(
             datetime.datetime(2018, 9, 21, 10, 0, 0)))
         self.assertEqual(event.end, timezone.make_aware(
             datetime.datetime(2018, 9, 21, 12, 0, 0)))
+        self.assertEqual(event.module.name, "AAA")
 
-        self.assertEqual(count, Course.objects.count() - 1)
         count += 1
+        module_count += 1
+
+        self.assertEqual(count, Course.objects.count())
+        self.assertEqual(module_count, Module.objects.count())
 
         events = [
             {
@@ -451,6 +459,9 @@ class UPS2018ParserTestCase(TestCase):
                 "name": "Cours quelconque", "type": "COURS/TD",
                 "group": self.group,
                 "room": self.room,
+                "id": "1",
+                "tag": ["aaa", "Cours"],
+                "meta": 1
             },
             {
                 "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque"
@@ -458,7 +469,8 @@ class UPS2018ParserTestCase(TestCase):
                 "name": "Cours quelconque",
                 "type": "COURS/TD",
                 "group": ngroup,
-                "notes": "Salle quelconque 3"
+                "notes": "Salle quelconque 3",
+                "id": "2"
             },
             {
                 "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque"
@@ -467,44 +479,50 @@ class UPS2018ParserTestCase(TestCase):
                 "type": "COURS/TD",
                 "group": self.group,
                 "notes": "Salle quelconque 3\nCommentaire",
+                "id": "3"
             },
             {
                 "text": "(10:00-12:00)<br>COURS/TD"
                 "<br>L3 Info s1 CMA<br>Salle quelconque 3",
                 "name": "COURS/TD",
                 "group": self.group,
-                "notes": "Salle quelconque 3"
+                "notes": "Salle quelconque 3",
+                "id": "4"
             },
             {
                 "text": "COURS/TD<br>L3 Info s1 CMA<br>Salle quelconque 3",
                 "name": "COURS/TD",
                 "group": self.group,
-                "notes": "Salle quelconque 3"
+                "notes": "Salle quelconque 3",
+                "id": "5"
             },
             {
                 "text": "L3 Info s1 CMA<br>Salle quelconque",
                 "name": "Sans nom",
                 "group": self.group,
-                "room": self.room
+                "room": self.room,
+                "id": "6"
             },
             {
                 "text": "L3 Info s1 CMA<br>Salle quelconque 3",
                 "name": "Sans nom",
                 "group": self.group,
-                "notes": "Salle quelconque 3"
+                "notes": "Salle quelconque 3",
+                "id": "7"
             },
             {
                 "text": "(10:00-12:00)<br>L3 Info s1 CMA<br>Salle quelconque",
                 "name": "Sans nom",
                 "group": self.group,
-                "room": self.room
+                "room": self.room,
+                "id": "8"
             }
         ]
 
         for e in events:
             event = get_event(
                 {"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00",
-                 "text": e["text"]},
+                 "text": e["text"], "id": e["id"], "tag": e.get("tag", [])},
                 timezone.make_aware(datetime.datetime(2018, 9, 21)),
                 timezone.make_aware(datetime.datetime(2018, 9, 1)),
                 timezone.make_aware(datetime.datetime(2018, 10, 1)),
@@ -513,6 +531,7 @@ class UPS2018ParserTestCase(TestCase):
             self.assertEqual(event.name, e["name"])
             self.assertIn(e["group"], event.groups.all())
             self.assertEqual(event.groups.count(), 1)
+            self.assertEqual(str(event.celcat_id), e["id"])
 
             if "type" in e:
                 self.assertEqual(event.type, e["type"])
@@ -530,9 +549,17 @@ class UPS2018ParserTestCase(TestCase):
             else:
                 self.assertIsNone(event.notes)
 
-            self.assertEqual(count, Course.objects.count() - 1)
+            if "tag" in e:
+                self.assertEqual(event.module.name, e["tag"][e["meta"]])
+                module_count += 1
+            else:
+                self.assertIsNone(event.module)
+
             count += 1
 
+            self.assertEqual(count, Course.objects.count())
+            self.assertEqual(module_count, Module.objects.count())
+
         event = get_event(
             {"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00",
              "text": "Global Event"},
@@ -560,11 +587,13 @@ class UPS2018ParserTestCase(TestCase):
             {"begin":
              timezone.make_aware(datetime.datetime(2018, 9, 21, 10, 00, 00)),
              "end":
-             timezone.make_aware(datetime.datetime(2018, 9, 21, 12, 00, 00))},
+             timezone.make_aware(datetime.datetime(2018, 9, 21, 12, 00, 00)),
+             "id": 0},
             {"begin":
              timezone.make_aware(datetime.datetime(2018, 10, 22, 10, 00, 00)),
              "end":
-             timezone.make_aware(datetime.datetime(2018, 10, 22, 12, 00, 00))}
+             timezone.make_aware(datetime.datetime(2018, 10, 22, 12, 00, 00)),
+             "id": 2}
         ]
 
         for i, course in enumerate(self.parser.get_events(
@@ -578,6 +607,8 @@ class UPS2018ParserTestCase(TestCase):
             self.assertIsNone(course.notes)
             self.assertEqual(course.begin, courses[i]["begin"])
             self.assertEqual(course.end, courses[i]["end"])
+            self.assertEqual(course.celcat_id, courses[i]["id"])
+            self.assertEqual(course.module, None)
 
         self.assertEqual(i, len(courses) - 1)
 
@@ -588,15 +619,21 @@ class UPS2018ParserTestCase(TestCase):
             [{
                 "start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00",
                 "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque"
-                "<br>L3 Info s1 CMA<br>Salle quelconque"
+                "<br>L3 Info s1 CMA<br>Salle quelconque",
+                "id": "0",
+                "tag": [],
             }], [{
                 "start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00",
                 "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque"
-                "<br>L3 Info s1 CMA<br>Salle quelconque"
+                "<br>L3 Info s1 CMA<br>Salle quelconque",
+                "id": "1",
+                "tag": [],
             }, {
                 "start": "2018-10-22T10:00:00", "end": "2018-10-22T12:00:00",
                 "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque"
-                "<br>L3 Info s1 CMA<br>Salle quelconque"
+                "<br>L3 Info s1 CMA<br>Salle quelconque",
+                "id": "2",
+                "tag": [],
             }], [], [], [], [], [], [], [], [], []])
 
     def test_get_update_date(self):
diff --git a/tests/data/2018/october.html b/tests/data/2018/october.html
index 6f326f9..ab3da0b 100644
--- a/tests/data/2018/october.html
+++ b/tests/data/2018/october.html
@@ -40,7 +40,7 @@ do_something();
 <script>
 function courses() {
 var v = {};
-v.events.list = [{"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque<br>L3 Info s1 CMA<br>Salle quelconque"}, {"start": "2018-10-22T10:00:00", "end": "2018-10-22T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque<br>L3 Info s1 CMA<br>Salle quelconque"}];;
+v.events.list = [{"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque<br>L3 Info s1 CMA<br>Salle quelconque", "id": "1", "tag": []}, {"start": "2018-10-22T10:00:00", "end": "2018-10-22T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque<br>L3 Info s1 CMA<br>Salle quelconque", "id": "2", "tag": []}];;
 }
 </script>
 
diff --git a/tests/data/2018/september.html b/tests/data/2018/september.html
index 3db6cbc..c81fc3b 100644
--- a/tests/data/2018/september.html
+++ b/tests/data/2018/september.html
@@ -41,7 +41,7 @@ alert("something");
 <script>
 function courses() {
 var v = {};
-v.events.list = [{"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque<br>L3 Info s1 CMA<br>Salle quelconque"}];;
+v.events.list = [{"start": "2018-09-21T10:00:00", "end": "2018-09-21T12:00:00", "text": "(10:00-12:00)<br>COURS/TD<br>Cours quelconque<br>L3 Info s1 CMA<br>Salle quelconque", "id": "0", "tag": []}];;
 }
 </script>