Merge branch 'stable/0.10.z' into prod/pa1ch/0.y.zv0.10.0-pa1ch

3 files changed, 128 insertions, 106 deletions

diff --git a/management/commands/_private.py b/management/commands/_private.py
index c140f51..8f195a1 100644
--- a/management/commands/_private.py
+++ b/management/commands/_private.py
@@ -13,33 +13,20 @@
 #    You should have received a copy of the GNU Affero General Public License
 #    along with celcatsanitizer.  If not, see <http://www.gnu.org/licenses/>.
 
+import datetime
+import re
+
 from bs4 import BeautifulSoup
 from django.utils import timezone
 
 from edt.models import Group, Room, Course
 from edt.utils import get_week
 
-import datetime
-import re
-
 import requests
 
-
-class Week:
-    def __init__(self, number, start):
-        self.number = number
-        self.start = timezone.make_aware(
-            datetime.datetime.strptime(start, "%d/%m/%Y"))
-
-    def get_day(self, id):
-        return self.start + datetime.timedelta(id)
-
-    @property
-    def year(self):
-        return self.start.year
-
 def add_time(date, time):
-    delta = datetime.timedelta(hours=time.hour, minutes=time.minute)
+    ptime = datetime.datetime.strptime(time, "%H:%M")
+    delta = datetime.timedelta(hours=ptime.hour, minutes=ptime.minute)
     return date + delta
 
 def consolidate_group(group):
@@ -75,7 +62,7 @@ def consolidate_group(group):
 
 def consolidate_groups(groups):
     for group in groups:
-        if group.parent == None:
+        if group.parent is None:
             consolidate_group(group)
 
 def delete_courses_in_week(timetable, year, week):
@@ -93,50 +80,69 @@ def get_from_db_or_create(cls, **kwargs):
 
     return obj
 
-def get_events(timetable, year, week, soup, weeks_in_soup):
+def get_event(timetable, event, event_week):
+    """Renvoie une classe Course à partir d’un événement récupéré par BS4"""
+    # On récupère la date de l’évènement à partir de la semaine
+    # et de la semaine référencée, puis l’heure de début et de fin
+    date = event_week + datetime.timedelta(int(event.day.text))
+    begin = add_time(date, event.starttime.text)
+    end = add_time(date, event.endtime.text)
+
+    # Création de l’objet cours
+    course = Course.objects.create(timetable=timetable, begin=begin, end=end)
+
+    # On récupère les groupes concernés par les cours, on les
+    # « consolide », puis on les insère dans l’objet cours.
+    groups = [get_from_db_or_create(Group, timetable=timetable,
+                                    celcat_name=item.text)
+              for item in event.resources.group.find_all("item")]
+    consolidate_groups(groups)
+    course.groups.add(*groups)
+
+    # On récupère le champ « remarque »
+    if event.notes is not None:
+        course.notes = event.notes.text
+
+    # On récupère le nom du cours
+    if event.resources.module is not None:
+        course.name = event.resources.module.item.text
+    else:
+        # Il est possible qu’un cours n’ait pas de nom. Oui oui.
+        # Qui sont les concepteurs de ce système ? Quels sont leurs
+        # réseaux ?
+        # Bref, dans ce cas, on déplace le champ « remarque » de
+        # l’objet dans le champ « nom ».
+        course.name, course.notes = course.notes, None
+
+    # Récupération du type de cours
+    if event.category is not None:
+        course.type = event.category.text
+
+    # Si un cours a une salle attribuée (oui, il est possible qu’il n’y
+    # en ait pas… qui sont ils, leurs réseaux, tout ça…), on les insère
+    # dans la base de données, et on les ajoute dans l’objet cours
+    if event.resources.room is not None:
+        rooms = [get_from_db_or_create(Room, name=item.text)
+                 for item in event.resources.room.find_all("item")]
+        course.rooms.add(*rooms)
+
+    return course
+
+def get_events(timetable, soup, weeks_in_soup, year=None, week=None):
+    """Récupère tous les cours disponibles dans l’emploi du temps Celcat.
+    Le traîtement se limitera à la semaine indiquée si il y en a une."""
     for event in soup.find_all("event"):
-        title = None
-        type_ = None
-        groups = None
-        rooms = None
-        notes = None
-
-        if weeks_in_soup[event.rawweeks.text].number == week and \
-           weeks_in_soup[event.rawweeks.text].year == year and \
+        event_week = weeks_in_soup[event.rawweeks.text]
+        event_week_num = event_week.isocalendar()[1] # Numéro de semaine
+
+        # On passe le traitement si la semaine de l’événement ne correspond pas
+        # à la semaine passée, ou qu’il ne contient pas de groupe ou n’a pas de
+        # date de début ou de fin.
+        if (event_week_num == week and event_week.year == year or \
+            year is None or week is None) and \
            event.resources.group is not None and \
            event.starttime is not None and event.endtime is not None:
-            date = weeks_in_soup[event.rawweeks.text].get_day(int(
-                event.day.text))
-
-            begin = add_time(date, datetime.datetime.strptime(
-                event.starttime.text, "%H:%M"))
-            end = add_time(date, datetime.datetime.strptime(
-                event.endtime.text, "%H:%M"))
-
-            groups = [get_from_db_or_create(Group, timetable=timetable,
-                                            celcat_name=item.text)
-                      for item in event.resources.group.find_all("item")]
-            consolidate_groups(groups)
-
-            if event.notes is not None:
-                notes = event.notes.text
-
-            if event.resources.module is not None:
-                title = event.resources.module.item.text
-            elif notes is not None:
-                title = notes
-                notes = None
-            else:
-                title = "Aucune information"
-
-            if event.category is not None:
-                type_ = event.category.text
-
-            if event.resources.room is not None:
-                rooms = [get_from_db_or_create(Room, name=item.text)
-                         for item in event.resources.room.find_all("item")]
-
-            yield title, type_, groups, rooms, notes, begin, end
+            yield get_event(timetable, event, event_week)
 
 def get_update_date(soup):
     # Explication de la regex
@@ -153,7 +159,7 @@ def get_update_date(soup):
     #                           (\d+)       au moins un nombre
     #                                :      un deux-points
     #                                 (\d+) au moins un nombre
-    datetime_regex = re.compile("(\d+)/(\d+)/(\d+)\s+(\d+):(\d+):(\d+)")
+    datetime_regex = re.compile(r"(\d+)/(\d+)/(\d+)\s+(\d+):(\d+):(\d+)")
     search = datetime_regex.search(soup.footer.text)
     if search is None:
         return None
@@ -163,16 +169,26 @@ def get_update_date(soup):
     return timezone.make_aware(date)
 
 def get_weeks(soup):
+    # Les semaines sont référencées de manière assez… exotique
+    # En gros, il y a une liste d’éléments span qui contiennent une sorte d’ID
+    # de la semaine, formaté de la manière suivante :
+    # NNNNNNNNNNNNNNNNNNNYNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN
+    # Tous sont de la même longueur, contiennent 51 N et un seul Y.
+    # Allez savoir pourquoi. Il se trouve dans la balise « alleventweeks ».
+    # Un paramètre du span (« date ») représente la date de début.
+    # Un cours contient donc un ID de semaine, puis le nombre de jours après le
+    # début de cette semaine.
     weeks = {}
-    for span in soup.find_all("span"):
-        weeks[span.alleventweeks.text] = Week(int(span.title.text),
-                                              span["date"])
+    for span in soup.find_all("span"): # Liste de toutes les semaines définies
+        # On parse la date et on la fait correspondre à l’ID
+        weeks[span.alleventweeks.text] = timezone.make_aware(
+            datetime.datetime.strptime(span["date"], "%d/%m/%Y"))
 
     return weeks
 
 def get_xml(url):
-    r = requests.get(url)
-    r.encoding = "utf8"
+    req = requests.get(url)
+    req.encoding = "utf8"
 
-    soup = BeautifulSoup(r.content, "html.parser")
+    soup = BeautifulSoup(req.content, "html.parser")
     return soup
diff --git a/management/commands/cleancourses.py b/management/commands/cleancourses.py
index ca2ef94..f6041ef 100644
--- a/management/commands/cleancourses.py
+++ b/management/commands/cleancourses.py
@@ -15,7 +15,7 @@
 
 from django.core.management.base import BaseCommand
 from django.db import transaction
-from edt.models import Course, Group, LastUpdate
+from edt.models import Course, Group
 
 
 class Command(BaseCommand):
@@ -29,10 +29,8 @@ class Command(BaseCommand):
             if options["timetable"] is None:
                 Course.objects.all().delete()
                 Group.objects.all().delete()
-                LastUpdate.objects.all().delete()
             else:
                 Course.objects.filter(timetable__id__in=options["timetable"]).delete()
                 Group.objects.filter(timetable__id__in=options["timetable"]).delete()
-                LastUpdate.objects.filter(timetable__id__in=options["timetable"]).delete()
 
         self.stdout.write(self.style.SUCCESS("Done."))
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index c82b0e4..76f0a7c 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -13,59 +13,63 @@
 #    You should have received a copy of the GNU Affero General Public License
 #    along with celcatsanitizer.  If not, see <http://www.gnu.org/licenses/>.
 
+import datetime
+
 from django.core.management.base import BaseCommand
 from django.db import transaction
+from django.db.models import Min
 from django.utils import timezone
 
-from edt.models import Timetable, LastUpdate, Course
+from edt.models import Course, Timetable
+from edt.utils import get_week
 from ._private import delete_courses_in_week, get_events, get_update_date, get_weeks, get_xml
 
-import datetime
-
 @transaction.atomic
-def process_timetable_week(timetable, year, week, soup, weeks_in_soup):
-    last_update_date = None
+def process_timetable_week(timetable, soup, weeks_in_soup, force, year=None, week=None):
+    criteria = {}
+    if year is not None and week is not None:
+        begin, end = get_week(year, week)
+        criteria["begin__gte"] = begin
+        criteria["begin__lt"] = end
+
+    last_update_date = Course.objects.filter(timetable=timetable, **criteria) \
+                                     .aggregate(Min("last_update")) \
+                                     ["last_update__min"]
     new_update_date = get_update_date(soup)
-    try:
-        last_update = LastUpdate.objects.get(timetable=timetable, year=year, week=week)
-        last_update_date = last_update.updated_at
-    except:
-        last_update = LastUpdate(timetable=timetable, year=year, week=week)
 
-    if last_update_date is not None and new_update_date is not None and \
+    if not force and last_update_date is not None and new_update_date is not None and \
        last_update_date >= new_update_date:
         return
 
-    delete_courses_in_week(timetable, year, week)
-    for name, type_, groups, rooms, notes, begin, end in \
-        get_events(timetable, year, week, soup, weeks_in_soup):
-        course = Course.objects.create(timetable=timetable, begin=begin, end=end)
-        course.name = name
-        course.type = type_
-        course.notes = notes
-
-        course.groups.add(*groups)
-        if rooms is not None:
-            course.rooms.add(*rooms)
+    if year is not None and week is not None:
+        delete_courses_in_week(timetable, year, week)
+    else:
+        Course.objects.filter(timetable=timetable,
+                              begin__gte=min(weeks_in_soup.values())).delete()
 
+    for course in get_events(timetable, soup, weeks_in_soup, year, week):
         course.save()
 
-    last_update.date = timezone.make_aware(datetime.datetime.now())
-    last_update.updated_at = new_update_date
-    last_update.save()
+    timetable.last_update_date = new_update_date
+    timetable.save()
 
-def process_timetable(timetable, year, weeks):
+def process_timetable(timetable, force, year=None, weeks=None):
     soup = get_xml(timetable.url)
     weeks_in_soup = get_weeks(soup)
 
-    for week in weeks:
-        process_timetable_week(timetable, year, week, soup, weeks_in_soup)
+    if year is not None and weeks is not None:
+        for week in weeks:
+            process_timetable_week(timetable, soup, weeks_in_soup, force, year, week)
+    else:
+        process_timetable_week(timetable, soup, weeks_in_soup, force)
 
 
 class Command(BaseCommand):
     help = "Fetches registered celcat timetables"
 
     def add_arguments(self, parser):
+        parser.add_argument("--all", const=True, default=False, action="store_const")
+        parser.add_argument("--force", const=True, default=False, action="store_const")
         parser.add_argument("--week", type=int, choices=range(1, 54), nargs="+")
         parser.add_argument("--year", type=int, nargs=1)
 
@@ -73,7 +77,9 @@ class Command(BaseCommand):
         year = None
         errcount = 0
 
-        if options["week"] is None:
+        if options["all"]:
+            weeks = None
+        elif options["week"] is None:
             _, week, day = timezone.now().isocalendar()
             if day >= 6:
                 year, week, _ = (timezone.now() + datetime.timedelta(weeks=1)).isocalendar()
@@ -81,18 +87,20 @@ class Command(BaseCommand):
         else:
             weeks = options["week"]
 
-        if options["year"] is None and year is None:
-            year = timezone.now().year
-        elif year is None:
-            year = options["year"][0]
+        if not options["all"]:
+            if options["year"] is None and year is None:
+                year = timezone.now().year
+            elif year is None:
+                year = options["year"][0]
 
         for timetable in Timetable.objects.all():
             self.stdout.write("Processing {0}".format(timetable))
 
             try:
-                process_timetable(timetable, year, weeks)
-            except Exception as e:
-                self.stderr.write(self.style.ERROR("Failed to process {0}: {1}".format(timetable, e)))
+                process_timetable(timetable, options["force"], year, weeks)
+            except Exception as exc:
+                self.stderr.write(
+                    self.style.ERROR("Failed to process {0}: {1}".format(timetable, exc)))
                 errcount += 1
 
         if errcount == 0: