From c5d409e7c38cd5dd5686ce2311928587796349f9 Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Tue, 12 Sep 2017 21:32:35 +0200
Subject: Lecture du contenu de la réponse avec r.content et non r.text pour
limiter les problèmes
---
management/commands/_private.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
(limited to 'management')
diff --git a/management/commands/_private.py b/management/commands/_private.py
index 17896c4..2b57599 100644
--- a/management/commands/_private.py
+++ b/management/commands/_private.py
@@ -159,5 +159,5 @@ def get_xml(url):
r = requests.get(url)
r.encoding = "utf8"
- soup = BeautifulSoup(r.text, "html.parser")
+ soup = BeautifulSoup(r.content, "html.parser")
return soup
--
cgit v1.2.1
From 32bd236f2c53bc8dfd515018a7ae0ec06f65c115 Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Sun, 10 Sep 2017 11:54:38 +0200
Subject: La consolidation a lieu pour le parent d’un groupe mais aussi ses
enfants
---
management/commands/_private.py | 31 +++++++++++++++++++++++--------
1 file changed, 23 insertions(+), 8 deletions(-)
(limited to 'management')
diff --git a/management/commands/_private.py b/management/commands/_private.py
index 2b57599..c140f51 100644
--- a/management/commands/_private.py
+++ b/management/commands/_private.py
@@ -45,19 +45,34 @@ def add_time(date, time):
def consolidate_group(group):
group_content_key = ("mention", "subgroup", "td", "tp")
group_content_list = group.group_info[1:]
- group_content = dict(zip(group_content_key, group_content_list))
- for i in range(len(group_content_list))[::-1]:
- del group_content[group_content_key[i]]
- group_content[group_content_key[i] + "__isnull"] = True
+ if group.subgroup is not None:
+ group_content = dict(zip(group_content_key, group_content_list))
- if group_content_list[i] is not None:
- break
+ for i in range(len(group_content_list))[::-1]:
+ del group_content[group_content_key[i]]
+ group_content[group_content_key[i] + "__isnull"] = True
- if "subgroup" in group_content:
- group.parent = Group.objects.filter(**group_content).first()
+ if group_content_list[i] is not None:
+ break
+
+ group.parent = Group.objects.filter(timetable=group.timetable,
+ **group_content).first()
group.save()
+ if group.tp is None:
+ group_content = dict(zip(group_content_key, group_content_list))
+ last_is_none = False
+
+ for i, key in enumerate(group_content_key):
+ if group_content_list[i] is None or last_is_none:
+ del group_content[key]
+ group_content[key + "__isnull"] = last_is_none
+ last_is_none = True
+
+ Group.objects.filter(timetable=group.timetable, parent__isnull=True,
+ **group_content).update(parent=group)
+
def consolidate_groups(groups):
for group in groups:
if group.parent == None:
--
cgit v1.2.1
From e30b0c673cae607175c5bae6df6e87d8b100bdf3 Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Sun, 24 Sep 2017 17:16:10 +0200
Subject: Suppression du modèle LastUpdate, utilisation du champ last_update à
la place.
---
management/commands/cleancourses.py | 4 +---
management/commands/timetables.py | 18 +++++++-----------
2 files changed, 8 insertions(+), 14 deletions(-)
(limited to 'management')
diff --git a/management/commands/cleancourses.py b/management/commands/cleancourses.py
index ca2ef94..f6041ef 100644
--- a/management/commands/cleancourses.py
+++ b/management/commands/cleancourses.py
@@ -15,7 +15,7 @@
from django.core.management.base import BaseCommand
from django.db import transaction
-from edt.models import Course, Group, LastUpdate
+from edt.models import Course, Group
class Command(BaseCommand):
@@ -29,10 +29,8 @@ class Command(BaseCommand):
if options["timetable"] is None:
Course.objects.all().delete()
Group.objects.all().delete()
- LastUpdate.objects.all().delete()
else:
Course.objects.filter(timetable__id__in=options["timetable"]).delete()
Group.objects.filter(timetable__id__in=options["timetable"]).delete()
- LastUpdate.objects.filter(timetable__id__in=options["timetable"]).delete()
self.stdout.write(self.style.SUCCESS("Done."))
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index c82b0e4..45a57ee 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -17,20 +17,19 @@ from django.core.management.base import BaseCommand
from django.db import transaction
from django.utils import timezone
-from edt.models import Timetable, LastUpdate, Course
+from edt.models import Timetable, Course
+from edt.utils import get_week
from ._private import delete_courses_in_week, get_events, get_update_date, get_weeks, get_xml
import datetime
@transaction.atomic
def process_timetable_week(timetable, year, week, soup, weeks_in_soup):
- last_update_date = None
+ begin, end = get_weeks(year, week)
+
+ last_update_date = Course.objects.filter(begin__gte=start, begin__lt=end).aggregate(Max("last_update"))["last_update__max"]
new_update_date = get_update_date(soup)
- try:
- last_update = LastUpdate.objects.get(timetable=timetable, year=year, week=week)
- last_update_date = last_update.updated_at
- except:
- last_update = LastUpdate(timetable=timetable, year=year, week=week)
+ updated_at = timezone.make_aware(datetime.datetime.now())
if last_update_date is not None and new_update_date is not None and \
last_update_date >= new_update_date:
@@ -43,6 +42,7 @@ def process_timetable_week(timetable, year, week, soup, weeks_in_soup):
course.name = name
course.type = type_
course.notes = notes
+ course.updated_at = updated_at
course.groups.add(*groups)
if rooms is not None:
@@ -50,10 +50,6 @@ def process_timetable_week(timetable, year, week, soup, weeks_in_soup):
course.save()
- last_update.date = timezone.make_aware(datetime.datetime.now())
- last_update.updated_at = new_update_date
- last_update.save()
-
def process_timetable(timetable, year, weeks):
soup = get_xml(timetable.url)
weeks_in_soup = get_weeks(soup)
--
cgit v1.2.1
From dc5618c3b3af18a5f6e1d50844097c745317125a Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Sun, 24 Sep 2017 17:36:40 +0200
Subject: Ajout de la date et de l’heure automatiquement lors de la création
d’un cours
---
management/commands/timetables.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
(limited to 'management')
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index 45a57ee..0716a91 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -15,6 +15,7 @@
from django.core.management.base import BaseCommand
from django.db import transaction
+from django.db.models import Max
from django.utils import timezone
from edt.models import Timetable, Course
@@ -27,9 +28,8 @@ import datetime
def process_timetable_week(timetable, year, week, soup, weeks_in_soup):
begin, end = get_weeks(year, week)
- last_update_date = Course.objects.filter(begin__gte=start, begin__lt=end).aggregate(Max("last_update"))["last_update__max"]
+ last_update_date = Course.objects.filter(begin__gte=begin, begin__lt=end).aggregate(Max("last_update"))["last_update__max"]
new_update_date = get_update_date(soup)
- updated_at = timezone.make_aware(datetime.datetime.now())
if last_update_date is not None and new_update_date is not None and \
last_update_date >= new_update_date:
@@ -42,7 +42,6 @@ def process_timetable_week(timetable, year, week, soup, weeks_in_soup):
course.name = name
course.type = type_
course.notes = notes
- course.updated_at = updated_at
course.groups.add(*groups)
if rooms is not None:
--
cgit v1.2.1
From a4fbe45dcdc89dbcfe50afd5058b0ecf09ed7642 Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Sun, 24 Sep 2017 17:37:21 +0200
Subject: Suppression des import inutilisés
---
management/commands/timetables.py | 1 -
1 file changed, 1 deletion(-)
(limited to 'management')
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index 0716a91..efdd611 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -19,7 +19,6 @@ from django.db.models import Max
from django.utils import timezone
from edt.models import Timetable, Course
-from edt.utils import get_week
from ._private import delete_courses_in_week, get_events, get_update_date, get_weeks, get_xml
import datetime
--
cgit v1.2.1
From 020d204d393ebf8c7f2bc9f6848cd4892b278f70 Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Sun, 24 Sep 2017 18:13:44 +0200
Subject: Récupération de la dernière mise à jour en filtrant par emploi du
temps Changement de la valeur par défaut de last_update, qui posait problème
get_week, pas get_weeks
---
management/commands/timetables.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
(limited to 'management')
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index efdd611..e136f53 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -19,15 +19,16 @@ from django.db.models import Max
from django.utils import timezone
from edt.models import Timetable, Course
+from edt.utils import get_week
from ._private import delete_courses_in_week, get_events, get_update_date, get_weeks, get_xml
import datetime
@transaction.atomic
def process_timetable_week(timetable, year, week, soup, weeks_in_soup):
- begin, end = get_weeks(year, week)
+ begin, end = get_week(year, week)
- last_update_date = Course.objects.filter(begin__gte=begin, begin__lt=end).aggregate(Max("last_update"))["last_update__max"]
+ last_update_date = Course.objects.filter(timetable=timetable, begin__gte=begin, begin__lt=end).aggregate(Max("last_update"))["last_update__max"]
new_update_date = get_update_date(soup)
if last_update_date is not None and new_update_date is not None and \
--
cgit v1.2.1
From aa7ccf13a8735f162e2ea859ad1d8ebe9f34f657 Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Mon, 25 Sep 2017 16:08:01 +0200
Subject: Utilisation de pylint pour améliorer la qualité du code
---
management/commands/_private.py | 20 ++++++++++----------
management/commands/timetables.py | 19 +++++++++++++------
2 files changed, 23 insertions(+), 16 deletions(-)
(limited to 'management')
diff --git a/management/commands/_private.py b/management/commands/_private.py
index c140f51..2d01e67 100644
--- a/management/commands/_private.py
+++ b/management/commands/_private.py
@@ -13,15 +13,15 @@
# You should have received a copy of the GNU Affero General Public License
# along with celcatsanitizer. If not, see .
+import datetime
+import re
+
from bs4 import BeautifulSoup
from django.utils import timezone
from edt.models import Group, Room, Course
from edt.utils import get_week
-import datetime
-import re
-
import requests
@@ -31,8 +31,8 @@ class Week:
self.start = timezone.make_aware(
datetime.datetime.strptime(start, "%d/%m/%Y"))
- def get_day(self, id):
- return self.start + datetime.timedelta(id)
+ def get_day(self, day_id):
+ return self.start + datetime.timedelta(day_id)
@property
def year(self):
@@ -75,7 +75,7 @@ def consolidate_group(group):
def consolidate_groups(groups):
for group in groups:
- if group.parent == None:
+ if group.parent is None:
consolidate_group(group)
def delete_courses_in_week(timetable, year, week):
@@ -153,7 +153,7 @@ def get_update_date(soup):
# (\d+) au moins un nombre
# : un deux-points
# (\d+) au moins un nombre
- datetime_regex = re.compile("(\d+)/(\d+)/(\d+)\s+(\d+):(\d+):(\d+)")
+ datetime_regex = re.compile(r"(\d+)/(\d+)/(\d+)\s+(\d+):(\d+):(\d+)")
search = datetime_regex.search(soup.footer.text)
if search is None:
return None
@@ -171,8 +171,8 @@ def get_weeks(soup):
return weeks
def get_xml(url):
- r = requests.get(url)
- r.encoding = "utf8"
+ req = requests.get(url)
+ req.encoding = "utf8"
- soup = BeautifulSoup(r.content, "html.parser")
+ soup = BeautifulSoup(req.content, "html.parser")
return soup
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index e136f53..f01ac3c 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -13,6 +13,8 @@
# You should have received a copy of the GNU Affero General Public License
# along with celcatsanitizer. If not, see .
+import datetime
+
from django.core.management.base import BaseCommand
from django.db import transaction
from django.db.models import Max
@@ -22,13 +24,15 @@ from edt.models import Timetable, Course
from edt.utils import get_week
from ._private import delete_courses_in_week, get_events, get_update_date, get_weeks, get_xml
-import datetime
-
@transaction.atomic
def process_timetable_week(timetable, year, week, soup, weeks_in_soup):
begin, end = get_week(year, week)
- last_update_date = Course.objects.filter(timetable=timetable, begin__gte=begin, begin__lt=end).aggregate(Max("last_update"))["last_update__max"]
+ last_update_date = Course.objects.filter(timetable=timetable,
+ begin__gte=begin,
+ begin__lt=end) \
+ .aggregate(Max("last_update")) \
+ ["last_update__max"]
new_update_date = get_update_date(soup)
if last_update_date is not None and new_update_date is not None and \
@@ -38,7 +42,9 @@ def process_timetable_week(timetable, year, week, soup, weeks_in_soup):
delete_courses_in_week(timetable, year, week)
for name, type_, groups, rooms, notes, begin, end in \
get_events(timetable, year, week, soup, weeks_in_soup):
- course = Course.objects.create(timetable=timetable, begin=begin, end=end)
+ course = Course.objects.create(timetable=timetable,
+ begin=begin,
+ end=end)
course.name = name
course.type = type_
course.notes = notes
@@ -86,8 +92,9 @@ class Command(BaseCommand):
try:
process_timetable(timetable, year, weeks)
- except Exception as e:
- self.stderr.write(self.style.ERROR("Failed to process {0}: {1}".format(timetable, e)))
+ except Exception as exc:
+ self.stderr.write(
+ self.style.ERROR("Failed to process {0}: {1}".format(timetable, exc)))
errcount += 1
if errcount == 0:
--
cgit v1.2.1
From a8d35aee63f073674993b8afde78a8d5c1517e05 Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Fri, 29 Sep 2017 21:19:01 +0200
Subject: get_events() renvoie des objets Course au lieu d’un tuple de données.
Ajout de commentaires dans la fonction get_events() Les paramètres year et
week des fonctions get_events(), process_timetable_week() et
process_timetable() sont maintenant optionnels.
---
management/commands/_private.py | 59 +++++++++++++++++++++++++--------------
management/commands/timetables.py | 25 ++++++-----------
2 files changed, 46 insertions(+), 38 deletions(-)
(limited to 'management')
diff --git a/management/commands/_private.py b/management/commands/_private.py
index 2d01e67..e018e3a 100644
--- a/management/commands/_private.py
+++ b/management/commands/_private.py
@@ -93,50 +93,67 @@ def get_from_db_or_create(cls, **kwargs):
return obj
-def get_events(timetable, year, week, soup, weeks_in_soup):
+def get_events(timetable, soup, weeks_in_soup, year=None, week=None):
+ """Récupère tous les cours disponibles dans l’emploi du temps Celcat.
+ Le traîtement se limitera à la semaine indiquée si il y en a une."""
for event in soup.find_all("event"):
- title = None
- type_ = None
- groups = None
- rooms = None
- notes = None
-
- if weeks_in_soup[event.rawweeks.text].number == week and \
- weeks_in_soup[event.rawweeks.text].year == year and \
+ event_week = weeks_in_soup[event.rawweeks.text]
+
+ # On passe le traitement si la semaine de l’événement ne correspond pas
+ # à la semaine passée, ou qu’il ne contient pas de groupe ou n’a pas de
+ # date de début ou de fin.
+ if (event_week.number == week and event_week.year == year or \
+ year is None or week is None) and \
event.resources.group is not None and \
event.starttime is not None and event.endtime is not None:
- date = weeks_in_soup[event.rawweeks.text].get_day(int(
- event.day.text))
-
+ # On récupère la date de l’évènement à partir de la semaine
+ # et de la semaine référencée, puis la date de début et de fin
+ date = event_week.get_day(int(event.day.text))
begin = add_time(date, datetime.datetime.strptime(
event.starttime.text, "%H:%M"))
end = add_time(date, datetime.datetime.strptime(
event.endtime.text, "%H:%M"))
+ # Création de l’objet cours
+ course = Course.objects.create(timetable=timetable, begin=begin,
+ end=end)
+
+ # On récupère les groupes concernés par les cours, on les
+ # « consolide », puis on les insère dans l’objet cours.
groups = [get_from_db_or_create(Group, timetable=timetable,
celcat_name=item.text)
for item in event.resources.group.find_all("item")]
consolidate_groups(groups)
+ course.groups.add(*groups)
+ # On récupère le champ « remarque »
if event.notes is not None:
- notes = event.notes.text
+ course.notes = event.notes.text
+ # On récupère le nom du cours
if event.resources.module is not None:
- title = event.resources.module.item.text
- elif notes is not None:
- title = notes
- notes = None
+ course.name = event.resources.module.item.text
else:
- title = "Aucune information"
-
+ # Il est possible qu’un cours n’ait pas de nom. Oui oui.
+ # Qui sont les concepteurs de ce système ? Quels sont leurs
+ # réseaux ?
+ # Bref, dans ce cas, on déplace le champ « remarque » de
+ # l’objet dans le champ « nom ».
+ course.name, course.notes = course.notes, None
+
+ # Récupération du type de cours
if event.category is not None:
- type_ = event.category.text
+ course.type = event.category.text
+ # Si un cours a une salle attribuée (oui, il est possible qu’il n’y
+ # en ait pas… qui sont ils, leurs réseaux, tout ça…), on les insère
+ # dans la base de données, et on les ajoute dans l’objet cours
if event.resources.room is not None:
rooms = [get_from_db_or_create(Room, name=item.text)
for item in event.resources.room.find_all("item")]
+ course.rooms.add(*rooms)
- yield title, type_, groups, rooms, notes, begin, end
+ yield course
def get_update_date(soup):
# Explication de la regex
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index f01ac3c..7ee63af 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -25,7 +25,7 @@ from edt.utils import get_week
from ._private import delete_courses_in_week, get_events, get_update_date, get_weeks, get_xml
@transaction.atomic
-def process_timetable_week(timetable, year, week, soup, weeks_in_soup):
+def process_timetable_week(timetable, soup, weeks_in_soup, year=None, week=None):
begin, end = get_week(year, week)
last_update_date = Course.objects.filter(timetable=timetable,
@@ -40,27 +40,18 @@ def process_timetable_week(timetable, year, week, soup, weeks_in_soup):
return
delete_courses_in_week(timetable, year, week)
- for name, type_, groups, rooms, notes, begin, end in \
- get_events(timetable, year, week, soup, weeks_in_soup):
- course = Course.objects.create(timetable=timetable,
- begin=begin,
- end=end)
- course.name = name
- course.type = type_
- course.notes = notes
-
- course.groups.add(*groups)
- if rooms is not None:
- course.rooms.add(*rooms)
-
+ for course in get_events(timetable, soup, weeks_in_soup, year, week):
course.save()
-def process_timetable(timetable, year, weeks):
+def process_timetable(timetable, year=None, weeks=None):
soup = get_xml(timetable.url)
weeks_in_soup = get_weeks(soup)
- for week in weeks:
- process_timetable_week(timetable, year, week, soup, weeks_in_soup)
+ if year is not None and weeks is not None:
+ for week in weeks:
+ process_timetable_week(timetable, soup, weeks_in_soup, year, week)
+ else:
+ process_timetable_week(timetable, soup, weeks_in_soup)
class Command(BaseCommand):
--
cgit v1.2.1
From 8d7b9f5e9156e9153b6060a8a7a0c33b188f21b7 Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Fri, 29 Sep 2017 21:39:13 +0200
Subject: Ajout d’une option pour récupérer tout un emploi du temps Pas encore
testé parce que j’aime vivre dangereusement
---
management/commands/timetables.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
(limited to 'management')
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index 7ee63af..279de89 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -60,12 +60,15 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument("--week", type=int, choices=range(1, 54), nargs="+")
parser.add_argument("--year", type=int, nargs=1)
+ parser.add_argument("--all", const=True, default=False, action="store_const")
def handle(self, *args, **options):
year = None
errcount = 0
- if options["week"] is None:
+ if options["all"]:
+ weeks = None
+ elif options["week"] is None:
_, week, day = timezone.now().isocalendar()
if day >= 6:
year, week, _ = (timezone.now() + datetime.timedelta(weeks=1)).isocalendar()
@@ -73,10 +76,11 @@ class Command(BaseCommand):
else:
weeks = options["week"]
- if options["year"] is None and year is None:
- year = timezone.now().year
- elif year is None:
- year = options["year"][0]
+ if not options["all"]:
+ if options["year"] is None and year is None:
+ year = timezone.now().year
+ elif year is None:
+ year = options["year"][0]
for timetable in Timetable.objects.all():
self.stdout.write("Processing {0}".format(timetable))
--
cgit v1.2.1
From bc2c2cae1fd6a52fb17771b4b537e07bca103987 Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Fri, 29 Sep 2017 22:09:51 +0200
Subject: Suppression de la classe Week \o/
---
management/commands/_private.py | 34 ++++++++++------------------------
1 file changed, 10 insertions(+), 24 deletions(-)
(limited to 'management')
diff --git a/management/commands/_private.py b/management/commands/_private.py
index e018e3a..0b26b4e 100644
--- a/management/commands/_private.py
+++ b/management/commands/_private.py
@@ -24,22 +24,9 @@ from edt.utils import get_week
import requests
-
-class Week:
- def __init__(self, number, start):
- self.number = number
- self.start = timezone.make_aware(
- datetime.datetime.strptime(start, "%d/%m/%Y"))
-
- def get_day(self, day_id):
- return self.start + datetime.timedelta(day_id)
-
- @property
- def year(self):
- return self.start.year
-
def add_time(date, time):
- delta = datetime.timedelta(hours=time.hour, minutes=time.minute)
+ ptime = datetime.datetime.strptime(time, "%H:%M")
+ delta = datetime.timedelta(hours=ptime.hour, minutes=ptime.minute)
return date + delta
def consolidate_group(group):
@@ -98,21 +85,20 @@ def get_events(timetable, soup, weeks_in_soup, year=None, week=None):
Le traîtement se limitera à la semaine indiquée si il y en a une."""
for event in soup.find_all("event"):
event_week = weeks_in_soup[event.rawweeks.text]
+ event_week_num = event_week.isocalendar()[1] # Numéro de semaine
# On passe le traitement si la semaine de l’événement ne correspond pas
# à la semaine passée, ou qu’il ne contient pas de groupe ou n’a pas de
# date de début ou de fin.
- if (event_week.number == week and event_week.year == year or \
+ if (event_week_num == week and event_week.year == year or \
year is None or week is None) and \
event.resources.group is not None and \
event.starttime is not None and event.endtime is not None:
# On récupère la date de l’évènement à partir de la semaine
- # et de la semaine référencée, puis la date de début et de fin
- date = event_week.get_day(int(event.day.text))
- begin = add_time(date, datetime.datetime.strptime(
- event.starttime.text, "%H:%M"))
- end = add_time(date, datetime.datetime.strptime(
- event.endtime.text, "%H:%M"))
+ # et de la semaine référencée, puis l’heure de début et de fin
+ date = event_week + datetime.timedelta(int(event.day.text))
+ begin = add_time(date, event.starttime.text)
+ end = add_time(date, event.endtime.text)
# Création de l’objet cours
course = Course.objects.create(timetable=timetable, begin=begin,
@@ -182,8 +168,8 @@ def get_update_date(soup):
def get_weeks(soup):
weeks = {}
for span in soup.find_all("span"):
- weeks[span.alleventweeks.text] = Week(int(span.title.text),
- span["date"])
+ weeks[span.alleventweeks.text] = timezone.make_aware(
+ datetime.datetime.strptime(span["date"], "%d/%m/%Y"))
return weeks
--
cgit v1.2.1
From bb15653aecf1bd13fd892eb54d04c8f3e0fee270 Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Fri, 29 Sep 2017 22:18:53 +0200
Subject: Ajout d’un paramètre à la commande timetables pour forcer la mise à
jour
---
management/commands/timetables.py | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
(limited to 'management')
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index 279de89..c5c6dfa 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -25,7 +25,7 @@ from edt.utils import get_week
from ._private import delete_courses_in_week, get_events, get_update_date, get_weeks, get_xml
@transaction.atomic
-def process_timetable_week(timetable, soup, weeks_in_soup, year=None, week=None):
+def process_timetable_week(timetable, soup, weeks_in_soup, force, year=None, week=None):
begin, end = get_week(year, week)
last_update_date = Course.objects.filter(timetable=timetable,
@@ -35,7 +35,7 @@ def process_timetable_week(timetable, soup, weeks_in_soup, year=None, week=None)
["last_update__max"]
new_update_date = get_update_date(soup)
- if last_update_date is not None and new_update_date is not None and \
+ if not force and last_update_date is not None and new_update_date is not None and \
last_update_date >= new_update_date:
return
@@ -43,24 +43,25 @@ def process_timetable_week(timetable, soup, weeks_in_soup, year=None, week=None)
for course in get_events(timetable, soup, weeks_in_soup, year, week):
course.save()
-def process_timetable(timetable, year=None, weeks=None):
+def process_timetable(timetable, force, year=None, weeks=None):
soup = get_xml(timetable.url)
weeks_in_soup = get_weeks(soup)
if year is not None and weeks is not None:
for week in weeks:
- process_timetable_week(timetable, soup, weeks_in_soup, year, week)
+ process_timetable_week(timetable, soup, weeks_in_soup, force, year, week)
else:
- process_timetable_week(timetable, soup, weeks_in_soup)
+ process_timetable_week(timetable, soup, weeks_in_soup, force)
class Command(BaseCommand):
help = "Fetches registered celcat timetables"
def add_arguments(self, parser):
+ parser.add_argument("--all", const=True, default=False, action="store_const")
+ parser.add_argument("--force", const=True, default=False, action="store_const")
parser.add_argument("--week", type=int, choices=range(1, 54), nargs="+")
parser.add_argument("--year", type=int, nargs=1)
- parser.add_argument("--all", const=True, default=False, action="store_const")
def handle(self, *args, **options):
year = None
@@ -86,7 +87,7 @@ class Command(BaseCommand):
self.stdout.write("Processing {0}".format(timetable))
try:
- process_timetable(timetable, year, weeks)
+ process_timetable(timetable, options["force"], year, weeks)
except Exception as exc:
self.stderr.write(
self.style.ERROR("Failed to process {0}: {1}".format(timetable, exc)))
--
cgit v1.2.1
From a796926c1d1e4ea19d1de205678be638fbfe8c06 Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Sat, 30 Sep 2017 11:51:54 +0200
Subject: Déplacement de la logique de création de l’objet Course dans sa
propre fonction
---
management/commands/_private.py | 95 +++++++++++++++++++++--------------------
1 file changed, 49 insertions(+), 46 deletions(-)
(limited to 'management')
diff --git a/management/commands/_private.py b/management/commands/_private.py
index 0b26b4e..23083e1 100644
--- a/management/commands/_private.py
+++ b/management/commands/_private.py
@@ -80,6 +80,54 @@ def get_from_db_or_create(cls, **kwargs):
return obj
+def get_event(timetable, event, event_week):
+ """Renvoie une classe Course à partir d’un événement récupéré par BS4"""
+ # On récupère la date de l’évènement à partir de la semaine
+ # et de la semaine référencée, puis l’heure de début et de fin
+ date = event_week + datetime.timedelta(int(event.day.text))
+ begin = add_time(date, event.starttime.text)
+ end = add_time(date, event.endtime.text)
+
+ # Création de l’objet cours
+ course = Course.objects.create(timetable=timetable, begin=begin, end=end)
+
+ # On récupère les groupes concernés par les cours, on les
+ # « consolide », puis on les insère dans l’objet cours.
+ groups = [get_from_db_or_create(Group, timetable=timetable,
+ celcat_name=item.text)
+ for item in event.resources.group.find_all("item")]
+ consolidate_groups(groups)
+ course.groups.add(*groups)
+
+ # On récupère le champ « remarque »
+ if event.notes is not None:
+ course.notes = event.notes.text
+
+ # On récupère le nom du cours
+ if event.resources.module is not None:
+ course.name = event.resources.module.item.text
+ else:
+ # Il est possible qu’un cours n’ait pas de nom. Oui oui.
+ # Qui sont les concepteurs de ce système ? Quels sont leurs
+ # réseaux ?
+ # Bref, dans ce cas, on déplace le champ « remarque » de
+ # l’objet dans le champ « nom ».
+ course.name, course.notes = course.notes, None
+
+ # Récupération du type de cours
+ if event.category is not None:
+ course.type = event.category.text
+
+ # Si un cours a une salle attribuée (oui, il est possible qu’il n’y
+ # en ait pas… qui sont ils, leurs réseaux, tout ça…), on les insère
+ # dans la base de données, et on les ajoute dans l’objet cours
+ if event.resources.room is not None:
+ rooms = [get_from_db_or_create(Room, name=item.text)
+ for item in event.resources.room.find_all("item")]
+ course.rooms.add(*rooms)
+
+ return course
+
def get_events(timetable, soup, weeks_in_soup, year=None, week=None):
"""Récupère tous les cours disponibles dans l’emploi du temps Celcat.
Le traîtement se limitera à la semaine indiquée si il y en a une."""
@@ -94,52 +142,7 @@ def get_events(timetable, soup, weeks_in_soup, year=None, week=None):
year is None or week is None) and \
event.resources.group is not None and \
event.starttime is not None and event.endtime is not None:
- # On récupère la date de l’évènement à partir de la semaine
- # et de la semaine référencée, puis l’heure de début et de fin
- date = event_week + datetime.timedelta(int(event.day.text))
- begin = add_time(date, event.starttime.text)
- end = add_time(date, event.endtime.text)
-
- # Création de l’objet cours
- course = Course.objects.create(timetable=timetable, begin=begin,
- end=end)
-
- # On récupère les groupes concernés par les cours, on les
- # « consolide », puis on les insère dans l’objet cours.
- groups = [get_from_db_or_create(Group, timetable=timetable,
- celcat_name=item.text)
- for item in event.resources.group.find_all("item")]
- consolidate_groups(groups)
- course.groups.add(*groups)
-
- # On récupère le champ « remarque »
- if event.notes is not None:
- course.notes = event.notes.text
-
- # On récupère le nom du cours
- if event.resources.module is not None:
- course.name = event.resources.module.item.text
- else:
- # Il est possible qu’un cours n’ait pas de nom. Oui oui.
- # Qui sont les concepteurs de ce système ? Quels sont leurs
- # réseaux ?
- # Bref, dans ce cas, on déplace le champ « remarque » de
- # l’objet dans le champ « nom ».
- course.name, course.notes = course.notes, None
-
- # Récupération du type de cours
- if event.category is not None:
- course.type = event.category.text
-
- # Si un cours a une salle attribuée (oui, il est possible qu’il n’y
- # en ait pas… qui sont ils, leurs réseaux, tout ça…), on les insère
- # dans la base de données, et on les ajoute dans l’objet cours
- if event.resources.room is not None:
- rooms = [get_from_db_or_create(Room, name=item.text)
- for item in event.resources.room.find_all("item")]
- course.rooms.add(*rooms)
-
- yield course
+ yield get_event(timetable, event, event_week)
def get_update_date(soup):
# Explication de la regex
--
cgit v1.2.1
From 45402cbdab2dfdbd9e2f72c70d27f966f932789b Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Sat, 30 Sep 2017 12:20:48 +0200
Subject: Ajout de commentaires à la fonction get_weeks()
---
management/commands/_private.py | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
(limited to 'management')
diff --git a/management/commands/_private.py b/management/commands/_private.py
index 23083e1..8f195a1 100644
--- a/management/commands/_private.py
+++ b/management/commands/_private.py
@@ -169,8 +169,18 @@ def get_update_date(soup):
return timezone.make_aware(date)
def get_weeks(soup):
+ # Les semaines sont référencées de manière assez… exotique
+ # En gros, il y a une liste d’éléments span qui contiennent une sorte d’ID
+ # de la semaine, formaté de la manière suivante :
+ # NNNNNNNNNNNNNNNNNNNYNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN
+ # Tous sont de la même longueur, contiennent 51 N et un seul Y.
+ # Allez savoir pourquoi. Il se trouve dans la balise « alleventweeks ».
+ # Un paramètre du span (« date ») représente la date de début.
+ # Un cours contient donc un ID de semaine, puis le nombre de jours après le
+ # début de cette semaine.
weeks = {}
- for span in soup.find_all("span"):
+ for span in soup.find_all("span"): # Liste de toutes les semaines définies
+ # On parse la date et on la fait correspondre à l’ID
weeks[span.alleventweeks.text] = timezone.make_aware(
datetime.datetime.strptime(span["date"], "%d/%m/%Y"))
--
cgit v1.2.1
From 7a34fc91521373dddacd1595354da53187d46a5d Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Sat, 30 Sep 2017 16:30:21 +0200
Subject: Suppression de tous les cours d’un edt si jamais aucune semaine n’est
fournie. Pas de filtre de début ou de fin dans le cas où aucune semaine n’est
fournie. Changement du type d’aggrégation (minimum au lieu de maximum).
---
management/commands/timetables.py | 26 ++++++++++++++++----------
1 file changed, 16 insertions(+), 10 deletions(-)
(limited to 'management')
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index c5c6dfa..58c86fd 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -17,29 +17,35 @@ import datetime
from django.core.management.base import BaseCommand
from django.db import transaction
-from django.db.models import Max
+from django.db.models import Min
from django.utils import timezone
-from edt.models import Timetable, Course
+from edt.models import Course, Timetable
from edt.utils import get_week
from ._private import delete_courses_in_week, get_events, get_update_date, get_weeks, get_xml
@transaction.atomic
def process_timetable_week(timetable, soup, weeks_in_soup, force, year=None, week=None):
- begin, end = get_week(year, week)
-
- last_update_date = Course.objects.filter(timetable=timetable,
- begin__gte=begin,
- begin__lt=end) \
- .aggregate(Max("last_update")) \
- ["last_update__max"]
+ criteria = {}
+ if year is not None and week is not None:
+ begin, end = get_week(year, week)
+ criteria["begin__gte"] = begin
+ criteria["begin__lt"] = end
+
+ last_update_date = Course.objects.filter(timetable=timetable, **criteria) \
+ .aggregate(Min("last_update")) \
+ ["last_update__min"]
new_update_date = get_update_date(soup)
if not force and last_update_date is not None and new_update_date is not None and \
last_update_date >= new_update_date:
return
- delete_courses_in_week(timetable, year, week)
+ if year is not None and week is not None:
+ delete_courses_in_week(timetable, year, week)
+ else:
+ Course.objects.filter(timetable=timetable).delete()
+
for course in get_events(timetable, soup, weeks_in_soup, year, week):
course.save()
--
cgit v1.2.1
From be6f084c336e285652088fe2d86694b2645a0b93 Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Sat, 30 Sep 2017 16:34:58 +0200
Subject: On ne supprime pas les éléments plus anciens que ceux référencés par
Celcat
---
management/commands/timetables.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
(limited to 'management')
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index 58c86fd..e82fd55 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -44,7 +44,8 @@ def process_timetable_week(timetable, soup, weeks_in_soup, force, year=None, wee
if year is not None and week is not None:
delete_courses_in_week(timetable, year, week)
else:
- Course.objects.filter(timetable=timetable).delete()
+ Course.objects.filter(timetable=timetable,
+ begin__gte=min(weeks_in_soup.values())).delete()
for course in get_events(timetable, soup, weeks_in_soup, year, week):
course.save()
--
cgit v1.2.1
From f0c61f1a3a26f8c76f43b21f7860d83041b7fe89 Mon Sep 17 00:00:00 2001
From: Alban Gruin
Date: Sun, 1 Oct 2017 16:44:53 +0200
Subject: Ajout d’une colonne pour stocker la date de dernière mise à jour
---
management/commands/timetables.py | 3 +++
1 file changed, 3 insertions(+)
(limited to 'management')
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index e82fd55..76f0a7c 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -50,6 +50,9 @@ def process_timetable_week(timetable, soup, weeks_in_soup, force, year=None, wee
for course in get_events(timetable, soup, weeks_in_soup, year, week):
course.save()
+ timetable.last_update_date = new_update_date
+ timetable.save()
+
def process_timetable(timetable, force, year=None, weeks=None):
soup = get_xml(timetable.url)
weeks_in_soup = get_weeks(soup)
--
cgit v1.2.1