aboutsummaryrefslogtreecommitdiff
path: root/management/commands
diff options
context:
space:
mode:
authorAlban Gruin2017-10-03 09:28:49 +0200
committerAlban Gruin2017-10-03 09:28:49 +0200
commitae3358c1296a02352409910a9ffcc2307d5ea87a (patch)
treeb47df3b99f639acc24d751cc0a6a73937c10069b /management/commands
parent9641fc9bb6ef2897c0d70e6d5d5ed3f8e295dbec (diff)
parent175fcd46f56d8c1c5d10a50b401c09c25cfcdf82 (diff)
Merge branch 'stable/0.10.z' into prod/pa1ch/0.y.zv0.10.0-pa1ch
Diffstat (limited to 'management/commands')
-rw-r--r--management/commands/_private.py150
-rw-r--r--management/commands/cleancourses.py4
-rw-r--r--management/commands/timetables.py80
3 files changed, 128 insertions, 106 deletions
diff --git a/management/commands/_private.py b/management/commands/_private.py
index c140f51..8f195a1 100644
--- a/management/commands/_private.py
+++ b/management/commands/_private.py
@@ -13,33 +13,20 @@
# You should have received a copy of the GNU Affero General Public License
# along with celcatsanitizer. If not, see <http://www.gnu.org/licenses/>.
+import datetime
+import re
+
from bs4 import BeautifulSoup
from django.utils import timezone
from edt.models import Group, Room, Course
from edt.utils import get_week
-import datetime
-import re
-
import requests
-
-class Week:
- def __init__(self, number, start):
- self.number = number
- self.start = timezone.make_aware(
- datetime.datetime.strptime(start, "%d/%m/%Y"))
-
- def get_day(self, id):
- return self.start + datetime.timedelta(id)
-
- @property
- def year(self):
- return self.start.year
-
def add_time(date, time):
- delta = datetime.timedelta(hours=time.hour, minutes=time.minute)
+ ptime = datetime.datetime.strptime(time, "%H:%M")
+ delta = datetime.timedelta(hours=ptime.hour, minutes=ptime.minute)
return date + delta
def consolidate_group(group):
@@ -75,7 +62,7 @@ def consolidate_group(group):
def consolidate_groups(groups):
for group in groups:
- if group.parent == None:
+ if group.parent is None:
consolidate_group(group)
def delete_courses_in_week(timetable, year, week):
@@ -93,50 +80,69 @@ def get_from_db_or_create(cls, **kwargs):
return obj
-def get_events(timetable, year, week, soup, weeks_in_soup):
+def get_event(timetable, event, event_week):
+ """Renvoie une classe Course à partir d’un événement récupéré par BS4"""
+ # On récupère la date de l’évènement à partir de la semaine
+ # et de la semaine référencée, puis l’heure de début et de fin
+ date = event_week + datetime.timedelta(int(event.day.text))
+ begin = add_time(date, event.starttime.text)
+ end = add_time(date, event.endtime.text)
+
+ # Création de l’objet cours
+ course = Course.objects.create(timetable=timetable, begin=begin, end=end)
+
+ # On récupère les groupes concernés par les cours, on les
+ # « consolide », puis on les insère dans l’objet cours.
+ groups = [get_from_db_or_create(Group, timetable=timetable,
+ celcat_name=item.text)
+ for item in event.resources.group.find_all("item")]
+ consolidate_groups(groups)
+ course.groups.add(*groups)
+
+ # On récupère le champ « remarque »
+ if event.notes is not None:
+ course.notes = event.notes.text
+
+ # On récupère le nom du cours
+ if event.resources.module is not None:
+ course.name = event.resources.module.item.text
+ else:
+ # Il est possible qu’un cours n’ait pas de nom. Oui oui.
+ # Qui sont les concepteurs de ce système ? Quels sont leurs
+ # réseaux ?
+ # Bref, dans ce cas, on déplace le champ « remarque » de
+ # l’objet dans le champ « nom ».
+ course.name, course.notes = course.notes, None
+
+ # Récupération du type de cours
+ if event.category is not None:
+ course.type = event.category.text
+
+ # Si un cours a une salle attribuée (oui, il est possible qu’il n’y
+ # en ait pas… qui sont ils, leurs réseaux, tout ça…), on les insère
+ # dans la base de données, et on les ajoute dans l’objet cours
+ if event.resources.room is not None:
+ rooms = [get_from_db_or_create(Room, name=item.text)
+ for item in event.resources.room.find_all("item")]
+ course.rooms.add(*rooms)
+
+ return course
+
+def get_events(timetable, soup, weeks_in_soup, year=None, week=None):
+ """Récupère tous les cours disponibles dans l’emploi du temps Celcat.
+ Le traîtement se limitera à la semaine indiquée si il y en a une."""
for event in soup.find_all("event"):
- title = None
- type_ = None
- groups = None
- rooms = None
- notes = None
-
- if weeks_in_soup[event.rawweeks.text].number == week and \
- weeks_in_soup[event.rawweeks.text].year == year and \
+ event_week = weeks_in_soup[event.rawweeks.text]
+ event_week_num = event_week.isocalendar()[1] # Numéro de semaine
+
+ # On passe le traitement si la semaine de l’événement ne correspond pas
+ # à la semaine passée, ou qu’il ne contient pas de groupe ou n’a pas de
+ # date de début ou de fin.
+ if (event_week_num == week and event_week.year == year or \
+ year is None or week is None) and \
event.resources.group is not None and \
event.starttime is not None and event.endtime is not None:
- date = weeks_in_soup[event.rawweeks.text].get_day(int(
- event.day.text))
-
- begin = add_time(date, datetime.datetime.strptime(
- event.starttime.text, "%H:%M"))
- end = add_time(date, datetime.datetime.strptime(
- event.endtime.text, "%H:%M"))
-
- groups = [get_from_db_or_create(Group, timetable=timetable,
- celcat_name=item.text)
- for item in event.resources.group.find_all("item")]
- consolidate_groups(groups)
-
- if event.notes is not None:
- notes = event.notes.text
-
- if event.resources.module is not None:
- title = event.resources.module.item.text
- elif notes is not None:
- title = notes
- notes = None
- else:
- title = "Aucune information"
-
- if event.category is not None:
- type_ = event.category.text
-
- if event.resources.room is not None:
- rooms = [get_from_db_or_create(Room, name=item.text)
- for item in event.resources.room.find_all("item")]
-
- yield title, type_, groups, rooms, notes, begin, end
+ yield get_event(timetable, event, event_week)
def get_update_date(soup):
# Explication de la regex
@@ -153,7 +159,7 @@ def get_update_date(soup):
# (\d+) au moins un nombre
# : un deux-points
# (\d+) au moins un nombre
- datetime_regex = re.compile("(\d+)/(\d+)/(\d+)\s+(\d+):(\d+):(\d+)")
+ datetime_regex = re.compile(r"(\d+)/(\d+)/(\d+)\s+(\d+):(\d+):(\d+)")
search = datetime_regex.search(soup.footer.text)
if search is None:
return None
@@ -163,16 +169,26 @@ def get_update_date(soup):
return timezone.make_aware(date)
def get_weeks(soup):
+ # Les semaines sont référencées de manière assez… exotique
+ # En gros, il y a une liste d’éléments span qui contiennent une sorte d’ID
+ # de la semaine, formaté de la manière suivante :
+ # NNNNNNNNNNNNNNNNNNNYNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN
+ # Tous sont de la même longueur, contiennent 51 N et un seul Y.
+ # Allez savoir pourquoi. Il se trouve dans la balise « alleventweeks ».
+ # Un paramètre du span (« date ») représente la date de début.
+ # Un cours contient donc un ID de semaine, puis le nombre de jours après le
+ # début de cette semaine.
weeks = {}
- for span in soup.find_all("span"):
- weeks[span.alleventweeks.text] = Week(int(span.title.text),
- span["date"])
+ for span in soup.find_all("span"): # Liste de toutes les semaines définies
+ # On parse la date et on la fait correspondre à l’ID
+ weeks[span.alleventweeks.text] = timezone.make_aware(
+ datetime.datetime.strptime(span["date"], "%d/%m/%Y"))
return weeks
def get_xml(url):
- r = requests.get(url)
- r.encoding = "utf8"
+ req = requests.get(url)
+ req.encoding = "utf8"
- soup = BeautifulSoup(r.content, "html.parser")
+ soup = BeautifulSoup(req.content, "html.parser")
return soup
diff --git a/management/commands/cleancourses.py b/management/commands/cleancourses.py
index ca2ef94..f6041ef 100644
--- a/management/commands/cleancourses.py
+++ b/management/commands/cleancourses.py
@@ -15,7 +15,7 @@
from django.core.management.base import BaseCommand
from django.db import transaction
-from edt.models import Course, Group, LastUpdate
+from edt.models import Course, Group
class Command(BaseCommand):
@@ -29,10 +29,8 @@ class Command(BaseCommand):
if options["timetable"] is None:
Course.objects.all().delete()
Group.objects.all().delete()
- LastUpdate.objects.all().delete()
else:
Course.objects.filter(timetable__id__in=options["timetable"]).delete()
Group.objects.filter(timetable__id__in=options["timetable"]).delete()
- LastUpdate.objects.filter(timetable__id__in=options["timetable"]).delete()
self.stdout.write(self.style.SUCCESS("Done."))
diff --git a/management/commands/timetables.py b/management/commands/timetables.py
index c82b0e4..76f0a7c 100644
--- a/management/commands/timetables.py
+++ b/management/commands/timetables.py
@@ -13,59 +13,63 @@
# You should have received a copy of the GNU Affero General Public License
# along with celcatsanitizer. If not, see <http://www.gnu.org/licenses/>.
+import datetime
+
from django.core.management.base import BaseCommand
from django.db import transaction
+from django.db.models import Min
from django.utils import timezone
-from edt.models import Timetable, LastUpdate, Course
+from edt.models import Course, Timetable
+from edt.utils import get_week
from ._private import delete_courses_in_week, get_events, get_update_date, get_weeks, get_xml
-import datetime
-
@transaction.atomic
-def process_timetable_week(timetable, year, week, soup, weeks_in_soup):
- last_update_date = None
+def process_timetable_week(timetable, soup, weeks_in_soup, force, year=None, week=None):
+ criteria = {}
+ if year is not None and week is not None:
+ begin, end = get_week(year, week)
+ criteria["begin__gte"] = begin
+ criteria["begin__lt"] = end
+
+ last_update_date = Course.objects.filter(timetable=timetable, **criteria) \
+ .aggregate(Min("last_update")) \
+ ["last_update__min"]
new_update_date = get_update_date(soup)
- try:
- last_update = LastUpdate.objects.get(timetable=timetable, year=year, week=week)
- last_update_date = last_update.updated_at
- except:
- last_update = LastUpdate(timetable=timetable, year=year, week=week)
- if last_update_date is not None and new_update_date is not None and \
+ if not force and last_update_date is not None and new_update_date is not None and \
last_update_date >= new_update_date:
return
- delete_courses_in_week(timetable, year, week)
- for name, type_, groups, rooms, notes, begin, end in \
- get_events(timetable, year, week, soup, weeks_in_soup):
- course = Course.objects.create(timetable=timetable, begin=begin, end=end)
- course.name = name
- course.type = type_
- course.notes = notes
-
- course.groups.add(*groups)
- if rooms is not None:
- course.rooms.add(*rooms)
+ if year is not None and week is not None:
+ delete_courses_in_week(timetable, year, week)
+ else:
+ Course.objects.filter(timetable=timetable,
+ begin__gte=min(weeks_in_soup.values())).delete()
+ for course in get_events(timetable, soup, weeks_in_soup, year, week):
course.save()
- last_update.date = timezone.make_aware(datetime.datetime.now())
- last_update.updated_at = new_update_date
- last_update.save()
+ timetable.last_update_date = new_update_date
+ timetable.save()
-def process_timetable(timetable, year, weeks):
+def process_timetable(timetable, force, year=None, weeks=None):
soup = get_xml(timetable.url)
weeks_in_soup = get_weeks(soup)
- for week in weeks:
- process_timetable_week(timetable, year, week, soup, weeks_in_soup)
+ if year is not None and weeks is not None:
+ for week in weeks:
+ process_timetable_week(timetable, soup, weeks_in_soup, force, year, week)
+ else:
+ process_timetable_week(timetable, soup, weeks_in_soup, force)
class Command(BaseCommand):
help = "Fetches registered celcat timetables"
def add_arguments(self, parser):
+ parser.add_argument("--all", const=True, default=False, action="store_const")
+ parser.add_argument("--force", const=True, default=False, action="store_const")
parser.add_argument("--week", type=int, choices=range(1, 54), nargs="+")
parser.add_argument("--year", type=int, nargs=1)
@@ -73,7 +77,9 @@ class Command(BaseCommand):
year = None
errcount = 0
- if options["week"] is None:
+ if options["all"]:
+ weeks = None
+ elif options["week"] is None:
_, week, day = timezone.now().isocalendar()
if day >= 6:
year, week, _ = (timezone.now() + datetime.timedelta(weeks=1)).isocalendar()
@@ -81,18 +87,20 @@ class Command(BaseCommand):
else:
weeks = options["week"]
- if options["year"] is None and year is None:
- year = timezone.now().year
- elif year is None:
- year = options["year"][0]
+ if not options["all"]:
+ if options["year"] is None and year is None:
+ year = timezone.now().year
+ elif year is None:
+ year = options["year"][0]
for timetable in Timetable.objects.all():
self.stdout.write("Processing {0}".format(timetable))
try:
- process_timetable(timetable, year, weeks)
- except Exception as e:
- self.stderr.write(self.style.ERROR("Failed to process {0}: {1}".format(timetable, e)))
+ process_timetable(timetable, options["force"], year, weeks)
+ except Exception as exc:
+ self.stderr.write(
+ self.style.ERROR("Failed to process {0}: {1}".format(timetable, exc)))
errcount += 1
if errcount == 0: