aboutsummaryrefslogtreecommitdiff
path: root/management
diff options
context:
space:
mode:
Diffstat (limited to 'management')
-rw-r--r--management/commands/missingtimetables.py76
-rw-r--r--management/commands/scraprooms.py36
-rw-r--r--management/commands/scraptimetables.py64
3 files changed, 176 insertions, 0 deletions
diff --git a/management/commands/missingtimetables.py b/management/commands/missingtimetables.py
new file mode 100644
index 0000000..1b38896
--- /dev/null
+++ b/management/commands/missingtimetables.py
@@ -0,0 +1,76 @@
+# Copyright (C) 2017 Alban Gruin
+#
+# celcatsanitizer is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# celcatsanitizer is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with celcatsanitizer. If not, see <http://www.gnu.org/licenses/>.
+
+from django.core.management.base import BaseCommand
+from ...models import Year
+from ._private import get_from_db_or_create, get_xml
+
+import json
+import re
+
+
+class Command(BaseCommand):
+ help = "List timetables not scrapped by scraptimetables"
+
+ def add_arguments(self, parser):
+ parser.add_argument("--url", type=str, required=True)
+
+ def handle(self, *args, **options):
+ gdict = {}
+ for year, name, finder in self.__get_finders(options["url"]):
+ soup = get_xml(finder)
+ handled = False
+ groups = []
+
+ for link in soup.find_all("a"):
+ if link.text.startswith("Groupe: "):
+ url_base = finder.rsplit("/", 1)
+ group_url = "/".join([url_base[0], link.parent.parent.find("a", attrs={"class": "xmllink"})["href"]])
+ if link.text[9:].startswith(year.name):
+ group_name = link.text[9:]
+ else:
+ group_name = year.name + " " + link.text[9:]
+
+ groups.append((group_name, group_url))
+
+ if "toutes sections et semestres confondus" in link.text:
+ handled = True
+ break
+
+ if not handled and len(groups) > 0:
+ gdict["{} {}".format(year, name)] = groups
+
+ print(json.dumps(gdict, indent=4))
+
+ def __get_finders(self, url):
+ soup = get_xml(url)
+ choose_regex = re.compile("^- Choisissez votre ([\w ]+) -$")
+
+ for form in soup.find_all("form"):
+ for i, option in enumerate(form.find_all("option")):
+ if i == 0 and option.text == "- Choisissez le niveau -":
+ break
+
+ search = choose_regex.search(option.text)
+ if search is not None:
+ current_year = get_from_db_or_create(Year, name=search.groups(0)[0])
+ else:
+ finder = option["value"].replace("finder", "finder2")
+ if option.text.startswith(current_year.name):
+ name = option.text[len(current_year.name):].strip()
+ else:
+ name = option.text
+
+ yield current_year, name, finder
diff --git a/management/commands/scraprooms.py b/management/commands/scraprooms.py
new file mode 100644
index 0000000..c448d61
--- /dev/null
+++ b/management/commands/scraprooms.py
@@ -0,0 +1,36 @@
+# Copyright (C) 2018 Alban Gruin
+#
+# celcatsanitizer is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# celcatsanitizer is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with celcatsanitizer. If not, see <http://www.gnu.org/licenses/>.
+
+from bs4 import BeautifulSoup
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+from edt.models import Room
+
+
+class Command(BaseCommand):
+ help = "Scraps rooms from file"
+
+ def add_arguments(self, parser):
+ parser.add_argument("--file", type=str, required=True)
+
+ @transaction.atomic
+ def handle(self, *args, **options):
+ with open(options["file"], "r") as rooms:
+ soup = BeautifulSoup(rooms.read(), "html.parser")
+
+ rooms = [option.text for option in soup.find_all("option")[1:]]
+ for room in set(rooms):
+ Room(name=room).save()
diff --git a/management/commands/scraptimetables.py b/management/commands/scraptimetables.py
new file mode 100644
index 0000000..98856a5
--- /dev/null
+++ b/management/commands/scraptimetables.py
@@ -0,0 +1,64 @@
+# Copyright (C) 2017-2018 Alban Gruin
+#
+# celcatsanitizer is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# celcatsanitizer is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with celcatsanitizer. If not, see <http://www.gnu.org/licenses/>.
+
+from bs4 import BeautifulSoup
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from edt.models import Source, Timetable, Year
+
+import re
+import requests
+
+
+class Command(BaseCommand):
+ help = "Fetch timetables from a specified URL"
+
+ def add_arguments(self, parser):
+ parser.add_argument("--url", type=str, required=True)
+
+ @transaction.atomic
+ def handle(self, *args, **options):
+ for year, name, source in self.__get_timetables(options["url"]):
+ source, _ = Source.objects.get_or_create(url=source)
+ timetable = Timetable(year=year, name=name, source=source)
+ timetable.save()
+
+ def __get_timetables(self, url):
+ req = requests.get(url)
+ soup = BeautifulSoup(req.content, "html.parser")
+ choose_regex = re.compile("^- Choisissez votre ([\w ]+) -$")
+ base_url = ("https://edt.univ-tlse3.fr/calendar/default.aspx"
+ "?View=month&Type=group&ResourceName=formation_")
+
+ for form in soup.find_all("form"):
+ for i, option in enumerate(form.find_all("option")):
+ if i == 0 and option.text == "- Choisissez le niveau -":
+ break
+
+ search = choose_regex.search(option.text)
+ if search is not None:
+ current_year, _ = Year.objects.get_or_create(
+ name=search.groups(0)[0])
+ else:
+ g = option["value"].split("-")[-1][:-5]
+ if g.endswith("s1"):
+ g = g[:-2]
+
+ url = base_url + g.upper()
+ if option.text.startswith(current_year.name):
+ name = option.text[len(current_year.name):].strip()
+ else:
+ name = option.text
+ yield current_year, name, url
generated by cgit v1.2.1 (git 2.18.0) at 2024-12-27 11:27:18 +0000