# Copyright (C) 2017 Alban Gruin # # celcatsanitizer is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published # by the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # celcatsanitizer is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with celcatsanitizer. If not, see . from django.core.management.base import BaseCommand from ...models import Year from ._private import get_from_db_or_create, get_xml import json import re class Command(BaseCommand): help = "List timetables not scrapped by scraptimetables" def add_arguments(self, parser): parser.add_argument("--url", type=str, required=True) def handle(self, *args, **options): gdict = {} for year, name, finder in self.__get_finders(options["url"]): soup = get_xml(finder) handled = False groups = [] for link in soup.find_all("a"): if link.text.startswith("Groupe: "): url_base = finder.rsplit("/", 1) group_url = "/".join([url_base[0], link.parent.parent.find("a", attrs={"class": "xmllink"})["href"]]) if link.text[9:].startswith(year.name): group_name = link.text[9:] else: group_name = year.name + " " + link.text[9:] groups.append((group_name, group_url)) if "toutes sections et semestres confondus" in link.text: handled = True break if not handled and len(groups) > 0: gdict["{} {}".format(year, name)] = groups print(json.dumps(gdict, indent=4)) def __get_finders(self, url): soup = get_xml(url) choose_regex = re.compile("^- Choisissez votre ([\w ]+) -$") for form in soup.find_all("form"): for i, option in enumerate(form.find_all("option")): if i == 0 and option.text == "- Choisissez le niveau -": break search = choose_regex.search(option.text) if search is not None: current_year = get_from_db_or_create(Year, name=search.groups(0)[0]) else: finder = option["value"].replace("finder", "finder2") if option.text.startswith(current_year.name): name = option.text[len(current_year.name):].strip() else: name = option.text yield current_year, name, finder