diff --git a/app/views.py b/app/views.py index 71a11be0259d3a1c5b3a5f4b821330b84dd6fcb4..af6fb0af71c70d514133c145f1bd252ba881c5db 100644 --- a/app/views.py +++ b/app/views.py @@ -18,7 +18,18 @@ router_main = Blueprint('router_sunjad', __name__) @require_jwt_token def get_courses(major_id): active_period = get_app_config("ACTIVE_PERIOD") - period = Period.objects(major_id=major_id, name=active_period).first() + period = Period.objects( + major_id=major_id, + name=active_period, + is_detail=True + ).first() + + if period is None: + period = Period.objects( + major_id=major_id, + name=active_period, + is_detail=False + ).first() return (jsonify(period.serialize()), 200) diff --git a/scraper/main.py b/scraper/main.py index f4c1c3a9d1daf8edee1b98f3b4ada2d4b51706c1..c56d62efc0fbeb079be380957f27a679b5ef85a2 100644 --- a/scraper/main.py +++ b/scraper/main.py @@ -25,7 +25,7 @@ GENERAL_SCHEDULE_URL = f"{BASE_URL}/Schedule/IndexOthers?fac={{fac}}&org={{org}} DEFAULT_CREDENTIAL = "01.00.12.01" -def scrape_courses(major_kd_org, period): +def scrape_courses(major_kd_org, period, skip_not_detail=False): username, password = fetch_credential(major_kd_org) if (username is not None) and (password is not None): req = requests.Session() @@ -36,16 +36,19 @@ def scrape_courses(major_kd_org, period): courses = create_courses(r.text, is_detail=True) return courses, True - username, password = fetch_credential(DEFAULT_CREDENTIAL) - fac, org = parse_kd_org(major_kd_org) - req = requests.Session() - r = req.post(AUTH_URL, data={'u': username, - 'p': password}, verify=False) - r = req.get(CHANGEROLE_URL) - r = req.get(GENERAL_SCHEDULE_URL.format( - fac=fac, org=org, period=period)) - courses = create_courses(r.text) - return courses, False + if not skip_not_detail: + username, password = fetch_credential(DEFAULT_CREDENTIAL) + fac, org = parse_kd_org(major_kd_org) + req = requests.Session() + r = req.post(AUTH_URL, data={'u': username, + 'p': password}, verify=False) + r = req.get(CHANGEROLE_URL) + r = req.get(GENERAL_SCHEDULE_URL.format( + fac=fac, org=org, period=period)) + courses = create_courses(r.text) + return courses, False + + return None, None def fetch_credential(major_kd_org): @@ -118,7 +121,7 @@ def create_courses(html, is_detail=False): lecturer=lecturers )) - except IndexError as e: + except (IndexError, ValueError) as e: pass if classes: diff --git a/sso/utils.py b/sso/utils.py index 31c76017d810796cd3c0a6584304797f8c88c4fd..5297f07f839eb36e62167d5218db48789095dd65 100644 --- a/sso/utils.py +++ b/sso/utils.py @@ -57,24 +57,33 @@ def process_sso_profile(sso_profile): major = Major(name=major_name, kd_org=major_kd_org) major.save() - period = Period.objects(major_id=major.id, name=period_name).first() - if period is None: - courses, is_detail = scrape_courses(major_kd_org, period_name) - - if not courses: - result = { - "err": True, - "major_name": major_name - } - return result - - period = Period( - major_id=major.id, - name=period_name, - courses=courses, - is_detail=is_detail - ) - period.save() + period_detail = Period.objects( + major_id=major.id, name=period_name, is_detail=True).first() + period_not_detail = Period.objects( + major_id=major.id, name=period_name, is_detail=False).first() + + if period_detail is None: + if period_not_detail is None: + courses, is_detail = scrape_courses(major_kd_org, period_name) + + if not courses: + result = { + "err": True, + "major_name": major_name + } + return result + else: + courses, is_detail = scrape_courses( + major_kd_org, period_name, skip_not_detail=True) + + if courses: + period = Period( + major_id=major.id, + name=period_name, + courses=courses, + is_detail=is_detail + ) + period.save() user = User.objects(npm=user_npm).first() if user is None: