onjen/scraperoog
1
0
Fork 0
Code Issues Pull Requests Projects Releases 4 Wiki Activity

Make error message for nonexistant id configurable

Browse Source
This commit is contained in:
Johannes Rothe 2023-09-17 12:33:19 +02:00
parent d7b07d02b5
commit 4e99a7f87b
Signed by: onjen
GPG Key ID: 73F092605AF3286C
2 changed files with 29 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
config.ini
View File

@ -3,3 +3,4 @@ haupt_url = https://www.spiekeroog-vermieter.de/suche/monatskalenderSite.htm?woh
data_url = https://www.spiekeroog-vermieter.de/suche/monatskalender.htm?wohnids=
von_id = 200
bis_id = 300
id_fehler = nicht aktiv

6
scraperoog/scrape.py
View File

@ -24,6 +24,7 @@ MAIN_URL = config['Allgemein'].get('haupt_url')
DATA_URL = config['Allgemein'].get('data_url')
FROM = config['Allgemein'].getint('von_id')
TO = config['Allgemein'].getint('bis_id')
ID_NONEXISTANT = config['Allgemein'].get('id_fehler')
DATEFORMAT = "%Y-%m-%d"
STATUS_MAPPING = {"DayF": 0, "DayB": 1, "DayFB": 0.5, "DayBF": 0.5}
@ -82,7 +83,8 @@ def convert_to_datestring(day: str, month: str, year: str) -> datetime:
async def request_data(index: int, client: AsyncClient) -> Optional[Entry]:
response_data = await client.get(DATA_URL + str(index), timeout=20.0)
if "Dieser Belegungskalender ist derzeit nicht aktiv." not in response_data.text:
if ID_NONEXISTANT in response_data.text:
return Entry(index=0)
response_title = await client.get(MAIN_URL + str(index), timeout=20.0)
title_soup = BeautifulSoup(response_title.text, "html.parser")
apartment = (
@ -109,8 +111,6 @@ async def request_data(index: int, client: AsyncClient) -> Optional[Entry]:
wohneinheit=apartment.encode("utf-8"),
availabilities=availabilities,
)
else:
return Entry(index=0)
async def extract_results() -> None: