Compare commits

..

No commits in common. "master" and "0.0.3" have entirely different histories.

3 changed files with 29 additions and 30 deletions

View File

@ -3,4 +3,3 @@ haupt_url = https://www.spiekeroog-vermieter.de/suche/monatskalenderSite.htm?woh
data_url = https://www.spiekeroog-vermieter.de/suche/monatskalender.htm?wohnids= data_url = https://www.spiekeroog-vermieter.de/suche/monatskalender.htm?wohnids=
von_id = 200 von_id = 200
bis_id = 300 bis_id = 300
id_fehler = nicht aktiv

View File

@ -1,6 +1,6 @@
[project] [project]
name = "scraperoog" name = "scraperoog"
version = "0.0.5" version = "0.0.3"
dependencies = [ dependencies = [
"tqdm", "tqdm",
"bs4", "bs4",

View File

@ -24,7 +24,6 @@ MAIN_URL = config['Allgemein'].get('haupt_url')
DATA_URL = config['Allgemein'].get('data_url') DATA_URL = config['Allgemein'].get('data_url')
FROM = config['Allgemein'].getint('von_id') FROM = config['Allgemein'].getint('von_id')
TO = config['Allgemein'].getint('bis_id') TO = config['Allgemein'].getint('bis_id')
ID_NONEXISTANT = config['Allgemein'].get('id_fehler')
DATEFORMAT = "%Y-%m-%d" DATEFORMAT = "%Y-%m-%d"
STATUS_MAPPING = {"DayF": 0, "DayB": 1, "DayFB": 0.5, "DayBF": 0.5} STATUS_MAPPING = {"DayF": 0, "DayB": 1, "DayFB": 0.5, "DayBF": 0.5}
@ -83,8 +82,7 @@ def convert_to_datestring(day: str, month: str, year: str) -> datetime:
async def request_data(index: int, client: AsyncClient) -> Optional[Entry]: async def request_data(index: int, client: AsyncClient) -> Optional[Entry]:
response_data = await client.get(DATA_URL + str(index), timeout=20.0) response_data = await client.get(DATA_URL + str(index), timeout=20.0)
if ID_NONEXISTANT in response_data.text: if "Dieser Belegungskalender ist derzeit nicht aktiv." not in response_data.text:
return Entry(index=0)
response_title = await client.get(MAIN_URL + str(index), timeout=20.0) response_title = await client.get(MAIN_URL + str(index), timeout=20.0)
title_soup = BeautifulSoup(response_title.text, "html.parser") title_soup = BeautifulSoup(response_title.text, "html.parser")
apartment = ( apartment = (
@ -111,6 +109,8 @@ async def request_data(index: int, client: AsyncClient) -> Optional[Entry]:
wohneinheit=apartment.encode("utf-8"), wohneinheit=apartment.encode("utf-8"),
availabilities=availabilities, availabilities=availabilities,
) )
else:
return Entry(index=0)
async def extract_results() -> None: async def extract_results() -> None: