onjen/scraperoog
1
0
Fork 0
Code Issues Pull Requests Projects Releases 4 Wiki Activity

Adapt to new error message thrown by the site

Browse Source
This commit is contained in:
Johannes Rothe 2022-08-01 22:17:31 +02:00
parent e6be0ab494
commit 8c6b672d90
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scraperoog/scrape.py
View File

@ -81,7 +81,7 @@ def convert_to_datestring(day: str, month: str, year: str) -> datetime:
async def request_data(index: int, client: AsyncClient) -> Optional[Entry]: async def request_data(index: int, client: AsyncClient) -> Optional[Entry]:
response_data = await client.get(DATA_URL + str(index), timeout=20.0) response_data = await client.get(DATA_URL + str(index), timeout=20.0)
if "Fehler aufgetreten" not in response_data.text: if "Die Darstellung ist derzeit deaktiviert" not in response_data.text:
response_title = await client.get(MAIN_URL + str(index), timeout=20.0) response_title = await client.get(MAIN_URL + str(index), timeout=20.0)
title_soup = BeautifulSoup(response_title.text, "lxml") title_soup = BeautifulSoup(response_title.text, "lxml")
apartment = ( apartment = (
@ -127,10 +127,12 @@ async def extract_results() -> None:
file.write(result.json()) file.write(result.json())
generate_csv(result) generate_csv(result)
def main() -> None:
if platform.system() == "Windows":
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
asyncio.run(extract_results())
if __name__ == "__main__": if __name__ == "__main__":
# with open("results.json", "r") as file: # with open("results.json", "r") as file:
# result = Result(**json.load(file)) # result = Result(**json.load(file))
if platform.system() == "Windows": main()
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
asyncio.run(extract_results())