Release 0.0.5

README.md

@@ -3,6 +3,10 @@
 `apt install python3-pydantic python3-httpx python3-bs4 python3-tqdm locales-all && sudo locale-gen`
 
 # Windows build
-1. install wine
+1. Install wine
-2. create venv and `pip install .`
+2. `winecfg` and configure windows 10
-3. `wine pyinstaller --paths=venv/lib/python3.10/site-packages/ --collect-submodules=lxml --onefile scraperoog/scrape.py`
+3. Install [python](https://www.python.org/downloads/windows/)
+4. Install pyinstaller in wine `wine pip install pyinstaller`
+5. Create venv and `pip install .`
+6. `wine pyinstaller --paths=venv/lib/python3.10/site-packages/ --onefile scraperoog/scrape.py`
+7. .exe is found under dist/

config.ini

@@ -3,3 +3,4 @@ haupt_url = https://www.spiekeroog-vermieter.de/suche/monatskalenderSite.htm?woh
 data_url = https://www.spiekeroog-vermieter.de/suche/monatskalender.htm?wohnids=
 von_id = 200
 bis_id = 300
+id_fehler = nicht aktiv

pyproject.toml

@@ -1,10 +1,9 @@
 [project]
 name = "scraperoog"
-version = "0.0.2"
+version = "0.0.5"
 dependencies = [
     "tqdm",
     "bs4",
-    "lxml",
     "httpx",
     "pydantic",
 ]

scraperoog/scrape.py

@@ -1,10 +1,11 @@
 import asyncio
 import csv
-import json
 import locale
-locale.setlocale(locale.LC_TIME, "German") # dates on that page are German
-import pickle
 import platform
+if platform.system() == "Windows":
+    locale.setlocale(locale.LC_TIME, "German")
+else:
+    locale.setlocale(locale.LC_TIME, "de_DE.utf_8")
 from configparser import ConfigParser
 from datetime import datetime
 from typing import List, Optional, Tuple
@@ -23,6 +24,7 @@ MAIN_URL = config['Allgemein'].get('haupt_url')
 DATA_URL = config['Allgemein'].get('data_url')
 FROM = config['Allgemein'].getint('von_id')
 TO = config['Allgemein'].getint('bis_id')
+ID_NONEXISTANT = config['Allgemein'].get('id_fehler')
 DATEFORMAT = "%Y-%m-%d"
 STATUS_MAPPING = {"DayF": 0, "DayB": 1, "DayFB": 0.5, "DayBF": 0.5}
 
@@ -81,16 +83,17 @@ def convert_to_datestring(day: str, month: str, year: str) -> datetime:
 
 async def request_data(index: int, client: AsyncClient) -> Optional[Entry]:
     response_data = await client.get(DATA_URL + str(index), timeout=20.0)
-    if "Die Darstellung ist derzeit deaktiviert" not in response_data.text:
+    if ID_NONEXISTANT in response_data.text:
+        return Entry(index=0)
     response_title = await client.get(MAIN_URL + str(index), timeout=20.0)
-        title_soup = BeautifulSoup(response_title.text, "lxml")
+    title_soup = BeautifulSoup(response_title.text, "html.parser")
     apartment = (
         title_soup.body.header.h2.get_text()
         .replace("\xa0", " ")
         .replace("Wohneinheit: ", "")
     )
 
-        data_soup = BeautifulSoup(response_data.text, "lxml")
+    data_soup = BeautifulSoup(response_data.text, "html.parser")
     valid_element = data_soup.find_all("td", attrs={"data-daynum": True})
     availabilities = []
     for elm in valid_element:
@@ -108,8 +111,6 @@ async def request_data(index: int, client: AsyncClient) -> Optional[Entry]:
         wohneinheit=apartment.encode("utf-8"),
         availabilities=availabilities,
     )
-    else:
-        return Entry(index=0)
 
 
 async def extract_results() -> None: