fblanke/tvstud-download-digital
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add no-processing mode

Browse Source
This commit is contained in:
Felix Blanke 2023-08-27 00:09:20 +02:00
parent efea733447
commit ca64496aba
1 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
download_digital.py
View File

@ -76,6 +76,7 @@ def construct_dataframe(
tag: str = "bez_data_2", tag: str = "bez_data_2",
grouped: bool = False, grouped: bool = False,
special_tag: str | None = None, special_tag: str | None = None,
no_processing: bool = False,
): ):
r = requests.get(url) r = requests.get(url)
soup = BeautifulSoup(r.text, "html.parser") soup = BeautifulSoup(r.text, "html.parser")
@ -89,9 +90,10 @@ def construct_dataframe(
bez_data = json.loads(substring[: substring.find("\n") - 1]) bez_data = json.loads(substring[: substring.find("\n") - 1])
data = {} data = {}
data["Bundesland"] = pd.Series( if not no_processing:
[bundesland_dict[k] for k in bez_data], index=list(bez_data.keys()) data["Bundesland"] = pd.Series(
) [bundesland_dict[k] for k in bez_data], index=list(bez_data.keys())
)
data["Bezirk"] = pd.Series( data["Bezirk"] = pd.Series(
[v["name"] for v in bez_data.values()], index=list(bez_data.keys()) [v["name"] for v in bez_data.values()], index=list(bez_data.keys())
) )
@ -112,7 +114,10 @@ def construct_dataframe(
data["Digitale Befragung"] = pd.Series(tot_col_data, index=tot_col_index) data["Digitale Befragung"] = pd.Series(tot_col_data, index=tot_col_index)
df = pd.DataFrame(data=data) df = pd.DataFrame(data=data)
df = df.astype({"Digitale Befragung": "Int32"}) df = df.astype({"Digitale Befragung": "Int32"})
if grouped:
if grouped and no_processing:
raise ValueError
elif grouped:
df = df.groupby("Bundesland", as_index=False)[["Digitale Befragung"]].sum() df = df.groupby("Bundesland", as_index=False)[["Digitale Befragung"]].sum()
return df return df
@ -124,8 +129,15 @@ def main(
dry_run: bool = False, dry_run: bool = False,
grouped: bool = False, grouped: bool = False,
special_tag: str | None = None, special_tag: str | None = None,
no_processing: bool = False,
) -> None: ) -> None:
df = construct_dataframe(url=url, tag=tag, grouped=grouped, special_tag=special_tag) df = construct_dataframe(
url=url,
tag=tag,
grouped=grouped,
special_tag=special_tag,
no_processing=no_processing,
)
if dry_run: if dry_run:
print(df) print(df)