fblanke/tvstud-download-digital
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add command for special tag

Browse Source
This commit is contained in:
Felix Blanke 2023-08-27 00:00:02 +02:00
parent 8aff06b217
commit efea733447
1 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
download_digital.py
View File

@ -75,6 +75,7 @@ def construct_dataframe(
url: str = "https://beschaeftigtenbefragung.verdi.de/", url: str = "https://beschaeftigtenbefragung.verdi.de/",
tag: str = "bez_data_2", tag: str = "bez_data_2",
grouped: bool = False, grouped: bool = False,
special_tag: str | None = None,
): ):
r = requests.get(url) r = requests.get(url)
soup = BeautifulSoup(r.text, "html.parser") soup = BeautifulSoup(r.text, "html.parser")
@ -97,10 +98,16 @@ def construct_dataframe(
tot_col_data = [] tot_col_data = []
tot_col_index = [] tot_col_index = []
for k, v in bez_data.items(): if special_tag:
if "tot" in v: for k, v in bez_data.items():
tot_col_data.append(v["tot"]) if "sp" in v and special_tag in v["sp"]:
tot_col_index.append(k) tot_col_data.append(v["sp"][special_tag])
tot_col_index.append(k)
else:
for k, v in bez_data.items():
if "tot" in v:
tot_col_data.append(v["tot"])
tot_col_index.append(k)
data["Digitale Befragung"] = pd.Series(tot_col_data, index=tot_col_index) data["Digitale Befragung"] = pd.Series(tot_col_data, index=tot_col_index)
df = pd.DataFrame(data=data) df = pd.DataFrame(data=data)
@ -116,8 +123,9 @@ def main(
tag: str = "bez_data_2", tag: str = "bez_data_2",
dry_run: bool = False, dry_run: bool = False,
grouped: bool = False, grouped: bool = False,
special_tag: str | None = None,
) -> None: ) -> None:
df = construct_dataframe(url=url, tag=tag, grouped=grouped) df = construct_dataframe(url=url, tag=tag, grouped=grouped, special_tag=special_tag)
if dry_run: if dry_run:
print(df) print(df)