download_digital.py

@@ -71,62 +71,40 @@ bundesland_dict = {
 }
 
 
-def get_bez_data(
-    tags: list[str], url: str = "https://beschaeftigtenbefragung.verdi.de/"
-) -> list[dict]:
+def construct_dataframe(
+    url: str = "https://beschaeftigtenbefragung.verdi.de/",
+    tag: str = "bez_data_2",
+    grouped: bool = False,
+):
     r = requests.get(url)
     soup = BeautifulSoup(r.text, "html.parser")
-    bez_data = []
-    for tag in tags:
-        for a in soup.find_all("script"):
-            script_contents = a.decode_contents()
-            if script_contents.find(tag) >= 0:
-                break
 
-        substring = script_contents[script_contents.find(tag) + len(tag) + 3 :]
-        bez_data.append(json.loads(substring[: substring.find("\n") - 1]))
-    return bez_data
+    for a in soup.find_all("script"):
+        script_contents = a.decode_contents()
+        if script_contents.find(tag) >= 0:
+            break
 
+    substring = script_contents[script_contents.find(tag) + len(tag) + 3 :]
+    bez_data = json.loads(substring[: substring.find("\n") - 1])
 
-def construct_dataframe(
-    bez_data: dict[str, dict],
-    grouped: bool = False,
-    special_tag: str | None = None,
-    no_processing: bool = False,
-):
     data = {}
-    if not no_processing:
-        data["Bundesland"] = pd.Series(
-            [bundesland_dict[k] for k in bez_data], index=list(bez_data.keys())
-        )
-        data["Bezirk"] = pd.Series(
-            [v["name"] for v in bez_data.values()], index=list(bez_data.keys())
-        )
-    else:
-        data["Landesbezirk"] = pd.Series(
-            [v["name"] for v in bez_data.values()], index=list(bez_data.keys())
-        )
+    data["Bundesland"] = pd.Series(
+        [bundesland_dict[k] for k in bez_data], index=list(bez_data.keys())
+    )
+    data["Bezirk"] = pd.Series(
+        [v["name"] for v in bez_data.values()], index=list(bez_data.keys())
+    )
 
     tot_col_data = []
     tot_col_index = []
-    if special_tag:
-        for k, v in bez_data.items():
-            if "sp" in v and special_tag in v["sp"]:
-                tot_col_data.append(v["sp"][special_tag])
-                tot_col_index.append(k)
-    else:
-        for k, v in bez_data.items():
-            if "tot" in v:
-                tot_col_data.append(v["tot"])
-                tot_col_index.append(k)
+    for k, v in bez_data.items():
+        if "tot" in v:
+            tot_col_data.append(v["tot"])
+            tot_col_index.append(k)
 
     data["Digitale Befragung"] = pd.Series(tot_col_data, index=tot_col_index)
     df = pd.DataFrame(data=data)
-    df = df.astype({"Digitale Befragung": "Int32"})
-
-    if grouped and no_processing:
-        raise ValueError
-    elif grouped:
+    if grouped:
         df = df.groupby("Bundesland", as_index=False)[["Digitale Befragung"]].sum()
 
     return df
@@ -137,16 +115,8 @@ def main(
     tag: str = "bez_data_2",
     dry_run: bool = False,
     grouped: bool = False,
-    special_tag: str | None = None,
-    no_processing: bool = False,
 ) -> None:
-    bez_data = get_bez_data([tag], url)[0]
-    df = construct_dataframe(
-        bez_data=bez_data,
-        grouped=grouped,
-        special_tag=special_tag,
-        no_processing=no_processing,
-    )
+    df = construct_dataframe(url=url, tag=tag, grouped=grouped)
 
     if dry_run:
         print(df)

templates/base.html

@@ -5,7 +5,6 @@
 
 <title>Digitale Beschäftigtenbefragung</title>
 
-<p>Bei einzelnen ver.di-Bezirken liegen Daten nur zu den Bezirken vor, bei denen TVStud einen Schwerpunkt bildet.</p>
 <p><a href="https://zusammen-geht-mehr.verdi.de/beschaeftigtenbefragung">Karte der digitalen Beschäftigtenbefragung</a></p>
 
 {{ tables|safe }}

wsgi.py

@@ -1,7 +1,7 @@
 import pandas as pd
 from flask import Flask, render_template, request
 
-from download_digital import construct_dataframe, get_bez_data
+from download_digital import construct_dataframe
 
 app = Flask(__name__)
 
@@ -9,20 +9,12 @@ app = Flask(__name__)
 @app.route("/")
 def tables(
     url: str = "https://beschaeftigtenbefragung.verdi.de/",
+    default_tag: str = "bez_data_2",
 ):
-    bez_data = get_bez_data(["bez_data_0", "bez_data_2"], url)
-
-    df = construct_dataframe(
-        bez_data=bez_data[0],
-        grouped=False,
-        special_tag="stud",
-    ).sort_values(
-        ["Digitale Befragung", "Bundesland", "Bezirk"], ascending=[False, True, True]
-    )
-
-    df_state = construct_dataframe(
-        bez_data=bez_data[1], grouped=False, no_processing=True
-    ).sort_values("Landesbezirk")
+    tag = request.args.get("tag")
+    if tag is None:
+        tag = default_tag
+    df = construct_dataframe(url=url, tag=tag, grouped=False)
 
     output_str = []
 
@@ -55,7 +47,9 @@ def tables(
             output_str.append(tfoot)
             output_str.append(table[idx:])
 
-    _print_as_html(df_state)
+    _print_as_html(
+        df.groupby("Bundesland", as_index=False)[["Digitale Befragung"]].sum()
+    )
     _print_as_html(df)
 
     return render_template("base.html", tables="\n".join(output_str))