Fix fallback

.gitignore

@@ -1,2 +1,4 @@
-data.html
+plots
+.venv
+cache
 __pycache__

download_digital.py

@@ -71,6 +71,24 @@ bundesland_dict = {
 }
 
 
+landesbezirk_dict = {
+    "100": "Nord",
+    "200": "Niedersachsen-Bremen",
+    "300": "Berlin-Brandenburg",
+    "400": "Nordrhein-Westfalen",
+    "500": "Rheinland-Pfalz-Saarland",
+    "600": "Hessen",
+    "700": "Sachsen, Sachsen-Anhalt, Thüringen",
+    "800": "Bayern",
+    "900": "Baden-Württemberg",
+    "1000": "Hamburg",
+}
+
+
+def get_landesbezirk(id: str):
+    return landesbezirk_dict[str((int(id) // 100) * 100)]
+
+
 def get_bez_data(
     tags: list[str], url: str = "https://beschaeftigtenbefragung.verdi.de/"
 ) -> list[dict]:
@@ -90,22 +108,22 @@ def get_bez_data(
 
 def construct_dataframe(
     bez_data: dict[str, dict],
-    grouped: bool = False,
     special_tag: str | None = None,
-    no_processing: bool = False,
 ):
     data = {}
-    if not no_processing:
+
+    first_key = next(iter(bez_data.keys()))
+    if first_key in landesbezirk_dict:
+        data["Landesbezirk"] = pd.Series(
+            [v["name"] for v in bez_data.values()], index=list(bez_data.keys())
+        )
+    else:
         data["Bundesland"] = pd.Series(
             [bundesland_dict[k] for k in bez_data], index=list(bez_data.keys())
         )
         data["Bezirk"] = pd.Series(
             [v["name"] for v in bez_data.values()], index=list(bez_data.keys())
         )
-    else:
-        data["Landesbezirk"] = pd.Series(
-            [v["name"] for v in bez_data.values()], index=list(bez_data.keys())
-        )
 
     tot_col_data = []
     tot_col_index = []
@@ -121,15 +139,7 @@ def construct_dataframe(
                 tot_col_index.append(k)
 
     data["Digitale Befragung"] = pd.Series(tot_col_data, index=tot_col_index)
-    df = pd.DataFrame(data=data)
+    return pd.DataFrame(data=data).astype({"Digitale Befragung": "Int32"})
-    df = df.astype({"Digitale Befragung": "Int32"})
-
-    if grouped and no_processing:
-        raise ValueError
-    elif grouped:
-        df = df.groupby("Bundesland", as_index=False)[["Digitale Befragung"]].sum()
-
-    return df
 
 
 def main(
@@ -138,24 +148,27 @@ def main(
     dry_run: bool = False,
     grouped: bool = False,
     special_tag: str | None = None,
-    no_processing: bool = False,
+    folder: str = "data",
+    name: str = "data",
+    sheet_name: str = "digital",
 ) -> None:
     bez_data = get_bez_data([tag], url)[0]
     df = construct_dataframe(
         bez_data=bez_data,
-        grouped=grouped,
         special_tag=special_tag,
-        no_processing=no_processing,
     )
 
+    if grouped:
+        df = df.groupby("Bundesland", as_index=False)[["Digitale Befragung"]].sum()
+
     if dry_run:
         print(df)
     else:
-        filename = f"data/{datetime.today().strftime('%Y-%m-%d')}_data.ods"
+        filename = f"{folder}/{datetime.today().strftime('%Y-%m-%d')}_{name}.ods"
         if Path(filename).exists():
             print("File already exists!")
         else:
-            df.to_excel(filename, sheet_name="digital")
+            df.to_excel(filename, sheet_name=sheet_name)
 
 
 if __name__ == "__main__":

plot.py

@@ -0,0 +1,22 @@
+import datetime
+from pathlib import Path
+
+import fire
+import matplotlib.pyplot as plt
+
+from wsgi import create_fig, create_plot_df
+
+
+def main(folder: str = "plots"):
+    fig, _df, _df_state, timestamp = create_fig()
+    timestamp = timestamp.replace(" ", "_")
+    timestamp = timestamp.replace(":", "-")
+    plot_df = create_plot_df(datetime.datetime.now(), _df_state)
+    print(plot_df.sum(1))
+    fig.savefig(
+        Path(folder) / f"digital_plot_{timestamp}.png", dpi=300, bbox_inches="tight"
+    )
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

regression.py

@@ -0,0 +1,128 @@
+import datetime
+from pathlib import Path
+
+import fire
+import matplotlib
+import matplotlib.dates as mdates
+import matplotlib.pyplot as plt
+import matplotlib.ticker as mtick
+import numpy as np
+import pandas as pd
+import scipy
+
+from wsgi import create_fig, create_plot_df, get_tables, plot
+
+
+def create_dfs(url: str = "https://beschaeftigtenbefragung.verdi.de/"):
+    try:
+        df, df_state, curr_datetime = get_tables(url)
+
+        df = df.sort_values(
+            ["Digitale Befragung", "Bundesland", "Bezirk"],
+            ascending=[False, True, True],
+        )
+
+        df_state = df_state.sort_values("Landesbezirk")
+        plot_df = create_plot_df(curr_datetime, df_state)
+
+    except Exception as e:
+        print(e)
+        last_file = sorted(Path("data").iterdir())[-1]
+        key = last_file.name[:10]
+
+        with (Path("data") / f"{key}_data.ods").open("rb") as ff:
+            df = pd.read_excel(ff, sheet_name="digital", index_col=0).astype(
+                {"Digitale Befragung": "Int32"}
+            )
+        with (Path("data") / f"{key}_state_data.ods").open("rb") as ff:
+            df_state = pd.read_excel(ff, sheet_name="digital", index_col=0).astype(
+                {"Digitale Befragung": "Int32"}
+            )
+
+        plot_df = create_plot_df(None, None)
+
+    return df, df_state, plot_df
+
+
+def main():
+    df, df_state, plot_df = create_dfs()
+
+    plot(plot_df, landesbez_str=[None], max_shading_date="2023-10-02")
+
+    plt.gcf().set_size_inches(10, 5)
+
+    target_time = pd.Timestamp("2023-10-01")
+    xlim = plt.xlim()
+    plt.xlim(xlim[0], pd.Timestamp("2023-10-02"))
+
+    plt.ylim(0, 3500 * 1.025)
+
+    data = plot_df.dropna().sum(1)
+    data = data.iloc[3:]
+    casted_timepoints = data.index.to_numpy().astype(np.int64)
+    reg = scipy.stats.linregress(casted_timepoints, data)
+
+    print(f"Regression R^2: {reg.rvalue**2:.6f}")
+
+    date_range = pd.date_range(start="2023-08-21 10:00:00", end=target_time)
+    date_range = date_range.to_series(index=np.arange(len(date_range)))
+    date_range.loc[len(date_range)] = target_time
+
+    regression_curve = lambda x: reg.intercept + reg.slope * x.astype(np.int64)
+    vals = regression_curve(date_range.to_numpy())
+    print(f"Projizierte Teilnahme am {target_time}: {vals[-1]:.2f}")
+    now = pd.Timestamp.now()
+    print(
+        f"Projizierte Teilnahme jetzt: {regression_curve(pd.Series([now]).to_numpy()).item():.2f}"
+    )
+
+    print()
+
+    for target in [1500, 2500, 3500]:
+        target_reached_date = (target - reg.intercept) / reg.slope
+        print(
+            f"Ziel {target} erreicht am {pd.Timestamp(target_reached_date).strftime('%Y-%m-%d %X')}"
+        )
+
+    num_skipped_days = 2
+
+    x = date_range.to_numpy().astype(np.int64)
+    curr_time = x[data.index.argmax() + num_skipped_days]
+
+    delta = 3500 - data[-1]
+
+    target_line = data[-1] + delta / (x[-1] - curr_time) * (
+        x[data.index.argmax() + num_skipped_days :] - curr_time
+    )
+
+    plt.plot(
+        date_range,
+        vals,
+        label=f"Lineare Regression ($R^2={reg.rvalue**2:.3f}$)",
+        color="tab:green",
+        zorder=1,
+    )
+    plt.plot(
+        date_range[data.index.argmax() + num_skipped_days :],
+        target_line,
+        label="Ziellinie",
+        color="tab:orange",
+        linestyle=":",
+        zorder=1,
+    )
+    # plt.gca().relim()      # make sure all the data fits
+    # plt.gca().autoscale()  # auto-scale
+    plt.xlabel("Zeit in Tagen ab dem 15.08.")
+    plt.axvline(x=target_time, color="tab:red", linestyle="--")
+    plt.legend()
+    plt.gca().xaxis.set_major_locator(matplotlib.ticker.NullLocator())
+    plt.gca().xaxis.set_major_locator(matplotlib.ticker.NullLocator())
+    plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%d.%m."))
+
+    plt.gca().set_xticks([target_time])
+    plt.title("Projektion Teilnahme an Digitaler Beschäftigtenbefragung")
+    plt.savefig("plots/regression.png", bbox_inches="tight", dpi=300)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

static/style.css

@@ -1,6 +1,13 @@
+main {
+    display: flex;
+    flex-flow: row wrap;
+    justify-content: space-around;
+    gap: 1em;
+    align-items: flex-start;
+}
+
 p {
     font-family: sans-serif;
-    margin: 10px;
 }
 
 a {
@@ -10,13 +17,11 @@ a {
 
 .dataframe {
     border-collapse: collapse;
-    margin: 25px 0;
     font-size: 0.9em;
     font-family: sans-serif;
     min-width: 400px;
     box-shadow: 0 0 20px rgba(0, 0, 0, 0.15);
-    float: left;
+    flex-grow: 1;
-    margin: 10px;
 }
 
 .dataframe thead tr {

templates/base.html

@@ -5,7 +5,30 @@
 
 <title>Digitale Beschäftigtenbefragung</title>
 
-<p>Bei einzelnen ver.di-Bezirken liegen Daten nur zu den Bezirken vor, bei denen TVStud einen Schwerpunkt bildet.</p>
-<p><a href="https://zusammen-geht-mehr.verdi.de/beschaeftigtenbefragung">Karte der digitalen Beschäftigtenbefragung</a></p>
 
-{{ tables|safe }}
+<article>
+<header>
+<p>Daten abgerufen: {{ timestamp }}</p>
+<p><i>Hinweis:</i> Die Daten entstammen der <a href="https://zusammen-geht-mehr.verdi.de/beschaeftigtenbefragung">Karte der digitalen Beschäftigtenbefragung</a> und sind dort nur für die ver.di-Bezirke auf Bezirksebene aufgelöst, bei denen TVStud auch als Schwerpunkt angegeben ist.</p>
+</header>
+
+<main>
+    {{ image|safe }}
+    {{ tables|safe }}
+</main>
+
+<footer>
+    <p>Ergebnisse einzelner Landesbezirke:
+        <a href="/">Bundesweit</a> |
+        <a href="/BaWü">Baden-Württemberg</a> |
+        <a href="/Bayern">Bayern</a> |
+        <a href="/BBR">Berlin-Brandenburg</a> |
+        <a href="/Hamburg">Hamburg</a> |
+        <a href="/NDS">Niedersachsen-Bremen</a> |
+        <a href="/Nord">Nord</a> |
+        <a href="/NRW">NRW</a> |
+        <a href="/RLP">Rheinland-Pfalz-Saarland</a> |
+        <a href="/SAT">Sachsen, Sachsen-Anhalt, Thüringen</a>
+    </p>
+</footer>
+</article>

wsgi.py

@@ -1,64 +1,445 @@
+import datetime
+import io
+import locale
+import os
+import time
+from itertools import chain
+from pathlib import Path
+
+import matplotlib.dates as mdates
+import matplotlib.pyplot as plt
+import matplotlib.ticker as mtick
+import numpy as np
 import pandas as pd
-from flask import Flask, render_template, request
+from flask import Flask, Markup, abort, render_template, request
+from flask_caching import Cache
 
-from download_digital import construct_dataframe, get_bez_data
+from download_digital import (
+    construct_dataframe,
+    get_bez_data,
+    get_landesbezirk,
+    landesbezirk_dict,
+)
 
+config = {
+    "CACHE_TYPE": "FileSystemCache",
+    "CACHE_DEFAULT_TIMEOUT": 300,
+    "CACHE_THRESHOLD": 50,
+    "CACHE_DIR": "cache",
+}
+
+abbrev_dict = {
+    "BBR": "Berlin-Brandenburg",
+    "BaWü": "Baden-Württemberg",
+    "NDS": "Niedersachsen-Bremen",
+    "NRW": "Nordrhein-Westfalen",
+    "RLP": "Rheinland-Pfalz-Saarland",
+    "SAT": "Sachsen, Sachsen-Anhalt, Thüringen",
+}
+
+os.environ["TZ"] = "Europe/Berlin"
+time.tzset()
+
+locale.setlocale(locale.LC_ALL, "de_DE.UTF-8")
 app = Flask(__name__)
+app.config.from_mapping(config)
+cache = Cache(app)
+
+
+def get_tables(url: str) -> tuple[pd.DataFrame, pd.DataFrame, datetime.datetime]:
+    bez_data = get_bez_data(["bez_data_0", "bez_data_2"], url)
+
+    df = construct_dataframe(bez_data=bez_data[0], special_tag="stud")
+    df_state = construct_dataframe(bez_data=bez_data[1])
+
+    return df, df_state, datetime.datetime.now()
+
+
+def create_plot_df(
+    curr_datetime,
+    current_df: pd.DataFrame | None,
+    data_folder: str = "data",
+    sheet_name: str = "digital",
+) -> pd.DataFrame:
+    data_dict = {}
+
+    ## Important: If multiple results are stored for the same date
+    ##            the last is used. So this relies on the Landesbezirk data
+    ##            to be stored with a filename that is lexigraphically larger
+    ##            than the single district results.
+
+    for f in sorted(Path(data_folder).iterdir()):
+        with f.open("rb") as ff:
+            df = pd.read_excel(ff, sheet_name=sheet_name, index_col=0)
+
+        if "Landesbezirk" not in df.columns:
+            df["Landesbezirk"] = df.index.map(get_landesbezirk)
+
+        df = df.astype({"Digitale Befragung": "Int32"})
+        df = df.groupby("Landesbezirk")[["Digitale Befragung"]].sum()
+
+        key = f.name[:10]
+        data_dict[key] = df["Digitale Befragung"]
+
+    df = pd.DataFrame(data=data_dict).T
+    max_date = df.index.max()
+
+    df.index = df.index.astype("datetime64[ns]") + pd.DateOffset(hours=10)
+
+    df = df.reindex(
+        pd.date_range(start="2023-08-15", end=max_date) + pd.DateOffset(hours=10)
+    )
+
+    if current_df is not None:
+        if "Landesbezirk" not in current_df.columns:
+            current_df["Landesbezirk"] = current_df.index.map(get_landesbezirk)
+        current_df = current_df.astype({"Digitale Befragung": "Int32"})
+        current_df = current_df.groupby("Landesbezirk")[["Digitale Befragung"]].sum()
+
+        df.loc[curr_datetime] = current_df["Digitale Befragung"]
+
+        if pd.isna(df.loc[df.index.max()][0]):
+            df = df.drop([df.index.max()])
+
+    return df
+
+
+def plot(
+    df: pd.DataFrame,
+    annotate_current: bool = False,
+    total_targets: tuple[int, ...] = (1500, 2500, 3500),
+    alpha: float | None = None,
+    landesbez_str: str | None = None,
+    fix_lims: bool = True,
+    max_shading_date=None,
+) -> str:
+    fig = plt.figure(dpi=300, figsize=(8.5, 5))
+
+    target_time = pd.Timestamp("2023-10-01")
+    plt.axvline(x=target_time, color="tab:green", linestyle=":")
+
+    if fix_lims:
+        for total_target in total_targets:
+            plt.axhline(y=total_target, color="#48a9be", linestyle="--")
+
+    for bez in landesbez_str:
+        series = df.sum(axis=1) if bez is None else df[bez]
+        plot_df = series.to_frame("Digitale Befragung").replace(0, np.nan)
+        plot_df = plot_df.astype({"Digitale Befragung": "float32"})
+        if not pd.isna(plot_df).all().item():
+            if alpha is not None:
+                plt.fill_between(
+                    plot_df.dropna().index,
+                    plot_df.dropna()["Digitale Befragung"],
+                    color="#e4004e",
+                    alpha=alpha,
+                )
+
+            (line,) = plt.plot(
+                plot_df.dropna().index,
+                plot_df.dropna()["Digitale Befragung"],
+                ls="--",
+                marker="o",
+                lw=1,
+                color="#e4004e" if bez is None or not fix_lims else None,
+                markersize=4,
+                label=bez if bez is not None else "Bundesweit",
+            )
+
+            if annotate_current and bez is None:
+                plt.annotate(
+                    "Jetzt",
+                    (
+                        plot_df.dropna().index[-1],
+                        plot_df.dropna()["Digitale Befragung"][-1] * 1.03,
+                    ),
+                    fontsize=8,
+                    ha="center",
+                )
+
+            plt.plot(
+                plot_df.index,
+                plot_df["Digitale Befragung"],
+                lw=1.5,
+                color=line.get_color(),
+                # label=bez,
+            )
+
+    plt.title("Teilnahme an Digitaler Beschäftigtenbefragung")
+    plt.ylabel("# Teilnahmen")
+
+    if fix_lims:
+        max_val = df.sum(axis=1).max().item()
+
+        nearest_target = np.array(total_targets, dtype=np.float32) - max_val
+        nearest_target[nearest_target <= 0] = np.inf
+        idx = np.argmin(nearest_target)
+
+        ceil_val = max(max_val, total_targets[idx])
+        plt.ylim(0, ceil_val * 1.04)
+    plt.legend()
+
+    # use timezone offset to center tick labels
+    plt.gca().xaxis.set_major_locator(
+        mdates.WeekdayLocator([mdates.TU], tz="Etc/GMT+12")
+    )
+    plt.gca().xaxis.set_minor_locator(mdates.DayLocator())
+    plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%a %d.%m."))
+
+    plt.grid(True, which="major", axis="y")
+    plt.grid(True, which="minor", axis="x")
+
+    plt.gca().tick_params("x", length=0, which="major")
+
+    def val_to_perc(val):
+        return 100 * val / total_targets[0]
+
+    def perc_to_val(perc):
+        return perc * total_targets[0] / 100
+
+    sec_ax = plt.gca().secondary_yaxis("right", functions=(val_to_perc, perc_to_val))
+    sec_ax.set_ylabel("# Teilnahmen [% Erfolg]")
+    sec_ax.yaxis.set_major_formatter(mtick.PercentFormatter())
+
+    xlim = plt.xlim()
+
+    # fill weekends
+    if max_shading_date is None:
+        max_shading_date = df.index.max() + datetime.timedelta(days=4)
+    days = pd.date_range(start="2023-08-14", end=max_shading_date)
+    for idx, day in enumerate(days[:-1]):
+        if day.weekday() >= 5:
+            plt.gca().axvspan(days[idx], days[idx + 1], alpha=0.2, color="gray")
+
+    # reset xlim
+    plt.xlim((xlim[0], pd.Timestamp("2023-10-02")))
+
+    plt.tight_layout()
+
+    return fig
+
+
+def create_fig(
+    url: str = "https://beschaeftigtenbefragung.verdi.de/",
+    importance_factor: float = 1.0,
+    landesbez_strs: list[str | None] | None = None,
+    fix_lims: bool = True,
+):
+    curr_datetime = datetime.datetime.now()
+    try:
+        df, df_state, curr_datetime = get_tables(url)
+
+        df = df.sort_values(
+            ["Digitale Befragung", "Bundesland", "Bezirk"],
+            ascending=[False, True, True],
+        )
+
+        df_state = df_state.sort_values("Landesbezirk")
+
+        plot_df = create_plot_df(curr_datetime, df_state)
+        annotate_current = True
+        timestamp = curr_datetime.strftime("%Y-%m-%d %H:%M:%S")
+
+    except Exception as e:
+        print(e)
+        last_file = sorted(Path("data").iterdir())[-1]
+        key = last_file.name[:10]
+
+        with (Path("data") / f"{key}_data.ods").open("rb") as ff:
+            df = pd.read_excel(ff, sheet_name="digital", index_col=0).astype(
+                {"Digitale Befragung": "Int32"}
+            )
+        with (Path("data") / f"{key}_state_data.ods").open("rb") as ff:
+            df_state = pd.read_excel(ff, sheet_name="digital", index_col=0).astype(
+                {"Digitale Befragung": "Int32"}
+            )
+
+        plot_df = create_plot_df(curr_datetime, df_state)
+        annotate_current = False
+        timestamp = Markup(f'<font color="red">{key} 10:00:00</font>')
+
+    total = plot_df.loc[curr_datetime].sum()
+    if landesbez_strs is None:
+        landesbez_strs = [None] + [
+            bez
+            for bez in plot_df.columns
+            if plot_df.loc[curr_datetime][bez] >= importance_factor * total
+        ]
+    return (
+        plot(
+            plot_df,
+            annotate_current=annotate_current,
+            landesbez_str=landesbez_strs,
+            fix_lims=fix_lims,
+        ),
+        df,
+        df_state,
+        timestamp,
+    )
+
+
+def convert_fig_to_svg(fig: plt.Figure) -> str:
+    # Convert plot to SVG image
+    imgdata = io.StringIO()
+    fig.savefig(imgdata, format="svg")
+    imgdata.seek(0)  # rewind the data
+
+    return imgdata.read()
+
+
+def _print_as_html(
+    df: pd.DataFrame,
+    output_str: list[str],
+    df_state: pd.DataFrame | None = None,
+    dropna: bool = True,
+) -> list[str]:
+    df = df.astype({"Digitale Befragung": "Int32"})
+    missing_df = (
+        df[["Digitale Befragung"]]
+        .isna()
+        .join(df[["Landesbezirk"]])
+        .groupby("Landesbezirk")
+        .sum()
+    )
+    total = df_state["Digitale Befragung"].sum() if df_state is not None else None
+
+    if df_state is not None:
+        for idx, row in missing_df.loc[
+            missing_df["Digitale Befragung"] == 1
+        ].iterrows():
+            df_tmp = df.loc[df["Landesbezirk"] == idx]
+            df_state_tmp = df_state.loc[df_state["Landesbezirk"] == idx]
+            missing_idx = df_tmp.loc[df_tmp.isna().any(axis=1)].iloc[0].name
+            df["Digitale Befragung"].loc[missing_idx] = (
+                df_state_tmp["Digitale Befragung"].sum()
+                - df_tmp["Digitale Befragung"].sum()
+            )
+
+        df = df.sort_values(
+            ["Digitale Befragung", "Landesbezirk", "Bezirk"],
+            ascending=[False, True, True],
+        )
+    if dropna:
+        df = df.dropna()
+
+    with pd.option_context("display.max_rows", None):
+        table = df.to_html(
+            index_names=False,
+            justify="left",
+            index=False,
+            classes="sortable dataframe",
+        )
+
+        tfoot = [
+            "  <tfoot>",
+            "    <tr>",
+            "      <td>Gesamt</td>",
+        ]
+        for i in range(len(df.columns) - 2):
+            tfoot.append("      <td></td>")
+        tfoot.extend(
+            [
+                f"      <td>{df['Digitale Befragung'].sum()}</td>",
+                "    </tr>",
+            ]
+        )
+        if total and (diff := total - df["Digitale Befragung"].sum()):
+            tfoot.append("    <tr>")
+            num_missing = missing_df["Digitale Befragung"].sum()
+            tfoot.append(
+                f"      <td>Weitere Bezirke ({num_missing})</td>"
+                if num_missing
+                else f"      <td>Weitere Bezirke</td>"
+            )
+            for i in range(len(df.columns) - 2):
+                tfoot.append("      <td></td>")
+            tfoot.extend(
+                [
+                    f"      <td>{diff}</td>",
+                    "    </tr>",
+                ]
+            )
+        tfoot.append("  </tfoot>")
+
+        tfoot = "\n".join(tfoot)
+        idx = table.index("</table>")
+        output_str.append(table[: idx - 1])
+        output_str.append(tfoot)
+        output_str.append(table[idx:])
+    return output_str
+
+
+@app.route("/<state>")
+@cache.cached(query_string=True)
+def state_dashboard(state: str):
+    if state in abbrev_dict:
+        state = abbrev_dict[state]
+
+    if state not in landesbezirk_dict.values():
+        abort(404)
+
+    importance_factor = request.args.get("importance")
+    if not importance_factor:
+        importance_factor = 1.0
+    else:
+        importance_factor = float(importance_factor)
+
+    fig, df, df_state, timestamp = create_fig(landesbez_strs=[state], fix_lims=False)
+    svg_string = convert_fig_to_svg(fig)
+    plt.close()
+
+    df["Bundesland"] = df.index.map(get_landesbezirk)
+    df = df.rename(columns={"Bundesland": "Landesbezirk"})
+
+    df_state = df_state.loc[df_state["Landesbezirk"] == state]
+    df = df.loc[df["Landesbezirk"] == state]
+
+    output_str = []
+    output_str = _print_as_html(df_state, output_str, dropna=False)
+    output_str = _print_as_html(df, output_str, df_state=df_state, dropna=False)
+
+    return render_template(
+        "base.html",
+        tables="\n".join(output_str),
+        timestamp=timestamp,
+        image=svg_string,
+    )
 
 
 @app.route("/")
-def tables(
+@cache.cached(query_string=True)
-    url: str = "https://beschaeftigtenbefragung.verdi.de/",
+def dashboard():
-):
+    importance_factor = request.args.get("importance")
-    bez_data = get_bez_data(["bez_data_0", "bez_data_2"], url)
+    if not importance_factor:
+        importance_factor = 1.0
+    else:
+        importance_factor = float(importance_factor)
 
-    df = construct_dataframe(
+    fig, df, df_state, timestamp = create_fig(importance_factor=importance_factor)
-        bez_data=bez_data[0],
+    svg_string = convert_fig_to_svg(fig)
-        grouped=False,
+    plt.close()
-        special_tag="stud",
-    ).sort_values(
-        ["Digitale Befragung", "Bundesland", "Bezirk"], ascending=[False, True, True]
-    )
 
-    df_state = construct_dataframe(
+    df["Bundesland"] = df.index.map(get_landesbezirk)
-        bez_data=bez_data[1], grouped=False, no_processing=True
+    df = df.rename(columns={"Bundesland": "Landesbezirk"})
-    ).sort_values("Landesbezirk")
 
     output_str = []
+    output_str = _print_as_html(df_state, output_str, dropna=False)
+    output_str = _print_as_html(df, output_str, df_state)
 
-    def _print_as_html(df: pd.DataFrame):
+    return render_template(
-        df = df.astype({"Digitale Befragung": "Int32"})
+        "base.html",
-        with pd.option_context("display.max_rows", None):
+        tables="\n".join(output_str),
-            table = df.to_html(
+        timestamp=timestamp,
-                index_names=False,
+        image=svg_string,
-                justify="left",
+    )
-                index=False,
-                classes="sortable dataframe",
-            )
 
-            tfoot = [
-                "  <tfoot>",
-                "      <td>Gesamt</td>",
-            ]
-            for i in range(len(df.columns) - 2):
-                tfoot.append("      <td/>")
-            tfoot.extend(
-                [
-                    f"      <td>{df['Digitale Befragung'].sum()}</td>",
-                    "    </tr>",
-                    "  </tfoot>",
-                ]
-            )
-            tfoot = "\n".join(tfoot)
-            idx = table.index("</table>")
-            output_str.append(table[: idx - 1])
-            output_str.append(tfoot)
-            output_str.append(table[idx:])
 
-    _print_as_html(df_state)
+@app.route("/total")
-    _print_as_html(df)
+@cache.cached(timeout=60)
-
+def total_result(url: str = "https://beschaeftigtenbefragung.verdi.de/"):
-    return render_template("base.html", tables="\n".join(output_str))
+    df, df_state, curr_datetime = get_tables(url)
+    total = df_state["Digitale Befragung"].sum().item()
+    return f"{total}"
 
 
 if __name__ == "__main__":