Fix fallback

plot.py

@@ -13,7 +13,9 @@ def main(folder: str = "plots"):
     timestamp = timestamp.replace(":", "-")
     plot_df = create_plot_df(datetime.datetime.now(), _df_state)
     print(plot_df.sum(1))
-    fig.savefig(Path(folder) / f"digital_plot_{timestamp}.png", dpi=300)
+    fig.savefig(
+        Path(folder) / f"digital_plot_{timestamp}.png", dpi=300, bbox_inches="tight"
+    )
 
 
 if __name__ == "__main__":

regression.py

@@ -0,0 +1,128 @@
+import datetime
+from pathlib import Path
+
+import fire
+import matplotlib
+import matplotlib.dates as mdates
+import matplotlib.pyplot as plt
+import matplotlib.ticker as mtick
+import numpy as np
+import pandas as pd
+import scipy
+
+from wsgi import create_fig, create_plot_df, get_tables, plot
+
+
+def create_dfs(url: str = "https://beschaeftigtenbefragung.verdi.de/"):
+    try:
+        df, df_state, curr_datetime = get_tables(url)
+
+        df = df.sort_values(
+            ["Digitale Befragung", "Bundesland", "Bezirk"],
+            ascending=[False, True, True],
+        )
+
+        df_state = df_state.sort_values("Landesbezirk")
+        plot_df = create_plot_df(curr_datetime, df_state)
+
+    except Exception as e:
+        print(e)
+        last_file = sorted(Path("data").iterdir())[-1]
+        key = last_file.name[:10]
+
+        with (Path("data") / f"{key}_data.ods").open("rb") as ff:
+            df = pd.read_excel(ff, sheet_name="digital", index_col=0).astype(
+                {"Digitale Befragung": "Int32"}
+            )
+        with (Path("data") / f"{key}_state_data.ods").open("rb") as ff:
+            df_state = pd.read_excel(ff, sheet_name="digital", index_col=0).astype(
+                {"Digitale Befragung": "Int32"}
+            )
+
+        plot_df = create_plot_df(None, None)
+
+    return df, df_state, plot_df
+
+
+def main():
+    df, df_state, plot_df = create_dfs()
+
+    plot(plot_df, landesbez_str=[None], max_shading_date="2023-10-02")
+
+    plt.gcf().set_size_inches(10, 5)
+
+    target_time = pd.Timestamp("2023-10-01")
+    xlim = plt.xlim()
+    plt.xlim(xlim[0], pd.Timestamp("2023-10-02"))
+
+    plt.ylim(0, 3500 * 1.025)
+
+    data = plot_df.dropna().sum(1)
+    data = data.iloc[3:]
+    casted_timepoints = data.index.to_numpy().astype(np.int64)
+    reg = scipy.stats.linregress(casted_timepoints, data)
+
+    print(f"Regression R^2: {reg.rvalue**2:.6f}")
+
+    date_range = pd.date_range(start="2023-08-21 10:00:00", end=target_time)
+    date_range = date_range.to_series(index=np.arange(len(date_range)))
+    date_range.loc[len(date_range)] = target_time
+
+    regression_curve = lambda x: reg.intercept + reg.slope * x.astype(np.int64)
+    vals = regression_curve(date_range.to_numpy())
+    print(f"Projizierte Teilnahme am {target_time}: {vals[-1]:.2f}")
+    now = pd.Timestamp.now()
+    print(
+        f"Projizierte Teilnahme jetzt: {regression_curve(pd.Series([now]).to_numpy()).item():.2f}"
+    )
+
+    print()
+
+    for target in [1500, 2500, 3500]:
+        target_reached_date = (target - reg.intercept) / reg.slope
+        print(
+            f"Ziel {target} erreicht am {pd.Timestamp(target_reached_date).strftime('%Y-%m-%d %X')}"
+        )
+
+    num_skipped_days = 2
+
+    x = date_range.to_numpy().astype(np.int64)
+    curr_time = x[data.index.argmax() + num_skipped_days]
+
+    delta = 3500 - data[-1]
+
+    target_line = data[-1] + delta / (x[-1] - curr_time) * (
+        x[data.index.argmax() + num_skipped_days :] - curr_time
+    )
+
+    plt.plot(
+        date_range,
+        vals,
+        label=f"Lineare Regression ($R^2={reg.rvalue**2:.3f}$)",
+        color="tab:green",
+        zorder=1,
+    )
+    plt.plot(
+        date_range[data.index.argmax() + num_skipped_days :],
+        target_line,
+        label="Ziellinie",
+        color="tab:orange",
+        linestyle=":",
+        zorder=1,
+    )
+    # plt.gca().relim()      # make sure all the data fits
+    # plt.gca().autoscale()  # auto-scale
+    plt.xlabel("Zeit in Tagen ab dem 15.08.")
+    plt.axvline(x=target_time, color="tab:red", linestyle="--")
+    plt.legend()
+    plt.gca().xaxis.set_major_locator(matplotlib.ticker.NullLocator())
+    plt.gca().xaxis.set_major_locator(matplotlib.ticker.NullLocator())
+    plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%d.%m."))
+
+    plt.gca().set_xticks([target_time])
+    plt.title("Projektion Teilnahme an Digitaler Beschäftigtenbefragung")
+    plt.savefig("plots/regression.png", bbox_inches="tight", dpi=300)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

templates/base.html

@@ -16,4 +16,19 @@
     {{ image|safe }}
     {{ tables|safe }}
 </main>
+
+<footer>
+    <p>Ergebnisse einzelner Landesbezirke:
+        <a href="/">Bundesweit</a> |
+        <a href="/BaWü">Baden-Württemberg</a> |
+        <a href="/Bayern">Bayern</a> |
+        <a href="/BBR">Berlin-Brandenburg</a> |
+        <a href="/Hamburg">Hamburg</a> |
+        <a href="/NDS">Niedersachsen-Bremen</a> |
+        <a href="/Nord">Nord</a> |
+        <a href="/NRW">NRW</a> |
+        <a href="/RLP">Rheinland-Pfalz-Saarland</a> |
+        <a href="/SAT">Sachsen, Sachsen-Anhalt, Thüringen</a>
+    </p>
+</footer>
 </article>

wsgi.py

@@ -14,7 +14,12 @@ import pandas as pd
 from flask import Flask, Markup, abort, render_template, request
 from flask_caching import Cache
 
-from download_digital import construct_dataframe, get_bez_data, get_landesbezirk, landesbezirk_dict
+from download_digital import (
+    construct_dataframe,
+    get_bez_data,
+    get_landesbezirk,
+    landesbezirk_dict,
+)
 
 config = {
     "CACHE_TYPE": "FileSystemCache",
@@ -26,6 +31,7 @@ config = {
 abbrev_dict = {
     "BBR": "Berlin-Brandenburg",
     "BaWü": "Baden-Württemberg",
+    "NDS": "Niedersachsen-Bremen",
     "NRW": "Nordrhein-Westfalen",
     "RLP": "Rheinland-Pfalz-Saarland",
     "SAT": "Sachsen, Sachsen-Anhalt, Thüringen",
@@ -40,13 +46,13 @@ app.config.from_mapping(config)
 cache = Cache(app)
 
 
-def get_tables(url: str) -> tuple[pd.DataFrame, pd.DataFrame]:
+def get_tables(url: str) -> tuple[pd.DataFrame, pd.DataFrame, datetime.datetime]:
     bez_data = get_bez_data(["bez_data_0", "bez_data_2"], url)
 
     df = construct_dataframe(bez_data=bez_data[0], special_tag="stud")
     df_state = construct_dataframe(bez_data=bez_data[1])
 
-    return df, df_state
+    return df, df_state, datetime.datetime.now()
 
 
 def create_plot_df(
@@ -76,11 +82,12 @@ def create_plot_df(
         data_dict[key] = df["Digitale Befragung"]
 
     df = pd.DataFrame(data=data_dict).T
+    max_date = df.index.max()
 
     df.index = df.index.astype("datetime64[ns]") + pd.DateOffset(hours=10)
 
     df = df.reindex(
-        pd.date_range(start="2023-08-15", end=curr_datetime) + pd.DateOffset(hours=10)
+        pd.date_range(start="2023-08-15", end=max_date) + pd.DateOffset(hours=10)
     )
 
     if current_df is not None:
@@ -98,22 +105,22 @@ def create_plot_df(
 
 
 def plot(
-    curr_datetime,
     df: pd.DataFrame,
     annotate_current: bool = False,
     total_targets: tuple[int, ...] = (1500, 2500, 3500),
     alpha: float | None = None,
     landesbez_str: str | None = None,
     fix_lims: bool = True,
+    max_shading_date=None,
 ) -> str:
-    fig = plt.figure(dpi=300)
+    fig = plt.figure(dpi=300, figsize=(8.5, 5))
 
-    # fill weekends
+    target_time = pd.Timestamp("2023-10-01")
-    max_date = curr_datetime + datetime.timedelta(days=1)
+    plt.axvline(x=target_time, color="tab:green", linestyle=":")
-    days = pd.date_range(start="2023-08-14", end=max_date)
+
-    for idx, day in enumerate(days[:-1]):
+    if fix_lims:
-        if day.weekday() >= 5:
+        for total_target in total_targets:
-            plt.gca().axvspan(days[idx], days[idx + 1], alpha=0.2, color="gray")
+            plt.axhline(y=total_target, color="#48a9be", linestyle="--")
 
     for bez in landesbez_str:
         series = df.sum(axis=1) if bez is None else df[bez]
@@ -169,7 +176,7 @@ def plot(
         idx = np.argmin(nearest_target)
 
         ceil_val = max(max_val, total_targets[idx])
-        plt.ylim(0, ceil_val * 1.025)
+        plt.ylim(0, ceil_val * 1.04)
     plt.legend()
 
     # use timezone offset to center tick labels
@@ -194,9 +201,18 @@ def plot(
     sec_ax.set_ylabel("# Teilnahmen [% Erfolg]")
     sec_ax.yaxis.set_major_formatter(mtick.PercentFormatter())
 
-    if fix_lims:
+    xlim = plt.xlim()
-        for total_target in total_targets:
+
-            plt.axhline(y=total_target, color="#48a9be", linestyle="--")
+    # fill weekends
+    if max_shading_date is None:
+        max_shading_date = df.index.max() + datetime.timedelta(days=4)
+    days = pd.date_range(start="2023-08-14", end=max_shading_date)
+    for idx, day in enumerate(days[:-1]):
+        if day.weekday() >= 5:
+            plt.gca().axvspan(days[idx], days[idx + 1], alpha=0.2, color="gray")
+
+    # reset xlim
+    plt.xlim((xlim[0], pd.Timestamp("2023-10-02")))
 
     plt.tight_layout()
 
@@ -211,7 +227,7 @@ def create_fig(
 ):
     curr_datetime = datetime.datetime.now()
     try:
-        df, df_state = get_tables(url)
+        df, df_state, curr_datetime = get_tables(url)
 
         df = df.sort_values(
             ["Digitale Befragung", "Bundesland", "Bezirk"],
@@ -238,7 +254,7 @@ def create_fig(
                 {"Digitale Befragung": "Int32"}
             )
 
-        plot_df = create_plot_df(curr_datetime)
+        plot_df = create_plot_df(curr_datetime, df_state)
         annotate_current = False
         timestamp = Markup(f'<font color="red">{key} 10:00:00</font>')
 
@@ -251,7 +267,6 @@ def create_fig(
         ]
     return (
         plot(
-            curr_datetime,
             plot_df,
             annotate_current=annotate_current,
             landesbez_str=landesbez_strs,
@@ -272,10 +287,41 @@ def convert_fig_to_svg(fig: plt.Figure) -> str:
     return imgdata.read()
 
 
-def _print_as_html(df: pd.DataFrame, output_str: list[str], total: int | None = None, dropna: bool = True) -> list[str]:
+def _print_as_html(
+    df: pd.DataFrame,
+    output_str: list[str],
+    df_state: pd.DataFrame | None = None,
+    dropna: bool = True,
+) -> list[str]:
     df = df.astype({"Digitale Befragung": "Int32"})
+    missing_df = (
+        df[["Digitale Befragung"]]
+        .isna()
+        .join(df[["Landesbezirk"]])
+        .groupby("Landesbezirk")
+        .sum()
+    )
+    total = df_state["Digitale Befragung"].sum() if df_state is not None else None
+
+    if df_state is not None:
+        for idx, row in missing_df.loc[
+            missing_df["Digitale Befragung"] == 1
+        ].iterrows():
+            df_tmp = df.loc[df["Landesbezirk"] == idx]
+            df_state_tmp = df_state.loc[df_state["Landesbezirk"] == idx]
+            missing_idx = df_tmp.loc[df_tmp.isna().any(axis=1)].iloc[0].name
+            df["Digitale Befragung"].loc[missing_idx] = (
+                df_state_tmp["Digitale Befragung"].sum()
+                - df_tmp["Digitale Befragung"].sum()
+            )
+
+        df = df.sort_values(
+            ["Digitale Befragung", "Landesbezirk", "Bezirk"],
+            ascending=[False, True, True],
+        )
     if dropna:
         df = df.dropna()
+
     with pd.option_context("display.max_rows", None):
         table = df.to_html(
             index_names=False,
@@ -297,16 +343,19 @@ def _print_as_html(df: pd.DataFrame, output_str: list[str], total: int | None =
                 "    </tr>",
             ]
         )
-        if total:
+        if total and (diff := total - df["Digitale Befragung"].sum()):
-            tfoot.extend([
+            tfoot.append("    <tr>")
-                "    <tr>",
+            num_missing = missing_df["Digitale Befragung"].sum()
-                "      <td>Weitere Bezirke</td>",
+            tfoot.append(
-            ])
+                f"      <td>Weitere Bezirke ({num_missing})</td>"
+                if num_missing
+                else f"      <td>Weitere Bezirke</td>"
+            )
             for i in range(len(df.columns) - 2):
                 tfoot.append("      <td></td>")
             tfoot.extend(
                 [
-                    f"      <td>{total - df['Digitale Befragung'].sum()}</td>",
+                    f"      <td>{diff}</td>",
                     "    </tr>",
                 ]
             )
@@ -347,7 +396,7 @@ def state_dashboard(state: str):
 
     output_str = []
     output_str = _print_as_html(df_state, output_str, dropna=False)
-    output_str = _print_as_html(df, output_str, total=df_state['Digitale Befragung'].sum(), dropna=False)
+    output_str = _print_as_html(df, output_str, df_state=df_state, dropna=False)
 
     return render_template(
         "base.html",
@@ -375,7 +424,7 @@ def dashboard():
 
     output_str = []
     output_str = _print_as_html(df_state, output_str, dropna=False)
-    output_str = _print_as_html(df, output_str, total=df_state['Digitale Befragung'].sum())
+    output_str = _print_as_html(df, output_str, df_state)
 
     return render_template(
         "base.html",
@@ -385,5 +434,13 @@ def dashboard():
     )
 
 
+@app.route("/total")
+@cache.cached(timeout=60)
+def total_result(url: str = "https://beschaeftigtenbefragung.verdi.de/"):
+    df, df_state, curr_datetime = get_tables(url)
+    total = df_state["Digitale Befragung"].sum().item()
+    return f"{total}"
+
+
 if __name__ == "__main__":
     app.run()