Compare commits

..

3 Commits

Author SHA1 Message Date
Felix Blanke
976ddc6c23 Add plots to gitignore 2023-08-29 15:40:02 +02:00
Felix Blanke
196b3f7576 Add plotting code 2023-08-29 15:39:36 +02:00
Felix Blanke
bee447b334 Encapsulate data creation for wsgi 2023-08-29 15:39:21 +02:00
3 changed files with 67 additions and 50 deletions

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
plots
.venv
cache
__pycache__

17
plot.py Normal file
View File

@ -0,0 +1,17 @@
from pathlib import Path
import fire
import matplotlib.pyplot as plt
from wsgi import create_fig
def main(folder: str = "plots"):
fig, _df, _df_state, timestamp = create_fig()
timestamp = timestamp.replace(" ", "_")
timestamp = timestamp.replace(":", "-")
fig.savefig(Path(folder) / f"digital_plot_{timestamp}.png", dpi=300)
if __name__ == "__main__":
fire.Fire(main)

99
wsgi.py
View File

@ -183,6 +183,53 @@ def plot(
return fig
def create_fig(url: str = "https://beschaeftigtenbefragung.verdi.de/", importance_factor: float = 1.0):
curr_datetime = datetime.datetime.now()
try:
df, df_state = get_tables(url)
df = df.sort_values(
["Digitale Befragung", "Bundesland", "Bezirk"],
ascending=[False, True, True],
)
df_state = df_state.sort_values("Landesbezirk")
plot_df = create_plot_df(curr_datetime, df_state)
annotate_current = True
timestamp = curr_datetime.strftime("%Y-%m-%d %H:%M:%S")
except Exception as e:
print(e)
last_file = sorted(Path("data").iterdir())[-1]
key = last_file.name[:10]
with (Path("data") / f"{key}_data.ods").open("rb") as ff:
df = pd.read_excel(ff, sheet_name="digital", index_col=0).astype(
{"Digitale Befragung": "Int32"}
)
with (Path("data") / f"{key}_state_data.ods").open("rb") as ff:
df_state = pd.read_excel(ff, sheet_name="digital", index_col=0).astype(
{"Digitale Befragung": "Int32"}
)
plot_df = create_plot_df(curr_datetime)
annotate_current = False
timestamp = Markup(f'<font color="red">{key} 10:00:00</font>')
total = plot_df.loc[curr_datetime].sum()
landesbez_strs = [None] + [
bez
for bez in plot_df.columns
if plot_df.loc[curr_datetime][bez] >= importance_factor * total
]
return plot(
curr_datetime,
plot_df,
annotate_current=annotate_current,
landesbez_str=landesbez_strs,
), df, df_state, timestamp
def convert_fig_to_svg(fig: plt.Figure) -> str:
# Convert plot to SVG image
@ -195,10 +242,7 @@ def convert_fig_to_svg(fig: plt.Figure) -> str:
@app.route("/")
@cache.cached()
def tables(
url: str = "https://beschaeftigtenbefragung.verdi.de/",
importance_factor: float = 1,
):
def tables():
def _print_as_html(df: pd.DataFrame):
df = df.astype({"Digitale Befragung": "Int32"})
with pd.option_context("display.max_rows", None):
@ -231,52 +275,7 @@ def tables(
output_str = []
curr_datetime = datetime.datetime.now()
try:
df, df_state = get_tables(url)
df = df.sort_values(
["Digitale Befragung", "Bundesland", "Bezirk"],
ascending=[False, True, True],
)
df_state = df_state.sort_values("Landesbezirk")
plot_df = create_plot_df(curr_datetime, df_state)
annotate_current = True
timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
except Exception as e:
print(e)
last_file = sorted(Path("data").iterdir())[-1]
key = last_file.name[:10]
with (Path("data") / f"{key}_data.ods").open("rb") as ff:
df = pd.read_excel(ff, sheet_name="digital", index_col=0).astype(
{"Digitale Befragung": "Int32"}
)
with (Path("data") / f"{key}_state_data.ods").open("rb") as ff:
df_state = pd.read_excel(ff, sheet_name="digital", index_col=0).astype(
{"Digitale Befragung": "Int32"}
)
plot_df = create_plot_df(curr_datetime)
annotate_current = False
timestamp = Markup(f'<font color="red">{key} 10:00:00</font>')
total = plot_df.loc[curr_datetime].sum()
landesbez_strs = [None] + [
bez
for bez in plot_df.columns
if plot_df.loc[curr_datetime][bez] >= importance_factor * total
]
fig = plot(
curr_datetime,
plot_df,
annotate_current=annotate_current,
landesbez_str=landesbez_strs,
)
fig, df, df_state, timestamp = create_fig()
svg_string = convert_fig_to_svg(fig)
plt.close()