fblanke/tvstud-download-digital
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move to flask

Browse Source
This commit is contained in:
Felix Blanke 2023-08-19 22:00:38 +02:00
parent 3109b1eddf
commit 27c196ee18
2 changed files with 64 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
app.py Normal file
View File

@ -0,0 +1,52 @@
import pandas as pd
from flask import Flask, render_template
from download_digital import construct_dataframe
app = Flask(__name__)
@app.route("/")
def tables(
url: str = "https://beschaeftigtenbefragung.verdi.de/",
tag: str = "bez_data_2",
):
df = construct_dataframe(url=url, tag=tag, grouped=False)
output_str = []
def _print_as_html(df: pd.DataFrame):
df = df.astype({"Digitale Befragung": "Int32"})
with pd.option_context("display.max_rows", None):
table = df.to_html(
index_names=False,
justify="left",
index=False,
classes="sortable dataframe",
)
tfoot = [
" <tfoot>",
" <td>Gesamt</td>",
]
for i in range(len(df.columns) - 2):
tfoot.append(" <td/>")
tfoot.extend(
[
f" <td>{df['Digitale Befragung'].sum()}</td>",
" </tr>",
" </tfoot>",
]
)
tfoot = "\n".join(tfoot)
idx = table.index("</table>")
output_str.append(table[: idx - 1])
output_str.append(tfoot)
output_str.append(table[idx:])
_print_as_html(df)
_print_as_html(
df.groupby("Bundesland", as_index=False)[["Digitale Befragung"]].sum()
)
return render_template("base.html", tables="\n".join(output_str))

56
download_digital.py
View File

@ -71,13 +71,11 @@ bundesland_dict = {
}
def main(
def construct_dataframe(
url: str = "https://beschaeftigtenbefragung.verdi.de/",
tag: str = "bez_data_2",
dry_run: bool = False,
grouped: bool = False,
html: bool = False,
) -> pd.DataFrame:
):
r = requests.get(url)
soup = BeautifulSoup(r.text, "html.parser")
@ -109,49 +107,19 @@ def main(
if grouped:
df = df.groupby("Bundesland", as_index=False)[["Digitale Befragung"]].sum()
if dry_run or html:
if html:
print("<link rel='stylesheet' href='style.css'>")
print("<script src='sorttable.js'></script>")
return df
def _print_as_html(df: pd.DataFrame):
df = df.astype({"Digitale Befragung": "Int32"})
with pd.option_context("display.max_rows", None):
if html:
table = df.to_html(
index_names=False,
justify="left",
index=False,
classes="sortable dataframe",
)
tfoot = [
" <tfoot>",
" <td>Gesamt</td>",
]
for i in range(len(df.columns) - 2):
tfoot.append(" <td/>")
tfoot.extend(
[
f" <td>{df['Digitale Befragung'].sum()}</td>",
" </tr>",
" </tfoot>",
]
)
tfoot = "\n".join(tfoot)
idx = table.index("</table>")
print(table[: idx - 1])
print(tfoot)
print(table[idx:])
else:
print(df)
_print_as_html(df)
if html:
_print_as_html(
df.groupby("Bundesland", as_index=False)[["Digitale Befragung"]].sum()
)
def main(
url: str = "https://beschaeftigtenbefragung.verdi.de/",
tag: str = "bez_data_2",
dry_run: bool = False,
grouped: bool = False,
) -> None:
df = construct_dataframe(url=url, tag=tag, grouped=grouped)
if dry_run:
print(df)
else:
filename = f"data/{datetime.today().strftime('%Y-%m-%d')}_data.ods"
if Path(filename).exists():