-
Notifications
You must be signed in to change notification settings - Fork 0
/
scrape.py
78 lines (68 loc) · 2.24 KB
/
scrape.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
# %%
import urllib.request
import markdownify
import pandas as pd
from bs4 import BeautifulSoup
# %%
url = (
"https://www.oryxspioenkop.com/2022/02/attack-on-europe-documenting-equipment.html"
)
html = urllib.request.urlopen(url)
soup = BeautifulSoup(html, "html.parser")
# %%
content = markdownify.markdownify(str(soup.find_all("h3")))
content = (
content[1:-1]
.replace("\n\n,", "\n\n")
.replace("### ", "")
.replace("Russia - ", "\n \n ## Russia\n \n ### Russia - ")
.replace("Ukraine - ", "\n \n ## Ukraine\n \n ### Ukraine - ")
)
# %%
with open("losses.md", "w") as f:
f.write(content)
with open("losses.html", "w") as f:
f.write(str(soup))
# %%
try:
# create markdown table
losses = {}
for line in content.split("\n"):
if "Ukraine" in line:
break
if "Russia" in line:
pass
elif len(line) > 2:
line_split = line.split("(")
if len(line_split) > 2:
vehicle_type = line_split[0] + " (" + line_split[1]
vehicle_type = vehicle_type.strip()
else:
vehicle_type = line_split[0].strip()
losses[vehicle_type] = {}
for i in ("destroyed: ", "damaged: ", "abandoned: ", "captured: "):
if i in line:
try:
str_splits = (
line.split(i)[1] # split on type
.split(",")[0]
.replace(")", "")
.strip()
# if there are still strings after the number
.split(" ")[0]
)
losses[vehicle_type][i.replace(": ", "")] = int(str_splits)
except ValueError as e:
print(e)
raise e
else:
losses[vehicle_type][i.replace(": ", "")] = 0
df = pd.DataFrame(losses).T
df["total"] = df.sum(axis=1)
df.loc["total"] = df.sum()
df.index.rename("Russian Losses", inplace=True)
with open("losses_table.md", "w") as f:
f.write(df.to_markdown())
except Exception as e:
print("error", e)
# %%