blob: 0a2bbae42bee232b8abb9563f8feed7a9dff4e73 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
|
import argparse
import json
import re
import sys
import time
import requests
KILL_HREF = re.compile(r"href=\"/kill/(\d+)/\"")
SYSTEM_AND_DATE = re.compile(r"/(\d+)/(\d+)/$")
def unique_kills_in(page):
so_far = set()
for match in KILL_HREF.finditer(page):
kill = match.group(1)
if kill not in so_far:
so_far.add(kill)
yield kill
def get_hash(kill):
response = requests.get("https://zkillboard.com/api/killID/{}/".format(kill))
response.raise_for_status()
data = response.json()
if len(data) > 1:
raise ValueError()
return data[0]["zkb"]["hash"]
def main():
parser = argparse.ArgumentParser()
parser.add_argument("url")
parser.add_argument("-o", "--output")
args = parser.parse_args()
response = requests.get(args.url)
response.raise_for_status()
page = response.text
output = []
for kill in unique_kills_in(page):
time.sleep(1.05) # Zkillboard is very sensitive.
output.append({"id": kill, "hash": get_hash(kill)})
if args.output:
filename = args.output
else:
match = SYSTEM_AND_DATE.search(args.url)
if match:
filename = "{}_{}.json".format(*match.groups())
else:
filename = "scrapped.json"
with open(filename, "w") as file:
file.write(json.dumps(output))
if __name__ == "__main__":
main()
|
