Updated test and stats to new crawl.json format

This commit is contained in:
renaud gaudin 2023-01-31 11:12:36 +00:00
parent 554fff5c87
commit 4b7e504d99
2 changed files with 4 additions and 3 deletions

View File

@ -47,8 +47,9 @@ def test_user_agent():
def test_stats_output():
with open("/output/crawl.json") as fh:
assert json.loads(fh.read()) == {
"numCrawled": 5,
"workersRunning": 0,
"crawled": 5,
"pending": 0,
"pendingPages": [],
"total": 5,
"limit": {"max": 0, "hit": False},
}

View File

@ -79,7 +79,7 @@ class ProgressFileWatcher:
limit.max = data["limit"]["max"]
limit.hit = data["limit"]["hit"]
return {
"done": data["numCrawled"],
"done": data["crawled"],
"total": int(data["total"] / 0.9),
"limit": limit.as_dict,
}