kiwix/zimit
1
0
Fork 0
mirror of https://github.com/openzim/zimit.git synced 2025-09-22 11:22:23 -04:00
Code Issues Packages Projects Releases Wiki Activity

Tidy code automatically

Browse Source
This commit is contained in:
benoit74 2023-11-23 08:50:59 +01:00
parent 216ac09d8c
commit 79d5f8bc7b
No known key found for this signature in database
GPG Key ID: B89606434FC7B530
1 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
zimit.py
View File

@ -30,6 +30,7 @@ from zimscraperlib.uri import rebuild_uri
DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
class ProgressFileWatcher:
def __init__(self, output_dir, stats_path):
self.crawl_path = output_dir / "crawl.json"
@ -153,9 +154,7 @@ def zimit(args=None):
type=int,
)
parser.add_argument(
"--limit", help="Limit crawl to this number of pages", type=int
)
parser.add_argument("--limit", help="Limit crawl to this number of pages", type=int)
parser.add_argument(
"--maxPageLimit",
@ -226,7 +225,7 @@ def zimit(args=None):
parser.add_argument(
"--userAgent",
help="Override default user-agent with specified value ; --userAgentSuffix is still applied",
default=DEFAULT_USER_AGENT
default=DEFAULT_USER_AGENT,
)
parser.add_argument(
@ -447,7 +446,11 @@ def check_url(url, user_agent, scope=None):
url = urllib.parse.urlparse(url)
try:
with requests.get(
url.geturl(), stream=True, allow_redirects=True, timeout=(12.2, 27), headers={"User-Agent": user_agent}
url.geturl(),
stream=True,
allow_redirects=True,
timeout=(12.2, 27),
headers={"User-Agent": user_agent},
) as resp:
resp.raise_for_status()
except requests.exceptions.RequestException as exc: