Merge pull request #318 from openzim/upgrade_deps

Upgrade dependencies

.pre-commit-config.yaml

@@ -7,15 +7,15 @@ repos:
   -   id: trailing-whitespace
   -   id: end-of-file-fixer
 - repo: https://github.com/psf/black
-  rev: "24.2.0"
+  rev: "24.4.2"
   hooks:
   -   id: black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.3.0
+  rev: v0.4.8
   hooks:
   - id: ruff
 - repo: https://github.com/RobertCraigie/pyright-python
-  rev: v1.1.352
+  rev: v1.1.367
   hooks:
   - id: pyright
     name: pyright (system)

CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Upgrade dependencies (especially warc2zim 2.0.1 and browsertrix crawler 1.2.0-beta.0) (#318)
+
+### Fixed
+
+- Crawler is not correctly checking disk size / usage (#305)
+
 ## [2.0.0] - 2024-06-04
 
 ### Added

Dockerfile

@@ -1,4 +1,4 @@
-FROM webrecorder/browsertrix-crawler:1.1.3
+FROM webrecorder/browsertrix-crawler:1.2.0-beta.0
 LABEL org.opencontainers.image.source https://github.com/openzim/zimit
 
 # add deadsnakes ppa for Python 3.12 on Ubuntu Jammy

pyproject.toml

@@ -8,36 +8,33 @@ requires-python = ">=3.12,<3.13"
 description = "Make ZIM file from any website through crawling"
 readme = "README.md"
 dependencies = [
-  "requests==2.31.0",
+  "requests==2.32.3",
   "inotify==0.2.10",
   "tld==0.13",
-  "warc2zim @ git+https://github.com/openzim/warc2zim@main",
+  "warc2zim==2.0.1",
 ]
 dynamic = ["authors", "classifiers", "keywords", "license", "version", "urls"]
 
 [tool.hatch.metadata.hooks.openzim-metadata]
 kind = "scraper"
 
-[tool.hatch.metadata]
-allow-direct-references = true  # to be removed once we use a released warc2zim version
-
 [project.optional-dependencies]
 scripts = [
   "invoke==2.2.0",
 ]
 lint = [
-  "black==24.2.0",
-  "ruff==0.3.0",
+  "black==24.4.2",
+  "ruff==0.4.8",
 ]
 check = [
-  "pyright==1.1.352",
+  "pyright==1.1.367",
 ]
 test = [
-  "pytest==8.0.2",
-  "coverage==7.4.3",
+  "pytest==8.2.2",
+  "coverage==7.5.3",
 ]
 dev = [
-  "pre-commit==3.6.2",
+  "pre-commit==3.7.1",
   "debugpy==1.8.1",
   "zimit[scripts]",
   "zimit[lint]",