kiwix/zimit
1
0
Fork 0
mirror of https://github.com/openzim/zimit.git synced 2025-09-22 11:22:23 -04:00
Code Issues Packages Projects Releases Wiki Activity

try using regular puppeteer, only copy deps from chrome image

Browse Source 
pywb: increase uwsgi processes, disable autoindex/autofetch for better perf
This commit is contained in:
Ilya Kreymer 2020-09-22 06:09:33 +00:00
parent b00c4262a7
commit f252245983
6 changed files with 17 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Dockerfile
View File

@ -10,16 +10,16 @@ ENV PROXY_HOST=localhost \
RUN pip install pywb uwsgi warc2zim RUN pip install pywb uwsgi warc2zim
COPY --from=chrome /opt/google/chrome/ /opt/google/chrome/ #COPY --from=chrome /opt/google/chrome/ /opt/google/chrome/
COPY --from=chrome /app/ /browser/ #COPY --from=chrome /app/ /browser/
COPY --from=chrome /usr/lib/x86_64-linux-gnu/ /usr/lib/x86_64-linux-gnu/ COPY --from=chrome /usr/lib/x86_64-linux-gnu/ /usr/lib/x86_64-linux-gnu/
COPY --from=chrome /lib/x86_64-linux-gnu/libdbus* /lib/x86_64-linux-gnu/ COPY --from=chrome /lib/x86_64-linux-gnu/libdbus* /lib/x86_64-linux-gnu/
COPY --from=chrome /usr/bin/run_forever /usr/bin/ #COPY --from=chrome /usr/bin/run_forever /usr/bin/
COPY --from=chrome /usr/bin/wget /usr/bin/ #COPY --from=chrome /usr/bin/wget /usr/bin/
COPY --from=chrome /usr/bin/certutil /usr/bin/ #COPY --from=chrome /usr/bin/certutil /usr/bin/
RUN ln -s /opt/google/chrome/google-chrome /usr/bin/google-chrome #RUN ln -s /opt/google/chrome/google-chrome /usr/bin/google-chrome
RUN useradd zimit --shell /bin/bash --create-home \ RUN useradd zimit --shell /bin/bash --create-home \
&& usermod -a -G sudo zimit \ && usermod -a -G sudo zimit \

4
config.yaml
View File

@ -9,9 +9,9 @@ proxy:
recorder: live recorder: live
autoindex: 10 #autoindex: 10
enable_auto_fetch: true #enable_auto_fetch: true
collections: collections:
live: $live live: $live

6
index.js
View File

@ -1,4 +1,4 @@
const puppeteer = require("puppeteer-core"); const puppeteer = require("puppeteer");
const { Cluster } = require("puppeteer-cluster"); const { Cluster } = require("puppeteer-cluster");
const child_process = require("child_process"); const child_process = require("child_process");
const fetch = require("node-fetch"); const fetch = require("node-fetch");
@ -20,14 +20,14 @@ async function run(params) {
// Puppeter Options // Puppeter Options
const puppeteerOptions = { const puppeteerOptions = {
headless: true, headless: true,
executablePath: "/usr/bin/google-chrome", //executablePath: "/usr/bin/google-chrome",
ignoreHTTPSErrors: true, ignoreHTTPSErrors: true,
args args
}; };
// Puppeteer Cluster init and options // Puppeteer Cluster init and options
const cluster = await Cluster.launch({ const cluster = await Cluster.launch({
concurrency: Cluster.CONCURRENCY_CONTEXT, concurrency: Cluster.CONCURRENCY_PAGE,
maxConcurrency: Number(params.workers) || 1, maxConcurrency: Number(params.workers) || 1,
skipDuplicateUrls: true, skipDuplicateUrls: true,
puppeteerOptions, puppeteerOptions,

2
package.json
View File

@ -8,8 +8,8 @@
"dependencies": { "dependencies": {
"abort-controller": "^3.0.0", "abort-controller": "^3.0.0",
"node-fetch": "^2.6.1", "node-fetch": "^2.6.1",
"puppeteer": "^5.3.0",
"puppeteer-cluster": "^0.22.0", "puppeteer-cluster": "^0.22.0",
"puppeteer-core": "^5.3.0",
"yargs": "^16.0.3" "yargs": "^16.0.3"
} }
} }

3
uwsgi.ini
View File

@ -12,13 +12,14 @@ if-env = VIRTUAL_ENV
venv = $(VIRTUAL_ENV) venv = $(VIRTUAL_ENV)
endif = endif =
gevent = 100 gevent = 200
#Not available until uwsgi 2.1 #Not available until uwsgi 2.1
#monkey-patching manually in pywb.apps.wayback #monkey-patching manually in pywb.apps.wayback
#gevent-early-monkey-patch = #gevent-early-monkey-patch =
# for uwsgi<2.1, set env when using gevent # for uwsgi<2.1, set env when using gevent
env = GEVENT_MONKEY_PATCH=1 env = GEVENT_MONKEY_PATCH=1
processes = 8
# specify config file here # specify config file here
env = PYWB_CONFIG_FILE=config.yaml env = PYWB_CONFIG_FILE=config.yaml

6
yarn.lock
View File

@ -347,10 +347,10 @@ puppeteer-cluster@^0.22.0:
dependencies: dependencies:
debug "^4.1.1" debug "^4.1.1"
puppeteer-core@^5.3.0: puppeteer@^5.3.0:
version "5.3.0" version "5.3.0"
resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-5.3.0.tgz#36d2e10132632c9cb73007f3f2880f4e6b655977" resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-5.3.0.tgz#0abf83d0f2d1273baf2b56885a813f8052903e33"
integrity sha512-+4wk+0dcDNg7AQqN41Q9r41U6iltAtknuVBI0aj0O/Vp8/4orgbFV0wn55wV5xRae//CucLPUnaczxZx7dz0UA== integrity sha512-GjqMk5GRro3TO0sw3QMsF1H7n+/jaK2OW45qMvqjYUyJ7y4oA//9auy969HHhTG3HZXaMxY/NWXF/NXlAFIvtw==
dependencies: dependencies:
debug "^4.1.0" debug "^4.1.0"
devtools-protocol "0.0.799653" devtools-protocol "0.0.799653"