kiwix/zimit
1
0
Fork 0
mirror of https://github.com/openzim/zimit.git synced 2025-09-22 11:22:23 -04:00
Code Issues Packages Projects Releases Wiki Activity

try using regular puppeteer, only copy deps from chrome image

Browse Source 
pywb: increase uwsgi processes, disable autoindex/autofetch for better perf
This commit is contained in:
Ilya Kreymer 2020-09-22 06:09:33 +00:00
parent b00c4262a7
commit f252245983
6 changed files with 17 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Dockerfile
View File

@ -10,16 +10,16 @@ ENV PROXY_HOST=localhost \
RUN pip install pywb uwsgi warc2zim
COPY --from=chrome /opt/google/chrome/ /opt/google/chrome/
#COPY --from=chrome /opt/google/chrome/ /opt/google/chrome/
COPY --from=chrome /app/ /browser/
#COPY --from=chrome /app/ /browser/
COPY --from=chrome /usr/lib/x86_64-linux-gnu/ /usr/lib/x86_64-linux-gnu/
COPY --from=chrome /lib/x86_64-linux-gnu/libdbus* /lib/x86_64-linux-gnu/
COPY --from=chrome /usr/bin/run_forever /usr/bin/
COPY --from=chrome /usr/bin/wget /usr/bin/
COPY --from=chrome /usr/bin/certutil /usr/bin/
#COPY --from=chrome /usr/bin/run_forever /usr/bin/
#COPY --from=chrome /usr/bin/wget /usr/bin/
#COPY --from=chrome /usr/bin/certutil /usr/bin/
RUN ln -s /opt/google/chrome/google-chrome /usr/bin/google-chrome
#RUN ln -s /opt/google/chrome/google-chrome /usr/bin/google-chrome
RUN useradd zimit --shell /bin/bash --create-home \
&& usermod -a -G sudo zimit \

4
config.yaml
View File

@ -9,9 +9,9 @@ proxy:
recorder: live
autoindex: 10
#autoindex: 10
enable_auto_fetch: true
#enable_auto_fetch: true
collections:
live: $live

6
index.js
View File

@ -1,4 +1,4 @@
const puppeteer = require("puppeteer-core");
const puppeteer = require("puppeteer");
const { Cluster } = require("puppeteer-cluster");
const child_process = require("child_process");
const fetch = require("node-fetch");
@ -20,14 +20,14 @@ async function run(params) {
// Puppeter Options
const puppeteerOptions = {
headless: true,
executablePath: "/usr/bin/google-chrome",
//executablePath: "/usr/bin/google-chrome",
ignoreHTTPSErrors: true,
args
};
// Puppeteer Cluster init and options
const cluster = await Cluster.launch({
concurrency: Cluster.CONCURRENCY_CONTEXT,
concurrency: Cluster.CONCURRENCY_PAGE,
maxConcurrency: Number(params.workers) || 1,
skipDuplicateUrls: true,
puppeteerOptions,

2
package.json
View File

@ -8,8 +8,8 @@
"dependencies": {
"abort-controller": "^3.0.0",
"node-fetch": "^2.6.1",
"puppeteer": "^5.3.0",
"puppeteer-cluster": "^0.22.0",
"puppeteer-core": "^5.3.0",
"yargs": "^16.0.3"
}
}

3
uwsgi.ini
View File

@ -12,13 +12,14 @@ if-env = VIRTUAL_ENV
venv = $(VIRTUAL_ENV)
endif =
gevent = 100
gevent = 200
#Not available until uwsgi 2.1
#monkey-patching manually in pywb.apps.wayback
#gevent-early-monkey-patch =
# for uwsgi<2.1, set env when using gevent
env = GEVENT_MONKEY_PATCH=1
processes = 8
# specify config file here
env = PYWB_CONFIG_FILE=config.yaml

6
yarn.lock
View File

@ -347,10 +347,10 @@ puppeteer-cluster@^0.22.0:
dependencies:
debug "^4.1.1"
puppeteer-core@^5.3.0:
puppeteer@^5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-5.3.0.tgz#36d2e10132632c9cb73007f3f2880f4e6b655977"
integrity sha512-+4wk+0dcDNg7AQqN41Q9r41U6iltAtknuVBI0aj0O/Vp8/4orgbFV0wn55wV5xRae//CucLPUnaczxZx7dz0UA==
resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-5.3.0.tgz#0abf83d0f2d1273baf2b56885a813f8052903e33"
integrity sha512-GjqMk5GRro3TO0sw3QMsF1H7n+/jaK2OW45qMvqjYUyJ7y4oA//9auy969HHhTG3HZXaMxY/NWXF/NXlAFIvtw==
dependencies:
debug "^4.1.0"
devtools-protocol "0.0.799653"