mirror of
https://github.com/openzim/zimit.git
synced 2025-09-24 04:30:11 -04:00
Merge pull request #2 from Natim/read_from_settings
Read settings from config.
This commit is contained in:
commit
146bd32f63
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
*.pyc
|
||||
__pycache__
|
||||
*.zim
|
||||
*.egg-info/
|
@ -1,14 +1,9 @@
|
||||
import subprocess
|
||||
import shlex
|
||||
|
||||
from pyramid.config import Configurator
|
||||
from pyramid.events import NewRequest
|
||||
from redis import Redis
|
||||
from rq import Queue
|
||||
|
||||
|
||||
|
||||
|
||||
def main(global_config, **settings):
|
||||
config = Configurator(settings=settings)
|
||||
config.registry.queue = Queue(connection=Redis())
|
||||
|
@ -1,39 +1,53 @@
|
||||
import tempfile
|
||||
import subprocess
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from cornice import Service
|
||||
from colander import MappingSchema, SchemaNode, String, drop
|
||||
from colander import MappingSchema, SchemaNode, String
|
||||
|
||||
zimwriterfs_bin = "/home/alexis/dev/openzim/zimwriterfs/zimwriterfs"
|
||||
httrack_bin = "/usr/bin/httrack"
|
||||
default_author = "Alexis Metaireau"
|
||||
HTTRACK_BIN = "/usr/bin/httrack"
|
||||
DEFAULT_AUTHOR = "BSF"
|
||||
|
||||
|
||||
def spawn(cmd):
|
||||
print cmd
|
||||
return subprocess.Popen(shlex.split(cmd))
|
||||
|
||||
def zim_it(config):
|
||||
location = download_website(config['url'])
|
||||
|
||||
def zim_it(config, settings):
|
||||
location = download_website(config['url'], settings)
|
||||
create_zim(location, config)
|
||||
|
||||
def download_website(url):
|
||||
|
||||
def download_website(url, settings):
|
||||
httrack_bin = settings.get('zimit.httrack_bin', HTTRACK_BIN)
|
||||
|
||||
if not os.path.exists(httrack_bin):
|
||||
raise OSError('%s does not exist.' % httrack_bin)
|
||||
|
||||
path = tempfile.mkdtemp("website")
|
||||
p = spawn("%s --path %s %s" % (httrack_bin, path, url))
|
||||
p.wait()
|
||||
shutil.copy('/home/alexis/dev/zimit/favicon.ico', path)
|
||||
shutil.copy('./favicon.ico', path)
|
||||
|
||||
return path
|
||||
|
||||
def create_zim(location, config):
|
||||
|
||||
def create_zim(location, config, settings):
|
||||
if 'zimit.zimwriterfs_bin' not in settings:
|
||||
raise ValueError('Please define zimit.zimwriterfs_bin config.')
|
||||
|
||||
if not os.path.exists(settings['zimit.zimwriterfs_bin']):
|
||||
raise OSError('%s does not exist.' % settings['zimit.zimwriterfs_bin'])
|
||||
|
||||
config.update({
|
||||
'bin': zimwriterfs_bin,
|
||||
'bin': settings['zimit.zimwriterfs_bin'],
|
||||
'location': location,
|
||||
'output': 'test.zim',
|
||||
'icon': 'favicon.ico',
|
||||
'publisher': 'Alexis Metaireau',
|
||||
'publisher': settings.get('zimit.default_author', DEFAULT_AUTHOR),
|
||||
})
|
||||
# Spawn zimwriterfs with the correct options.
|
||||
p = spawn(('{bin} -w "{welcome}" -l "{language}" -t "{title}"'
|
||||
@ -41,13 +55,14 @@ def create_zim(location, config):
|
||||
' -p "{publisher}" {location} {output}').format(**config))
|
||||
p.wait()
|
||||
|
||||
|
||||
class WebSiteSchema(MappingSchema):
|
||||
url = SchemaNode(String(), location="body", type='str')
|
||||
title = SchemaNode(String(), location="body", type='str')
|
||||
email = SchemaNode(String(), location="body", type='str')
|
||||
description = SchemaNode(String(), default="-",
|
||||
location="body", type='str')
|
||||
author = SchemaNode(String(), default=default_author,
|
||||
author = SchemaNode(String(), default=None,
|
||||
location="body", type='str')
|
||||
welcome = SchemaNode(String(), default="index.html",
|
||||
location="body", type='str')
|
||||
@ -57,8 +72,10 @@ class WebSiteSchema(MappingSchema):
|
||||
|
||||
webpage = Service(name='website', path='/website')
|
||||
|
||||
|
||||
@webpage.post(schema=WebSiteSchema)
|
||||
def crawl_new_website(request):
|
||||
request.queue.enqueue(zim_it, request.validated, timeout=1800)
|
||||
request.queue.enqueue(zim_it, request.validated,
|
||||
request.registry.settings, timeout=1800)
|
||||
request.response.status_code = 201
|
||||
return {'success': True}
|
||||
|
Loading…
x
Reference in New Issue
Block a user