Merge pull request #2 from Natim/read_from_settings

Read settings from config.
This commit is contained in:
Alexis Metaireau 2016-01-10 12:16:43 +01:00
commit 146bd32f63
3 changed files with 36 additions and 20 deletions

4
.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
*.pyc
__pycache__
*.zim
*.egg-info/

View File

@ -1,14 +1,9 @@
import subprocess
import shlex
from pyramid.config import Configurator
from pyramid.events import NewRequest
from redis import Redis
from rq import Queue
def main(global_config, **settings):
config = Configurator(settings=settings)
config.registry.queue = Queue(connection=Redis())

View File

@ -1,39 +1,53 @@
import tempfile
import subprocess
import os
import shlex
import shutil
import subprocess
import tempfile
from cornice import Service
from colander import MappingSchema, SchemaNode, String, drop
from colander import MappingSchema, SchemaNode, String
zimwriterfs_bin = "/home/alexis/dev/openzim/zimwriterfs/zimwriterfs"
httrack_bin = "/usr/bin/httrack"
default_author = "Alexis Metaireau"
HTTRACK_BIN = "/usr/bin/httrack"
DEFAULT_AUTHOR = "BSF"
def spawn(cmd):
print cmd
return subprocess.Popen(shlex.split(cmd))
def zim_it(config):
location = download_website(config['url'])
def zim_it(config, settings):
location = download_website(config['url'], settings)
create_zim(location, config)
def download_website(url):
def download_website(url, settings):
httrack_bin = settings.get('zimit.httrack_bin', HTTRACK_BIN)
if not os.path.exists(httrack_bin):
raise OSError('%s does not exist.' % httrack_bin)
path = tempfile.mkdtemp("website")
p = spawn("%s --path %s %s" % (httrack_bin, path, url))
p.wait()
shutil.copy('/home/alexis/dev/zimit/favicon.ico', path)
shutil.copy('./favicon.ico', path)
return path
def create_zim(location, config):
def create_zim(location, config, settings):
if 'zimit.zimwriterfs_bin' not in settings:
raise ValueError('Please define zimit.zimwriterfs_bin config.')
if not os.path.exists(settings['zimit.zimwriterfs_bin']):
raise OSError('%s does not exist.' % settings['zimit.zimwriterfs_bin'])
config.update({
'bin': zimwriterfs_bin,
'bin': settings['zimit.zimwriterfs_bin'],
'location': location,
'output': 'test.zim',
'icon': 'favicon.ico',
'publisher': 'Alexis Metaireau',
'publisher': settings.get('zimit.default_author', DEFAULT_AUTHOR),
})
# Spawn zimwriterfs with the correct options.
p = spawn(('{bin} -w "{welcome}" -l "{language}" -t "{title}"'
@ -41,13 +55,14 @@ def create_zim(location, config):
' -p "{publisher}" {location} {output}').format(**config))
p.wait()
class WebSiteSchema(MappingSchema):
url = SchemaNode(String(), location="body", type='str')
title = SchemaNode(String(), location="body", type='str')
email = SchemaNode(String(), location="body", type='str')
description = SchemaNode(String(), default="-",
location="body", type='str')
author = SchemaNode(String(), default=default_author,
author = SchemaNode(String(), default=None,
location="body", type='str')
welcome = SchemaNode(String(), default="index.html",
location="body", type='str')
@ -57,8 +72,10 @@ class WebSiteSchema(MappingSchema):
webpage = Service(name='website', path='/website')
@webpage.post(schema=WebSiteSchema)
def crawl_new_website(request):
request.queue.enqueue(zim_it, request.validated, timeout=1800)
request.queue.enqueue(zim_it, request.validated,
request.registry.settings, timeout=1800)
request.response.status_code = 201
return {'success': True}