PlaggerがわからないのでDeferredで
なんかplugableな感じにしたいんだけどって考えてたらdeferredでいい気がして来た。
関数の組み合わせでなんでもできるようになりそうな気がする。
数珠つなぎで実行して結果を渡していくんであればdeferred使えばいいじゃん。
例えばこんなの。
core/__init__.py
from twisted.web import client from twisted.internet.defer import Deferred import os, os.path, types def parse_mod_func(callback): dot = callback.rindex('.') return callback[:dot], callback[dot+1:] def get_mod_func(calback): mod_name, func_name = parse_mod_func(callback) return getattr(__import__(mod_name, {}, {}, ['']), func_name) def create_units(funcs, *args, **kwargs): d = Deferred() d = add_units(d, funcs, *args, **kwargs) return Units(d) def create_web_units(funcs, url, method='GET', cookies=None, postdata=None, headers=None, *args, **kwargs): d = client.getPage(url, method=method, cookies=cookies, postdata=postdata, headers=headers) args = (url,)+args if cookies: kwargs['cookies'] = cookies if method: kwargs['method'] = method if postdata: kwargs['postdata'] = postdata if headers: kwargs['headers'] = headers d = add_units(d, funcs, *args, **kwargs) return Units(d) def add_units(deferred, funcs, *args, **kwargs): func_list = parse_func(funcs) for func in func_list: deferred.addCallback(func, *args, **kwargs) return deferred def parse_func(funcs): list = [] for func in funcs: if type(func) == types.StringType: func = get_mod_func(func) list.append(func) return list class Units(object): def __init__(self, deferred): self.deferred = deferred def execute(self, *args, **kwargs): self.deferred.callback(*args, **kwargs)
基本的にdeferredを周りを作るコアに対し結果を処理するのが関数(unit)。
unit/net.py
from BeautifulSoup import * from types import * import core, cookielib, urllib, feedparser def make_cookie(ignore): cookie = cookielib.CookieJar() return cookie def load_cookie(file): cookie = make_cookie(0) cookie.load(file) return cookie def get_data(data, url, *args, **kwargs): return data def get_links(data, url, *args, **kwargs): soup = BeautifulSoup(data) hrefs = soup.findAll(href=True) links = [] for href in hrefs: link = href['href'] if link: links.append(str(link).encode()) return links def create_routing(functions, method='GET', cookies=None, postdata=None, headers=None): def _routing(links, url, method='GET', cookies=cookies, postdata=postdata, headers=headers, *args, **kwargs): print cookies if type(links) == ListType: for link in links: core.create_web_units(functions, link, method, cookies, postdata, headers, *args, **kwargs) else: core.create_web_units(functions, links, method, cookies, postdata, headers, *args, **kwargs) return _routing def craate_sync_routing def _routing(links, url, method='GET', cookies=cookies, postdata=postdata, headers=headers, *args, **kwargs): defer = None if type(links) == ListType: for link in links: if not defer: defer = core.create_web_units(functions, link, method, cookies, postdata, headers, *args, **kwargs) else: def _callcack(data, url, *args, **kwargs): defer = core.create_web_units(functions, link, method, cookies, postdata, headers, *args, **kwargs) defer.addCallback(_callcack) else: core.create_web_units(functions, links, method, cookies, postdata, headers, *args, **kwargs) return _routing def get_feed(data, url, *args, **kwargs): feeds = [] d = feedparser.parse(data) charset = d.encording version = d.version for entry in d.entries: value = None if version = "atom": value = entry.content[0].value elif entry.has_key('description'): value = entry.description else: pass link = None if entry.has_key("link"): link = entry.link else: link = entry.id feeds.append((encode.title, link, value)) return feeds
ネット周りの関数。
こいつらは用途別にunitパッケージに放り込む。
これを数珠つなぎに実行していけばいいだけなので数珠つなぎに読み込むloaderをこんな感じで作る。
loader.py
import os import os.path import settings import core def load_module(): plugins = settings.plugins plugins_module = [] for plugin in plugins: plugins_module.append(__import__(plugin, {}, {}, [''])) return plugins_module def create_units(): plugins = load_module() for p in plugins: _create_unit(p) def _create_unit(plugin): if hasattr(plugin, 'plugin_name'): kwargs = {} for k in dir(plugin): if k.startswith('arg'): key = k[k.rindex('_')+1:] kwargs[key] = getattr(plugin, k) functions = getattr(plugin, 'main_units') if kwargs.has_key('url'): url = kwargs['url'] del kwargs['url'] core.create_web_units(functions, url, **kwargs) else: core.create_units(functions, **kwargs)
loaderは基本のsetting.pyの内容、さらにunitをまとめたplugin単位で関数をまとめてから実行する。
pluginはこんな感じで
plugin/example.py
from unit.net import * from unit.simple import * plugin_name = 'example' arg_url = 'http://localhost/' arg_method = 'GET' arg_cookies = {} link_units=( get_data, debug_net_file('/tmp'), ) main_units=( get_data, get_links, create_routing(link_units ), )
setting.py
plugins = (
'plugin.example',
)
loaderはmain_unitのタプルから情報を読み込む。
この例だと
- dataを返す
- dataからlinkを取得し、返す
- linkを元に更にlinkをたどるdeferredを作る
って感じ。
メインの実行はこんな感じ。
定期実行はTimerServiceで。
間隔は固定だけど。
from twisted.application.internet import TimerService from twisted.internet import reactor from twisted.web import client import core import loader import settings class Service(object): def __init__(self): pass def main_loop(self): loader.create_units() def start(self): service = TimerService(60*15, self.main_loop) reactor.callLater(1, service.startService) reactor.run() def stop(self): if reactor.running: reactor.stop() if __name__ == '__main__': Service().start()
とまあこんな感じで実行。
うくく。
