From 84965018a4f5c71bf777e700c3dc07d34a79eaf5 Mon Sep 17 00:00:00 2001 From: Daniel Pierce Date: Tue, 23 Jan 2024 00:04:55 -0500 Subject: [PATCH] Major update - Update bottle.py - Use logging instead of print (fixes buffered output) - Catch failed callback errors - New config format allowing multiple tags and with regexp matching - Fixed tests --- .github/workflows/python-app.yml | 4 +- README.md | 11 ++- bottle.py | 127 +++++++++++++++++++++---------- example.config | 14 ++++ forklift.config.example | 9 --- forklift.py | 47 +++++++++--- requirements.txt | 2 +- test_forklift.py | 17 +++-- 8 files changed, 160 insertions(+), 71 deletions(-) create mode 100644 example.config delete mode 100644 forklift.config.example diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 4786eef..985409e 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -16,10 +16,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.11 uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.11 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/README.md b/README.md index d490df8..7be53c5 100644 --- a/README.md +++ b/README.md @@ -9,16 +9,21 @@ Listener for Docker Hub webhooks. `python forklift.py` ## Configuration -- Copy `forklift.config.example` to `forklift.config` +- Copy `example.config` to `forklift.config` - Set a secure `api_key` - Set `docker_root` to the directory that the target script is run relative to. - Setup `valid_containers` - Top level key is `{repository name}/{image_name}`. - - `target` points to a script to run inside `docker_root`. - - `tag` allows only the specified tag to trigger the `target` script. + - `tag` allows only the specified tag to trigger the `target` script. This is interpreted as a regular expression. + - `target` is a path relative to `docker_root` to the script to run. +- Create a Docker Hub webhook that calls `https://your_host/hook?apikey={your_apikey}` ## Development `pip install -r requirements_dev.txt` +### Design Notes +The shell command used to restart a container should use only strings from the program configuration, +and must not contain any fragments sourced from the request data. + ### Running Tests `pytest` diff --git a/bottle.py b/bottle.py index 3a51b38..54ee51d 100644 --- a/bottle.py +++ b/bottle.py @@ -16,7 +16,7 @@ from __future__ import with_statement __author__ = 'Marcel Hellkamp' -__version__ = '0.12.13' +__version__ = '0.12.25' __license__ = 'MIT' # The gevent server adapter needs to patch some modules before they are imported @@ -35,13 +35,12 @@ if _cmd_options.server and _cmd_options.server.startswith('gevent'): import gevent.monkey; gevent.monkey.patch_all() -import base64, cgi, email.utils, functools, hmac, imp, itertools, mimetypes,\ - os, re, subprocess, sys, tempfile, threading, time, warnings +import base64, cgi, email.utils, functools, hmac, itertools, mimetypes,\ + os, re, subprocess, sys, tempfile, threading, time, warnings, hashlib from datetime import date as datedate, datetime, timedelta from tempfile import TemporaryFile from traceback import format_exc, print_exc -from inspect import getargspec from unicodedata import normalize @@ -84,10 +83,21 @@ def _e(): return sys.exc_info()[1] from urllib.parse import urlencode, quote as urlquote, unquote as urlunquote urlunquote = functools.partial(urlunquote, encoding='latin1') from http.cookies import SimpleCookie - from collections import MutableMapping as DictMixin + if py >= (3, 3, 0): + from collections.abc import MutableMapping as DictMixin + from types import ModuleType as new_module + else: + from collections import MutableMapping as DictMixin + from imp import new_module import pickle from io import BytesIO from configparser import ConfigParser + from inspect import getfullargspec + def getargspec(func): + spec = getfullargspec(func) + kwargs = makelist(spec[0]) + makelist(spec.kwonlyargs) + return kwargs, spec[1], spec[2], spec[3] + basestring = str unicode = str json_loads = lambda s: json_lds(touni(s)) @@ -102,8 +112,10 @@ def _raise(*a): raise a[0](a[1]).with_traceback(a[2]) from Cookie import SimpleCookie from itertools import imap import cPickle as pickle + from imp import new_module from StringIO import StringIO as BytesIO from ConfigParser import SafeConfigParser as ConfigParser + from inspect import getargspec if py25: msg = "Python 2.5 support may be dropped in future versions of Bottle." warnings.warn(msg, DeprecationWarning) @@ -296,7 +308,7 @@ def add_filter(self, name, func): rule_syntax = re.compile('(\\\\*)'\ '(?:(?::([a-zA-Z_][a-zA-Z_0-9]*)?()(?:#(.*?)#)?)'\ '|(?:<([a-zA-Z_][a-zA-Z_0-9]*)?(?::([a-zA-Z_]*)'\ - '(?::((?:\\\\.|[^\\\\>]+)+)?)?)?>))') + '(?::((?:\\\\.|[^\\\\>])+)?)?)?>))') def _itertokens(self, rule): offset, prefix = 0, '' @@ -434,7 +446,7 @@ def match(self, environ): nocheck = set(methods) for method in set(self.static) - nocheck: if path in self.static[method]: - allowed.add(verb) + allowed.add(method) for method in set(self.dyna_regexes) - allowed - nocheck: for combined, rules in self.dyna_regexes[method]: match = combined(path) @@ -553,7 +565,7 @@ def get_callback_args(self): def get_config(self, key, default=None): ''' Lookup a config field and return its value, first checking the route.config, then route.app.config.''' - for conf in (self.config, self.app.conifg): + for conf in (self.config, self.app.config): if key in conf: return conf[key] return default @@ -842,17 +854,19 @@ def default_error_handler(self, res): return tob(template(ERROR_PAGE_TEMPLATE, e=res)) def _handle(self, environ): - path = environ['bottle.raw_path'] = environ['PATH_INFO'] - if py3k: - try: - environ['PATH_INFO'] = path.encode('latin1').decode('utf8') - except UnicodeError: - return HTTPError(400, 'Invalid path string. Expected UTF-8') - try: + environ['bottle.app'] = self request.bind(environ) response.bind() + + path = environ['bottle.raw_path'] = environ['PATH_INFO'] + if py3k: + try: + environ['PATH_INFO'] = path.encode('latin1').decode('utf8') + except UnicodeError: + return HTTPError(400, 'Invalid path string. Expected UTF-8') + try: self.trigger_hook('before_request') route, args = self.router.match(environ) @@ -1081,6 +1095,7 @@ def forms(self): :class:`FormsDict`. All keys and values are strings. File uploads are stored separately in :attr:`files`. """ forms = FormsDict() + forms.recode_unicode = self.POST.recode_unicode for name, item in self.POST.allitems(): if not isinstance(item, FileUpload): forms[name] = item @@ -1104,6 +1119,7 @@ def files(self): """ files = FormsDict() + files.recode_unicode = self.POST.recode_unicode for name, item in self.POST.allitems(): if isinstance(item, FileUpload): files[name] = item @@ -1229,15 +1245,16 @@ def POST(self): newline='\n') elif py3k: args['encoding'] = 'utf8' + post.recode_unicode = False data = cgi.FieldStorage(**args) self['_cgi.FieldStorage'] = data #http://bugs.python.org/issue18394#msg207958 data = data.list or [] for item in data: - if item.filename: + if item.filename is None: + post[item.name] = item.value + else: post[item.name] = FileUpload(item.file, item.name, item.filename, item.headers) - else: - post[item.name] = item.value return post @property @@ -1557,7 +1574,7 @@ def iter_headers(self): @property def headerlist(self): - ''' WSGI conform list of (header, value) tuples. ''' + """ WSGI conform list of (header, value) tuples. """ out = [] headers = list(self._headers.items()) if 'Content-Type' not in self._headers: @@ -1565,10 +1582,12 @@ def headerlist(self): if self._status_code in self.bad_headers: bad_headers = self.bad_headers[self._status_code] headers = [h for h in headers if h[0] not in bad_headers] - out += [(name, val) for name, vals in headers for val in vals] + out += [(name, val) for (name, vals) in headers for val in vals] if self._cookies: for c in self._cookies.values(): - out.append(('Set-Cookie', c.OutputString())) + out.append(('Set-Cookie', _hval(c.OutputString()))) + if py3k: + out = [(k, v.encode('utf8').decode('latin1')) for (k, v) in out] return out content_type = HeaderProperty('Content-Type') @@ -1738,7 +1757,7 @@ def apply(self, callback, route): def wrapper(*a, **ka): try: rv = callback(*a, **ka) - except HTTPError: + except HTTPResponse: rv = _e() if isinstance(rv, dict): @@ -1779,7 +1798,7 @@ def __init__(self, name, impmask): ''' Create a virtual package that redirects imports (see PEP 302). ''' self.name = name self.impmask = impmask - self.module = sys.modules.setdefault(name, imp.new_module(name)) + self.module = sys.modules.setdefault(name, new_module(name)) self.module.__dict__.update({'__file__': __file__, '__path__': [], '__all__': [], '__loader__': self}) sys.meta_path.append(self) @@ -2577,7 +2596,7 @@ def parse_range_header(header, maxlen=0): def _parse_qsl(qs): r = [] - for pair in qs.replace(';','&').split('&'): + for pair in qs.split('&'): if not pair: continue nv = pair.split('=', 1) if len(nv) != 2: nv.append('') @@ -2595,7 +2614,7 @@ def _lscmp(a, b): def cookie_encode(data, key): ''' Encode and sign a pickle-able object. Return a (byte) string ''' msg = base64.b64encode(pickle.dumps(data, -1)) - sig = base64.b64encode(hmac.new(tob(key), msg).digest()) + sig = base64.b64encode(hmac.new(tob(key), msg, digestmod=hashlib.md5).digest()) return tob('!') + sig + tob('?') + msg @@ -2604,7 +2623,7 @@ def cookie_decode(data, key): data = tob(data) if cookie_is_encoded(data): sig, msg = data.split(tob('?'), 1) - if _lscmp(sig[1:], base64.b64encode(hmac.new(tob(key), msg).digest())): + if _lscmp(sig[1:], base64.b64encode(hmac.new(tob(key), msg, digestmod=hashlib.md5).digest())): return pickle.loads(base64.b64decode(msg)) return None @@ -2681,6 +2700,7 @@ def auth_basic(check, realm="private", text="Access denied"): ''' Callback decorator to require HTTP auth (basic). TODO: Add route(check_auth=...) parameter. ''' def decorator(func): + @functools.wraps(func) def wrapper(*a, **ka): user, password = request.auth or (None, None) if user is None or not check(user, password): @@ -2784,7 +2804,11 @@ class server_cls(server_cls): class CherryPyServer(ServerAdapter): def run(self, handler): # pragma: no cover - from cherrypy import wsgiserver + depr("The wsgi server part of cherrypy was split into a new " + "project called 'cheroot'. Use the 'cheroot' server " + "adapter instead of cherrypy.") + from cherrypy import wsgiserver # This will fail for CherryPy >= 9 + self.options['bind_addr'] = (self.host, self.port) self.options['wsgi_app'] = handler @@ -2807,6 +2831,25 @@ def run(self, handler): # pragma: no cover server.stop() +class CherootServer(ServerAdapter): + def run(self, handler): # pragma: no cover + from cheroot import wsgi + from cheroot.ssl import builtin + self.options['bind_addr'] = (self.host, self.port) + self.options['wsgi_app'] = handler + certfile = self.options.pop('certfile', None) + keyfile = self.options.pop('keyfile', None) + chainfile = self.options.pop('chainfile', None) + server = wsgi.Server(**self.options) + if certfile and keyfile: + server.ssl_adapter = builtin.BuiltinSSLAdapter( + certfile, keyfile, chainfile) + try: + server.start() + finally: + server.stop() + + class WaitressServer(ServerAdapter): def run(self, handler): from waitress import serve @@ -2830,7 +2873,7 @@ def run(self, handler): class FapwsServer(ServerAdapter): - """ Extremely fast webserver using libev. See http://www.fapws.org/ """ + """ Extremely fast webserver using libev. See https://github.com/william-os4y/fapws3 """ def run(self, handler): # pragma: no cover import fapws._evwsgi as evwsgi from fapws import base, config @@ -2904,14 +2947,16 @@ class GeventServer(ServerAdapter): * See gevent.wsgi.WSGIServer() documentation for more options. """ def run(self, handler): - from gevent import wsgi, pywsgi, local + from gevent import pywsgi, local if not isinstance(threading.local(), local.local): msg = "Bottle requires gevent.monkey.patch_all() (before import)" raise RuntimeError(msg) - if not self.options.pop('fast', None): wsgi = pywsgi - self.options['log'] = None if self.quiet else 'default' + if self.options.pop('fast', None): + depr('The "fast" option has been deprecated and removed by Gevent.') + if self.quiet: + self.options['log'] = None address = (self.host, self.port) - server = wsgi.WSGIServer(address, handler, **self.options) + server = pywsgi.WSGIServer(address, handler, **self.options) if 'BOTTLE_CHILD' in os.environ: import signal signal.signal(signal.SIGINT, lambda s, f: server.stop()) @@ -2972,7 +3017,9 @@ def run(self, handler): class AutoServer(ServerAdapter): """ Untested. """ - adapters = [WaitressServer, PasteServer, TwistedServer, CherryPyServer, WSGIRefServer] + adapters = [WaitressServer, PasteServer, TwistedServer, CherryPyServer, + CherootServer, WSGIRefServer] + def run(self, handler): for sa in self.adapters: try: @@ -2986,6 +3033,7 @@ def run(self, handler): 'wsgiref': WSGIRefServer, 'waitress': WaitressServer, 'cherrypy': CherryPyServer, + 'cheroot': CherootServer, 'paste': PasteServer, 'fapws3': FapwsServer, 'tornado': TornadoServer, @@ -3154,7 +3202,7 @@ def run(self): files = dict() for module in list(sys.modules.values()): - path = getattr(module, '__file__', '') + path = getattr(module, '__file__', '') or '' if path[-4:] in ('.pyo', '.pyc'): path = path[:-1] if path and exists(path): files[path] = mtime(path) @@ -3418,7 +3466,7 @@ class StplParser(object): _re_cache = {} #: Cache for compiled re patterns # This huge pile of voodoo magic splits python code into 8 different tokens. # 1: All kinds of python strings (trust me, it works) - _re_tok = '((?m)[urbURB]?(?:\'\'(?!\')|""(?!")|\'{6}|"{6}' \ + _re_tok = '([urbURB]?(?:\'\'(?!\')|""(?!")|\'{6}|"{6}' \ '|\'(?:[^\\\\\']|\\\\.)+?\'|"(?:[^\\\\"]|\\\\.)+?"' \ '|\'{3}(?:[^\\\\]|\\\\.|\\n)+?\'{3}' \ '|"{3}(?:[^\\\\]|\\\\.|\\n)+?"{3}))' @@ -3426,8 +3474,8 @@ class StplParser(object): # 2: Comments (until end of line, but not the newline itself) _re_tok += '|(#.*)' # 3,4: Open and close grouping tokens - _re_tok += '|([\[\{\(])' - _re_tok += '|([\]\}\)])' + _re_tok += '|([\\[\\{\\(])' + _re_tok += '|([\\]\\}\\)])' # 5,6: Keywords that start or continue a python block (only start of line) _re_tok += '|^([ \\t]*(?:if|for|while|with|try|def|class)\\b)' \ '|^([ \\t]*(?:elif|else|except|finally)\\b)' @@ -3441,7 +3489,8 @@ class StplParser(object): # Match the start tokens of code areas in a template _re_split = '(?m)^[ \t]*(\\\\?)((%(line_start)s)|(%(block_start)s))(%%?)' # Match inline statements (may contain python strings) - _re_inl = '%%(inline_start)s((?:%s|[^\'"\n]*?)+)%%(inline_end)s' % _re_inl + _re_inl = '(?m)%%(inline_start)s((?:%s|[^\'"\n])*?)%%(inline_end)s' % _re_inl + _re_tok = '(?m)' + _re_tok default_syntax = '<% %> % {{ }}' @@ -3642,7 +3691,7 @@ def wrapper(*args, **kwargs): tplvars.update(result) return template(tpl_name, **tplvars) elif result is None: - return template(tpl_name, defaults) + return template(tpl_name, **defaults) return result return wrapper return decorator diff --git a/example.config b/example.config new file mode 100644 index 0000000..b276c0a --- /dev/null +++ b/example.config @@ -0,0 +1,14 @@ +[bottle] +debug = False + +[forklift] +port = 8080 +api_key = 12345 +docker_root = /srv/docker +valid_containers = {'docker_user/image_name': + [{'tag': 'latest', 'target': 'target_dir/update'}, + {'tag': 'main', 'target': 'target_dir/update'}, + {'tag': 'test', 'target': 'test_dir/update'}], + 'other_user/other_name': + [{'tag': 'stable-.+', 'target': 'other_target_dir/update'}] + } diff --git a/forklift.config.example b/forklift.config.example deleted file mode 100644 index 9d2969b..0000000 --- a/forklift.config.example +++ /dev/null @@ -1,9 +0,0 @@ -[bottle] -debug = False - -[forklift] -port = 8080 -api_key = 12345 -docker_root = /srv/docker -valid_containers = {'docker_user/image_name': {'target': 'target_dir/update', 'tag': 'latest'}, - 'other_user/other_name': {'target': 'other_target_dir/update', 'tag': 'stable-0.1-rc1'}} diff --git a/forklift.py b/forklift.py index cae56f7..6df0ca5 100644 --- a/forklift.py +++ b/forklift.py @@ -1,5 +1,5 @@ import bottle -import sys, os, subprocess, urllib.request +import sys, os, subprocess, logging, re, urllib.request from bottle import get, post, request, abort, run from json import dumps from ast import literal_eval @@ -7,6 +7,7 @@ sys.version_info.major >= 3 or sys.exit('ERROR: Python 3 required') app = bottle.default_app() app.config.load_config('./forklift.config') +logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(message)s') @get('/status') def status(): @@ -15,30 +16,52 @@ def status(): @post('/hook') def hook(): request.params.apikey == app.config['forklift.api_key'] or abort(403, 'Forbidden') - print("Request params: {}".format(dumps(request.params.dict))) + logging.debug("Request params: {}".format(dumps(request.params.dict))) params = request.json or abort(400, 'Params not found') - params['push_data']['tag'] == 'main' or abort(304, 'Not modified') + + # Get data from webhook data + tag = params['push_data']['tag'] repo_name = params['repository']['repo_name'] - container = validate_container(repo_name) - container or abort(404, "Valid container {}:{} not found".format(repo_name, container)) + + # Verify a configuration exists for this repo_name and tag combo + container = validate_container(repo_name, tag) + container or abort(404, "Valid container {}/{}:{} not found".format(repo_name, container, tag)) + + # Send a callback request validate_webhook(params.get('callback_url'), 'success') + + # Exec the restart command output = restart(container) or abort(500, 'Restart failed') return "OK\n{}".format(output) -def validate_container(key): - return literal_eval(app.config['forklift.valid_containers'])[key] +def validate_container(repo_name, tag): + try: + container = literal_eval(app.config['forklift.valid_containers'])[repo_name] + target = [t['target'] for t in container if re.compile(t['tag']).fullmatch(tag)][0] + except (IndexError, KeyError) as e: + logging.warning("Invalid container {}:{}".format(repo_name, tag)) + return False + else: + return target def validate_webhook(url, state): if url == None: return data = dumps({'state':state}).encode() req = urllib.request.Request(url=url, data=data, method='POST') - print("Validating webhook: {} => {}".format(state, url)) - return urllib.request.urlopen(req) + logging.debug("Validating webhook: {} => {}".format(state, url)) + try: + callback_resp = urllib.request.urlopen(req) + except URLError as e: + logging.warning("Callback webhook ({}) failed: {}".format(url, e.reason)) + finally: + return callback_resp def restart(container_name): - cmd = "{}/{}/bin/update".format(app.config['forklift.docker_root'], container_name) - print("Running command: {}".format(cmd)) + logging.info("Restarting {}".format(container_name)) + cmd = "{}/{}".format(app.config['forklift.docker_root'], container_name) + logging.debug("Running command: {}".format(cmd)) return subprocess.check_output(cmd) # return os.spawnl(os.P_NOWAIT, cmd) -run(host='0.0.0.0', port=app.config['forklift.port'], debug=False, reloader=False) +if __name__ == '__main__': + run(host='0.0.0.0', port=app.config['forklift.port'], debug=False, reloader=False) diff --git a/requirements.txt b/requirements.txt index f2a6868..4e045a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -bottle ~= 0.12.19 +bottle ~= 0.12.25 diff --git a/test_forklift.py b/test_forklift.py index f397a19..ccaa8c5 100644 --- a/test_forklift.py +++ b/test_forklift.py @@ -3,7 +3,7 @@ import subprocess forklift_app = forklift.app -forklift_app.config.load_config('./forklift.config.example') +forklift_app.config.load_config('./example.config') app = WebTestApp(forklift_app) @@ -25,7 +25,7 @@ def test_hook_no_json(): def test_hook_tag_mismatch(): data = {'push_data': {'tag': 'badtag'}, 'repository': {'repo_name': 'docker_user/image_name'}} - assert app.post_json('/hook?apikey=12345', data, expect_errors=True).status == '304 Not Modified' + assert app.post_json('/hook?apikey=12345', data, expect_errors=True).status == '404 Not Found' def test_hook_invalid_container(): @@ -47,7 +47,14 @@ def test_hook_success(mocker): subprocess.check_output.assert_called_once_with('/srv/docker/target_dir/update') -def test_hook_success_other(mocker): +def test_hook_success_alt_tag(mocker): + mocker.patch('subprocess.check_output', return_value="Mocked subprocess call success") + data = {'push_data': {'tag': 'main'}, 'repository': {'repo_name': 'docker_user/image_name'}} + assert app.post_json('/hook?apikey=12345', data).status == '200 OK' + subprocess.check_output.assert_called_once_with('/srv/docker/target_dir/update') + + +def test_hook_success_other_regexp(mocker): mocker.patch('subprocess.check_output', return_value="Mocked subprocess call success") data = {'push_data': {'tag': 'stable-0.1-rc1'}, 'repository': {'repo_name': 'other_user/other_name'}} assert app.post_json('/hook?apikey=12345', data).status == '200 OK' @@ -55,5 +62,5 @@ def test_hook_success_other(mocker): def test_validate_container(): - assert not forklift.validate_container('bad_docker_user/image_name') - assert forklift.validate_container('docker_user/image_name')['target'] == 'target_dir/update' + assert not forklift.validate_container('bad_docker_user/image_name', 'latest') + assert forklift.validate_container('docker_user/image_name', 'latest') == 'target_dir/update'