diff options
author | Alexander Shorin <kxepal@apache.org> | 2014-11-15 18:09:06 +0300 |
---|---|---|
committer | Alexander Shorin <kxepal@apache.org> | 2014-11-15 18:12:50 +0300 |
commit | f2a5c33b072ed204710f9efd168170a32dcd4798 (patch) | |
tree | 3455cd1b6cbf360ed144ae0145ae09e200ab311e | |
parent | 701bf2ee8ee7103be2cf99fda50ba016e24e4713 (diff) | |
download | couchdb-f2a5c33b072ed204710f9efd168170a32dcd4798.tar.gz |
Make Python scripts compatible with both 2.x and 3.x series
-rwxr-xr-x | dev/run | 403 | ||||
-rwxr-xr-x | test/javascript/run | 25 |
2 files changed, 224 insertions, 204 deletions
@@ -13,61 +13,166 @@ # the License. import atexit -import contextlib as ctx +import contextlib +import functools import glob -import httplib -import optparse as op +import inspect +import optparse import os import re -import select import subprocess as sp import sys import time -import traceback -import urllib import uuid from pbkdf2 import pbkdf2_hex -# clipped down from e.g. '0x594fc30efe7746318d7d79684a15cfd0L' -COMMON_SALT = hex(uuid.uuid4().int)[2:-1] +COMMON_SALT = uuid.uuid4().hex -USAGE = "%prog [options] [command to run...]" -DEV_PATH = os.path.dirname(os.path.abspath(__file__)) -COUCHDB = os.path.dirname(DEV_PATH) +try: + from urllib import urlopen +except ImportError: + from urllib.request import urlopen -DEFAULT_N = 3 -PROCESSES = [] +try: + import httplib as httpclient +except ImportError: + import http.client as httpclient -def init_log_dir(): - logdir = os.path.join(DEV_PATH, "logs") - if not os.path.exists(logdir): - os.makedirs(logdir) +def log(msg): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + callargs = dict(zip(inspect.getargspec(func).args, args)) + callargs.update(kwargs) + sys.stdout.write(msg.format(**callargs) + '... ') + sys.stdout.flush() + try: + res = func(*args, **kwargs) + except: + sys.stdout.write('failed\n') + raise + else: + sys.stdout.write('ok\n') + return res + finally: + sys.stdout.flush() + return wrapper + return decorator + + +def main(): + ctx = setup() + startup(ctx) + if ctx['cmd']: + run_command(ctx['cmd']) + else: + join(ctx) + + +def setup(): + opts, args = setup_argparse() + ctx = setup_context(opts, args) + setup_dirs(ctx) + check_beams(ctx) + setup_configs(ctx) + return ctx + + +def setup_argparse(): + parser = optparse.OptionParser(description='Runs CouchDB 2.0 dev cluster') + parser.add_option('-a', '--admin', metavar='USER:PASS', default=None, + help="Add an admin account to the development cluster") + parser.add_option("-n", "--nodes", metavar="nodes", default=3, + type=int, + help="Number of development nodes to be spun up") + return parser.parse_args() + + +def setup_context(opts, args): + fpath = os.path.abspath(__file__) + return {'N': opts.nodes, + 'admin': opts.admin, + 'nodes': ['node%d' % (i + 1) for i in range(opts.nodes)], + 'devdir': os.path.dirname(fpath), + 'rootdir': os.path.dirname(os.path.dirname(fpath)), + 'cmd': ' '.join(args), + 'procs': []} + + +@log('Setup environment') +def setup_dirs(ctx): + ensure_dir_exists(ctx['devdir'], 'data') + ensure_dir_exists(ctx['devdir'], 'logs') -def init_beams(): - # Including this for people that forget to run - # make dev. - for fname in glob.glob(os.path.join(DEV_PATH, "*.erl")): - cmd = [ - "erlc", - "-o", DEV_PATH + os.sep, - fname - ] - sp.check_call(cmd) +def ensure_dir_exists(root, *segments): + path = os.path.join(root, *segments) + if not os.path.exists(path): + os.makedirs(path) + return path -def hack_default_ini(opts, node, args, contents): +@log('Ensure CouchDB is built') +def check_beams(ctx): + for fname in glob.glob(os.path.join(ctx['devdir'], "*.erl")): + sp.check_call(["erlc", "-o", ctx['devdir'] + os.sep, fname]) + + +@log('Prepare configuration files') +def setup_configs(ctx): + for idx, node in enumerate(ctx['nodes']): + cluster_port, backend_port = get_ports(idx) + env = { + "prefix": ctx['rootdir'], + "package_author_name": "The Apache Software Foundation", + "data_dir": ensure_dir_exists("lib", node, "data"), + "view_index_dir": ensure_dir_exists("lib", node, "data"), + "node_name": "-name %s@127.0.0.1" % node, + "cluster_port": cluster_port, + "backend_port": backend_port + } + write_config(ctx, node, env) + + +def get_ports(idnode): + return ((10000 * idnode) + 5984, (10000 * idnode) + 5986) + + +def write_config(ctx, node, env): + etc_src = os.path.join(ctx['rootdir'], "rel", "overlay", "etc") + etc_tgt = ensure_dir_exists(ctx['devdir'], "lib", node, "etc") + + for fname in glob.glob(os.path.join(etc_src, "*")): + base = os.path.basename(fname) + tgt = os.path.join(etc_tgt, base) + + with open(fname) as handle: + content = handle.read() + + for key in env: + content = re.sub("{{%s}}" % key, str(env[key]), content) + + if base == "default.ini": + content = hack_default_ini(ctx, node, content) + elif base == "local.ini": + content = hack_local_ini(ctx, content) + + with open(tgt, "w") as handle: + handle.write(content) + + +def hack_default_ini(ctx, node, content): # Replace log file - logfile = os.path.join(DEV_PATH, "logs", "%s.log" % node) + logfile = os.path.join(ctx['devdir'], "logs", "%s.log" % node) repl = "file = %s" % logfile - contents = re.sub("(?m)^file.*$", repl, contents) + contents = re.sub("(?m)^file.*$", repl, content) # Replace couchjs command - couchjs = os.path.join(COUCHDB, "src", "couch", "priv", "couchjs") - mainjs = os.path.join(COUCHDB, "share", "server", "main.js") - coffeejs = os.path.join(COUCHDB, "share", "server", "main-coffee.js") + couchjs = os.path.join(ctx['rootdir'], "src", "couch", "priv", "couchjs") + mainjs = os.path.join(ctx['rootdir'], "share", "server", "main.js") + coffeejs = os.path.join(ctx['rootdir'], "share", "server", "main-coffee.js") repl = "javascript = %s %s" % (couchjs, mainjs) contents = re.sub("(?m)^javascript.*$", repl, contents) @@ -78,9 +183,22 @@ def hack_default_ini(opts, node, args, contents): return contents -def hashify(pwd, salt=COMMON_SALT): +def hack_local_ini(ctx, contents): + # make sure all three nodes have the same secret + secret_line = "secret = %s\n" % COMMON_SALT + previous_line = "; require_valid_user = false\n" + contents = contents.replace(previous_line, previous_line + secret_line) + # if --admin user:password on invocation, make sure all three nodes + # have the same hashed password + if ctx['admin'] is None: + return contents + usr, pwd = ctx['admin'].split(":", 1) + return contents + "\n%s = %s" % (usr, hashify(pwd)) + + +def hashify(pwd, salt=COMMON_SALT, iterations=10, keylen=20): """ - Implements password hasshing according to: + Implements password hashing according to: - https://issues.apache.org/jira/browse/COUCHDB-1060 - https://issues.apache.org/jira/secure/attachment/12492631/0001-Integrate-PBKDF2.patch @@ -89,72 +207,44 @@ def hashify(pwd, salt=COMMON_SALT): >>> hashify(candeira) -pbkdf2-99eb34d97cdaa581e6ba7b5386e112c265c5c670,d1d2d4d8909c82c81b6c8184429a0739,10 """ - iterations = 10 - keylen = 20 derived_key = pbkdf2_hex(pwd, salt, iterations, keylen) return "-pbkdf2-%s,%s,%s" % (derived_key, salt, iterations) -def hack_local_ini(opts, node, args, contents): - # make sure all three nodes have the same secret - secret_line = "secret = %s\n" % COMMON_SALT - previous_line = "; require_valid_user = false\n" - contents = contents.replace(previous_line, previous_line + secret_line) - # if --admin user:password on invocation, make sure all three nodes - # have the same hashed password - if opts.admin is None: - return contents - usr, pwd = opts.admin.split(":", 1) - return contents + "\n%s = %s" % (usr, hashify(pwd)) +def startup(ctx): + atexit.register(kill_processes, ctx) + boot_nodes(ctx) + join_nodes("127.0.0.1", 15986, ctx) -def write_config(opts, node, args): - etc_src = os.path.join(COUCHDB, "rel", "overlay", "etc") - etc_tgt = os.path.join(DEV_PATH, "lib", node, "etc") - if not os.path.exists(etc_tgt): - os.makedirs(etc_tgt) - etc_files = glob.glob(os.path.join(etc_src, "*")) - for fname in etc_files: - base = os.path.basename(fname) - tgt = os.path.join(etc_tgt, base) - with open(fname) as handle: - contents = handle.read() - for key in args: - contents = re.sub("{{%s}}" % key, args[key], contents) - if base == "default.ini": - contents = hack_default_ini(opts, node, args, contents) - elif base == "local.ini": - contents = hack_local_ini(opts, node, args, contents) - with open(tgt, "w") as handle: - handle.write(contents) +def kill_processes(ctx): + for proc in ctx['procs']: + if proc.returncode is None: + proc.kill() -def write_configs(opts): - datadir = os.path.join(DEV_PATH, "data") - if not os.path.exists(datadir): - os.makedirs(datadir) - for i in range(1, N+1): - node = "node%d" % i - args = { - "prefix": COUCHDB, - "package_author_name": "The Apache Software Foundation", - "data_dir": os.path.join(DEV_PATH, "lib", node, "data"), - "view_index_dir": os.path.join(DEV_PATH, "lib", node, "data"), - "node_name": "-name %s@127.0.0.1" % node, - "cluster_port": str((10000 * i) + 5984), - "backend_port" : str((10000 * i) + 5986) - } - if not os.path.exists(args["data_dir"]): - os.makedirs(args["data_dir"]) - write_config(opts, node, args) +def boot_nodes(ctx): + for node in ctx['nodes']: + ctx['procs'].append(boot_node(node, ctx)) + + ensure_nodes_stated(ctx) + +@log('Ensure all nodes are run') +def ensure_nodes_stated(ctx): + for _ in range(30): + if all_nodes_alive(ctx): + break + time.sleep(1) -def all_nodes_alive(n): - for i in range(1, n+1): - url = "http://127.0.0.1:{0}/".format(local_port(i)) + +def all_nodes_alive(ctx): + for num in range(ctx['N']): + local_port, _ = get_ports(num) + url = "http://127.0.0.1:{0}/".format(local_port) while True: try: - with ctx.closing(urllib.urlopen(url)) as resp: + with contextlib.closing(urlopen(url)): pass except IOError: time.sleep(0.25) @@ -163,132 +253,65 @@ def all_nodes_alive(n): return True -def local_port(n): - return 10000 * n + 5986 - - -def node_port(n): - return 10000 * n + 5984 - - -def boot_node(node): - apps = os.path.join(COUCHDB, "src") +@log('Start node {node}') +def boot_node(node, ctx): + erl_libs = os.path.join(ctx['rootdir'], "src") env = os.environ.copy() - env["ERL_LIBS"] = os.pathsep.join([apps]) + env["ERL_LIBS"] = os.pathsep.join([erl_libs]) + + node_etcdir = os.path.join(ctx['devdir'], "lib", node, "etc") + reldir = os.path.join(ctx['rootdir'], "rel") cmd = [ "erl", - "-args_file", os.path.join(DEV_PATH, "lib", node, "etc", "vm.args"), - "-config", os.path.join(COUCHDB, "rel", "files", "sys"), + "-args_file", os.path.join(node_etcdir, "vm.args"), + "-config", os.path.join(reldir, "files", "sys"), "-couch_ini", - os.path.join(DEV_PATH, "lib", node, "etc", "default.ini"), - os.path.join(DEV_PATH, "lib", node, "etc", "local.ini"), - "-reltool_config", os.path.join(COUCHDB, "rel", "reltool.config"), + os.path.join(node_etcdir, "default.ini"), + os.path.join(node_etcdir, "local.ini"), + "-reltool_config", os.path.join(reldir, "reltool.config"), "-parent_pid", str(os.getpid()), - "-pa", DEV_PATH, - "-pa", os.path.join(COUCHDB, "src", "*"), + "-pa", ctx['devdir'], + "-pa", os.path.join(erl_libs, "*"), "-s", "boot_node" ] - logfname = os.path.join(DEV_PATH, "logs", "%s.log" % node) - log = open(logfname, "w") - return sp.Popen( - cmd, - stdin=sp.PIPE, - stdout=log, - stderr=sp.STDOUT, - env=env) - - -def connect_nodes(host, port): - global N - for i in range(1, N+1): + logfname = os.path.join(ctx['devdir'], "logs", "%s.log" % node) + log = open(logfname, "wb") + return sp.Popen(cmd, stdin=sp.PIPE, stdout=log, stderr=sp.STDOUT, env=env) + + +@log('Join nodes into cluster') +def join_nodes(host, port, ctx): + for node in ctx['nodes']: body = "{}" - conn = httplib.HTTPConnection(host, port) - conn.request("PUT", "/nodes/node%d@127.0.0.1" % i, body) + conn = httpclient.HTTPConnection(host, port) + conn.request("PUT", "/nodes/%s@127.0.0.1" % node, body) resp = conn.getresponse() if resp.status not in (200, 201, 202, 409): - print resp.reason + print('Failed to join %s into cluster' % node, resp.reason) exit(1) -def kill_processes(): - global PROCESSES - for p in PROCESSES: - if p.returncode is None: - p.kill() - - -def boot_nodes(): - global N, PROCESSES - for i in range(1, N+1): - p = boot_node("node%d" % i) - PROCESSES.append(p) - - for i in range(30): - if all_nodes_alive(N): - break - time.sleep(1) - - -def reboot_nodes(): - kill_processes() - boot_nodes() +@log('Developers cluster is set up at http://127.0.0.1:15984. Time to hack!') +def join(ctx): + while True: + for proc in ctx['procs']: + if proc.returncode is not None: + exit(1) + time.sleep(2) +@log('Exec command {cmd}') def run_command(cmd): p = sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sys.stderr) while True: line = p.stdout.readline() if not line: break - try: - eval(line) - except: - traceback.print_exc() - exit(1) + eval(line) p.wait() exit(p.returncode) -def wait_for_procs(): - global PROCESSES - while True: - for p in PROCESSES: - if p.returncode is not None: - exit(1) - time.sleep(2) - - -def options(): - return [ - op.make_option("-a", "--admin", metavar="USER:PASS", default=None, - help="Add an admin account to the development cluster"), - op.make_option("-n", "--nodes", metavar="N", default=DEFAULT_N, - type="int", help="Number of development nodes to be spun up") - ] - - -def main(): - parser = op.OptionParser(usage=USAGE, option_list=options()) - opts, args = parser.parse_args() - - global N - N = opts.nodes - - init_log_dir() - init_beams() - write_configs(opts) - - atexit.register(kill_processes) - - boot_nodes() - connect_nodes("127.0.0.1", 15986) - - if len(args): - run_command(" ".join(args)) - else: - wait_for_procs() - - if __name__ == "__main__": try: main() diff --git a/test/javascript/run b/test/javascript/run index 883fd37c1..2b6b382c2 100755 --- a/test/javascript/run +++ b/test/javascript/run @@ -12,17 +12,11 @@ # License for the specific language governing permissions and limitations under # the License. -import atexit -import contextlib as ctx import glob import optparse as op import os -import re -import select import subprocess as sp import sys -import time -import urllib USAGE = "%prog [options] [command to run...]" @@ -53,13 +47,15 @@ def mkformatter(tests): clear = "\033[0m" if not sys.stderr.isatty(): green, read, clear = "", "", "" + def _colorized(passed): if passed: return green + "pass" + clear else: return red + "fail" + clear + def _fmt(test): - if isinstance(test, basestring): + if isinstance(test, str): padding = (longest - len(test)) * " " sys.stderr.write(test + " " + padding) sys.stderr.flush() @@ -69,6 +65,7 @@ def mkformatter(tests): else: sys.stderr.write(_colorized(test) + os.linesep) sys.stderr.flush() + return _fmt @@ -76,11 +73,11 @@ def run_couchjs(test, fmt): fmt(test) cmd = [COUCHJS, "-H"] + SCRIPTS + [test, RUNNER] p = sp.Popen( - cmd, - stdin = sp.PIPE, - stdout = sp.PIPE, - stderr = sys.stderr - ) + cmd, + stdin=sp.PIPE, + stdout=sp.PIPE, + stderr=sys.stderr + ) while True: line = p.stdout.readline() if not line: @@ -97,7 +94,8 @@ def run_couchjs(test, fmt): def options(): return [ op.make_option("-s", "--start", metavar="FILENAME", default=None, - help="Start from the given filename if multiple files are passed") + help="Start from the given filename if multiple files " + "are passed") ] @@ -135,7 +133,6 @@ def main(): run_couchjs(test, fmt) - if __name__ == "__main__": try: main() |