diff --git a/.travis.yml b/.travis.yml index 134a1030..e582fd17 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ python: - "3.4" - "3.5" install: - - "pip install -r requirements.txt" - - "pip install -r requirements-dev.txt" + - "pip install -r requirements/developer.pip" script: - "py.test ./tests/test_style.py" diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000..1bae3265 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,11 @@ +# Flintrock Change Log + + +## Unreleased + +* Nothing yet. + + +## 0.1.0 - 2015-12-11 + +* Initial release. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d92dd126..94901f1f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,19 +1,16 @@ # Contributing Guide -*Note: This guide was inspired in part by the excellent [Phabricator contributor docs](https://secure.phabricator.com/book/phabcontrib/).* - -**Flintrock is still under heavy development and has no releases at this time.** - -**Until we make an initial release, most PRs will be rejected.** We are likely already fixing or making that thing you want to open a PR for, and major components are rapidly evolving so your PR is likely to be invalidated by upcoming changes you may not know about. - +There are many ways to contribute to Flintrock. ## Contributing Thanks -When we put our time and enthusiasm into an open source project like this, we hope that somewhere out there we are helping people by saving them time and frustration. But most of the time, we'll never know what kind of impact we made, if any, on people's lives. +When we put our time and enthusiasm into an open source project like this, we hope that somewhere out there we are putting a smile on someone's face. + +Most of the time we'll never know, though. When people reach out within an open source community, it's typically to report a problem, ask for help, or share an idea. -Hearing first-hand of the positive impact we had on someone else's day, even if it is minor, can be a huge boost of joy and motivation. +That's a bummer, because hearing first-hand that we made a positive impact on someone else's day, even if it's minor, can be a huge boost of joy and motivation. -Don't underestimate the power of a thank you. If Flintrock helped you in some way, share your story, no matter how long or short, and know that at times it can be the most valuable way to contribute to the project. +Don't underestimate the power of a thank you. If Flintrock helped you in some way, share your story, even if it's "trivial", and know that at times this can be the most valuable way to contribute to the project. ## Contributing Money diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..98745d47 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,10 @@ +# See: https://docs.python.org/3/distutils/commandref.html +graft flintrock + +include README.md +include CHANGELOG.md +include COPYRIGHT +include LICENSE + +global-exclude *.py[cod] __pycache__ .DS_Store +global-exclude config.yaml diff --git a/README.md b/README.md index 6e0504ec..37953b76 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -![Flintrock logo](flintrock-logo.png) +![Flintrock logo](https://raw.githubusercontent.com/nchammas/flintrock/master/flintrock-logo.png) [![Build Status](https://travis-ci.org/nchammas/flintrock.svg)](https://travis-ci.org/nchammas/flintrock) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/nchammas/flintrock) Flintrock is a command-line tool and library for launching [Apache Spark](http://spark.apache.org/) clusters. -**Flintrock is currently undergoing heavy development. Until we make a 1.0 release, you probably should not use Flintrock unless you are ready to keep up with frequent changes to how it works.** Python hackers or heavy spark-ec2 users who are looking to experiment with something new are welcome to try Flintrock out and potentially even [contribute](CONTRIBUTING.md). +**Flintrock is currently undergoing heavy development. Until we make a 1.0 release, you probably should not use Flintrock unless you are ready to keep up with frequent changes to how it works.** Python hackers or heavy spark-ec2 users who are looking to experiment with something new are welcome to try Flintrock out and potentially even [contribute](https://github.com/nchammas/flintrock/blob/master/CONTRIBUTING.md). ## Usage @@ -16,7 +16,7 @@ Here's a quick way to launch a cluster on EC2, assuming you already have an [AWS flintrock launch test-cluster \ --num-slaves 1 \ --no-install-hdfs \ - --spark-version 1.5.1 \ + --spark-version 1.5.2 \ --ec2-key-name key_name \ --ec2-identity-file /path/to/key.pem \ --ec2-ami ami-60b6c60a \ @@ -40,7 +40,7 @@ Other things you can do with Flintrock include: ```sh flintrock login test-cluster flintrock describe test-cluster -flintrock run-command test-cluster 'yum install -y package' +flintrock run-command test-cluster 'sudo yum install -y package' flintrock copy-file test-cluster /local/path /remote/path ``` @@ -56,31 +56,48 @@ That's not all. Flintrock has a few more [features](#features) that you may find ## Installation -Before using Flintrock, take a quick look at the [copyright](COPYRIGHT) notice and [license](LICENSE) and make sure you're OK with their terms. +Before using Flintrock, take a quick look at the [copyright](https://github.com/nchammas/flintrock/blob/master/COPYRIGHT) notice and [license](https://github.com/nchammas/flintrock/blob/master/LICENSE) and make sure you're OK with their terms. -Flintrock requires Python 3.4 or newer. Since we don't have any releases yet, the only way to install Flintrock at the moment is as follows: +Flintrock requires Python 3.4 or newer. It's currently been tested only on OS X, but it should run on all POSIX systems. We have plans to [add Windows support](https://github.com/nchammas/flintrock/issues/46) in the future, too. + +Eventually, we also plan to release stand-alone executables so that you can install Flintrock without having to worry about having Python installed. + +### Release version + +To get the latest release of Flintrock, simply run [pip](https://pip.pypa.io/en/stable/): + +``` +python3 -m pip install flintrock +``` + +This will install Flintrock and place it on your path. You should be good to go now! + +You'll probably want to get started with the following two commands: ```sh -# Download Flintrock. -git clone https://github.com/nchammas/flintrock +flintrock --help +flintrock configure +``` + +### Development version + +If you like living on the edge, or if you want to [contribute](https://github.com/nchammas/flintrock/blob/master/CONTRIBUTING.md), you can install the development version of Flintrock like this: -# Set your defaults. +```sh +git clone https://github.com/nchammas/flintrock cd flintrock -cp config.yaml.template config.yaml -# vi config.yaml -# Install Flintrock's dependencies. +# Setup a virtual environment. +# Optional, but *strongly recommended*. python3 -m venv venv source venv/bin/activate -python3 -m pip install -r requirements.txt -deactivate -# You're good to go now. -./flintrock --help +# Install Flintrock. +# If you want to contribute, install the developer requirements. +python3 -m pip install -r requirements/user.pip +python3 -m pip install -r requirements/developer.pip ``` -Eventually, we plan to release binaries so that you can install Flintrock without having to worry about having Python installed. - ## Use Cases @@ -137,7 +154,9 @@ flintrock launch test-cluster --num-slaves 10 ### Configurable CLI Defaults -Flintrock lets you persist your desired configuration to a file (called `config.yaml` by default) so that you don't have to keep typing out the same options over and over at the command line. +Flintrock lets you persist your desired configuration to a YAML file so that you don't have to keep typing out the same options over and over at the command line. + +To setup and edit the default config file, call `flintrock configure`. You can also point Flintrock to a non-default config file by using the `--config` option. #### Sample `config.yaml` @@ -204,7 +223,7 @@ Flintrock automatically configures any available [ephemeral storage](http://docs ### Tests -Flintrock comes with a set of automated, end-to-end [tests](./tests). These tests help us develop Flintrock with confidence and guarantee a certain level of quality. +Flintrock comes with a set of automated, end-to-end [tests](https://github.com/nchammas/flintrock/tree/master/tests). These tests help us develop Flintrock with confidence and guarantee a certain level of quality. ### Low-level Provider Options @@ -257,4 +276,4 @@ There are a few additional peeves I had with spark-ec2 -- some of which are diff ## About the Flintrock Logo -The [Flintrock logo](flintrock-logo.png) was created using [Highbrow Cafetorium JNL](http://www.myfonts.com/fonts/jnlevine/highbrow-cafetorium/) and [this icon](https://thenounproject.com/term/stars/40856/). Licenses to use both the font and icon were purchased from their respective owners. +The [Flintrock logo](https://github.com/nchammas/flintrock/blob/master/flintrock-logo.png) was created using [Highbrow Cafetorium JNL](http://www.myfonts.com/fonts/jnlevine/highbrow-cafetorium/) and [this icon](https://thenounproject.com/term/stars/40856/). Licenses to use both the font and icon were purchased from their respective owners. diff --git a/flintrock b/flintrock deleted file mode 100755 index ed90691a..00000000 --- a/flintrock +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -venv/bin/python3 -Wdefault::DeprecationWarning flintrock.py "$@" diff --git a/flintrock/__init__.py b/flintrock/__init__.py new file mode 100644 index 00000000..b794fd40 --- /dev/null +++ b/flintrock/__init__.py @@ -0,0 +1 @@ +__version__ = '0.1.0' diff --git a/flintrock/__main__.py b/flintrock/__main__.py new file mode 100644 index 00000000..d4a5319d --- /dev/null +++ b/flintrock/__main__.py @@ -0,0 +1,5 @@ +# See: https://docs.python.org/3/library/__main__.html +from .flintrock import main + +if __name__ == '__main__': + main() diff --git a/config.yaml.template b/flintrock/config.yaml.template similarity index 100% rename from config.yaml.template rename to flintrock/config.yaml.template diff --git a/flintrock.py b/flintrock/flintrock.py similarity index 96% rename from flintrock.py rename to flintrock/flintrock.py index 004a1652..739ee76b 100644 --- a/flintrock.py +++ b/flintrock/flintrock.py @@ -32,6 +32,7 @@ import string import sys import shlex +import shutil import subprocess import pprint import asyncio @@ -53,7 +54,9 @@ import paramiko import yaml -FLINTROCK_ROOT_DIR = os.path.dirname(os.path.realpath(__file__)) +from flintrock import __version__ + +THIS_DIR = os.path.dirname(os.path.realpath(__file__)) def timeit(func): @@ -67,6 +70,15 @@ def wrapper(*args, **kwargs): return wrapper +def get_config_file() -> str: + """ + Get the path to Flintrock's default configuration file. + """ + config_dir = click.get_app_dir(app_name='Flintrock') + config_file = os.path.join(config_dir, 'config.yaml') + return config_file + + def generate_ssh_key_pair() -> namedtuple('KeyPair', ['public', 'private']): """ Generate an SSH key pair that the cluster can use for intra-cluster @@ -119,7 +131,7 @@ def cluster_info_to_template_mapping( """ template_mapping = {} - for k, v in vars(cluster_info).items(): + for k, v in cluster_info._asdict().items(): if k == 'slave_hosts': template_mapping.update({k: '\n'.join(v)}) elif k == 'storage_dirs': @@ -167,7 +179,7 @@ def install( with ssh_client.open_sftp() as sftp: sftp.put( - localpath='./get-best-apache-mirror.py', + localpath=os.path.join(THIS_DIR, 'get-best-apache-mirror.py'), remotepath='/tmp/get-best-apache-mirror.py') ssh_check_output( @@ -189,12 +201,13 @@ def configure( self, ssh_client: paramiko.client.SSHClient, cluster_info: ClusterInfo): + # TODO: os.walk() through these files. template_paths = [ - './hadoop/conf/masters', - './hadoop/conf/slaves', - './hadoop/conf/hadoop-env.sh', - './hadoop/conf/core-site.xml', - './hadoop/conf/hdfs-site.xml'] + 'hadoop/conf/masters', + 'hadoop/conf/slaves', + 'hadoop/conf/hadoop-env.sh', + 'hadoop/conf/core-site.xml', + 'hadoop/conf/hdfs-site.xml'] for template_path in template_paths: ssh_check_output( @@ -204,7 +217,7 @@ def configure( """.format( f=shlex.quote( get_formatted_template( - path="templates/" + template_path, + path=os.path.join(THIS_DIR, "templates", template_path), mapping=cluster_info_to_template_mapping( cluster_info=cluster_info, module='hdfs'))), @@ -278,7 +291,7 @@ def install( if self.version: with ssh_client.open_sftp() as sftp: sftp.put( - localpath='./install-spark.sh', + localpath=os.path.join(THIS_DIR, 'install-spark.sh'), remotepath='/tmp/install-spark.sh') sftp.chmod(path='/tmp/install-spark.sh', mode=0o755) ssh_check_output( @@ -325,8 +338,8 @@ def configure( This method is master/slave-agnostic. """ template_paths = [ - './spark/conf/spark-env.sh', - './spark/conf/slaves'] + 'spark/conf/spark-env.sh', + 'spark/conf/slaves'] for template_path in template_paths: ssh_check_output( client=ssh_client, @@ -335,7 +348,7 @@ def configure( """.format( f=shlex.quote( get_formatted_template( - path="templates/" + template_path, + path=os.path.join(THIS_DIR, "templates", template_path), mapping=cluster_info_to_template_mapping( cluster_info=cluster_info, module='spark'))), @@ -416,9 +429,9 @@ def health_check(self, master_host: str): @click.group() -@click.option('--config', default=FLINTROCK_ROOT_DIR + '/config.yaml') +@click.option('--config', default=get_config_file()) @click.option('--provider', default='ec2', type=click.Choice(['ec2'])) -@click.version_option(version='dev') # TODO: Replace with setuptools auto-detect. +@click.version_option(version=__version__) @click.pass_context def cli(cli_context, config, provider): """ @@ -428,15 +441,15 @@ def cli(cli_context, config, provider): """ cli_context.obj['provider'] = provider - if os.path.exists(config): + if os.path.isfile(config): with open(config) as f: config_raw = yaml.safe_load(f) config_map = config_to_click(normalize_keys(config_raw)) cli_context.default_map = config_map else: - if config != (FLINTROCK_ROOT_DIR + '/config.yaml'): - raise FileNotFoundError(errno.ENOENT, 'No such file or directory', config) + if config != get_config_file(): + raise FileNotFoundError(errno.ENOENT, 'No such file', config) @cli.command() @@ -637,7 +650,7 @@ def get_or_create_ec2_security_groups( # TODO: Add rules in one shot. for rule in client_rules: try: - flintrock_group.authorize(**vars(rule)) + flintrock_group.authorize(**rule._asdict()) except boto.exception.EC2ResponseError as e: if e.error_code != 'InvalidPermission.Duplicate': print("Error adding rule: {r}".format(r=rule)) @@ -675,7 +688,7 @@ def get_or_create_ec2_security_groups( # TODO: Add rules in one shot. for rule in cluster_rules: try: - cluster_group.authorize(**vars(rule)) + cluster_group.authorize(**rule._asdict()) except boto.exception.EC2ResponseError as e: if e.error_code != 'InvalidPermission.Duplicate': print("Error adding rule: {r}".format(r=rule)) @@ -1030,7 +1043,7 @@ def provision_node( with client.open_sftp() as sftp: sftp.put( - localpath='./setup-ephemeral-storage.py', + localpath=os.path.join(THIS_DIR, 'setup-ephemeral-storage.py'), remotepath='/tmp/setup-ephemeral-storage.py') print("[{h}] Configuring ephemeral storage...".format(h=host)) @@ -1670,8 +1683,8 @@ def run_command( Examples: - ./flintrock run-command my-cluster 'touch /tmp/flintrock' - ./flintrock run-command my-cluster -- yum install -y package + flintrock run-command my-cluster 'touch /tmp/flintrock' + flintrock run-command my-cluster -- yum install -y package Flintrock will return a non-zero code if any of the cluster nodes raises an error while running the command. @@ -1792,8 +1805,8 @@ def copy_file( Examples: - ./flintrock copy-file my-cluster /tmp/file.102.txt /tmp/file.txt - ./flintrock copy-file my-cluster /tmp/spark-defaults.conf /tmp/ + flintrock copy-file my-cluster /tmp/file.102.txt /tmp/file.txt + flintrock copy-file my-cluster /tmp/spark-defaults.conf /tmp/ Flintrock will return a non-zero code if any of the cluster nodes raises an error. """ @@ -1978,10 +1991,37 @@ def config_to_click(config: dict) -> dict: 'copy-file': ec2_configs, } + # TODO: Use a different name. click is a module. return click -if __name__ == "__main__": - # TODO: try, catch - # print and exit from exception metadata +@cli.command() +@click.option('--locate', is_flag=True, default=False, + help="Don't open an editor. " + "Just open the folder containing the configuration file.") +@click.pass_context +def configure(cli_context, locate): + """ + Configure Flintrock's defaults. + + This will open Flintrock's configuration file in your default YAML editor so + you can set your defaults. + """ + config_file = get_config_file() + + if not os.path.isfile(config_file): + print("Initializing config file from template...") + os.makedirs(os.path.dirname(config_file), exist_ok=True) + shutil.copyfile( + src=os.path.join(THIS_DIR, 'config.yaml.template'), + dst=config_file) + os.chmod(config_file, mode=0o644) + + click.launch(config_file, locate=locate) + + +def main(): + # We pass in obj so we can add attributes to it, like provider, which + # get shared by all commands. + # See: http://click.pocoo.org/6/api/#click.Context cli(obj={}) diff --git a/get-best-apache-mirror.py b/flintrock/get-best-apache-mirror.py similarity index 100% rename from get-best-apache-mirror.py rename to flintrock/get-best-apache-mirror.py diff --git a/install-spark.sh b/flintrock/install-spark.sh similarity index 100% rename from install-spark.sh rename to flintrock/install-spark.sh diff --git a/setup-ephemeral-storage.py b/flintrock/setup-ephemeral-storage.py similarity index 100% rename from setup-ephemeral-storage.py rename to flintrock/setup-ephemeral-storage.py diff --git a/templates/hadoop/conf/core-site.xml b/flintrock/templates/hadoop/conf/core-site.xml similarity index 100% rename from templates/hadoop/conf/core-site.xml rename to flintrock/templates/hadoop/conf/core-site.xml diff --git a/templates/hadoop/conf/hadoop-env.sh b/flintrock/templates/hadoop/conf/hadoop-env.sh similarity index 100% rename from templates/hadoop/conf/hadoop-env.sh rename to flintrock/templates/hadoop/conf/hadoop-env.sh diff --git a/templates/hadoop/conf/hdfs-site.xml b/flintrock/templates/hadoop/conf/hdfs-site.xml similarity index 100% rename from templates/hadoop/conf/hdfs-site.xml rename to flintrock/templates/hadoop/conf/hdfs-site.xml diff --git a/templates/hadoop/conf/masters b/flintrock/templates/hadoop/conf/masters similarity index 100% rename from templates/hadoop/conf/masters rename to flintrock/templates/hadoop/conf/masters diff --git a/templates/hadoop/conf/slaves b/flintrock/templates/hadoop/conf/slaves similarity index 100% rename from templates/hadoop/conf/slaves rename to flintrock/templates/hadoop/conf/slaves diff --git a/templates/spark/conf/slaves b/flintrock/templates/spark/conf/slaves similarity index 100% rename from templates/spark/conf/slaves rename to flintrock/templates/spark/conf/slaves diff --git a/templates/spark/conf/spark-env.sh b/flintrock/templates/spark/conf/spark-env.sh similarity index 100% rename from templates/spark/conf/spark-env.sh rename to flintrock/templates/spark/conf/spark-env.sh diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 10fb6876..00000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,2 +0,0 @@ -pep8==1.6.2 -pytest==2.8.3 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 48f7ba8c..00000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -boto==2.38.0 -click==5.1 -paramiko==1.15.4 -PyYAML==3.11 diff --git a/requirements/developer.pip b/requirements/developer.pip new file mode 100644 index 00000000..2afda7ed --- /dev/null +++ b/requirements/developer.pip @@ -0,0 +1,3 @@ +-r user.pip +pep8 >= 1.6.2 +pytest >= 2.8.4 diff --git a/requirements/maintainer.pip b/requirements/maintainer.pip new file mode 100644 index 00000000..0e23a1d9 --- /dev/null +++ b/requirements/maintainer.pip @@ -0,0 +1,4 @@ +-r developer.pip +wheel >= 0.26.0 +twine >= 1.6.4 +pypandoc >= 1.1.2 diff --git a/requirements/user.pip b/requirements/user.pip new file mode 100644 index 00000000..6cd67c6d --- /dev/null +++ b/requirements/user.pip @@ -0,0 +1,7 @@ +# NOTE: Run pip from Flintrock's root directory, not from the directory containing +# this file. +# NOTE: The `-e .` syntax lets us reuse the requirements already specified under +# `install_requires` in setup.py. +# See: https://caremad.io/2013/07/setup-vs-requirement/ +setuptools >= 18.7.1 +-e . diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..b6b3fdb6 --- /dev/null +++ b/setup.py @@ -0,0 +1,60 @@ +import setuptools + +try: + import pypandoc + long_description = pypandoc.convert('README.md', 'rst') +except ImportError: + long_description = open('README.md').read() + +from flintrock import __version__ + +setuptools.setup( + name='Flintrock', + version=__version__, + description='A command-line tool for launching Apache Spark clusters.', + long_description=long_description, + url='https://github.com/nchammas/flintrock', + author='Nicholas Chammas', + author_email='nicholas.chammas@gmail.com', + license='Apache License 2.0', + + # See: https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + 'Development Status :: 3 - Alpha', + + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + + 'Topic :: Utilities', + 'Environment :: Console', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: POSIX', + + 'License :: OSI Approved :: Apache Software License', + + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + ], + keywords=['Apache Spark'], + + packages=setuptools.find_packages(), + include_package_data=True, + + # We pin dependencies because sometimes projects do not + # strictly follow semantic versioning, so new "feature" + # releases end up making backwards-incompatible changes. + install_requires=[ + 'boto == 2.38.0', + 'click == 5.1', + 'paramiko == 1.15.4', + 'PyYAML == 3.11' + ], + + entry_points={ + 'console_scripts': [ + 'flintrock = flintrock.__main__:main', + ], + }, +) diff --git a/tests/README.md b/tests/README.md index 033f9826..e45d22c3 100644 --- a/tests/README.md +++ b/tests/README.md @@ -5,7 +5,7 @@ Use the tests in this directory to help you catch bugs as you work on Flintrock. The instructions here assume the following things: 1. You're working from Flintrock's root directory. -2. You're working in a Python [virtual environment](https://docs.python.org/3/library/venv.html). (We set this up under `venv/` when we [installed Flintrock](../README.md#Installation).) +2. You're working in a Python [virtual environment](https://docs.python.org/3/library/venv.html). (We set this up under `venv/` when we [installed Flintrock](../README.md#development-version).) 3. You're running Python 3.5+. 4. You've already setup your Flintrock config file and can launch clusters. @@ -15,7 +15,7 @@ The instructions here assume the following things: Flintrock's tests have their own dependencies which you can install as follows: ```sh -python3 -m pip install -r requirements-dev.txt +python3 -m pip install -r requirements/developer.pip ``` Among other things, this will make [pytest](http://pytest.readthedocs.org/en/latest/) available at the command line. We'll use it to run our tests. @@ -28,10 +28,10 @@ Among other things, this will make [pytest](http://pytest.readthedocs.org/en/lat To run all of Flintrock's tests, just run: ```sh -py.test --verbose +py.test ``` -The complete test run is quite long, so you'll want the `--verbose` flag so you can track pytest's progress. +Keep in mind that the complete test run is quite long. ## Style Tests diff --git a/tests/conftest.py b/tests/conftest.py index 1a289548..927a1b8c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,7 @@ def random_string(): def launch_cluster(cluster_name, instance_type): p = subprocess.run([ - './flintrock', 'launch', cluster_name, + 'flintrock', 'launch', cluster_name, '--num-slaves', '1', '--install-hdfs', '--hdfs-version', HADOOP_VERSION, @@ -31,13 +31,13 @@ def launch_cluster(cluster_name, instance_type): def stop_cluster(cluster_name): p = subprocess.run([ - './flintrock', 'stop', cluster_name, '--assume-yes']) + 'flintrock', 'stop', cluster_name, '--assume-yes']) assert p.returncode == 0 def start_cluster(cluster_name): p = subprocess.run([ - './flintrock', 'start', cluster_name]) + 'flintrock', 'start', cluster_name]) assert p.returncode == 0 @@ -67,7 +67,7 @@ def running_cluster(request): def destroy(): p = subprocess.run([ - './flintrock', 'destroy', cluster_name, '--assume-yes']) + 'flintrock', 'destroy', cluster_name, '--assume-yes']) assert p.returncode == 0 request.addfinalizer(destroy) @@ -78,7 +78,7 @@ def destroy(): def stopped_cluster(request): cluster_name = 'running-cluster-' + random_string() p = subprocess.run([ - './flintrock', 'launch', cluster_name, + 'flintrock', 'launch', cluster_name, '--num-slaves', '1', '--no-install-hdfs', '--no-install-spark', @@ -86,12 +86,12 @@ def stopped_cluster(request): assert p.returncode == 0 p = subprocess.run([ - './flintrock', 'stop', cluster_name, '--assume-yes']) + 'flintrock', 'stop', cluster_name, '--assume-yes']) assert p.returncode == 0 def destroy(): p = subprocess.run([ - './flintrock', 'destroy', cluster_name, '--assume-yes']) + 'flintrock', 'destroy', cluster_name, '--assume-yes']) assert p.returncode == 0 request.addfinalizer(destroy) @@ -105,7 +105,7 @@ def remote_file(request, running_cluster): """ file_path = '/tmp/remote_dummy_file_for_testing' p = subprocess.run([ - './flintrock', 'run-command', running_cluster, '--', + 'flintrock', 'run-command', running_cluster, '--', 'echo -e "{data}" > {path}'.format( data='test\n' * 3, path=file_path)]) @@ -113,7 +113,7 @@ def remote_file(request, running_cluster): def destroy(): p = subprocess.run([ - './flintrock', 'run-command', running_cluster, '--', + 'flintrock', 'run-command', running_cluster, '--', 'rm', '-f', file_path]) assert p.returncode == 0 request.addfinalizer(destroy) diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py index b2e46d57..ca27b216 100644 --- a/tests/test_acceptance.py +++ b/tests/test_acceptance.py @@ -7,7 +7,7 @@ def test_describe_stopped_cluster(stopped_cluster): p = subprocess.run([ - './flintrock', 'describe', stopped_cluster], + 'flintrock', 'describe', stopped_cluster], stdout=subprocess.PIPE) assert p.returncode == 0 assert p.stdout.startswith(stopped_cluster.encode()) @@ -15,7 +15,7 @@ def test_describe_stopped_cluster(stopped_cluster): def test_try_launching_duplicate_stopped_cluster(stopped_cluster): p = subprocess.run([ - './flintrock', 'launch', stopped_cluster], + 'flintrock', 'launch', stopped_cluster], stderr=subprocess.PIPE) assert p.returncode == 1 assert p.stderr.startswith(b"Cluster already exists: ") @@ -23,7 +23,7 @@ def test_try_launching_duplicate_stopped_cluster(stopped_cluster): def test_try_launching_duplicate_cluster(running_cluster): p = subprocess.run([ - './flintrock', 'launch', running_cluster], + 'flintrock', 'launch', running_cluster], stderr=subprocess.PIPE) assert p.returncode == 1 assert p.stderr.startswith(b"Cluster already exists: ") @@ -31,7 +31,7 @@ def test_try_launching_duplicate_cluster(running_cluster): def test_describe_running_cluster(running_cluster): p = subprocess.run([ - './flintrock', 'describe', running_cluster], + 'flintrock', 'describe', running_cluster], stdout=subprocess.PIPE) assert p.returncode == 0 assert p.stdout.startswith(running_cluster.encode()) @@ -39,13 +39,13 @@ def test_describe_running_cluster(running_cluster): def test_run_command_on_running_cluster(running_cluster): p = subprocess.run([ - './flintrock', 'run-command', running_cluster, '--', 'ls', '-l']) + 'flintrock', 'run-command', running_cluster, '--', 'ls', '-l']) assert p.returncode == 0 def test_copy_file_on_running_cluster(running_cluster, local_file): p = subprocess.run([ - './flintrock', 'copy-file', running_cluster, local_file, '/tmp/copied_from_local']) + 'flintrock', 'copy-file', running_cluster, local_file, '/tmp/copied_from_local']) assert p.returncode == 0 @@ -53,12 +53,12 @@ def test_hdfs_on_running_cluster(running_cluster, remote_file): hdfs_path = '/hdfs_file' p = subprocess.run([ - './flintrock', 'run-command', running_cluster, '--master-only', '--', + 'flintrock', 'run-command', running_cluster, '--master-only', '--', './hadoop/bin/hdfs', 'dfs', '-put', remote_file, hdfs_path]) assert p.returncode == 0 p = subprocess.run([ - './flintrock', 'run-command', running_cluster, '--', + 'flintrock', 'run-command', running_cluster, '--', './hadoop/bin/hdfs', 'dfs', '-cat', hdfs_path]) assert p.returncode == 0 @@ -66,12 +66,12 @@ def test_hdfs_on_running_cluster(running_cluster, remote_file): def test_spark_on_running_cluster(running_cluster, remote_file): # TODO: Run a real query; e.g. sc.parallelize(range(10)).count() p = subprocess.run([ - './flintrock', 'run-command', running_cluster, '--', + 'flintrock', 'run-command', running_cluster, '--', './spark/bin/pyspark', '--help']) assert p.returncode == 0 p = subprocess.run([ - './flintrock', 'describe', running_cluster, '--master-hostname-only'], + 'flintrock', 'describe', running_cluster, '--master-hostname-only'], stdout=subprocess.PIPE) master_address = p.stdout.strip().decode('utf-8') assert p.returncode == 0 @@ -88,21 +88,21 @@ def test_operations_against_non_existent_cluster(): for command in ['describe', 'stop', 'start', 'login', 'destroy']: p = subprocess.run( - ['./flintrock', command, cluster_name], + ['flintrock', command, cluster_name], stderr=subprocess.PIPE) assert p.returncode == 1 assert p.stderr.startswith(expected_error_message) for command in ['run-command']: p = subprocess.run( - ['./flintrock', command, cluster_name, 'ls'], + ['flintrock', command, cluster_name, 'ls'], stderr=subprocess.PIPE) assert p.returncode == 1 assert p.stderr.startswith(expected_error_message) for command in ['copy-file']: p = subprocess.run( - ['./flintrock', command, cluster_name, __file__, '/remote/path'], + ['flintrock', command, cluster_name, __file__, '/remote/path'], stderr=subprocess.PIPE) assert p.returncode == 1 assert p.stderr.startswith(expected_error_message) diff --git a/tests/test_style.py b/tests/test_style.py index 3bac7345..b692a9c8 100644 --- a/tests/test_style.py +++ b/tests/test_style.py @@ -10,8 +10,8 @@ os.path.realpath(__file__)))) TEST_TARGETS = [ - 'flintrock.py', - 'setup-ephemeral-storage.py', + 'setup.py', + 'flintrock/', 'tests/'] TEST_PATHS = [ @@ -26,6 +26,7 @@ def test_code_compiles(): result = compileall.compile_file(path) # NOTE: This is not publicly documented, but a return of 1 means # the compilation succeeded. + # See: http://bugs.python.org/issue25768 assert result == 1 diff --git a/tox.ini b/tox.ini index 1d7b87ff..e2f160f1 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,5 @@ max-line-length=100 ignore=E501 [pytest] -minversion=2.8.3 norecursedirs=venv -addopts=--exitfirst +addopts=--verbose --exitfirst