diff --git a/.gitignore b/.gitignore index 796dc06763..2e0abdaa40 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,6 @@ jobs.groovy # Vagrant dev items Vagrantfile .vagrant/ + +# Travis-CI encrypted files +.travis/secrets* diff --git a/.travis.yml b/.travis.yml index f369a10652..99f1796285 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,26 +13,48 @@ cache: pip: true apt: true +before_install: + # See ClusterHQ/segredos/ci-platform/all/acceptance.yaml + # Includes the private SSH key used by `run-acceptance-tests` to log into and + # provision acceptance test nodes. + - openssl aes-256-cbc -K $encrypted_02804a7cf34b_key -iv $encrypted_02804a7cf34b_iv + -in .travis/secrets.tgz.enc -out .travis/secrets.tgz -d + - tar --verbose --extract --file .travis/secrets.tgz + - eval "$(ssh-agent -s)" + - chmod 600 .travis/secrets/id_rsa_vagrant_buildbot + - ssh-add .travis/secrets/id_rsa_vagrant_buildbot + install: + - pip install --requirement requirements/flocker.txt + - pip install --requirement requirements/flocker-dev.txt + - pip install --requirement requirements/admin.txt - pip install tox awscli + # Install some Flocker specific Travis-CI tools. + - pip install .travis + # Install Flocker itself so that run-acceptance-tests has access to flocker-ca + # command + - pip install . env: + - FLOCKER_BUILDER=acceptance:aws:centos-7:aws + - FLOCKER_BUILDER=acceptance:aws:ubuntu-16.04:aws + - FLOCKER_BUILDER=acceptance:aws:ubuntu-14.04:aws - FLOCKER_BUILDER=lint - - FLOCKER_BUILDER=docs-lint - - FLOCKER_BUILDER=docs-spelling - - FLOCKER_BUILDER=docs-linkcheck - - FLOCKER_BUILDER=docs-html + - FLOCKER_BUILDER=docs:lint + - FLOCKER_BUILDER=docs:spelling + - FLOCKER_BUILDER=docs:linkcheck + - FLOCKER_BUILDER=docs:html matrix: allow_failures: - - env: FLOCKER_BUILDER=docs-lint - - env: FLOCKER_BUILDER=docs-linkcheck + - env: FLOCKER_BUILDER=docs:lint + - env: FLOCKER_BUILDER=docs:linkcheck script: - - .travis/script + - flocker-travis-script after_script: - - .travis/after_script + - flocker-travis-after-script addons: apt: diff --git a/.travis/flocker_travis/__init__.py b/.travis/flocker_travis/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.travis/after_script b/.travis/flocker_travis/after_script.py old mode 100755 new mode 100644 similarity index 54% rename from .travis/after_script rename to .travis/flocker_travis/after_script.py index 978bbb4931..a1491a18b0 --- a/.travis/after_script +++ b/.travis/flocker_travis/after_script.py @@ -12,10 +12,13 @@ """ from os import environ -from subprocess import call +from subprocess import call, check_call +import sys +from .common import BuildHandler -def publish_staging_docs(build_directory): + +def publish_staging_docs(): """ Copy the HTML generated by the docs-html build to an S3 bucket. """ @@ -37,21 +40,33 @@ def publish_staging_docs(build_directory): return call([ "aws", "--region", "us-east-1", "s3", "sync", "--delete", - "build/" + build_directory, + "build/html", "s3://clusterhq-staging-docs/" + target_directory ]) -HANDLERS = { - "docs-html": publish_staging_docs, -} - +def archive_artifacts(*ignored): + """ + Archive everything in the `build/` directory to S3. + """ + check_call([ + "aws", "s3", "sync", + "build", + "/".join([ + # XXX Using this because the Jenkins S3 user has access to it. + "s3://clusterhq-dev-archive/flocker/artifacts", + environ["TRAVIS_BUILD_NUMBER"], -def main(): - handler_name = environ["FLOCKER_BUILDER"] - handler = HANDLERS.get(handler_name) - if handler: - return handler(handler_name) + ]) + ]) -if __name__ == "__main__": - raise SystemExit(main()) +main = BuildHandler( + handlers={ + "docs:html": publish_staging_docs, + "docs": archive_artifacts, + "acceptance": archive_artifacts, + "": lambda *build_labels: sys.stderr.write( + "WARNING: no handler for {}\n".format(build_labels).encode('utf8') + ) + } +).main diff --git a/.travis/flocker_travis/common.py b/.travis/flocker_travis/common.py new file mode 100644 index 0000000000..9be9943774 --- /dev/null +++ b/.travis/flocker_travis/common.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python2 +# +# Copyright ClusterHQ Inc. See LICENSE file for details. + +""" +Run a build step. + +Travis calls this during the `script` phase of its build lifecycle. + * https://docs.travis-ci.com/user/customizing-the-build + +Set ``FLOCKER_BUILDER`` environment variable before calling this script. +""" +from functools import partial +from os import environ + + +class BuildHandler(object): + def __init__(self, handlers, default_handler=None): + self._handlers = handlers + self._default_handler = default_handler + + def _get_handler(self, build_label): + arguments = [] + parts = build_label.split(":") + while True: + key = ":".join(parts) + handler = self._handlers.get(key) + if handler: + break + arguments.append( + parts.pop() + ) + else: + raise KeyError("Handler not found", build_label, self._handlers) + + return partial(handler, *reversed(arguments)) + + def main(self): + build_label = environ["FLOCKER_BUILDER"] + handler = self._get_handler(build_label) + return handler() diff --git a/.travis/flocker_travis/script.py b/.travis/flocker_travis/script.py new file mode 100644 index 0000000000..fd02a594d8 --- /dev/null +++ b/.travis/flocker_travis/script.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python2 +# +# Copyright ClusterHQ Inc. See LICENSE file for details. + +""" +Run a build step. + +Travis calls this during the `script` phase of its build lifecycle. + * https://docs.travis-ci.com/user/customizing-the-build + +Set ``FLOCKER_BUILDER`` environment variable before calling this script. +""" +import os +from subprocess import call + +from .common import BuildHandler + + +def tox(tox_env): + return call(["tox", "-e", tox_env]) + + +def docs(build_type): + return tox("docs-" + build_type) + + +def lint(): + return tox("lint") + + +def acceptance(provider, distribution, dataset_backend): + build_dir = "/".join([ + os.environ["TRAVIS_BUILD_DIR"], + "build", + os.environ["FLOCKER_BUILDER"] + ]) + os.makedirs(build_dir) + command = [ + os.environ["TRAVIS_BUILD_DIR"] + "/admin/run-acceptance-tests", + "--provider", provider, + "--distribution", distribution, + "--dataset-backend", dataset_backend, + "--config-file", + os.environ["TRAVIS_BUILD_DIR"] + "/.travis/secrets/acceptance.yml", + "--", + "flocker.acceptance.endtoend.test_diagnostics", + ] + return call(command, cwd=build_dir) + + +main = BuildHandler( + handlers={ + "acceptance": acceptance, + "lint": lint, + "docs": docs, + } +).main diff --git a/.travis/script b/.travis/script deleted file mode 100755 index b6aed85803..0000000000 --- a/.travis/script +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python2 -# -# Copyright ClusterHQ Inc. See LICENSE file for details. - -""" -Run a build step. - -Travis calls this during the `script` phase of its build lifecycle. - * https://docs.travis-ci.com/user/customizing-the-build - -Set ``FLOCKER_BUILDER`` environment variable before calling this script. -""" - -from os import environ -from subprocess import call - - -def tox(tox_env): - return call(["tox", "-e", tox_env]) - - -BUILDERS = { - "lint": tox, - "docs-lint": tox, - "docs-spelling": tox, - "docs-linkcheck": tox, - "docs-html": tox, -} - - -def main(): - builder_name = environ["FLOCKER_BUILDER"] - builder = BUILDERS[builder_name] - return builder(builder_name) - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.travis/secrets.tgz.enc b/.travis/secrets.tgz.enc new file mode 100644 index 0000000000..d04c789f6d Binary files /dev/null and b/.travis/secrets.tgz.enc differ diff --git a/.travis/setup.py b/.travis/setup.py new file mode 100644 index 0000000000..1a2b23746c --- /dev/null +++ b/.travis/setup.py @@ -0,0 +1,16 @@ +# Copyright ClusterHQ Inc. See LICENSE file for details. + +""" +A package of Python scripts for use in Travis-CI builds. +""" +from setuptools import setup, find_packages +setup( + name="flocker_travis", + packages=find_packages(), + entry_points={ + 'console_scripts': [ + 'flocker-travis-script = flocker_travis.script:main', + 'flocker-travis-after-script = flocker_travis.after_script:main', + ], + }, +) diff --git a/build.yaml b/build.yaml.disabled similarity index 100% rename from build.yaml rename to build.yaml.disabled diff --git a/flocker/provision/_aws.py b/flocker/provision/_aws.py index 7e5800e24f..2532bc5651 100644 --- a/flocker/provision/_aws.py +++ b/flocker/provision/_aws.py @@ -80,7 +80,7 @@ def _enable_boto_logging(): logger.setLevel(logging.DEBUG) logger.addHandler(EliotLogHandler()) -_enable_boto_logging() +# _enable_boto_logging() class FailedToRun(Exception):