Merge pull request #2 from bldrvnlw/master

pyhton-tsne now building and testing successfully

python-tsne/.bumpversion.cfg

@@ -0,0 +1,10 @@
+[bumpversion]
+current_version = 0.3.1
+commit = True
+tag = True
+tag_name = python-tsne-{new_version}
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)
+serialize = {major}.{minor}.{patch}
+
+[bumpversion:file:Dockerfile]
+

python-tsne/Dockerfile

@@ -1,14 +1,35 @@
 FROM hbpmip/python-mip:0.2.2
 
+MAINTAINER [email protected]
+
+ENV DOCKER_IMAGE=hbpmip/python-tsne:0.3.1 \
+    FUNCTION=python-tsne
+
 # This COPY may be replaced by a build step when the atsne is open sourced (BvL 19/09/2017)
 COPY atsne/ /atsne/
 COPY requirements.txt /requirements.txt
-COPY tsne.py /main.py
+COPY tsne.py /tsne.py
+
+RUN pip install -r /requirements.txt
+RUN apt-get update -y -qq && \
+    apt-get install -y qt5-default && \
+    apt-get install -y libgomp1 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
-RUN conda install -y --file /requirements.txt
+# For dev purposes (i.e. docker exec) use the following noop daemon
+# ENTRYPOINT ["tail", "-f", "/dev/null"]
+ENTRYPOINT ["python", "/tsne.py"]
 
-ENV DOCKER_IMAGE=hbpmip/python-tsne
-ENV MODEL example
-ENV FUNCTION python-tsne
-ENV CODE tsne
-ENV NAME tsne
+LABEL org.label-schema.build-date=$BUILD_DATE \
+      org.label-schema.name="hbpmip/python-tsne" \
+      org.label-schema.description="Python wrapper for approximated tSNE" \
+      org.label-schema.url="https://github.com/LREN-CHUV/algorithm-repository" \
+      org.label-schema.vcs-type="git" \
+      org.label-schema.vcs-url="https://github.com/LREN-CHUV/algorithm-repository.git" \
+      org.label-schema.vcs-ref=$VCS_REF \
+      org.label-schema.version="$VERSION" \
+      org.label-schema.vendor="LKEB LUMC" \
+      org.label-schema.license="AGPLv3" \
+      org.label-schema.docker.dockerfile="Dockerfile" \
+      org.label-schema.schema-version="1.0"

python-tsne/build.sh

@@ -1,8 +1,32 @@
 #!/usr/bin/env bash
-if groups $USER | grep &>/dev/null '\bdocker\b'; then
+
+set -o pipefail  # trace ERR through pipes
+set -o errtrace  # trace ERR through 'time command' and other functions
+set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
+
+get_script_dir () {
+     SOURCE="${BASH_SOURCE[0]}"
+
+     while [ -h "$SOURCE" ]; do
+          DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+          SOURCE="$( readlink "$SOURCE" )"
+          [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
+     done
+     cd -P "$( dirname "$SOURCE" )"
+     pwd
+}
+
+cd "$(get_script_dir)"
+
+if [[ $NO_SUDO || -n "$CIRCLECI" ]]; then
+  CAPTAIN="captain"
+elif groups $USER | grep &>/dev/null '\bdocker\b'; then
   CAPTAIN="captain"
 else
   CAPTAIN="sudo captain"
 fi
 
-$CAPTAIN build
+BUILD_DATE=$(date -Iseconds) \
+  VCS_REF=$(git describe --tags --dirty) \
+  VERSION=$(git describe --tags --dirty) \
+  $CAPTAIN build

python-tsne/captain.yml

@@ -5,3 +5,5 @@ target_image:
     - echo "Preparing python-tsne"
   post:
     - echo "Finished python-tsne"
+  test:
+    - ./tests/test.sh

python-tsne/dev.sh

@@ -3,14 +3,13 @@
 docker run \
 -e "JOB_ID=$(uuidgen)" \
 -e "NODE=federation" \
--e "PARAM_variables=iq" \
--e "PARAM_covariables=college_math,score_math_course1,score_math_course2" \
--e "PARAM_grouping=" \
--e "PARAM_meta={\"iq\":{\"code\":\"iq\",\"type\":\"real\"},\"college_math\":{\"code\":\"college_math\",\"type\":\"real\"},\"score_math_course1\":{\"code\":\"score_math_course1\",\"type\":\"real\"},\"score_math_course2\":{\"code\":\"score_math_course2\",\"type\":\"real\"}}" \
--e "PARAM_query=select iq,college_math,score_math_course1,score_math_course2 from sample_data where iq is not null and college_math is not null and score_math_course1 is not null and score_math_course2 is not null" \
+-e "PARAM_variables=Name" \
+-e "PARAM_covariables=SepalLength,SepalWidth,PetalLength,PetalWidth" \
+-e "PARAM_query=select Name,SepalLength,SepalWidth,PetalLength,PetalWidth from sample_data where Name is not null and SepalLength is not null and SepalWidth is not null and PetalLength is not null and PetalWidth is not null" \
+-e "PARAM_meta={\"Name\":{\"code\":\"Name\",\"type\":\"string\"},\"SepalLength\":{\"code\":\"SepalLength\",\"type\":\"real\"},\"SepalWidth\":{\"code\":\"SepalWidth\",\"type\":\"real\"},\"PetalLength\":{\"code\":\"PetalLength\",\"type\":\"real\"}, \"PetalWidth\":{\"code\":\"PetalWidth\",\"type\":\"real\"}}" \
 -e "IN_JDBC_URL=jdbc:postgresql://172.18.0.1:5432/features" \
--e "IN_JDBC_USER=postgres" \
--e "IN_JDBC_PASSWORD=test" \
+-e "IN_JDBC_USER=features" \
+-e "IN_JDBC_PASSWORD=featurespwd" \
 -e "OUT_JDBC_URL=jdbc:postgresql://172.18.0.1:5432/woken" \
 -e "OUT_JDBC_USER=woken" \
 -e "OUT_JDBC_PASSWORD=wokenpwd" \

python-tsne/publish.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+
+set -o pipefail  # trace ERR through pipes
+set -o errtrace  # trace ERR through 'time command' and other functions
+set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
+
+get_script_dir () {
+     SOURCE="${BASH_SOURCE[0]}"
+
+     while [ -h "$SOURCE" ]; do
+          DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+          SOURCE="$( readlink "$SOURCE" )"
+          [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
+     done
+     cd -P "$( dirname "$SOURCE" )"
+     pwd
+}
+
+WORKSPACE=$(get_script_dir)
+
+if pgrep -lf sshuttle > /dev/null ; then
+  echo "sshuttle detected. Please close this program as it messes with networking and prevents builds inside Docker to work"
+  exit 1
+fi
+
+if [ $NO_SUDO ]; then
+  CAPTAIN="captain"
+elif groups $USER | grep &>/dev/null '\bdocker\b'; then
+  CAPTAIN="captain"
+else
+  CAPTAIN="sudo captain"
+fi
+
+# Build
+echo "Build the project..."
+./build.sh
+./tests/test.sh
+echo "[ok] Done"
+
+count=$(git status --porcelain | wc -l)
+if test $count -gt 0; then
+  git status
+  echo "Not all files have been committed in Git. Release aborted"
+  exit 1
+fi
+
+select_part() {
+  local choice=$1
+  case "$choice" in
+      "Patch release")
+          bumpversion patch
+          ;;
+      "Minor release")
+          bumpversion minor
+          ;;
+      "Major release")
+          bumpversion major
+          ;;
+      *)
+          read -p "Version > " version
+          bumpversion --new-version=$version all
+          ;;
+  esac
+}
+
+git pull --tags
+# Look for a version tag in Git. If not found, ask the user to provide one
+[ $(git tag --points-at HEAD | grep python-tsne | wc -l) == 1 ] || (
+  latest_version=$(bumpversion --dry-run --list patch | grep current_version | sed -r s,"^.*=",, || echo '0.0.1')
+  echo
+  echo "Current commit has not been tagged with a version. Latest known version is $latest_version."
+  echo
+  echo 'What do you want to release?'
+  PS3='Select the version increment> '
+  options=("Patch release" "Minor release" "Major release" "Release with a custom version")
+  select choice in "${options[@]}";
+  do
+    select_part "$choice"
+    break
+  done
+  updated_version=$(bumpversion --dry-run --list patch | grep current_version | sed -r s,"^.*=",,)
+  read -p "Release version $updated_version? [y/N] > " ok
+  if [ "$ok" != "y" ]; then
+    echo "Release aborted"
+    exit 1
+  fi
+)
+
+updated_version=$(bumpversion --dry-run --list patch | grep current_version | sed -r s,"^.*=",,)
+
+# Build again to update the version
+echo "Build the project for distribution..."
+./build.sh
+./tests/test.sh
+echo "[ok] Done"
+
+git push
+git push --tags
+
+# Push on Docker Hub
+# WARNING: Requires captain 1.1.0 to push user tags
+BUILD_DATE=$(date -Iseconds) \
+  VCS_REF=$updated_version \
+  VERSION=$updated_version \
+  WORKSPACE=$WORKSPACE \
+  $CAPTAIN push target_image --branch-tags=false --commit-tags=false --tag $updated_version
+
+# Notify on slack
+sed "s/USER/${USER^}/" $WORKSPACE/slack.json > $WORKSPACE/.slack.json
+sed -i.bak "s/VERSION/$updated_version/" $WORKSPACE/.slack.json
+curl -k -X POST --data-urlencode payload@$WORKSPACE/.slack.json https://hbps1.chuv.ch/slack/dev-activity
+rm -f $WORKSPACE/.slack.json $WORKSPACE/.slack.json.bak

python-tsne/requirements.txt

@@ -1,3 +1,4 @@
 pandas==0.20.3
-numpy=1.13.1
-scipy=0.19.1
+numpy==1.13.1
+scipy==0.19.1
+io_helper==0.2.2

python-tsne/test/docker-compose.yml

@@ -0,0 +1,95 @@
+---
+
+version: '2'
+services:
+
+  db:
+    image: postgres:9.6.5-alpine
+    hostname: db
+# Enable for foreground debugging
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_PASSWORD: test
+
+  wait_dbs:
+    image: "waisbrot/wait"
+    restart: "no"
+    environment:
+      TARGETS: "db:5432"
+      TIMEOUT: 60
+
+  create_dbs:
+    image: "hbpmip/create-databases:1.0.0"
+    restart: "no"
+    environment:
+      DB_HOST: db
+      DB_PORT: 5432
+      DB_ADMIN_USER: postgres
+      DB_ADMIN_PASSWORD: test
+      DB1: features
+      USER1: features
+      PASSWORD1: featurespwd
+      DB2: woken
+      USER2: woken
+      PASSWORD2: wokenpwd
+    depends_on:
+      - db
+
+  sample_data_db_setup:
+    image: "hbpmip/sample-data-db-setup:latest"
+    container_name: "data-db-setup"
+    restart: "no"
+    environment:
+      FLYWAY_DBMS: postgresql
+      FLYWAY_HOST: db
+      FLYWAY_PORT: 5432
+      FLYWAY_DATABASE_NAME: features
+      FLYWAY_USER: postgres
+      FLYWAY_PASSWORD: test
+    depends_on:
+      - db
+
+  woken_db_setup:
+    image: "hbpmip/woken-db-setup:1.0.2"
+    container_name: "woken-db-setup"
+    restart: "no"
+    environment:
+      FLYWAY_DBMS: postgresql
+      FLYWAY_HOST: db
+      FLYWAY_PORT: 5432
+      FLYWAY_DATABASE_NAME: woken
+      FLYWAY_USER: postgres
+      FLYWAY_PASSWORD: test
+    depends_on:
+      - db
+
+  tsne:
+    image: "hbpmip/python-tsne:latest"
+    container_name: "tsne"
+    restart: "no"
+    environment:
+      FUNCTION: python-tsne
+      NODE: job_test
+      JOB_ID: 2
+      IN_JDBC_DRIVER: org.postgresql.Driver
+      IN_JDBC_URL: jdbc:postgresql://db:5432/features
+      IN_JDBC_USER: features
+      IN_JDBC_PASSWORD: featurespwd
+      OUT_JDBC_DRIVER: org.postgresql.Driver
+      OUT_JDBC_URL: jdbc:postgresql://db:5432/woken
+      OUT_JDBC_USER: woken
+      OUT_JDBC_PASSWORD: wokenpwd
+      PARAM_variables: "name"
+      PARAM_covariables: "sepal_length,sepal_width,petal_length,petal_width"
+      PARAM_grouping: ""
+      PARAM_query: "SELECT name,sepal_length,sepal_width,petal_length,petal_width FROM iris"
+      PARAM_meta: "{\"name\":{\"code\":\"name\",\"type\":\"string\"},\"sepal_length\":{\"code\":\"sepal_length\",\"type\":\"real\"},\"sepal_width\":{\"code\":\"sepal_width\",\"type\":\"real\"},\"petal_length\":{\"code\":\"petal_length\",\"type\":\"real\"}, \"petal_width\":{\"code\":\"petal_width\",\"type\":\"real\"}}"
+      PARAM_MODEL_dependent_is_label: "True"
+      PARAM_MODEL_perplexity: 30
+      PARAM_MODEL_do_zscore: "True"
+      PARAM_MODEL_iterations: 1000
+      PARAM_MODEL_theta: 0.5
+      PARAM_MODEL_target_dimensions: 2
+    links:
+      - "db:db"