merged from release

.gitignore

@@ -90,3 +90,4 @@ ENV/
 
 #pycharm
 .idea/
+pyshifu.iml

hello-/.HEAD

@@ -0,0 +1 @@
+master

hello-/ModelConfig.json

@@ -0,0 +1,100 @@
+{
+  "basic" : {
+    "name" : "hello-",
+    "author" : "haifwu",
+    "description" : "Created at 2017-10-24 09:12:55",
+    "version" : "0.11.0",
+    "runMode" : "LOCAL",
+    "postTrainOn" : false,
+    "customPaths" : { }
+  },
+  "dataSet" : {
+    "source" : "LOCAL",
+    "dataPath" : "/Users/haifwu/Shifu/pyshifu/pyshifu/java/bin/../example/cancer-judgement/DataStore/DataSet1",
+    "validationDataPath" : null,
+    "dataDelimiter" : "|",
+    "headerPath" : "/Users/haifwu/Shifu/pyshifu/pyshifu/java/bin/../example/cancer-judgement/DataStore/DataSet1/.pig_header",
+    "headerDelimiter" : "|",
+    "filterExpressions" : "",
+    "weightColumnName" : "",
+    "targetColumnName" : "diagnosis",
+    "posTags" : [ "M" ],
+    "negTags" : [ "B" ],
+    "missingOrInvalidValues" : [ "", "*", "#", "?", "null", "~" ],
+    "metaColumnNameFile" : "columns/meta.column.names",
+    "categoricalColumnNameFile" : "columns/categorical.column.names"
+  },
+  "stats" : {
+    "maxNumBin" : 10,
+    "cateMaxNumBin" : 0,
+    "binningMethod" : "EqualPositive",
+    "sampleRate" : 1.0,
+    "sampleNegOnly" : false,
+    "binningAlgorithm" : "SPDTI",
+    "psiColumnName" : ""
+  },
+  "varSelect" : {
+    "forceEnable" : true,
+    "candidateColumnNameFile" : null,
+    "forceSelectColumnNameFile" : "columns/forceselect.column.names",
+    "forceRemoveColumnNameFile" : "columns/forceremove.column.names",
+    "filterEnable" : true,
+    "filterNum" : 200,
+    "filterBy" : "KS",
+    "filterOutRatio" : 0.05,
+    "missingRateThreshold" : 0.98,
+    "correlationThreshold" : 1.0,
+    "minIvThreshold" : 0.0,
+    "minKsThreshold" : 0.0,
+    "postCorrelationMetric" : "IV",
+    "params" : null
+  },
+  "normalize" : {
+    "stdDevCutOff" : 4.0,
+    "sampleRate" : 1.0,
+    "sampleNegOnly" : false,
+    "normType" : "ZSCALE"
+  },
+  "train" : {
+    "baggingNum" : 1,
+    "baggingWithReplacement" : false,
+    "baggingSampleRate" : 1.0,
+    "validSetRate" : 0.2,
+    "numTrainEpochs" : 200,
+    "isContinuous" : false,
+    "workerThreadCount" : 4,
+    "algorithm" : "NN",
+    "params" : {
+      "Propagation" : "R",
+      "LearningRate" : 0.1,
+      "NumHiddenNodes" : [ 50 ],
+      "NumHiddenLayers" : 1,
+      "RegularizedConstant" : 0.0,
+      "ActivationFunc" : [ "tanh" ]
+    },
+    "customPaths" : { }
+  },
+  "evals" : [ {
+    "name" : "Eval1",
+    "dataSet" : {
+      "source" : "LOCAL",
+      "dataPath" : "/Users/haifwu/Shifu/pyshifu/pyshifu/java/bin/../example/cancer-judgement/DataStore/EvalSet1",
+      "validationDataPath" : null,
+      "dataDelimiter" : "|",
+      "headerPath" : "/Users/haifwu/Shifu/pyshifu/pyshifu/java/bin/../example/cancer-judgement/DataStore/EvalSet1/.pig_header",
+      "headerDelimiter" : "|",
+      "filterExpressions" : "",
+      "weightColumnName" : "",
+      "targetColumnName" : "diagnosis",
+      "posTags" : [ "M" ],
+      "negTags" : [ "B" ],
+      "missingOrInvalidValues" : [ "", "*", "#", "?", "null", "~" ],
+      "metaColumnNameFile" : "columns/Eval1.meta.column.names"
+    },
+    "performanceBucketNum" : 10,
+    "performanceScoreSelector" : "mean",
+    "scoreMetaColumnNameFile" : "columns/Eval1score.meta.column.names",
+    "customPaths" : { },
+    "gbtConvertToProb" : true
+  } ]
+}

pyshifu/core/shell.py

@@ -12,14 +12,14 @@ def __init__(self):
         self._model_dir = None  # model dir
 
         self._model_config_file_name = "ModelConfig.json"
-        self._model_config_file = None  # full path of the model config file
+        self.model_config_file = None  # full path of the model config file
 
         self._os_platform = Helper.get_os_platform()
         self._shifu_home = Helper.get_shifu_home()   # home directory where shifu script with java package
         self._main_script = path.join(self._shifu_home, "java", "bin", "shifu")  # script to running command
 
     def _sanity_check(self):
-        if self._model_config_file is None or not path.exists(self._model_config_file):
+        if self.model_config_file is None or not path.exists(self.model_config_file):
             print('[Error] This is not a model set folder because no ModelConfig.json found in working dir: %s, '
                   'please change folder to your model set folder.' % self._work_dir)
             return False
@@ -29,7 +29,7 @@ def _change_to_model_dir(self):
         if self._model_dir is None:
             raise ValueError("Model dir is not set!")
         chdir(self._model_dir)
-        print("Directory changed to " + getcwd())
+        # print("Directory changed to " + getcwd())
 
     def __set_work_dir(self, work_dir):
         if work_dir is not None:
@@ -43,12 +43,16 @@ def __set_model_name(self, model_name):
             raise FieldException("Setting model name while working dir is None!")
         self._name = model_name
         self._model_dir = path.join(self._work_dir, self._name)
-        self._model_config_file = path.join(self._model_dir, self._model_config_file_name)
+        self.model_config_file = path.join(self._model_dir, self._model_config_file_name)
 
     def _init_working_directory(self, work_directory, model_name):
         self.__set_work_dir(work_directory)
         self.__set_model_name(model_name)
 
+    def __run_command(self, command):
+        command_list = ['bash', self._main_script, command]
+        return Helper.run_shell(command_list)
+
 
 
 

pyshifu/shifu.py

@@ -17,14 +17,13 @@ def new(self, name, work_dir=None):
 
         command_list = ['bash', self._main_script, 'new', self._name]
         status, output = Helper.run_shell(command_list)
-        if status is CommandRunningStatus.SUCCESS:
+        if status == CommandRunningStatus.SUCCESS:
             self._change_to_model_dir()
-            print("You can edit file:" + self._model_config_file)
-            Helper.edit_file(self._os_platform, self._model_config_file)
-
-    def __run_command(self, command):
-        command_list = ['bash', self._main_script, command]
-        Helper.run_shell(command_list)
+            print ("Configure your ModelConfig.json in %s or directly do initialization step by 'shifu.init()'" %
+                   self.model_config_file)
+            Helper.edit_file(self._os_platform, self.model_config_file)
+        else:
+            print ("Shifu.new() failed! Please check if you successfully install pyshifu.")
 
     def init(self):
         self.__run_command("init")

setup.py

@@ -18,7 +18,7 @@
 
 # Change this line to the module name you want to create
 __title__ = "pyshifu"
-__version__ = "0.0.7"
+__version__ = "0.0.8"
 __summary__ = "An end-to-end machine learning and data mining framework on Hadoop."
 __uri__ = "https://github.com/shifuml/pyshifu"