move mill modules into subdirectory

build.yaml

@@ -735,7 +735,7 @@ steps:
       # This was likely from running mill in server mode; mill main exits before
       # the jar has finished being written.
       time retry make shadowJar
-      if [ ! -f out/assembly.dest/out.jar ]; then
+      if [ ! -f out/hail/assembly.dest/out.jar ]; then
           echo 'no out.jar found after mill assembly returned. going to sleep'
           sleep 5
       fi
@@ -774,7 +774,7 @@ steps:
 
       # See `build_hail_jar_and_wheel`
       time retry make shadowJar
-      if [ ! -f out/assembly.dest/out.jar ]; then
+      if [ ! -f out/hail/assembly.dest/out.jar ]; then
           echo 'no out.jar found after mill assembly returned. going to sleep'
           sleep 5
       fi
@@ -807,7 +807,7 @@ steps:
 
       # See `build_hail_jar_and_wheel`
       time retry make shadowTestJar
-      if [ ! -f out/test/assembly.dest/out.jar ]; then
+      if [ ! -f out/hail/test/assembly.dest/out.jar ]; then
           echo 'no out.jar found after mill assembly returned. going to sleep'
           sleep 5
       fi
@@ -832,7 +832,7 @@ steps:
       cd /io/repo/hail
 
       time tar czf test.tar.gz -C python test pytest.ini
-      time tar czf resources.tar.gz -C src/test resources
+      time tar czf resources.tar.gz -C hail/test resources
       time tar czf data.tar.gz -C python/hail/docs data
       time TESTNG_SPLITS=5 python3 generate_splits.py
       time tar czf splits.tar.gz testng-splits-*.xml
@@ -867,7 +867,7 @@ steps:
 
       # See `build_hail_jar_and_wheel`
       time retry make shadowJar
-      if [ ! -f out/assembly.dest/out.jar ]; then
+      if [ ! -f out/hail/assembly.dest/out.jar ]; then
           echo 'no out.jar found after mill assembly returned. going to sleep'
           sleep 5
       fi
@@ -965,7 +965,7 @@ steps:
       export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
       export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
       python3 -m hailtop.aiotools.copy 'null' '[
-      {"from": "src/test/resources",
+      {"from": "hail/test/resources",
        "to":   "{{ global.test_storage_uri }}/{{ token }}/test/resources"},
       {"from": "python/hail/docs/data",
        "to":   "{{ global.test_storage_uri }}/{{ token }}/doctest/data"}
@@ -976,8 +976,8 @@ steps:
           valueFrom: default_ns.name
         mountPath: /test-gsa-key
     inputs:
-      - from: /repo/hail/src/test/resources
-        to: /io/repo/hail/src/test/resources
+      - from: /repo/hail/hail/test/resources
+        to: /io/repo/hail/hail/test/resources
       - from: /repo/hail/python/hail/docs/data
         to: /io/repo/hail/python/hail/docs/data
     dependsOn:
@@ -996,15 +996,15 @@ steps:
     script: |
       set -ex
       cd /io
-      mkdir -p src/test
-      tar xzf resources.tar.gz -C src/test
+      mkdir -p hail/test
+      tar xzf resources.tar.gz -C hail/test
       tar xzf splits.tar.gz
       export HAIL_TEST_SKIP_R=1
       java -cp hail-test.jar:$SPARK_HOME/jars/* org.testng.TestNG -listener is.hail.LogTestListener testng-splits-$HAIL_RUN_IMAGE_SPLIT_INDEX.xml
     inputs:
       - from: /resources.tar.gz
         to: /io/resources.tar.gz
-      - from: /derived/debug/hail/out/test/assembly.dest/out.jar
+      - from: /derived/debug/hail/out/hail/test/assembly.dest/out.jar
         to: /io/hail-test.jar
       - from: /splits.tar.gz
         to: /io/splits.tar.gz
@@ -2444,7 +2444,7 @@ steps:
           valueFrom: default_ns.name
         mountPath: /batch-gsa-key
     inputs:
-      - from: /derived/release/hail/out/assembly.dest/out.jar
+      - from: /derived/release/hail/out/hail/assembly.dest/out.jar
         to: /io/hail.jar
       - from: /git_version
         to: /io/git_version
@@ -3387,7 +3387,7 @@ steps:
       # #14452 - must build wheel with release `hailtop/hailctl/deploy.yaml`
       # use `-o` to prevent `mill` from rebuilding the "SHADOW_JAR"
       make upload-artifacts HAIL_RELEASE_MODE=1 DEPLOY_REMOTE=origin \
-        -o out/assembly.dest/out.jar
+        -o out/hail/assembly.dest/out.jar
 
       echo Setting arguments and invoking release.sh.
 
@@ -3416,8 +3416,8 @@ steps:
         to: /io/git_version
       - from: /repo
         to: /io/repo
-      - from: /derived/release/hail/out/assembly.dest/out.jar
-        to: /io/repo/hail/out/assembly.dest/out.jar
+      - from: /derived/release/hail/out/hail/assembly.dest/out.jar
+        to: /io/repo/hail/out/hail/assembly.dest/out.jar
       - from: /azure-wheel
         to: /io/azure-wheel
       - from: /www.tar.gz
@@ -3664,7 +3664,7 @@ steps:
     inputs:
       - from: /resources.tar.gz
         to: /io/resources.tar.gz
-      - from: /derived/debug/hail/out/test/assembly.dest/out.jar
+      - from: /derived/debug/hail/out/hail/test/assembly.dest/out.jar
         to: /io/hail-test.jar
       - from: /repo/hail/testng-fs.xml
         to: /io/testng-fs.xml
@@ -3706,7 +3706,7 @@ steps:
     inputs:
       - from: /resources.tar.gz
         to: /io/resources.tar.gz
-      - from: /derived/debug/hail/out/test/assembly.dest/out.jar
+      - from: /derived/debug/hail/out/hail/test/assembly.dest/out.jar
         to: /io/hail-test.jar
       - from: /repo/hail/testng-services.xml
         to: /io/testng-services.xml

hail/Makefile

@@ -62,50 +62,50 @@ mill-clean:
 # be recomputed.
 FORCE:
 
-SHADOW_JAR := out/assembly.dest/out.jar
+SHADOW_JAR := out/hail/assembly.dest/out.jar
 ifdef HAIL_COMPILE_NATIVES
 $(SHADOW_JAR): native-lib-prebuilt
 endif
 $(SHADOW_JAR): FORCE
-	$(MILL) $(MILLOPTS) assembly
+	$(MILL) $(MILLOPTS) hail.assembly
 
 .PHONY: shadowJar
 shadowJar: $(SHADOW_JAR)
 
-NON_SHADOW_JAR := out/jar.dest/out.jar
+NON_SHADOW_JAR := out/hail/jar.dest/out.jar
 ifdef HAIL_COMPILE_NATIVES
 $(NON_SHADOW_JAR): native-lib-prebuilt
 endif
 $(NON_SHADOW_JAR): FORCE
-	$(MILL) $(MILLOPTS) jar
+	$(MILL) $(MILLOPTS) hail.jar
 
-SHADOW_TEST_JAR := out/test/assembly.dest/out.jar
+SHADOW_TEST_JAR := out/hail/test/assembly.dest/out.jar
 ifdef HAIL_COMPILE_NATIVES
 $(SHADOW_TEST_JAR): native-lib-prebuilt
 endif
 $(SHADOW_TEST_JAR): FORCE
-	$(MILL) $(MILLOPTS) test.assembly
+	$(MILL) $(MILLOPTS) hail.test.assembly
 
 .PHONY: shadowTestJar
 shadowTestJar: $(SHADOW_TEST_JAR)
 
-EXTRA_CLASSPATH := out/writeRunClasspath.dest/runClasspath
+EXTRA_CLASSPATH := out/hail/writeRunClasspath.dest/runClasspath
 $(EXTRA_CLASSPATH): FORCE
-	$(MILL) $(MILLOPTS) writeRunClasspath
+	$(MILL) $(MILLOPTS) hail.writeRunClasspath
 
 .PHONY: jvm-test
 ifdef HAIL_COMPILE_NATIVES
 jvm-test: native-lib-prebuilt
 endif
 jvm-test:
-	$(MILL) $(MILLOPTS) test testng.xml
+	$(MILL) $(MILLOPTS) hail.test testng.xml
 
 .PHONY: services-jvm-test
 ifdef HAIL_COMPILE_NATIVES
 services-jvm-test: native-lib-prebuilt
 endif
 services-jvm-test:
-	$(MILL) $(MILLOPTS) test testng-services.xml
+	$(MILL) $(MILLOPTS) hail.test testng-services.xml
 
 .PHONY: fs-jvm-test
 ifdef HAIL_COMPILE_NATIVES
@@ -118,7 +118,7 @@ fs-jvm-test: upload-remote-test-resources
 	HAIL_DEFAULT_NAMESPACE=$(NAMESPACE) \
 	HAIL_FS_TEST_CLOUD_RESOURCES_URI=$(CLOUD_HAIL_TEST_RESOURCES_DIR)fs \
 	HAIL_TEST_STORAGE_URI=$(TEST_STORAGE_URI) \
-	$(MILL) $(MILLOPTS) test testng-fs.xml
+	$(MILL) $(MILLOPTS) hail.test testng-fs.xml
 
 # end mill integration
 

hail/build.mill

@@ -72,7 +72,35 @@ object Deps {
   }
 }
 
-trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule { outer =>
+object env extends Module {
+  def scalaVersion: T[String] = Task.Input {
+    val v = T.ctx().env.getOrElse("SCALA_VERSION", "2.12.20")
+    if (!v.startsWith("2.12"))
+      Result.Failure("Hail currently supports only Scala 2.12")
+    else
+      v
+  }
+
+  def javaVersion: T[String] = Task.Input {
+    System.getProperty("java.version")
+  }
+
+  def sparkVersion: T[String] = Task.Input {
+    Result.Success(T.ctx().env.getOrElse("SPARK_VERSION", "3.5.0"))
+  }
+
+  def debugMode: T[Boolean] = Task.Input {
+    val isDebug = !T.ctx().env.contains("HAIL_RELEASE_MODE")
+    T.log.info(s"Building in ${if (isDebug) "debug" else "release"} mode")
+    isDebug
+  }
+
+  def debugOrRelease: Task[String] = Task.Anon {
+    if (debugMode()) "debug" else "release"
+  }
+}
+
+trait HailModule extends ScalaModule with ScalafmtModule with ScalafixModule { outer =>
   override def scalaVersion: T[String] = build.env.scalaVersion()
 
   override def javacOptions: T[Seq[String]] = Seq(
@@ -106,7 +134,7 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule
   override def bspCompileClasspath: T[Agg[UnresolvedPath]] =
     super.bspCompileClasspath() ++ resources().map(p => UnresolvedPath.ResolvedPath(p.path))
 
-  trait HailTests extends SbtTests with TestNg with ScalafmtModule with ScalafixModule {
+  trait HailTests extends ScalaTests with TestNg with ScalafmtModule with ScalafixModule {
     override def forkArgs: T[Seq[String]] = Seq("-Xss4m", "-Xmx4096M")
 
     override def ivyDeps: T[Agg[Dep]] =
@@ -124,35 +152,7 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule
   }
 }
 
-object `package` extends RootModule with HailScalaModule { outer =>
-
-  object env extends Module {
-    def scalaVersion: T[String] = Task.Input {
-      val v = T.ctx().env.getOrElse("SCALA_VERSION", "2.12.20")
-      if (!v.startsWith("2.12"))
-        Result.Failure("Hail currently supports only Scala 2.12")
-      else
-        v
-    }
-
-    def javaVersion: T[String] = Task.Input {
-      System.getProperty("java.version")
-    }
-
-    def sparkVersion: T[String] = Task.Input {
-      Result.Success(T.ctx().env.getOrElse("SPARK_VERSION", "3.5.0"))
-    }
-
-    def debugMode: T[Boolean] = Task.Input {
-      val isDebug = !T.ctx().env.contains("HAIL_RELEASE_MODE")
-      T.log.info(s"Building in ${if (isDebug) "debug" else "release"} mode")
-      isDebug
-    }
-
-    def debugOrRelease: Task[String] = Task.Anon {
-      if (debugMode()) "debug" else "release"
-    }
-  }
+object hail extends HailModule { outer =>
 
   def buildInfo: T[PathRef] = T {
     val revision = VcsVersion.vcsState().currentRevision
@@ -170,8 +170,7 @@ object `package` extends RootModule with HailScalaModule { outer =>
 
   override def moduleDeps: Seq[JavaModule] = Seq(memory)
 
-  override def resources: T[Seq[PathRef]] = Seq(
-    PathRef(millSourcePath / "src" / "main" / "resources"),
+  override def resources: T[Seq[PathRef]] = super.resources() ++ Seq(
     PathRef(millSourcePath / "prebuilt" / "lib"),
     buildInfo(),
   )
@@ -260,7 +259,7 @@ object `package` extends RootModule with HailScalaModule { outer =>
       )
 
     override def sources: T[Seq[PathRef]] = Task.Sources {
-      Seq(PathRef(this.millSourcePath / os.up / "src" / env.debugOrRelease() / "java"))
+      Seq(PathRef(this.millSourcePath / env.debugOrRelease() / "src"))
     }
   }
 

hail/generate_splits.py

@@ -23,14 +23,14 @@ def partition(k: int, ls: List[T]) -> List[List[T]]:
     return out
 
 
-sp.run("find src/test \\( -name \"*.scala\" -o -name \"*.java\" \\) -type f > classes", shell=True, check=True)
+sp.run("find hail/test/src \\( -name \"*.scala\" -o -name \"*.java\" \\) -type f > classes", shell=True, check=True)
 
 n_splits = int(os.environ['TESTNG_SPLITS'])
 
 with open('classes', 'r') as f:
     foo = f.readlines()
     classes = [
-        x.replace('src/test/scala/', '').replace('.scala\n', '').replace('.java\n', '').replace('/', '.') for x in foo
+        x.replace('hail/test/src/', '').replace('.scala\n', '').replace('.java\n', '').replace('/', '.') for x in foo
     ]
     classes = [cls for cls in classes if not cls.startswith('is.hail.services') if not cls.startswith('is.hail.io.fs')]