From 239c0b8620038879ea19f1e4d30bfe20e2b7c64e Mon Sep 17 00:00:00 2001 From: Christopher Vittal Date: Tue, 9 Dec 2025 17:20:34 -0500 Subject: [PATCH 01/51] Update pip lockfiles (#15206) Primarily for dependabot alerts about urllib3 ## Security Assessment - This change potentially impacts the Hail Batch instance as deployed by Broad Institute in GCP ### Impact Rating - This change has a low security impact ### Impact Description Dependency update to resolve security issue. ### Appsec Review - [x] Required: The impact has been assessed and approved by appsec --- ci/pinned-requirements.txt | 2 +- gear/pinned-requirements.txt | 4 ++-- hail/python/dev/pinned-requirements.txt | 24 ++++++++++----------- hail/python/hailtop/pinned-requirements.txt | 10 ++++----- hail/python/pinned-requirements.txt | 12 +++++------ 5 files changed, 25 insertions(+), 27 deletions(-) diff --git a/ci/pinned-requirements.txt b/ci/pinned-requirements.txt index a3d51674383..b4f44972374 100644 --- a/ci/pinned-requirements.txt +++ b/ci/pinned-requirements.txt @@ -64,7 +64,7 @@ uritemplate==4.2.0 # via # -c ci/../gear/pinned-requirements.txt # gidgethub -urllib3==2.5.0 +urllib3==2.6.1 # via # -c ci/../gear/pinned-requirements.txt # -c ci/../hail/python/dev/pinned-requirements.txt diff --git a/gear/pinned-requirements.txt b/gear/pinned-requirements.txt index 033f00d6feb..e4ef4edb785 100644 --- a/gear/pinned-requirements.txt +++ b/gear/pinned-requirements.txt @@ -121,7 +121,7 @@ propcache==0.4.1 # yarl proto-plus==1.26.1 # via google-api-core -protobuf==6.33.1 +protobuf==6.33.2 # via # google-api-core # google-cloud-profiler @@ -195,7 +195,7 @@ typing-extensions==4.15.0 # multidict uritemplate==4.2.0 # via google-api-python-client -urllib3==2.5.0 +urllib3==2.6.1 # via # -c gear/../hail/python/dev/pinned-requirements.txt # -c gear/../hail/python/hailtop/pinned-requirements.txt diff --git a/hail/python/dev/pinned-requirements.txt b/hail/python/dev/pinned-requirements.txt index 38a44fc4e89..798dd99dcb3 100644 --- a/hail/python/dev/pinned-requirements.txt +++ b/hail/python/dev/pinned-requirements.txt @@ -16,7 +16,7 @@ aiosignal==1.4.0 # aiohttp alabaster==0.7.16 # via sphinx -anyio==4.11.0 +anyio==4.12.0 # via # httpx # jupyter-server @@ -52,7 +52,7 @@ babel==2.17.0 # via # jupyterlab-server # sphinx -beautifulsoup4==4.14.2 +beautifulsoup4==4.14.3 # via nbconvert bleach==6.3.0 # via nbconvert @@ -115,7 +115,7 @@ docutils==0.18.1 # sphinx-rtd-theme entrypoints==0.4 # via jupyter-client -exceptiongroup==1.3.0 +exceptiongroup==1.3.1 # via # anyio # ipython @@ -130,7 +130,7 @@ fastjsonschema==2.21.2 # via nbformat filelock==3.20.0 # via virtualenv -fonttools==4.60.1 +fonttools==4.61.0 # via matplotlib fqdn==1.5.1 # via jsonschema @@ -294,7 +294,7 @@ nbformat==5.10.4 # nbconvert # nbsphinx # notebook -nbsphinx==0.9.7 +nbsphinx==0.9.8 # via -r hail/python/dev/requirements.txt nest-asyncio==1.6.0 # via @@ -352,7 +352,7 @@ pillow==12.0.0 # -c hail/python/dev/../pinned-requirements.txt # -r hail/python/dev/requirements.txt # matplotlib -platformdirs==4.5.0 +platformdirs==4.5.1 # via # jupyter-core # pylint @@ -476,7 +476,7 @@ rfc3986-validator==0.1.1 # jupyter-events rfc3987-syntax==1.1.0 # via jsonschema -rpds-py==0.29.0 +rpds-py==0.30.0 # via # jsonschema # referencing @@ -494,8 +494,6 @@ six==1.17.0 # asttokens # python-dateutil # rfc3339-validator -sniffio==1.3.1 - # via anyio snowballstemmer==3.0.1 # via sphinx soupsieve==2.8 @@ -535,7 +533,7 @@ terminado==0.18.1 # jupyter-server # jupyter-server-terminals # notebook -tinycss2==1.5.0 +tinycss2==1.5.1 # via nbconvert toml==0.10.2 # via curlylint @@ -618,13 +616,13 @@ tzdata==2025.2 # arrow uri-template==1.3.0 # via jsonschema -urllib3==2.5.0 +urllib3==2.6.1 # via # -c hail/python/dev/../pinned-requirements.txt # requests -uv==0.9.10 +uv==0.9.16 # via -r hail/python/dev/requirements.txt -uv-build==0.9.10 +uv-build==0.9.16 # via -r hail/python/dev/requirements.txt virtualenv==20.35.4 # via pre-commit diff --git a/hail/python/hailtop/pinned-requirements.txt b/hail/python/hailtop/pinned-requirements.txt index eff22825f68..d8dff76e7d4 100644 --- a/hail/python/hailtop/pinned-requirements.txt +++ b/hail/python/hailtop/pinned-requirements.txt @@ -28,9 +28,9 @@ azure-mgmt-storage==20.1.0 # via -r hail/python/hailtop/requirements.txt azure-storage-blob==12.27.1 # via -r hail/python/hailtop/requirements.txt -boto3==1.41.0 +boto3==1.42.4 # via -r hail/python/hailtop/requirements.txt -botocore==1.41.0 +botocore==1.42.4 # via # -r hail/python/hailtop/requirements.txt # boto3 @@ -104,7 +104,7 @@ nest-asyncio==1.6.0 # via -r hail/python/hailtop/requirements.txt oauthlib==3.3.1 # via requests-oauthlib -orjson==3.11.4 +orjson==3.11.5 # via -r hail/python/hailtop/requirements.txt propcache==0.4.1 # via @@ -146,7 +146,7 @@ rich==12.6.0 # typer rsa==4.9.1 # via google-auth -s3transfer==0.14.0 +s3transfer==0.16.0 # via boto3 shellingham==1.5.4 # via typer @@ -170,7 +170,7 @@ typing-extensions==4.15.0 # janus # multidict # typer -urllib3==2.5.0 +urllib3==2.6.1 # via # botocore # requests diff --git a/hail/python/pinned-requirements.txt b/hail/python/pinned-requirements.txt index da7ede4479e..4472c3b51b1 100644 --- a/hail/python/pinned-requirements.txt +++ b/hail/python/pinned-requirements.txt @@ -55,11 +55,11 @@ azure-storage-blob==12.27.1 # -r hail/python/hailtop/requirements.txt bokeh==3.4.3 # via -r hail/python/requirements.txt -boto3==1.41.0 +boto3==1.42.4 # via # -c hail/python/hailtop/pinned-requirements.txt # -r hail/python/hailtop/requirements.txt -botocore==1.41.0 +botocore==1.42.4 # via # -c hail/python/hailtop/pinned-requirements.txt # -r hail/python/hailtop/requirements.txt @@ -187,7 +187,7 @@ oauthlib==3.3.1 # via # -c hail/python/hailtop/pinned-requirements.txt # requests-oauthlib -orjson==3.11.4 +orjson==3.11.5 # via # -c hail/python/hailtop/pinned-requirements.txt # -r hail/python/hailtop/requirements.txt @@ -281,7 +281,7 @@ rsa==4.9.1 # via # -c hail/python/hailtop/pinned-requirements.txt # google-auth -s3transfer==0.14.0 +s3transfer==0.16.0 # via # -c hail/python/hailtop/pinned-requirements.txt # boto3 @@ -325,7 +325,7 @@ typing-extensions==4.15.0 # typer tzdata==2025.2 # via pandas -urllib3==2.5.0 +urllib3==2.6.1 # via # -c hail/python/hailtop/pinned-requirements.txt # botocore @@ -336,7 +336,7 @@ uvloop==0.21.0 # -r hail/python/hailtop/requirements.txt wrapt==1.17.3 # via deprecated -xyzservices==2025.10.0 +xyzservices==2025.11.0 # via bokeh yarl==1.22.0 # via From 5d54486fcac06f6b7b2e8af380f812d1552343fc Mon Sep 17 00:00:00 2001 From: grohli <22306963+grohli@users.noreply.github.com> Date: Wed, 10 Dec 2025 14:48:07 -0500 Subject: [PATCH 02/51] Remove unnecessary accessConfigs parameter from VM config API payload (#15205) ## Change Description Fixes https://github.com/hail-is/hail-security/issues/67 Currently, worker VMs are created with external IP addresses. However, there is no operational reason why they actually need those, as we don't actually need/want our workers to be accessible from the public internet. This change updates our worker VM config (sent to GCP for worker creation) to exclude giving an external IP. ## Security Assessment - This change potentially impacts the Hail Batch instance as deployed by Broad Institute in GCP ### Impact Rating - This change has a medium security impact ### Impact Description While removing public IPs from our workers makes things generally more secure, this change still relates to changing our worker VM configuration and communication. ### Appsec Review - [ ] Required: The impact has been assessed and approved by appsec --- batch/batch/cloud/gcp/driver/create_instance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/batch/batch/cloud/gcp/driver/create_instance.py b/batch/batch/cloud/gcp/driver/create_instance.py index fadcf5981f8..82f4f454b55 100644 --- a/batch/batch/cloud/gcp/driver/create_instance.py +++ b/batch/batch/cloud/gcp/driver/create_instance.py @@ -129,7 +129,6 @@ def scheduling() -> dict: 'network': 'global/networks/default', 'subnetwork': f'regions/{region}/subnetworks/default', 'networkTier': 'PREMIUM', - 'accessConfigs': [{'type': 'ONE_TO_ONE_NAT', 'name': 'external-nat'}], } ], 'scheduling': scheduling(), From e6acdb2087d1243a67a7f475b2ca2deffe0fcc3f Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Mon, 15 Dec 2025 14:30:48 -0500 Subject: [PATCH 03/51] [query] mill multi-file builds (#15207) What? - Refactors hail/build.mill to use mill's multi-file builds. - Organises dependencies via their maven coordinate to avoid typing full coordinate each time. - Uses scala 3 syntax Why? - modules like `shadeazure` and `ir-gen` are independent of the root module's cross value - listing all mvn coordinates is useful for a subsequent change where I define boms for various pyspark distributions. - Scala 3 syntax is a lot nicer. I don't think the overheads are particularly strenuous. This change cannot impact the Hail Batch instance as deployed by Broad Institute in GCP --- hail/build.mill | 530 +++++------------- hail/hail/ir-gen/package.mill | 28 + hail/hail/memory/package.mill | 16 + hail/hail/package.mill | 192 +++++++ hail/hail/shadeazure/package.mill | 42 ++ hail/mill-build/build.mill | 113 ++-- hail/mill-build/mill-build/build.mill | 5 + .../mill-build/mill-build/src/BuildMode.scala | 6 + .../mill-build/src/MvnCoordinate.scala | 410 ++++++++++++++ 9 files changed, 898 insertions(+), 444 deletions(-) create mode 100644 hail/hail/ir-gen/package.mill create mode 100644 hail/hail/memory/package.mill create mode 100644 hail/hail/package.mill create mode 100644 hail/hail/shadeazure/package.mill create mode 100644 hail/mill-build/mill-build/build.mill create mode 100644 hail/mill-build/mill-build/src/BuildMode.scala create mode 100644 hail/mill-build/mill-build/src/MvnCoordinate.scala diff --git a/hail/build.mill b/hail/build.mill index b27546b8c91..7d088127dcb 100644 --- a/hail/build.mill +++ b/hail/build.mill @@ -1,439 +1,167 @@ //| mill-version: 1.0.4 //| mill-jvm-version: 11 - package build -import com.goyeau.mill.scalafix.ScalafixModule import mill.* -import mill.api.{BuildCtx, Result} +import mill.api.Result import mill.scalalib.* -import mill.scalalib.Assembly.* -import mill.scalalib.TestModule.TestNg import mill.scalalib.scalafmt.ScalafmtModule -import mill.util.{Jvm, VcsVersion} + +import millbuild.BuildConfig.* +import millbuild.BuildMode.* +import millbuild.MvnCoordinate.* + +import com.goyeau.mill.scalafix.ScalafixModule import org.typelevel.scalacoptions.* -import upickle.default.ReadWriter -object Settings { +object Settings: val hailMajorMinorVersion = "0.2" val hailPatchVersion = "137" val scalaMinorVersions = Map("2.12" -> "2.12.20", "2.12.13" -> "2.12.13", "2.13" -> "2.13.13") -} -object Deps { - - object Asm { - val `asm-bom` = mvn"org.ow2.asm:asm-bom:9.9" - val asm = mvn"org.ow2.asm:asm" - val `asm-analysis` = mvn"org.ow2.asm:asm-analysis" - val `asm-util` = mvn"org.ow2.asm:asm-util" - } - - object Breeze { - // WARNING WARNING WARNING - // Before changing the breeze version review: - // - https://hail.zulipchat.com/#narrow/stream/123011-Hail-Query-Dev/topic/new.20spark.20ndarray.20failures/near/41645 - // - https://github.com/hail-is/hail/pull/11555 - val breeze = mvn"org.scalanlp::breeze:1.1" - val `breeze-natives` = mvn"org.scalanlp::breeze-natives:1.1" - } - object GoogleCloud { - val `libraries-bom` = mvn"com.google.cloud:libraries-bom:26.66.0" - val `google-auth-library-oauth2-http` = mvn"com.google.auth:google-auth-library-oauth2-http" - val `google-cloud-storage` = mvn"com.google.cloud:google-cloud-storage" - } - - object Spark { - def core: Task[Dep] = Task.Anon(mvn"org.apache.spark::spark-core:${Env.sparkVersion()}") - def mllib: Task[Dep] = Task.Anon(mvn"org.apache.spark::spark-mllib:${Env.sparkVersion()}") - - // direct dependencies provided by Spark at runtime - val avro = mvn"org.apache.avro:avro:1.11.2" - val `commons-codec` = mvn"commons-codec:commons-codec:1.16.1" - val `commons-io` = mvn"commons-io:commons-io:2.16.1" - val `commons-lang3` = mvn"org.apache.commons:commons-lang3:3.12.0" - val `commons-math3` = mvn"org.apache.commons:commons-math3:3.6.1" - val `hadoop-client-api` = mvn"org.apache.hadoop:hadoop-client-api:3.3.4" - val httpclient = mvn"org.apache.httpcomponents:httpclient:4.5.14" - val `jackson-core` = mvn"com.fasterxml.jackson.core:jackson-core:2.15.2" - val `json4s-jackson` = mvn"org.json4s::json4s-jackson:3.7.0-M11" - val `l4z-java` = mvn"org.lz4:lz4-java:1.8.0" - val `log4j-bom` = mvn"org.apache.logging.log4j:log4j-bom:2.22.0" - val `logj4-api` = mvn"org.apache.logging.log4j:log4j-api" - val `log4j-core` = mvn"org.apache.logging.log4j:log4j-core" - val `zstd-jni` = mvn"com.github.luben:zstd-jni:1.5.5-4" - } - - val `elasticsearch-spark` = mvn"org.elasticsearch::elasticsearch-spark-30:9.2.1" - val freemarker = mvn"org.freemarker:freemarker:2.3.31" - val htsjdk = mvn"com.github.samtools:htsjdk:3.0.5" - val jdistlib = mvn"net.sourceforge.jdistlib:jdistlib:0.4.5" - val jna = mvn"net.java.dev.jna:jna:5.13.0" - val `junixsocket-core` = mvn"com.kohlschutter.junixsocket:junixsocket-core:2.6.1" - val `log4j-api-scala` = mvn"org.apache.logging.log4j::log4j-api-scala:13.1.0" - val netlib = mvn"com.github.fommil.netlib:all:1.1.2" - val `scala-collection-compat` = mvn"org.scala-lang.modules::scala-collection-compat:2.13.0" - // provides @nowarn212 and @nowarn213 - val `scalac-compat-annotation` = mvn"org.typelevel::scalac-compat-annotation:0.1.4" - val sourcecode = mvn"com.lihaoyi::sourcecode:0.4.2" +object Env extends Module: + def sparkVersion: T[String] = + Task.Input { + Task.env.getOrElse("SPARK_VERSION", "3.5.3") + } - object Plugins { - val `better-monadic-for` = mvn"com.olegpy::better-monadic-for:0.3.1" - } -} -enum BuildMode derives ReadWriter { - case dev, ci, release -} +trait HailJavaModule extends JavaModule: + override def javacOptions: T[Seq[String]] = { + val `java-1.8` = + if System.getProperty("java.version").startsWith("1.8") + then Seq("-Xlint:-processing") + else Seq() -object Env extends Module { - def javaVersion: T[String] = Task.Input { - System.getProperty("java.version") - } + val `optim?` = + if buildMode == Release + then Seq("-O") + else Seq("-g") - def sparkVersion: T[String] = Task.Input { - Result.Success(Task.env.getOrElse("SPARK_VERSION", "3.5.3")) + `java-1.8` ++ `optim?` ++ Seq( + "-Xlint:all", + "-Werror", + ) } - val buildMode: BuildMode = BuildMode.valueOf(millbuild.BuildConfig.buildMode) -} -trait HailModule extends ScalaModule with ScalafmtModule with ScalafixModule { outer => - // crossValue is defined by the RootHailModule#CrossValue trait, which this will be mixed with. - // It propagates the cross value from the root `hail` module. - def crossValue: String - override def scalaVersion: T[String] = Task { Settings.scalaMinorVersions(crossValue) } +trait HailScalaModule extends ScalaModule, HailJavaModule, ScalafmtModule, ScalafixModule: + outer => - override def scalafixIvyDeps: T[Seq[Dep]] = Seq( - mvn"io.github.tanin47::scalafix-forbidden-symbol:1.0.0", - ) + override def scalafixIvyDeps: T[Seq[Dep]] = + Seq( + `scalafix-forbidden-symbol` :: "1.0.0", + ) override def depManagement: T[Seq[Dep]] = Seq( - Deps.`scala-collection-compat`, + `scala-collection-compat` :: "2.13.0", ) - override def javacOptions: T[Seq[String]] = Seq( - "-Xlint:all", - "-Werror", - if (Env.buildMode == BuildMode.release) "-O" else "-g", - ) ++ (if (!Env.javaVersion().startsWith("1.8")) Seq("-Xlint:-processing") else Seq()) + override def scalacOptions: T[Seq[String]] = + Task { + val scalaVersion = this.scalaVersion() - override def scalacOptions: T[Seq[String]] = Task { - val disabledOptions = Set( - ScalacOptions.warnUnusedParams, ScalacOptions.privateWarnUnusedParams, - ScalacOptions.warnUnusedImplicits, ScalacOptions.privateWarnUnusedImplicits, - ScalacOptions.warnUnusedExplicits, - // we don't enable the unused imports warning in 2.12, as the wconf origin filter doesn't work - ScalacOptions.privateWarnUnusedImports, - // 2.13 added the -Wnonunit-discard flag, which is a stronger check than -Wvalue-discard - ScalacOptions.warnValueDiscard, ScalacOptions.privateWarnValueDiscard, + val base = + ScalacOptions.default + + ScalacOptions.source213 + + ScalacOptions.advancedOption("no-patmat-analysis") - ScalacOptions.lintInferAny, - ScalacOptions.languageExperimentalMacros, - ) - - val baseOptions = ScalacOptions.default + - ScalacOptions.source213 + - ScalacOptions.advancedOption("no-patmat-analysis") - - val additionalOptions = Env.buildMode match { - case BuildMode.`dev` => - ScalacOptions.verboseOptions - // enable to help debug accessing unitialized fields - // + ScalacOptions.checkInit - case BuildMode.`ci` => - Set(ScalacOptions.fatalWarnings) - case BuildMode.`release` => + val disabled = Set( - ScalacOptions.fatalWarnings, - ScalacOptions.optimizerMethodLocal, - ScalacOptions.optimizerOption(":-closure-invocations"), + ScalacOptions.warnUnusedParams, + ScalacOptions.privateWarnUnusedParams, + ScalacOptions.warnUnusedImplicits, + ScalacOptions.privateWarnUnusedImplicits, + ScalacOptions.warnUnusedExplicits, + // we don't enable the unused imports warning in 2.12, as the wconf origin filter doesn't work + ScalacOptions.privateWarnUnusedImports, + // 2.13 added the -Wnonunit-discard flag, which is a stronger check than -Wvalue-discard + ScalacOptions.warnValueDiscard, + ScalacOptions.privateWarnValueDiscard, + ScalacOptions.lintInferAny, + ScalacOptions.languageExperimentalMacros, ) - } - - val finalOptions = ScalacOptions.tokensForVersion( - ScalaVersion.unsafeFromString(scalaVersion()), - baseOptions -- disabledOptions ++ additionalOptions - ) - if (scalaVersion().startsWith("2.13")) { - finalOptions ++ - Seq( - // collection types - // value ArrayStack in package mutable is deprecated (since 2.13.0): Use Stack instead of ArrayStack; it now uses an array-based implementation - "-Wconf:cat=deprecation&msg=value ArrayStack in package mutable is deprecated:s", - // generic to immutable default - // collection methods - // method copyArrayToImmutableIndexedSeq in class LowPriorityImplicits2 is deprecated (since 2.13.0): implicit conversions from Array to immutable.IndexedSeq are implemented by copying; use `toIndexedSeq` explicitly if you want to copy, or use the more efficient non-copying ArraySeq.unsafeWrapArray - "-Wconf:cat=deprecation&msg=method copyArrayToImmutableIndexedSeq in class LowPriorityImplicits2 is deprecated:s", - // Passing an explicit array value to a Scala varargs method is deprecated (since 2.13.0) and will result in a defensive copy; Use the more efficient non-copying ArraySeq.unsafeWrapArray or an explicit toIndexedSeq call - "-Wconf:cat=deprecation&msg=Passing an explicit array value to a Scala varargs method is deprecated:s", - // method any2stringadd in object Predef is deprecated (since 2.13.0): Implicit injection of + is deprecated. Convert to String to call + - "-Wconf:cat=deprecation&msg=any2stringadd:s", - // method + in class Char is deprecated (since 2.13.0): Adding a number and a String is deprecated. Use the string interpolation `s"$num$str"` - raw"-Wconf:cat=deprecation&msg=method \+ in class Char is deprecated:s", - // method toIterable in trait Iterable is deprecated (since 2.13.7): toIterable is internal and will be made protected; its name is similar to `toList` or `toSeq`, but it doesn't copy non-immutable collections - "-Wconf:cat=deprecation&msg=method toIterable in trait Iterable is deprecated:s", - // method toIterator in class IterableOnceExtensionMethods is deprecated (since 2.13.0): Use .iterator instead - "-Wconf:cat=deprecation&msg=toIterator:s", - // method foreach in class IterableOnceExtensionMethods is deprecated (since 2.13.0): Use .iterator.foreach(...) instead - "-Wconf:cat=deprecation&msg=foreach:s", - // method nonEmpty in class IterableOnceExtensionMethods is deprecated (since 2.13.0): Use .iterator.nonEmpty instead - "-Wconf:cat=deprecation&msg=method nonEmpty in class IterableOnceExtensionMethods is deprecated:s", - // method view in trait IndexedSeqOps is deprecated (since 2.13.0): Use .view.slice(from, until) instead of .view(from, until) - "-Wconf:cat=deprecation&msg=method view in trait IndexedSeqOps is deprecated:s", - // method reverseContents in class StringBuilder is deprecated (since 2.13.0): Use reverseInPlace instead - "-Wconf:cat=deprecation&msg=method reverseContents in class StringBuilder is deprecated:s", - // method + in trait MapOps is deprecated (since 2.13.0): Consider requiring an immutable Map or fall back to Map.concat. - raw"-Wconf:cat=deprecation&msg=method \+ in trait MapOps is deprecated:s", - "-Wconf:cat=deprecation&msg=method - in trait MapOps is deprecated:s", - // method mapValues in trait MapOps is deprecated (since 2.13.0): Use .view.mapValues(f). A future version will include a strict version of this method (for now, .view.mapValues(f).toMap). - "-Wconf:cat=deprecation&msg=method mapValues in trait MapOps is deprecated:s", - // method filterKeys in trait MapOps is deprecated (since 2.13.0): Use .view.filterKeys(f). A future version will include a strict version of this method (for now, .view.filterKeys(p).toMap). - "-Wconf:cat=deprecation&msg=method filterKeys in trait MapOps is deprecated:s", - // method - in trait SetOps is deprecated (since 2.13.0): Consider requiring an immutable Set or fall back to Set.diff - "-Wconf:cat=deprecation&msg=method - in trait SetOps is deprecated:s", - // method reverseMap in trait SeqOps is deprecated (since 2.13.0): Use .reverseIterator.map(f).to(...) instead of .reverseMap(f) - "-Wconf:cat=deprecation&msg=method reverseMap in trait SeqOps is deprecated:s", - // method transform in trait SeqOps is deprecated (since 2.13.0): Use `mapInPlace` on an `IndexedSeq` instead - "-Wconf:cat=deprecation&msg=method transform in trait SeqOps is deprecated:s", - // other - // Widening conversion from Long to Double is deprecated because it loses precision. Write `.toDouble` instead. - "-Wconf:cat=deprecation&msg=Widening conversion from Long to Double:s", - "-Wconf:cat=deprecation&msg=symbol literal is deprecated:s", - raw"-Wconf:cat=unused-imports&origin=scala\.collection\.compat\._:s", - raw"-Wconf:cat=unused-imports&origin=scala\.Option\.option2Iterable:s", - raw"-Wconf:cat=unused-imports&origin=is\.hail\.utils\.compat\._:s", - raw"-Wconf:msg=unused value of type org\.scalatest\..*Assertion:s", - raw"-Wconf:msg=unused value of type org\.mockito\.stubbing\.ScalaOngoingStubbing:s", - ) - } else { - finalOptions :+ "-Wconf:msg=legacy-binding:s" - } - } -} - -object hail extends Cross[RootHailModule](millbuild.BuildConfig.enabledScalaVersions) - -trait RootHailModule extends CrossScalaModule with HailModule { outer => - def buildInfo: T[PathRef] = Task { - val revision = VcsVersion.vcsState().currentRevision - os.write( - Task.dest / "build-info.properties", - s"""[Build Metadata] - |revision=$revision - |sparkVersion=${Env.sparkVersion()} - |hailPipVersion=${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion} - |hailBuildConfiguration=${Env.buildMode} - |""".stripMargin, - ) - PathRef(Task.dest) - } - - override def moduleDeps: Seq[JavaModule] = Seq(memory) - override def resources: T[Seq[PathRef]] = super.resources() ++ Seq( - BuildCtx.withFilesystemCheckerDisabled { - PathRef(BuildCtx.workspaceRoot / "prebuilt" / "lib") - }, - buildInfo(), - ) - - override def generatedSources: T[Seq[PathRef]] = Task { - Seq(`ir-gen`.generate()) - } - - override def unmanagedClasspath: T[Seq[PathRef]] = - Seq(shadedazure.assembly()) - - // omit unmanagedClasspath from the jar - override def jar: T[PathRef] = Task { - val jar = Task.dest / "out.jar" - Jvm.createJar(jar, (resources() ++ Seq(compile().classes)).map(_.path).filter(os.exists), manifest()) - PathRef(jar) - } - - override def bomMvnDeps: T[Seq[Dep]] = Seq( - Deps.Asm.`asm-bom`, - Deps.GoogleCloud.`libraries-bom`, - Deps.Spark.`log4j-bom`, - ) - - override def mvnDeps: T[Seq[Dep]] = Seq( - Deps.Asm.asm, - Deps.Asm.`asm-analysis`, - Deps.Asm.`asm-util`, - Deps.GoogleCloud.`google-auth-library-oauth2-http` - .excludeOrg( - "commons-codec", - "org.apache.httpcomponents", - "org.slf4j", - ), - Deps.GoogleCloud.`google-cloud-storage` - .excludeOrg( - "com.fasterxml.jackson.core", - "commons-codec", - "org.slf4j", - ), - Deps.`elasticsearch-spark` - .excludeOrg( - "commons-logging", - "org.apache.spark", - "org.slf4j", - ), - Deps.freemarker, - Deps.htsjdk.excludeOrg("*"), - Deps.jdistlib.excludeOrg("*"), - Deps.`log4j-api-scala`.excludeOrg("*"), - Deps.jna, - Deps.`scala-collection-compat`, - Deps.sourcecode, - ) - - override def runMvnDeps: T[Seq[Dep]] = Seq( - Deps.Breeze.`breeze-natives` - .excludeOrg( - "org.apache.commons", - "org.slf4j", - ), - Deps.netlib, - Deps.`junixsocket-core`, - ) - - override def compileMvnDeps: T[Seq[Dep]] = Seq( - Deps.Breeze.breeze, - Deps.Spark.core().excludeOrg("org.scalanlp"), // Hail has an explicit dependency on Breeze 1.1 - Deps.Spark.mllib().excludeOrg("org.scalanlp"), // Hail has an explicit dependency on Breeze 1.1 - Deps.Spark.avro, - Deps.Spark.`commons-codec`, - Deps.Spark.`commons-io`, - Deps.Spark.`commons-lang3`, - Deps.Spark.`commons-math3`, - Deps.Spark.httpclient, - Deps.Spark.`hadoop-client-api`, - Deps.Spark.`jackson-core`, - Deps.Spark.`json4s-jackson`, - Deps.Spark.`l4z-java`, - Deps.Spark.`logj4-api`, - Deps.Spark.`log4j-core`, - Deps.Spark.`zstd-jni`, - Deps.`scalac-compat-annotation`, - ) - - override def assemblyRules: Seq[Rule] = super.assemblyRules ++ Seq( - Rule.Exclude("META-INF/INDEX.LIST"), - Rule.ExcludePattern("^scala/(?!collection/compat).*"), - Rule.AppendPattern("META-INF/services/.*", "\n"), - Rule.Relocate("breeze.**", "is.hail.relocated.@0"), - Rule.Relocate("com.google.**", "is.hail.relocated.@0"), - Rule.Relocate("scala.collection.compat.**", "is.hail.relocated.@0"), - ) - - override def scalacPluginMvnDeps: T[Seq[Dep]] = Seq( - Deps.Plugins.`better-monadic-for`, - ) - - def writeRunClasspath: T[PathRef] = Task { - os.write( - Task.dest / "runClasspath", - runClasspath().map(_.path).mkString(":"), - ) - PathRef(Task.dest) - } - - object `ir-gen` extends HailModule with DefaultTaskModule with CrossValue { - override def defaultTask(): String = "generate" - - override def mvnDeps = Seq( - mvn"com.lihaoyi::mainargs:0.7.7", - mvn"com.lihaoyi::os-lib:0.10.7", - mvn"com.lihaoyi::sourcecode:0.4.2", - ) - - def generate: T[PathRef] = Task { - runner().run(Args("--path", Task.dest).value) - PathRef(Task.dest) - } - } - - object memory extends JavaModule with CrossValue { - override def zincIncrementalCompilation: T[Boolean] = false + val fromBuildMode = + buildMode match { + case Dev => + ScalacOptions.verboseOptions + // + ScalacOptions.checkInit // enable to help debug accessing uninitialized fields + case CI => + Set(ScalacOptions.fatalWarnings) + case Release => + Set( + ScalacOptions.fatalWarnings, + ScalacOptions.optimizerMethodLocal, + ScalacOptions.optimizerOption(":-closure-invocations"), + ) + } + + val richOptions = + ScalacOptions.tokensForVersion( + ScalaVersion.unsafeFromString(scalaVersion), + base -- disabled ++ fromBuildMode, + ) - override def javacOptions: T[Seq[String]] = - outer.javacOptions() ++ ( - if (Env.javaVersion().startsWith("1.8")) Seq( - "-XDenableSunApiLintControl", - "-Xlint:-sunapi", + val rawOptions = + if scalaVersion < "2.12" || scalaVersion >= "3" + then sys.error(s"scalaVersion must be 2.12 or 2.13. Got '$scalaVersion'.") + else if scalaVersion < "2.13" + then Seq("-Wconf:msg=legacy-binding:s") + else Seq( + // collection types + // value ArrayStack in package mutable is deprecated (since 2.13.0): Use Stack instead of ArrayStack; it now uses an array-based implementation + "-Wconf:cat=deprecation&msg=value ArrayStack in package mutable is deprecated:s", + // generic to immutable default + // collection methods + // method copyArrayToImmutableIndexedSeq in class LowPriorityImplicits2 is deprecated (since 2.13.0): implicit conversions from Array to immutable.IndexedSeq are implemented by copying; use `toIndexedSeq` explicitly if you want to copy, or use the more efficient non-copying ArraySeq.unsafeWrapArray + "-Wconf:cat=deprecation&msg=method copyArrayToImmutableIndexedSeq in class LowPriorityImplicits2 is deprecated:s", + // Passing an explicit array value to a Scala varargs method is deprecated (since 2.13.0) and will result in a defensive copy; Use the more efficient non-copying ArraySeq.unsafeWrapArray or an explicit toIndexedSeq call + "-Wconf:cat=deprecation&msg=Passing an explicit array value to a Scala varargs method is deprecated:s", + // method any2stringadd in object Predef is deprecated (since 2.13.0): Implicit injection of + is deprecated. Convert to String to call + + "-Wconf:cat=deprecation&msg=any2stringadd:s", + // method + in class Char is deprecated (since 2.13.0): Adding a number and a String is deprecated. Use the string interpolation `s"$num$str"` + raw"-Wconf:cat=deprecation&msg=method \+ in class Char is deprecated:s", + // method toIterable in trait Iterable is deprecated (since 2.13.7): toIterable is internal and will be made protected; its name is similar to `toList` or `toSeq`, but it doesn't copy non-immutable collections + "-Wconf:cat=deprecation&msg=method toIterable in trait Iterable is deprecated:s", + // method toIterator in class IterableOnceExtensionMethods is deprecated (since 2.13.0): Use .iterator instead + "-Wconf:cat=deprecation&msg=toIterator:s", + // method foreach in class IterableOnceExtensionMethods is deprecated (since 2.13.0): Use .iterator.foreach(...) instead + "-Wconf:cat=deprecation&msg=foreach:s", + // method nonEmpty in class IterableOnceExtensionMethods is deprecated (since 2.13.0): Use .iterator.nonEmpty instead + "-Wconf:cat=deprecation&msg=method nonEmpty in class IterableOnceExtensionMethods is deprecated:s", + // method view in trait IndexedSeqOps is deprecated (since 2.13.0): Use .view.slice(from, until) instead of .view(from, until) + "-Wconf:cat=deprecation&msg=method view in trait IndexedSeqOps is deprecated:s", + // method reverseContents in class StringBuilder is deprecated (since 2.13.0): Use reverseInPlace instead + "-Wconf:cat=deprecation&msg=method reverseContents in class StringBuilder is deprecated:s", + // method + in trait MapOps is deprecated (since 2.13.0): Consider requiring an immutable Map or fall back to Map.concat. + raw"-Wconf:cat=deprecation&msg=method \+ in trait MapOps is deprecated:s", + "-Wconf:cat=deprecation&msg=method - in trait MapOps is deprecated:s", + // method mapValues in trait MapOps is deprecated (since 2.13.0): Use .view.mapValues(f). A future version will include a strict version of this method (for now, .view.mapValues(f).toMap). + "-Wconf:cat=deprecation&msg=method mapValues in trait MapOps is deprecated:s", + // method filterKeys in trait MapOps is deprecated (since 2.13.0): Use .view.filterKeys(f). A future version will include a strict version of this method (for now, .view.filterKeys(p).toMap). + "-Wconf:cat=deprecation&msg=method filterKeys in trait MapOps is deprecated:s", + // method - in trait SetOps is deprecated (since 2.13.0): Consider requiring an immutable Set or fall back to Set.diff + "-Wconf:cat=deprecation&msg=method - in trait SetOps is deprecated:s", + // method reverseMap in trait SeqOps is deprecated (since 2.13.0): Use .reverseIterator.map(f).to(...) instead of .reverseMap(f) + "-Wconf:cat=deprecation&msg=method reverseMap in trait SeqOps is deprecated:s", + // method transform in trait SeqOps is deprecated (since 2.13.0): Use `mapInPlace` on an `IndexedSeq` instead + "-Wconf:cat=deprecation&msg=method transform in trait SeqOps is deprecated:s", + // other + // Widening conversion from Long to Double is deprecated because it loses precision. Write `.toDouble` instead. + "-Wconf:cat=deprecation&msg=Widening conversion from Long to Double:s", + "-Wconf:cat=deprecation&msg=symbol literal is deprecated:s", + raw"-Wconf:cat=unused-imports&origin=scala\.collection\.compat\._:s", + raw"-Wconf:cat=unused-imports&origin=scala\.Option\.option2Iterable:s", + raw"-Wconf:cat=unused-imports&origin=is\.hail\.utils\.compat\._:s", + raw"-Wconf:msg=unused value of type org\.scalatest\..*Assertion:s", + raw"-Wconf:msg=unused value of type org\.mockito\.stubbing\.ScalaOngoingStubbing:s", ) - else Seq() - ) - override def sources: T[Seq[PathRef]] = Task.Sources { - val debugOrRelease = if (Env.buildMode == BuildMode.ci) "debug" else "release" - os.sub / debugOrRelease / "src" + richOptions ++ rawOptions } - } - - object test extends ScalaTests with TestNg with ScalafmtModule with ScalafixModule { - override def forkArgs: T[Seq[String]] = - Seq("-Xss4m", "-Xmx4096M") - - override def scalafixIvyDeps: T[Seq[Dep]] = - outer.scalafixIvyDeps() - - override def compileMvnDeps: T[Seq[Dep]] = - outer.compileMvnDeps() - override def mvnDeps: T[Seq[Dep]] = - outer.mvnDeps() ++ Seq( - mvn"com.google.inject:guice:5.1.0", - mvn"org.scalacheck::scalacheck:1.18.1", - mvn"org.scalatest::scalatest:3.2.19", - mvn"org.scalatest::scalatest-shouldmatchers:3.2.19", - mvn"org.scalatestplus::scalacheck-1-18:3.2.19.0", - mvn"org.scalatestplus::testng-7-10:3.2.19.0", - mvn"org.mockito::mockito-scala:1.17.31", - ) - - override def runMvnDeps: T[Seq[Dep]] = - outer.runMvnDeps() - - override def assemblyRules: Seq[Rule] = - outer.assemblyRules - } - - object shadedazure extends JavaModule { - override def mvnDeps: T[Seq[Dep]] = Seq( - mvn"com.azure:azure-storage-blob:12.22.0", - mvn"com.azure:azure-core-http-netty:1.13.7", - mvn"com.azure:azure-identity:1.8.3", - ) - - override def assemblyRules: Seq[Rule] = Seq( - Rule.ExcludePattern("META-INF/*.RSA"), - Rule.ExcludePattern("META-INF/*.SF"), - Rule.ExcludePattern("META-INF/*.DSA"), - Rule.Relocate("com.azure.**", "is.hail.shadedazure.@0"), - Rule.Relocate("com.ctc.**", "is.hail.shadedazure.@0"), - Rule.Relocate("com.fasterxml.**", "is.hail.shadedazure.@0"), - Rule.Relocate("com.microsoft.**", "is.hail.shadedazure.@0"), - Rule.Relocate("com.nimbusds.**", "is.hail.shadedazure.@0"), - Rule.Relocate("com.sun.**", "is.hail.shadedazure.@0"), - Rule.Relocate("io.netty.**", "is.hail.shadedazure.@0"), - Rule.Relocate("is.hail.**", "is.hail.shadedazure.@0"), - Rule.Relocate("net.jcip.**", "is.hail.shadedazure.@0"), - Rule.Relocate("net.minidev.**", "is.hail.shadedazure.@0"), - Rule.Relocate("org.apache.**", "is.hail.shadedazure.@0"), - Rule.Relocate("org.codehaus.**", "is.hail.shadedazure.@0"), - Rule.Relocate("org.objectweb.**", "is.hail.shadedazure.@0"), - Rule.Relocate("org.reactivestreams.**", "is.hail.shadedazure.@0"), - Rule.Relocate("org.slf4j.**", "is.hail.shadedazure.@0"), - Rule.Relocate("reactor.adapter.**", "is.hail.shadedazure.@0"), - Rule.Relocate("reactor.core.**", "is.hail.shadedazure.@0"), - Rule.Relocate("reactor.netty.**", "is.hail.shadedazure.@0"), - Rule.Relocate("reactor.util.**", "is.hail.shadedazure.@0"), - ) - } + trait HailScalaTests extends ScalaTests, HailScalaModule -} diff --git a/hail/hail/ir-gen/package.mill b/hail/hail/ir-gen/package.mill new file mode 100644 index 00000000000..2f5cbbc5fa2 --- /dev/null +++ b/hail/hail/ir-gen/package.mill @@ -0,0 +1,28 @@ +package build.hail.`ir-gen` + +import mill.{Args, DefaultTaskModule, PathRef, T, Task} +import mill.scalalib.Dep + +import millbuild.MvnCoordinate.* + +object `package` extends build.HailScalaModule, DefaultTaskModule: + + override def scalaVersion: T[String] = + "2.13.18" + + override def defaultTask(): String = + "generate" + + override def mvnDeps: T[Seq[Dep]] = + Seq[Dep]( + `mainargs` :: "0.7.7", + `os-lib` :: "0.10.7", + `sourcecode` :: "0.4.2", + `scala-collection-compat`, + ) + + def generate: T[PathRef] = + Task { + runner().run(Args("--path", Task.dest).value) + PathRef(Task.dest) + } \ No newline at end of file diff --git a/hail/hail/memory/package.mill b/hail/hail/memory/package.mill new file mode 100644 index 00000000000..6e74496fb9b --- /dev/null +++ b/hail/hail/memory/package.mill @@ -0,0 +1,16 @@ +package build.hail.memory + +import mill.{Cross, T} + +object `package` extends Cross[Memory]("debug", "release") +trait Memory extends build.HailJavaModule, Cross.Module[String]: + + override def moduleDir: os.Path = + super.moduleDir / crossValue + + override def javacOptions: T[Seq[String]] = + super.javacOptions() ++ ( + if scala.util.Properties.versionString.startsWith("1.8") + then Seq("-XDenableSunApiLintControl", "-Xlint:-sunapi") + else Seq() + ) \ No newline at end of file diff --git a/hail/hail/package.mill b/hail/hail/package.mill new file mode 100644 index 00000000000..ecfabf6f41f --- /dev/null +++ b/hail/hail/package.mill @@ -0,0 +1,192 @@ +package build.hail + +import mill.* +import mill.api.{BuildCtx, Result} +import mill.scalalib.* +import mill.scalalib.Assembly.* +import mill.scalalib.TestModule.TestNg +import mill.util.{Jvm, VcsVersion} + +import build.Env +import build.Settings +import build.HailScalaModule + +import millbuild.BuildConfig.* +import millbuild.BuildMode.* +import millbuild.MvnCoordinate.* + +object `package` extends Cross[RootHailModule](enabledScalaVersions) + +trait RootHailModule extends CrossScalaModule, HailScalaModule: + outer => + + override def crossScalaVersion: String = + Settings.scalaMinorVersions(crossValue) + + def buildInfo: T[PathRef] = Task { + val revision = VcsVersion.vcsState().currentRevision + os.write( + Task.dest / "build-info.properties", + s"""[Build Metadata] + |revision=$revision + |sparkVersion=${Env.sparkVersion()} + |hailPipVersion=${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion} + |hailBuildConfiguration=${buildMode.toString.toLowerCase} + |""".stripMargin, + ) + PathRef(Task.dest) + } + + override def moduleDeps: Seq[JavaModule] = + Seq( + build.hail.memory(if buildMode == CI then "debug" else "release"), + ) + + override def resources: T[Seq[PathRef]] = + super.resources() ++ Seq( + BuildCtx.withFilesystemCheckerDisabled { + PathRef(BuildCtx.workspaceRoot / "prebuilt" / "lib") + }, + buildInfo(), + ) + + override def generatedSources: T[Seq[PathRef]] = + Task { + Seq(build.hail.`ir-gen`.generate()) + } + + override def unmanagedClasspath: T[Seq[PathRef]] = + Seq(build.hail.shadeazure.assembly()) + + // omit unmanagedClasspath from the jar + override def jar: T[PathRef] = + Task { + val jar = Task.dest / "out.jar" + Jvm.createJar(jar, (resources() ++ Seq(compile().classes)).map(_.path).filter(os.exists), manifest()) + PathRef(jar) + } + + override def bomMvnDeps: T[Seq[Dep]] = + Seq( + `asm-bom` :: "9.9", + `libraries-bom` :: "26.66.0", + `log4j-bom` :: "2.22.0", + ) + + override def compileMvnDeps: T[Seq[Dep]] = + Seq[Dep]( + // WARNING WARNING WARNING + // Before changing the breeze version review: + // - https://hail.zulipchat.com/#narrow/stream/123011-Hail-Query-Dev/topic/new.20spark.20ndarray.20failures/near/41645 + // - https://github.com/hail-is/hail/pull/11555 + `breeze` :: "1.1", + `avro` :: "1.11.2", + `commons-codec` :: "1.16.1", + `commons-io` :: "2.16.1", + `commons-lang3` :: "3.12.0", + `commons-math3` :: "3.6.1", + `hadoop-client-api` :: "3.3.4", + `httpclient` :: "4.5.14", + `jackson-core` :: "2.15.2", + `json4s-jackson` :: "3.7.0-M11", + `lz4-java` :: "1.8.0", + `log4j-api`, + `log4j-core`, + `zstd-jni` :: "1.5.5-4", + `scalac-compat-annotation` :: "0.1.4", + (`spark-core` :: Env.sparkVersion()).excludeOrg("org.scalanlp"), + (`spark-mllib` :: Env.sparkVersion()).excludeOrg("org.scalanlp"), + ) + + override def mvnDeps: T[Seq[Dep]] = + Seq[Dep]( + `all` :: "1.1.2", + `asm`, + `asm-analysis`, + `asm-util`, + `google-auth-library-oauth2-http` + .mvn + .excludeOrg( + "commons-codec", + "org.apache.httpcomponents", + "org.slf4j", + ), + `google-cloud-storage` + .mvn + .excludeOrg( + "com.fasterxml.jackson.core", + "commons-codec", + "org.slf4j", + ), + (`elasticsearch-spark` :: "9.2.1") + .excludeOrg( + "commons-logging", + "org.apache.spark", + "org.slf4j", + ), + `freemarker` :: "2.3.31", + (`htsjdk` :: "3.0.5").excludeOrg("*"), + (`jdistlib` :: "0.4.5").excludeOrg("*"), + `jna` :: "5.13.0", + (`log4j-api-scala` :: "13.1.0").excludeOrg("*"), + `scala-collection-compat`, + `sourcecode` :: "0.4.2", + ) + + override def runMvnDeps: T[Seq[Dep]] = + Seq( + (`breeze-natives` :: "1.1") + .excludeOrg( + "org.apache.commons", + "org.slf4j", + ), + `junixsocket-core` :: "2.6.1", + ) + + override def assemblyRules: Seq[Rule] = + super.assemblyRules ++ Seq( + Rule.Exclude("META-INF/INDEX.LIST"), + Rule.ExcludePattern("^scala/(?!collection/compat).*"), + Rule.AppendPattern("META-INF/services/.*", "\n"), + Rule.Relocate("breeze.**", "is.hail.relocated.@0"), + Rule.Relocate("com.google.**", "is.hail.relocated.@0"), + Rule.Relocate("scala.collection.compat.**", "is.hail.relocated.@0"), + ) + + override def scalacPluginMvnDeps: T[Seq[Dep]] = + Seq( + `better-monadic-for` :: "0.3.1", + ) + + def writeRunClasspath: T[PathRef] = + Task { + os.write( + Task.dest / "runClasspath", + runClasspath().map(_.path).mkString(":"), + ) + PathRef(Task.dest) + } + + object test extends HailScalaTests, TestNg, CrossValue: + override def forkArgs: T[Seq[String]] = + Seq("-Xss4m", "-Xmx4096M") + + override def compileMvnDeps: T[Seq[Dep]] = + outer.compileMvnDeps() + + override def mvnDeps: T[Seq[Dep]] = + outer.mvnDeps() ++ Seq( + `guice` :: "5.1.0", + `mockito-scala` :: "1.17.31", + `scalacheck` :: "1.18.1", + `scalacheck-1-18` :: "3.2.19.0", + `scalatest` :: "3.2.19", + `scalatest-shouldmatchers` :: "3.2.19", + `testng-7-10` :: "3.2.19.0", + ) + + override def runMvnDeps: T[Seq[Dep]] = + outer.runMvnDeps() + + override def assemblyRules: Seq[Rule] = + outer.assemblyRules diff --git a/hail/hail/shadeazure/package.mill b/hail/hail/shadeazure/package.mill new file mode 100644 index 00000000000..6862b9b175a --- /dev/null +++ b/hail/hail/shadeazure/package.mill @@ -0,0 +1,42 @@ +package build.hail.shadeazure + +import mill.T +import mill.javalib.{Dep, JavaModule} +import mill.javalib.Assembly.Rule + +import millbuild.MvnCoordinate.* + +object `package` extends JavaModule: + + override def mvnDeps: T[Seq[Dep]] = + Seq( + `azure-core-http-netty` :: "1.13.7", + `azure-identity` :: "1.8.3", + `azure-storage-blob` :: "12.22.0", + ) + + override def assemblyRules: Seq[Rule] = + Seq( + Rule.ExcludePattern("META-INF/*.RSA"), + Rule.ExcludePattern("META-INF/*.SF"), + Rule.ExcludePattern("META-INF/*.DSA"), + Rule.Relocate("com.azure.**", "is.hail.shadedazure.@0"), + Rule.Relocate("com.ctc.**", "is.hail.shadedazure.@0"), + Rule.Relocate("com.fasterxml.**", "is.hail.shadedazure.@0"), + Rule.Relocate("com.microsoft.**", "is.hail.shadedazure.@0"), + Rule.Relocate("com.nimbusds.**", "is.hail.shadedazure.@0"), + Rule.Relocate("com.sun.**", "is.hail.shadedazure.@0"), + Rule.Relocate("io.netty.**", "is.hail.shadedazure.@0"), + Rule.Relocate("is.hail.**", "is.hail.shadedazure.@0"), + Rule.Relocate("net.jcip.**", "is.hail.shadedazure.@0"), + Rule.Relocate("net.minidev.**", "is.hail.shadedazure.@0"), + Rule.Relocate("org.apache.**", "is.hail.shadedazure.@0"), + Rule.Relocate("org.codehaus.**", "is.hail.shadedazure.@0"), + Rule.Relocate("org.objectweb.**", "is.hail.shadedazure.@0"), + Rule.Relocate("org.reactivestreams.**", "is.hail.shadedazure.@0"), + Rule.Relocate("org.slf4j.**", "is.hail.shadedazure.@0"), + Rule.Relocate("reactor.adapter.**", "is.hail.shadedazure.@0"), + Rule.Relocate("reactor.core.**", "is.hail.shadedazure.@0"), + Rule.Relocate("reactor.netty.**", "is.hail.shadedazure.@0"), + Rule.Relocate("reactor.util.**", "is.hail.shadedazure.@0"), + ) \ No newline at end of file diff --git a/hail/mill-build/build.mill b/hail/mill-build/build.mill index 267cea4f7f8..d890846d080 100644 --- a/hail/mill-build/build.mill +++ b/hail/mill-build/build.mill @@ -1,61 +1,88 @@ package build -import mill.*, scalalib.* +import mill.T +import mill.api.{PathRef, Task, TaskCtx} import mill.meta.MillBuildRootModule +import mill.scalalib.Dep -object `package` extends MillBuildRootModule { - def mvnDeps = Seq( - mvn"com.goyeau::mill-scalafix::0.6.0", - mvn"org.typelevel::scalac-options:0.1.8", - ) +import millbuild.BuildMode +import millbuild.MvnCoordinate.* + +import scala.collection.immutable.ArraySeq + +extension [A, F[+X] <: Iterable[X]] (as: F[A]) + def quoted: Iterable[String] = + as.map(i => s"\"$i\"") + + infix def or[B >: A](alternative: => F[B]): F[B] = + if as.nonEmpty then as else alternative + + +object `package` extends MillBuildRootModule: + + private val SupportedScalaVersions = + Set("2.12", "2.12.13", "2.13") + + override def mvnDeps: T[Seq[Dep]] = + Seq( + `mill-scalafix` :::: "0.6.0", + `scalac-options` :: "0.1.8", + ) + + private def customSources: T[Seq[PathRef]] = + Task.Sources("mill-build/src") + + override def sources: T[Seq[PathRef]] = + super.sources() ++ customSources() def generatedSources: T[Seq[PathRef]] = Task { os.write( Task.dest / "BuildConfig.scala", - s""" - |package millbuild - |object BuildConfig{ - | def buildMode = "${buildMode()}" - | def enabledScalaVersions = IndexedSeq(${enabledScalaVersions().map(v => s"\"$v\"").mkString(", ")}) + s"""package millbuild + | + |import scala.collection.immutable.* + | + |object BuildConfig { + | val buildMode = + | BuildMode.${buildMode().getOrElse(BuildMode.Release)} + | val enabledScalaVersions = + | ${(enabledScalaVersions() or SupportedScalaVersions.to(ArraySeq)).quoted} |} """.stripMargin ) super.generatedSources() ++ Seq(PathRef(Task.dest)) } - def configDir: T[PathRef] = Task.Source("config") - - def envScalaVersion: T[Option[String]] = Task.Input { - Task.env.get("SCALA_VERSION") - } + def getConfig(name: String)(implicit ctx: TaskCtx.Env): Seq[String] = + ctx.env + .get(name map { case '-' => '_'; case c => c.toUpper }) + .orElse { + val path = moduleDir / "config" / name + Option.when(os.exists(path))(os.read(path)) + } + .to(ArraySeq) + .flatMap(_.split(',').map(_.strip)) - def enabledScalaVersions: T[Seq[String]] = Task { - val path = configDir().path / "scala-version" - val supportedScalaVersions = IndexedSeq("2.12", "2.12.13", "2.13") - envScalaVersion() - .orElse(Option.when(os.exists(path))(os.read(path).strip)) - .map { scalaVersion => + def enabledScalaVersions: T[Seq[String]] = + Task.Input { + getConfig("scala-version").tapEach { ver => require( - supportedScalaVersions.contains(scalaVersion), - s"SCALA_VERSION must be set to one of 2.12.13, 2.12, or 2.13; found $scalaVersion" + SupportedScalaVersions `contains` ver, + s"SCALA_VERSION may be one of ${SupportedScalaVersions.quoted.mkString(",")}; found '$ver'." ) - IndexedSeq(scalaVersion) - }.getOrElse(supportedScalaVersions) - } + } + } - def envBuildMode: T[Option[String]] = Task.Input { - Task.env.get("HAIL_BUILD_MODE") - } - - def buildMode: T[String] = Task { - val path = configDir().path / "hail-build-mode" - val raw = envBuildMode().orElse(Option.when(os.exists(path))(os.read(path).strip)) - .getOrElse("Release") - val supportedBuildModes = IndexedSeq("dev", "ci", "release") - require( - supportedBuildModes.contains(raw.toLowerCase), - s"HAIL_BUILD_MODE must be set to one of 'dev', 'ci' or 'release'; found '$raw'" - ) - raw.toLowerCase - } -} \ No newline at end of file + def buildMode: T[Option[BuildMode]] = + Task.Input { + getConfig("hail-build-mode").headOption.map { raw => + try + BuildMode.valueOf(raw) + catch { + case _: Throwable => + sys.error( + s"HAIL_BUILD_MODE must be set to one of ${BuildMode.values.mkString("{", ", ", "}")}; found '$raw'" + ) + } + } + } \ No newline at end of file diff --git a/hail/mill-build/mill-build/build.mill b/hail/mill-build/mill-build/build.mill new file mode 100644 index 00000000000..988c54d772a --- /dev/null +++ b/hail/mill-build/mill-build/build.mill @@ -0,0 +1,5 @@ +package build + +import mill.meta.MillBuildRootModule + +object `package` extends MillBuildRootModule \ No newline at end of file diff --git a/hail/mill-build/mill-build/src/BuildMode.scala b/hail/mill-build/mill-build/src/BuildMode.scala new file mode 100644 index 00000000000..6139555414c --- /dev/null +++ b/hail/mill-build/mill-build/src/BuildMode.scala @@ -0,0 +1,6 @@ +package millbuild + +import upickle.default.ReadWriter + +enum BuildMode derives ReadWriter: + case Dev, CI, Release \ No newline at end of file diff --git a/hail/mill-build/mill-build/src/MvnCoordinate.scala b/hail/mill-build/mill-build/src/MvnCoordinate.scala new file mode 100644 index 00000000000..31b83b98c61 --- /dev/null +++ b/hail/mill-build/mill-build/src/MvnCoordinate.scala @@ -0,0 +1,410 @@ +package millbuild + +import mill.scalalib.* + +object MvnCoordinate: + + extension (coordinate: String) { + infix def ::(version: String): Dep = mvn"$coordinate:$version" + infix def ::::(version: String): Dep = mvn"$coordinate::$version" + def mvn: Dep = mvn"$coordinate" + } + + implicit def mvnCoordinateToDep: String => Dep = + _.mvn + + val `accessors-smart` = "net.minidev:accessors-smart" + val `activation` = "javax.activation:activation" + val `aircompressor` = "io.airlift:aircompressor" + val `algebra` = "org.typelevel::algebra" + val `all` = "com.github.fommil.netlib:all" + val `annotations` = "org.jetbrains:annotations" + val `antlr4-runtime` = "org.antlr:antlr4-runtime" + val `antlr-runtime` = "org.antlr:antlr-runtime" + val `aopalliance` = "aopalliance:aopalliance" + val `aopalliance-repackaged` = "org.glassfish.hk2.external:aopalliance-repackaged" + val `apacheds-i18n` = "org.apache.directory.server:apacheds-i18n" + val `apacheds-kerberos-codec` = "org.apache.directory.server:apacheds-i18n" + val `api-asn1-api` = "org.apache.directory.api:api-asn1-api" + val `api-util` = "org.apache.directory.api:api-asn1-api" + val `asm` = "org.ow2.asm:asm" + val `asm-analysis` = "org.ow2.asm:asm-analysis" + val `asm-bom` = "org.ow2.asm:asm-bom" + val `asm-util` = "org.ow2.asm:asm-util" + val `arpack` = "dev.ludovic.netlib:arpack" + val `arpack_combined_all` = "dev.ludovic.netlib:arpack_combined_all" + val `arrow-format` = "org.apache.arrow:arrow-format" + val `arrow-memory` = "org.apache.arrow:arrow-memory" + val `arrow-memory-core` = "org.apache.arrow:arrow-memory-core" + val `arrow-memory-netty` = "org.apache.arrow:arrow-memory-netty" + val `arrow-memory-netty-buffer-patch` = "org.apache.arrow:arrow-memory-netty-buffer-patch" + val `arrow-vector` = "org.apache.arrow:arrow-vector" + val `audience-annotations` = "org.apache.yetus:audience-annotations" + val `automation` = "dk.bricks.automation:automation" + val `avro` = "org.apache.avro:avro" + val `avro-ipc` = "org.apache.avro:avro-ipc" + val `avro-mapred` = "org.apache.avro:avro-mapred" + val `azure-core-http-netty` = "com.azure:azure-core-http-netty" + val `azure-identity` = "com.azure:azure-identity" + val `azure-storage-blob` = "com.azure:azure-storage-blob" + val `bcprov-jdk18on` = "org.bouncycastle:bcprov-jdk18on" + val `better-monadic-for` = "com.olegpy::better-monadic-for" + val `blas` = "dev.ludovic.netlib:blas" + val `bonecp` = "org.jolbox:bonecp" + val `breeze` = "org.scalanlp::breeze" + val `breeze-macros` = "org.scalanlp::breeze-macros" + val `breeze-natives` = "org.scalanlp::breeze-natives" + val `cats-kernel` = "org.typelevel::cats-kernel" + val `chill` = "org.twitter::chill" + val `chill-java` = "org.twitter:chill-java" + val `commons-beanutils` = "commons-beanutils:commons-beanutils" + val `commons-cli` = "commons-cli:commons-cli" + val `commons-codec` = "commons-codec:commons-codec" + val `commons-collections` = "org.apache.commons:commons-collections" + val `commons-collections4` = "org.apache.commons:commons-collections4" + val `commons-compiler` = "org.codehaus.janino:commons-compiler" + val `commons-compress` = "org.apache.commons:commons-compress" + val `commons-configuration` = "commons-configuration:commons-configuration" + val `commons-configuration2` = "org.apache.commons:commons-configuration2" + val `commons-crypto` = "org.apache.commons:commons-crypto" + val `commons-daemon` = "commons-daemon:commons-daemon" + val `commons-dhcp` = "commons-dhcp:commons-dhcp" + val `commons-digester` = "commons-digester:commons-digester" + val `commons-httpclient` = "commons-httpclient:commons-httpclient" + val `commons-io` = "commons-io:commons-io" + val `commons-lang` = "commons-lang:commons-lang" + val `commons-lang3` = "org.apache.commons:commons-lang3" + val `commons-logging` = "commons-logging:commons-logging" + val `commons-math3` = "org.apache.commons:commons-math3" + val `commons-net` = "commons-net:commons-net" + val `commons-pool` = "commons-pool:commons-pool" + val `commons-text` = "org.apache.commons:commons-text" + val `compress-lzf` = "com.ning:compress-lzf" + val `core` = "com.github.fommil.netlib:core" + val `curator-client` = "org.apache.curator:curator-client" + val `curator-framework` = "org.apache.curator:curator-framework" + val `curator-recipes` = "org.apache.curator:curator-recipes" + val `datanucleus-api-jdo` = "org.datanucleus:datanucleus-api-jdo" + val `datanucleus-core` = "org.datanucleus:datanucleus-core" + val `datanucleus-rdbms` = "org.datanucleus:datanucleus-rdbms" + val `datasketches-java` = "org.apache.datasketches:datasketches-java" + val `datasketches-memory` = "org.apache.datasketches:datasketches-memory" + val `derby` = "org.apache.derby:derby" + val `dnsjava` = "dnsjava:dnsjava" + val `dropwizard-metrics-hadoop-metrics2-reporter` = "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter" + val `ehcache` = "org.ehcache:ehcache" + val `elasticsearch-spark` = "org.elasticsearch::elasticsearch-spark-30" + val `error_prone_annotations` = "com.google.errorprone:error_prone_annotations" + val `failureaccess` = "com.google.guava:failureaccess" + val `freemarker` = "org.freemarker:freemarker" + val `flatbuffers-java` = "com.google.flatbuffers:flatbuffers-java" + val `gcs-connector` = "com.google.cloud.bigdataoss:gcs-connector" + val `generex` = "com.github.mifmif:generex" + val `geronimo-jcache_1.0_spec` = "org.apache.geronimo.specs:geronimo-jcache_1.0_spec" + val `gson` = "com.google.code.gson:gson" + val `guava` = "com.google.guava:guava" + val `guice` = "com.google.inject:guice" + val `guice-servlet` = "com.google.inject:guice-servlet" + val `google-auth-library-oauth2-http` = "com.google.auth:google-auth-library-oauth2-http" + val `google-cloud-storage` = "com.google.cloud:google-cloud-storage" + val `hadoop-annotations` = "org.apache.hadoop:hadoop-annotations" + val `hadoop-auth` = "org.apache.hadoop:hadoop-auth" + val `hadoop-client` = "org.apache.hadoop:hadoop-client" + val `hadoop-client-api` = "org.apache.hadoop:hadoop-client-api" + val `hadoop-client-runtime` = "org.apache.hadoop:hadoop-client-runtime" + val `hadoop-common` = "org.apache.hadoop:hadoop-common" + val `hadoop-hdfs` = "org.apache.hadoop:hadoop-hdfs" + val `hadoop-hdfs-client` = "org.apache.hadoop:hadoop-hdfs-client" + val `hadoop-mapreduce-client-app` = "org.apache.hadoop:hadoop-mapreduce-client-app" + val `hadoop-mapreduce-client-common` = "org.apache.hadoop:hadoop-mapreduce-client-common" + val `hadoop-mapreduce-client-core` = "org.apache.hadoop:hadoop-mapreduce-client-core" + val `hadoop-mapreduce-client-jobclient` = "org.apache.hadoop:hadoop-mapreduce-client-jobclient" + val `hadoop-mapreduce-client-shuffle` = "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + val `hadoop-shaded-guava` = "org.apache.hadoop.thirdparty:hadoop-shaded-guava" + val `hadoop-yarn-api` = "org.apache.hadoop:hadoop-yarn-server-api" + val `hadoop-yarn-client` = "org.apache.hadoop:hadoop-yarn-client" + val `hadoop-yarn-common` = "org.apache.hadoop:hadoop-yarn-common" + val `hadoop-yarn-registry` = "org.apache.hadoop:hadoop-yarn-registry" + val `hadoop-yarn-server-common` = "org.apache.hadoop:hadoop-yarn-server-common" + val `hadoop-yarn-server-web-proxy` = "org.apache.hadoop:hadoop-yarn-server-web-proxy" + val `HikariCP` = "com.zaxxer:HakariCP" + val `hive-beeline` = "org.apache.hive:hive-beeline" + val `hive-cli` = "org.apache.hive:hive-cli" + val `hive-common` = "org.apache.hive:hive-common" + val `hive-exec` = "org.apache.hive:hive-exec" + val `hive-jdbc` = "org.apache.hive:hive-jdbc" + val `hive-llap-common` = "org.apache.hive:hive-llap-common" + val `hive-metastore` = "org.apache.hive:hive-metastore" + val `hive-serde` = "org.apache.hive:hive-serde" + val `hive-service-rpc` = "org.apache.hive:hive-service-rpc" + val `hive-shims-0.23` = "org.apache.hive:hive-shims-0.23" + val `hive-shims` = "org.apache.hive:hive-shims" + val `hive-shims-common` = "org.apache.hive:hive-shims-common" + val `hive-shims-scheduler` = "org.apache.hive:hive-shims-scheduler" + val `hive-storage-api` = "org.apache.hive:hive-storage-api" + val `hive-vector-code-gen` = "org.apache.hive:hive-vector-code-gen" + val `hk2-api` = "org.glassfish.hk2:hk2-api" + val `hk2-locator` = "org.glassfish.hk2:hk2-locator" + val `hk2-utils` = "org.glassfish.hk2:hk2-utils" + val `htsjdk` = "com.github.samtools:htsjdk" + val `htrace-core4` = "org.apache.htrace:htrace-core4" + val `httpclient` = "org.apache.httpcomponents:httpclient" + val `httpcore` = "org.apache.httpcomponents:httpcore" + val `icu4j` = "icu4j:icu4j" + val `istack-commons-runtime` = "com.sun.istack:istack-commons-runtime" + val `ivy` = "org.apache.ivy:ivy" + val `j2objc-annotations` = "com.google.j2objc:j2objc-annotations" + val `jackson-annotations` = "com.fasterxml.jackson.core:jackson-annotations" + val `jackson-core` = "com.fasterxml.jackson.core:jackson-core" + val `jackson-core-asl` = "org.codehaus.jackson:jackson-core-asl" + val `jackson-databind` = "com.fasterxml.jackson.core:jackson-databind" + val `jackson-dataformat-yaml` = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" + val `jackson-datatype-jsr310` = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" + val `jackson-jaxrs` = "com.fasterxml.jackson.jaxrs:jackson-jaxrs" + val `jackson-jaxrs-base` = "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base" + val `jackson-jaxrs-json-provider` = "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider" + val `jackson-mapper-asl` = "org.codehaus.jackson:jackson-mapper-asl" + val `jackson-module-jaxb-annotations` = "com.fasterxml.jackson.module::jackson-module-jaxb-annotations" + val `jackson-module-paranamer` = "com.fasterxml.jackson.module::jackson-module-paranamer" + val `jackson-module-scala` = "com.fasterxml.jackson.module::jackson-module-scala" + val `jackson-xc` = "org.codehaus.jackson:jackson-xc" + val `jakarta.activation-api` = "jakarta.activation:jakarta.activation-api" + val `jakarta.annotation-api` = "jakarta.annotation:jakarta.annotation-api" + val `jakarta.inject` = "jakarta.inject:jakarta.inject" + val `jakarta.inject-api` = "jakarta.inject:jakarta.inject-api" + val `jakarta.servlet-api` = "jakarta.servlet.jakarta.servlet-api" + val `jakarta.validation-api` = "jakarta.validation.jakarta.validation-api" + val `jakarta.ws.rs-api` = "jakarta.ws.rs:jakarta.ws.rs-api" + val `jakarta.xml.bind-api` = "jakara.xml.bind:jakarta.xml.bind-api" + val `janino` = "org.codehaus.janino:janino" + val `javassist` = "org.javassist:javassist" + val `javax.inject` = "javax.inject:javax.inject" + val `javax.jdo` = "org.datanucleus:javax.jdo" + val `javax.servlet-api` = "javax.servet:javax.servlet-api" + val `javolution` = "org.javolution:javolution" + val `jaxb-api` = "javax.xml.bind:jaxb-api" + val `jaxb-core` = "org.glassfish.jaxb:jaxb-core" + val `jaxb-runtime` = "org.glassfish.jaxb:jaxb-runtime" + val `jcl-over-slf4j` = "org.slf4j:jcl-over-slf4j" + val `jdistlib` = "net.sourceforge.jdistlib:jdistlib" + val `jdo-api` = "javax.jdo:jdo-api" + val `jersey-client` = "org.glassfish.jersey.core:jersey-client" + val `jersey-common` = "org.glassfish.jersey.core:jersey-common" + val `jersey-container-servlet` = "org.glassfish.jersey.containers:jersey-container-servlet" + val `jersey-container-servlet-core` = "org.glassfish.jersey.containers:jersey-container-servlet-core" + val `jersey-media-jaxb` = "org.glassfish.jersey.media:jersey-media-jaxb" + val `jersey-hk2` = "org.glassfish.jersey.inject:jersey-hk2" + val `jersey-server` = "org.glassfish.jersey.core:jersey-server" + val `jjwt-api` = "io.jsonwebtoken:jjwt-api" + val `jjwt-impl` = "io.jsonwebtoken:jjwt-impl" + val `jjwt-jackson` = "io.jsonwebtoken:jjwt-jackson" + val `jetty` = "org.mortbay.jetty:jetty" + val `jetty-sslengine` = "org.mortbay.jetty:jetty-sslengine" + val `jetty-util` = "org.mortbay.jetty:jetty-util" + val `JLargeArrays` = "pl.edu.icm:JLargeArrays" + val `jline` = "jline:jline" + val `joda-time` = "joda-time:joda-time" + val `jodd-core` = "org.jodd:joda-core" + val `jpam` = "net.sf.jpam:jpam" + val `jna` = "net.java.dev.jna:jna" + val `json` = "org.json:json" + val `json4s-ast` = "org.json4s::json4s-ast" + val `json4s-core` = "org.json4s::json4s-core" + val `json4s-jackson` = "org.json4s::json4s-jackson" + val `json4s-scalap` = "org.json4s::json4s-scalap" + val `json-smart` = "net.minidev:json-smart" + val `jsp-api` = "javax.servlet.api:jsp-api" + val `jsr305` = "com.google.code.findbugs:jsr305" + val `jta` = "javax.transaction:jta" + val `JTransforms` = "com.github.wendykierp:JTransforms" + val `jul-to-slf4j` = "org.slf4j:jul-to-slf4j" + val `junixsocket-core` = "com.kohlschutter.junixsocket:junixsocket-core" + val `kerb-admin` = "org.apache.kerby:kerb-admin" + val `kerb-client` = "org.apache.kerby:kerb-client" + val `kerb-common` = "org.apache.kerby:kerb-common" + val `kerb-core` = "org.apache.kerby:kerb-core" + val `kerb-crypto` = "org.apache.kerby:kerb-crypto" + val `kerb-identity` = "org.apache.kerby:kerb-identity" + val `kerb-server` = "org.apache.kerby:kerb-server" + val `kerb-simplekdc` = "org.apache.kerby:kerb-simplekdc" + val `kerb-util` = "org.apache.kerby:kerb-util" + val `kerb-xdr` = "org.apache.kerby:kerb-xdr" + val `kryo-shaded` = "com.esotericsoftware:kryo-shaded" + val `kubernetes-client` = "io.fabric8:kubernetes-client" + val `kubernetes-client-api` = "io.fabric8:kubernetes-client-api" + val `kubernetes-httpclient-okhttp` = "io.fabric8:kubernetes-httpclient-okhttp" + val `kubernetes-httpclient-vertex` = "io.fabric8:kubernetes-httpclient-vertex" + val `kubernetes-model` = "io.fabric8:kubernetes-model" + val `kubernetes-model-adminssionregistration` = "io.fabric8:kubernetes-admissionregistration" + val `kubernetes-model-apiextensions` = "io.fabric8:kubernetes-apiextensions" + val `kubernetes-model-apps` = "io.fabric8:kubernetes-apps" + val `kubernetes-model-autoscaling` = "io.fabric8:kubernetes-autoscaling" + val `kubernetes-model-batch` = "io.fabric8:kubernetes-batch" + val `kubernetes-model-certificates` = "io.fabric8:kubernetes-certificates" + val `kubernetes-model-common` = "io.fabric8:kubernetes-common" + val `kubernetes-model-coordination` = "io.fabric8:kubernetes-coordination" + val `kubernetes-model-core` = "io.fabric8:kubernetes-core" + val `kubernetes-model-discovery` = "io.fabric8:kubernetes-discovery" + val `kubernetes-model-events` = "io.fabric8:kubernetes-events" + val `kubernetes-model-extensions` = "io.fabric8:kubernetes-extensions" + val `kubernetes-model-flowcontrol` = "io.fabric8:kubernetes-flowcontrol" + val `kubernetes-model-gatewayapi` = "io.fabric8:kubernetes-gatewayapi" + val `kubernetes-model-metrics` = "io.fabric8:kubernetes-metrics" + val `kubernetes-model-networking` = "io.fabric8:kubernetes-networking" + val `kubernetes-model-node` = "io.fabric8:kubernetes-node" + val `kubernetes-model-policy` = "io.fabric8:kubernetes-policy" + val `kubernetes-model-rbac` = "io.fabric8:kubernetes-rbac" + val `kubernetes-model-resource` = "io.fabric8:kubernetes-resource" + val `kubernetes-model-scheduling` = "io.fabric8:kubernetes-scheduling" + val `kubernetes-model-settings` = "io.fabric8:kubernetes-settings" + val `kubernetes-model-storageclass` = "io.fabric8:kubernetes-storageclass" + val `lapack` = "dev.ludovic.netlib:lapack" + val `leveldbjni-all` = "org.fusesource.leveldbjni:leveldbjni-all" + val `libfb303` = "org.apache.trift:libfb303" + val `libraries-bom` = "com.google.cloud:libraries-bom" + val `libthrift` = "org.apache.thrift:libthrift" + val `listenablefuture` = "com.google.guava:listenablefuture" + val `log4j` = "log4j:log4j" + val `log4j-1.2-api` = "org.apache.logging.log4j:log4j-1.2-api" + val `log4j-api` = "org.apache.logging.log4j:log4j-api" + val `log4j-api-scala` = "org.apache.logging.log4j::log4j-api-scala" + val `log4j-bom` = "org.apache.logging.log4j:log4j-bom" + val `log4j-core` = "org.apache.logging.log4j:log4j-core" + val `log4j-layout-template-json` = "org.apache.logging.log4j:log4j-layout-template-json" + val `log4j-slf4j2-impl` = "org.apache.logging.log4j::log4j-slf4j2-impl" + val `logging-interceptor` = "org.squareup.okhttp:logging-interceptor" + val `lz4-java` = "org.lz4:lz4-java" + val `machinist` = "org.typelevel::machinist" + val `macro-compat` = "org.typelevel::macro-compat" + val `mainargs` = "com.lihaoyi::mainargs" + val `mesos` = "org.apache.mesos:mesos" + val `metrics-core` = "io.dropwizard.metrics:metrics-core" + val `metrics-graphite` = "io.dropwizard.metrics:metrics-graphite" + val `metrics-jmx` = "io.dropwizard.metrics:metrics-jmx" + val `metrics-json` = "io.dropwizard.metrics:metrics-json" + val `metrics-jvm` = "io.dropwizard.metrics:metrics-jvm" + val `mill-scalafix` = "com.goyeau::mill-scalafix" + val `minlog` = "com.esotericsoftware:minlog" + val `mockito-scala` = "org.mockito::mockito-scala" + val `netty-all` = "io.netty:netty-all" + val `netty-buffer` = "io.netty:netty-buffer" + val `netty-codec` = "io.netty:netty-codec" + val `netty-codec-dns` = "io.netty:netty-codec-dns" + val `netty-codec-http2` = "io.netty:netty-codec-http2" + val `netty-codec-http` = "io.netty:netty-codec-http" + val `netty-codec-socks` = "io.netty:netty-codec-socks" + val `netty-common` = "io.netty:netty-common" + val `netty-handler` = "io.netty:netty-handler" + val `netty-handler-proxy` = "io.netty:netty-handler-proxy" + val `netty-resolver` = "io.netty:netty-resolver" + val `netty-resolver-dns` = "io.netty:netty-resolver-dns" + val `netty-tcnative-boringssl-static` = "io.netty:netty-tcnative-boringssl-static" + val `netty-tcnative-classes` = "io.netty:netty-tcnative-classes" + val `netty-transport` = "io.netty:netty-transport" + val `netty-transport-classes-epoll` = "io.netty:netty-transport-classes-epoll" + val `netty-transport-classes-kqueue` = "io.netty:netty-transport-classes-kqueue" + val `netty-transport-native-epoll` = "io.netty:netty-transport-native-epoll" + val `netty-transport-native-kqueue` = "io.netty:netty-transport-native-kqueue" + val `netty-transport-native-unix-common` = "io.netty:netty-transport-native-unix-common" + val `nimbus-jose-jwt` = "com.nimbusds:nimbus-jose-jwt" + val `objenesis` = "org.objenesis:objenesis" + val `okhttp` = "org.squareup.okhttp" + val `okio` = "org.squareup.okio" + val `opencsv` = "com.opencsv:opencsv" + val `orc-core` = "org.apache.orc:orc-core" + val `orc-format` = "org.apache.orc:orc-format" + val `orc-mapreduce` = "org.apache.orc:orc-mapreduce" + val `orc-shims` = "org.apache.orc:orc-shims" + val `oro` = "oro:oro" + val `os-lib` = "com.lihaoyi::os-lib" + val `osgi-resource-locator` = "org.glassfish.hk2:osgi-resource-locator" + val `paranamer` = "org.throughtworks.paranamer:paranamer" + val `parquet-column` = "org.apache.parquet:parquet-column" + val `parquet-common` = "org.apache.parquet:parquet-common" + val `parquet-format` = "org.apache.parquet:parquet-format" + val `parquet-format-structures` = "org.apache.parquet:parquet-format-structures" + val `parquet-encoding` = "org.apache.parquet:parquet-encoding" + val `parquet-hadoop` = "org.apache.parquet:parquet-hadoop" + val `parquet-jackson` = "org.apache.parquet:parquet-jackson" + val `protobuf-java` = "org.google.protobuf:protobuf-java" + val `pickle` = "net.razorvine:pickle" + val `py4j` = "net.sf.py4j:py4j" + val `pyrolite` = "net.razorvine:pyrolite" + val `pyspark-bom` = "org.apache.spark::pyspark-bom" + val `re2j` = "com.google.re2j:re2j" + val `RoaringBitmap` = "org.roaringbitmap:RoaringBitmap" + val `rocksdbjni` = "org.rocksdb:rocksdbjni" + val `scala-collection-compat` = "org.scala-lang.modules::scala-collection-compat" + val `scalac-compat-annotation` = "org.typelevel::scalac-compat-annotation" + val `scala-compiler` = "org.scala-lang:scala-compiler" + val `scala-library` = "org.scala-lang:scala-library" + val `scala-parallel-collections` = "org.scala-lang.modules::scala-parallel-collections" + val `scala-parser-combinators` = "org.scala-lang.modules::scala-parser-combinators" + val `scala-reflect` = "org.scala-lang:scala-reflect" + val `scala-xml` = "org.scala-lang.modules::scala-xml" + val `scalac-options` = "org.typelevel::scalac-options" + val `scalacheck` = "org.scalacheck::scalacheck" + val `scalacheck-1-18` = "org.scalatestplus::scalacheck-1-18" + val `scalatest` = "org.scalatest::scalatest" + val `scalatest-shouldmatchers` = "org.scalatest::scalatest-shouldmatchers" + val `scalafix-forbidden-symbol` = "io.github.tanin47::scalafix-forbidden-symbol" + val `shims` = "org.roaringbitmap:shims" + val `shapeless` = "com.chuusai::shapeless" + val `slf4j-api` = "org.slf4j:slf4j-api" + val `slf4j-log4j12` = "org.slf4j:slf4j-log4j12" + val `snakeyaml` = "org.yaml:snakeyaml" + val `snakeyaml-engine` = "org.yaml:snakeyaml-engine" + val `snappy-java` = "org.xerial.snappy:snappy-java" + val `sourcecode` = "com.lihaoyi::sourcecode" + val `spark-catalyst` = "org.apache.spark::spark-catalyst" + val `spark-common-utils` = "org.apache.spark::spark-common-utils" + val `spark-core` = "org.apache.spark::spark-core" + val `spark-graphx` = "org.apache.spark::spark-graphx" + val `spark-hive` = "org.apache.spark::spark-hive" + val `spark-hive-thriftserver` = "org.apache.spark::spark-hive-thirftserver" + val `spark-kubernetes` = "org.apache.spark::spark-kubernetes" + val `spark-kvstore` = "org.apache.spark::spark-kvstore" + val `spark-launcher` = "org.apache.spark::spark-launcher" + val `spark-mesos` = "org.apache.spark::spark-mesos" + val `spark-mllib` = "org.apache.spark::spark-mllib" + val `spark-mllib-local` = "org.apache.spark::spark-mllib-local" + val `spark-network-common` = "org.apache.spark::spark-network-common" + val `spark-network-shuffle` = "org.apache.spark::spark-network-shuffle" + val `spark-repl` = "org.apache.spark::spark-repl" + val `spark-sketch` = "org.apache.spark::spark-sketch" + val `spark-sql` = "org.apache.spark::spark-sql" + val `spark-sql-api` = "org.apache.spark::spark-sql-api" + val `spark-streaming` = "org.apache.spark::spark-streaming" + val `spark-tags` = "org.apache.spark::spark-tags" + val `spark-unsafe` = "org.apache.spark::spark-unsafe" + val `spark-variant` = "org.apache.spark::spark-variant" + val `spark-yarn` = "org.apache.spark::spark-yarn" + val `spire` = "org.typelevel::spire" + val `spire-macros` = "org.typelevel::spire-macros" + val `spire-platform` = "org.typelevel::spire-platform" + val `spire-util` = "org.typelevel::spire-util" + val `ST4` = "org.antlr:ST4" + val `stax2-api` = "org.codehaus.woodstox:stax2-api" + val `stax-api` = "javax.xml.stream:stax-api" + val `stream` = "org.clearspring.analytics:stream" + val `super-csv` = "net.sf.supercsv:super-csv" + val `testng-7-10` = "org.scalatestplus::testng-7-10" + val `threeten-extra` = "org.threeten:threeten-extra" + val `tink` = "org.google.crypto.tink:tink" + val `transaction-api` = "javax.transaction:transaction-api" + val `univocity-parsers` = "com.univocity:univocity-parsers" + val `vertx-auth-common` = "io.vertx:vertex-auth-common" + val `vertx-core` = "io.vertx:vertex-core" + val `vertx-web-client` = "io.vertx:vertex-web-client" + val `vertx-web-common` = "io.vertx:vertex-web-common" + val `xbean-asm7-shaded` = "org.apache.xbean:xbean-asm7-shaded" + val `xbean-asm9-shaded` = "org.apache.xbean:xbean-asm9-shaded" + val `xercesImpl` = "xerces:xercesImpl" + val `xml-apis` = "xml-apis:xml-apis" + val `xmlenc` = "xmlenc:xmlenc" + val `xmlschema-core` = "org.apache.ws.xmlschema:xmlschema-core" + val `xz` = "org.tukaani:xz" + val `zjsonpatch` = "io.fabric8:zjsonpatch" + val `zookeeper` = "org.apache.zookeeper" + val `zookeeper-jute` = "org.apache.zookeeper-jute" + val `zstd-jni` = "com.github.luben:zstd-jni" \ No newline at end of file From 4abeb1f817cdc6ff04781ae39b5016fcc740d5cf Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 16 Dec 2025 11:01:40 -0500 Subject: [PATCH 04/51] [auth] Remove low-risk reflected xss finding (#15213) ## Change Description Stop mirroring back an invalid "next" page specified at login. ## Security Assessment - This change potentially impacts the Hail Batch instance as deployed by Broad Institute in GCP ### Impact Rating - This change has a low security impact ### Impact Description Prevents a potential reflected XSS ### Appsec Review - [x] Required: The impact has been assessed and approved by appsec --- auth/auth/auth.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/auth/auth/auth.py b/auth/auth/auth.py index a5447aabd74..5e6feb17d23 100644 --- a/auth/auth/auth.py +++ b/auth/auth/auth.py @@ -219,9 +219,7 @@ def validate_next_page_url(next_page): actual_next_page_domain = urlparse(next_page).netloc if actual_next_page_domain not in valid_next_domains: - raise web.HTTPBadRequest( - text=f'Invalid next page: \'{next_page}\'. Domain \'{actual_next_page_domain}\' not in {valid_next_domains}' - ) + raise web.HTTPBadRequest(text='Invalid next page.') @routes.get('/healthcheck') From 0718551962fe87d0eb942925a7d5e197b80bf666 Mon Sep 17 00:00:00 2001 From: Christopher Vittal Date: Tue, 16 Dec 2025 17:38:52 -0500 Subject: [PATCH 05/51] [auth] Add 'inactive' status to the list of account errors (#15210) A user reported a 500 when trying to log in on zulip ([#Hail Batch support > unable to log in -- 500 error](https://hail.zulipchat.com/#narrow/channel/223457-Hail-Batch-support/topic/unable.20to.20log.20in.20--.20500.20error/with/563247607)). We run into this internal server error because we were failing [this](https://github.com/hail-is/hail/blob/5d54486fcac06f6b7b2e8af380f812d1552343fc/auth/auth/auth.py#L438) assertion, as were not handling the 'inactive' status. This change adds the 'inactive' status to the list of errors, and also makes sure to set an http error status when rendering the `account-errors.html` template. ## Security Assessment - This change potentially impacts the Hail Batch instance as deployed by Broad Institute in GCP ### Impact Rating - This change has low security impact ### Impact Description Change the response codes to one portion of the login flow, and make it so that the server doesn't fail when handling login of an inactive user. ### Appsec Review - [x] Required: The impact has been assessed and approved by appsec --- auth/auth/auth.py | 6 ++++-- web_common/web_common/web_common.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/auth/auth/auth.py b/auth/auth/auth.py index 5e6feb17d23..f88a0dabc1b 100644 --- a/auth/auth/auth.py +++ b/auth/auth/auth.py @@ -420,9 +420,11 @@ async def callback(request) -> web.Response: raise web.HTTPFound(creating_url) - if user['state'] in ('deleting', 'deleted'): + if user['state'] in ('deleting', 'deleted', 'inactive'): page_context = {'username': user['username'], 'state': user['state'], 'login_id': user['login_id']} - return await render_template('auth', request, user, 'account-error.html', page_context) + return await render_template( + 'auth', request, user, 'account-error.html', page_context, status_code=web.HTTPUnauthorized.status_code + ) if user['state'] == 'creating': if caller == 'signup': diff --git a/web_common/web_common/web_common.py b/web_common/web_common/web_common.py index 8f3c3cb664d..6e82d21b630 100644 --- a/web_common/web_common/web_common.py +++ b/web_common/web_common/web_common.py @@ -83,6 +83,8 @@ async def render_template( userdata: Optional[UserData], file: str, page_context: Dict[str, Any], + *, + status_code: int = 200, ) -> web.Response: if request.headers.get('x-hail-return-jinja-context'): if userdata and userdata['is_developer']: @@ -100,7 +102,7 @@ async def render_template( context['use_tailwind'] = service in TAILWIND_SERVICES context['csrf_token'] = csrf_token - response = aiohttp_jinja2.render_template(file, request, context) + response = aiohttp_jinja2.render_template(file, request, context, status=status_code) response.set_cookie('_csrf', csrf_token, secure=True, httponly=True, samesite='strict') return response From 42504e35e5ab5f63e023cdced206e1a9895d360d Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Mon, 5 Jan 2026 12:06:52 -0500 Subject: [PATCH 06/51] [query] supplant `build-info.properties` with cgen (#15211) Replace generated `build-info.properties` with a generated Scala package object. Renamed terms to be less shouty. Retired useless `BuildConfiguration` enum. This change cannot impact the Hail Batch instance as deployed by Broad Institute in GCP --- hail/.gitignore | 1 - hail/hail/package.mill | 40 +++++----- .../is/hail/utils/compat/package.scala | 3 +- hail/hail/src/is/hail/asm4s/CodeBuilder.scala | 17 +++-- .../src/is/hail/backend/ExecutionCache.scala | 2 +- .../backend/driver/BatchQueryDriver.scala | 4 +- .../hail/backend/service/ServiceBackend.scala | 4 +- .../src/is/hail/backend/service/Worker.scala | 4 +- .../is/hail/backend/spark/SparkBackend.scala | 2 +- .../src/is/hail/expr/ir/MatrixValue.scala | 10 +-- .../src/is/hail/expr/ir/MatrixWriter.scala | 2 +- .../src/is/hail/expr/ir/TableWriter.scala | 2 +- hail/hail/src/is/hail/io/vcf/ExportVCF.scala | 2 +- hail/hail/src/is/hail/package.scala | 73 ------------------- hail/hail/src/is/hail/utils/package.scala | 2 +- .../expr/ir/analyses/SemanticHashSuite.scala | 4 +- .../is/hail/services/BatchClientSuite.scala | 4 +- hail/python/hail/backend/py4j_backend.py | 2 +- 18 files changed, 55 insertions(+), 123 deletions(-) delete mode 100644 hail/hail/src/is/hail/package.scala diff --git a/hail/.gitignore b/hail/.gitignore index d72c8184c41..186f5ce05e7 100644 --- a/hail/.gitignore +++ b/hail/.gitignore @@ -73,7 +73,6 @@ src/main/c/.cxx.vsn src/main/c/headers src/main/c/lib src/main/c/libsimdpp-2* -src/main/resources/build-info.properties src/main/resources/include/catch.hpp env/ www/docs diff --git a/hail/hail/package.mill b/hail/hail/package.mill index ecfabf6f41f..e73beeb8b3a 100644 --- a/hail/hail/package.mill +++ b/hail/hail/package.mill @@ -23,36 +23,42 @@ trait RootHailModule extends CrossScalaModule, HailScalaModule: override def crossScalaVersion: String = Settings.scalaMinorVersions(crossValue) - def buildInfo: T[PathRef] = Task { - val revision = VcsVersion.vcsState().currentRevision - os.write( - Task.dest / "build-info.properties", - s"""[Build Metadata] - |revision=$revision - |sparkVersion=${Env.sparkVersion()} - |hailPipVersion=${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion} - |hailBuildConfiguration=${buildMode.toString.toLowerCase} - |""".stripMargin, - ) - PathRef(Task.dest) - } - override def moduleDeps: Seq[JavaModule] = Seq( build.hail.memory(if buildMode == CI then "debug" else "release"), ) - + override def resources: T[Seq[PathRef]] = super.resources() ++ Seq( BuildCtx.withFilesystemCheckerDisabled { PathRef(BuildCtx.workspaceRoot / "prebuilt" / "lib") }, - buildInfo(), ) override def generatedSources: T[Seq[PathRef]] = Task { - Seq(build.hail.`ir-gen`.generate()) + os.write( + Task.dest / "is" / "hail" / "package.scala", + f"""package is + | + |package object hail { + | // FIXME: This should be a compiler switch/flag + | val EmitTracebackInAssertions = ${buildMode != Release} + | + | val PipVersion = "${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion}" + | val Revision = "${VcsVersion.vcsState().currentRevision}" + | val SparkVersion = "${Env.sparkVersion()}" + | + | // FIXME: probably should use tags or something to choose English name + | val PrettyVersion = PipVersion + "-" + Revision.substring(0, 12) + |}""".stripMargin, + createFolders = true, + ) + + Seq( + PathRef(Task.dest), + build.hail.`ir-gen`.generate(), + ) } override def unmanagedClasspath: T[Seq[PathRef]] = diff --git a/hail/hail/src-2.12/is/hail/utils/compat/package.scala b/hail/hail/src-2.12/is/hail/utils/compat/package.scala index be947323f92..5682feb5b06 100644 --- a/hail/hail/src-2.12/is/hail/utils/compat/package.scala +++ b/hail/hail/src-2.12/is/hail/utils/compat/package.scala @@ -8,8 +8,7 @@ import scala.reflect.ClassTag package compat { class ArrayOps[A: ClassTag](val a: WrappedArray[A]) { - def sortInPlace[B >: A]()(implicit ct: ClassTag[B], ord: Ordering[B]) - : WrappedArray[A] = { + def sortInPlace[B >: A]()(implicit ct: ClassTag[B], ord: Ordering[B]): WrappedArray[A] = { scala.util.Sorting.stableSort(a.array.asInstanceOf[Array[B]]) a } diff --git a/hail/hail/src/is/hail/asm4s/CodeBuilder.scala b/hail/hail/src/is/hail/asm4s/CodeBuilder.scala index 16470cb68ca..81a59574916 100644 --- a/hail/hail/src/is/hail/asm4s/CodeBuilder.scala +++ b/hail/hail/src/is/hail/asm4s/CodeBuilder.scala @@ -1,6 +1,6 @@ package is.hail.asm4s -import is.hail.{lir, HAIL_BUILD_CONFIGURATION} +import is.hail.{lir, EmitTracebackInAssertions} import is.hail.utils.{toRichIterable, Traceback} import scala.collection.compat._ @@ -274,13 +274,14 @@ trait CodeBuilderLike { def _assert(cond: => Code[Boolean], msgs: Code[String]*): Unit = { val message = msgs.reduce(_.concat(_)) - if (HAIL_BUILD_CONFIGURATION.isDebug) { - val traceback = mb.cb.modb.getObject[Throwable](new Traceback().fillInStackTrace()) - val assertion = Code.newInstance[AssertionError, String, Throwable](message, traceback) - if_(cond, {}, _throw(assertion)) - } else { - if_(cond, {}, _throw(Code.newInstance[AssertionError, Object](message))) - } + + val assertion = + if (EmitTracebackInAssertions) { + val traceback = mb.cb.modb.getObject[Throwable](new Traceback().fillInStackTrace()) + Code.newInstance[AssertionError, String, Throwable](message, traceback) + } else Code.newInstance[AssertionError, Object](message) + + if_(cond, {}, _throw(assertion)) } } diff --git a/hail/hail/src/is/hail/backend/ExecutionCache.scala b/hail/hail/src/is/hail/backend/ExecutionCache.scala index 62fcd1960a9..2cea980a049 100644 --- a/hail/hail/src/is/hail/backend/ExecutionCache.scala +++ b/hail/hail/src/is/hail/backend/ExecutionCache.scala @@ -27,7 +27,7 @@ case object ExecutionCache { if (Option(flags.get(Flags.UseFastRestarts)).isEmpty) noCache else fsCache( fs, - Option(flags.get(Flags.Cachedir)).getOrElse(s"$tmpdir/hail/${is.hail.HAIL_PIP_VERSION}"), + Option(flags.get(Flags.Cachedir)).getOrElse(s"$tmpdir/hail/${is.hail.PipVersion}"), ) def fsCache(fs: FS, cachedir: String): ExecutionCache = { diff --git a/hail/hail/src/is/hail/backend/driver/BatchQueryDriver.scala b/hail/hail/src/is/hail/backend/driver/BatchQueryDriver.scala index eb4d49b7d48..7ca1e978e20 100644 --- a/hail/hail/src/is/hail/backend/driver/BatchQueryDriver.scala +++ b/hail/hail/src/is/hail/backend/driver/BatchQueryDriver.scala @@ -1,6 +1,6 @@ package is.hail.backend.driver -import is.hail.{HAIL_PRETTY_VERSION, HailFeatureFlags} +import is.hail.{HailFeatureFlags, PrettyVersion} import is.hail.annotations.Memory import is.hail.asm4s.HailClassLoader import is.hail.backend.{Backend, ExecuteContext, OwningTempFileManager} @@ -134,7 +134,7 @@ object BatchQueryDriver extends HttpLikeRpc with Logging { val inputURL = argv(5) val outputURL = argv(6) - logger.info(f"${getClass.getName} $HAIL_PRETTY_VERSION") + logger.info(f"${getClass.getName} $PrettyVersion") sys.env.get("HAIL_SSL_CONFIG_DIR").foreach(tls.setSSLConfigFromDir) diff --git a/hail/hail/src/is/hail/backend/service/ServiceBackend.scala b/hail/hail/src/is/hail/backend/service/ServiceBackend.scala index 9d2820a18ac..fc7e891ca22 100644 --- a/hail/hail/src/is/hail/backend/service/ServiceBackend.scala +++ b/hail/hail/src/is/hail/backend/service/ServiceBackend.scala @@ -1,6 +1,6 @@ package is.hail.backend.service -import is.hail.HAIL_REVISION +import is.hail.Revision import is.hail.backend._ import is.hail.backend.Backend.PartitionFn import is.hail.backend.local.LocalTaskContext @@ -82,7 +82,7 @@ object ServiceBackend { new ServiceBackend( name, client, - GitRevision(HAIL_REVISION), + GitRevision(Revision), BatchConfig(batchId, 0), workerConfig, ) diff --git a/hail/hail/src/is/hail/backend/service/Worker.scala b/hail/hail/src/is/hail/backend/service/Worker.scala index d00de18b6de..cfb9c55babf 100644 --- a/hail/hail/src/is/hail/backend/service/Worker.scala +++ b/hail/hail/src/is/hail/backend/service/Worker.scala @@ -1,6 +1,6 @@ package is.hail.backend.service -import is.hail.{HAIL_REVISION, HailFeatureFlags} +import is.hail.{HailFeatureFlags, Revision} import is.hail.asm4s._ import is.hail.backend.Backend.PartitionFn import is.hail.backend.HailTaskContext @@ -176,7 +176,7 @@ object Worker extends Logging { sys.env.get("HAIL_SSL_CONFIG_DIR").foreach(tls.setSSLConfigFromDir) - logger.info(s"${getClass.getName} $HAIL_REVISION") + logger.info(s"${getClass.getName} $Revision") logger.info(s"running partition $partition root '$root' with scratch directory '$scratchDir'") timer.start(s"partition $partition") diff --git a/hail/hail/src/is/hail/backend/spark/SparkBackend.scala b/hail/hail/src/is/hail/backend/spark/SparkBackend.scala index 723adcc45a7..a8d87497f96 100644 --- a/hail/hail/src/is/hail/backend/spark/SparkBackend.scala +++ b/hail/hail/src/is/hail/backend/spark/SparkBackend.scala @@ -109,7 +109,7 @@ object SparkBackend extends Logging { : SparkConf = { require(blockSize >= 0) - checkSparkCompatibility(is.hail.HAIL_SPARK_VERSION, org.apache.spark.SPARK_VERSION) + checkSparkCompatibility(is.hail.SparkVersion, org.apache.spark.SPARK_VERSION) val conf = new SparkConf().setAppName(appName) diff --git a/hail/hail/src/is/hail/expr/ir/MatrixValue.scala b/hail/hail/src/is/hail/expr/ir/MatrixValue.scala index 36c3e164dfd..4c815cb800b 100644 --- a/hail/hail/src/is/hail/expr/ir/MatrixValue.scala +++ b/hail/hail/src/is/hail/expr/ir/MatrixValue.scala @@ -107,7 +107,7 @@ case class MatrixValue( val colsSpec = TableSpecParameters( FileFormat.version.rep, - is.hail.HAIL_PRETTY_VERSION, + is.hail.PrettyVersion, "../references", typ.colsTableType.copy(key = FastSeq[String]()), Map( @@ -144,7 +144,7 @@ case class MatrixValue( val globalsSpec = TableSpecParameters( FileFormat.version.rep, - is.hail.HAIL_PRETTY_VERSION, + is.hail.PrettyVersion, "../references", TableType(typ.globalType, FastSeq(), TStruct.empty), Map( @@ -175,7 +175,7 @@ case class MatrixValue( val rowsSpec = TableSpecParameters( FileFormat.version.rep, - is.hail.HAIL_PRETTY_VERSION, + is.hail.PrettyVersion, "../references", typ.rowsTableType, Map( @@ -190,7 +190,7 @@ case class MatrixValue( val entriesSpec = TableSpecParameters( FileFormat.version.rep, - is.hail.HAIL_PRETTY_VERSION, + is.hail.PrettyVersion, "../references", TableType(entriesRVType, FastSeq(), typ.globalType), Map( @@ -218,7 +218,7 @@ case class MatrixValue( val spec = MatrixTableSpecParameters( FileFormat.version.rep, - is.hail.HAIL_PRETTY_VERSION, + is.hail.PrettyVersion, "references", typ, Map( diff --git a/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala b/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala index cf299fc098a..ded0be716c4 100644 --- a/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala +++ b/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala @@ -654,7 +654,7 @@ class MatrixSpecHelper( def write(fs: FS, nCols: Long, partCounts: Array[Long]): Unit = { val spec = MatrixTableSpecParameters( FileFormat.version.rep, - is.hail.HAIL_PRETTY_VERSION, + is.hail.PrettyVersion, "references", typ, Map( diff --git a/hail/hail/src/is/hail/expr/ir/TableWriter.scala b/hail/hail/src/is/hail/expr/ir/TableWriter.scala index 0cad81ba033..0f8c1660cd8 100644 --- a/hail/hail/src/is/hail/expr/ir/TableWriter.scala +++ b/hail/hail/src/is/hail/expr/ir/TableWriter.scala @@ -439,7 +439,7 @@ class TableSpecHelper( def write(fs: FS, partCounts: Array[Long], distinctlyKeyed: Boolean): Unit = { val spec = TableSpecParameters( FileFormat.version.rep, - is.hail.HAIL_PRETTY_VERSION, + is.hail.PrettyVersion, refRelPath, typ, Map( diff --git a/hail/hail/src/is/hail/io/vcf/ExportVCF.scala b/hail/hail/src/is/hail/io/vcf/ExportVCF.scala index 3c7463401e0..56726747a22 100644 --- a/hail/hail/src/is/hail/io/vcf/ExportVCF.scala +++ b/hail/hail/src/is/hail/io/vcf/ExportVCF.scala @@ -149,7 +149,7 @@ object ExportVCF extends Logging { val sb = new StringBuilder() sb.append("##fileformat=VCFv4.2\n") - sb.append(s"##hailversion=${hail.HAIL_PRETTY_VERSION}\n") + sb.append(s"##hailversion=${hail.PrettyVersion}\n") entryType.fields.foreach { f => val attrs = getAttributes("format", f.name, metadata).getOrElse(Map.empty[String, String]) diff --git a/hail/hail/src/is/hail/package.scala b/hail/hail/src/is/hail/package.scala deleted file mode 100644 index bda02673f3c..00000000000 --- a/hail/hail/src/is/hail/package.scala +++ /dev/null @@ -1,73 +0,0 @@ -package is - -import java.io.InputStream - -package object hail { - - private object HailBuildInfo { - - import is.hail.utils._ - - import java.util.Properties - - val ( - hail_revision: String, - hail_spark_version: String, - hail_pip_version: String, - hail_build_configuration: BuildConfiguration, - ) = - loadFromResource[(String, String, String, BuildConfiguration)]("build-info.properties") { - (is: InputStream) => - val unknownProp = "" - val props = new Properties() - props.load(is) - ( - props.getProperty("revision", unknownProp), - props.getProperty("sparkVersion", unknownProp), - props.getProperty("hailPipVersion", unknownProp), { - val c = props.getProperty("hailBuildConfiguration", "debug") - BuildConfiguration.parseString(c).getOrElse( - throw new IllegalArgumentException( - s"Illegal 'hailBuildConfiguration' entry in 'build-info.properties': '$c'." - ) - ) - }, - ) - } - } - - val HAIL_REVISION = HailBuildInfo.hail_revision - val HAIL_SPARK_VERSION = HailBuildInfo.hail_spark_version - val HAIL_PIP_VERSION = HailBuildInfo.hail_pip_version - - // FIXME: probably should use tags or something to choose English name - val HAIL_PRETTY_VERSION = HAIL_PIP_VERSION + "-" + HAIL_REVISION.substring(0, 12) - - val HAIL_BUILD_CONFIGURATION = HailBuildInfo.hail_build_configuration -} - -sealed trait BuildConfiguration extends Product with Serializable { - def isDebug: Boolean -} - -object BuildConfiguration { - case object Release extends BuildConfiguration { - override def isDebug: Boolean = false - } - - case object Debug extends BuildConfiguration { - override def isDebug: Boolean = true - } - - case object CI extends BuildConfiguration { - override def isDebug: Boolean = true - } - - def parseString(c: String): Option[BuildConfiguration] = - c match { - case "release" => Some(BuildConfiguration.Release) - case "dev" => Some(BuildConfiguration.Debug) - case "ci" => Some(BuildConfiguration.CI) - case _ => None - } -} diff --git a/hail/hail/src/is/hail/utils/package.scala b/hail/hail/src/is/hail/utils/package.scala index 37535404e15..e162f2dded0 100644 --- a/hail/hail/src/is/hail/utils/package.scala +++ b/hail/hail/src/is/hail/utils/package.scala @@ -745,7 +745,7 @@ package object utils using(new OutputStreamWriter(fs.create(path + "/README.txt"))) { out => out.write( s"""This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - | Written with version $HAIL_PRETTY_VERSION + | Written with version $PrettyVersion | Created at ${dateFormat.format(new Date())}""".stripMargin ) } diff --git a/hail/hail/test/src/is/hail/expr/ir/analyses/SemanticHashSuite.scala b/hail/hail/test/src/is/hail/expr/ir/analyses/SemanticHashSuite.scala index 9b88be8b9cc..f2c7091183c 100644 --- a/hail/hail/test/src/is/hail/expr/ir/analyses/SemanticHashSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/analyses/SemanticHashSuite.scala @@ -1,6 +1,6 @@ package is.hail.expr.ir.analyses -import is.hail.{HAIL_PRETTY_VERSION, HailSuite} +import is.hail.{HailSuite, PrettyVersion} import is.hail.expr.ir._ import is.hail.expr.ir.defs._ import is.hail.io.fs.{FS, FakeFS, FakeURL, FileListEntry} @@ -365,7 +365,7 @@ class SemanticHashSuite extends HailSuite { override def file_version: Int = 1 override def hail_version: String = - HAIL_PRETTY_VERSION + PrettyVersion override def components: Map[String, ComponentSpec] = Map.empty override def toJValue: JValue = ??? diff --git a/hail/hail/test/src/is/hail/services/BatchClientSuite.scala b/hail/hail/test/src/is/hail/services/BatchClientSuite.scala index 7c340611378..bfdf96b0852 100644 --- a/hail/hail/test/src/is/hail/services/BatchClientSuite.scala +++ b/hail/hail/test/src/is/hail/services/BatchClientSuite.scala @@ -1,6 +1,6 @@ package is.hail.services -import is.hail.{services, HAIL_REVISION} +import is.hail.{services, Revision} import is.hail.backend.service.Main import is.hail.services.JobGroupStates.Failure import is.hail.services.oauth2.CloudCredentials @@ -165,7 +165,7 @@ class BatchClientSuite extends TestNGSuite { always_run = false, process = JvmJob( command = Array(Main.TEST), - spec = GitRevision(HAIL_REVISION), + spec = GitRevision(Revision), profile = false, ), ) diff --git a/hail/python/hail/backend/py4j_backend.py b/hail/python/hail/backend/py4j_backend.py index ba60788a3c3..c99ce3d85e7 100644 --- a/hail/python/hail/backend/py4j_backend.py +++ b/hail/python/hail/backend/py4j_backend.py @@ -82,7 +82,7 @@ def raise_when_mismatched_hail_versions(jvm: JVMView) -> None: ) _is = getattr(jvm, 'is') - jar_version = scala_package_object(_is.hail).HAIL_PRETTY_VERSION() + jar_version = scala_package_object(_is.hail).PrettyVersion() if jar_version != __version__: raise RuntimeError( f"Hail version mismatch between JAR and Python library\n JAR: {jar_version}\n Python: {__version__}" From 3458ff8cd363509dd56bb2951306b8c37ef71ba0 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Wed, 7 Jan 2026 13:00:40 -0500 Subject: [PATCH 07/51] [terraform] Dynamic cloudnat ports and all regions (#15212) ## Change Description Updates cloudnat to have a more realistic number of ports, allow the number to float, and add routers and nats for all regions where worker VMs might be created ## Security Assessment Delete all except the correct answer: - This change potentially impacts the Hail Batch instance as deployed by Broad Institute in GCP ### Impact Rating - This change has a medium security impact ### Impact Description Networking change, but in analogy to what already exists, and in the service of adding a secure layer between our worker vms and the outside world ### Appsec Review - [x] Required: The impact has been assessed and approved by appsec --- infra/gcp-broad/hail-is/global.tfvars | 3 ++- infra/gcp-broad/main.tf | 30 ++++++++++++++++--------- infra/gcp/main.tf | 32 ++++++++++++++++++--------- 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/infra/gcp-broad/hail-is/global.tfvars b/infra/gcp-broad/hail-is/global.tfvars index 4f2be4f5094..3e3248b3443 100644 --- a/infra/gcp-broad/hail-is/global.tfvars +++ b/infra/gcp-broad/hail-is/global.tfvars @@ -7,7 +7,8 @@ github_organization = "hail-is" # batch_gcp_regions is a JSON array of string, the names of the gcp # regions to schedule over in Batch. E.g. "[\"us-central1\"]" -batch_gcp_regions = "[\"us-central1\"]" +# Note: doesn't update the k8s secret in gcp-broad, but does impact which cloudnat resources are created. +batch_gcp_regions = "[\"us-central1\",\"us-east1\",\"us-east4\",\"us-west1\",\"us-west2\",\"us-west3\",\"us-west4\"]" gcp_project = "hail-vdc" diff --git a/infra/gcp-broad/main.tf b/infra/gcp-broad/main.tf index 75b27ea7411..9a4bfe8b9a1 100644 --- a/infra/gcp-broad/main.tf +++ b/infra/gcp-broad/main.tf @@ -508,20 +508,30 @@ resource "google_compute_address" "internal_gateway" { region = var.gcp_region } -# Cloud Router for GKE node outbound NAT -resource "google_compute_router" "gke_node_outbound_router" { - name = "gke-node-outbound-router" - region = var.gcp_region - network = google_compute_network.default.id +# Parse batch_gcp_regions to create Cloud NAT for all regions where batch instances can be created +locals { + batch_regions = jsondecode(var.batch_gcp_regions) +} + +# Cloud Routers for outbound NAT in all batch regions +resource "google_compute_router" "outbound_router" { + for_each = toset(local.batch_regions) + name = "outbound-router-${each.key}" + region = each.key + network = google_compute_network.default.id } -# Cloud NAT Gateway for GKE private node outbound internet access -resource "google_compute_router_nat" "gke_node_outbound_nat" { - name = "gke-node-outbound-nat" - router = google_compute_router.gke_node_outbound_router.name - region = var.gcp_region +# Cloud NAT Gateways for outbound internet access in all batch regions +resource "google_compute_router_nat" "outbound_nat" { + for_each = google_compute_router.outbound_router + name = "outbound-nat-${each.key}" + router = each.value.name + region = each.key nat_ip_allocate_option = "AUTO_ONLY" source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" + enable_dynamic_port_allocation = true + min_ports_per_vm = 500 + max_ports_per_vm = 8000 log_config { enable = true diff --git a/infra/gcp/main.tf b/infra/gcp/main.tf index 832b35fbd6e..41e20c89f78 100644 --- a/infra/gcp/main.tf +++ b/infra/gcp/main.tf @@ -380,21 +380,31 @@ resource "google_compute_address" "internal_gateway" { region = var.gcp_region } -# Cloud Router for GKE node outbound NAT -resource "google_compute_router" "gke_node_outbound_router" { - name = "gke-node-outbound-router" - region = var.gcp_region - network = google_compute_network.default.id +# Parse batch_gcp_regions to create Cloud NAT for all regions where batch instances can be created +locals { + batch_regions = jsondecode(var.batch_gcp_regions) +} + +# Cloud Routers for outbound NAT in all batch regions +resource "google_compute_router" "outbound_router" { + for_each = toset(local.batch_regions) + name = "outbound-router-${each.key}" + region = each.key + network = google_compute_network.default.id } -# Cloud NAT Gateway for GKE private node outbound internet access -resource "google_compute_router_nat" "gke_node_outbound_nat" { - name = "gke-node-outbound-nat" - router = google_compute_router.gke_node_outbound_router.name - region = var.gcp_region +# Cloud NAT Gateways for outbound internet access in all batch regions +resource "google_compute_router_nat" "outbound_nat" { + for_each = google_compute_router.outbound_router + name = "outbound-nat-${each.key}" + router = each.value.name + region = each.key nat_ip_allocate_option = "AUTO_ONLY" source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" - + enable_dynamic_port_allocation = true + min_ports_per_vm = 500 + max_ports_per_vm = 8000 + log_config { enable = true filter = "ERRORS_ONLY" From 5dcb3c9961bf02092cfb59ec42c2a45f12422c30 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Mon, 12 Jan 2026 16:01:08 -0500 Subject: [PATCH 08/51] [hailctl] Direct to the batch ui for dev deploy builds (#15214) Updates the dev deploy build script to link us out to the `batch.hail.is` UI pages rather than `ci.hail.is` pages. This change cannot impact the Hail Batch instance as deployed by Broad Institute in GCP --- hail/python/hailtop/hailctl/dev/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hail/python/hailtop/hailctl/dev/cli.py b/hail/python/hailtop/hailctl/dev/cli.py index 613a4626857..6b365a420ef 100644 --- a/hail/python/hailtop/hailctl/dev/cli.py +++ b/hail/python/hailtop/hailctl/dev/cli.py @@ -60,7 +60,7 @@ async def _deploy(branch: str, steps: List[str], excluded_steps: List[str], extr extra_config_dict = unpack_key_value_inputs(extra_config) async with CIClient(deploy_config) as ci_client: batch_id = await ci_client.dev_deploy_branch(branch, steps, excluded_steps, extra_config_dict) - url = deploy_config.url('ci', f'/batches/{batch_id}') + url = deploy_config.url('batch', f'/batches/{batch_id}') print(f'Created deploy batch, see {url}') if open: webbrowser.open(url) From cb54af43bfa2cbd1ec2f1e72cf835288ce017b81 Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Mon, 12 Jan 2026 19:52:57 -0500 Subject: [PATCH 09/51] [query] Replace Deprecated `AnyRefMap` with `HashMap` (#15221) Later versions of scalac will issue the following warning: ``` object AnyRefMap in package mutable is deprecated (since 2.13.16): Use scala.collection.mutable.HashMap instead for better performance. ``` Seems fair enough. Spark have also followed suit: https://gitmemories.com/apache/spark/issues/48128 This change has no impact on the Broad-managed hail batch deployment in GCP. --- hail/.scalafix.conf | 3 ++- .../src-2.12/is/hail/utils/compat/mutable/package.scala | 4 ++++ .../src-2.13/is/hail/utils/compat/mutable/package.scala | 3 +++ hail/hail/src/is/hail/expr/ir/NormalizeNames.scala | 3 +-- hail/hail/src/is/hail/expr/ir/PruneDeadFields.scala | 6 +++--- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/hail/.scalafix.conf b/hail/.scalafix.conf index cc8cb91078d..dab76d2e8d0 100644 --- a/hail/.scalafix.conf +++ b/hail/.scalafix.conf @@ -31,5 +31,6 @@ RemoveUnused { } ForbiddenSymbol.symbols = [ - "scala.collection.compat.immutable.ArraySeq" + "scala.collection.mutable.AnyRefMap", + "scala.collection.compat.immutable.ArraySeq", ] diff --git a/hail/hail/src-2.12/is/hail/utils/compat/mutable/package.scala b/hail/hail/src-2.12/is/hail/utils/compat/mutable/package.scala index 7c2ded2f508..e692c5644c1 100644 --- a/hail/hail/src-2.12/is/hail/utils/compat/mutable/package.scala +++ b/hail/hail/src-2.12/is/hail/utils/compat/mutable/package.scala @@ -1,6 +1,10 @@ package is.hail.utils.compat package object mutable { + // scalafix:off ForbiddenSymbol + type AnyRefMap[K <: AnyRef, V] = scala.collection.mutable.AnyRefMap[K, V] + val AnyRefMap = scala.collection.mutable.AnyRefMap + type Growable[-A] = scala.collection.generic.Growable[A] type Shrinkable[-A] = scala.collection.generic.Shrinkable[A] } diff --git a/hail/hail/src-2.13/is/hail/utils/compat/mutable/package.scala b/hail/hail/src-2.13/is/hail/utils/compat/mutable/package.scala index 1918e4e5942..7057944cb3e 100644 --- a/hail/hail/src-2.13/is/hail/utils/compat/mutable/package.scala +++ b/hail/hail/src-2.13/is/hail/utils/compat/mutable/package.scala @@ -1,6 +1,9 @@ package is.hail.utils.compat package object mutable { + type AnyRefMap[K, V] = scala.collection.mutable.HashMap[K, V] + val AnyRefMap = scala.collection.mutable.HashMap + type Growable[-A] = scala.collection.mutable.Growable[A] type GrowableCompat[-A] = Growable[A] type Shrinkable[-A] = scala.collection.mutable.Shrinkable[A] diff --git a/hail/hail/src/is/hail/expr/ir/NormalizeNames.scala b/hail/hail/src/is/hail/expr/ir/NormalizeNames.scala index 738fe308cbd..08217fd65ff 100644 --- a/hail/hail/src/is/hail/expr/ir/NormalizeNames.scala +++ b/hail/hail/src/is/hail/expr/ir/NormalizeNames.scala @@ -7,7 +7,6 @@ import is.hail.types.virtual.Type import is.hail.utils.StackSafe._ import scala.annotation.tailrec -import scala.collection.mutable object NormalizeNames { def apply[T <: BaseIR](allowFreeVariables: Boolean = false)(ctx: ExecuteContext, ir: T): T = @@ -99,7 +98,7 @@ class NormalizeNames(freeVariables: Set[Name]) { } Recur(newName, args, typ).mapChildrenStackSafe(normalizeIR(_, env)) case ir => - val bindingsMap = mutable.AnyRefMap.empty[Name, Name] + val bindingsMap = is.hail.utils.compat.mutable.AnyRefMap.empty[Name, Name] val updateEnv: (BindingEnv[Name], Bindings[Type]) => BindingEnv[Name] = if (needsRenaming(ir)) { (env, bindings) => val bindingsNames = bindings.map((name, _) => bindingsMap.getOrElseUpdate(name, gen())) diff --git a/hail/hail/src/is/hail/expr/ir/PruneDeadFields.scala b/hail/hail/src/is/hail/expr/ir/PruneDeadFields.scala index 7bad42e7218..46f98b9ac2b 100644 --- a/hail/hail/src/is/hail/expr/ir/PruneDeadFields.scala +++ b/hail/hail/src/is/hail/expr/ir/PruneDeadFields.scala @@ -1385,7 +1385,7 @@ object PruneDeadFields extends Logging { recurMax(ir, 0) case Block(bindings, _) => - val typeStates = mutable.AnyRefMap.empty[Name, TypeState] + val typeStates = is.hail.utils.compat.mutable.AnyRefMap.empty[Name, TypeState] recurWithTypeStates(ir, bindings.length, requestedType, typeStates) for (i <- bindings.indices.reverse) recurWithTypeStates(ir, i, typeStates(bindings(i).name).newType, typeStates) @@ -1497,7 +1497,7 @@ object PruneDeadFields extends Logging { case StreamFold2(_, accum, valueName, seq, _) => recur(ir, 2 * accum.length + 1, requestedType) - val seqBindings = mutable.AnyRefMap.empty[Name, TypeState] + val seqBindings = is.hail.utils.compat.mutable.AnyRefMap.empty[Name, TypeState] seq.indices.foreach(i => recurMaxWithTypeStates(ir, accum.length + 1 + i, seqBindings)) accum.indices.foreach(i => recurMax(ir, i + 1)) recur(ir, 0, TStream(seqBindings(valueName).newType)) @@ -1607,7 +1607,7 @@ object PruneDeadFields extends Logging { recur(ir, 1, requestedType) case RunAggScan(_, name, _, _, _, _) => - val bindings = mutable.AnyRefMap.empty[Name, TypeState] + val bindings = is.hail.utils.compat.mutable.AnyRefMap.empty[Name, TypeState] recurWithTypeStates(ir, 3, TIterable.elementType(requestedType), bindings) recurMaxWithTypeStates(ir, 2, bindings) From 3033aa4da26fe1c0156734831453d2be24ca9fa5 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 15 Jan 2026 16:07:56 -0500 Subject: [PATCH 10/51] [terraform] Make nat maxes higher, powers of 2 (#15228) ## Change Description Sets the max connections per VM higher, and makes it a power of 2, as required by the API ## Security Assessment - This change potentially impacts the Hail Batch instance as deployed by Broad Institute in GCP ### Impact Rating - This change has a low security impact ### Impact Description Small config change to un-break the terraform file and give more connection space to VMs ### Appsec Review - [x] Required: The impact has been assessed and approved by appsec --- infra/gcp-broad/main.tf | 4 ++-- infra/gcp/main.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/gcp-broad/main.tf b/infra/gcp-broad/main.tf index 9a4bfe8b9a1..2d6fc45ddb8 100644 --- a/infra/gcp-broad/main.tf +++ b/infra/gcp-broad/main.tf @@ -530,8 +530,8 @@ resource "google_compute_router_nat" "outbound_nat" { nat_ip_allocate_option = "AUTO_ONLY" source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" enable_dynamic_port_allocation = true - min_ports_per_vm = 500 - max_ports_per_vm = 8000 + min_ports_per_vm = 512 + max_ports_per_vm = 16384 log_config { enable = true diff --git a/infra/gcp/main.tf b/infra/gcp/main.tf index 41e20c89f78..6d8dffc058d 100644 --- a/infra/gcp/main.tf +++ b/infra/gcp/main.tf @@ -402,8 +402,8 @@ resource "google_compute_router_nat" "outbound_nat" { nat_ip_allocate_option = "AUTO_ONLY" source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" enable_dynamic_port_allocation = true - min_ports_per_vm = 500 - max_ports_per_vm = 8000 + min_ports_per_vm = 512 + max_ports_per_vm = 16384 log_config { enable = true From 12d25413dd320a731d0eeaea90491691070e3131 Mon Sep 17 00:00:00 2001 From: Patrick Schultz Date: Fri, 16 Jan 2026 10:35:22 -0500 Subject: [PATCH 11/51] [compiler] misc 2.13 deprecation fixes (#15226) ## Change Description Fixes a bunch of the easy-to-fix 2.13 deprecation warnings. ## Security Assessment - This change cannot impact the Hail Batch instance as deployed by Broad Institute in GCP --- hail/build.mill | 30 ------------- hail/hail/ir-gen/src/Main.scala | 4 +- hail/hail/src/is/hail/HailFeatureFlags.scala | 6 +-- .../hail/backend/driver/Py4JQueryDriver.scala | 3 +- hail/hail/src/is/hail/expr/ir/Emit.scala | 5 ++- hail/hail/src/is/hail/expr/ir/Interpret.scala | 9 +++- .../src/is/hail/expr/ir/MatrixWriter.scala | 2 +- hail/hail/src/is/hail/expr/ir/Parser.scala | 2 +- hail/hail/src/is/hail/expr/ir/Pretty.scala | 42 +++++++++---------- .../src/is/hail/expr/ir/Requiredness.scala | 4 +- .../expr/ir/functions/StringFunctions.scala | 6 +-- .../src/is/hail/sparkextras/BlockedRDD.scala | 8 ++-- .../types/physical/PCanonicalStruct.scala | 2 +- .../src/is/hail/types/virtual/TDict.scala | 4 +- .../src/is/hail/types/virtual/TStruct.scala | 2 +- .../src/is/hail/utils/FlipbookIterator.scala | 1 + .../src/is/hail/utils/PartitionCounts.scala | 2 +- hail/hail/src/is/hail/utils/package.scala | 8 ++-- .../test/src/is/hail/expr/ir/IRSuite.scala | 4 +- 19 files changed, 59 insertions(+), 85 deletions(-) diff --git a/hail/build.mill b/hail/build.mill index 7d088127dcb..d0fc454f91f 100644 --- a/hail/build.mill +++ b/hail/build.mill @@ -120,38 +120,9 @@ trait HailScalaModule extends ScalaModule, HailJavaModule, ScalafmtModule, Scala "-Wconf:cat=deprecation&msg=method copyArrayToImmutableIndexedSeq in class LowPriorityImplicits2 is deprecated:s", // Passing an explicit array value to a Scala varargs method is deprecated (since 2.13.0) and will result in a defensive copy; Use the more efficient non-copying ArraySeq.unsafeWrapArray or an explicit toIndexedSeq call "-Wconf:cat=deprecation&msg=Passing an explicit array value to a Scala varargs method is deprecated:s", - // method any2stringadd in object Predef is deprecated (since 2.13.0): Implicit injection of + is deprecated. Convert to String to call + - "-Wconf:cat=deprecation&msg=any2stringadd:s", - // method + in class Char is deprecated (since 2.13.0): Adding a number and a String is deprecated. Use the string interpolation `s"$num$str"` - raw"-Wconf:cat=deprecation&msg=method \+ in class Char is deprecated:s", - // method toIterable in trait Iterable is deprecated (since 2.13.7): toIterable is internal and will be made protected; its name is similar to `toList` or `toSeq`, but it doesn't copy non-immutable collections - "-Wconf:cat=deprecation&msg=method toIterable in trait Iterable is deprecated:s", - // method toIterator in class IterableOnceExtensionMethods is deprecated (since 2.13.0): Use .iterator instead - "-Wconf:cat=deprecation&msg=toIterator:s", - // method foreach in class IterableOnceExtensionMethods is deprecated (since 2.13.0): Use .iterator.foreach(...) instead - "-Wconf:cat=deprecation&msg=foreach:s", - // method nonEmpty in class IterableOnceExtensionMethods is deprecated (since 2.13.0): Use .iterator.nonEmpty instead - "-Wconf:cat=deprecation&msg=method nonEmpty in class IterableOnceExtensionMethods is deprecated:s", - // method view in trait IndexedSeqOps is deprecated (since 2.13.0): Use .view.slice(from, until) instead of .view(from, until) - "-Wconf:cat=deprecation&msg=method view in trait IndexedSeqOps is deprecated:s", - // method reverseContents in class StringBuilder is deprecated (since 2.13.0): Use reverseInPlace instead - "-Wconf:cat=deprecation&msg=method reverseContents in class StringBuilder is deprecated:s", - // method + in trait MapOps is deprecated (since 2.13.0): Consider requiring an immutable Map or fall back to Map.concat. - raw"-Wconf:cat=deprecation&msg=method \+ in trait MapOps is deprecated:s", - "-Wconf:cat=deprecation&msg=method - in trait MapOps is deprecated:s", - // method mapValues in trait MapOps is deprecated (since 2.13.0): Use .view.mapValues(f). A future version will include a strict version of this method (for now, .view.mapValues(f).toMap). - "-Wconf:cat=deprecation&msg=method mapValues in trait MapOps is deprecated:s", - // method filterKeys in trait MapOps is deprecated (since 2.13.0): Use .view.filterKeys(f). A future version will include a strict version of this method (for now, .view.filterKeys(p).toMap). - "-Wconf:cat=deprecation&msg=method filterKeys in trait MapOps is deprecated:s", - // method - in trait SetOps is deprecated (since 2.13.0): Consider requiring an immutable Set or fall back to Set.diff - "-Wconf:cat=deprecation&msg=method - in trait SetOps is deprecated:s", - // method reverseMap in trait SeqOps is deprecated (since 2.13.0): Use .reverseIterator.map(f).to(...) instead of .reverseMap(f) - "-Wconf:cat=deprecation&msg=method reverseMap in trait SeqOps is deprecated:s", // method transform in trait SeqOps is deprecated (since 2.13.0): Use `mapInPlace` on an `IndexedSeq` instead "-Wconf:cat=deprecation&msg=method transform in trait SeqOps is deprecated:s", // other - // Widening conversion from Long to Double is deprecated because it loses precision. Write `.toDouble` instead. - "-Wconf:cat=deprecation&msg=Widening conversion from Long to Double:s", "-Wconf:cat=deprecation&msg=symbol literal is deprecated:s", raw"-Wconf:cat=unused-imports&origin=scala\.collection\.compat\._:s", raw"-Wconf:cat=unused-imports&origin=scala\.Option\.option2Iterable:s", @@ -164,4 +135,3 @@ trait HailScalaModule extends ScalaModule, HailJavaModule, ScalafmtModule, Scala } trait HailScalaTests extends ScalaTests, HailScalaModule - diff --git a/hail/hail/ir-gen/src/Main.scala b/hail/hail/ir-gen/src/Main.scala index 6dc7537f620..f9990f74dca 100644 --- a/hail/hail/ir-gen/src/Main.scala +++ b/hail/hail/ir-gen/src/Main.scala @@ -192,7 +192,7 @@ object IRDSL_Impl extends IRDSL { override def copyWithNewChildren(newChildren: SeqRepr[Child]): Repr[Child] = { assert(newChildren.hasStaticLen(1)) - ChildRepr(typ, newChildren(0) + s".asInstanceOf[$typ]") + ChildRepr(typ, s"${newChildren(0)}.asInstanceOf[$typ]") } } @@ -247,7 +247,7 @@ object IRDSL_Impl extends IRDSL { case (SeqRepr.Static(Seq(), _), r) => r case (l, SeqRepr.Static(Seq(), _)) => l case (SeqRepr.Static(l, t), SeqRepr.Static(r, _)) => SeqRepr.Static(l ++ r, t) - case _ => SeqRepr.Dynamic(this + " ++ " + other, eltType) + case _ => SeqRepr.Dynamic(s"$this ++ $other", eltType) } } diff --git a/hail/hail/src/is/hail/HailFeatureFlags.scala b/hail/hail/src/is/hail/HailFeatureFlags.scala index b44360b7e48..72ea8c62174 100644 --- a/hail/hail/src/is/hail/HailFeatureFlags.scala +++ b/hail/hail/src/is/hail/HailFeatureFlags.scala @@ -73,13 +73,13 @@ class HailFeatureFlags private ( } def +(feature: (String, String)): HailFeatureFlags = - new HailFeatureFlags(flags + (feature._1 -> feature._2)) + new HailFeatureFlags(flags.clone() += feature._1 -> feature._2) def define(feature: String): HailFeatureFlags = - new HailFeatureFlags(flags + (feature -> "1")) + new HailFeatureFlags(flags.clone() += feature -> "1") def -(feature: String): HailFeatureFlags = - new HailFeatureFlags(flags - feature) + new HailFeatureFlags(flags.clone() -= feature) def get(flag: String): String = flags(flag) diff --git a/hail/hail/src/is/hail/backend/driver/Py4JQueryDriver.scala b/hail/hail/src/is/hail/backend/driver/Py4JQueryDriver.scala index 2d85760e4f2..54dc24b3e6e 100644 --- a/hail/hail/src/is/hail/backend/driver/Py4JQueryDriver.scala +++ b/hail/hail/src/is/hail/backend/driver/Py4JQueryDriver.scala @@ -21,6 +21,7 @@ import is.hail.utils.ExecutionTimer.Timings import is.hail.variant.ReferenceGenome import scala.annotation.nowarn +import scala.collection.compat._ import scala.collection.mutable import scala.jdk.CollectionConverters._ @@ -302,7 +303,7 @@ final class Py4JQueryDriver(backend: Backend) extends Closeable with Logging { def pyGrepReturn(regex: String, files: Seq[String], maxLines: Int) : Array[(String, Array[String])] = - fileAndLineCounts(regex, files, maxLines).mapValues(_.map(_.value)).toArray + fileAndLineCounts(regex, files, maxLines).view.mapValues(_.map(_.value)).toArray private[this] def addReference(rg: ReferenceGenome): Unit = ReferenceGenome.addFatalOnCollision(references, FastSeq(rg)) diff --git a/hail/hail/src/is/hail/expr/ir/Emit.scala b/hail/hail/src/is/hail/expr/ir/Emit.scala index bdfb6785efc..26929b62e02 100644 --- a/hail/hail/src/is/hail/expr/ir/Emit.scala +++ b/hail/hail/src/is/hail/expr/ir/Emit.scala @@ -23,6 +23,7 @@ import is.hail.types.physical.stypes.interfaces._ import is.hail.types.physical.stypes.primitives._ import is.hail.types.virtual._ import is.hail.utils._ +import is.hail.utils.compat.immutable.ArraySeq import is.hail.variant.ReferenceGenome import scala.annotation.{nowarn, tailrec} @@ -68,8 +69,8 @@ case class EmitEnv(bindings: Env[EmitValue], inputValues: IndexedSeq[EmitValue]) def asParams(freeVariables: Env[Unit]) : (IndexedSeq[ParamType], IndexedSeq[Value[_]], (EmitCodeBuilder, Int) => EmitEnv) = { - val m = bindings.m.filterKeys(freeVariables.contains) - val bindingNames = m.keys.toArray + val m = bindings.m + val bindingNames = m.keys.filter(freeVariables.contains).to(ArraySeq) val paramTypes = bindingNames.map(name => m(name).emitType.paramType) ++ inputValues.map(_.emitType.paramType) val params = diff --git a/hail/hail/src/is/hail/expr/ir/Interpret.scala b/hail/hail/src/is/hail/expr/ir/Interpret.scala index 3bcf95b4cfa..90f93b60453 100644 --- a/hail/hail/src/is/hail/expr/ir/Interpret.scala +++ b/hail/hail/src/is/hail/expr/ir/Interpret.scala @@ -18,6 +18,7 @@ import is.hail.types.virtual._ import is.hail.utils._ import is.hail.utils.compat.immutable.ArraySeq +import scala.collection.compat._ import scala.collection.mutable import org.apache.spark.sql.Row @@ -426,7 +427,9 @@ object Interpret extends Logging { case GroupByKey(collection) => interpret(collection, env, args).asInstanceOf[IndexedSeq[Row]] .groupBy { case Row(k, _) => k } + .view .mapValues { elt: IndexedSeq[Row] => elt.map { case Row(_, v) => v } } + .toMap case StreamTake(a, len) => val aValue = interpret(a, env, args) val lenValue = interpret(len, env, args) @@ -780,11 +783,13 @@ object Interpret extends Logging { if (struct != null) fieldOrder match { case Some(fds) => - val newValues = fields.toMap.mapValues(interpret(_, env, args)) + val m = fields.toMap val oldIndices = old.typ.asInstanceOf[TStruct].fields.map(f => f.name -> f.index).toMap Row.fromSeq(fds.map(name => - newValues.getOrElse(name, struct.asInstanceOf[Row].get(oldIndices(name))) + m.get(name).map(interpret(_, env, args)).getOrElse( + struct.asInstanceOf[Row].get(oldIndices(name)) + ) )) case None => var t = old.typ.asInstanceOf[TStruct] diff --git a/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala b/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala index ded0be716c4..2d1f11c573a 100644 --- a/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala +++ b/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala @@ -607,7 +607,7 @@ case class SplitPartitionNativeWriter( cb += buff.close() } - stages.flatMap(_.toIterable).zip(filenames).foreach { case (source, destination) => + stages.flatten.zip(filenames).foreach { case (source, destination) => cb += mb.getFS.invoke[String, String, Boolean, Unit]( "copy", source, diff --git a/hail/hail/src/is/hail/expr/ir/Parser.scala b/hail/hail/src/is/hail/expr/ir/Parser.scala index 57b71de9300..4fb22596adf 100644 --- a/hail/hail/src/is/hail/expr/ir/Parser.scala +++ b/hail/hail/src/is/hail/expr/ir/Parser.scala @@ -2108,7 +2108,7 @@ object IRParser { } def parse[T](s: String, f: (TokenIterator) => T): T = { - val it = IRLexer.parse(s).toIterator.buffered + val it = IRLexer.parse(s).iterator.buffered f(it) } diff --git a/hail/hail/src/is/hail/expr/ir/Pretty.scala b/hail/hail/src/is/hail/expr/ir/Pretty.scala index 84ea2570ace..f76f4041862 100644 --- a/hail/hail/src/is/hail/expr/ir/Pretty.scala +++ b/hail/hail/src/is/hail/expr/ir/Pretty.scala @@ -126,7 +126,7 @@ class Pretty( def prettyLongs(x: IndexedSeq[Long], elideLiterals: Boolean): Doc = { val truncate = elideLiterals && x.length > MAX_VALUES_TO_LOG - val view = if (truncate) x.view else x.view(0, MAX_VALUES_TO_LOG) + val view = if (truncate) x.view else x.view.slice(0, MAX_VALUES_TO_LOG) val docs = view.map(i => text(i.toString)) concat(docs.intersperse[Doc]( "(", @@ -137,7 +137,7 @@ class Pretty( def prettyInts(x: IndexedSeq[Int], elideLiterals: Boolean): Doc = { val truncate = elideLiterals && x.length > MAX_VALUES_TO_LOG - val view = if (truncate) x.view else x.view(0, MAX_VALUES_TO_LOG) + val view = if (truncate) x.view else x.view.slice(0, MAX_VALUES_TO_LOG) val docs = view.map(i => text(i.toString)) concat(docs.intersperse[Doc]( "(", @@ -439,26 +439,26 @@ class Pretty( Pretty.prettyBooleanLiteral(dropCols), Pretty.prettyBooleanLiteral(dropRows), if (elideLiterals) reader.renderShort() - else '"' + StringEscapeUtils.escapeString(JsonMethods.compact(reader.toJValue)) + '"', + else s""""${StringEscapeUtils.escapeString(JsonMethods.compact(reader.toJValue))}"""", ) case MatrixWrite(_, writer) => - single('"' + StringEscapeUtils.escapeString( - Serialization.write(writer)(MatrixWriter.formats) - ) + '"') + single( + s""""${StringEscapeUtils.escapeString(Serialization.write(writer)(MatrixWriter.formats))}"""" + ) case MatrixMultiWrite(_, writer) => - single('"' + StringEscapeUtils.escapeString( - Serialization.write(writer)(MatrixNativeMultiWriter.formats) - ) + '"') + single( + s""""${StringEscapeUtils.escapeString(Serialization.write(writer)(MatrixNativeMultiWriter.formats))}"""" + ) case BlockMatrixRead(reader) => - single('"' + StringEscapeUtils.escapeString(JsonMethods.compact(reader.toJValue)) + '"') + single(s""""${StringEscapeUtils.escapeString(JsonMethods.compact(reader.toJValue))}"""") case BlockMatrixWrite(_, writer) => - single('"' + StringEscapeUtils.escapeString( - Serialization.write(writer)(BlockMatrixWriter.formats) - ) + '"') + single( + s""""${StringEscapeUtils.escapeString(Serialization.write(writer)(BlockMatrixWriter.formats))}"""" + ) case BlockMatrixMultiWrite(_, writer) => - single('"' + StringEscapeUtils.escapeString( - Serialization.write(writer)(BlockMatrixWriter.formats) - ) + '"') + single( + s""""${StringEscapeUtils.escapeString(Serialization.write(writer)(BlockMatrixWriter.formats))}"""" + ) case BlockMatrixBroadcast(_, inIndexExpr, shape, blockSize) => FastSeq( prettyInts(inIndexExpr, elideLiterals), @@ -515,16 +515,16 @@ class Pretty( if (typ == tr.fullType) "None" else typ.parsableString(), Pretty.prettyBooleanLiteral(dropRows), if (elideLiterals) tr.renderShort() - else '"' + StringEscapeUtils.escapeString(JsonMethods.compact(tr.toJValue)) + '"', + else s""""${StringEscapeUtils.escapeString(JsonMethods.compact(tr.toJValue))}"""", ) case TableWrite(_, writer) => single( - '"' + StringEscapeUtils.escapeString(Serialization.write(writer)(TableWriter.formats)) + '"' + s""""${StringEscapeUtils.escapeString(Serialization.write(writer)(TableWriter.formats))}"""" ) case TableMultiWrite(_, writer) => - single('"' + StringEscapeUtils.escapeString( - Serialization.write(writer)(WrappedMatrixNativeMultiWriter.formats) - ) + '"') + single( + s""""${StringEscapeUtils.escapeString(Serialization.write(writer)(WrappedMatrixNativeMultiWriter.formats))}"""" + ) case TableKeyBy(_, keys, isSorted) => FastSeq(prettyIdentifiers(keys), Pretty.prettyBooleanLiteral(isSorted)) case TableRange(n, nPartitions) => FastSeq(n.toString, nPartitions.toString) diff --git a/hail/hail/src/is/hail/expr/ir/Requiredness.scala b/hail/hail/src/is/hail/expr/ir/Requiredness.scala index 13b4e5b1831..dc5629ea27d 100644 --- a/hail/hail/src/is/hail/expr/ir/Requiredness.scala +++ b/hail/hail/src/is/hail/expr/ir/Requiredness.scala @@ -805,9 +805,9 @@ class Requiredness(val usesAndDefs: UsesAndDefs, ctx: ExecuteContext) { lookup(old) match { case oldReq: RStruct => requiredness.union(oldReq.required) - val fieldMap = fields.toMap.mapValues(lookup) + val fieldMap = fields.toMap tcoerce[RStruct](requiredness).fields.foreach { f => - f.typ.unionFrom(fieldMap.getOrElse(f.name, oldReq.field(f.name))) + f.typ.unionFrom(fieldMap.get(f.name).map(lookup).getOrElse(oldReq.field(f.name))) } case _ => fields.foreach { case (n, f) => tcoerce[RStruct](requiredness).field(n).unionFrom(lookup(f)) diff --git a/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala index a8fd0eb010e..5b1fbe3acaf 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala @@ -26,11 +26,7 @@ import org.json4s.JValue import org.json4s.jackson.JsonMethods object StringFunctions extends RegistryFunctions { - def reverse(s: String): String = { - val sb = new StringBuilder - sb.append(s) - sb.reverseContents().result() - } + def reverse(s: String): String = s.reverse def upper(s: String): String = s.toUpperCase diff --git a/hail/hail/src/is/hail/sparkextras/BlockedRDD.scala b/hail/hail/src/is/hail/sparkextras/BlockedRDD.scala index 9006acbe901..aef65f7c131 100644 --- a/hail/hail/src/is/hail/sparkextras/BlockedRDD.scala +++ b/hail/hail/src/is/hail/sparkextras/BlockedRDD.scala @@ -2,6 +2,7 @@ package is.hail.sparkextras import is.hail.utils._ +import scala.collection.compat._ import scala.reflect.ClassTag import org.apache.spark.{Dependency, NarrowDependency, Partition, TaskContext} @@ -54,15 +55,12 @@ class BlockedRDD[T]( val locationAvail = range.flatMap(i => prev.preferredLocations(prevPartitions(i)) ) - .groupBy(identity) - .mapValues(_.length) + .counter() if (locationAvail.isEmpty) return FastSeq.empty[String] val m = locationAvail.values.max - locationAvail.filter(_._2 == m) - .keys - .toFastSeq + locationAvail.filter(_._2 == m).keys.toFastSeq } } diff --git a/hail/hail/src/is/hail/types/physical/PCanonicalStruct.scala b/hail/hail/src/is/hail/types/physical/PCanonicalStruct.scala index 455ba1daeb7..a1b4a2063c8 100644 --- a/hail/hail/src/is/hail/types/physical/PCanonicalStruct.scala +++ b/hail/hail/src/is/hail/types/physical/PCanonicalStruct.scala @@ -112,7 +112,7 @@ final case class PCanonicalStruct(fields: IndexedSeq[PField], required: Boolean ab += fields(i) i += 1 } - val it = fieldsToInsert.toIterator + val it = fieldsToInsert.iterator while (it.hasNext) { val (name, typ) = it.next() if (fieldIdx.contains(name)) { diff --git a/hail/hail/src/is/hail/types/virtual/TDict.scala b/hail/hail/src/is/hail/types/virtual/TDict.scala index b30851c05fb..a67fb9f5eea 100644 --- a/hail/hail/src/is/hail/types/virtual/TDict.scala +++ b/hail/hail/src/is/hail/types/virtual/TDict.scala @@ -4,6 +4,8 @@ import is.hail.annotations.{Annotation, ExtendedOrdering} import is.hail.backend.HailStateManager import is.hail.utils._ +import scala.collection.compat._ + import org.json4s.jackson.JsonMethods final case class TDict(keyType: Type, valueType: Type) extends TContainer { @@ -75,7 +77,7 @@ final case class TDict(keyType: Type, valueType: Type) extends TContainer { return identity val subsetValue = valueType.valueSubsetter(subdict.valueType) - (a: Any) => a.asInstanceOf[Map[Any, Any]].mapValues(subsetValue) + (a: Any) => a.asInstanceOf[Map[Any, Any]].view.mapValues(subsetValue) } override def arrayElementsRepr: TArray = TArray(elementType) diff --git a/hail/hail/src/is/hail/types/virtual/TStruct.scala b/hail/hail/src/is/hail/types/virtual/TStruct.scala index f28bad29363..15ead3ff969 100644 --- a/hail/hail/src/is/hail/types/virtual/TStruct.scala +++ b/hail/hail/src/is/hail/types/virtual/TStruct.scala @@ -296,7 +296,7 @@ final case class TStruct(fields: IndexedSeq[Field]) extends TBaseStruct { ab += fields(i) i += 1 } - val it = fieldsToInsert.toIterator + val it = fieldsToInsert.iterator while (it.hasNext) { val (name, typ) = it.next() if (fieldIdx.contains(name)) { diff --git a/hail/hail/src/is/hail/utils/FlipbookIterator.scala b/hail/hail/src/is/hail/utils/FlipbookIterator.scala index fe27fbf4481..8da883b73e6 100644 --- a/hail/hail/src/is/hail/utils/FlipbookIterator.scala +++ b/hail/hail/src/is/hail/utils/FlipbookIterator.scala @@ -184,6 +184,7 @@ abstract class FlipbookIterator[A] extends BufferedIterator[A] { self => ) @nowarn213("msg=GenTraversableOnce in package collection is deprecated") + @nowarn213("msg=toIterator in class IterableOnceExtensionMethods is deprecated") override def flatMap[B](f: A => scala.collection.GenTraversableOnce[B]): FlipbookIterator[B] = FlipbookIterator( new StateMachine[B] { diff --git a/hail/hail/src/is/hail/utils/PartitionCounts.scala b/hail/hail/src/is/hail/utils/PartitionCounts.scala index 1ddc6d990a7..225313cfbba 100644 --- a/hail/hail/src/is/hail/utils/PartitionCounts.scala +++ b/hail/hail/src/is/hail/utils/PartitionCounts.scala @@ -35,7 +35,7 @@ object PartitionCounts { def getTailPCs(original: IndexedSeq[Long], n: Long): IndexedSeq[Long] = getPCSubsetOffset(n, original.reverseIterator) match { case Some(PCSubsetOffset(lastIdx, nKeep, _)) => - (0 to lastIdx).reverseMap { i => + (0 to lastIdx).reverse.map { i => if (i == lastIdx) nKeep else diff --git a/hail/hail/src/is/hail/utils/package.scala b/hail/hail/src/is/hail/utils/package.scala index e162f2dded0..dfd2e0bf005 100644 --- a/hail/hail/src/is/hail/utils/package.scala +++ b/hail/hail/src/is/hail/utils/package.scala @@ -53,7 +53,7 @@ package utils { sealed trait AnyFailAllFail[C[_]] { def apply[T](ts: IterableOnce[Option[T]])(implicit cbf: Factory[T, C[T]]): Option[C[T]] = { val b = cbf.newBuilder - for (t <- ts) + for (t <- ts.iterator) if (t.isEmpty) return None else @@ -683,13 +683,13 @@ package object utils def makeJavaMap[K, V](x: IterableOnce[(K, V)]): java.util.HashMap[K, V] = { val m = new java.util.HashMap[K, V] - x.foreach { case (k, v) => m.put(k, v) } + x.iterator.foreach { case (k, v) => m.put(k, v) } m } def makeJavaSet[K](x: IterableOnce[K]): java.util.HashSet[K] = { val m = new java.util.HashSet[K] - x.foreach(m.add) + x.iterator.foreach(m.add) m } @@ -699,7 +699,7 @@ package object utils key: T => K, value: T => V, ): collection.Map[K, V] = { - val it = ts.toIterator + val it = ts.iterator val m = mutable.Map[K, V]() while (it.hasNext) { val t = it.next() diff --git a/hail/hail/test/src/is/hail/expr/ir/IRSuite.scala b/hail/hail/test/src/is/hail/expr/ir/IRSuite.scala index eb70ae0a9d0..73a1fece5be 100644 --- a/hail/hail/test/src/is/hail/expr/ir/IRSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/IRSuite.scala @@ -4428,11 +4428,11 @@ class IRSuite extends HailSuite { assert(interval1.isDisjointFrom(kord, interval2)) } - val splitterValueDuplicated = splitters.counter().mapValues(_ > 1) + val splitterValueCounts = splitters.counter() val intBuilder = new IntArrayBuilder() splitters.toSet.toIndexedSeq.sorted.foreach { e => intBuilder.add(e) - if (splitterValueDuplicated(e)) { + if (splitterValueCounts(e) > 1) { intBuilder.add(e) } } From 1e1dbf67afadc67f5ecb600241bd25b95bbbbde5 Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Tue, 20 Jan 2026 14:46:35 -0500 Subject: [PATCH 12/51] [query] replace `readRepeatedly` with built-in `readNBytes` (#15231) This change cannot impact the Hail Batch instance as deployed by Broad Institute in GCP --- hail/hail/src/is/hail/io/plink/LoadPlink.scala | 2 +- .../hail/utils/richUtils/RichInputStream.scala | 18 +----------------- .../is/hail/io/compress/BGzipCodecSuite.scala | 4 ++-- 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/hail/hail/src/is/hail/io/plink/LoadPlink.scala b/hail/hail/src/is/hail/io/plink/LoadPlink.scala index afed2c87f12..14649b640a8 100644 --- a/hail/hail/src/is/hail/io/plink/LoadPlink.scala +++ b/hail/hail/src/is/hail/io/plink/LoadPlink.scala @@ -492,7 +492,7 @@ class MatrixPLINKReader( offset = newOffset } - is.readFully(input, 0, input.length) + is.readFully(input) rvb.start(requestedPType) rvb.startStruct() diff --git a/hail/hail/src/is/hail/utils/richUtils/RichInputStream.scala b/hail/hail/src/is/hail/utils/richUtils/RichInputStream.scala index 61b5ab8f34f..ccda9720f67 100644 --- a/hail/hail/src/is/hail/utils/richUtils/RichInputStream.scala +++ b/hail/hail/src/is/hail/utils/richUtils/RichInputStream.scala @@ -9,23 +9,7 @@ class RichInputStream(val in: InputStream) extends AnyVal { readFully(to, 0, to.length) def readFully(to: Array[Byte], toOff: Int, n: Int): Unit = { - val nRead = readRepeatedly(to, toOff, n) + val nRead = in.readNBytes(to, toOff, n) if (nRead < n) fatal(s"Premature end of file: expected $n bytes, found $nRead") } - - def readRepeatedly(to: Array[Byte], toOff: Int, n: Int): Int = { - assert(toOff + n <= to.length) - var read = 0 - var endOfStream = false - while (read < n && !endOfStream) { - val r = in.read(to, toOff + read, n - read) - if (r < 0) - endOfStream = true - else - read += r - } - read - } - - def readRepeatedly(to: Array[Byte]): Int = readRepeatedly(to, 0, to.length) } diff --git a/hail/hail/test/src/is/hail/io/compress/BGzipCodecSuite.scala b/hail/hail/test/src/is/hail/io/compress/BGzipCodecSuite.scala index 0c30b9813df..ca2c672ba05 100644 --- a/hail/hail/test/src/is/hail/io/compress/BGzipCodecSuite.scala +++ b/hail/hail/test/src/is/hail/io/compress/BGzipCodecSuite.scala @@ -241,8 +241,8 @@ class BGzipCodecSuite extends HailSuite with ScalaCheckDrivenPropertyChecks { assert(decompIS.getVirtualOffset() == vOff); uncompIS.seek(uOff.toLong + extra) - val decompRead = decompIS.readRepeatedly(decompData) - val uncompRead = uncompIS.readRepeatedly(uncompData) + val decompRead = decompIS.readNBytes(decompData, 0, decompData.length) + val uncompRead = uncompIS.readNBytes(uncompData, 0, uncompData.length) assert( decompRead == uncompRead, From e68a2a07604d368b33183b7c5d7c584ac0f744db Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 22 Jan 2026 10:20:50 -0500 Subject: [PATCH 13/51] [docs] Document refreshing the trivy scanner gsa key (#15235) ## Change Description Add documentation for refreshing the trivy-scanner gsa key ## Security Assessment - This change cannot impact the Hail Batch instance as deployed by Broad Institute in GCP --- .../refreshing-trivy-scanner-github-secret.md | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 dev-docs/refreshing-trivy-scanner-github-secret.md diff --git a/dev-docs/refreshing-trivy-scanner-github-secret.md b/dev-docs/refreshing-trivy-scanner-github-secret.md new file mode 100644 index 00000000000..e5222b498ca --- /dev/null +++ b/dev-docs/refreshing-trivy-scanner-github-secret.md @@ -0,0 +1,90 @@ +# Refreshing the trivy-scanner GitHub Secret + +The `GOOGLE_GAR_CREDENTIALS` GitHub secret is used by the Trivy Security Scan workflow +(`.github/workflows/trivy-scan.yml`) to authenticate with Google Cloud and access +images from Google Artifact Registry (GAR) for vulnerability scanning. + +This secret contains a Google Service Account (GSA) key in JSON format. Service account +keys should be rotated periodically for security. This guide walks through the process +of creating a new GSA key and updating the GitHub secret. + +## Prerequisites + +- Access to the Google Cloud Console for the relevant project +- Admin access to the GitHub repository to update secrets +- Knowledge of which Google Service Account is used for the trivy-scanner workflow + +## Creating a New GSA Key in Google Cloud Console + +1. **Navigate to the Google Cloud Console** + - Go to https://console.cloud.google.com + - Select the appropriate project (e.g., `hail-vdc`) + +2. **Open the Service Accounts page** + - In the left navigation menu, go to **IAM & Admin** → **Service Accounts** + - Or navigate directly to: https://console.cloud.google.com/iam-admin/serviceaccounts + +3. **Find the Service Account** + - Locate the `trivy-scanner` service account. + - Click on the service account name to open its details page + +4. **Create a New Key** + - Click on the **KEYS** tab + - Click **ADD KEY** → **Create new key** + - Select **JSON** as the key type + - Click **CREATE** + - The JSON key file will automatically download to your computer + +5. **Save the Key Securely** + - The downloaded file contains sensitive credentials + - Store it temporarily in a secure location until you've updated the GitHub secret + +## Copying the Key to GitHub Secret + +1. **Open the Key File** + - Open the downloaded JSON key file in a text editor + - The file contains a JSON object with fields like `type`, `project_id`, `private_key_id`, + `private_key`, `client_email`, etc. + +2. **Copy the Entire Contents** + - Select all contents of the JSON file (the entire JSON object) + - Copy it to your clipboard + - Ensure you copy the complete JSON, including all opening and closing braces + +3. **Navigate to GitHub Repository Settings** + - Go to the repository: https://github.com/hail-is/hail + - Click on **Settings** (requires admin access) + - In the left sidebar, click **Secrets and variables** → **Actions** + +4. **Update the Secret** + - Find the `GOOGLE_GAR_CREDENTIALS` secret in the list + - Click **Update** (or if it doesn't exist, click **New repository secret**) + - Paste the entire JSON key contents into the **Secret** field + - Click **Update secret** (or **Add secret**) + - You might need to authenticate with 2FA to update the secret + +5. **Verify the Update** + - The secret should now show as "Last updated: now" + - You can verify it works by running the Trivy Security Scan workflow manually + - Go to **Actions** → **Trivy Security Scan** → **Run workflow** + +## Cleanup + +After successfully updating the GitHub secret: + +1. **Delete the Old Keys** + - Return to the Google Cloud Console Service Accounts page + - Open the service account's **KEYS** tab + - Identify the old key (it will have an older creation date) + - Click the **Delete** icon (trash can) next to the old key + - Confirm the deletion + +2. **Delete the Local Key File** + - Securely delete the downloaded JSON key file from your local machine + - Consider using `shred` or secure deletion tools if the file contained highly sensitive data + +## Troubleshooting + +- **Authentication errors in workflow**: Verify the JSON was copied completely and correctly +- **Permission denied errors**: Ensure the service account has the necessary Artifact Registry permissions +- **Key not found**: Verify you're using the correct service account for the trivy-scanner workflow From fb3343b265275b0b635441928f88542b1d8cca5c Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Fri, 23 Jan 2026 13:27:49 -0500 Subject: [PATCH 14/51] [query] remove tunion (#15232) TUnion was never implemented and so is safe to delete. This change does not impact the Broad-managed hail batch deployment in GCP. --- hail/hail/src/is/hail/expr/ir/Parser.scala | 8 -- .../is/hail/types/TypeWithRequiredness.scala | 27 ---- .../src/is/hail/types/virtual/TUnion.scala | 134 ------------------ hail/python/hail/expr/__init__.py | 2 - .../expr/expressions/expression_typecheck.py | 48 +------ hail/python/hail/expr/type_parsing.py | 11 +- hail/python/hail/expr/types.py | 95 +------------ hail/python/test/hail/expr/test_types.py | 7 - 8 files changed, 5 insertions(+), 327 deletions(-) delete mode 100644 hail/hail/src/is/hail/types/virtual/TUnion.scala diff --git a/hail/hail/src/is/hail/expr/ir/Parser.scala b/hail/hail/src/is/hail/expr/ir/Parser.scala index 4fb22596adf..279b4490e07 100644 --- a/hail/hail/src/is/hail/expr/ir/Parser.scala +++ b/hail/hail/src/is/hail/expr/ir/Parser.scala @@ -538,12 +538,6 @@ object IRParser { punctuation(it, "}") val fields = args.zipWithIndex.map { case ((id, t), i) => Field(id, t, i) } TStruct(fields) - case "Union" => - punctuation(it, "{") - val args = repsepUntil(it, type_field, PunctuationToken(","), PunctuationToken("}")) - punctuation(it, "}") - val cases = args.zipWithIndex.map { case ((id, t), i) => Case(id, t, i) } - TUnion(cases) case "Void" => TVoid } typ @@ -2151,8 +2145,6 @@ object IRParser { def parseStructType(code: String): TStruct = tcoerce[TStruct](parse(code, type_expr)) - def parseUnionType(code: String): TUnion = tcoerce[TUnion](parse(code, type_expr)) - def parseRVDType(code: String): RVDType = parse(code, rvd_type_expr) def parseTableType(code: String): TableType = parse(code, table_type_expr) diff --git a/hail/hail/src/is/hail/types/TypeWithRequiredness.scala b/hail/hail/src/is/hail/types/TypeWithRequiredness.scala index 40aa288434f..07aefe4c367 100644 --- a/hail/hail/src/is/hail/types/TypeWithRequiredness.scala +++ b/hail/hail/src/is/hail/types/TypeWithRequiredness.scala @@ -53,12 +53,6 @@ object BaseTypeWithRequiredness { assert(rf.index == f.index) check(rf.typ, f.typ) } - case r: RUnion => - val union = typ.asInstanceOf[TUnion] - r.cases.lazyZip(union.cases).foreach { (rc, c) => - assert(rc._1 == c.name) - check(rc._2, c.typ) - } } } } @@ -74,7 +68,6 @@ object TypeWithRequiredness { case t: TInterval => RInterval(apply(t.pointType), apply(t.pointType)) case t: TStruct => RStruct.fromNamesAndTypes(t.fields.map(f => f.name -> apply(f.typ))) case t: TTuple => RTuple.fromNamesAndTypes(t.fields.map(f => f.name -> apply(f.typ))) - case t: TUnion => RUnion(t.cases.map(c => c.name -> apply(c.typ))) } } @@ -570,26 +563,6 @@ case class RTuple(fields: IndexedSeq[RField]) extends RBaseStruct { s"RTuple[${fields.map(f => s"${f.index}: ${f.typ.toString}").mkString(",")}]" } -case class RUnion(cases: IndexedSeq[(String, TypeWithRequiredness)]) extends TypeWithRequiredness { - val children: IndexedSeq[TypeWithRequiredness] = cases.map(_._2) - def _unionLiteral(a: Annotation): Unit = ??? - def _matchesPType(pt: PType): Boolean = ??? - def _unionPType(pType: PType): Unit = ??? - def _unionEmitType(emitType: EmitType): Unit = ??? - - def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RUnion = { - assert(newChildren.length == cases.length) - RUnion(Array.tabulate(cases.length)(i => - cases(i)._1 -> tcoerce[TypeWithRequiredness](newChildren(i)) - )) - } - - def canonicalPType(t: Type): PType = ??? - - def _toString: String = - s"RStruct[${cases.map { case (n, t) => s"$n: ${t.toString}" }.mkString(",")}]" -} - object RTable { def apply(rowStruct: RStruct, globStruct: RStruct, key: IndexedSeq[String]): RTable = RTable( diff --git a/hail/hail/src/is/hail/types/virtual/TUnion.scala b/hail/hail/src/is/hail/types/virtual/TUnion.scala deleted file mode 100644 index 88e7a7335d1..00000000000 --- a/hail/hail/src/is/hail/types/virtual/TUnion.scala +++ /dev/null @@ -1,134 +0,0 @@ -package is.hail.types.virtual - -import is.hail.annotations.ExtendedOrdering -import is.hail.backend.HailStateManager -import is.hail.expr.ir.IRParser -import is.hail.utils._ -import is.hail.utils.compat.immutable.ArraySeq - -import scala.collection.compat._ - -import org.json4s.CustomSerializer -import org.json4s.JsonAST.JString - -final case class Case(name: String, typ: Type, index: Int) { - - def unify(cf: Case): Boolean = - name == cf.name && - typ.unify(cf.typ) && - index == cf.index - - def pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = { - if (compact) { - sb.append(prettyIdentifier(name)) - sb.append(":") - } else { - sb.append(" " * indent) - sb.append(prettyIdentifier(name)) - sb.append(": ") - } - typ.pretty(sb, indent, compact) - } -} - -class TUnionSerializer extends CustomSerializer[TUnion](format => - ( - { case JString(s) => IRParser.parseUnionType(s) }, - { case t: TUnion => JString(t.parsableString()) }, - ) - ) - -object TUnion { - val empty: TUnion = TUnion(FastSeq()) - - def apply(args: (String, Type)*): TUnion = - TUnion(args - .iterator - .zipWithIndex - .map { case ((n, t), i) => Case(n, t, i) } - .toArray) -} - -final case class TUnion(cases: IndexedSeq[Case]) extends Type { - lazy val types: Array[Type] = cases.map(_.typ).toArray - - lazy val caseIdx: collection.Map[String, Int] = toMapFast(cases)(_.name, _.index) - - lazy val caseNames: Array[String] = cases.map(_.name).toArray - - def size: Int = cases.length - - override def unify(concrete: Type): Boolean = concrete match { - case TUnion(cfields) => - cases.length == cfields.length && - cases.lazyZip(cfields).forall { case (f, cf) => - f.unify(cf) - } - case _ => false - } - - override def subst(): TUnion = TUnion(cases.map(f => f.copy(typ = f.typ.subst()))) - - def index(str: String): Option[Int] = caseIdx.get(str) - - def selfCase(name: String): Option[Case] = caseIdx.get(name).map(i => cases(i)) - - def hasCase(name: String): Boolean = caseIdx.contains(name) - - def getCase(name: String): Case = cases(caseIdx(name)) - - def fieldType(name: String): Type = types(caseIdx(name)) - - def rename(m: Map[String, String]): TUnion = { - val newFieldsBuilder = ArraySeq.newBuilder[(String, Type)] - cases.foreach { fd => - val n = fd.name - newFieldsBuilder += (m.getOrElse(n, n) -> fd.typ) - } - TUnion(newFieldsBuilder.result(): _*) - } - - def _toPretty: String = { - val sb = new StringBuilder - _pretty(sb, 0, compact = true) - sb.result() - } - - override def _typeCheck(a: Any): Boolean = ??? - - override def pyString(sb: StringBuilder): Unit = { - sb ++= "union{" - cases.foreachBetween({ field => - sb ++= prettyIdentifier(field.name) ++= ": ": Unit - field.typ.pyString(sb) - })(sb ++= ", ") - sb += '}' - } - - override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = { - if (compact) { - sb ++= "Union{" - cases.foreachBetween(_.pretty(sb, indent, compact))(sb += ',') - sb += '}' - } else { - if (size == 0) - sb ++= "Union { }" - else { - sb ++= "Union {\n" - cases.foreachBetween(_.pretty(sb, indent + 4, compact))(sb ++= ",\n") - sb += '\n' ++= (" " * indent) += '}': Unit - } - } - } - - override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = ??? - - override def isIsomorphicTo(t: Type): Boolean = - t match { - case u: TUnion => - size == u.size && - cases.lazyZip(u.cases).forall(_.typ isIsomorphicTo _.typ) - case _ => - false - } -} diff --git a/hail/python/hail/expr/__init__.py b/hail/python/hail/expr/__init__.py index 3f2fd66b4d3..381d2f4b1f3 100644 --- a/hail/python/hail/expr/__init__.py +++ b/hail/python/hail/expr/__init__.py @@ -282,7 +282,6 @@ tstream, tstruct, ttuple, - tunion, tvariable, tvoid, types_match, @@ -558,7 +557,6 @@ 'tstruct', 'ttable', 'ttuple', - 'tunion', 'tuple', 'tvariable', 'tvoid', diff --git a/hail/python/hail/expr/expressions/expression_typecheck.py b/hail/python/hail/expr/expressions/expression_typecheck.py index 5c6f5679795..f81554a5b2d 100644 --- a/hail/python/hail/expr/expressions/expression_typecheck.py +++ b/hail/python/hail/expr/expressions/expression_typecheck.py @@ -21,7 +21,6 @@ tstream, tstruct, ttuple, - tunion, ) from hail.typecheck import TypeChecker, TypecheckFailure from hail.utils.java import escape_parsable @@ -443,45 +442,6 @@ def _coerce(self, x: Expression): return hl.struct(**{name: c.coerce(x[name]) for name, c in self.fields.items()}) -class UnionCoercer(ExprCoercer): - def __init__(self, cases: Optional[Dict[str, ExprCoercer]] = None): - super(UnionCoercer, self).__init__() - self.cases = cases - - @property - def str_t(self) -> str: - if self.cases is None: - return 'union' - else: - case_strs = ', '.join(f'{escape_parsable(name)}: {c.str_t}' for name, c in self.cases.items()) - return f'union{{{case_strs}}})' - - def _requires_conversion(self, t: HailType) -> bool: - assert isinstance(t, tunion) - if self.cases is None: - return False - else: - return any(c._requires_conversion(t[name]) for name, c in self.cases.items()) - - def can_coerce(self, t: HailType): - if self.cases is None: - return isinstance(t, tunion) - else: - return ( - isinstance(t, tunion) - and len(t) == len(self.cases) - and all( - expected[0] == actual[0] and expected[1].can_coerce(actual[1]) - for expected, actual in zip(self.cases.items(), t.items()) - ) - ) - - def _coerce(self, x: Expression): - assert isinstance(x, hl.expr.StructExpression) - assert list(x.keys()) == list(self.cases.keys()) - raise NotImplementedError() - - class OneOfExprCoercer(ExprCoercer): def __init__(self, *options: ExprCoercer): super(OneOfExprCoercer, self).__init__() @@ -526,7 +486,6 @@ def _coerce(self, x: Expression) -> Expression: expr_dict = DictCoercer expr_tuple = TupleCoercer expr_struct = StructCoercer -expr_union = UnionCoercer expr_numeric = expr_oneof(expr_int32, expr_int64, expr_float32, expr_float64) primitives: Dict[HailType, ExprCoercer] = { @@ -541,8 +500,8 @@ def _coerce(self, x: Expression) -> Expression: def coercer_from_dtype(t: HailType) -> ExprCoercer: - if t in primitives: - return primitives[t] + if (c := primitives.get(t)) is not None: + return c elif isinstance(t, tlocus): return expr_locus(t.reference_genome) elif isinstance(t, tinterval): @@ -562,5 +521,4 @@ def coercer_from_dtype(t: HailType) -> ExprCoercer: elif isinstance(t, tstruct): return expr_struct({name: coercer_from_dtype(t_) for name, t_ in t.items()}) else: - assert isinstance(t, tunion) - return expr_union({name: coercer_from_dtype(t_) for name, t_ in t.items()}) + assert False, f'Unknown dtype: {t}' diff --git a/hail/python/hail/expr/type_parsing.py b/hail/python/hail/expr/type_parsing.py index f3f03b19d67..8d95b70a0cd 100644 --- a/hail/python/hail/expr/type_parsing.py +++ b/hail/python/hail/expr/type_parsing.py @@ -6,7 +6,7 @@ from . import types type_grammar_str = r""" - type = _ ( array / bool / call / dict / interval / int64 / int32 / float32 / float64 / locus / ndarray / rng_state / set / stream / struct / str / tuple / union / void / variable ) _ + type = _ ( array / bool / call / dict / interval / int64 / int32 / float32 / float64 / locus / ndarray / rng_state / set / stream / struct / str / tuple / void / variable ) _ variable = "?" simple_identifier (":" simple_identifier)? void = "void" / "tvoid" int64 = "int64" / "tint64" @@ -23,7 +23,6 @@ stream = ("tstream" / "stream") _ "<" type ">" dict = ("tdict" / "dict") _ "<" type "," type ">" struct = ("tstruct" / "struct") _ "{" (fields / _) "}" - union = ("tunion" / "union") _ "{" (fields / _) "}" tuple = ("ttuple" / "tuple") _ "(" ((type ("," type)*) / _) ")" fields = field ("," field)* field = identifier ":" type @@ -112,14 +111,6 @@ def visit_struct(self, node, visited_children): fields = maybe_fields[0] return types.tstruct(**dict(fields)) - def visit_union(self, node, visited_children): - tunion, _, brace, maybe_fields, brace = visited_children # noqa: PLW0128 - if not maybe_fields: - return types.tunion() - else: - fields = maybe_fields[0] - return types.tunion(**dict(fields)) - def visit_tuple(self, node, visited_children): ttuple, _, paren, [maybe_types], paren = visited_children # noqa: PLW0128 if not maybe_types: diff --git a/hail/python/hail/expr/types.py b/hail/python/hail/expr/types.py index 4c053855052..8aa9ffce9f5 100644 --- a/hail/python/hail/expr/types.py +++ b/hail/python/hail/expr/types.py @@ -51,7 +51,6 @@ 'tstream', 'tstruct', 'ttuple', - 'tunion', 'tvariable', 'tvoid', 'types_match', @@ -1505,97 +1504,6 @@ def _get_context(self): return HailTypeContext.union(*self.values()) -class tunion(HailType, Mapping): - @typecheck_method(case_types=hail_type) - def __init__(self, **case_types): - """Tagged union type. Values of type union represent one of several - heterogenous, named cases. - - Parameters - ---------- - cases : keyword args of :class:`.HailType` - The union cases. - - """ - - super(tunion, self).__init__() - self._case_types = case_types - self._cases = tuple(case_types) - - @property - def cases(self): - """Return union case names. - - Returns - ------- - :obj:`tuple` of :class:`str` - Tuple of union case names - """ - return self._cases - - @typecheck_method(item=oneof(int, str)) - def __getitem__(self, item): - if isinstance(item, int): - item = self._cases[item] - return self._case_types[item] - - def __iter__(self): - return iter(self._case_types) - - def __len__(self): - return len(self._cases) - - def __str__(self): - return "union{{{}}}".format(', '.join('{}: {}'.format(escape_parsable(f), str(t)) for f, t in self.items())) - - def _eq(self, other): - return ( - isinstance(other, tunion) and self._cases == other._cases and all(self[c] == other[c] for c in self._cases) - ) - - def _pretty(self, b, indent, increment): - if not self._cases: - b.append('union {}') - return - - pre_indent = indent - indent += increment - b.append('union {') - for i, (f, t) in enumerate(self.items()): - if i > 0: - b.append(', ') - b.append('\n') - b.append(' ' * indent) - b.append('{}: '.format(escape_parsable(f))) - t._pretty(b, indent, increment) - b.append('\n') - b.append(' ' * pre_indent) - b.append('}') - - def _parsable_string(self): - return "Union{{{}}}".format( - ','.join('{}:{}'.format(escape_parsable(f), t._parsable_string()) for f, t in self.items()) - ) - - def unify(self, t): - if not (isinstance(t, tunion) and len(self) == len(t)): - return False - for (f1, t1), (f2, t2) in zip(self.items(), t.items()): - if not (f1 == f2 and t1.unify(t2)): - return False - return True - - def subst(self): - return tunion(**{f: t.subst() for f, t in self.items()}) - - def clear(self): - for f, t in self.items(): - t.clear() - - def _get_context(self): - return HailTypeContext.union(*self.values()) - - class ttuple(HailType, Sequence): """Hail type for tuples. @@ -2267,7 +2175,7 @@ def is_container(t) -> bool: @typecheck(t=HailType) def is_compound(t) -> bool: - return is_container(t) or isinstance(t, (tstruct, tunion, ttuple, tndarray)) + return is_container(t) or isinstance(t, (tstruct, ttuple, tndarray)) def types_match(left, right) -> bool: @@ -2334,7 +2242,6 @@ class tvariable(HailType): 'float64': lambda x: x == tfloat64, 'locus': lambda x: isinstance(x, tlocus), 'struct': lambda x: isinstance(x, tstruct), - 'union': lambda x: isinstance(x, tunion), 'tuple': lambda x: isinstance(x, ttuple), } diff --git a/hail/python/test/hail/expr/test_types.py b/hail/python/test/hail/expr/test_types.py index 20c0b589b2b..0e62243ffe9 100644 --- a/hail/python/test/hail/expr/test_types.py +++ b/hail/python/test/hail/expr/test_types.py @@ -21,7 +21,6 @@ tstr, tstruct, ttuple, - tunion, ) from hail.utils.java import Env @@ -59,9 +58,6 @@ def types_to_test(self): tstruct(a=tfloat64, bb=tint32, c=tbool), tstruct(a=tint32, b=tint32), tstruct(**{'___': tint32, '_ . _': tint32}), - tunion(), - tunion(a=tint32, b=tstr), - tunion(**{'!@#$%^&({[': tstr}), ttuple(tstr, tint32), ttuple(tarray(tint32), tstr, tstr, tint32, tbool), ttuple(), @@ -141,8 +137,6 @@ def test_get_context(self): tndarray(tstr, 1), tstruct(), tstruct(x=tint32, y=tint64, z=tarray(tset(tstr))), - tunion(), - tunion(a=tint32, b=tstr), ttuple(tstr, tint32), ttuple(), ], @@ -157,7 +151,6 @@ def test_get_context(self): tinterval(tl1), tset(tinterval(tl1)), tstruct(a=tint32, b=tint32, c=tarray(tl1)), - tunion(a=tint32, b=tl1), ttuple(tarray(tint32), tl1, tstr, tint32, tbool), ], {"GRCh37"}, From a8a3cd25c1c33c5cefa845c23c28390f2dda8cd0 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 29 Jan 2026 11:06:14 -0500 Subject: [PATCH 15/51] [Services] Bump python deps (#15218) ## Change Description Python dependency bump ## Security Assessment This change potentially impacts the Hail Batch instance as deployed by Broad Institute in GCP ### Impact Rating - This change has a medium security impact ### Impact Description Dependency updates ### Appsec Review - [x] Required: The impact has been assessed and approved by appsec --- batch/pinned-requirements.txt | 6 +- ci/pinned-requirements.txt | 4 +- gear/pinned-requirements.txt | 25 ++++---- hail/python/dev/pinned-requirements.txt | 66 ++++++++++----------- hail/python/hailtop/pinned-requirements.txt | 24 ++++---- hail/python/pinned-requirements.txt | 32 +++++----- web_common/pinned-requirements.txt | 2 +- 7 files changed, 74 insertions(+), 85 deletions(-) diff --git a/batch/pinned-requirements.txt b/batch/pinned-requirements.txt index 2a15a2da9b6..1e682fe1a37 100644 --- a/batch/pinned-requirements.txt +++ b/batch/pinned-requirements.txt @@ -1,6 +1,6 @@ # This file was autogenerated by uv via the following command: # uv pip compile --python-version 3.10 --python-platform linux batch/requirements.txt --output-file=batch/pinned-requirements.txt -aiodocker==0.24.0 +aiodocker==0.25.0 # via -r batch/requirements.txt aiohappyeyeballs==2.6.1 # via @@ -9,7 +9,7 @@ aiohappyeyeballs==2.6.1 # -c batch/../hail/python/pinned-requirements.txt # -c batch/../web_common/pinned-requirements.txt # aiohttp -aiohttp==3.13.2 +aiohttp==3.13.3 # via # -c batch/../gear/pinned-requirements.txt # -c batch/../hail/python/dev/pinned-requirements.txt @@ -120,7 +120,7 @@ typing-extensions==4.15.0 # -c batch/../web_common/pinned-requirements.txt # aiosignal # multidict -tzdata==2025.2 +tzdata==2025.3 # via # -c batch/../hail/python/dev/pinned-requirements.txt # -c batch/../hail/python/pinned-requirements.txt diff --git a/ci/pinned-requirements.txt b/ci/pinned-requirements.txt index b4f44972374..3714f3376da 100644 --- a/ci/pinned-requirements.txt +++ b/ci/pinned-requirements.txt @@ -1,6 +1,6 @@ # This file was autogenerated by uv via the following command: # uv pip compile --python-version 3.10 --python-platform linux ci/requirements.txt --output-file=ci/pinned-requirements.txt -certifi==2025.11.12 +certifi==2026.1.4 # via # -c ci/../gear/pinned-requirements.txt # -c ci/../hail/python/dev/pinned-requirements.txt @@ -64,7 +64,7 @@ uritemplate==4.2.0 # via # -c ci/../gear/pinned-requirements.txt # gidgethub -urllib3==2.6.1 +urllib3==2.6.3 # via # -c ci/../gear/pinned-requirements.txt # -c ci/../hail/python/dev/pinned-requirements.txt diff --git a/gear/pinned-requirements.txt b/gear/pinned-requirements.txt index e4ef4edb785..090ef6b6e9e 100644 --- a/gear/pinned-requirements.txt +++ b/gear/pinned-requirements.txt @@ -6,7 +6,7 @@ aiohappyeyeballs==2.6.1 # -c gear/../hail/python/hailtop/pinned-requirements.txt # -c gear/../hail/python/pinned-requirements.txt # aiohttp -aiohttp==3.13.2 +aiohttp==3.13.3 # via # -c gear/../hail/python/dev/pinned-requirements.txt # -c gear/../hail/python/hailtop/pinned-requirements.txt @@ -35,12 +35,7 @@ attrs==25.4.0 # -c gear/../hail/python/hailtop/pinned-requirements.txt # -c gear/../hail/python/pinned-requirements.txt # aiohttp -cachetools==6.2.2 - # via - # -c gear/../hail/python/hailtop/pinned-requirements.txt - # -c gear/../hail/python/pinned-requirements.txt - # google-auth -certifi==2025.11.12 +certifi==2026.1.4 # via # -c gear/../hail/python/dev/pinned-requirements.txt # -c gear/../hail/python/hailtop/pinned-requirements.txt @@ -60,11 +55,11 @@ frozenlist==1.8.0 # -c gear/../hail/python/pinned-requirements.txt # aiohttp # aiosignal -google-api-core==2.28.1 +google-api-core==2.29.0 # via google-api-python-client google-api-python-client==2.187.0 # via google-cloud-profiler -google-auth==2.43.0 +google-auth==2.47.0 # via # -c gear/../hail/python/hailtop/pinned-requirements.txt # -c gear/../hail/python/pinned-requirements.txt @@ -72,7 +67,7 @@ google-auth==2.43.0 # google-api-python-client # google-auth-httplib2 # google-cloud-profiler -google-auth-httplib2==0.2.1 +google-auth-httplib2==0.3.0 # via # google-api-python-client # google-cloud-profiler @@ -107,7 +102,7 @@ packaging==25.0 # -r gear/requirements.txt prometheus-async==19.2.0 # via -r gear/requirements.txt -prometheus-client==0.23.1 +prometheus-client==0.24.0 # via # -c gear/../hail/python/dev/pinned-requirements.txt # -r gear/requirements.txt @@ -119,9 +114,9 @@ propcache==0.4.1 # -c gear/../hail/python/pinned-requirements.txt # aiohttp # yarl -proto-plus==1.26.1 +proto-plus==1.27.0 # via google-api-core -protobuf==6.33.2 +protobuf==6.33.4 # via # google-api-core # google-cloud-profiler @@ -142,7 +137,7 @@ pymysql==1.1.2 # via # -r gear/requirements.txt # aiomysql -pyparsing==3.2.5 +pyparsing==3.3.1 # via # -c gear/../hail/python/dev/pinned-requirements.txt # httplib2 @@ -195,7 +190,7 @@ typing-extensions==4.15.0 # multidict uritemplate==4.2.0 # via google-api-python-client -urllib3==2.6.1 +urllib3==2.6.3 # via # -c gear/../hail/python/dev/pinned-requirements.txt # -c gear/../hail/python/hailtop/pinned-requirements.txt diff --git a/hail/python/dev/pinned-requirements.txt b/hail/python/dev/pinned-requirements.txt index 798dd99dcb3..692fd7c75dd 100644 --- a/hail/python/dev/pinned-requirements.txt +++ b/hail/python/dev/pinned-requirements.txt @@ -4,7 +4,7 @@ aiohappyeyeballs==2.6.1 # via # -c hail/python/dev/../pinned-requirements.txt # aiohttp -aiohttp==3.13.2 +aiohttp==3.13.3 # via # -c hail/python/dev/../pinned-requirements.txt # aiohttp-devtools @@ -16,7 +16,7 @@ aiosignal==1.4.0 # aiohttp alabaster==0.7.16 # via sphinx -anyio==4.12.0 +anyio==4.12.1 # via # httpx # jupyter-server @@ -58,7 +58,7 @@ bleach==6.3.0 # via nbconvert build==1.1.1 # via -r hail/python/dev/requirements.txt -certifi==2025.11.12 +certifi==2026.1.4 # via # -c hail/python/dev/../pinned-requirements.txt # httpcore @@ -92,7 +92,7 @@ curlylint==0.13.1 # via -r hail/python/dev/requirements.txt cycler==0.12.1 # via matplotlib -debugpy==1.8.17 +debugpy==1.8.19 # via ipykernel decorator==4.4.2 # via @@ -128,9 +128,9 @@ executing==2.2.1 # stack-data fastjsonschema==2.21.2 # via nbformat -filelock==3.20.0 +filelock==3.20.3 # via virtualenv -fonttools==4.61.0 +fonttools==4.61.1 # via matplotlib fqdn==1.5.1 # via jsonschema @@ -147,7 +147,7 @@ httpcore==1.0.9 # via httpx httpx==0.28.1 # via jupyterlab -identify==2.6.15 +identify==2.6.16 # via pre-commit idna==3.11 # via @@ -159,7 +159,7 @@ idna==3.11 # yarl imagesize==1.4.1 # via sphinx -importlib-metadata==8.7.0 +importlib-metadata==8.7.1 # via build iniconfig==2.3.0 # via pytest @@ -170,7 +170,7 @@ ipykernel==6.29.5 # jupyterlab # nbclassic # notebook -ipython==8.37.0 +ipython==8.38.0 # via # ipykernel # ipywidgets @@ -197,11 +197,11 @@ jinja2==3.1.6 # nbsphinx # notebook # sphinx -json5==0.12.1 +json5==0.13.0 # via jupyterlab-server jsonpointer==3.0.0 # via jsonschema -jsonschema==4.25.1 +jsonschema==4.26.0 # via # jupyter-events # jupyterlab-server @@ -242,7 +242,7 @@ jupyter-server==2.17.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.5.0 +jupyterlab==4.5.2 # via jupyter jupyterlab-pygments==0.3.0 # via nbconvert @@ -259,7 +259,7 @@ markupsafe==3.0.3 # -c hail/python/dev/../pinned-requirements.txt # jinja2 # nbconvert -matplotlib==3.10.7 +matplotlib==3.10.8 # via -r hail/python/dev/requirements.txt matplotlib-inline==0.2.1 # via @@ -267,7 +267,7 @@ matplotlib-inline==0.2.1 # ipython mccabe==0.7.0 # via pylint -mistune==3.1.4 +mistune==3.2.0 # via nbconvert mock==5.1.0 # via -r hail/python/dev/requirements.txt @@ -278,7 +278,7 @@ multidict==6.7.0 # yarl nbclassic==1.3.3 # via notebook -nbclient==0.10.2 +nbclient==0.10.4 # via nbconvert nbconvert==7.13.1 # via @@ -303,11 +303,11 @@ nest-asyncio==1.6.0 # jupyter-client # nbclassic # notebook -nodeenv==1.9.1 +nodeenv==1.10.0 # via # pre-commit # pyright -nodejs-wheel-binaries==24.11.1 +nodejs-wheel-binaries==24.12.0 # via pyright notebook==6.5.7 # via @@ -347,7 +347,7 @@ pathspec==0.12.1 # via curlylint pexpect==4.9.0 # via ipython -pillow==12.0.0 +pillow==12.1.0 # via # -c hail/python/dev/../pinned-requirements.txt # -r hail/python/dev/requirements.txt @@ -361,7 +361,7 @@ pluggy==1.6.0 # via pytest pre-commit==3.8.0 # via -r hail/python/dev/requirements.txt -prometheus-client==0.23.1 +prometheus-client==0.24.0 # via # jupyter-server # notebook @@ -374,7 +374,7 @@ propcache==0.4.1 # -c hail/python/dev/../pinned-requirements.txt # aiohttp # yarl -psutil==7.1.3 +psutil==7.2.1 # via ipykernel ptyprocess==0.7.0 # via @@ -399,11 +399,11 @@ pygments==2.19.2 # sphinx pylint==3.3.9 # via -r hail/python/dev/requirements.txt -pyparsing==3.2.5 +pyparsing==3.3.1 # via matplotlib pyproject-hooks==1.2.0 # via build -pyright==1.1.407 +pyright==1.1.408 # via -r hail/python/dev/requirements.txt pytest==7.4.4 # via @@ -482,7 +482,7 @@ rpds-py==0.30.0 # referencing ruff==0.11.13 # via -r hail/python/dev/requirements.txt -send2trash==1.8.3 +send2trash==2.0.0 # via # jupyter-server # notebook @@ -496,7 +496,7 @@ six==1.17.0 # rfc3339-validator snowballstemmer==3.0.1 # via sphinx -soupsieve==2.8 +soupsieve==2.8.1 # via beautifulsoup4 sphinx==6.2.1 # via @@ -537,7 +537,7 @@ tinycss2==1.5.1 # via nbconvert toml==0.10.2 # via curlylint -tomli==2.3.0 +tomli==2.4.0 # via # build # jupyterlab @@ -545,7 +545,7 @@ tomli==2.3.0 # pytest tomlkit==0.13.3 # via pylint -tornado==6.5.2 +tornado==6.5.4 # via # -c hail/python/dev/../pinned-requirements.txt # ipykernel @@ -577,7 +577,7 @@ types-decorator==5.2.0.20251101 # via -r hail/python/dev/requirements.txt types-deprecated==1.3.1.20251101 # via -r hail/python/dev/requirements.txt -types-pymysql==1.1.0.20250916 +types-pymysql==1.1.0.20251220 # via -r hail/python/dev/requirements.txt types-python-dateutil==2.9.0.20251115 # via -r hail/python/dev/requirements.txt @@ -585,7 +585,7 @@ types-pyyaml==6.0.12.20250915 # via -r hail/python/dev/requirements.txt types-requests==2.31.0.6 # via -r hail/python/dev/requirements.txt -types-setuptools==80.9.0.20250822 +types-setuptools==80.9.0.20251223 # via -r hail/python/dev/requirements.txt types-six==1.17.0.20251009 # via -r hail/python/dev/requirements.txt @@ -610,21 +610,21 @@ typing-extensions==4.15.0 # pyright # referencing # virtualenv -tzdata==2025.2 +tzdata==2025.3 # via # -c hail/python/dev/../pinned-requirements.txt # arrow uri-template==1.3.0 # via jsonschema -urllib3==2.6.1 +urllib3==2.6.3 # via # -c hail/python/dev/../pinned-requirements.txt # requests -uv==0.9.16 +uv==0.9.24 # via -r hail/python/dev/requirements.txt -uv-build==0.9.16 +uv-build==0.9.24 # via -r hail/python/dev/requirements.txt -virtualenv==20.35.4 +virtualenv==20.36.1 # via pre-commit watchfiles==1.1.1 # via aiohttp-devtools diff --git a/hail/python/hailtop/pinned-requirements.txt b/hail/python/hailtop/pinned-requirements.txt index d8dff76e7d4..7bcd1590641 100644 --- a/hail/python/hailtop/pinned-requirements.txt +++ b/hail/python/hailtop/pinned-requirements.txt @@ -4,7 +4,7 @@ aiodns==2.0.0 # via -r hail/python/hailtop/requirements.txt aiohappyeyeballs==2.6.1 # via aiohttp -aiohttp==3.13.2 +aiohttp==3.13.3 # via -r hail/python/hailtop/requirements.txt aiosignal==1.4.0 # via aiohttp @@ -14,7 +14,7 @@ attrs==25.4.0 # via aiohttp azure-common==1.1.28 # via azure-mgmt-storage -azure-core==1.36.0 +azure-core==1.38.0 # via # azure-identity # azure-mgmt-core @@ -26,18 +26,16 @@ azure-mgmt-core==1.6.0 # via azure-mgmt-storage azure-mgmt-storage==20.1.0 # via -r hail/python/hailtop/requirements.txt -azure-storage-blob==12.27.1 +azure-storage-blob==12.28.0 # via -r hail/python/hailtop/requirements.txt -boto3==1.42.4 +boto3==1.42.26 # via -r hail/python/hailtop/requirements.txt -botocore==1.42.4 +botocore==1.42.26 # via # -r hail/python/hailtop/requirements.txt # boto3 # s3transfer -cachetools==6.2.2 - # via google-auth -certifi==2025.11.12 +certifi==2026.1.4 # via # msrest # requests @@ -64,13 +62,13 @@ frozenlist==1.8.0 # -r hail/python/hailtop/requirements.txt # aiohttp # aiosignal -google-auth==2.43.0 +google-auth==2.47.0 # via # -r hail/python/hailtop/requirements.txt # google-auth-oauthlib google-auth-oauthlib==0.8.0 # via -r hail/python/hailtop/requirements.txt -humanize==4.14.0 +humanize==4.15.0 # via -r hail/python/hailtop/requirements.txt idna==3.11 # via @@ -116,7 +114,7 @@ pyasn1==0.6.1 # rsa pyasn1-modules==0.4.2 # via google-auth -pycares==4.11.0 +pycares==5.0.1 # via aiodns pycparser==2.23 # via cffi @@ -158,7 +156,7 @@ sortedcontainers==2.4.0 # via -r hail/python/hailtop/requirements.txt tabulate==0.9.0 # via -r hail/python/hailtop/requirements.txt -typer==0.20.0 +typer==0.21.1 # via -r hail/python/hailtop/requirements.txt typing-extensions==4.15.0 # via @@ -170,7 +168,7 @@ typing-extensions==4.15.0 # janus # multidict # typer -urllib3==2.6.1 +urllib3==2.6.3 # via # botocore # requests diff --git a/hail/python/pinned-requirements.txt b/hail/python/pinned-requirements.txt index 4472c3b51b1..6fa442bd59e 100644 --- a/hail/python/pinned-requirements.txt +++ b/hail/python/pinned-requirements.txt @@ -8,7 +8,7 @@ aiohappyeyeballs==2.6.1 # via # -c hail/python/hailtop/pinned-requirements.txt # aiohttp -aiohttp==3.13.2 +aiohttp==3.13.3 # via # -c hail/python/hailtop/pinned-requirements.txt # -r hail/python/hailtop/requirements.txt @@ -30,7 +30,7 @@ azure-common==1.1.28 # via # -c hail/python/hailtop/pinned-requirements.txt # azure-mgmt-storage -azure-core==1.36.0 +azure-core==1.38.0 # via # -c hail/python/hailtop/pinned-requirements.txt # azure-identity @@ -49,27 +49,23 @@ azure-mgmt-storage==20.1.0 # via # -c hail/python/hailtop/pinned-requirements.txt # -r hail/python/hailtop/requirements.txt -azure-storage-blob==12.27.1 +azure-storage-blob==12.28.0 # via # -c hail/python/hailtop/pinned-requirements.txt # -r hail/python/hailtop/requirements.txt bokeh==3.4.3 # via -r hail/python/requirements.txt -boto3==1.42.4 +boto3==1.42.26 # via # -c hail/python/hailtop/pinned-requirements.txt # -r hail/python/hailtop/requirements.txt -botocore==1.42.4 +botocore==1.42.26 # via # -c hail/python/hailtop/pinned-requirements.txt # -r hail/python/hailtop/requirements.txt # boto3 # s3transfer -cachetools==6.2.2 - # via - # -c hail/python/hailtop/pinned-requirements.txt - # google-auth -certifi==2025.11.12 +certifi==2026.1.4 # via # -c hail/python/hailtop/pinned-requirements.txt # msrest @@ -114,7 +110,7 @@ frozenlist==1.8.0 # -r hail/python/hailtop/requirements.txt # aiohttp # aiosignal -google-auth==2.43.0 +google-auth==2.47.0 # via # -c hail/python/hailtop/pinned-requirements.txt # -r hail/python/hailtop/requirements.txt @@ -123,7 +119,7 @@ google-auth-oauthlib==0.8.0 # via # -c hail/python/hailtop/pinned-requirements.txt # -r hail/python/hailtop/requirements.txt -humanize==4.14.0 +humanize==4.15.0 # via # -c hail/python/hailtop/pinned-requirements.txt # -r hail/python/hailtop/requirements.txt @@ -201,7 +197,7 @@ pandas==2.3.3 # bokeh parsimonious==0.11.0 # via -r hail/python/requirements.txt -pillow==12.0.0 +pillow==12.1.0 # via bokeh plotly==5.24.1 # via -r hail/python/requirements.txt @@ -221,7 +217,7 @@ pyasn1-modules==0.4.2 # via # -c hail/python/hailtop/pinned-requirements.txt # google-auth -pycares==4.11.0 +pycares==5.0.1 # via # -c hail/python/hailtop/pinned-requirements.txt # aiodns @@ -306,9 +302,9 @@ tabulate==0.9.0 # -r hail/python/hailtop/requirements.txt tenacity==9.1.2 # via plotly -tornado==6.5.2 +tornado==6.5.4 # via bokeh -typer==0.20.0 +typer==0.21.1 # via # -c hail/python/hailtop/pinned-requirements.txt # -r hail/python/hailtop/requirements.txt @@ -323,9 +319,9 @@ typing-extensions==4.15.0 # janus # multidict # typer -tzdata==2025.2 +tzdata==2025.3 # via pandas -urllib3==2.6.1 +urllib3==2.6.3 # via # -c hail/python/hailtop/pinned-requirements.txt # botocore diff --git a/web_common/pinned-requirements.txt b/web_common/pinned-requirements.txt index bdfa04834cc..21a5bb662da 100644 --- a/web_common/pinned-requirements.txt +++ b/web_common/pinned-requirements.txt @@ -6,7 +6,7 @@ aiohappyeyeballs==2.6.1 # -c web_common/../hail/python/dev/pinned-requirements.txt # -c web_common/../hail/python/pinned-requirements.txt # aiohttp -aiohttp==3.13.2 +aiohttp==3.13.3 # via # -c web_common/../gear/pinned-requirements.txt # -c web_common/../hail/python/dev/pinned-requirements.txt From 3eb08274bb32589ef0e201b19bcd1e433b29dc0c Mon Sep 17 00:00:00 2001 From: Christopher Vittal Date: Thu, 29 Jan 2026 13:43:37 -0500 Subject: [PATCH 16/51] [query] Remove ENumpyBinaryNDArray (#15219) There have been open FIXMEs to remove this EType in favor of custom value readers and writers. This change does that. ## Security Assessment - This change cannot impact the Hail Batch instance as deployed by Broad Institute in GCP --- .../src/is/hail/expr/ir/BlockMatrixIR.scala | 10 +- .../is/hail/expr/ir/BlockMatrixWriter.scala | 10 +- .../src/is/hail/expr/ir/ValueReader.scala | 44 ++++++++- .../src/is/hail/expr/ir/ValueWriter.scala | 29 +++++- .../types/encoded/ENumpyBinaryNDArray.scala | 96 ------------------- 5 files changed, 77 insertions(+), 112 deletions(-) delete mode 100644 hail/hail/src/is/hail/types/encoded/ENumpyBinaryNDArray.scala diff --git a/hail/hail/src/is/hail/expr/ir/BlockMatrixIR.scala b/hail/hail/src/is/hail/expr/ir/BlockMatrixIR.scala index 6f964bdce49..7b58a58b8ea 100644 --- a/hail/hail/src/is/hail/expr/ir/BlockMatrixIR.scala +++ b/hail/hail/src/is/hail/expr/ir/BlockMatrixIR.scala @@ -5,10 +5,10 @@ import is.hail.backend.ExecuteContext import is.hail.expr.Nat import is.hail.expr.ir.defs._ import is.hail.expr.ir.lowering.{BMSContexts, BlockMatrixStage2, LowererUnsupportedOperation} -import is.hail.io.{StreamBufferSpec, TypedCodecSpec} +import is.hail.io.TypedCodecSpec import is.hail.io.fs.FS import is.hail.linalg.{BlockMatrix, BlockMatrixMetadata, MatrixSparsity} -import is.hail.types.encoded.{EBlockMatrixNDArray, EFloat64, ENumpyBinaryNDArray} +import is.hail.types.encoded.{EBlockMatrixNDArray, EFloat64} import is.hail.types.virtual._ import is.hail.utils._ import is.hail.utils.compat.immutable.ArraySeq @@ -216,11 +216,7 @@ case class BlockMatrixBinaryReader(path: String, shape: IndexedSeq[Long], blockS } override def lower(ctx: ExecuteContext, evalCtx: IRBuilder): BlockMatrixStage2 = { - // FIXME numpy should be it's own value reader - val readFromNumpyEType = ENumpyBinaryNDArray(nRows, nCols, true) - val readFromNumpySpec = - TypedCodecSpec(readFromNumpyEType, TNDArray(TFloat64, Nat(2)), new StreamBufferSpec()) - val reader = ETypeValueReader(readFromNumpySpec) + val reader = NumpyBinaryValueReader(nRows, nCols) val nd = evalCtx.memoize(ReadValue(Str(path), reader, TNDArray(TFloat64, nDimsBase = Nat(2)))) val typ = fullType diff --git a/hail/hail/src/is/hail/expr/ir/BlockMatrixWriter.scala b/hail/hail/src/is/hail/expr/ir/BlockMatrixWriter.scala index f44f90b1f7e..4002cc9f4ea 100644 --- a/hail/hail/src/is/hail/expr/ir/BlockMatrixWriter.scala +++ b/hail/hail/src/is/hail/expr/ir/BlockMatrixWriter.scala @@ -6,11 +6,11 @@ import is.hail.backend.ExecuteContext import is.hail.expr.Nat import is.hail.expr.ir.defs.{MetadataWriter, Str, UUID4, WriteMetadata, WriteValue} import is.hail.expr.ir.lowering.{BlockMatrixStage2, LowererUnsupportedOperation} -import is.hail.io.{StreamBufferSpec, TypedCodecSpec} +import is.hail.io.TypedCodecSpec import is.hail.io.fs.FS import is.hail.linalg.{BlockMatrix, BlockMatrixMetadata, MatrixSparsity} import is.hail.types.TypeWithRequiredness -import is.hail.types.encoded.{EBlockMatrixNDArray, ENumpyBinaryNDArray, EType} +import is.hail.types.encoded.{EBlockMatrixNDArray, EType} import is.hail.types.virtual._ import is.hail.utils._ import is.hail.utils.richUtils.RichDenseMatrixDouble @@ -185,11 +185,7 @@ case class BlockMatrixBinaryWriter(path: String) extends BlockMatrixWriter { eltR: TypeWithRequiredness, ): IR = { val nd = s.collectLocal(evalCtx, "block_matrix_binary_writer") - - // FIXME remove numpy encoder - val etype = ENumpyBinaryNDArray(s.typ.nRows, s.typ.nCols, true) - val spec = TypedCodecSpec(etype, TNDArray(s.typ.elementType, Nat(2)), new StreamBufferSpec()) - val writer = ETypeValueWriter(spec) + val writer = NumpyBinaryValueWriter(s.typ.nRows, s.typ.nCols) WriteValue(nd, Str(path), writer) } } diff --git a/hail/hail/src/is/hail/expr/ir/ValueReader.scala b/hail/hail/src/is/hail/expr/ir/ValueReader.scala index 0305502b818..9cf1db6e547 100644 --- a/hail/hail/src/is/hail/expr/ir/ValueReader.scala +++ b/hail/hail/src/is/hail/expr/ir/ValueReader.scala @@ -2,12 +2,15 @@ package is.hail.expr.ir import is.hail.annotations.Region import is.hail.asm4s._ -import is.hail.io.{AbstractTypedCodecSpec, BufferSpec, TypedCodecSpec} +import is.hail.io.{AbstractTypedCodecSpec, BufferSpec, StreamBufferSpec, TypedCodecSpec} import is.hail.types.TypeWithRequiredness import is.hail.types.encoded._ import is.hail.types.physical._ import is.hail.types.physical.stypes.SValue +import is.hail.types.physical.stypes.concrete.SNDArrayPointer +import is.hail.types.physical.stypes.primitives.SFloat64Value import is.hail.types.virtual._ +import is.hail.utils._ import java.io.InputStream @@ -19,6 +22,7 @@ object ValueReader { override val typeHints = ShortTypeHints( List( classOf[ETypeValueReader], + classOf[NumpyBinaryValueReader], classOf[AbstractTypedCodecSpec], classOf[TypedCodecSpec], ), @@ -51,3 +55,41 @@ final case class ETypeValueReader(spec: AbstractTypedCodecSpec) extends ValueRea ret } } + +final case class NumpyBinaryValueReader(nRows: Long, nCols: Long) extends ValueReader { + final private val st = SNDArrayPointer(PCanonicalNDArray(PFloat64(true), 2, false)) + + override def unionRequiredness(requestedType: Type, requiredness: TypeWithRequiredness): Unit = + requiredness.fromPType(st.pType.setRequired(true)) + + override def readValue( + cb: EmitCodeBuilder, + t: Type, + region: Value[Region], + is: Value[InputStream], + ): SValue = { + val pt = st.pType + + val stride0 = const(nCols * pt.elementType.byteSize) + val stride1 = const(pt.elementType.byteSize) + + val in = cb.memoize(new StreamBufferSpec().buildCodeInputBuffer(is)) + val (tFirstElementAddress, tFinisher) = + pt.constructDataFunction(IndexedSeq(nRows, nCols), IndexedSeq(stride0, stride1), cb, region) + val currElementAddress = + cb.newLocal[Long]("eblockmatrix_ndarray_currElementAddress", tFirstElementAddress) + + val i = cb.newLocal[Long]("i") + cb.for_( + cb.assign(i, 0L), + i < nRows * nCols, + cb.assign(i, i + 1L), { + val elem = SFloat64Value(cb.memoize(in.readDouble())) + pt.elementType.storeAtAddress(cb, currElementAddress, region, elem, false) + cb.assign(currElementAddress, currElementAddress + pt.elementType.byteSize) + }, + ) + + tFinisher(cb) + } +} diff --git a/hail/hail/src/is/hail/expr/ir/ValueWriter.scala b/hail/hail/src/is/hail/expr/ir/ValueWriter.scala index 1205006008c..f5fed6728bc 100644 --- a/hail/hail/src/is/hail/expr/ir/ValueWriter.scala +++ b/hail/hail/src/is/hail/expr/ir/ValueWriter.scala @@ -1,11 +1,13 @@ package is.hail.expr.ir import is.hail.asm4s._ -import is.hail.io.{AbstractTypedCodecSpec, BufferSpec, TypedCodecSpec} +import is.hail.io.{AbstractTypedCodecSpec, BufferSpec, StreamBufferSpec, TypedCodecSpec} import is.hail.types.encoded._ import is.hail.types.physical._ import is.hail.types.physical.stypes.SValue +import is.hail.types.physical.stypes.primitives.SFloat64 import is.hail.types.virtual._ +import is.hail.utils._ import java.io.OutputStream @@ -17,6 +19,7 @@ object ValueWriter { override val typeHints = ShortTypeHints( List( classOf[ETypeValueWriter], + classOf[NumpyBinaryValueWriter], classOf[AbstractTypedCodecSpec], classOf[TypedCodecSpec], ), @@ -43,3 +46,27 @@ final case class ETypeValueWriter(spec: AbstractTypedCodecSpec) extends ValueWri cb += ob.invoke[Unit]("flush") } } + +final case class NumpyBinaryValueWriter(nRows: Long, nCols: Long) extends ValueWriter { + override def writeValue(cb: EmitCodeBuilder, value: SValue, os: Value[OutputStream]): Unit = { + val ob = cb.memoize(new StreamBufferSpec().buildCodeOutputBuffer(os)) + val ndarray = value.asNDArray + assert(ndarray.st.elementType == SFloat64) + val i = cb.newLocal[Long]("i") + val j = cb.newLocal[Long]("j") + + cb.for_( + cb.assign(i, 0L), + i < nRows, + cb.assign(i, i + 1L), + cb.for_( + cb.assign(j, 0L), + j < nCols, + cb.assign(j, j + 1L), + cb += ob.writeDouble(ndarray.loadElement(FastSeq(i, j), cb).asFloat64.value), + ), + ) + + cb += ob.invoke[Unit]("flush") + } +} diff --git a/hail/hail/src/is/hail/types/encoded/ENumpyBinaryNDArray.scala b/hail/hail/src/is/hail/types/encoded/ENumpyBinaryNDArray.scala deleted file mode 100644 index e3e5a015e14..00000000000 --- a/hail/hail/src/is/hail/types/encoded/ENumpyBinaryNDArray.scala +++ /dev/null @@ -1,96 +0,0 @@ -package is.hail.types.encoded - -import is.hail.annotations.Region -import is.hail.asm4s._ -import is.hail.expr.ir.EmitCodeBuilder -import is.hail.io.{InputBuffer, OutputBuffer} -import is.hail.types.physical.PCanonicalNDArray -import is.hail.types.physical.stypes.{SType, SValue} -import is.hail.types.physical.stypes.concrete.SNDArrayPointer -import is.hail.types.physical.stypes.interfaces.SNDArrayValue -import is.hail.types.physical.stypes.primitives.SFloat64 -import is.hail.types.virtual.{TNDArray, Type} -import is.hail.utils.FastSeq - -// FIXME numpy format should not be a hail native serialized format, move this to ValueReader/Writer -final case class ENumpyBinaryNDArray(nRows: Long, nCols: Long, required: Boolean) extends EType { - type DecodedPType = PCanonicalNDArray - val elementType = EFloat64(true) - - def setRequired(newRequired: Boolean): ENumpyBinaryNDArray = - ENumpyBinaryNDArray(nRows, nCols, newRequired) - - def _decodedSType(requestedType: Type): SType = { - val elementPType = elementType.decodedPType(requestedType.asInstanceOf[TNDArray].elementType) - SNDArrayPointer(PCanonicalNDArray(elementPType, 2, false)) - } - - override def _buildEncoder(cb: EmitCodeBuilder, v: SValue, out: Value[OutputBuffer]): Unit = { - val ndarray = v.asInstanceOf[SNDArrayValue] - assert(ndarray.st.elementType == SFloat64) - val i = cb.newLocal[Long]("i") - val j = cb.newLocal[Long]("j") - val writeElemF = elementType.buildEncoder(ndarray.st.elementType, cb.emb.ecb) - - cb.for_( - cb.assign(i, 0L), - i < nRows, - cb.assign(i, i + 1L), - cb.for_( - cb.assign(j, 0L), - j < nCols, - cb.assign(j, j + 1L), - writeElemF(cb, ndarray.loadElement(FastSeq(i, j), cb), out), - ), - ) - - } - - override def _buildDecoder( - cb: EmitCodeBuilder, - t: Type, - region: Value[Region], - in: Value[InputBuffer], - ): SValue = { - val st = decodedSType(t).asInstanceOf[SNDArrayPointer] - val pt = st.pType - val readElemF = elementType.buildInplaceDecoder(pt.elementType, cb.emb.ecb) - - val stride0 = cb.newLocal[Long]("stride0", nCols * pt.elementType.byteSize) - val stride1 = cb.newLocal[Long]("stride1", pt.elementType.byteSize) - - val n = cb.newLocal[Long]("length", nRows * nCols) - - val (tFirstElementAddress, tFinisher) = - pt.constructDataFunction(IndexedSeq(nRows, nCols), IndexedSeq(stride0, stride1), cb, region) - val currElementAddress = - cb.newLocal[Long]("eblockmatrix_ndarray_currElementAddress", tFirstElementAddress) - - val i = cb.newLocal[Long]("i") - cb.for_( - cb.assign(i, 0L), - i < n, - cb.assign(i, i + 1L), { - readElemF(cb, region, currElementAddress, in) - cb.assign(currElementAddress, currElementAddress + pt.elementType.byteSize) - }, - ) - - tFinisher(cb) - } - - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = - ??? - - override def _asIdent: String = - s"${nRows}by${nCols}_numpy_array_of_${elementType.asIdent}" - - def _toPretty = s"ENDArray[$elementType]" - - override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false): Unit = { - sb ++= "ENDArray[" - elementType.pretty(sb, indent, compact) - sb += ']' - } - -} From 0a67c1d6324d8e6b4f2b569aa4cc2db5101e21b0 Mon Sep 17 00:00:00 2001 From: Christopher Vittal Date: Thu, 29 Jan 2026 14:33:42 -0500 Subject: [PATCH 17/51] [query] Fix BlockMatrix persist (#15233) Use the regular backend `persist`/`unpersist` for BlockMatrix persist rather than a call to the nonexistent `persist_blockmatrix` that was added in #12864. Fixes #15229 ## Security Assessment - This change cannot impact the Hail Batch instance as deployed by Broad Institute in GCP --- hail/python/hail/backend/backend.py | 4 ++-- hail/python/hail/linalg/blockmatrix.py | 15 ++++++--------- hail/python/test/hail/linalg/test_linalg.py | 10 ++++++++++ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/hail/python/hail/backend/backend.py b/hail/python/hail/backend/backend.py index 7f63f268ac6..868266c8eed 100644 --- a/hail/python/hail/backend/backend.py +++ b/hail/python/hail/backend/backend.py @@ -339,9 +339,9 @@ def import_fam(self, path: str, quant_pheno: bool, delimiter: str, missing: str) return orjson.loads(fam_json_bytes) def persist(self, dataset: Dataset) -> Dataset: - from hail.context import TemporaryFilename + from hail.context import TemporaryDirectory - tempfile = TemporaryFilename(prefix=f'persist_{type(dataset).__name__}') + tempfile = TemporaryDirectory(prefix=f'persist_{type(dataset).__name__}', ensure_exists=False) persisted = dataset.checkpoint(tempfile.__enter__()) self._persisted_locations[persisted] = (tempfile, dataset) return persisted diff --git a/hail/python/hail/linalg/blockmatrix.py b/hail/python/hail/linalg/blockmatrix.py index 175e3dadff5..b6c0693e6bd 100644 --- a/hail/python/hail/linalg/blockmatrix.py +++ b/hail/python/hail/linalg/blockmatrix.py @@ -66,7 +66,7 @@ typecheck, typecheck_method, ) -from hail.utils import local_path_uri, new_local_temp_file, new_temp_file, storage_level, with_local_temp_file +from hail.utils import local_path_uri, new_local_temp_file, new_temp_file, with_local_temp_file from hail.utils.java import Env block_matrix_type = lazy() @@ -1328,7 +1328,6 @@ def cache(self): """ return self.persist('MEMORY_ONLY') - @typecheck_method(storage_level=storage_level) def persist(self, storage_level='MEMORY_AND_DISK'): """Persists this block matrix in memory or on disk. @@ -1340,11 +1339,6 @@ def persist(self, storage_level='MEMORY_AND_DISK'): pipelines. This method is not a substitution for :meth:`.BlockMatrix.write`, which stores a permanent file. - Most users should use the "MEMORY_AND_DISK" storage level. See the `Spark - documentation - `__ - for a more in-depth discussion of persisting data. - Parameters ---------- storage_level : str @@ -1353,12 +1347,15 @@ def persist(self, storage_level='MEMORY_AND_DISK'): MEMORY_ONLY_SER_2, MEMORY_AND_DISK, MEMORY_AND_DISK_2, MEMORY_AND_DISK_SER, MEMORY_AND_DISK_SER_2, OFF_HEAP + This parameter no longer does anything and has been retained for + compatibility. + Returns ------- :class:`.BlockMatrix` Persisted block matrix. """ - return Env.backend().persist_blockmatrix(self) + return Env.backend().persist(self) def unpersist(self): """Unpersists this block matrix from memory/disk. @@ -1373,7 +1370,7 @@ def unpersist(self): :class:`.BlockMatrix` Unpersisted block matrix. """ - return Env.backend().unpersist_blockmatrix(self) + return Env.backend().unpersist(self) def __pos__(self): return self diff --git a/hail/python/test/hail/linalg/test_linalg.py b/hail/python/test/hail/linalg/test_linalg.py index df7fbcb787e..655238152bf 100644 --- a/hail/python/test/hail/linalg/test_linalg.py +++ b/hail/python/test/hail/linalg/test_linalg.py @@ -254,6 +254,16 @@ def test_block_matrix_from_file(): _assert_eq(array, data) +def test_block_matrix_persist_unpersist(): + data = np.random.rand(10, 11) + bm = BlockMatrix.from_numpy(data, block_size=5) + bm_persist = bm.persist() + _assert_eq(data, bm_persist.to_numpy()) + + bm_unpersist = bm_persist.unpersist() + _assert_eq(data, bm_unpersist.to_numpy()) + + @fails_service_backend() @fails_local_backend() def test_numpy_round_trip_force_blocking(): From 3fae026ec43b5ce9b535f05907a8075f27a66be2 Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Thu, 29 Jan 2026 15:14:25 -0500 Subject: [PATCH 18/51] [query] remove `RichRegex` (#15236) While this was convenient for 2.12 (build-in method for 2.13), it was triggering a pattern compilation for every fileListEntry. Lifting the pattern out removes the need for this utility. This change does not impact the blah blah blah --- hail/hail/src/is/hail/io/bgen/LoadBgen.scala | 10 ++++------ hail/hail/src/is/hail/utils/richUtils/Implicits.scala | 3 --- hail/hail/src/is/hail/utils/richUtils/RichRegex.scala | 7 ------- 3 files changed, 4 insertions(+), 16 deletions(-) delete mode 100644 hail/hail/src/is/hail/utils/richUtils/RichRegex.scala diff --git a/hail/hail/src/is/hail/io/bgen/LoadBgen.scala b/hail/hail/src/is/hail/io/bgen/LoadBgen.scala index 90fda6bff12..47cc3c32965 100644 --- a/hail/hail/src/is/hail/io/bgen/LoadBgen.scala +++ b/hail/hail/src/is/hail/io/bgen/LoadBgen.scala @@ -156,6 +156,9 @@ object LoadBgen extends Logging { ) } + private[this] lazy val EntryPattern = + ".*part-[0-9]+(-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})?".r.pattern + def getAllFileListEntries(fs: FS, files: Array[String]): Array[FileListEntry] = { val badFiles = ArraySeq.newBuilder[String] @@ -172,9 +175,7 @@ object LoadBgen extends Logging { if (fileListEntry.isDirectory) fs.listDirectory(file) .filter(fileListEntry => - ".*part-[0-9]+(-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})?".r.matches( - fileListEntry.getPath - ) + EntryPattern.matcher(fileListEntry.getPath).matches() ) else Array(fileListEntry) @@ -192,9 +193,6 @@ object LoadBgen extends Logging { fileListEntries } - def getAllFilePaths(fs: FS, files: Array[String]): Array[String] = - getAllFileListEntries(fs, files).map(_.getPath.toString) - def getBgenFileMetadata( ctx: ExecuteContext, files: Array[FileListEntry], diff --git a/hail/hail/src/is/hail/utils/richUtils/Implicits.scala b/hail/hail/src/is/hail/utils/richUtils/Implicits.scala index d38d74d5638..d7c8b5ddf8b 100644 --- a/hail/hail/src/is/hail/utils/richUtils/Implicits.scala +++ b/hail/hail/src/is/hail/utils/richUtils/Implicits.scala @@ -10,7 +10,6 @@ import is.hail.utils.compat.immutable.ArraySeq import scala.collection.compat._ import scala.collection.mutable import scala.reflect.ClassTag -import scala.util.matching.Regex import java.io.InputStream @@ -92,8 +91,6 @@ trait Implicits { implicit def toRichContextRDDLong(r: ContextRDD[Long]): RichContextRDDLong = new RichContextRDDLong(r) - implicit def toRichRegex(r: Regex): RichRegex = new RichRegex(r) - implicit def toRichRow(r: Row): RichRow = new RichRow(r) implicit def toRichSC(sc: SparkContext): RichSparkContext = new RichSparkContext(sc) diff --git a/hail/hail/src/is/hail/utils/richUtils/RichRegex.scala b/hail/hail/src/is/hail/utils/richUtils/RichRegex.scala deleted file mode 100644 index 1a2f0b2f006..00000000000 --- a/hail/hail/src/is/hail/utils/richUtils/RichRegex.scala +++ /dev/null @@ -1,7 +0,0 @@ -package is.hail.utils.richUtils - -import scala.util.matching.Regex - -class RichRegex(r: Regex) { - def matches(s: String): Boolean = r.pattern.matcher(s).matches() -} From 9e1e01d8bf1a435d37f6afd833a8fa4a441680f7 Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Thu, 29 Jan 2026 21:53:46 -0500 Subject: [PATCH 19/51] [query] fix LiftMeOut blockargs (#15245) LiftMeOut takes a non-strict argument. This change does not affect the broad-managed batch service in gcp. --- hail/hail/src/is/hail/expr/ir/Pretty.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hail/hail/src/is/hail/expr/ir/Pretty.scala b/hail/hail/src/is/hail/expr/ir/Pretty.scala index f76f4041862..4dee49c50fe 100644 --- a/hail/hail/src/is/hail/expr/ir/Pretty.scala +++ b/hail/hail/src/is/hail/expr/ir/Pretty.scala @@ -685,6 +685,8 @@ class Pretty( def blockArgs(ir: BaseIR, i: Int): Option[IndexedSeq[(Name, String)]] = ir match { case If(_, _, _) => if (i > 0) Some(FastSeq()) else None + case _: LiftMeOut => + Some(ArraySeq.empty) case _: Switch => if (i > 0) Some(FastSeq()) else None case TailLoop(name, args, _, _) => if (i == args.length) From ca9193db57ef0c42370c6ca21f3856ed3545d57f Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Fri, 30 Jan 2026 11:45:13 -0500 Subject: [PATCH 20/51] [query] sort block args matching (#15247) Non-functional change only. Quality of life change for me while I scan through IR generation and try to determine which child IRs are strict or non-strict and update their corresponding block-args in the pretty printer. This change does not affect the broad-managed batch service in gcp. --- hail/hail/src/is/hail/expr/ir/Pretty.scala | 338 +++++++++++---------- 1 file changed, 175 insertions(+), 163 deletions(-) diff --git a/hail/hail/src/is/hail/expr/ir/Pretty.scala b/hail/hail/src/is/hail/expr/ir/Pretty.scala index 4dee49c50fe..48937793130 100644 --- a/hail/hail/src/is/hail/expr/ir/Pretty.scala +++ b/hail/hail/src/is/hail/expr/ir/Pretty.scala @@ -681,171 +681,183 @@ class Pretty( def ssaStyle(ir: BaseIR): String = { def childIsStrict(ir: BaseIR, i: Int): Boolean = blockArgs(ir, i).isEmpty + def some(as: (Name, String)*) = Some(as.toFastSeq) - def blockArgs(ir: BaseIR, i: Int): Option[IndexedSeq[(Name, String)]] = ir match { - case If(_, _, _) => - if (i > 0) Some(FastSeq()) else None - case _: LiftMeOut => - Some(ArraySeq.empty) - case _: Switch => - if (i > 0) Some(FastSeq()) else None - case TailLoop(name, args, _, _) => if (i == args.length) - Some(args.map { case (name, _) => name -> "loopvar" } :+ - name -> "loop") - else None - case StreamMap(_, name, _) => - if (i == 1) Some(Array(name -> "elt")) else None - case StreamZip(as, names, _, _, _) => - if (i == as.length) Some(names.map(_ -> "elt")) else None - case StreamZipJoin(as, _, curKey, curVals, _) => - if (i == as.length) - Some(Array(curKey -> "key", curVals -> "elts")) - else - None - case StreamFor(_, name, _) => - if (i == 1) Some(Array(name -> "elt")) else None - case StreamFlatMap(_, name, _) => - if (i == 1) Some(Array(name -> "elt")) else None - case StreamFilter(_, name, _) => - if (i == 1) Some(Array(name -> "elt")) else None - case StreamTakeWhile(_, name, _) => - if (i == 1) Some(Array(name -> "elt")) else None - case StreamDropWhile(_, name, _) => - if (i == 1) Some(Array(name -> "elt")) else None - case StreamFold(_, _, accumName, valueName, _) => - if (i == 2) Some(Array(accumName -> "accum", valueName -> "elt")) else None - case StreamFold2(_, accum, valueName, _, _) => - if (i <= accum.length) - None - else if (i < 2 * accum.length + 1) - Some(Array(valueName -> "elt") ++ accum.map { case (name, _) => name -> "accum" }) - else - Some(accum.map { case (name, _) => name -> "accum" }) - case RunAggScan(_, name, _, _, _, _) => - if (i == 2 || i == 3) Some(Array(name -> "elt")) else None - case StreamScan(_, _, accumName, valueName, _) => - if (i == 2) Some(Array(accumName -> "accum", valueName -> "elt")) else None - case StreamAggScan(_, name, _) => - if (i == 1) Some(FastSeq(name -> "elt")) else None - case StreamJoinRightDistinct(_, _, _, _, l, r, _, _) => - if (i == 2) Some(Array(l -> "l_elt", r -> "r_elt")) else None - case StreamLeftIntervalJoin(_, _, _, _, l, r, _) => - if (i == 2) Some(Array(l -> "l_elt", r -> "r_elts")) else None - case ArraySort(_, left, right, _) => - if (i == 1) Some(Array(left -> "l", right -> "r")) else None - case AggArrayPerElement(_, elementName, indexName, _, _, _) => - if (i == 1) Some(Array(elementName -> "elt", indexName -> "idx")) else None - case AggFold(_, _, _, accumName, otherAccumName, _) => - if (i == 1) Some(Array(accumName -> "accum")) - else if (i == 2) Some(Array(accumName -> "l", otherAccumName -> "r")) - else None - case NDArrayMap(_, name, _) => - if (i == 1) Some(Array(name -> "elt")) else None - case NDArrayMap2(_, _, lName, rName, _, _) => if (i == 2) - Some(Array(lName -> "l_elt", rName -> "r_elt")) - else - None - case CollectDistributedArray(_, _, cname, gname, _, _, _, _) => - if (i == 2) Some(Array(cname -> "ctx", gname -> "g")) else None - case TableAggregate(_, _) => - if (i == 1) Some(Array(TableIR.globalName -> "g", TableIR.rowName -> "row")) else None - case MatrixAggregate(_, _) => - if (i == 1) Some(Array( - MatrixIR.globalName -> "g", - MatrixIR.colName -> "col", - MatrixIR.rowName -> "row", - MatrixIR.entryName -> "entry", - )) - else None - case TableFilter(_, _) => - if (i == 1) Some(Array(TableIR.globalName -> "g", TableIR.rowName -> "row")) else None - case TableMapGlobals(_, _) => - if (i == 1) Some(Array(TableIR.globalName -> "g")) else None - case TableMapRows(_, _) => - if (i == 1) Some(Array(TableIR.globalName -> "g", TableIR.rowName -> "row")) else None - case TableAggregateByKey(_, _) => - if (i == 1) Some(Array(TableIR.globalName -> "g", TableIR.rowName -> "row")) else None - case TableKeyByAndAggregate(_, _, _, _, _) => - if (i == 1 || i == 2) - Some(Array(TableIR.globalName -> "g", TableIR.rowName -> "row")) - else None - case TableMapPartitions(_, g, p, _, _, _) => - if (i == 1) Some(Array(g -> "g", p -> "part")) else None - case MatrixMapRows(_, _) => - if (i == 1) Some(Array( - MatrixIR.globalName -> "g", - MatrixIR.rowName -> "row", - MatrixIR.colName -> "col", - MatrixIR.entryName -> "entry", - Name("n_cols") -> "n_cols", - )) - else None - case MatrixFilterRows(_, _) => - if (i == 1) Some(Array(MatrixIR.globalName -> "g", MatrixIR.rowName -> "row")) else None - case MatrixMapCols(_, _, _) => - if (i == 1) Some(Array( - MatrixIR.globalName -> "g", - MatrixIR.rowName -> "row", - MatrixIR.colName -> "col", - MatrixIR.entryName -> "entry", - Name("n_rows") -> "n_rows", - )) - else None - case MatrixFilterCols(_, _) => - if (i == 1) Some(Array(MatrixIR.globalName -> "g", MatrixIR.colName -> "col")) else None - case MatrixMapEntries(_, _) => - if (i == 1) Some(Array( - MatrixIR.globalName -> "g", - MatrixIR.colName -> "col", - MatrixIR.rowName -> "row", - MatrixIR.entryName -> "entry", - )) - else None - case MatrixFilterEntries(_, _) => - if (i == 1) Some(Array( - MatrixIR.globalName -> "g", - MatrixIR.colName -> "col", - MatrixIR.rowName -> "row", - MatrixIR.entryName -> "entry", - )) - else None - case MatrixMapGlobals(_, _) => - if (i == 1) Some(Array(MatrixIR.globalName -> "g")) else None - case MatrixAggregateColsByKey(_, _, _) => - if (i == 1) - Some(Array( - MatrixIR.globalName -> "g", - MatrixIR.rowName -> "row", - MatrixIR.colName -> "col", - MatrixIR.entryName -> "entry", - )) - else if (i == 2) - Some(Array(MatrixIR.globalName -> "g", MatrixIR.colName -> "col")) - else - None - case MatrixAggregateRowsByKey(_, _, _) => - if (i == 1) - Some(Array( - MatrixIR.globalName -> "g", - MatrixIR.rowName -> "row", - MatrixIR.colName -> "col", - MatrixIR.entryName -> "entry", - )) - else if (i == 2) - Some(Array(MatrixIR.globalName -> "g", MatrixIR.rowName -> "row")) - else + lazy val matrixBlockArgs = + some( + MatrixIR.globalName -> "g", + MatrixIR.colName -> "col", + MatrixIR.rowName -> "row", + MatrixIR.entryName -> "entry", + ) + + lazy val tableBlockArgs = + some( + TableIR.globalName -> "g", + TableIR.rowName -> "row", + ) + + def blockArgs(ir: BaseIR, i: Int): Option[IndexedSeq[(Name, String)]] = + ir match { + case AggArrayPerElement(_, elementName, indexName, _, _, _) => + if (i == 1) some(elementName -> "elt", indexName -> "idx") + else None + case AggExplode(_, name, _, _) => + if (i == 1) some(name -> "elt") + else None + case AggFold(_, _, _, accumName, otherAccumName, _) => + if (i == 1) some(accumName -> "accum") + else if (i == 2) some(accumName -> "l", otherAccumName -> "r") + else None + case ArraySort(_, left, right, _) => + if (i == 1) some(left -> "l", right -> "r") + else None + case _: Coalesce => + some() + case BlockMatrixMap(_, eltName, _, _) => + if (i == 1) some(eltName -> "elt") + else None + case BlockMatrixMap2(_, _, lName, rName, _, _) => + if (i == 2) some(lName -> "l", rName -> "r") + else None + case CollectDistributedArray(_, _, cname, gname, _, _, _, _) => + if (i == 2) some(cname -> "ctx", gname -> "g") + else None + case _: ConsoleLog => + if (i == 1) some() + else None + case _: If => + if (i > 0) some() + else None + case _: LiftMeOut => + some() + case _: MatrixAggregate => + if (i == 1) matrixBlockArgs + else None + case _: MatrixAggregateColsByKey => + if (i == 1) matrixBlockArgs + else if (i == 2) some(MatrixIR.globalName -> "g", MatrixIR.colName -> "col") + else None + case _: MatrixAggregateRowsByKey => + if (i == 1) matrixBlockArgs + else if (i == 2) some(MatrixIR.globalName -> "g", MatrixIR.rowName -> "row") + else None + case _: MatrixFilterCols => + if (i == 1) some(MatrixIR.globalName -> "g", MatrixIR.colName -> "col") + else None + case _: MatrixFilterEntries => + if (i == 1) matrixBlockArgs + else None + case _: MatrixFilterRows => + if (i == 1) some(MatrixIR.globalName -> "g", MatrixIR.rowName -> "row") + else None + case _: MatrixMapCols => + if (i == 1) matrixBlockArgs map { _ :+ (Name("n_rows") -> "n_rows") } + else None + case _: MatrixMapEntries => + if (i == 1) matrixBlockArgs + else None + case _: MatrixMapGlobals => + if (i == 1) some(MatrixIR.globalName -> "g") + else None + case _: MatrixMapRows => + if (i == 1) matrixBlockArgs map { _ :+ (Name("n_rows") -> "n_rows") } + else None + case NDArrayMap(_, name, _) => + if (i == 1) some(name -> "elt") + else None + case NDArrayMap2(_, _, lName, rName, _, _) => + if (i == 2) some(lName -> "l_elt", rName -> "r_elt") + else None + case RunAggScan(_, name, _, _, _, _) => + if (i == 2 || i == 3) some(name -> "elt") + else None + case StreamAgg(_, name, _) => + if (i == 1) some(name -> "elt") + else None + case StreamAggScan(_, name, _) => + if (i == 1) some(name -> "elt") + else None + case StreamDropWhile(_, name, _) => + if (i == 1) some(name -> "elt") + else None + case StreamFlatMap(_, name, _) => + if (i == 1) some(name -> "elt") + else None + case StreamFilter(_, name, _) => + if (i == 1) some(name -> "elt") + else None + case StreamFold(_, _, accumName, valueName, _) => + if (i == 2) some(accumName -> "accum", valueName -> "elt") + else None + case StreamFold2(_, accum, valueName, _, _) => + if (i <= accum.length) None + else if (i < 2 * accum.length + 1) Some((valueName -> "elt") +: accum.map { + case (name, _) => name -> "accum" + }) + else Some(accum.map { case (name, _) => name -> "accum" }) + case StreamFor(_, name, _) => + if (i == 1) some(name -> "elt") + else None + case StreamJoinRightDistinct(_, _, _, _, l, r, _, _) => + if (i == 2) some(l -> "l_elt", r -> "r_elt") + else None + case StreamLeftIntervalJoin(_, _, _, _, l, r, _) => + if (i == 2) some(l -> "l_elt", r -> "r_elts") + else None + case StreamMap(_, name, _) => + if (i == 1) some(name -> "elt") + else None + case StreamScan(_, _, accumName, valueName, _) => + if (i == 2) some(accumName -> "accum", valueName -> "elt") + else None + case StreamTakeWhile(_, name, _) => + if (i == 1) some(name -> "elt") + else None + case StreamZip(as, names, _, _, _) => + if (i == as.length) Some(names.map(_ -> "elt")) + else None + case StreamZipJoin(as, _, curKey, curVals, _) => + if (i == as.length) some(curKey -> "key", curVals -> "elts") + else None + case StreamZipJoinProducers(_, cname, _, _, k, v, _) => + if (i == 1) some(cname -> "elt") + else if (i == 2) some(k -> "key", v -> "val") + else None + case _: Switch => + if (i > 0) some() + else None + case _: TableAggregate => + if (i == 1) tableBlockArgs + else None + case _: TableAggregateByKey => + if (i == 1) tableBlockArgs + else None + case _: TableFilter => + if (i == 1) tableBlockArgs + else None + case _: TableKeyByAndAggregate => + if (i == 1 || i == 2) tableBlockArgs + else None + case _: TableMapGlobals => + if (i == 1) some(TableIR.globalName -> "g") + else None + case TableMapPartitions(_, g, p, _, _, _) => + if (i == 1) some(g -> "g", p -> "part") + else None + case _: TableMapRows => + if (i == 1) tableBlockArgs + else None + case TailLoop(name, args, _, _) => + if (i == args.length) Some(args.map { case (name, _) => + name -> "loopvar" + } :+ name -> "loop") + else None + case _: Trap => + some() + case _ => None - case BlockMatrixMap(_, eltName, _, _) => - if (i == 1) Some(Array(eltName -> "elt")) else None - case BlockMatrixMap2(_, _, lName, rName, _, _) => - if (i == 2) Some(Array(lName -> "l", rName -> "r")) else None - case AggExplode(_, name, _, _) => - if (i == 1) Some(Array(name -> "elt")) else None - case StreamAgg(_, name, _) => - if (i == 1) Some(Array(name -> "elt")) else None - case _ => - None - } + } var identCounter: Int = 0 val idents = collection.mutable.Map.empty[String, Int] From 3ed78fd245c1a573d4fda7142636bcd52e282fdf Mon Sep 17 00:00:00 2001 From: John Marshall Date: Sat, 31 Jan 2026 05:50:17 +1300 Subject: [PATCH 21/51] Match error message regardless of single/double quote marks (#15237) This message can contain '...' or "..." in different locales. ## Security Assessment [Edited by Chris L: - This change might impact the Hail Batch instance as deployed by Broad Institute in GCP This change prevents an accidental DOS on our worker VMs whereby the docker pull mechanism gets stuck in an infinite loop of retrying futile requests. --- batch/batch/worker/worker.py | 5 ++++- hail/python/hailtop/utils/utils.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/batch/batch/worker/worker.py b/batch/batch/worker/worker.py index 525bf20786b..9bfeeff57b4 100644 --- a/batch/batch/worker/worker.py +++ b/batch/batch/worker/worker.py @@ -531,7 +531,10 @@ async def pull(): if e.status == 404 and 'pull access denied' in e.message: raise ImageCannotBePulled from e if e.status == 500 and ( - 'Permission "artifactregistry.repositories.downloadArtifacts" denied on resource' in e.message + ( + 'artifactregistry.repositories.downloadArtifacts' in e.message + and 'denied on resource' in e.message + ) or 'Caller does not have permission' in e.message or 'unauthorized' in e.message ): diff --git a/hail/python/hailtop/utils/utils.py b/hail/python/hailtop/utils/utils.py index 88d0af92706..6b3c4b67bd4 100644 --- a/hail/python/hailtop/utils/utils.py +++ b/hail/python/hailtop/utils/utils.py @@ -682,7 +682,8 @@ def is_transient_error(e: BaseException) -> bool: return False if ( e.status == 500 - and 'Permission "artifactregistry.repositories.downloadArtifacts" denied on resource' in e.message + and 'artifactregistry.repositories.downloadArtifacts' in e.message + and 'denied on resource' in e.message ): return False if e.status == 500 and 'denied: retrieving permissions failed' in e.message: From b6c56435f257768fd00005403b7e1dc1a64026a1 Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Fri, 30 Jan 2026 13:18:56 -0500 Subject: [PATCH 22/51] [query] printer/parser rules for StreamBufferedAggregate (#15241) Adding for completeness. This operation is used internally by the compiler and isn't exposed to python. We were never getting an assertion from the prettyprinter as this operation is generated and immediately executed in LowerAndExecuteShuffles. No affect batch. --- hail/hail/src/is/hail/expr/ir/Parser.scala | 10 ++++++++++ hail/hail/src/is/hail/expr/ir/Pretty.scala | 10 +++++++++- hail/hail/test/src/is/hail/expr/ir/IRSuite.scala | 10 ++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/hail/hail/src/is/hail/expr/ir/Parser.scala b/hail/hail/src/is/hail/expr/ir/Parser.scala index 279b4490e07..3f75d42677a 100644 --- a/hail/hail/src/is/hail/expr/ir/Parser.scala +++ b/hail/hail/src/is/hail/expr/ir/Parser.scala @@ -1069,6 +1069,16 @@ object IRParser { elem <- ir_value_expr(ctx)(it) } yield LowerBoundOnOrderedCollection(col, elem, onKey) case "GroupByKey" => ir_value_expr(ctx)(it).map(GroupByKey) + case "StreamBufferedAggregate" => + val n = name(it) + val aggSigs = p_agg_sigs(ctx)(it) + val size = int32_literal(it) + for { + stream <- ir_value_expr(ctx)(it) + init <- ir_value_expr(ctx)(it) + key <- ir_value_expr(ctx)(it) + seq <- ir_value_expr(ctx)(it) + } yield StreamBufferedAggregate(stream, init, key, seq, n, aggSigs, size) case "StreamMap" => val n = name(it) for { diff --git a/hail/hail/src/is/hail/expr/ir/Pretty.scala b/hail/hail/src/is/hail/expr/ir/Pretty.scala index 48937793130..5679ca4edea 100644 --- a/hail/hail/src/is/hail/expr/ir/Pretty.scala +++ b/hail/hail/src/is/hail/expr/ir/Pretty.scala @@ -135,10 +135,13 @@ class Pretty( )) } + def prettyInt(x: Int): Doc = + text(x.toString) + def prettyInts(x: IndexedSeq[Int], elideLiterals: Boolean): Doc = { val truncate = elideLiterals && x.length > MAX_VALUES_TO_LOG val view = if (truncate) x.view else x.view.slice(0, MAX_VALUES_TO_LOG) - val docs = view.map(i => text(i.toString)) + val docs = view.map(prettyInt) concat(docs.intersperse[Doc]( "(", softline, @@ -277,6 +280,8 @@ class Pretty( case MakeArray(_, typ) => single(typ.parsableString()) case MakeStream(_, typ, requiresMemoryManagementPerElement) => FastSeq(typ.parsableString(), Pretty.prettyBooleanLiteral(requiresMemoryManagementPerElement)) + case StreamBufferedAggregate(_, _, _, _, name, sigs, size) => + ArraySeq(prettyName(name), prettyPhysicalAggSigs(sigs), prettyInt(size)) case StreamIota(_, _, requiresMemoryManagementPerElement) => FastSeq(Pretty.prettyBooleanLiteral(requiresMemoryManagementPerElement)) case StreamRange(_, _, _, requiresMemoryManagementPerElement, errorID) => @@ -778,6 +783,9 @@ class Pretty( case StreamAggScan(_, name, _) => if (i == 1) some(name -> "elt") else None + case s: StreamBufferedAggregate => + if (i >= 2) some(s.name -> "elt") + else None case StreamDropWhile(_, name, _) => if (i == 1) some(name -> "elt") else None diff --git a/hail/hail/test/src/is/hail/expr/ir/IRSuite.scala b/hail/hail/test/src/is/hail/expr/ir/IRSuite.scala index 73a1fece5be..8e7a464a7c7 100644 --- a/hail/hail/test/src/is/hail/expr/ir/IRSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/IRSuite.scala @@ -22,6 +22,7 @@ import is.hail.types.physical.stypes.primitives.SInt32 import is.hail.types.virtual._ import is.hail.types.virtual.TIterable.elementType import is.hail.utils.{FastSeq, _} +import is.hail.utils.compat.immutable.ArraySeq import is.hail.variant.{Call2, Locus} import scala.collection.compat._ @@ -3429,6 +3430,15 @@ class IRSuite extends HailSuite { }, forIR(st)(_ => Void()) -> Array(st), streamAggIR(st)(x => ApplyAggOp(Sum())(Cast(x, TInt64))) -> Array(st), + StreamBufferedAggregate( + st, + Void(), + MakeStruct(ArraySeq("l" -> l)), + Void(), + l.name, + ArraySeq(pCollectSig), + 27, + ) -> Array(st), streamAggScanIR(st)(x => ApplyScanOp(Sum())(Cast(x, TInt64))) -> Array(st), RunAgg( Begin(FastSeq( From 2cbc8c20b6565e886cdad9ae51eb52ccffb5ea33 Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Fri, 30 Jan 2026 14:54:14 -0500 Subject: [PATCH 23/51] [query] AggFilter blockargs (#15250) Adds blockargs for AggFilter This change does not affect the broad-managed batch service in GCP. --- hail/hail/src/is/hail/expr/ir/Pretty.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hail/hail/src/is/hail/expr/ir/Pretty.scala b/hail/hail/src/is/hail/expr/ir/Pretty.scala index 5679ca4edea..0d8969d1aa0 100644 --- a/hail/hail/src/is/hail/expr/ir/Pretty.scala +++ b/hail/hail/src/is/hail/expr/ir/Pretty.scala @@ -710,6 +710,9 @@ class Pretty( case AggExplode(_, name, _, _) => if (i == 1) some(name -> "elt") else None + case _: AggFilter => + if (i == 1) some() + else None case AggFold(_, _, _, accumName, otherAccumName, _) => if (i == 1) some(accumName -> "accum") else if (i == 2) some(accumName -> "l", otherAccumName -> "r") From 3c34d5ce5308e617753177f4f05410361dd49a47 Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Fri, 30 Jan 2026 17:23:18 -0500 Subject: [PATCH 24/51] [query] AggGroupBy blockargs (#15249) Missing AggGroupBy blockargs. This change cannot affect the broad-managed batch service in gcp. --- hail/hail/src/is/hail/expr/ir/Pretty.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hail/hail/src/is/hail/expr/ir/Pretty.scala b/hail/hail/src/is/hail/expr/ir/Pretty.scala index 0d8969d1aa0..035a8804fe9 100644 --- a/hail/hail/src/is/hail/expr/ir/Pretty.scala +++ b/hail/hail/src/is/hail/expr/ir/Pretty.scala @@ -717,17 +717,20 @@ class Pretty( if (i == 1) some(accumName -> "accum") else if (i == 2) some(accumName -> "l", otherAccumName -> "r") else None + case _: AggGroupBy => + if (i == 1) some() + else None case ArraySort(_, left, right, _) => if (i == 1) some(left -> "l", right -> "r") else None - case _: Coalesce => - some() case BlockMatrixMap(_, eltName, _, _) => if (i == 1) some(eltName -> "elt") else None case BlockMatrixMap2(_, _, lName, rName, _, _) => if (i == 2) some(lName -> "l", rName -> "r") else None + case _: Coalesce => + some() case CollectDistributedArray(_, _, cname, gname, _, _, _, _) => if (i == 2) some(cname -> "ctx", gname -> "g") else None From 4f87ee5aa5ca5c0591a716624415c27de39eb495 Mon Sep 17 00:00:00 2001 From: Patrick Schultz Date: Tue, 3 Feb 2026 11:12:21 -0500 Subject: [PATCH 25/51] [compiler] enforce use of `override` keyword using wartremover (#15238) ## Change Description Use the wartremover scala compiler plugin to enforce using the `override` keyword when implementing/overriding a method from a parent class. This is generally considered a best practice, and e.g. helps with refactoring when deleting a method or changing its signature, as any implementing/overriding methods in child classes will now produce compiler errors. ## Security Assessment - This change cannot impact the Hail Batch instance as deployed by Broad Institute in GCP --- hail/build.mill | 8 +- hail/hail/ir-gen/src/Main.scala | 8 +- hail/hail/package.mill | 4 +- .../is/hail/annotations/BroadcastValue.scala | 8 +- .../src/is/hail/annotations/ChunkCache.scala | 2 +- .../hail/annotations/ExtendedOrdering.scala | 15 +- .../hail/annotations/OrderedRVIterator.scala | 6 +- .../hail/src/is/hail/annotations/Region.scala | 2 +- .../is/hail/annotations/RegionMemory.scala | 2 +- .../src/is/hail/annotations/RegionPool.scala | 2 +- .../is/hail/annotations/UnsafeOrdering.scala | 2 - .../src/is/hail/annotations/UnsafeRow.scala | 16 +- .../is/hail/annotations/ValueVisitor.scala | 36 ++-- .../annotations/WritableRegionValue.scala | 12 +- .../hail/src/is/hail/asm4s/ClassBuilder.scala | 11 +- hail/hail/src/is/hail/asm4s/Code.scala | 60 +++--- hail/hail/src/is/hail/asm4s/CodeBuilder.scala | 4 +- .../src/is/hail/asm4s/GenericTypeInfo.scala | 8 +- hail/hail/src/is/hail/asm4s/package.scala | 32 +-- .../src/is/hail/backend/ExecuteContext.scala | 2 +- .../src/is/hail/backend/HailTaskContext.scala | 2 +- .../backend/driver/BatchQueryDriver.scala | 2 +- .../hail/backend/driver/Py4JQueryDriver.scala | 2 +- .../is/hail/backend/local/LocalBackend.scala | 8 +- .../hail/backend/service/ServiceBackend.scala | 8 +- .../is/hail/backend/spark/SparkBackend.scala | 10 +- .../compatibility/LegacyBufferSpecs.scala | 6 +- .../hail/compatibility/LegacyRVDSpecs.scala | 12 +- .../src/is/hail/cxx/RegionValueIterator.scala | 4 +- .../experimental/ExperimentalFunctions.scala | 2 +- hail/hail/src/is/hail/expr/Parser.scala | 2 +- .../expr/ir/AbstractMatrixTableSpec.scala | 14 +- .../is/hail/expr/ir/AbstractTableSpec.scala | 22 +- .../src/is/hail/expr/ir/BinarySearch.scala | 6 +- .../src/is/hail/expr/ir/BlockMatrixIR.scala | 42 ++-- .../is/hail/expr/ir/BlockMatrixWriter.scala | 34 +-- hail/hail/src/is/hail/expr/ir/Compile.scala | 8 +- hail/hail/src/is/hail/expr/ir/Emit.scala | 8 +- .../is/hail/expr/ir/EmitClassBuilder.scala | 7 +- .../src/is/hail/expr/ir/EmitCodeBuilder.scala | 6 +- hail/hail/src/is/hail/expr/ir/Env.scala | 26 +-- .../hail/expr/ir/ExtractIntervalFilters.scala | 38 ++-- .../src/is/hail/expr/ir/FreeVariables.scala | 2 +- .../src/is/hail/expr/ir/GenericLines.scala | 28 +-- .../is/hail/expr/ir/GenericTableValue.scala | 2 +- hail/hail/src/is/hail/expr/ir/IR.scala | 15 +- hail/hail/src/is/hail/expr/ir/MatrixIR.scala | 50 ++--- .../src/is/hail/expr/ir/MatrixWriter.scala | 62 +++--- hail/hail/src/is/hail/expr/ir/Param.scala | 16 +- hail/hail/src/is/hail/expr/ir/Parser.scala | 12 +- .../expr/ir/SpecializedArrayBuilders.scala | 10 +- hail/hail/src/is/hail/expr/ir/TableIR.scala | 97 ++++----- .../hail/src/is/hail/expr/ir/TableValue.scala | 20 +- .../src/is/hail/expr/ir/TableWriter.scala | 46 ++-- .../src/is/hail/expr/ir/ValueReader.scala | 10 +- .../src/is/hail/expr/ir/ValueWriter.scala | 2 +- .../is/hail/expr/ir/agg/AggregatorState.scala | 30 +-- .../expr/ir/agg/ApproxCDFAggregator.scala | 18 +- .../ArrayElementLengthCheckAggregator.scala | 27 +-- .../expr/ir/agg/CallStatsAggregator.scala | 22 +- .../hail/expr/ir/agg/CollectAggregator.scala | 22 +- .../expr/ir/agg/CollectAsSetAggregator.scala | 15 +- .../is/hail/expr/ir/agg/CountAggregator.scala | 9 +- .../hail/expr/ir/agg/DensifyAggregator.scala | 19 +- .../expr/ir/agg/DownsampleAggregator.scala | 20 +- .../hail/expr/ir/agg/GroupedAggregator.scala | 18 +- .../expr/ir/agg/ImputeTypeAggregator.scala | 11 +- .../ir/agg/LinearRegressionAggregator.scala | 9 +- .../hail/expr/ir/agg/MonoidAggregator.scala | 23 +- .../expr/ir/agg/NDArraySumAggregator.scala | 3 +- .../expr/ir/agg/PrevNonNullAggregator.scala | 9 +- .../ir/agg/ReservoirSampleAggregator.scala | 29 ++- .../is/hail/expr/ir/agg/TakeAggregator.scala | 20 +- .../hail/expr/ir/agg/TakeByAggregator.scala | 20 +- .../ir/functions/ApproxCDFFunctions.scala | 2 +- .../expr/ir/functions/ArrayFunctions.scala | 2 +- .../expr/ir/functions/CallFunctions.scala | 2 +- .../expr/ir/functions/DictFunctions.scala | 2 +- .../is/hail/expr/ir/functions/Functions.scala | 2 +- .../expr/ir/functions/GenotypeFunctions.scala | 2 +- .../expr/ir/functions/IntervalFunctions.scala | 2 +- .../expr/ir/functions/LocusFunctions.scala | 2 +- .../expr/ir/functions/MathFunctions.scala | 2 +- .../ir/functions/RandomSeededFunctions.scala | 2 +- .../functions/ReferenceGenomeFunctions.scala | 2 +- .../ir/functions/RelationalFunctions.scala | 8 +- .../hail/expr/ir/functions/SetFunctions.scala | 2 +- .../expr/ir/functions/StringFunctions.scala | 2 +- .../TableCalculateNewPartitions.scala | 7 +- .../expr/ir/functions/UtilFunctions.scala | 2 +- .../expr/ir/lowering/LowerBlockMatrixIR.scala | 24 +-- .../ir/lowering/LowerDistributedSort.scala | 15 +- .../hail/expr/ir/lowering/LoweringPass.scala | 15 +- .../expr/ir/lowering/RVDToTableStage.scala | 7 +- .../expr/ir/orderings/BinaryOrdering.scala | 2 +- .../is/hail/expr/ir/streams/EmitStream.scala | 28 +-- .../is/hail/expr/ir/streams/StreamUtils.scala | 4 +- hail/hail/src/is/hail/io/BufferSpecs.scala | 73 +++---- .../hail/src/is/hail/io/ByteArrayReader.scala | 2 +- hail/hail/src/is/hail/io/Decoder.scala | 10 +- .../src/is/hail/io/DoubleInputBuffer.scala | 4 +- hail/hail/src/is/hail/io/Encoder.scala | 14 +- .../is/hail/io/HadoopFSDataBinaryReader.scala | 4 +- hail/hail/src/is/hail/io/IndexBTree.scala | 2 +- .../is/hail/io/IndexedBinaryBlockReader.scala | 10 +- .../is/hail/io/IndexedBinaryInputFormat.scala | 2 +- hail/hail/src/is/hail/io/InputBuffers.scala | 199 +++++++++--------- hail/hail/src/is/hail/io/OutputBuffers.scala | 124 +++++------ .../hail/io/RichContextRDDRegionValue.scala | 12 +- hail/hail/src/is/hail/io/TypedCodecSpec.scala | 15 +- .../is/hail/io/avro/AvroPartitionReader.scala | 8 +- .../src/is/hail/io/avro/AvroTableReader.scala | 6 +- .../src/is/hail/io/bgen/BgenSettings.scala | 2 +- hail/hail/src/is/hail/io/bgen/LoadBgen.scala | 28 +-- .../src/is/hail/io/fs/AzureStorageFS.scala | 16 +- hail/hail/src/is/hail/io/fs/FS.scala | 43 ++-- .../src/is/hail/io/fs/GoogleStorageFS.scala | 14 +- hail/hail/src/is/hail/io/fs/HadoopFS.scala | 40 ++-- hail/hail/src/is/hail/io/fs/RouterFS.scala | 18 +- hail/hail/src/is/hail/io/gen/ExportGen.scala | 2 +- .../io/hadoop/ByteArrayOutputFormat.scala | 4 +- .../src/is/hail/io/index/IndexReader.scala | 6 +- .../src/is/hail/io/index/IndexWriter.scala | 2 +- .../hail/src/is/hail/io/plink/LoadPlink.scala | 10 +- .../src/is/hail/io/tabix/TabixReader.scala | 5 +- hail/hail/src/is/hail/io/vcf/LoadVCF.scala | 28 +-- .../is/hail/io/vcf/TabixReadVCFIterator.scala | 10 +- hail/hail/src/is/hail/linalg/BLAS.scala | 2 +- .../hail/src/is/hail/linalg/BlockMatrix.scala | 88 ++++---- hail/hail/src/is/hail/linalg/LAPACK.scala | 2 +- .../src/is/hail/linalg/MatrixSparsity.scala | 2 +- hail/hail/src/is/hail/linalg/RowMatrix.scala | 13 +- hail/hail/src/is/hail/lir/Blocks.scala | 4 +- hail/hail/src/is/hail/lir/Locals.scala | 4 +- hail/hail/src/is/hail/lir/X.scala | 54 ++--- .../hail/src/is/hail/methods/ForceCount.scala | 4 +- hail/hail/src/is/hail/methods/IBD.scala | 6 +- .../is/hail/methods/LinearRegression.scala | 8 +- .../src/is/hail/methods/LocalLDPrune.scala | 4 +- .../is/hail/methods/LogisticRegression.scala | 4 +- .../methods/MatrixExportEntriesByCol.scala | 6 +- .../src/is/hail/methods/NPartitions.scala | 4 +- hail/hail/src/is/hail/methods/Nirvana.scala | 4 +- hail/hail/src/is/hail/methods/PCA.scala | 4 +- hail/hail/src/is/hail/methods/PCRelate.scala | 4 +- .../is/hail/methods/PoissonRegression.scala | 4 +- hail/hail/src/is/hail/methods/Skat.scala | 4 +- .../src/is/hail/rvd/AbstractRVDSpec.scala | 30 +-- hail/hail/src/is/hail/rvd/RVD.scala | 18 +- hail/hail/src/is/hail/rvd/RVDContext.scala | 2 +- .../hail/src/is/hail/rvd/RVDPartitioner.scala | 4 +- hail/hail/src/is/hail/rvd/RVDType.scala | 8 +- .../src/is/hail/services/JSONLogLayout.scala | 9 +- .../src/is/hail/sparkextras/BlockedRDD.scala | 2 +- .../src/is/hail/sparkextras/ContextRDD.scala | 14 +- .../is/hail/sparkextras/IndexReadRDD.scala | 2 +- .../MapPartitionsWithValueRDD.scala | 2 +- .../RepartitionedOrderedRDD2.scala | 6 +- .../src/is/hail/stats/LeveneHaldane.scala | 15 +- .../hail/stats/LogisticRegressionModel.scala | 20 +- .../hail/stats/PoissonRegressionModel.scala | 6 +- hail/hail/src/is/hail/stats/eigSymD.scala | 8 +- hail/hail/src/is/hail/types/Box.scala | 2 +- .../is/hail/types/TypeWithRequiredness.scala | 80 +++---- .../src/is/hail/types/encoded/EArray.scala | 10 +- .../is/hail/types/encoded/EBaseStruct.scala | 10 +- .../src/is/hail/types/encoded/EBinary.scala | 10 +- .../types/encoded/EBlockMatrixNDArray.scala | 8 +- .../src/is/hail/types/encoded/EBoolean.scala | 10 +- .../encoded/EDictAsUnsortedArrayOfPairs.scala | 20 +- .../src/is/hail/types/encoded/EFloat32.scala | 10 +- .../src/is/hail/types/encoded/EFloat64.scala | 10 +- .../src/is/hail/types/encoded/EInt32.scala | 10 +- .../src/is/hail/types/encoded/EInt64.scala | 10 +- .../types/encoded/ENDArrayColumnMajor.scala | 4 +- .../src/is/hail/types/encoded/ERNGState.scala | 10 +- .../hail/types/encoded/EStructOfArrays.scala | 55 ++--- .../src/is/hail/types/encoded/EType.scala | 2 +- .../is/hail/types/encoded/EUnsortedSet.scala | 20 +- .../physical/PArrayBackedContainer.scala | 6 +- .../is/hail/types/physical/PBaseStruct.scala | 4 +- .../src/is/hail/types/physical/PBinary.scala | 2 +- .../src/is/hail/types/physical/PBoolean.scala | 8 +- .../hail/types/physical/PCanonicalArray.scala | 16 +- .../types/physical/PCanonicalBinary.scala | 50 ++--- .../hail/types/physical/PCanonicalCall.scala | 16 +- .../hail/types/physical/PCanonicalDict.scala | 4 +- .../hail/types/physical/PCanonicalLocus.scala | 28 +-- .../types/physical/PCanonicalNDArray.scala | 55 +++-- .../hail/types/physical/PCanonicalSet.scala | 4 +- .../types/physical/PCanonicalString.scala | 27 +-- .../hail/types/physical/PCanonicalTuple.scala | 4 +- .../src/is/hail/types/physical/PDict.scala | 4 +- .../src/is/hail/types/physical/PFloat32.scala | 6 +- .../src/is/hail/types/physical/PFloat64.scala | 6 +- .../src/is/hail/types/physical/PInt32.scala | 6 +- .../src/is/hail/types/physical/PInt64.scala | 6 +- .../is/hail/types/physical/PInterval.scala | 4 +- .../is/hail/types/physical/PPrimitive.scala | 12 +- .../src/is/hail/types/physical/PStruct.scala | 6 +- .../hail/types/physical/PSubsetStruct.scala | 20 +- .../src/is/hail/types/physical/PTuple.scala | 2 +- .../hail/types/physical/PUnrealizable.scala | 4 +- .../src/is/hail/types/physical/PVoid.scala | 8 +- .../types/physical/StoredSTypePType.scala | 9 +- .../physical/stypes/SingleCodeSCode.scala | 103 ++++++--- .../stypes/concrete/SCanonicalCall.scala | 22 +- .../stypes/concrete/SIndexablePointer.scala | 4 +- .../stypes/concrete/SNDArrayPointer.scala | 4 +- .../stypes/concrete/SNDArraySlice.scala | 3 +- .../physical/stypes/concrete/SRNGState.scala | 18 +- .../stypes/concrete/SStringPointer.scala | 6 +- .../stypes/concrete/SStructView.scala | 2 +- .../stypes/concrete/SUnreachable.scala | 3 +- .../stypes/interfaces/SBaseStruct.scala | 6 +- .../stypes/interfaces/SContainer.scala | 2 +- .../stypes/interfaces/SInterval.scala | 2 +- .../physical/stypes/interfaces/SNDArray.scala | 6 +- .../physical/stypes/interfaces/SStream.scala | 4 +- .../stypes/primitives/SPrimitive.scala | 2 +- .../is/hail/types/virtual/MatrixType.scala | 2 +- .../src/is/hail/types/virtual/TArray.scala | 6 +- .../is/hail/types/virtual/TBaseStruct.scala | 2 +- .../src/is/hail/types/virtual/TBinary.scala | 6 +- .../src/is/hail/types/virtual/TBoolean.scala | 4 +- .../src/is/hail/types/virtual/TCall.scala | 4 +- .../src/is/hail/types/virtual/TDict.scala | 4 +- .../src/is/hail/types/virtual/TFloat32.scala | 4 +- .../src/is/hail/types/virtual/TFloat64.scala | 2 +- .../src/is/hail/types/virtual/TInt32.scala | 4 +- .../src/is/hail/types/virtual/TInt64.scala | 4 +- .../src/is/hail/types/virtual/TInterval.scala | 4 +- .../src/is/hail/types/virtual/TLocus.scala | 4 +- .../src/is/hail/types/virtual/TNDArray.scala | 4 +- .../src/is/hail/types/virtual/TRNGState.scala | 4 +- .../hail/src/is/hail/types/virtual/TSet.scala | 4 +- .../src/is/hail/types/virtual/TStream.scala | 4 +- .../src/is/hail/types/virtual/TString.scala | 4 +- .../src/is/hail/types/virtual/TStruct.scala | 2 +- .../src/is/hail/types/virtual/TTuple.scala | 2 +- .../src/is/hail/types/virtual/TVariable.scala | 2 +- .../src/is/hail/types/virtual/TableType.scala | 2 +- .../hail/src/is/hail/types/virtual/Type.scala | 2 +- .../hail/utils/AbsoluteFuzzyComparable.scala | 4 +- .../utils/BufferedAggregatorIterator.scala | 4 +- hail/hail/src/is/hail/utils/Cache.scala | 2 +- .../src/is/hail/utils/FlipbookIterator.scala | 64 +++--- hail/hail/src/is/hail/utils/MultiArray2.scala | 10 +- .../src/is/hail/utils/NumericImplicits.scala | 8 +- hail/hail/src/is/hail/utils/NumericPair.scala | 16 +- .../hail/utils/RelativeFuzzyComparable.scala | 4 +- .../hail/utils/SpillingCollectIterator.scala | 4 +- hail/hail/src/is/hail/utils/package.scala | 6 +- .../richUtils/ByteTrackingOutputStream.scala | 2 +- .../hail/utils/richUtils/RichContextRDD.scala | 4 +- .../richUtils/RichIndexedRowMatrix.scala | 5 +- .../hail/utils/richUtils/RichIterable.scala | 22 +- .../hail/utils/richUtils/RichIterator.scala | 22 +- .../src/is/hail/utils/richUtils/RichRDD.scala | 12 +- .../is/hail/utils/richUtils/RichString.scala | 4 +- hail/hail/src/is/hail/variant/Call.scala | 4 +- .../src/is/hail/variant/ReferenceGenome.scala | 3 +- .../is/hail/variant/RegionValueVariant.scala | 2 +- .../is/hail/backend/ServiceBackendSuite.scala | 2 + .../src/is/hail/expr/ir/EmitStreamSuite.scala | 2 +- .../src/is/hail/expr/ir/FunctionSuite.scala | 2 +- .../expr/ir/MissingArrayBuilderSuite.scala | 2 +- .../src/is/hail/expr/ir/OrderingSuite.scala | 4 +- .../test/src/is/hail/expr/ir/PruneSuite.scala | 14 +- .../ir/agg/StagedBlockLinkedListSuite.scala | 4 +- .../is/hail/utils/FlipbookIteratorSuite.scala | 16 +- 271 files changed, 1845 insertions(+), 1695 deletions(-) diff --git a/hail/build.mill b/hail/build.mill index d0fc454f91f..6ac432e8277 100644 --- a/hail/build.mill +++ b/hail/build.mill @@ -59,6 +59,11 @@ trait HailScalaModule extends ScalaModule, HailJavaModule, ScalafmtModule, Scala `scala-collection-compat` :: "2.13.0", ) + def scalacPluginMvnDeps = Seq( + mvn"org.wartremover::wartremover:3.4.1", + mvn"org.wartremover::wartremover-contrib:2.3.5", + ) + override def scalacOptions: T[Seq[String]] = Task { val scalaVersion = this.scalaVersion() @@ -103,7 +108,8 @@ trait HailScalaModule extends ScalaModule, HailJavaModule, ScalafmtModule, Scala ScalacOptions.tokensForVersion( ScalaVersion.unsafeFromString(scalaVersion), base -- disabled ++ fromBuildMode, - ) + ) :+ "-P:wartremover:only-warn-traverser:org.wartremover.contrib.warts.MissingOverride" + // :+ "-P:wartremover:only-warn-traverser:org.wartremover.contrib.warts.UnsafeInheritance" val rawOptions = if scalaVersion < "2.12" || scalaVersion >= "3" diff --git a/hail/hail/ir-gen/src/Main.scala b/hail/hail/ir-gen/src/Main.scala index f9990f74dca..910c27a3d63 100644 --- a/hail/hail/ir-gen/src/Main.scala +++ b/hail/hail/ir-gen/src/Main.scala @@ -25,7 +25,7 @@ trait IRDSL { protected def oneOrMore[T](x: Type[T]): Type[Seq[T]] protected def optional[T](x: Type[T]): Type[Option[T]] - def name: Type[Name] = att("Name").asInstanceOf[Type[Name]] + final def name: Type[Name] = att("Name").asInstanceOf[Type[Name]] def child: Type[Child] def tableChild: Type[Child] def matrixChild: Type[Child] @@ -81,10 +81,10 @@ trait IRDSL { // Implicits for common names - implicit def nameDefaultName(t: Type[Name]): Declaration[Name] = + final implicit def nameDefaultName(t: Type[Name]): Declaration[Name] = in("name", t) - implicit def childDefaultName(t: Type[Child]): Declaration[Child] = + final implicit def childDefaultName(t: Type[Child]): Declaration[Child] = in("child", t) } @@ -558,7 +558,7 @@ object IRDSL_Impl extends IRDSL { s"object $name extends ${name}CompanionExt\n" else "" - def generateDef: String = companionDef + classDef + "\n" + override def generateDef: String = companionDef + classDef + "\n" } } diff --git a/hail/hail/package.mill b/hail/hail/package.mill index e73beeb8b3a..8083c7a3910 100644 --- a/hail/hail/package.mill +++ b/hail/hail/package.mill @@ -160,9 +160,7 @@ trait RootHailModule extends CrossScalaModule, HailScalaModule: ) override def scalacPluginMvnDeps: T[Seq[Dep]] = - Seq( - `better-monadic-for` :: "0.3.1", - ) + super.scalacPluginMvnDeps() :+ (`better-monadic-for` :: "0.3.1") def writeRunClasspath: T[PathRef] = Task { diff --git a/hail/hail/src/is/hail/annotations/BroadcastValue.scala b/hail/hail/src/is/hail/annotations/BroadcastValue.scala index ffd4cec0779..941e407ccc0 100644 --- a/hail/hail/src/is/hail/annotations/BroadcastValue.scala +++ b/hail/hail/src/is/hail/annotations/BroadcastValue.scala @@ -99,9 +99,9 @@ trait BroadcastRegionValue extends Logging { case class BroadcastRow(ctx: ExecuteContext, value: RegionValue, t: PStruct) extends BroadcastRegionValue { - def javaValue: UnsafeRow = UnsafeRow.readBaseStruct(t, value.region, value.offset) + override def javaValue: UnsafeRow = UnsafeRow.readBaseStruct(t, value.region, value.offset) - def safeJavaValue: Row = SafeRow.read(t, value).asInstanceOf[Row] + override def safeJavaValue: Row = SafeRow.read(t, value).asInstanceOf[Row] def cast(newT: PStruct): BroadcastRow = { assert(t.virtualType == newT.virtualType) @@ -128,9 +128,9 @@ case class BroadcastIndexedSeq( t: PArray, ) extends BroadcastRegionValue { - def safeJavaValue: IndexedSeq[Row] = SafeRow.read(t, value).asInstanceOf[IndexedSeq[Row]] + override def safeJavaValue: IndexedSeq[Row] = SafeRow.read(t, value).asInstanceOf[IndexedSeq[Row]] - def javaValue: UnsafeIndexedSeq = new UnsafeIndexedSeq(t, value.region, value.offset) + override def javaValue: UnsafeIndexedSeq = new UnsafeIndexedSeq(t, value.region, value.offset) def cast(newT: PArray): BroadcastIndexedSeq = { assert(t.virtualType == newT.virtualType) diff --git a/hail/hail/src/is/hail/annotations/ChunkCache.scala b/hail/hail/src/is/hail/annotations/ChunkCache.scala index 7ca5deb69cb..22eddec08ee 100644 --- a/hail/hail/src/is/hail/annotations/ChunkCache.scala +++ b/hail/hail/src/is/hail/annotations/ChunkCache.scala @@ -87,7 +87,7 @@ private class ChunkCache(allocator: Long => Long, freer: Long => Unit) { } // BiConsumer needed to work with scala 2.11.12 bigChunkCache.forEach(new BiConsumer[Long, LongArrayBuilder]() { - def accept(key: Long, value: LongArrayBuilder): Unit = + override def accept(key: Long, value: LongArrayBuilder): Unit = while (value.size > 0) freeChunkFromMemory(pool, value.pop()) }) } diff --git a/hail/hail/src/is/hail/annotations/ExtendedOrdering.scala b/hail/hail/src/is/hail/annotations/ExtendedOrdering.scala index 87655121305..eb5278320f4 100644 --- a/hail/hail/src/is/hail/annotations/ExtendedOrdering.scala +++ b/hail/hail/src/is/hail/annotations/ExtendedOrdering.scala @@ -9,7 +9,8 @@ object ExtendedOrdering { new ExtendedOrdering { val missingEqual = _missingEqual - def compareNonnull(x: T, y: T): Int = ord.compare(x.asInstanceOf[S], y.asInstanceOf[S]) + override def compareNonnull(x: T, y: T): Int = + ord.compare(x.asInstanceOf[S], y.asInstanceOf[S]) override def ltNonnull(x: T, y: T): Boolean = ord.lt(x.asInstanceOf[S], y.asInstanceOf[S]) @@ -24,7 +25,7 @@ object ExtendedOrdering { new ExtendedOrdering { val missingEqual = _missingEqual - def compareNonnull(x: T, y: T): Int = { + override def compareNonnull(x: T, y: T): Int = { val xit = x.asInstanceOf[Iterable[T]].iterator val yit = y.asInstanceOf[Iterable[T]].iterator @@ -93,7 +94,7 @@ object ExtendedOrdering { // ord can be null if the element type is a TVariable val elemOrd = if (ord != null) ord.toOrdering else null - def compareNonnull(x: T, y: T): Int = + override def compareNonnull(x: T, y: T): Int = itOrd.compareNonnull( x.asInstanceOf[Array[T]].sorted(elemOrd).toFastSeq, y.asInstanceOf[Array[T]].sorted(elemOrd).toFastSeq, @@ -124,7 +125,7 @@ object ExtendedOrdering { val missingEqual = _missingEqual - def compareNonnull(x: T, y: T): Int = + override def compareNonnull(x: T, y: T): Int = saOrd.compareNonnull( x.asInstanceOf[Iterable[T]].toArray, y.asInstanceOf[Iterable[T]].toArray, @@ -158,7 +159,7 @@ object ExtendedOrdering { private def toArrayOfT(x: T): Array[T] = x.asInstanceOf[Map[_, _]].iterator.map { case (k, v) => Row(k, v): T }.toArray - def compareNonnull(x: T, y: T): Int = + override def compareNonnull(x: T, y: T): Int = saOrd.compareNonnull( toArrayOfT(x), toArrayOfT(y), @@ -384,7 +385,7 @@ abstract class ExtendedOrdering extends Serializable { override def reverse: ExtendedOrdering = outer - def compareNonnull(x: T, y: T): Int = outer.compareNonnull(y, x) + override def compareNonnull(x: T, y: T): Int = outer.compareNonnull(y, x) override def ltNonnull(x: T, y: T): Boolean = outer.ltNonnull(y, x) @@ -394,7 +395,7 @@ abstract class ExtendedOrdering extends Serializable { } def toOrdering: Ordering[T] = new Ordering[T] { - def compare(x: T, y: T): Int = outer.compare(x, y) + override def compare(x: T, y: T): Int = outer.compare(x, y) override def lt(x: T, y: T): Boolean = outer.lt(x, y) diff --git a/hail/hail/src/is/hail/annotations/OrderedRVIterator.scala b/hail/hail/src/is/hail/annotations/OrderedRVIterator.scala index a51d12cb8bc..bbdf51d1a37 100644 --- a/hail/hail/src/is/hail/annotations/OrderedRVIterator.scala +++ b/hail/hail/src/is/hail/annotations/OrderedRVIterator.scala @@ -95,7 +95,7 @@ case class OrderedRVIterator( } } - def advance(): Unit = { + override def advance(): Unit = { left.advance() setValue() } @@ -190,9 +190,9 @@ case class OrderedRVIterator( private val rvb = new RegionValueBuilder(sm, consumerRegion) private val rv = RegionValue() - def hasNext: Boolean = bit.hasNext || q.nonEmpty + override def hasNext: Boolean = bit.hasNext || q.nonEmpty - def next(): RegionValue = { + override def next(): RegionValue = { if (q.isEmpty) { do { val rv = bit.next() diff --git a/hail/hail/src/is/hail/annotations/Region.scala b/hail/hail/src/is/hail/annotations/Region.scala index 1031e019dbe..d8420ca70f3 100644 --- a/hail/hail/src/is/hail/annotations/Region.scala +++ b/hail/hail/src/is/hail/annotations/Region.scala @@ -417,7 +417,7 @@ final class Region protected[annotations] ( def trackSharedChunk(addr: Long): Unit = memory.trackSharedChunk(addr) - def close(): Unit = + override def close(): Unit = invalidate() def addReferenceTo(r: Region): Unit = diff --git a/hail/hail/src/is/hail/annotations/RegionMemory.scala b/hail/hail/src/is/hail/annotations/RegionMemory.scala index 02e45222f4f..b0957f7734f 100644 --- a/hail/hail/src/is/hail/annotations/RegionMemory.scala +++ b/hail/hail/src/is/hail/annotations/RegionMemory.scala @@ -226,7 +226,7 @@ final class RegionMemory(pool: RegionPool) extends AutoCloseable { offsetWithinBlock = 0L } - def close(): Unit = + override def close(): Unit = free() def numChunks: Int = bigChunks.size diff --git a/hail/hail/src/is/hail/annotations/RegionPool.scala b/hail/hail/src/is/hail/annotations/RegionPool.scala index 492554be335..0e02f7d716a 100644 --- a/hail/hail/src/is/hail/annotations/RegionPool.scala +++ b/hail/hail/src/is/hail/annotations/RegionPool.scala @@ -173,7 +173,7 @@ final class RegionPool private (strictMemoryCheck: Boolean, threadName: String, private[this] var closed: Boolean = false - def close(): Unit = { + override def close(): Unit = { if (closed) return closed = true diff --git a/hail/hail/src/is/hail/annotations/UnsafeOrdering.scala b/hail/hail/src/is/hail/annotations/UnsafeOrdering.scala index 507531c8876..64de70999d6 100644 --- a/hail/hail/src/is/hail/annotations/UnsafeOrdering.scala +++ b/hail/hail/src/is/hail/annotations/UnsafeOrdering.scala @@ -1,8 +1,6 @@ package is.hail.annotations abstract class UnsafeOrdering extends Ordering[Long] with Serializable { - def compare(o1: Long, o2: Long): Int - def compare(rv1: RegionValue, rv2: RegionValue): Int = compare(rv1.offset, rv2.offset) diff --git a/hail/hail/src/is/hail/annotations/UnsafeRow.scala b/hail/hail/src/is/hail/annotations/UnsafeRow.scala index 6ef46397745..bcc7506fceb 100644 --- a/hail/hail/src/is/hail/annotations/UnsafeRow.scala +++ b/hail/hail/src/is/hail/annotations/UnsafeRow.scala @@ -13,10 +13,10 @@ import org.apache.spark.sql.Row import sun.reflect.generics.reflectiveObjects.NotImplementedException trait UnKryoSerializable extends KryoSerializable { - def write(kryo: Kryo, output: Output): Unit = + override def write(kryo: Kryo, output: Output): Unit = throw new NotImplementedException() - def read(kryo: Kryo, input: Input): Unit = + override def read(kryo: Kryo, input: Input): Unit = throw new NotImplementedException() } @@ -28,7 +28,7 @@ class UnsafeIndexedSeq( val length: Int = t.loadLength(aoff) - def apply(i: Int): Annotation = { + override def apply(i: Int): Annotation = { if (i < 0 || i >= length) throw new IndexOutOfBoundsException(i.toString) if (t.isElementDefined(aoff, i)) { @@ -156,19 +156,19 @@ class UnsafeRow(val t: PBaseStruct, var region: Region, var offset: Long) def set(rv: RegionValue): Unit = set(rv.region, rv.offset) - def length: Int = t.size + override def length: Int = t.size private def assertDefined(i: Int): Unit = if (isNullAt(i)) throw new NullPointerException(s"null value at index $i") - def get(i: Int): Any = + override def get(i: Int): Any = if (isNullAt(i)) null else UnsafeRow.read(t.types(i), region, t.loadField(offset, i)) - def copy(): Row = new UnsafeRow(t, region, offset) + override def copy(): Row = new UnsafeRow(t, region, offset) def pretty(): String = Region.pretty(t, offset) @@ -328,12 +328,12 @@ class UnsafeNDArray(val pnd: PNDArray, val region: Region, val ndAddr: Long) ext val elementType = pnd.elementType.virtualType val coordStorageArray = new Array[Long](shape.size) - def lookupElement(indices: IndexedSeq[Long]): Annotation = { + override def lookupElement(indices: IndexedSeq[Long]): Annotation = { val elementAddress = pnd.getElementAddress(indices, ndAddr) UnsafeRow.read(pnd.elementType, region, pnd.elementType.unstagedLoadFromNested(elementAddress)) } - def getRowMajorElements(): IndexedSeq[Annotation] = { + override def getRowMajorElements(): IndexedSeq[Annotation] = { val indices = (0 until pnd.nDims).map(_ => 0L).toArray var curIdx = indices.size - 1 var idxIntoFlat = 0 diff --git a/hail/hail/src/is/hail/annotations/ValueVisitor.scala b/hail/hail/src/is/hail/annotations/ValueVisitor.scala index ff69277c889..25eedb54e62 100644 --- a/hail/hail/src/is/hail/annotations/ValueVisitor.scala +++ b/hail/hail/src/is/hail/annotations/ValueVisitor.scala @@ -45,50 +45,50 @@ final class PrettyVisitor extends ValueVisitor { @inline def result(): String = sb.result() - @inline def visitMissing(t: PType): Unit = + @inline override def visitMissing(t: PType): Unit = sb.append("NA") - @inline def visitBoolean(b: Boolean): Unit = + @inline override def visitBoolean(b: Boolean): Unit = sb.append(b) - @inline def visitInt32(i: Int): Unit = + @inline override def visitInt32(i: Int): Unit = sb.append(i) - @inline def visitInt64(l: Long): Unit = + @inline override def visitInt64(l: Long): Unit = sb.append(l) - @inline def visitFloat32(f: Float): Unit = + @inline override def visitFloat32(f: Float): Unit = sb.append(f) - @inline def visitFloat64(d: Double): Unit = + @inline override def visitFloat64(d: Double): Unit = sb.append(d) - @inline def visitBinary(a: Array[Byte]): Unit = + @inline override def visitBinary(a: Array[Byte]): Unit = sb.append("bytes...") - @inline def visitString(s: String): Unit = + @inline override def visitString(s: String): Unit = sb.append(s) - @inline def enterStruct(t: PStruct): Unit = + @inline override def enterStruct(t: PStruct): Unit = sb.append("{") - def enterField(f: PField): Unit = { + override def enterField(f: PField): Unit = { if (f.index > 0) sb += ',' sb += ' ' ++= f.name ++= ": ": Unit } - @inline def leaveField(): Unit = {} + @inline override def leaveField(): Unit = {} - @inline def leaveStruct(): Unit = + @inline override def leaveStruct(): Unit = sb.append(" }") - @inline def enterTuple(t: PTuple): Unit = + @inline override def enterTuple(t: PTuple): Unit = sb.append('(') - @inline def leaveTuple(): Unit = + @inline override def leaveTuple(): Unit = sb.append(')') - def enterArray(t: PContainer, length: Int): Unit = { + override def enterArray(t: PContainer, length: Int): Unit = { t match { case _: PSet => sb.append("Set") case _: PDict => sb.append("Dict") @@ -98,13 +98,13 @@ final class PrettyVisitor extends ValueVisitor { sb += '[' ++= length.toString += ';': Unit } - @inline def leaveArray(): Unit = + @inline override def leaveArray(): Unit = sb.append("]") - @inline def enterElement(i: Int): Unit = { + @inline override def enterElement(i: Int): Unit = { if (i > 0) sb += ',' sb += ' ' } - @inline def leaveElement(): Unit = {} + @inline override def leaveElement(): Unit = {} } diff --git a/hail/hail/src/is/hail/annotations/WritableRegionValue.scala b/hail/hail/src/is/hail/annotations/WritableRegionValue.scala index 4b864b94901..71871b660c5 100644 --- a/hail/hail/src/is/hail/annotations/WritableRegionValue.scala +++ b/hail/hail/src/is/hail/annotations/WritableRegionValue.scala @@ -125,7 +125,7 @@ class RegionValuePriorityQueue( popped.region.close() } - def iterator: Iterator[RegionValue] = queue.iterator + override def iterator: Iterator[RegionValue] = queue.iterator } class RegionValueArrayBuffer(val t: PType, region: Region, sm: HailStateManager) @@ -140,7 +140,7 @@ class RegionValueArrayBuffer(val t: PType, region: Region, sm: HailStateManager) override def knownSize: Int = idx.length - def addOne(rv: RegionValue): this.type = + override def addOne(rv: RegionValue): this.type = this.append(rv.region, rv.offset) def append(fromRegion: Region, fromOffset: Long): this.type = { @@ -165,7 +165,7 @@ class RegionValueArrayBuffer(val t: PType, region: Region, sm: HailStateManager) this } - def clear(): Unit = { + override def clear(): Unit = { region.clear() idx.clear() rvb.clear() // remove @@ -174,16 +174,16 @@ class RegionValueArrayBuffer(val t: PType, region: Region, sm: HailStateManager) private var itIdx = 0 private val it = new Iterator[RegionValue] { - def next(): RegionValue = { + override def next(): RegionValue = { value.setOffset(idx(itIdx)) itIdx += 1 value } - def hasNext: Boolean = itIdx < idx.size + override def hasNext: Boolean = itIdx < idx.size } - def iterator: Iterator[RegionValue] = { + override def iterator: Iterator[RegionValue] = { itIdx = 0 it } diff --git a/hail/hail/src/is/hail/asm4s/ClassBuilder.scala b/hail/hail/src/is/hail/asm4s/ClassBuilder.scala index eee035ed9a4..5208b0e6543 100644 --- a/hail/hail/src/is/hail/asm4s/ClassBuilder.scala +++ b/hail/hail/src/is/hail/asm4s/ClassBuilder.scala @@ -209,7 +209,7 @@ class ModuleBuilder() { trait WrappedClassBuilder[C] extends WrappedModuleBuilder { def cb: ClassBuilder[C] - def modb: ModuleBuilder = cb.modb + override def modb: ModuleBuilder = cb.modb def className: String = cb.className @@ -499,7 +499,7 @@ class ClassBuilder[C]( new (HailClassLoader => C) with java.io.Serializable { @transient @volatile private var theClass: Class[_] = null - def apply(hcl: HailClassLoader): C = { + override def apply(hcl: HailClassLoader): C = { if (theClass == null) { this.synchronized { if (theClass == null) { @@ -518,7 +518,7 @@ class ClassBuilder[C]( new LocalRef[C](new lir.Parameter(null, 0, ti)) val fieldBuilder: SettableBuilder = new SettableBuilder { - def newSettable[T](name: String)(implicit tti: TypeInfo[T]): Settable[T] = + override def newSettable[T](name: String)(implicit tti: TypeInfo[T]): Settable[T] = genFieldThisRef[T](name) } @@ -625,7 +625,7 @@ object FunctionBuilder { trait WrappedMethodBuilder[C] extends WrappedClassBuilder[C] { def mb: MethodBuilder[C] - def cb: ClassBuilder[C] = mb.cb + override def cb: ClassBuilder[C] = mb.cb def methodName: String = mb.methodName @@ -673,7 +673,8 @@ class MethodBuilder[C]( cb.lclass.newMethod(methodName, parameterTypeInfo, returnTypeInfo, isStatic) val localBuilder: SettableBuilder = new SettableBuilder { - def newSettable[T](name: String)(implicit tti: TypeInfo[T]): Settable[T] = newLocal[T](name) + override def newSettable[T](name: String)(implicit tti: TypeInfo[T]): Settable[T] = + newLocal[T](name) } def this_ : Value[C] = diff --git a/hail/hail/src/is/hail/asm4s/Code.scala b/hail/hail/src/is/hail/asm4s/Code.scala index d9e7a71ba15..338d9f798fd 100644 --- a/hail/hail/src/is/hail/asm4s/Code.scala +++ b/hail/hail/src/is/hail/asm4s/Code.scala @@ -1082,7 +1082,7 @@ object Code { def _empty: Value[Unit] = Value.fromLIR[Unit](null: lir.ValueX) def _throwAny[T <: java.lang.Throwable]: Thrower[T] = new Thrower[T] { - def apply[U](cerr: Code[T])(implicit uti: TypeInfo[U]): Code[U] = { + override def apply[U](cerr: Code[T])(implicit uti: TypeInfo[U]): Code[U] = { if (uti eq UnitInfo) { cerr.end.append(lir.throwx(cerr.v)) val newC = new VCode(cerr.start, cerr.end, null) @@ -1214,7 +1214,7 @@ object Code { c.v match { case v: lir.LdcX => val t = new Value[T] { - def get: Code[T] = Code(lir.ldcInsn(v.a, v.ti)) + override def get: Code[T] = Code(lir.ldcInsn(v.a, v.ti)) } return f(t) // You can't forward local references here because the local might have changed @@ -1302,27 +1302,27 @@ class VCode[+T]( // val stack = Thread.currentThread().getStackTrace // var clearStack: Array[StackTraceElement] = _ - def start: lir.Block = { + override def start: lir.Block = { check() _start } - def end: lir.Block = { + override def end: lir.Block = { check() _end } - def v: lir.ValueX = { + override def v: lir.ValueX = { check() _v } - def check(): Unit = + override def check(): Unit = /* if (_start == null) { println(clearStack.mkString("\n")) println("-----") * println(stack.mkString("\n")) } */ assert(_start != null) - def clear(): Unit = { + override def clear(): Unit = { /* if (clearStack != null) { println(clearStack.mkString("\n")) } assert(clearStack == null) * clearStack = Thread.currentThread().getStackTrace */ @@ -1364,17 +1364,17 @@ class CCode( _Lfalse } - def start: lir.Block = { + override def start: lir.Block = { checkV() _start } - def end: lir.Block = { + override def end: lir.Block = { checkV() _end } - def v: lir.ValueX = { + override def v: lir.ValueX = { checkV() _v } @@ -1408,7 +1408,7 @@ class CCode( assert(_entry != null) } - def check(): Unit = { + override def check(): Unit = { if (_kind == null || _kind == CodeKind.C) assert(_entry != null) else { @@ -1417,7 +1417,7 @@ class CCode( } } - def clear(): Unit = { + override def clear(): Unit = { _entry = null _Ltrue = null _Lfalse = null @@ -1462,15 +1462,15 @@ class ConstCodeBoolean(val b: Boolean) extends Code[Boolean] { def toCCode: CCode = vc.toCCode - def start: lir.Block = vc.start + override def start: lir.Block = vc.start - def end: lir.Block = vc.end + override def end: lir.Block = vc.end - def v: lir.ValueX = vc.v + override def v: lir.ValueX = vc.v - def check(): Unit = vc.check() + override def check(): Unit = vc.check() - def clear(): Unit = vc.clear() + override def clear(): Unit = vc.clear() } class CodeBoolean(val lhs: Code[Boolean]) extends AnyVal { @@ -1851,27 +1851,27 @@ class CodeLabel(val L: lir.Block) extends Code[Unit] { def isImplemented: Boolean = L.wellFormed - def start: lir.Block = { + override def start: lir.Block = { check() _start } - def end: lir.Block = { + override def end: lir.Block = { check() _start } - def v: lir.ValueX = { + override def v: lir.ValueX = { check() null } - def check(): Unit = + override def check(): Unit = /* if (_start == null) { println(clearStack.mkString("\n")) println("-----") * println(stack.mkString("\n")) } */ assert(_start != null) - def clear(): Unit = + override def clear(): Unit = /* if (clearStack != null) { println(clearStack.mkString("\n")) } assert(clearStack == null) * clearStack = Thread.currentThread().getStackTrace */ _start = null @@ -2004,7 +2004,7 @@ object FieldRef { object Value { def fromLIR[T](v: => lir.ValueX): Value[T] = new Value[T] { - def get: Code[T] = Code(v) + override def get: Code[T] = Code(v) } } @@ -2040,23 +2040,23 @@ class ThisLazyFieldRef[T: TypeInfo](cb: ClassBuilder[_], name: String, setup: Co class ThisFieldRef[T: TypeInfo](cb: ClassBuilder[_], f: Field[T]) extends Settable[T] { def name: String = f.name - def get: Code[T] = f.get(cb.this_) + override def get: Code[T] = f.get(cb.this_) - def store(rhs: Code[T]): Code[Unit] = f.put(cb.this_, rhs) + override def store(rhs: Code[T]): Code[Unit] = f.put(cb.this_, rhs) } class StaticFieldRef[T: TypeInfo](f: StaticField[T]) extends Settable[T] { def name: String = f.name - def get: Code[T] = f.get() + override def get: Code[T] = f.get() - def store(rhs: Code[T]): Code[Unit] = f.put(rhs) + override def store(rhs: Code[T]): Code[Unit] = f.put(rhs) } class LocalRef[T](val l: lir.Local) extends Settable[T] { - def get: Code[T] = Code(lir.load(l)) + override def get: Code[T] = Code(lir.load(l)) - def store(rhs: Code[T]): Code[Unit] = { + override def store(rhs: Code[T]): Code[Unit] = { assert(rhs.v != null) rhs.end.append(lir.store(l, rhs.v)) val newC = new VCode(rhs.start, rhs.end, null) @@ -2090,7 +2090,7 @@ class FieldRef[T, S](f: reflect.Field)(implicit tct: ClassTag[T], sti: TypeInfo[ def getField(lhs: Value[T]): Value[S] = new Value[S] { - def get: Code[S] = self.getField(if (lhs != null) lhs.get else null) + override def get: Code[S] = self.getField(if (lhs != null) lhs.get else null) } def getField(lhs: Code[T]): Code[S] = diff --git a/hail/hail/src/is/hail/asm4s/CodeBuilder.scala b/hail/hail/src/is/hail/asm4s/CodeBuilder.scala index 81a59574916..5dc3b48f445 100644 --- a/hail/hail/src/is/hail/asm4s/CodeBuilder.scala +++ b/hail/hail/src/is/hail/asm4s/CodeBuilder.scala @@ -286,7 +286,7 @@ trait CodeBuilderLike { } class CodeBuilder(val mb: MethodBuilder[_], var code: Code[Unit]) extends CodeBuilderLike { - def isOpenEnded: Boolean = + override def isOpenEnded: Boolean = code.isOpenEnded override def append(c: Code[Unit]): Unit = { @@ -306,7 +306,7 @@ class CodeBuilder(val mb: MethodBuilder[_], var code: Code[Unit]) extends CodeBu def uncheckedAppend(c: Code[Unit]): Unit = code = Code(code, c) - def result(): Code[Unit] = { + override def result(): Code[Unit] = { val tmp = code code = Code._empty tmp diff --git a/hail/hail/src/is/hail/asm4s/GenericTypeInfo.scala b/hail/hail/src/is/hail/asm4s/GenericTypeInfo.scala index 372cdaaf82a..c6d35efeb09 100644 --- a/hail/hail/src/is/hail/asm4s/GenericTypeInfo.scala +++ b/hail/hail/src/is/hail/asm4s/GenericTypeInfo.scala @@ -16,7 +16,7 @@ object GenericTypeInfo { final class GenericTypeInfo[T: TypeInfo]() extends MaybeGenericTypeInfo[T] { val base = typeInfo[T] - def castFromGeneric(cb: CodeBuilderLike, _x: Value[_]): Value[T] = { + override def castFromGeneric(cb: CodeBuilderLike, _x: Value[_]): Value[T] = { val x = coerce[AnyRef](_x) base match { case _: IntInfo.type => @@ -44,7 +44,7 @@ final class GenericTypeInfo[T: TypeInfo]() extends MaybeGenericTypeInfo[T] { } } - def castToGeneric(cb: CodeBuilderLike, x: Value[T]): Value[_] = base match { + override def castToGeneric(cb: CodeBuilderLike, x: Value[T]): Value[_] = base match { case _: IntInfo.type => cb.memoize(Code.boxInt(coerce[Int](x))) case _: LongInfo.type => @@ -78,8 +78,8 @@ object NotGenericTypeInfo { } final class NotGenericTypeInfo[T: TypeInfo]() extends MaybeGenericTypeInfo[T] { - def castFromGeneric(cb: CodeBuilderLike, x: Value[_]): Value[T] = coerce[T](x) - def castToGeneric(cb: CodeBuilderLike, x: Value[T]): Value[_] = x + override def castFromGeneric(cb: CodeBuilderLike, x: Value[_]): Value[T] = coerce[T](x) + override def castToGeneric(cb: CodeBuilderLike, x: Value[T]): Value[_] = x val base = typeInfo[T] val generic = base diff --git a/hail/hail/src/is/hail/asm4s/package.scala b/hail/hail/src/is/hail/asm4s/package.scala index b3d24da6e10..49ce98e7f0d 100644 --- a/hail/hail/src/is/hail/asm4s/package.scala +++ b/hail/hail/src/is/hail/asm4s/package.scala @@ -51,7 +51,7 @@ package asm4s { val astoreOp = AASTORE val returnOp = ARETURN - def newArray(): AbstractInsnNode = new TypeInsnNode(ANEWARRAY, iname) + override def newArray(): AbstractInsnNode = new TypeInsnNode(ANEWARRAY, iname) override def uninitializedValue: Value[_] = Code._uncheckednull(this) } @@ -65,7 +65,7 @@ package asm4s { val astoreOp = AASTORE val returnOp = ARETURN - def newArray() = new TypeInsnNode(ANEWARRAY, iname) + override def newArray() = new TypeInsnNode(ANEWARRAY, iname) override def uninitializedValue: Value[_] = Code._null[Array[T]](this) } @@ -130,7 +130,7 @@ package object asm4s { val returnOp = IRETURN val newarrayOp = NEWARRAY - def newArray() = new IntInsnNode(NEWARRAY, T_BOOLEAN) + override def newArray() = new IntInsnNode(NEWARRAY, T_BOOLEAN) override def uninitializedValue: Value[_] = const(false) } @@ -144,7 +144,7 @@ package object asm4s { val returnOp = IRETURN val newarrayOp = NEWARRAY - def newArray() = new IntInsnNode(NEWARRAY, T_BYTE) + override def newArray() = new IntInsnNode(NEWARRAY, T_BYTE) override def uninitializedValue: Value[_] = const(0.toByte) } @@ -158,7 +158,7 @@ package object asm4s { val returnOp = IRETURN val newarrayOp = NEWARRAY - def newArray() = new IntInsnNode(NEWARRAY, T_SHORT) + override def newArray() = new IntInsnNode(NEWARRAY, T_SHORT) override def uninitializedValue: Value[_] = const(0.toShort) } @@ -171,7 +171,7 @@ package object asm4s { val astoreOp = IASTORE val returnOp = IRETURN - def newArray() = new IntInsnNode(NEWARRAY, T_INT) + override def newArray() = new IntInsnNode(NEWARRAY, T_INT) override def uninitializedValue: Value[_] = const(0) } @@ -185,7 +185,7 @@ package object asm4s { val returnOp = LRETURN override val slots = 2 - def newArray() = new IntInsnNode(NEWARRAY, T_LONG) + override def newArray() = new IntInsnNode(NEWARRAY, T_LONG) override def uninitializedValue: Value[_] = const(0L) } @@ -198,7 +198,7 @@ package object asm4s { val astoreOp = FASTORE val returnOp = FRETURN - def newArray() = new IntInsnNode(NEWARRAY, T_FLOAT) + override def newArray() = new IntInsnNode(NEWARRAY, T_FLOAT) override def uninitializedValue: Value[_] = const(0f) } @@ -212,7 +212,7 @@ package object asm4s { val returnOp = DRETURN override val slots = 2 - def newArray() = new IntInsnNode(NEWARRAY, T_DOUBLE) + override def newArray() = new IntInsnNode(NEWARRAY, T_DOUBLE) override def uninitializedValue: Value[_] = const(0d) } @@ -226,21 +226,21 @@ package object asm4s { val returnOp = IRETURN override val slots = 2 - def newArray() = new IntInsnNode(NEWARRAY, T_CHAR) + override def newArray() = new IntInsnNode(NEWARRAY, T_CHAR) override def uninitializedValue: Value[_] = const(0.toChar) } implicit object UnitInfo extends TypeInfo[Unit] { val desc = "V" - def loadOp = ??? - def storeOp = ??? - def aloadOp = ??? - def astoreOp = ??? + override def loadOp = ??? + override def storeOp = ??? + override def aloadOp = ??? + override def astoreOp = ??? val returnOp = RETURN override def slots = ??? - def newArray() = ??? + override def newArray() = ??? override def uninitializedValue: Value[_] = Code._empty } @@ -354,7 +354,7 @@ package object asm4s { } implicit def const(b: Boolean): Value[Boolean] = new Value[Boolean] { - def get: Code[Boolean] = new ConstCodeBoolean(b) + override def get: Code[Boolean] = new ConstCodeBoolean(b) } implicit def const(i: Int): Value[Int] = _const(i, IntInfo) diff --git a/hail/hail/src/is/hail/backend/ExecuteContext.scala b/hail/hail/src/is/hail/backend/ExecuteContext.scala index e03577c61d6..8e5b1b4856d 100644 --- a/hail/hail/src/is/hail/backend/ExecuteContext.scala +++ b/hail/hail/src/is/hail/backend/ExecuteContext.scala @@ -161,7 +161,7 @@ class ExecuteContext( def shouldLogIR(): Boolean = !shouldNotLogIR() - def close(): Unit = { + override def close(): Unit = { tempFileManager.close() taskContext.close() } diff --git a/hail/hail/src/is/hail/backend/HailTaskContext.scala b/hail/hail/src/is/hail/backend/HailTaskContext.scala index 41f82c7cba4..459888e6141 100644 --- a/hail/hail/src/is/hail/backend/HailTaskContext.scala +++ b/hail/hail/src/is/hail/backend/HailTaskContext.scala @@ -44,7 +44,7 @@ abstract class HailTaskContext extends AutoCloseable with Logging { f } - def close(): Unit = { + override def close(): Unit = { logger.info( s"TaskReport: stage=${stageId()}, partition=${partitionId()}, attempt=${attemptNumber()}, " + s"peakBytes=${thePool.getHighestTotalUsage}, peakBytesReadable=${formatSpace(thePool.getHighestTotalUsage)}, " + diff --git a/hail/hail/src/is/hail/backend/driver/BatchQueryDriver.scala b/hail/hail/src/is/hail/backend/driver/BatchQueryDriver.scala index 7ca1e978e20..af143f46e4d 100644 --- a/hail/hail/src/is/hail/backend/driver/BatchQueryDriver.scala +++ b/hail/hail/src/is/hail/backend/driver/BatchQueryDriver.scala @@ -239,7 +239,7 @@ private class HailSocketAPIOutputStream( def writeString(s: String): Unit = writeBytes(s.getBytes(StandardCharsets.UTF_8)) - def close(): Unit = + override def close(): Unit = if (!closed) { out.close() closed = true diff --git a/hail/hail/src/is/hail/backend/driver/Py4JQueryDriver.scala b/hail/hail/src/is/hail/backend/driver/Py4JQueryDriver.scala index 54dc24b3e6e..d2de329597e 100644 --- a/hail/hail/src/is/hail/backend/driver/Py4JQueryDriver.scala +++ b/hail/hail/src/is/hail/backend/driver/Py4JQueryDriver.scala @@ -474,7 +474,7 @@ final class Py4JQueryDriver(backend: Backend) extends Closeable with Logging { // Note that simply calling httpServer.start() from a non-daemon thread will spawn a // non-daemon thread itself. private[this] val thread = new Thread(new Runnable() { - def run(): Unit = httpServer.start() + override def run(): Unit = httpServer.start() }) thread.setDaemon(true) diff --git a/hail/hail/src/is/hail/backend/local/LocalBackend.scala b/hail/hail/src/is/hail/backend/local/LocalBackend.scala index a869f30b483..cb98e53d533 100644 --- a/hail/hail/src/is/hail/backend/local/LocalBackend.scala +++ b/hail/hail/src/is/hail/backend/local/LocalBackend.scala @@ -40,7 +40,7 @@ object LocalBackend extends Backend with Logging { is.hail.linalg.registerImplOpMulMatrix_DMD_DVD_eq_DVD - def broadcast[T: ClassTag](value: T): BroadcastValue[T] = + override def broadcast[T: ClassTag](value: T): BroadcastValue[T] = new LocalBroadcastValue[T](value) private[this] var stageIdx: Int = 0 @@ -100,9 +100,9 @@ object LocalBackend extends Backend with Logging { } } - def defaultParallelism: Int = 1 + override def defaultParallelism: Int = 1 - def close(): Unit = + override def close(): Unit = synchronized { stageIdx = 0 } private[this] def _jvmLowerAndExecute( @@ -140,7 +140,7 @@ object LocalBackend extends Backend with Logging { ): TableReader = LowerDistributedSort.distributedSort(ctx, stage, sortFields, rt, nPartitions) - def tableToTableStage(ctx: ExecuteContext, inputIR: TableIR, analyses: LoweringAnalyses) + override def tableToTableStage(ctx: ExecuteContext, inputIR: TableIR, analyses: LoweringAnalyses) : TableStage = LowerTableIR.applyTable(inputIR, DArrayLowering.All, ctx, analyses) } diff --git a/hail/hail/src/is/hail/backend/service/ServiceBackend.scala b/hail/hail/src/is/hail/backend/service/ServiceBackend.scala index fc7e891ca22..290f01bddf5 100644 --- a/hail/hail/src/is/hail/backend/service/ServiceBackend.scala +++ b/hail/hail/src/is/hail/backend/service/ServiceBackend.scala @@ -112,11 +112,11 @@ class ServiceBackend( Executors.newFixedThreadPool(MaxAvailableGcsConnections) } - def defaultParallelism: Int = 4 + override def defaultParallelism: Int = 4 - def broadcast[T: ClassTag](_value: T): BroadcastValue[T] = + override def broadcast[T: ClassTag](_value: T): BroadcastValue[T] = new BroadcastValue[T] with Serializable { - def value: T = _value + override def value: T = _value } override def runtimeContext(ctx: ExecuteContext): DriverRuntimeContext = @@ -409,7 +409,7 @@ class ServiceBackend( ): TableReader = LowerDistributedSort.distributedSort(ctx, inputStage, sortFields, rt, nPartitions) - def tableToTableStage(ctx: ExecuteContext, inputIR: TableIR, analyses: LoweringAnalyses) + override def tableToTableStage(ctx: ExecuteContext, inputIR: TableIR, analyses: LoweringAnalyses) : TableStage = LowerTableIR.applyTable(inputIR, DArrayLowering.All, ctx, analyses) } diff --git a/hail/hail/src/is/hail/backend/spark/SparkBackend.scala b/hail/hail/src/is/hail/backend/spark/SparkBackend.scala index a8d87497f96..2ed1be9f55f 100644 --- a/hail/hail/src/is/hail/backend/spark/SparkBackend.scala +++ b/hail/hail/src/is/hail/backend/spark/SparkBackend.scala @@ -32,7 +32,7 @@ import org.apache.spark.sql.SparkSession import sourcecode.Enclosing class SparkBroadcastValue[T](bc: Broadcast[T]) extends BroadcastValue[T] with Serializable { - def value: T = bc.value + override def value: T = bc.value } object SparkTaskContext { @@ -231,7 +231,7 @@ class SparkBackend(val spark: SparkSession) extends Backend with Logging { val sc: SparkContext = spark.sparkContext - def broadcast[T: ClassTag](value: T): BroadcastValue[T] = + override def broadcast[T: ClassTag](value: T): BroadcastValue[T] = new SparkBroadcastValue[T](sc.broadcast(value)) override def runtimeContext(ctx: ExecuteContext): DriverRuntimeContext = @@ -316,12 +316,12 @@ class SparkBackend(val spark: SparkSession) extends Backend with Logging { } } - def defaultParallelism: Int = + override def defaultParallelism: Int = sc.defaultParallelism override def asSpark(implicit E: Enclosing): SparkBackend = this - def close(): Unit = + override def close(): Unit = SparkBackend.synchronized { assert(this eq SparkBackend.theSparkBackend) SparkBackend.theSparkBackend = null @@ -424,7 +424,7 @@ class SparkBackend(val spark: SparkSession) extends Backend with Logging { RVDTableReader(RVD.unkeyed(rowPType, orderedCRDD), globalsLit, rt) } - def tableToTableStage(ctx: ExecuteContext, inputIR: TableIR, analyses: LoweringAnalyses) + override def tableToTableStage(ctx: ExecuteContext, inputIR: TableIR, analyses: LoweringAnalyses) : TableStage = CanLowerEfficiently(ctx, inputIR) match { case Some(failReason) => diff --git a/hail/hail/src/is/hail/compatibility/LegacyBufferSpecs.scala b/hail/hail/src/is/hail/compatibility/LegacyBufferSpecs.scala index 95d595ac6b0..d412782cd99 100644 --- a/hail/hail/src/is/hail/compatibility/LegacyBufferSpecs.scala +++ b/hail/hail/src/is/hail/compatibility/LegacyBufferSpecs.scala @@ -6,7 +6,7 @@ import is.hail.io.compress.LZ4 final case class LZ4BlockBufferSpec(blockSize: Int, child: BlockBufferSpec) extends LZ4BlockBufferSpecCommon { - def lz4 = LZ4.hc - def stagedlz4: Code[LZ4] = Code.invokeScalaObject0[LZ4](LZ4.getClass, "hc") - def typeName = "LZ4BlockBufferSpec" + override def lz4 = LZ4.hc + override def stagedlz4: Code[LZ4] = Code.invokeScalaObject0[LZ4](LZ4.getClass, "hc") + override def typeName = "LZ4BlockBufferSpec" } diff --git a/hail/hail/src/is/hail/compatibility/LegacyRVDSpecs.scala b/hail/hail/src/is/hail/compatibility/LegacyRVDSpecs.scala index 26366b0293f..0e5cc7aca53 100644 --- a/hail/hail/src/is/hail/compatibility/LegacyRVDSpecs.scala +++ b/hail/hail/src/is/hail/compatibility/LegacyRVDSpecs.scala @@ -109,7 +109,7 @@ trait ShimRVDSpec extends AbstractRVDSpec { val shim: AbstractRVDSpec - final def key: IndexedSeq[String] = shim.key + final override def key: IndexedSeq[String] = shim.key override def partitioner(sm: HailStateManager): RVDPartitioner = shim.partitioner(sm) @@ -148,12 +148,12 @@ case class UnpartitionedRVDSpec private ( ) extends AbstractRVDSpec { private val (rowVType: TStruct, rowEType) = LegacyEncodedTypeParser.parseTypeAndEType(rowType) - def partitioner(sm: HailStateManager): RVDPartitioner = + override def partitioner(sm: HailStateManager): RVDPartitioner = RVDPartitioner.unkeyed(sm, partFiles.length) - def key: IndexedSeq[String] = FastSeq() + override def key: IndexedSeq[String] = FastSeq() - def typedCodecSpec: AbstractTypedCodecSpec = + override def typedCodecSpec: AbstractTypedCodecSpec = TypedCodecSpec(rowEType.setRequired(true), rowVType, codecSpec.child) val attrs: Map[String, String] = Map.empty @@ -167,9 +167,9 @@ case class OrderedRVDSpec private ( ) extends AbstractRVDSpec { private val lRvdType = LegacyEncodedTypeParser.parseLegacyRVDType(rvdType) - def key: IndexedSeq[String] = lRvdType.key + override def key: IndexedSeq[String] = lRvdType.key - def partitioner(sm: HailStateManager): RVDPartitioner = { + override def partitioner(sm: HailStateManager): RVDPartitioner = { val rangeBoundsType = TArray(TInterval(lRvdType.keyType)) new RVDPartitioner( sm, diff --git a/hail/hail/src/is/hail/cxx/RegionValueIterator.scala b/hail/hail/src/is/hail/cxx/RegionValueIterator.scala index 9b4cc37f4cd..d5c1ff22760 100644 --- a/hail/hail/src/is/hail/cxx/RegionValueIterator.scala +++ b/hail/hail/src/is/hail/cxx/RegionValueIterator.scala @@ -4,7 +4,7 @@ import is.hail.annotations.RegionValue class RegionValueIterator(it: Iterator[RegionValue]) extends Iterator[Long] { - def next(): Long = it.next().offset + override def next(): Long = it.next().offset - def hasNext: Boolean = it.hasNext + override def hasNext: Boolean = it.hasNext } diff --git a/hail/hail/src/is/hail/experimental/ExperimentalFunctions.scala b/hail/hail/src/is/hail/experimental/ExperimentalFunctions.scala index e19680bbdc7..ff2c891ece8 100644 --- a/hail/hail/src/is/hail/experimental/ExperimentalFunctions.scala +++ b/hail/hail/src/is/hail/experimental/ExperimentalFunctions.scala @@ -8,7 +8,7 @@ import is.hail.types.virtual.{TArray, TFloat64, TInt32, Type} object ExperimentalFunctions extends RegistryFunctions { - def registerAll(): Unit = { + override def registerAll(): Unit = { val experimentalPackageClass = Class.forName("is.hail.experimental.package$") registerScalaFunction( diff --git a/hail/hail/src/is/hail/expr/Parser.scala b/hail/hail/src/is/hail/expr/Parser.scala index 24e234507a1..8d764ec38ac 100644 --- a/hail/hail/src/is/hail/expr/Parser.scala +++ b/hail/hail/src/is/hail/expr/Parser.scala @@ -86,7 +86,7 @@ object Parser extends JavaTokenParsers { def oneOfLiteral(a: IndexedSeq[String]): Parser[String] = new Parser[String] { private[this] val root = ParseTrieNode.generate(a) - def apply(in: Input): ParseResult[String] = { + override def apply(in: Input): ParseResult[String] = { var _in = in var node = root diff --git a/hail/hail/src/is/hail/expr/ir/AbstractMatrixTableSpec.scala b/hail/hail/src/is/hail/expr/ir/AbstractMatrixTableSpec.scala index e1fcbae7dbb..600434e33dc 100644 --- a/hail/hail/src/is/hail/expr/ir/AbstractMatrixTableSpec.scala +++ b/hail/hail/src/is/hail/expr/ir/AbstractMatrixTableSpec.scala @@ -154,7 +154,7 @@ abstract class AbstractMatrixTableSpec extends RelationalSpec { def entriesSpec: AbstractTableSpec - def indexed: Boolean = rowsSpec.indexed + override def indexed: Boolean = rowsSpec.indexed } object MatrixTableSpec { @@ -219,17 +219,17 @@ class MatrixTableSpec( val rowsSpec: AbstractTableSpec, val entriesSpec: AbstractTableSpec, ) extends AbstractMatrixTableSpec { - def references_rel_path: String = params.references_rel_path + override def references_rel_path: String = params.references_rel_path - def file_version: Int = params.file_version + override def file_version: Int = params.file_version - def hail_version: String = params.hail_version + override def hail_version: String = params.hail_version - def matrix_type: MatrixType = params.matrix_type + override def matrix_type: MatrixType = params.matrix_type - def components: Map[String, ComponentSpec] = params.components + override def components: Map[String, ComponentSpec] = params.components - def toJValue: JValue = + override def toJValue: JValue = decomposeWithName(params, "MatrixTableSpec")(RelationalSpec.formats) } diff --git a/hail/hail/src/is/hail/expr/ir/AbstractTableSpec.scala b/hail/hail/src/is/hail/expr/ir/AbstractTableSpec.scala index 4bf3e1f9205..eaa382c95d5 100644 --- a/hail/hail/src/is/hail/expr/ir/AbstractTableSpec.scala +++ b/hail/hail/src/is/hail/expr/ir/AbstractTableSpec.scala @@ -29,13 +29,13 @@ sealed abstract class SortOrder { } case object Ascending extends SortOrder { - def serialize: Byte = 0.toByte - def parsableString(): String = "A" + override def serialize: Byte = 0.toByte + override def parsableString(): String = "A" } case object Descending extends SortOrder { - def serialize: Byte = 1.toByte - def parsableString(): String = "D" + override def serialize: Byte = 1.toByte + override def parsableString(): String = "D" } object SortField { @@ -59,7 +59,7 @@ abstract class AbstractTableSpec extends RelationalSpec { def globalsSpec: AbstractRVDSpec - def indexed: Boolean = rowsSpec.indexed + override def indexed: Boolean = rowsSpec.indexed } object TableSpec { @@ -99,16 +99,16 @@ class TableSpec( val globalsSpec: AbstractRVDSpec, val rowsSpec: AbstractRVDSpec, ) extends AbstractTableSpec { - def file_version: Int = params.file_version + override def file_version: Int = params.file_version - def hail_version: String = params.hail_version + override def hail_version: String = params.hail_version - def components: Map[String, ComponentSpec] = params.components + override def components: Map[String, ComponentSpec] = params.components - def references_rel_path: String = params.references_rel_path + override def references_rel_path: String = params.references_rel_path - def table_type: TableType = params.table_type + override def table_type: TableType = params.table_type - def toJValue: JValue = + override def toJValue: JValue = decomposeWithName(params, "TableSpec")(RelationalSpec.formats) } diff --git a/hail/hail/src/is/hail/expr/ir/BinarySearch.scala b/hail/hail/src/is/hail/expr/ir/BinarySearch.scala index aa99b6cbee7..beb10d84345 100644 --- a/hail/hail/src/is/hail/expr/ir/BinarySearch.scala +++ b/hail/hail/src/is/hail/expr/ir/BinarySearch.scala @@ -12,7 +12,7 @@ object BinarySearch { ltNeedle: IEmitCode => Code[Boolean], gtNeedle: IEmitCode => Code[Boolean], ): Comparator = new Comparator { - def apply( + override def apply( cb: EmitCodeBuilder, elt: IEmitCode, ifLtNeedle: => Unit, @@ -29,7 +29,7 @@ object BinarySearch { } def fromCompare(compare: IEmitCode => Value[Int]): Comparator = new Comparator { - def apply( + override def apply( cb: EmitCodeBuilder, elt: IEmitCode, ifLtNeedle: => Unit, @@ -42,7 +42,7 @@ object BinarySearch { } def fromPred(pred: IEmitCode => Code[Boolean]): Comparator = new Comparator { - def apply( + override def apply( cb: EmitCodeBuilder, elt: IEmitCode, ifLtNeedle: => Unit, diff --git a/hail/hail/src/is/hail/expr/ir/BlockMatrixIR.scala b/hail/hail/src/is/hail/expr/ir/BlockMatrixIR.scala index 7b58a58b8ea..5b47d3f46da 100644 --- a/hail/hail/src/is/hail/expr/ir/BlockMatrixIR.scala +++ b/hail/hail/src/is/hail/expr/ir/BlockMatrixIR.scala @@ -46,7 +46,7 @@ object BlockMatrixIR { } sealed abstract class BlockMatrixIR extends BaseIR { - def typ: BlockMatrixType + override def typ: BlockMatrixType protected[ir] def execute(ctx: ExecuteContext): BlockMatrix = fatal("tried to execute unexecutable IR:\n" + Pretty(ctx, this)) @@ -126,7 +126,7 @@ class BlockMatrixNativeReader( val params: BlockMatrixNativeReaderParameters, val metadata: BlockMatrixMetadata, ) extends BlockMatrixReader { - def pathsUsed: Seq[String] = Array(params.path) + override def pathsUsed: Seq[String] = Array(params.path) lazy val fullType: BlockMatrixType = { val sparsity = MatrixSparsity.fromLinearCoords( @@ -137,7 +137,7 @@ class BlockMatrixNativeReader( BlockMatrixType(TFloat64, metadata.nRows, metadata.nCols, metadata.blockSize, sparsity) } - def apply(ctx: ExecuteContext): BlockMatrix = { + override def apply(ctx: ExecuteContext): BlockMatrix = { val key = ("BlockMatrixNativeReader.apply", params.path) if (ctx.memo.contains(key)) { ctx.memo(key).asInstanceOf[BlockMatrix] @@ -195,7 +195,7 @@ class BlockMatrixNativeReader( case class BlockMatrixBinaryReader(path: String, shape: IndexedSeq[Long], blockSize: Int) extends BlockMatrixReader { - def pathsUsed: Seq[String] = Array(path) + override def pathsUsed: Seq[String] = Array(path) val IndexedSeq(nRows, nCols) = shape BlockMatrixIR.checkFitsIntoArray(nRows, nCols) @@ -203,7 +203,7 @@ case class BlockMatrixBinaryReader(path: String, shape: IndexedSeq[Long], blockS lazy val fullType: BlockMatrixType = BlockMatrixType.dense(TFloat64, nRows, nCols, blockSize) - def apply(ctx: ExecuteContext): BlockMatrix = { + override def apply(ctx: ExecuteContext): BlockMatrix = { val breezeMatrix = RichDenseMatrixDouble.importFromDoubles( ctx.fs, @@ -258,9 +258,9 @@ object BlockMatrixPersistReader { } case class BlockMatrixPersistReader(id: String, typ: BlockMatrixType) extends BlockMatrixReader { - def pathsUsed: Seq[String] = FastSeq() + override def pathsUsed: Seq[String] = FastSeq() lazy val fullType: BlockMatrixType = typ - def apply(ctx: ExecuteContext): BlockMatrix = ctx.BlockMatrixCache(id) + override def apply(ctx: ExecuteContext): BlockMatrix = ctx.BlockMatrixCache(id) } case class BlockMatrixMap(child: BlockMatrixIR, eltName: Name, f: IR, needsDense: Boolean) @@ -368,23 +368,23 @@ abstract class SparsityStrategy { } case object UnionBlocks extends SparsityStrategy { - def exists(leftBlock: Boolean, rightBlock: Boolean): Boolean = leftBlock || rightBlock + override def exists(leftBlock: Boolean, rightBlock: Boolean): Boolean = leftBlock || rightBlock - def mergeSparsity(left: MatrixSparsity, right: MatrixSparsity): MatrixSparsity = + override def mergeSparsity(left: MatrixSparsity, right: MatrixSparsity): MatrixSparsity = MatrixSparsity.union(left, right) } case object IntersectionBlocks extends SparsityStrategy { - def exists(leftBlock: Boolean, rightBlock: Boolean): Boolean = leftBlock && rightBlock + override def exists(leftBlock: Boolean, rightBlock: Boolean): Boolean = leftBlock && rightBlock - def mergeSparsity(left: MatrixSparsity, right: MatrixSparsity): MatrixSparsity = + override def mergeSparsity(left: MatrixSparsity, right: MatrixSparsity): MatrixSparsity = MatrixSparsity.intersection(left, right) } case object NeedsDense extends SparsityStrategy { - def exists(leftBlock: Boolean, rightBlock: Boolean): Boolean = true + override def exists(leftBlock: Boolean, rightBlock: Boolean): Boolean = true - def mergeSparsity(left: MatrixSparsity, right: MatrixSparsity): MatrixSparsity = { + override def mergeSparsity(left: MatrixSparsity, right: MatrixSparsity): MatrixSparsity = { assert(left.nRows == right.nRows && left.nCols == right.nCols) assert(!left.isSparse && !right.isSparse) left @@ -782,7 +782,7 @@ case class BlockMatrixFilter( case class BlockMatrixDensify(child: BlockMatrixIR) extends BlockMatrixIR { override lazy val typ: BlockMatrixType = child.typ.densify - def blockCostIsLinear: Boolean = child.blockCostIsLinear + override def blockCostIsLinear: Boolean = child.blockCostIsLinear val childrenSeq: IndexedSeq[BaseIR] = FastSeq(child) @@ -806,7 +806,7 @@ sealed abstract class BlockMatrixSparsifier { case class BandSparsifier(blocksOnly: Boolean, l: Long, u: Long) extends BlockMatrixSparsifier { val typ: Type = TTuple(TInt64, TInt64) - def definedBlocks(childType: BlockMatrixType): MatrixSparsity = { + override def definedBlocks(childType: BlockMatrixType): MatrixSparsity = { val lowerBlock = java.lang.Math.floorDiv(l, childType.blockSize).toInt val upperBlock = java.lang.Math.floorDiv(u + childType.blockSize - 1, childType.blockSize).toInt @@ -826,7 +826,7 @@ case class BandSparsifier(blocksOnly: Boolean, l: Long, u: Long) extends BlockMa override def sparsify(ctx: ExecuteContext, bm: BlockMatrix): BlockMatrix = bm.filterBand(l, u, blocksOnly) - def pretty(): String = + override def pretty(): String = s"(BandSparsifier ${Pretty.prettyBooleanLiteral(blocksOnly)} $l $u)" } @@ -838,7 +838,7 @@ case class RowIntervalSparsifier( ) extends BlockMatrixSparsifier { val typ: Type = TTuple(TArray(TInt64), TArray(TInt64)) - def definedBlocks(childType: BlockMatrixType): MatrixSparsity = { + override def definedBlocks(childType: BlockMatrixType): MatrixSparsity = { val blockStarts = starts.grouped(childType.blockSize).map(idxs => childType.getBlockIdx(idxs.min)).toArray val blockStops = @@ -852,7 +852,7 @@ case class RowIntervalSparsifier( override def sparsify(ctx: ExecuteContext, bm: BlockMatrix): BlockMatrix = bm.filterRowIntervals(ctx, starts.toArray, stops.toArray, blocksOnly) - def pretty(): String = + override def pretty(): String = s"(RowIntervalSparsifier ${Pretty.prettyBooleanLiteral(blocksOnly)} ${starts.mkString("(", " ", ")")} ${stops.mkString("(", " ", ")")})" } @@ -861,7 +861,7 @@ case class RectangleSparsifier(rectangles: IndexedSeq[IndexedSeq[Long]]) extends BlockMatrixSparsifier { val typ: Type = TArray(TInt64) - def definedBlocks(childType: BlockMatrixType): MatrixSparsity = { + override def definedBlocks(childType: BlockMatrixType): MatrixSparsity = { val definedBlocks = rectangles.flatMap { case IndexedSeq(rowStart, rowEnd, colStart, colEnd) => val rs = childType.getBlockIdx(java.lang.Math.max(rowStart, 0)) val re = childType.getBlockIdx(java.lang.Math.min(rowEnd - 1, childType.nRows)) + 1 @@ -880,7 +880,7 @@ case class RectangleSparsifier(rectangles: IndexedSeq[IndexedSeq[Long]]) override def sparsify(ctx: ExecuteContext, bm: BlockMatrix): BlockMatrix = bm.filterRectangles(rectangles.flatten.toArray) - def pretty(): String = + override def pretty(): String = s"(RectangleSparsifier ${rectangles.flatten.mkString("(", " ", ")")})" } @@ -908,7 +908,7 @@ case class BlockMatrixSparsify( override lazy val typ: BlockMatrixType = child.typ.copy(sparsity = sparsifier.definedBlocks(child.typ)) - def blockCostIsLinear: Boolean = child.blockCostIsLinear + override def blockCostIsLinear: Boolean = child.blockCostIsLinear val childrenSeq: IndexedSeq[BaseIR] = Array(child) diff --git a/hail/hail/src/is/hail/expr/ir/BlockMatrixWriter.scala b/hail/hail/src/is/hail/expr/ir/BlockMatrixWriter.scala index 4002cc9f4ea..7d60ae95f78 100644 --- a/hail/hail/src/is/hail/expr/ir/BlockMatrixWriter.scala +++ b/hail/hail/src/is/hail/expr/ir/BlockMatrixWriter.scala @@ -56,12 +56,12 @@ case class BlockMatrixNativeWriter( forceRowMajor: Boolean, stageLocally: Boolean, ) extends BlockMatrixWriter { - def pathOpt: Option[String] = Some(path) + override def pathOpt: Option[String] = Some(path) - def apply(ctx: ExecuteContext, bm: BlockMatrix): Unit = + override def apply(ctx: ExecuteContext, bm: BlockMatrix): Unit = bm.write(ctx, path, overwrite, forceRowMajor, stageLocally) - def loweredTyp: Type = TVoid + override def loweredTyp: Type = TVoid override def lower( ctx: ExecuteContext, @@ -128,9 +128,9 @@ case class BlockMatrixNativeMetadataWriter( } } - def annotationType: Type = TArray(TString) + override def annotationType: Type = TArray(TString) - def writeMetadata( + override def writeMetadata( writeAnnotations: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region], @@ -164,9 +164,9 @@ case class BlockMatrixNativeMetadataWriter( } case class BlockMatrixBinaryWriter(path: String) extends BlockMatrixWriter { - def pathOpt: Option[String] = Some(path) + override def pathOpt: Option[String] = Some(path) - def apply(ctx: ExecuteContext, bm: BlockMatrix): String = { + override def apply(ctx: ExecuteContext, bm: BlockMatrix): String = { RichDenseMatrixDouble.exportToDoubles( ctx.fs, path, @@ -176,7 +176,7 @@ case class BlockMatrixBinaryWriter(path: String) extends BlockMatrixWriter { path } - def loweredTyp: Type = TString + override def loweredTyp: Type = TString override def lower( ctx: ExecuteContext, @@ -191,12 +191,12 @@ case class BlockMatrixBinaryWriter(path: String) extends BlockMatrixWriter { } case class BlockMatrixPersistWriter(id: String, storageLevel: String) extends BlockMatrixWriter { - def pathOpt: Option[String] = None + override def pathOpt: Option[String] = None - def apply(ctx: ExecuteContext, bm: BlockMatrix): Unit = + override def apply(ctx: ExecuteContext, bm: BlockMatrix): Unit = ctx.BlockMatrixCache += id -> bm.persist(storageLevel) - def loweredTyp: Type = TVoid + override def loweredTyp: Type = TVoid } case class BlockMatrixRectanglesWriter( @@ -206,12 +206,12 @@ case class BlockMatrixRectanglesWriter( binary: Boolean, ) extends BlockMatrixWriter { - def pathOpt: Option[String] = Some(path) + override def pathOpt: Option[String] = Some(path) - def apply(ctx: ExecuteContext, bm: BlockMatrix): Unit = + override def apply(ctx: ExecuteContext, bm: BlockMatrix): Unit = bm.exportRectangles(ctx, path, rectangles, delimiter, binary) - def loweredTyp: Type = TVoid + override def loweredTyp: Type = TVoid } abstract class BlockMatrixMultiWriter { @@ -223,7 +223,7 @@ case class BlockMatrixBinaryMultiWriter( overwrite: Boolean, ) extends BlockMatrixMultiWriter { - def apply(ctx: ExecuteContext, bms: IndexedSeq[BlockMatrix]): Unit = + override def apply(ctx: ExecuteContext, bms: IndexedSeq[BlockMatrix]): Unit = BlockMatrix.binaryWriteBlockMatrices(ctx, bms, prefix, overwrite) def loweredTyp: Type = TVoid @@ -239,7 +239,7 @@ case class BlockMatrixTextMultiWriter( customFilenames: Option[Array[String]], ) extends BlockMatrixMultiWriter { - def apply(ctx: ExecuteContext, bms: IndexedSeq[BlockMatrix]): Unit = + override def apply(ctx: ExecuteContext, bms: IndexedSeq[BlockMatrix]): Unit = BlockMatrix.exportBlockMatrices( ctx, bms, @@ -261,7 +261,7 @@ case class BlockMatrixNativeMultiWriter( forceRowMajor: Boolean, ) extends BlockMatrixMultiWriter { - def apply(ctx: ExecuteContext, bms: IndexedSeq[BlockMatrix]): Unit = + override def apply(ctx: ExecuteContext, bms: IndexedSeq[BlockMatrix]): Unit = BlockMatrix.writeBlockMatrices(ctx, bms, prefix, overwrite, forceRowMajor) def loweredTyp: Type = TVoid diff --git a/hail/hail/src/is/hail/expr/ir/Compile.scala b/hail/hail/src/is/hail/expr/ir/Compile.scala index 2a3b6369904..b60ee982a70 100644 --- a/hail/hail/src/is/hail/expr/ir/Compile.scala +++ b/hail/hail/src/is/hail/expr/ir/Compile.scala @@ -156,7 +156,7 @@ object CompileIterator { private var _stepped = false private var _hasNext = false - def hasNext: Boolean = { + override def hasNext: Boolean = { if (!_stepped) { _hasNext = step() _stepped = true @@ -164,7 +164,7 @@ object CompileIterator { _hasNext } - def next(): java.lang.Long = { + override def next(): java.lang.Long = { if (!hasNext) Iterator.empty.next(): Unit // throw _stepped = false stepFunction.loadAddress() @@ -309,7 +309,7 @@ object CompileIterator { new LongIteratorWrapper { val stepFunction: TMPStepFunction = outerStepFunction - def step(): Boolean = stepFunction.apply(null, v0, part) + override def step(): Boolean = stepFunction.apply(null, v0, part) } }, ) @@ -345,7 +345,7 @@ object CompileIterator { new LongIteratorWrapper { val stepFunction: TableStageToRVDStepFunction = outerStepFunction - def step(): Boolean = stepFunction.apply(null, v0, v1) + override def step(): Boolean = stepFunction.apply(null, v0, v1) } }, ) diff --git a/hail/hail/src/is/hail/expr/ir/Emit.scala b/hail/hail/src/is/hail/expr/ir/Emit.scala index 26929b62e02..a2dd8288548 100644 --- a/hail/hail/src/is/hail/expr/ir/Emit.scala +++ b/hail/hail/src/is/hail/expr/ir/Emit.scala @@ -2498,7 +2498,7 @@ class Emit[C](val ctx: EmitContext, val cb: EmitClassBuilder[C]) { val M = shapeArray(0) val N = shapeArray(1) val K = new Value[Long] { - def get: Code[Long] = (M < N).mux(M, N) + override def get: Code[Long] = (M < N).mux(M, N) } val LDA = new Value[Long] { override def get: Code[Long] = @@ -3746,7 +3746,7 @@ object NDArrayEmitter { val notBroadcasted = 1L Array.tabulate(nDims)(dim => new Value[Long] { - def get: Code[Long] = + override def get: Code[Long] = (shapeArray(dim) > 1L).mux(notBroadcasted, broadcasted) * loopVars(dim) } ) @@ -3757,7 +3757,7 @@ object NDArrayEmitter { val notBroadcasted = 1L shapeArray.map(shapeElement => new Value[Long] { - def get: Code[Long] = (shapeElement > 1L).mux(notBroadcasted, broadcasted) + override def get: Code[Long] = (shapeElement > 1L).mux(notBroadcasted, broadcasted) } ) } @@ -3766,7 +3766,7 @@ object NDArrayEmitter { : IndexedSeq[Value[Long]] = indices.zip(broadcastMask).map { case (index, flag) => new Value[Long] { - def get: Code[Long] = index * flag + override def get: Code[Long] = index * flag } } diff --git a/hail/hail/src/is/hail/expr/ir/EmitClassBuilder.scala b/hail/hail/src/is/hail/expr/ir/EmitClassBuilder.scala index 3557da46bc4..463baac0f74 100644 --- a/hail/hail/src/is/hail/expr/ir/EmitClassBuilder.scala +++ b/hail/hail/src/is/hail/expr/ir/EmitClassBuilder.scala @@ -135,7 +135,7 @@ trait WrappedEmitModuleBuilder { trait WrappedEmitClassBuilder[C] extends WrappedEmitModuleBuilder { def ecb: EmitClassBuilder[C] - def emodb: EmitModuleBuilder = ecb.emodb + override def emodb: EmitModuleBuilder = ecb.emodb def cb: ClassBuilder[C] = ecb.cb @@ -944,7 +944,8 @@ final class EmitClassBuilder[C](val emodb: EmitModuleBuilder, val cb: ClassBuild new ((HailClassLoader, FS, HailTaskContext, Region) => C) with java.io.Serializable { @transient @volatile private var theClass: Class[_] = null - def apply(hcl: HailClassLoader, fs: FS, htc: HailTaskContext, region: Region): C = { + override def apply(hcl: HailClassLoader, fs: FS, htc: HailTaskContext, region: Region) + : C = { if (theClass == null) { this.synchronized { if (theClass == null) { @@ -1438,7 +1439,7 @@ class EmitMethodBuilder[C]( trait WrappedEmitMethodBuilder[C] extends WrappedEmitClassBuilder[C] { def emb: EmitMethodBuilder[C] - def ecb: EmitClassBuilder[C] = emb.ecb + override def ecb: EmitClassBuilder[C] = emb.ecb // wrapped MethodBuilder methods def mb: MethodBuilder[C] = emb.mb diff --git a/hail/hail/src/is/hail/expr/ir/EmitCodeBuilder.scala b/hail/hail/src/is/hail/expr/ir/EmitCodeBuilder.scala index c4f170be77a..840f1424828 100644 --- a/hail/hail/src/is/hail/expr/ir/EmitCodeBuilder.scala +++ b/hail/hail/src/is/hail/expr/ir/EmitCodeBuilder.scala @@ -37,10 +37,10 @@ object EmitCodeBuilder { } class EmitCodeBuilder(val emb: EmitMethodBuilder[_], var code: Code[Unit]) extends CodeBuilderLike { - def isOpenEnded: Boolean = + override def isOpenEnded: Boolean = code.isOpenEnded - def mb: MethodBuilder[_] = emb.mb + override def mb: MethodBuilder[_] = emb.mb override def append(c: Code[Unit]): Unit = code = Code(code, c) @@ -54,7 +54,7 @@ class EmitCodeBuilder(val emb: EmitMethodBuilder[_], var code: Code[Unit]) exten L.clear() } - def result(): Code[Unit] = { + override def result(): Code[Unit] = { val tmp = code code = Code._empty tmp diff --git a/hail/hail/src/is/hail/expr/ir/Env.scala b/hail/hail/src/is/hail/expr/ir/Env.scala index d4fd11e8d56..2abf566f083 100644 --- a/hail/hail/src/is/hail/expr/ir/Env.scala +++ b/hail/hail/src/is/hail/expr/ir/Env.scala @@ -99,7 +99,7 @@ case class BindingEnv[V]( newEnv } - def extend(bindings: Bindings[V]): BindingEnv[V] = { + override def extend(bindings: Bindings[V]): BindingEnv[V] = { val Bindings(all, eval, agg, scan, relational, _) = bindings var newEnv = modifyWithoutNewBindings(bindings) if (all.nonEmpty) { @@ -150,9 +150,9 @@ case class BindingEnv[V]( def allEmpty: Boolean = eval.isEmpty && agg.forall(_.isEmpty) && scan.forall(_.isEmpty) && relational.isEmpty - def promoteAgg: BindingEnv[V] = copy(eval = agg.get, agg = None) + override def promoteAgg: BindingEnv[V] = copy(eval = agg.get, agg = None) - def promoteScan: BindingEnv[V] = copy(eval = scan.get, scan = None) + override def promoteScan: BindingEnv[V] = copy(eval = scan.get, scan = None) def promoteScope(scope: Int): BindingEnv[V] = scope match { case Scope.EVAL => this @@ -160,17 +160,17 @@ case class BindingEnv[V]( case Scope.SCAN => promoteScan } - def noAgg: BindingEnv[V] = copy(agg = None) + override def noAgg: BindingEnv[V] = copy(agg = None) - def noScan: BindingEnv[V] = copy(scan = None) + override def noScan: BindingEnv[V] = copy(scan = None) - def createAgg: BindingEnv[V] = + override def createAgg: BindingEnv[V] = copy(agg = Some(eval), scan = scan.map(_ => Env.empty)) - def createScan: BindingEnv[V] = + override def createScan: BindingEnv[V] = copy(scan = Some(eval), agg = agg.map(_ => Env.empty)) - def onlyRelational(keepAggCapabilities: Boolean = false): BindingEnv[V] = + override def onlyRelational(keepAggCapabilities: Boolean = false): BindingEnv[V] = BindingEnv( agg = if (keepAggCapabilities) agg.map(_ => Env.empty) else None, scan = if (keepAggCapabilities) scan.map(_ => Env.empty) else None, @@ -180,18 +180,18 @@ case class BindingEnv[V]( def bindEval(name: Name, v: V): BindingEnv[V] = copy(eval = eval.bind(name, v)) - def bindEval(bindings: (Name, V)*): BindingEnv[V] = + override def bindEval(bindings: (Name, V)*): BindingEnv[V] = copy(eval = eval.bindIterable(bindings)) def deleteEval(name: Name): BindingEnv[V] = copy(eval = eval.delete(name)) def deleteEval(names: IndexedSeq[Name]): BindingEnv[V] = copy(eval = eval.delete(names)) - def noEval: BindingEnv[V] = copy(eval = Env.empty) + override def noEval: BindingEnv[V] = copy(eval = Env.empty) def bindAgg(name: Name, v: V): BindingEnv[V] = copy(agg = Some(agg.get.bind(name, v))) - def bindAgg(bindings: (Name, V)*): BindingEnv[V] = + override def bindAgg(bindings: (Name, V)*): BindingEnv[V] = copy(agg = Some(agg.get.bindIterable(bindings))) def aggOrEmpty: Env[V] = agg.getOrElse(Env.empty) @@ -199,13 +199,13 @@ case class BindingEnv[V]( def bindScan(name: Name, v: V): BindingEnv[V] = copy(scan = Some(scan.get.bind(name, v))) - def bindScan(bindings: (Name, V)*): BindingEnv[V] = + override def bindScan(bindings: (Name, V)*): BindingEnv[V] = copy(scan = Some(scan.get.bindIterable(bindings))) def bindRelational(name: Name, v: V): BindingEnv[V] = copy(relational = relational.bind(name, v)) - def bindRelational(bindings: (Name, V)*): BindingEnv[V] = + override def bindRelational(bindings: (Name, V)*): BindingEnv[V] = copy(relational = relational.bind(bindings: _*)) def scanOrEmpty: Env[V] = scan.getOrElse(Env.empty) diff --git a/hail/hail/src/is/hail/expr/ir/ExtractIntervalFilters.scala b/hail/hail/src/is/hail/expr/ir/ExtractIntervalFilters.scala index ec4e984577d..f7ddadfdf01 100644 --- a/hail/hail/src/is/hail/expr/ir/ExtractIntervalFilters.scala +++ b/hail/hail/src/is/hail/expr/ir/ExtractIntervalFilters.scala @@ -278,11 +278,11 @@ class ExtractIntervalFilters(ctx: ExecuteContext, keyType: TStruct) extends Logg // booleans derived from key comparisons), we don't store top valued fields, // and let all missing fields be top implicitly. private case class ConcreteStruct(fields: Map[String, Value]) extends StructValue { - def apply(field: String): Value = fields.getOrElse(field, top) + override def apply(field: String): Value = fields.getOrElse(field, top) - def values: Iterable[Value] = fields.values + override def values: Iterable[Value] = fields.values - def isKeyPrefix: Boolean = fields.values.view.zipWithIndex.forall { + override def isKeyPrefix: Boolean = fields.values.view.zipWithIndex.forall { case (f: KeyField, i2) => f.idx == i2 case _ => false } @@ -505,10 +505,10 @@ class ExtractIntervalFilters(ctx: ExecuteContext, keyType: TStruct) extends Logg private case class ConstantStruct(value: Row, t: TStruct) extends StructValue with ConstantValue { - def apply(field: String): Value = this(t.field(field)) - def values: Iterable[Value] = t.fields.map(apply) + override def apply(field: String): Value = this(t.field(field)) + override def values: Iterable[Value] = t.fields.map(apply) private def apply(field: Field): ConstantValue = ConstantValue(value(field.index), field.typ) - def isKeyPrefix: Boolean = false + override def isKeyPrefix: Boolean = false } private case class ConstantBool(value: Boolean, keySet: KeySet) @@ -546,15 +546,15 @@ class ExtractIntervalFilters(ctx: ExecuteContext, keyType: TStruct) extends Logg } private case class KeyFieldStruct(idx: Int) extends StructValue with KeyField { - def apply(field: String): Value = Top - def values: Iterable[Value] = Iterable.empty - def isKeyPrefix: Boolean = false + override def apply(field: String): Value = Top + override def values: Iterable[Value] = Iterable.empty + override def isKeyPrefix: Boolean = false } private case object Top extends StructValue with BoolValue { - def apply(field: String): Value = Top - def values: Iterable[Value] = Iterable.empty - def isKeyPrefix: Boolean = false + override def apply(field: String): Value = Top + override def values: Iterable[Value] = Iterable.empty + override def isKeyPrefix: Boolean = false override def trueBound: KeySet = KeySetLattice.top override def falseBound: KeySet = KeySetLattice.top override def naBound: KeySet = KeySetLattice.top @@ -563,9 +563,9 @@ class ExtractIntervalFilters(ctx: ExecuteContext, keyType: TStruct) extends Logg } private case object Bottom extends StructValue with BoolValue { - def apply(field: String): Value = ??? - def values: Iterable[Value] = ??? - def isKeyPrefix: Boolean = ??? + override def apply(field: String): Value = ??? + override def values: Iterable[Value] = ??? + override def isKeyPrefix: Boolean = ??? override def trueBound: KeySet = KeySetLattice.bottom override def falseBound: KeySet = KeySetLattice.bottom override def naBound: KeySet = KeySetLattice.bottom @@ -573,10 +573,10 @@ class ExtractIntervalFilters(ctx: ExecuteContext, keyType: TStruct) extends Logg override def isNA: BoolValue = Bottom } - def top: StructValue with BoolValue = Top - def bottom: StructValue with BoolValue = Bottom + override def top: StructValue with BoolValue = Top + override def bottom: StructValue with BoolValue = Bottom - def join(l: Value, r: Value): Value = (l, r) match { + override def join(l: Value, r: Value): Value = (l, r) match { case (Bottom, x) => x case (x, Bottom) => x case (l: ConstantValue, r: ConstantValue) if l.value == r.value => l @@ -599,7 +599,7 @@ class ExtractIntervalFilters(ctx: ExecuteContext, keyType: TStruct) extends Logg case _ => Top } - def meet(l: Value, r: Value): Value = (l, r) match { + override def meet(l: Value, r: Value): Value = (l, r) match { case (Top, x) => x case (x, Top) => x case (l: ConstantValue, r: ConstantValue) if l.value == r.value => l diff --git a/hail/hail/src/is/hail/expr/ir/FreeVariables.scala b/hail/hail/src/is/hail/expr/ir/FreeVariables.scala index cf3b01e0f9d..82f57ff4b5a 100644 --- a/hail/hail/src/is/hail/expr/ir/FreeVariables.scala +++ b/hail/hail/src/is/hail/expr/ir/FreeVariables.scala @@ -27,7 +27,7 @@ case class FreeVariableBindingEnv( aggVars: Option[FreeVariableEnv], scanVars: Option[FreeVariableEnv], ) extends GenericBindingEnv[FreeVariableBindingEnv, Type] { - def extend(bindings: Bindings[Type]): FreeVariableBindingEnv = { + override def extend(bindings: Bindings[Type]): FreeVariableBindingEnv = { val Bindings(all, eval, agg, scan, relational, dropEval) = bindings var newEnv = this if (dropEval) newEnv = newEnv.noEval diff --git a/hail/hail/src/is/hail/expr/ir/GenericLines.scala b/hail/hail/src/is/hail/expr/ir/GenericLines.scala index c08a1fd4d10..e120907ad29 100644 --- a/hail/hail/src/is/hail/expr/ir/GenericLines.scala +++ b/hail/hail/src/is/hail/expr/ir/GenericLines.scala @@ -20,9 +20,9 @@ trait CloseableIterator[T] extends Iterator[T] with AutoCloseable object CloseableIterator { def empty[T]: CloseableIterator[T] = new CloseableIterator[T] { - def close(): Unit = () - def hasNext: Boolean = false - def next(): T = throw new NoSuchElementException + override def close(): Unit = () + override def hasNext: Boolean = false + override def next(): T = throw new NoSuchElementException } } @@ -54,7 +54,7 @@ object GenericLines extends Logging { val bgzIS = new BGzipInputStream(rawIS, start, end, SplittableCompressionCodec.READ_MODE.BYBLOCK) new ProxyInputStream(bgzIS) with Positioned { - def getPosition: Long = bgzIS.getVirtualOffset + override def getPosition: Long = bgzIS.getVirtualOffset } } else { assert(!split || filePerPartition) @@ -222,7 +222,7 @@ object GenericLines extends Logging { private var consumed = false - def hasNext: Boolean = { + override def hasNext: Boolean = { if (consumed) { readLine() consumed = false @@ -230,7 +230,7 @@ object GenericLines extends Logging { line != null } - def next(): GenericLine = { + override def next(): GenericLine = { if (consumed) readLine() assert(line != null) @@ -240,7 +240,7 @@ object GenericLines extends Logging { line } - def close(): Unit = + override def close(): Unit = if (!closed) { is.close() closed = true @@ -364,8 +364,8 @@ object GenericLines extends Logging { private[this] var l = lines.next() private[this] var curIdx: Long = lines.getCurIdx() private[this] val inner = new Iterator[GenericLine] { - def hasNext: Boolean = l != null - def next(): GenericLine = { + override def hasNext: Boolean = l != null + override def next(): GenericLine = { assert(l != null) val n = l val idx = curIdx @@ -390,9 +390,9 @@ object GenericLines extends Logging { start <= pos && pos <= end } - def hasNext: Boolean = inner.hasNext - def next(): GenericLine = inner.next() - def close(): Unit = lines.close() + override def hasNext: Boolean = inner.hasNext + override def next(): GenericLine = inner.next() + override def close(): Unit = lines.close() } } } @@ -458,12 +458,12 @@ class GenericLinesRDD( body: (Any) => CloseableIterator[GenericLine], ) extends RDD[GenericLine](SparkBackend.sparkContext, Seq()) { - protected def getPartitions: Array[Partition] = + override protected def getPartitions: Array[Partition] = contexts.iterator.zipWithIndex.map { case (c, i) => new GenericLinesRDDPartition(i, c) }.toArray - def compute(split: Partition, context: TaskContext): Iterator[GenericLine] = { + override def compute(split: Partition, context: TaskContext): Iterator[GenericLine] = { val it = body(split.asInstanceOf[GenericLinesRDDPartition].context) TaskContext.get().addTaskCompletionListener[Unit](_ => it.close()): Unit it diff --git a/hail/hail/src/is/hail/expr/ir/GenericTableValue.scala b/hail/hail/src/is/hail/expr/ir/GenericTableValue.scala index 7347df780c5..5082151133b 100644 --- a/hail/hail/src/is/hail/expr/ir/GenericTableValue.scala +++ b/hail/hail/src/is/hail/expr/ir/GenericTableValue.scala @@ -136,7 +136,7 @@ class PartitionIteratorLongReader( } } - def toJValue: JValue = + override def toJValue: JValue = JObject( "category" -> JString("PartitionIteratorLongReader"), "fullRowType" -> Extraction.decompose(fullRowType)(PartitionReader.formats), diff --git a/hail/hail/src/is/hail/expr/ir/IR.scala b/hail/hail/src/is/hail/expr/ir/IR.scala index 9341d1bb82a..4074a6a314b 100644 --- a/hail/hail/src/is/hail/expr/ir/IR.scala +++ b/hail/hail/src/is/hail/expr/ir/IR.scala @@ -401,11 +401,11 @@ package defs { } abstract class SimplePartitionWriter extends PartitionWriter { - def ctxType: Type = TString + override def ctxType: Type = TString - def returnType: Type = TString + override def returnType: Type = TString - def unionTypeRequiredness( + override def unionTypeRequiredness( r: TypeWithRequiredness, ctxType: TypeWithRequiredness, streamType: RIterable, @@ -425,7 +425,7 @@ package defs { def postConsume(cb: EmitCodeBuilder, os: Value[OutputStream]): Unit = () - final def consumeStream( + final override def consumeStream( ctx: ExecuteContext, cb: EmitCodeBuilder, stream: StreamProducer, @@ -463,8 +463,11 @@ package defs { } final case class SimpleMetadataWriter(val annotationType: Type) extends MetadataWriter { - def writeMetadata(writeAnnotations: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region]) - : Unit = + override def writeMetadata( + writeAnnotations: => IEmitCode, + cb: EmitCodeBuilder, + region: Value[Region], + ): Unit = writeAnnotations.consume(cb, {}, _ => ()) } diff --git a/hail/hail/src/is/hail/expr/ir/MatrixIR.scala b/hail/hail/src/is/hail/expr/ir/MatrixIR.scala index 87ec5e0cb7c..efb45ba0d61 100644 --- a/hail/hail/src/is/hail/expr/ir/MatrixIR.scala +++ b/hail/hail/src/is/hail/expr/ir/MatrixIR.scala @@ -51,7 +51,7 @@ object MatrixIR { } sealed abstract class MatrixIR extends BaseIR { - def typ: MatrixType + override def typ: MatrixType override protected def copyWithNewChildren(newChildren: IndexedSeq[BaseIR]): MatrixIR } @@ -261,7 +261,7 @@ class MatrixNativeReader( val params: MatrixNativeReaderParameters, spec: AbstractMatrixTableSpec, ) extends MatrixReader { - def pathsUsed: Seq[String] = FastSeq(params.path) + override def pathsUsed: Seq[String] = FastSeq(params.path) override def renderShort(): String = s"(MatrixNativeReader ${params.path} ${params.options.map(_.renderShort()).getOrElse("")})" @@ -271,13 +271,13 @@ class MatrixNativeReader( .sum .toInt) - def partitionCounts: Option[IndexedSeq[Long]] = + override def partitionCounts: Option[IndexedSeq[Long]] = if (params.options.isEmpty) Some(spec.partitionCounts) else None - def fullMatrixTypeWithoutUIDs: MatrixType = spec.matrix_type + override def fullMatrixTypeWithoutUIDs: MatrixType = spec.matrix_type - def rowUIDType = TTuple(TInt64, TInt64) - def colUIDType = TTuple(TInt64, TInt64) + override def rowUIDType = TTuple(TInt64, TInt64) + override def colUIDType = TTuple(TInt64, TInt64) override def lower( ctx: ExecuteContext, @@ -364,7 +364,7 @@ class MatrixNativeReader( } } - def toJValue: JValue = { + override def toJValue: JValue = { implicit val formats: Formats = DefaultFormats decomposeWithName(params, "MatrixNativeReader") } @@ -403,12 +403,12 @@ case class MatrixRangeReader( val params: MatrixRangeReaderParameters, nPartitionsAdj: Int, ) extends MatrixReader { - def pathsUsed: Seq[String] = FastSeq() + override def pathsUsed: Seq[String] = FastSeq() - def rowUIDType = TInt64 - def colUIDType = TInt64 + override def rowUIDType = TInt64 + override def colUIDType = TInt64 - def fullMatrixTypeWithoutUIDs: MatrixType = MatrixType( + override def fullMatrixTypeWithoutUIDs: MatrixType = MatrixType( globalType = TStruct.empty, colKey = Array("col_idx"), colType = TStruct("col_idx" -> TInt32), @@ -458,7 +458,7 @@ case class MatrixRangeReader( ht } - def toJValue: JValue = { + override def toJValue: JValue = { implicit val formats: Formats = DefaultFormats decomposeWithName(params, "MatrixRangeReader") } @@ -529,7 +529,7 @@ case class MatrixFilterCols(child: MatrixIR, pred: IR) extends MatrixIR with Pre MatrixFilterCols(newChildren(0).asInstanceOf[MatrixIR], newChildren(1).asInstanceOf[IR]) } - def typ: MatrixType = child.typ + override def typ: MatrixType = child.typ override def preservesRowsOrColsFrom: BaseIR = child } @@ -544,7 +544,7 @@ case class MatrixFilterRows(child: MatrixIR, pred: IR) MatrixFilterRows(newChildren(0).asInstanceOf[MatrixIR], newChildren(1).asInstanceOf[IR]) } - def typ: MatrixType = child.typ + override def typ: MatrixType = child.typ override def preservesRowsOrColsFrom: BaseIR = child } @@ -558,7 +558,7 @@ case class MatrixChooseCols(child: MatrixIR, oldIndices: IndexedSeq[Int]) MatrixChooseCols(newChildren(0).asInstanceOf[MatrixIR], oldIndices) } - def typ: MatrixType = child.typ + override def typ: MatrixType = child.typ override def preservesRowsOrColsFrom: BaseIR = child } @@ -750,7 +750,7 @@ case class MatrixFilterEntries(child: MatrixIR, pred: IR) MatrixFilterEntries(newChildren(0).asInstanceOf[MatrixIR], newChildren(1).asInstanceOf[IR]) } - def typ: MatrixType = child.typ + override def typ: MatrixType = child.typ override def preservesRowsOrColsFrom: BaseIR = child } @@ -829,7 +829,7 @@ case class MatrixExplodeRows(child: MatrixIR, path: IndexedSeq[String]) case class MatrixRepartition(child: MatrixIR, n: Int, strategy: Int) extends MatrixIR with PreservesRows with PreservesCols { - def typ: MatrixType = child.typ + override def typ: MatrixType = child.typ lazy val childrenSeq: IndexedSeq[BaseIR] = FastSeq(child) @@ -846,7 +846,7 @@ case class MatrixRepartition(child: MatrixIR, n: Int, strategy: Int) case class MatrixUnionRows(childrenSeq: IndexedSeq[MatrixIR]) extends MatrixIR { require(childrenSeq.length > 1) - def typ: MatrixType = childrenSeq.head.typ + override def typ: MatrixType = childrenSeq.head.typ override protected def copyWithNewChildren(newChildren: IndexedSeq[BaseIR]): MatrixUnionRows = MatrixUnionRows(newChildren.asInstanceOf[IndexedSeq[MatrixIR]]) @@ -854,7 +854,7 @@ case class MatrixUnionRows(childrenSeq: IndexedSeq[MatrixIR]) extends MatrixIR { case class MatrixDistinctByRow(child: MatrixIR) extends MatrixIR with PreservesOrRemovesRows with PreservesCols { - def typ: MatrixType = child.typ + override def typ: MatrixType = child.typ lazy val childrenSeq: IndexedSeq[BaseIR] = FastSeq(child) @@ -869,7 +869,7 @@ case class MatrixDistinctByRow(child: MatrixIR) case class MatrixRowsHead(child: MatrixIR, n: Long) extends MatrixIR with PreservesCols { require(n >= 0) - def typ: MatrixType = child.typ + override def typ: MatrixType = child.typ lazy val childrenSeq: IndexedSeq[BaseIR] = Array(child) @@ -883,7 +883,7 @@ case class MatrixRowsHead(child: MatrixIR, n: Long) extends MatrixIR with Preser case class MatrixColsHead(child: MatrixIR, n: Int) extends MatrixIR with PreservesRows { require(n >= 0) - def typ: MatrixType = child.typ + override def typ: MatrixType = child.typ lazy val childrenSeq: IndexedSeq[BaseIR] = Array(child) @@ -897,7 +897,7 @@ case class MatrixColsHead(child: MatrixIR, n: Int) extends MatrixIR with Preserv case class MatrixRowsTail(child: MatrixIR, n: Long) extends MatrixIR with PreservesCols { require(n >= 0) - def typ: MatrixType = child.typ + override def typ: MatrixType = child.typ lazy val childrenSeq: IndexedSeq[BaseIR] = Array(child) @@ -911,7 +911,7 @@ case class MatrixRowsTail(child: MatrixIR, n: Long) extends MatrixIR with Preser case class MatrixColsTail(child: MatrixIR, n: Int) extends MatrixIR with PreservesRows { require(n >= 0) - def typ: MatrixType = child.typ + override def typ: MatrixType = child.typ lazy val childrenSeq: IndexedSeq[BaseIR] = Array(child) @@ -1032,9 +1032,9 @@ case class MatrixFilterIntervals(child: MatrixIR, intervals: IndexedSeq[Interval case class RelationalLetMatrixTable(name: Name, value: IR, body: MatrixIR) extends MatrixIR with PreservesRows { - def typ: MatrixType = body.typ + override def typ: MatrixType = body.typ - def childrenSeq: IndexedSeq[BaseIR] = Array(value, body) + override def childrenSeq: IndexedSeq[BaseIR] = Array(value, body) override protected def copyWithNewChildren(newChildren: IndexedSeq[BaseIR]): MatrixIR = { val IndexedSeq(newValue: IR, newBody: MatrixIR) = newChildren diff --git a/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala b/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala index 2d1f11c573a..6ea72091372 100644 --- a/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala +++ b/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala @@ -61,7 +61,7 @@ case class WrappedMatrixWriter( entriesFieldName: String, colKey: IndexedSeq[String], ) extends TableWriter { - def path: String = writer.path + override def path: String = writer.path override def lower(ctx: ExecuteContext, ts: TableStage, r: RTable): IR = writer.lower(colsFieldName, entriesFieldName, colKey, ctx, ts, r) @@ -409,9 +409,9 @@ case class SplitPartitionNativeWriter( val keyType = spec1.encodedVirtualType.asInstanceOf[TStruct].select(keyFieldNames)._1 - def ctxType: Type = TString + override def ctxType: Type = TString - def returnType: Type = TStruct( + override def returnType: Type = TStruct( "filePath" -> TString, "partitionCounts" -> TInt64, "distinctlyKeyed" -> TBoolean, @@ -419,7 +419,7 @@ case class SplitPartitionNativeWriter( "lastKey" -> keyType, ) - def unionTypeRequiredness( + override def unionTypeRequiredness( r: TypeWithRequiredness, ctxType: TypeWithRequiredness, streamType: RIterable, @@ -434,7 +434,7 @@ case class SplitPartitionNativeWriter( r.union(streamType.required) } - def consumeStream( + override def consumeStream( ctx: ExecuteContext, cb: EmitCodeBuilder, stream: StreamProducer, @@ -686,9 +686,9 @@ case class MatrixSpecWriter( refRelPath: String, log: Boolean, ) extends MetadataWriter { - def annotationType: Type = TStruct("cols" -> TInt64, "rows" -> TArray(TInt64)) + override def annotationType: Type = TStruct("cols" -> TInt64, "rows" -> TArray(TInt64)) - def writeMetadata( + override def writeMetadata( writeAnnotations: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region], @@ -847,9 +847,9 @@ case class VCFPartitionWriter( val (infoExists, infoIdx) = ExportVCF.lookupVAField(typ.rowType, "info", "INFO", None) - def returnType: Type = TString + override def returnType: Type = TString - def unionTypeRequiredness( + override def unionTypeRequiredness( r: TypeWithRequiredness, ctxType: TypeWithRequiredness, streamType: RIterable, @@ -858,7 +858,7 @@ case class VCFPartitionWriter( r.union(streamType.required) } - final def consumeStream( + final override def consumeStream( ctx: ExecuteContext, cb: EmitCodeBuilder, stream: StreamProducer, @@ -1238,7 +1238,8 @@ case class VCFExportFinalizer( exportType: String, tabix: Boolean, ) extends MetadataWriter { - def annotationType: Type = TStruct("cols" -> TArray(typ.colType), "partFiles" -> TArray(TString)) + override def annotationType: Type = + TStruct("cols" -> TArray(typ.colType), "partFiles" -> TArray(TString)) private def header(cb: EmitCodeBuilder, annotations: SBaseStructValue): Code[String] = { val mb = cb.emb @@ -1268,8 +1269,11 @@ case class VCFExportFinalizer( ) } - def writeMetadata(writeAnnotations: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region]) - : Unit = { + override def writeMetadata( + writeAnnotations: => IEmitCode, + cb: EmitCodeBuilder, + region: Value[Region], + ): Unit = { val ctx: ExecuteContext = cb.emb.ctx val ext = ctx.fs.getCodecExtension(outputPath) @@ -1451,7 +1455,7 @@ case class MatrixGENWriter( final case class GenVariantWriter(typ: MatrixType, entriesFieldName: String, precision: Int) extends SimplePartitionWriter { - def consumeElement( + override def consumeElement( cb: EmitCodeBuilder, element: EmitCode, os: Value[OutputStream], @@ -1551,7 +1555,7 @@ final case class GenVariantWriter(typ: MatrixType, entriesFieldName: String, pre } final class GenSampleWriter extends SimplePartitionWriter { - def consumeElement( + override def consumeElement( cb: EmitCodeBuilder, element: EmitCode, os: Value[OutputStream], @@ -1675,7 +1679,7 @@ case class BGENPartitionWriter( override def returnType: TStruct = TStruct("partFile" -> TString, "numVariants" -> TInt64, "dropped" -> TInt64) - def unionTypeRequiredness( + override def unionTypeRequiredness( r: TypeWithRequiredness, ctxType: TypeWithRequiredness, streamType: RIterable, @@ -1684,7 +1688,7 @@ case class BGENPartitionWriter( r.union(streamType.required) } - final def consumeStream( + final override def consumeStream( ctx: ExecuteContext, cb: EmitCodeBuilder, stream: StreamProducer, @@ -1962,7 +1966,7 @@ case class BGENPartitionWriter( case class BGENExportFinalizer(typ: MatrixType, path: String, exportType: String, compression: Int) extends MetadataWriter { - def annotationType: Type = TStruct( + override def annotationType: Type = TStruct( "cols" -> TArray(typ.colType), "results" -> TArray(TStruct( "partFile" -> TString, @@ -1971,8 +1975,11 @@ case class BGENExportFinalizer(typ: MatrixType, path: String, exportType: String )), ) - def writeMetadata(writeAnnotations: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region]) - : Unit = { + override def writeMetadata( + writeAnnotations: => IEmitCode, + cb: EmitCodeBuilder, + region: Value[Region], + ): Unit = { val annotations = writeAnnotations.getOrAssert(cb).asBaseStruct val colValues = annotations.loadField(cb, "cols").getOrAssert(cb).asIndexable val sampleIds = cb.memoize(Code.newArray[String](colValues.loadLength)) @@ -2154,14 +2161,14 @@ case class MatrixPLINKWriter( case class PLINKPartitionWriter(typ: MatrixType, entriesFieldName: String) extends PartitionWriter { val ctxType = TStruct("bedFile" -> TString, "bimFile" -> TString) - def returnType = TStruct("bedFile" -> TString, "bimFile" -> TString) + override def returnType = TStruct("bedFile" -> TString, "bimFile" -> TString) val locusIdx = typ.rowType.fieldIdx("locus") val allelesIdx = typ.rowType.fieldIdx("alleles") val varidIdx = typ.rowType.fieldIdx("varid") val cmPosIdx = typ.rowType.fieldIdx("cm_position") - def unionTypeRequiredness( + override def unionTypeRequiredness( r: TypeWithRequiredness, ctxType: TypeWithRequiredness, streamType: RIterable, @@ -2170,7 +2177,7 @@ case class PLINKPartitionWriter(typ: MatrixType, entriesFieldName: String) exten r.union(streamType.required) } - final def consumeStream( + final override def consumeStream( ctx: ExecuteContext, cb: EmitCodeBuilder, stream: StreamProducer, @@ -2294,10 +2301,13 @@ object PLINKExportFinalizer { case class PLINKExportFinalizer(typ: MatrixType, path: String, headerPath: String) extends MetadataWriter { - def annotationType: Type = TArray(TStruct("bedFile" -> TString, "bimFile" -> TString)) + override def annotationType: Type = TArray(TStruct("bedFile" -> TString, "bimFile" -> TString)) - def writeMetadata(writeAnnotations: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region]) - : Unit = { + override def writeMetadata( + writeAnnotations: => IEmitCode, + cb: EmitCodeBuilder, + region: Value[Region], + ): Unit = { val paths = writeAnnotations.getOrAssert(cb).asIndexable val bedFiles = cb.memoize(Code.newArray[String](paths.loadLength + 1)) // room for header val bimFiles = cb.memoize(Code.newArray[String](paths.loadLength)) diff --git a/hail/hail/src/is/hail/expr/ir/Param.scala b/hail/hail/src/is/hail/expr/ir/Param.scala index f9935540e52..dd1c19a0537 100644 --- a/hail/hail/src/is/hail/expr/ir/Param.scala +++ b/hail/hail/src/is/hail/expr/ir/Param.scala @@ -10,13 +10,13 @@ sealed trait ParamType { } case class CodeParamType(ti: TypeInfo[_]) extends ParamType { - def nCodes: Int = 1 + override def nCodes: Int = 1 override def toString: String = s"CodeParam($ti)" } case class SCodeParamType(st: SType) extends ParamType { - def nCodes: Int = st.nSettables + override def nCodes: Int = st.nSettables override def toString: String = s"SCodeParam($st, $nCodes)" } @@ -34,25 +34,25 @@ trait EmitParamType extends ParamType { ts :+ BooleanInfo } - final def nCodes: Int = valueTupleTypes.length + final override def nCodes: Int = valueTupleTypes.length protected def definedValueTupleTypes(): IndexedSeq[TypeInfo[_]] } case class SingleCodeEmitParamType(required: Boolean, sct: SingleCodeType) extends EmitParamType { - def virtualType: Type = sct.virtualType + override def virtualType: Type = sct.virtualType - def definedValueTupleTypes(): IndexedSeq[TypeInfo[_]] = FastSeq(sct.ti) + override def definedValueTupleTypes(): IndexedSeq[TypeInfo[_]] = FastSeq(sct.ti) override def toString: String = s"SingleCodeEmitParamType($required, $sct)" } case class SCodeEmitParamType(et: EmitType) extends EmitParamType { - def required: Boolean = et.required + override def required: Boolean = et.required - def virtualType: Type = et.st.virtualType + override def virtualType: Type = et.st.virtualType - def definedValueTupleTypes(): IndexedSeq[TypeInfo[_]] = et.st.settableTupleTypes() + override def definedValueTupleTypes(): IndexedSeq[TypeInfo[_]] = et.st.settableTupleTypes() } sealed trait Param diff --git a/hail/hail/src/is/hail/expr/ir/Parser.scala b/hail/hail/src/is/hail/expr/ir/Parser.scala index 3f75d42677a..99ba3c74234 100644 --- a/hail/hail/src/is/hail/expr/ir/Parser.scala +++ b/hail/hail/src/is/hail/expr/ir/Parser.scala @@ -35,23 +35,23 @@ abstract class Token extends Positional { } final case class IdentifierToken(value: String) extends Token { - def getName: String = "identifier" + override def getName: String = "identifier" } final case class StringToken(value: String) extends Token { - def getName: String = "string" + override def getName: String = "string" } final case class IntegerToken(value: Long) extends Token { - def getName: String = "integer" + override def getName: String = "integer" } final case class FloatToken(value: Double) extends Token { - def getName: String = "float" + override def getName: String = "float" } final case class PunctuationToken(value: String) extends Token { - def getName: String = "punctuation" + override def getName: String = "punctuation" } object IRLexer extends JavaTokenParsers { @@ -66,7 +66,7 @@ object IRLexer extends JavaTokenParsers { def quotedLiteral(delim: Char, what: String): Parser[String] = new Parser[String] { - def apply(in: Input): ParseResult[String] = { + override def apply(in: Input): ParseResult[String] = { var r = in val source = in.source diff --git a/hail/hail/src/is/hail/expr/ir/SpecializedArrayBuilders.scala b/hail/hail/src/is/hail/expr/ir/SpecializedArrayBuilders.scala index 55b3f7f7d8a..a33b602e196 100644 --- a/hail/hail/src/is/hail/expr/ir/SpecializedArrayBuilders.scala +++ b/hail/hail/src/is/hail/expr/ir/SpecializedArrayBuilders.scala @@ -150,7 +150,7 @@ final class IntMissingArrayBuilder(initialCapacity: Int) b(i) } - def ensureCapacity(n: Int): Unit = { + override def ensureCapacity(n: Int): Unit = { if (b.length < n) { val newCapacity = math.max(n, b.length * 2) val newb = new Array[Int](newCapacity) @@ -210,7 +210,7 @@ final class LongMissingArrayBuilder(initialCapacity: Int) b(i) } - def ensureCapacity(n: Int): Unit = { + override def ensureCapacity(n: Int): Unit = { if (b.length < n) { val newCapacity = math.max(n, b.length * 2) val newb = new Array[Long](newCapacity) @@ -270,7 +270,7 @@ final class FloatMissingArrayBuilder(initialCapacity: Int) b(i) } - def ensureCapacity(n: Int): Unit = { + override def ensureCapacity(n: Int): Unit = { if (b.length < n) { val newCapacity = math.max(n, b.length * 2) val newb = new Array[Float](newCapacity) @@ -330,7 +330,7 @@ final class DoubleMissingArrayBuilder(initialCapacity: Int) b(i) } - def ensureCapacity(n: Int): Unit = { + override def ensureCapacity(n: Int): Unit = { if (b.length < n) { val newCapacity = math.max(n, b.length * 2) val newb = new Array[Double](newCapacity) @@ -390,7 +390,7 @@ final class BooleanMissingArrayBuilder(initialCapacity: Int) b(i) } - def ensureCapacity(n: Int): Unit = { + override def ensureCapacity(n: Int): Unit = { if (b.length < n) { val newCapacity = math.max(n, b.length * 2) val newb = new Array[Boolean](newCapacity) diff --git a/hail/hail/src/is/hail/expr/ir/TableIR.scala b/hail/hail/src/is/hail/expr/ir/TableIR.scala index 736a32563e3..b855317cafe 100644 --- a/hail/hail/src/is/hail/expr/ir/TableIR.scala +++ b/hail/hail/src/is/hail/expr/ir/TableIR.scala @@ -50,7 +50,7 @@ object TableIR { } sealed abstract class TableIR extends BaseIR { - def typ: TableType + override def typ: TableType override protected def copyWithNewChildren(newChildren: IndexedSeq[BaseIR]): TableIR } @@ -612,7 +612,7 @@ case class PartitionRVDReader(rvd: RVD, uidFieldName: String) extends PartitionR tr } - def emitStream( + override def emitStream( ctx: ExecuteContext, cb: EmitCodeBuilder, mb: EmitMethodBuilder[_], @@ -736,12 +736,12 @@ case class PartitionRVDReader(rvd: RVD, uidFieldName: String) extends PartitionR } } - def toJValue: JValue = + override def toJValue: JValue = JString("") // cannot be parsed, but need a printout for Pretty } trait AbstractNativeReader extends PartitionReader { - def uidFieldName: String + override def uidFieldName: String def spec: AbstractTypedCodecSpec @@ -758,16 +758,16 @@ trait AbstractNativeReader extends PartitionReader { tr } - def fullRowType: TStruct = spec.encodedVirtualType.asInstanceOf[TStruct] + override def fullRowType: TStruct = spec.encodedVirtualType.asInstanceOf[TStruct] .insertFields(Array(uidFieldName -> TTuple(TInt64, TInt64))) } case class PartitionNativeReader(spec: AbstractTypedCodecSpec, uidFieldName: String) extends AbstractNativeReader { - def contextType: Type = TStruct("partitionIndex" -> TInt64, "partitionPath" -> TString) + override def contextType: Type = TStruct("partitionIndex" -> TInt64, "partitionPath" -> TString) - def emitStream( + override def emitStream( ctx: ExecuteContext, cb: EmitCodeBuilder, mb: EmitMethodBuilder[_], @@ -857,7 +857,7 @@ case class PartitionNativeReader(spec: AbstractTypedCodecSpec, uidFieldName: Str } } - def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) + override def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) } case class PartitionNativeIntervalReader( @@ -876,9 +876,9 @@ case class PartitionNativeIntervalReader( lazy val contextType: Type = RVDPartitioner.intervalIRRepresentation(partitioner.kType) require(partitioner.kType.size > 0) - def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) + override def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) - def emitStream( + override def emitStream( ctx: ExecuteContext, cb: EmitCodeBuilder, mb: EmitMethodBuilder[_], @@ -1177,7 +1177,7 @@ case class PartitionNativeReaderIndexed( key: IndexedSeq[String], uidFieldName: String, ) extends AbstractNativeReader { - def contextType: Type = TStruct( + override def contextType: Type = TStruct( "partitionIndex" -> TInt64, "partitionPath" -> TString, "indexPath" -> TString, @@ -1186,7 +1186,7 @@ case class PartitionNativeReaderIndexed( ), ) - def emitStream( + override def emitStream( ctx: ExecuteContext, cb: EmitCodeBuilder, mb: EmitMethodBuilder[_], @@ -1315,7 +1315,7 @@ case class PartitionNativeReaderIndexed( } } - def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) + override def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) } // Result uses the uid field name and values from the right input, and ignores @@ -1323,9 +1323,9 @@ case class PartitionNativeReaderIndexed( case class PartitionZippedNativeReader(left: PartitionReader, right: PartitionReader) extends PartitionReader { - def uidFieldName = right.uidFieldName + override def uidFieldName = right.uidFieldName - def contextType: Type = TStruct( + override def contextType: Type = TStruct( "leftContext" -> left.contextType, "rightContext" -> right.contextType, ) @@ -1356,7 +1356,7 @@ case class PartitionZippedNativeReader(left: PartitionReader, right: PartitionRe TStruct.concat(leftStruct, rightStruct) } - def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) + override def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) override def emitStream( ctx: ExecuteContext, @@ -1460,12 +1460,15 @@ case class PartitionZippedNativeIntervalReader( private lazy val zippedReader = PartitionZippedNativeReader(rowsReader, entriesReader) - def contextType = rowsReader.contextType - def fullRowType = zippedReader.fullRowType - def rowRequiredness(requestedType: TStruct): RStruct = zippedReader.rowRequiredness(requestedType) - def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) + override def contextType = rowsReader.contextType + override def fullRowType = zippedReader.fullRowType + + override def rowRequiredness(requestedType: TStruct): RStruct = + zippedReader.rowRequiredness(requestedType) + + override def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) - def emitStream( + override def emitStream( ctx: ExecuteContext, cb: EmitCodeBuilder, mb: EmitMethodBuilder[_], @@ -1491,7 +1494,7 @@ case class PartitionZippedIndexedNativeReader( uidFieldName: String, ) extends PartitionReader { - def contextType: Type = { + override def contextType: Type = { TStruct( "partitionIndex" -> TInt64, "leftPartitionPath" -> TString, @@ -1518,7 +1521,7 @@ case class PartitionZippedIndexedNativeReader( (leftStruct, rightStruct) } - def rowRequiredness(requestedType: TStruct): RStruct = { + override def rowRequiredness(requestedType: TStruct): RStruct = { val (leftStruct, rightStruct) = splitRequestedTypes(requestedType) val rt = TypeWithRequiredness(requestedType).asInstanceOf[RStruct] val pt = specLeft.decodedPType(leftStruct).asInstanceOf[PStruct].insertFields( @@ -1538,13 +1541,13 @@ case class PartitionZippedIndexedNativeReader( ), ) - def fullRowType: TStruct = + override def fullRowType: TStruct = (specLeft.encodedVirtualType.asInstanceOf[TStruct] ++ specRight.encodedVirtualType.asInstanceOf[ TStruct ]) .insertFields(Array(uidFieldName -> TTuple(TInt64, TInt64, TInt64, TInt64))) - def emitStream( + override def emitStream( ctx: ExecuteContext, cb: EmitCodeBuilder, mb: EmitMethodBuilder[_], @@ -1697,7 +1700,7 @@ case class PartitionZippedIndexedNativeReader( } } - def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) + override def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) } case class TableNativeReaderParameters( @@ -1709,25 +1712,25 @@ class TableNativeReader( val params: TableNativeReaderParameters, val spec: AbstractTableSpec, ) extends TableReaderWithExtraUID { - def pathsUsed: Seq[String] = Array(params.path) + override def pathsUsed: Seq[String] = Array(params.path) val filterIntervals: Boolean = params.options.map(_.filterIntervals).getOrElse(false) - def partitionCounts: Option[IndexedSeq[Long]] = + override def partitionCounts: Option[IndexedSeq[Long]] = if (params.options.isDefined) None else Some(spec.partitionCounts) override def isDistinctlyKeyed: Boolean = spec.isDistinctlyKeyed - def uidType = TTuple(TInt64, TInt64) + override def uidType = TTuple(TInt64, TInt64) - def fullTypeWithoutUIDs = spec.table_type + override def fullTypeWithoutUIDs = spec.table_type override def concreteRowRequiredness(ctx: ExecuteContext, requestedType: TableType) : VirtualTypeWithReq = VirtualTypeWithReq(tcoerce[PStruct](spec.rowsComponent.rvdSpec(ctx.fs, params.path) .typedCodecSpec.encodedType.decodedPType(requestedType.rowType))) - protected def uidRequiredness: VirtualTypeWithReq = + override protected def uidRequiredness: VirtualTypeWithReq = VirtualTypeWithReq(PCanonicalTuple(true, PInt64Required, PInt64Required)) override def globalRequiredness(ctx: ExecuteContext, requestedType: TableType) @@ -1813,7 +1816,7 @@ case class TableNativeZippedReader( specLeft: AbstractTableSpec, specRight: AbstractTableSpec, ) extends TableReaderWithExtraUID { - def pathsUsed: Seq[String] = FastSeq(pathLeft, pathRight) + override def pathsUsed: Seq[String] = FastSeq(pathLeft, pathRight) override def renderShort(): String = s"(TableNativeZippedReader $pathLeft $pathRight ${options.map(_.renderShort()).getOrElse("")})" @@ -1830,7 +1833,7 @@ case class TableNativeZippedReader( require(specLeft.partitionCounts sameElements specRight.partitionCounts) require(specLeft.version == specRight.version) - def partitionCounts: Option[IndexedSeq[Long]] = + override def partitionCounts: Option[IndexedSeq[Long]] = if (intervals.isEmpty) Some(specLeft.partitionCounts) else None override def uidType = TTuple(TInt64, TInt64) @@ -1974,7 +1977,7 @@ case class TableFromBlockMatrixNativeReader( params: TableFromBlockMatrixNativeReaderParameters, metadata: BlockMatrixMetadata, ) extends TableReaderWithExtraUID { - def pathsUsed: Seq[String] = FastSeq(params.path) + override def pathsUsed: Seq[String] = FastSeq(params.path) val partitionRanges = (0 until params.nPartitions).map { i => val nRows = metadata.nRows @@ -2040,7 +2043,7 @@ case class TableFromBlockMatrixNativeReader( override def toJValue: JValue = decomposeWithName(params, "TableFromBlockMatrixNativeReader")(TableReader.formats) - def renderShort(): String = defaultRender() + override def renderShort(): String = defaultRender() } object TableRead { @@ -2192,7 +2195,7 @@ case class TableRange(n: Int, nPartitions: Int) extends TableIR { case class TableFilter(child: TableIR, pred: IR) extends TableIR with PreservesOrRemovesRows { val childrenSeq: IndexedSeq[BaseIR] = Array(child, pred) - def typ: TableType = child.typ + override def typ: TableType = child.typ override protected def copyWithNewChildren(newChildren: IndexedSeq[BaseIR]): TableFilter = { assert(newChildren.length == 2) @@ -2205,7 +2208,7 @@ case class TableFilter(child: TableIR, pred: IR) extends TableIR with PreservesO case class TableHead(child: TableIR, n: Long) extends TableIR { require(n >= 0, fatal(s"TableHead: n must be non-negative! Found '$n'.")) lazy val childrenSeq: IndexedSeq[BaseIR] = FastSeq(child) - def typ: TableType = child.typ + override def typ: TableType = child.typ override protected def copyWithNewChildren(newChildren: IndexedSeq[BaseIR]): TableHead = { val IndexedSeq(newChild: TableIR) = newChildren @@ -2216,7 +2219,7 @@ case class TableHead(child: TableIR, n: Long) extends TableIR { case class TableTail(child: TableIR, n: Long) extends TableIR { require(n >= 0, fatal(s"TableTail: n must be non-negative! Found '$n'.")) lazy val childrenSeq: IndexedSeq[BaseIR] = FastSeq(child) - def typ: TableType = child.typ + override def typ: TableType = child.typ override protected def copyWithNewChildren(newChildren: IndexedSeq[BaseIR]): TableTail = { val IndexedSeq(newChild: TableIR) = newChildren @@ -2232,7 +2235,7 @@ object RepartitionStrategy { case class TableRepartition(child: TableIR, n: Int, strategy: Int) extends TableIR with PreservesRows { - def typ: TableType = child.typ + override def typ: TableType = child.typ lazy val childrenSeq: IndexedSeq[BaseIR] = FastSeq(child) @@ -2466,7 +2469,7 @@ case class TableUnion(childrenSeq: IndexedSeq[TableIR]) extends TableIR { override protected def copyWithNewChildren(newChildren: IndexedSeq[BaseIR]): TableUnion = TableUnion(newChildren.map(_.asInstanceOf[TableIR])) - def typ: TableType = childrenSeq(0).typ + override def typ: TableType = childrenSeq(0).typ } case class MatrixRowsTable(child: MatrixIR) extends TableIR with PreservesRows { @@ -2477,7 +2480,7 @@ case class MatrixRowsTable(child: MatrixIR) extends TableIR with PreservesRows { MatrixRowsTable(newChildren(0).asInstanceOf[MatrixIR]) } - def typ: TableType = child.typ.rowsTableType + override def typ: TableType = child.typ.rowsTableType override def preservesRowsOrColsFrom: BaseIR = child } @@ -2490,7 +2493,7 @@ case class MatrixColsTable(child: MatrixIR) extends TableIR { MatrixColsTable(newChildren(0).asInstanceOf[MatrixIR]) } - def typ: TableType = child.typ.colsTableType + override def typ: TableType = child.typ.colsTableType } case class MatrixEntriesTable(child: MatrixIR) extends TableIR { @@ -2502,7 +2505,7 @@ case class MatrixEntriesTable(child: MatrixIR) extends TableIR { MatrixEntriesTable(newChildren(0).asInstanceOf[MatrixIR]) } - def typ: TableType = child.typ.entriesTableType + override def typ: TableType = child.typ.entriesTableType } case class TableDistinct(child: TableIR) extends TableIR with PreservesOrRemovesRows { @@ -2513,7 +2516,7 @@ case class TableDistinct(child: TableIR) extends TableIR with PreservesOrRemoves TableDistinct(newChild.asInstanceOf[TableIR]) } - def typ: TableType = child.typ + override def typ: TableType = child.typ override def preservesRowsOrColsFrom: BaseIR = child } @@ -2715,14 +2718,14 @@ case class BlockMatrixToTable(child: BlockMatrixIR) extends TableIR { case class RelationalLetTable(name: Name, value: IR, body: TableIR) extends TableIR with PreservesRows { - def typ: TableType = body.typ + override def typ: TableType = body.typ - def childrenSeq: IndexedSeq[BaseIR] = Array(value, body) + override def childrenSeq: IndexedSeq[BaseIR] = Array(value, body) override protected def copyWithNewChildren(newChildren: IndexedSeq[BaseIR]): TableIR = { val IndexedSeq(newValue: IR, newBody: TableIR) = newChildren RelationalLetTable(name, newValue, newBody) } - def preservesRowsOrColsFrom: BaseIR = body + override def preservesRowsOrColsFrom: BaseIR = body } diff --git a/hail/hail/src/is/hail/expr/ir/TableValue.scala b/hail/hail/src/is/hail/expr/ir/TableValue.scala index e2ec7d70069..6ba78b71a88 100644 --- a/hail/hail/src/is/hail/expr/ir/TableValue.scala +++ b/hail/hail/src/is/hail/expr/ir/TableValue.scala @@ -49,23 +49,23 @@ sealed trait TableExecuteIntermediate { } case class TableValueIntermediate(tv: TableValue) extends TableExecuteIntermediate { - def asTableStage(ctx: ExecuteContext): TableStage = + override def asTableStage(ctx: ExecuteContext): TableStage = RVDToTableStage(tv.rvd, tv.globals.toEncodedLiteral(ctx.theHailClassLoader)) - def asTableValue(ctx: ExecuteContext): TableValue = tv + override def asTableValue(ctx: ExecuteContext): TableValue = tv - def partitioner: RVDPartitioner = tv.rvd.partitioner + override def partitioner: RVDPartitioner = tv.rvd.partitioner } case class TableStageIntermediate(ts: TableStage) extends TableExecuteIntermediate { - def asTableStage(ctx: ExecuteContext): TableStage = ts + override def asTableStage(ctx: ExecuteContext): TableStage = ts - def asTableValue(ctx: ExecuteContext): TableValue = { + override def asTableValue(ctx: ExecuteContext): TableValue = { val (globals, rvd) = TableStageToRVD(ctx, ts) TableValue(ctx, TableType(ts.rowType, ts.key, ts.globalType), globals, rvd) } - def partitioner: RVDPartitioner = ts.partitioner + override def partitioner: RVDPartitioner = ts.partitioner } object TableValue extends Logging { @@ -529,7 +529,7 @@ case class TableValue(ctx: ExecuteContext, typ: TableType, globals: BroadcastRow val rowKey: WritableRegionValue = WritableRegionValue(sm, keyType, ctx.freshRegion()) val consumerRegion: Region = ctx.region - def hasNext: Boolean = { + override def hasNext: Boolean = { if (isEnd || (current == 0 && !it.hasNext)) { isEnd = true return false @@ -539,7 +539,7 @@ case class TableValue(ctx: ExecuteContext, typ: TableType, globals: BroadcastRow true } - def next(): Long = { + override def next(): Long = { if (!hasNext) throw new java.util.NoSuchElementException() @@ -641,9 +641,9 @@ case class TableValue(ctx: ExecuteContext, typ: TableType, globals: BroadcastRow new Iterator[Long] { private[this] var i = 0 - def hasNext: Boolean = i < len + override def hasNext: Boolean = i < len - def next(): Long = { + override def next(): Long = { val ret = rowF(ctx.region, ptr, i) i += 1 ret diff --git a/hail/hail/src/is/hail/expr/ir/TableWriter.scala b/hail/hail/src/is/hail/expr/ir/TableWriter.scala index 0f8c1660cd8..294ae944217 100644 --- a/hail/hail/src/is/hail/expr/ir/TableWriter.scala +++ b/hail/hail/src/is/hail/expr/ir/TableWriter.scala @@ -196,10 +196,10 @@ case class PartitionNativeWriter( ) extends PartitionWriter { val keyType = spec.encodedVirtualType.asInstanceOf[TStruct].select(keyFields)._1 - def ctxType = PartitionNativeWriter.ctxType + override def ctxType = PartitionNativeWriter.ctxType val returnType = PartitionNativeWriter.returnType(keyType, trackTotalBytes) - def unionTypeRequiredness( + override def unionTypeRequiredness( r: TypeWithRequiredness, ctxType: TypeWithRequiredness, streamType: RIterable, @@ -383,7 +383,7 @@ case class PartitionNativeWriter( } } - def consumeStream( + override def consumeStream( ctx: ExecuteContext, cb: EmitCodeBuilder, stream: StreamProducer, @@ -402,9 +402,9 @@ case class PartitionNativeWriter( } case class RVDSpecWriter(path: String, spec: RVDSpecMaker) extends MetadataWriter { - def annotationType: Type = TArray(TString) + override def annotationType: Type = TArray(TString) - def writeMetadata( + override def writeMetadata( writeAnnotations: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region], @@ -469,14 +469,14 @@ case class TableSpecWriter( refRelPath: String, log: Boolean, ) extends MetadataWriter { - def annotationType: Type = TArray(TStruct( + override def annotationType: Type = TArray(TStruct( "partitionCounts" -> TInt64, "distinctlyKeyed" -> TBoolean, "firstKey" -> typ.keyType, "lastKey" -> typ.keyType, )) - def writeMetadata( + override def writeMetadata( writeAnnotations: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region], @@ -576,9 +576,10 @@ case class RelationalSetup(path: String, overwrite: Boolean, refs: Option[TableT )) ) - def annotationType: Type = TVoid + override def annotationType: Type = TVoid - def writeMetadata(ignored: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region]): Unit = { + override def writeMetadata(ignored: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region]) + : Unit = { if (overwrite) cb += cb.emb.getFS.invoke[String, Boolean, Unit]("delete", path, true) else @@ -605,9 +606,9 @@ case class RelationalSetup(path: String, overwrite: Boolean, refs: Option[TableT } case class RelationalCommit(path: String) extends MetadataWriter { - def annotationType: Type = TStruct() + override def annotationType: Type = TStruct() - def writeMetadata( + override def writeMetadata( writeAnnotations: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region], @@ -627,9 +628,9 @@ case class RelationalWriter( overwrite: Boolean, maybeRefs: Option[(String, Set[String])], ) extends MetadataWriter { - def annotationType: Type = TVoid + override def annotationType: Type = TVoid - def writeMetadata( + override def writeMetadata( writeAnnotations: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region], @@ -734,7 +735,7 @@ case class TableTextPartitionWriter(rowType: TStruct, delimiter: String, writeHe cb += os.invoke[Int, Unit]("write", '\n') } - def consumeElement( + override def consumeElement( cb: EmitCodeBuilder, element: EmitCode, os: Value[OutputStream], @@ -808,10 +809,13 @@ case class TableTextFinalizer( header: Boolean = true, exportType: String = ExportType.CONCATENATED, ) extends MetadataWriter { - def annotationType: Type = TArray(TString) + override def annotationType: Type = TArray(TString) - def writeMetadata(writeAnnotations: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region]) - : Unit = { + override def writeMetadata( + writeAnnotations: => IEmitCode, + cb: EmitCodeBuilder, + region: Value[Region], + ): Unit = { val ctx: ExecuteContext = cb.emb.ctx val ext = ctx.fs.getCodecExtension(outputPath) val partPaths = writeAnnotations.getOrFatal(cb, "write annotations cannot be missing!") @@ -1064,7 +1068,7 @@ case class TableNativeFanoutWriter( class PartitionNativeFanoutWriter( targets: IndexedSeq[FanoutWriterTarget] ) extends PartitionWriter { - def consumeStream( + override def consumeStream( ctx: ExecuteContext, cb: EmitCodeBuilder, stream: StreamProducer, @@ -1097,12 +1101,12 @@ class PartitionNativeFanoutWriter( ) } - def ctxType = TString + override def ctxType = TString - def returnType: TTuple = + override def returnType: TTuple = TTuple(targets.map(target => target.rowWriter.returnType): _*) - def unionTypeRequiredness( + override def unionTypeRequiredness( returnType: TypeWithRequiredness, ctxType: TypeWithRequiredness, streamType: RIterable, diff --git a/hail/hail/src/is/hail/expr/ir/ValueReader.scala b/hail/hail/src/is/hail/expr/ir/ValueReader.scala index 9cf1db6e547..0c0c007fe11 100644 --- a/hail/hail/src/is/hail/expr/ir/ValueReader.scala +++ b/hail/hail/src/is/hail/expr/ir/ValueReader.scala @@ -44,11 +44,15 @@ abstract class ValueReader { } final case class ETypeValueReader(spec: AbstractTypedCodecSpec) extends ValueReader { - def unionRequiredness(requestedType: Type, requiredness: TypeWithRequiredness): Unit = + override def unionRequiredness(requestedType: Type, requiredness: TypeWithRequiredness): Unit = requiredness.fromPType(spec.encodedType.decodedPType(requestedType)) - def readValue(cb: EmitCodeBuilder, t: Type, region: Value[Region], is: Value[InputStream]) - : SValue = { + override def readValue( + cb: EmitCodeBuilder, + t: Type, + region: Value[Region], + is: Value[InputStream], + ): SValue = { val decoder = spec.encodedType.buildDecoder(t, cb.emb.ecb) val ib = cb.memoize(spec.buildCodeInputBuffer(is)) val ret = decoder.apply(cb, region, ib) diff --git a/hail/hail/src/is/hail/expr/ir/ValueWriter.scala b/hail/hail/src/is/hail/expr/ir/ValueWriter.scala index f5fed6728bc..6af93230739 100644 --- a/hail/hail/src/is/hail/expr/ir/ValueWriter.scala +++ b/hail/hail/src/is/hail/expr/ir/ValueWriter.scala @@ -39,7 +39,7 @@ abstract class ValueWriter { } final case class ETypeValueWriter(spec: AbstractTypedCodecSpec) extends ValueWriter { - def writeValue(cb: EmitCodeBuilder, value: SValue, os: Value[OutputStream]): Unit = { + override def writeValue(cb: EmitCodeBuilder, value: SValue, os: Value[OutputStream]): Unit = { val encoder = spec.encodedType.buildEncoder(value.st, cb.emb.ecb) val ob = cb.memoize(spec.buildCodeOutputBuffer(os)) encoder.apply(cb, value, ob) diff --git a/hail/hail/src/is/hail/expr/ir/agg/AggregatorState.scala b/hail/hail/src/is/hail/expr/ir/agg/AggregatorState.scala index b65cdb663aa..3c77a11e952 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/AggregatorState.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/AggregatorState.scala @@ -75,19 +75,19 @@ trait RegionBackedAggState extends AggregatorState { protected val r: Settable[Region] = kb.genFieldThisRef[Region]() val region: Value[Region] = r - def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = + override def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = cb += region.getNewRegion(const(regionSize)) - def createState(cb: EmitCodeBuilder): Unit = + override def createState(cb: EmitCodeBuilder): Unit = cb.if_(region.isNull, cb.assign(r, Region.stagedCreate(regionSize, kb.pool()))) - def load( + override def load( cb: EmitCodeBuilder, regionLoader: (EmitCodeBuilder, Value[Region]) => Unit, src: Value[Long], ): Unit = regionLoader(cb, r) - def store( + override def store( cb: EmitCodeBuilder, regionStorer: (EmitCodeBuilder, Value[Region]) => Unit, dest: Value[Long], @@ -127,7 +127,7 @@ trait PointerBasedRVAState extends RegionBackedAggState { }, ) - def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = + override def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = copyFromAddress(cb, cb.memoize(Region.loadAddress(src))) def copyFromAddress(cb: EmitCodeBuilder, src: Value[Long]): Unit @@ -189,7 +189,7 @@ abstract class AbstractTypedRegionBackedAggState(val ptype: PType) extends Regio ptype.loadCheapSCode(cb, storageType.loadField(off, 0)), ) - def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = { + override def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = { newState(cb, off) storageType.storeAtAddress( cb, @@ -200,13 +200,13 @@ abstract class AbstractTypedRegionBackedAggState(val ptype: PType) extends Regio ) } - def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { + override def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { val codecSpec = TypedCodecSpec(kb.ctx, storageType, codec) val enc = codecSpec.encodedType.buildEncoder(storageType.sType, kb) (cb, ob: Value[OutputBuffer]) => enc(cb, storageType.loadCheapSCode(cb, off), ob) } - def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { + override def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { val codecSpec = TypedCodecSpec(kb.ctx, storageType, codec) val dec = codecSpec.encodedType.buildDecoder(storageType.virtualType, kb) @@ -232,23 +232,23 @@ class PrimitiveRVAState(val vtypes: Array[VirtualTypeWithReq], val kb: EmitClass def foreachField(f: (Int, EmitSettable) => Unit): Unit = (0 until nFields).foreach(i => f(i, fields(i))) - def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = {} + override def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = {} - def createState(cb: EmitCodeBuilder): Unit = {} + override def createState(cb: EmitCodeBuilder): Unit = {} private[this] def loadVarsFromRegion(cb: EmitCodeBuilder, srcc: Code[Long]): Unit = { val pv = storageType.loadCheapSCode(cb, srcc) foreachField((i, es) => cb.assign(es, pv.loadField(cb, i))) } - def load( + override def load( cb: EmitCodeBuilder, regionLoader: (EmitCodeBuilder, Value[Region]) => Unit, src: Value[Long], ): Unit = loadVarsFromRegion(cb, src) - def store( + override def store( cb: EmitCodeBuilder, regionStorer: (EmitCodeBuilder, Value[Region]) => Unit, dest: Value[Long], @@ -261,9 +261,9 @@ class PrimitiveRVAState(val vtypes: Array[VirtualTypeWithReq], val kb: EmitClass false, ) - def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = loadVarsFromRegion(cb, src) + override def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = loadVarsFromRegion(cb, src) - def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { + override def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { (cb, ob: Value[OutputBuffer]) => foreachField { case (_, es) => if (es.emitType.required) { @@ -281,7 +281,7 @@ class PrimitiveRVAState(val vtypes: Array[VirtualTypeWithReq], val kb: EmitClass } } - def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { + override def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { (cb, ib: Value[InputBuffer]) => foreachField { case (_, es) => if (es.emitType.required) { diff --git a/hail/hail/src/is/hail/expr/ir/agg/ApproxCDFAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/ApproxCDFAggregator.scala index 9d68f82e47b..f5abcec3919 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/ApproxCDFAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/ApproxCDFAggregator.scala @@ -61,9 +61,10 @@ class ApproxCDFState(val kb: EmitClassBuilder[_]) extends AggregatorState { aggr.invoke[Region, Long]("rvResult", region), ) - def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = cb += region.getNewRegion(regionSize) + override def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = + cb += region.getNewRegion(regionSize) - def createState(cb: EmitCodeBuilder): Unit = + override def createState(cb: EmitCodeBuilder): Unit = cb.if_(region.isNull, cb.assign(r, Region.stagedCreate(regionSize, kb.pool()))) override def load( @@ -144,11 +145,13 @@ class ApproxCDFState(val kb: EmitClassBuilder[_]) extends AggregatorState { class ApproxCDFAggregator extends StagedAggregator { type State = ApproxCDFState - def resultEmitType: EmitType = EmitType(SBaseStructPointer(QuantilesAggregator.resultPType), true) + override def resultEmitType: EmitType = + EmitType(SBaseStructPointer(QuantilesAggregator.resultPType), true) + val initOpTypes: Seq[Type] = FastSeq(TInt32) val seqOpTypes: Seq[Type] = FastSeq(TFloat64) - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { val Array(k) = init k.toI(cb) .consume( @@ -158,7 +161,7 @@ class ApproxCDFAggregator extends StagedAggregator { ) } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { val Array(x) = seq x.toI(cb) .consume( @@ -168,7 +171,7 @@ class ApproxCDFAggregator extends StagedAggregator { ) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -177,6 +180,7 @@ class ApproxCDFAggregator extends StagedAggregator { ): Unit = state.comb(cb, other) - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = IEmitCode.present(cb, state.result(cb, region)) } diff --git a/hail/hail/src/is/hail/expr/ir/agg/ArrayElementLengthCheckAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/ArrayElementLengthCheckAggregator.scala index 580b3050214..ee36b4793de 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/ArrayElementLengthCheckAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/ArrayElementLengthCheckAggregator.scala @@ -27,11 +27,11 @@ class ArrayElementState(val kb: EmitClassBuilder[_], val nested: StateTuple) private val aoff: Settable[Long] = kb.genFieldThisRef[Long]("arrayrva_aoff") private def regionOffset(eltIdx: Value[Int]): Value[Int] = new Value[Int] { - def get: Code[Int] = (eltIdx + 1) * nStates + override def get: Code[Int] = (eltIdx + 1) * nStates } private def statesOffset(eltIdx: Value[Int]): Value[Long] = new Value[Long] { - def get: Code[Long] = arrayType.loadElement(typ.loadField(off, 1), eltIdx) + override def get: Code[Long] = arrayType.loadElement(typ.loadField(off, 1), eltIdx) } val initContainer: TupleAggregatorState = new TupleAggregatorState( @@ -39,7 +39,7 @@ class ArrayElementState(val kb: EmitClassBuilder[_], val nested: StateTuple) nested, region, new Value[Long] { - def get: Code[Long] = typ.loadField(off, 0) + override def get: Code[Long] = typ.loadField(off, 0) }, ) @@ -118,7 +118,7 @@ class ArrayElementState(val kb: EmitClassBuilder[_], val nested: StateTuple) def store(cb: EmitCodeBuilder): Unit = container.store(cb) - def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { + override def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { val serializers = nested.states.map(_.serialize(codec)); { (cb: EmitCodeBuilder, ob: Value[OutputBuffer]) => loadInit(cb) @@ -140,7 +140,7 @@ class ArrayElementState(val kb: EmitClassBuilder[_], val nested: StateTuple) } } - def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { + override def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { val deserializers = nested.states.map(_.deserialize(codec)); { (cb: EmitCodeBuilder, ib: Value[InputBuffer]) => init( @@ -157,7 +157,7 @@ class ArrayElementState(val kb: EmitClassBuilder[_], val nested: StateTuple) } } - def copyFromAddress(cb: EmitCodeBuilder, src: Value[Long]): Unit = { + override def copyFromAddress(cb: EmitCodeBuilder, src: Value[Long]): Unit = { init(cb, cb => initContainer.copyFrom(cb, cb.memoize(typ.loadField(src, 0))), initLen = false) cb.if_( typ.isFieldMissing(cb, src, 1), { @@ -188,7 +188,7 @@ class ArrayElementLengthCheckAggregator(nestedAggs: Array[StagedAggregator], kno val seqOpTypes: Seq[Type] = FastSeq(TInt32) // inits all things - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { if (knownLength) { val Array(len, inits) = init state.init(cb, cb => cb += inits.asVoid(), initLen = false) @@ -205,7 +205,7 @@ class ArrayElementLengthCheckAggregator(nestedAggs: Array[StagedAggregator], kno } // does a length check on arrays - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { assert(seq.length == 1) val len = seq.head len.toI(cb).consume( @@ -223,7 +223,7 @@ class ArrayElementLengthCheckAggregator(nestedAggs: Array[StagedAggregator], kno ) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -312,12 +312,12 @@ class ArrayElementwiseOpAggregator(nestedAggs: Array[StagedAggregator]) extends override def resultEmitType = EmitType(SIndexablePointer(resultPType), false) - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = throw new UnsupportedOperationException( "State must be initialized by ArrayElementLengthCheckAggregator." ) - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { val Array(eltIdx, seqOps) = seq eltIdx.toI(cb).consume( cb, @@ -336,7 +336,7 @@ class ArrayElementwiseOpAggregator(nestedAggs: Array[StagedAggregator]) extends ) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -347,7 +347,8 @@ class ArrayElementwiseOpAggregator(nestedAggs: Array[StagedAggregator]) extends "State must be combined by ArrayElementLengthCheckAggregator." ) - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = throw new UnsupportedOperationException( "Result must be defined by ArrayElementLengthCheckAggregator." ) diff --git a/hail/hail/src/is/hail/expr/ir/agg/CallStatsAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/CallStatsAggregator.scala index 4720c29a8a5..970640fa8d1 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/CallStatsAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/CallStatsAggregator.scala @@ -97,13 +97,14 @@ class CallStatsState(val kb: EmitClassBuilder[_]) extends PointerBasedRVAState { cb += Region.storeInt(addr, updater(Region.loadInt(addr))) } - def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { (cb, ob) => - val codecSpec = TypedCodecSpec(kb.ctx, CallStatsState.stateType, codec) - codecSpec.encodedType.buildEncoder(CallStatsState.stateType.sType, kb) - .apply(cb, CallStatsState.stateType.loadCheapSCode(cb, off), ob) + override def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { + (cb, ob) => + val codecSpec = TypedCodecSpec(kb.ctx, CallStatsState.stateType, codec) + codecSpec.encodedType.buildEncoder(CallStatsState.stateType.sType, kb) + .apply(cb, CallStatsState.stateType.loadCheapSCode(cb, off), ob) } - def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { + override def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { (cb: EmitCodeBuilder, ib: Value[InputBuffer]) => val codecSpec = TypedCodecSpec(kb.ctx, CallStatsState.stateType, codec) val decValue = codecSpec.encodedType.buildDecoder(CallStatsState.stateType.virtualType, kb) @@ -113,7 +114,7 @@ class CallStatsState(val kb: EmitClassBuilder[_]) extends PointerBasedRVAState { loadNAlleles(cb) } - def copyFromAddress(cb: EmitCodeBuilder, src: Value[Long]): Unit = { + override def copyFromAddress(cb: EmitCodeBuilder, src: Value[Long]): Unit = { cb.assign( off, CallStatsState.stateType.store( @@ -137,7 +138,7 @@ class CallStatsAggregator extends StagedAggregator { val initOpTypes: Seq[Type] = FastSeq(TInt32) val seqOpTypes: Seq[Type] = FastSeq(TCall) - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { val Array(nAlleles) = init val addr = state.kb.genFieldThisRef[Long]() val n = state.kb.genFieldThisRef[Int]() @@ -175,7 +176,7 @@ class CallStatsAggregator extends StagedAggregator { ) } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { val Array(call) = seq call.toI(cb).consume( @@ -203,7 +204,7 @@ class CallStatsAggregator extends StagedAggregator { ) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -236,7 +237,8 @@ class CallStatsAggregator extends StagedAggregator { ) } - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = { + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = { val rt = CallStatsState.resultPType val addr = cb.memoize(rt.allocate(region), "call_stats_aggregator_result_addr") rt.stagedInitialize(cb, addr, setMissing = false) diff --git a/hail/hail/src/is/hail/expr/ir/agg/CollectAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/CollectAggregator.scala index 907fa809639..9fbcc594196 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/CollectAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/CollectAggregator.scala @@ -20,11 +20,11 @@ class CollectAggState(val elemVType: VirtualTypeWithReq, val kb: EmitClassBuilde val region: Value[Region] = r val bll = new StagedBlockLinkedList(elemType, kb) - def storageType = bll.storageType + override def storageType = bll.storageType override def regionSize: Region.Size = Region.REGULAR - def createState(cb: EmitCodeBuilder): Unit = + override def createState(cb: EmitCodeBuilder): Unit = cb.if_( region.isNull, { cb.assign(r, Region.stagedCreate(regionSize, kb.pool())) @@ -32,7 +32,8 @@ class CollectAggState(val elemVType: VirtualTypeWithReq, val kb: EmitClassBuilde }, ) - def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = cb += region.getNewRegion(regionSize) + override def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = + cb += region.getNewRegion(regionSize) override def load( cb: EmitCodeBuilder, @@ -57,17 +58,17 @@ class CollectAggState(val elemVType: VirtualTypeWithReq, val kb: EmitClassBuilde ) } - def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = { + override def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = { val copyBll = new StagedBlockLinkedList(elemType, kb) copyBll.load(cb, src) bll.initWithDeepCopy(cb, region, copyBll) } - def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { + override def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { (cb, ib) => bll.serialize(cb, region, ib) } - def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { + override def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { (cb, ib) => bll.init(cb, region) bll.deserialize(cb, region, ib) @@ -82,15 +83,15 @@ class CollectAggregator(val elemType: VirtualTypeWithReq) extends StagedAggregat val initOpTypes: Seq[Type] = Array[Type]() val seqOpTypes: Seq[Type] = Array[Type](elemType.t) - protected def _initOp(cb: EmitCodeBuilder, state: State, args: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, args: Array[EmitCode]): Unit = { assert(args.isEmpty) state.bll.init(cb, state.region) } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = state.bll.push(cb, state.region, seq(0)) - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -99,7 +100,8 @@ class CollectAggregator(val elemType: VirtualTypeWithReq) extends StagedAggregat ): Unit = state.bll.append(cb, state.region, other.bll) - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = // deepCopy is handled by the blocked linked list IEmitCode.present( cb, diff --git a/hail/hail/src/is/hail/expr/ir/agg/CollectAsSetAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/CollectAsSetAggregator.scala index 6eade3802fd..45b7b11e7f5 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/CollectAsSetAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/CollectAsSetAggregator.scala @@ -137,14 +137,14 @@ class AppendOnlySetState(val kb: EmitClassBuilder[_], vt: VirtualTypeWithReq) ) } - def copyFromAddress(cb: EmitCodeBuilder, src: Value[Long]): Unit = { + override def copyFromAddress(cb: EmitCodeBuilder, src: Value[Long]): Unit = { cb.assign(off, region.allocate(typ.alignment, typ.byteSize)) cb.assign(size, Region.loadInt(typ.loadField(src, 0))) tree.init(cb) tree.deepCopy(cb, cb.memoize(Region.loadAddress(typ.loadField(src, 1)))) } - def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { + override def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { (cb: EmitCodeBuilder, ob: Value[OutputBuffer]) => tree.bulkStore(cb, ob) { (cb, ob, srcCode) => val src = cb.newLocal("aoss_ser_src", srcCode) @@ -159,7 +159,7 @@ class AppendOnlySetState(val kb: EmitClassBuilder[_], vt: VirtualTypeWithReq) } } - def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { + override def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { val kDec = et.buildDecoder(t.virtualType, kb) { (cb: EmitCodeBuilder, ib: Value[InputBuffer]) => @@ -184,17 +184,17 @@ class CollectAsSetAggregator(elem: VirtualTypeWithReq) extends StagedAggregator val initOpTypes: Seq[Type] = Array[Type]() val seqOpTypes: Seq[Type] = Array[Type](elem.t) - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { assert(init.length == 0) state.init(cb) } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { val Array(elt) = seq state.insert(cb, elt) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -203,7 +203,8 @@ class CollectAsSetAggregator(elem: VirtualTypeWithReq) extends StagedAggregator ): Unit = other.foreach(cb)((cb, k) => state.insert(cb, k)) - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = { + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = { val (pushElement, finish) = arrayRep.constructFromFunctions(cb, region, state.size, deepCopy = true) state.foreach(cb)((cb, elt) => pushElement(cb, elt.toI(cb))) diff --git a/hail/hail/src/is/hail/expr/ir/agg/CountAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/CountAggregator.scala index c0b447de0cb..ddf66942058 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/CountAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/CountAggregator.scala @@ -16,21 +16,21 @@ object CountAggregator extends StagedAggregator { val initOpTypes: Seq[Type] = Array[Type]() val seqOpTypes: Seq[Type] = Array[Type]() - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { assert(init.length == 0) assert(state.vtypes.head.r.required) val ev = state.fields(0) cb.assign(ev, EmitCode.present(cb.emb, primitive(const(0L)))) } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { assert(seq.length == 0) assert(state.vtypes.head.r.required) val ev = state.fields(0) cb.assign(ev, EmitCode.present(cb.emb, primitive(cb.memoize(ev.pv.asInt64.value + 1L)))) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -46,7 +46,8 @@ object CountAggregator extends StagedAggregator { ) } - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = { + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = { assert(state.vtypes.head.r.required) val ev = state.fields(0) ev.toI(cb).map(cb)(sv => sv.copyToRegion(cb, region, sv.st)) diff --git a/hail/hail/src/is/hail/expr/ir/agg/DensifyAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/DensifyAggregator.scala index 760701c5355..624daa537ab 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/DensifyAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/DensifyAggregator.scala @@ -34,10 +34,10 @@ class DensifyState(val arrayVType: VirtualTypeWithReq, val kb: EmitClassBuilder[ private val length = kb.genFieldThisRef[Int]("densify_len") private val arrayAddr = kb.genFieldThisRef[Long]("densify_addr") - def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = + override def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = cb += region.getNewRegion(regionSize) - def createState(cb: EmitCodeBuilder): Unit = + override def createState(cb: EmitCodeBuilder): Unit = cb.if_(region.isNull, cb.assign(r, Region.stagedCreate(regionSize, kb.pool()))) override def load( @@ -60,7 +60,7 @@ class DensifyState(val arrayVType: VirtualTypeWithReq, val kb: EmitClassBuilder[ cb += Region.storeAddress(dest, arrayAddr) } - def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { + override def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { val codecSpec = TypedCodecSpec(kb.ctx, arrayStorageType, codec) (cb: EmitCodeBuilder, ob: Value[OutputBuffer]) => { @@ -71,7 +71,7 @@ class DensifyState(val arrayVType: VirtualTypeWithReq, val kb: EmitClassBuilder[ } } - def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { + override def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { val codecSpec = TypedCodecSpec(kb.ctx, arrayStorageType, codec) (cb: EmitCodeBuilder, ib: Value[InputBuffer]) => { @@ -155,7 +155,7 @@ class DensifyState(val arrayVType: VirtualTypeWithReq, val kb: EmitClassBuilder[ def result(cb: EmitCodeBuilder, region: Value[Region]): SIndexablePointerValue = arrayStorageType.loadCheapSCode(cb, arrayAddr) - def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = { + override def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = { cb.assign( arrayAddr, arrayStorageType.store( @@ -183,7 +183,7 @@ class DensifyAggregator(val arrayVType: VirtualTypeWithReq) extends StagedAggreg val initOpTypes: Seq[Type] = Array(TInt32) val seqOpTypes: Seq[Type] = Array(resultEmitType.virtualType) - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { assert(init.length == 1) val Array(sizeTriplet) = init sizeTriplet.toI(cb) @@ -194,12 +194,12 @@ class DensifyAggregator(val arrayVType: VirtualTypeWithReq) extends StagedAggreg ) } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { val Array(elt: EmitCode) = seq state.seqOp(cb, elt) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -207,7 +207,8 @@ class DensifyAggregator(val arrayVType: VirtualTypeWithReq) extends StagedAggreg other: DensifyState, ): Unit = state.combine(cb, other) - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = { + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = { val resultInWrongRegion = state.result(cb, region) // deepCopy needs to be done here val ptrInRightRegion = pt.store(cb, region, resultInWrongRegion, true) diff --git a/hail/hail/src/is/hail/expr/ir/agg/DownsampleAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/DownsampleAggregator.scala index 1b4a3f79b6b..b6dbe9cdcfe 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/DownsampleAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/DownsampleAggregator.scala @@ -76,9 +76,10 @@ class DownsampleState( val oldRegion: Settable[Region] = kb.genFieldThisRef[Region]("old_region") - def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = cb += region.getNewRegion(regionSize) + override def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = + cb += region.getNewRegion(regionSize) - def createState(cb: EmitCodeBuilder): Unit = + override def createState(cb: EmitCodeBuilder): Unit = cb.if_(region.isNull, cb.assign(r, Region.stagedCreate(regionSize, kb.pool()))) val binType = PCanonicalStruct(required = true, "x" -> PInt32Required, "y" -> PInt32Required) @@ -216,7 +217,7 @@ class DownsampleState( ) } - def copyFrom(cb: EmitCodeBuilder, _src: Value[Long]): Unit = { + override def copyFrom(cb: EmitCodeBuilder, _src: Value[Long]): Unit = { val mb = kb.genEmitMethod("downsample_copy", FastSeq[ParamType](LongInfo), UnitInfo) val src = mb.getCodeParam[Long](1) @@ -235,7 +236,7 @@ class DownsampleState( cb.invokeVoid(mb, cb.this_, _src) } - def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { + override def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { (cb: EmitCodeBuilder, ob: Value[OutputBuffer]) => val mb = kb.genEmitMethod( "downsample_serialize", @@ -268,7 +269,7 @@ class DownsampleState( cb.invokeVoid(mb, cb.this_, ob) } - def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { + override def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { val binDec = binET.buildInplaceDecoderMethod(binType, kb) val pointDec = pointET.buildInplaceDecoderMethod(pointType, kb) @@ -721,7 +722,7 @@ class DownsampleAggregator(arrayType: VirtualTypeWithReq) extends StagedAggregat val initOpTypes: Seq[Type] = Array(TInt32) val seqOpTypes: Seq[Type] = Array(TFloat64, TFloat64, arrayType.t) - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { val Array(nDivisions) = init nDivisions.toI(cb) .consume( @@ -731,13 +732,13 @@ class DownsampleAggregator(arrayType: VirtualTypeWithReq) extends StagedAggregat ) } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { val Array(x, y, label) = seq state.insert(cb, x, y, label) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -745,7 +746,8 @@ class DownsampleAggregator(arrayType: VirtualTypeWithReq) extends StagedAggregat other: DownsampleState, ): Unit = state.merge(cb, other) - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = // deepCopy is handled by state.resultArray IEmitCode.present(cb, state.resultArray(cb, region, resultPType)) } diff --git a/hail/hail/src/is/hail/expr/ir/agg/GroupedAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/GroupedAggregator.scala index c1ce565ad86..45ec86a4abf 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/GroupedAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/GroupedAggregator.scala @@ -51,7 +51,7 @@ class GroupedBTreeKey( } val regionIdx: Value[Int] = new Value[Int] { - def get: Code[Int] = Region.loadInt(storageType.fieldOffset(offset, 1)) + override def get: Code[Int] = Region.loadInt(storageType.fieldOffset(offset, 1)) } val container = new TupleAggregatorState(kb, states, region, containerOffset(offset), regionIdx) @@ -89,7 +89,7 @@ class GroupedBTreeKey( cb += Region.storeInt(storageType.fieldOffset(off, 1), idx) def containerOffset(off: Value[Long]): Value[Long] = new Value[Long] { - def get: Code[Long] = storageType.fieldOffset(off, 2) + override def get: Code[Long] = storageType.fieldOffset(off, 2) } override def isEmpty(cb: EmitCodeBuilder, off: Code[Long]): Value[Boolean] = @@ -153,7 +153,7 @@ class DictState( private val _elt = kb.genFieldThisRef[Long]() private val initStatesOffset: Value[Long] = new Value[Long] { - def get: Code[Long] = typ.loadField(off, 0) + override def get: Code[Long] = typ.loadField(off, 0) } val initContainer: TupleAggregatorState = @@ -246,13 +246,13 @@ class DictState( ) } - def copyFromAddress(cb: EmitCodeBuilder, src: Value[Long]): Unit = { + override def copyFromAddress(cb: EmitCodeBuilder, src: Value[Long]): Unit = { init(cb, cb => initContainer.copyFrom(cb, cb.memoize(typ.loadField(src, 0)))) cb.assign(size, Region.loadInt(typ.loadField(src, 1))) tree.deepCopy(cb, cb.memoize(Region.loadAddress(typ.loadField(src, 2)))) } - def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { + override def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { val serializers = nested.states.map(_.serialize(codec)) { (cb: EmitCodeBuilder, ob: Value[OutputBuffer]) => @@ -275,7 +275,7 @@ class DictState( } } - def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { + override def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { val deserializers = nested.states.map(_.deserialize(codec)) { (cb: EmitCodeBuilder, ib: Value[InputBuffer]) => @@ -314,17 +314,17 @@ class GroupedAggregator(ktV: VirtualTypeWithReq, nestedAggs: Array[StagedAggrega val initOpTypes: Seq[Type] = Array(TVoid) val seqOpTypes: Seq[Type] = Array(ktV.t, TVoid) - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { val Array(inits) = init state.init(cb, cb => cb += inits.asVoid()) } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { val Array(key, seqs) = seq state.withContainer(cb, key, (cb) => cb += seqs.asVoid()) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], diff --git a/hail/hail/src/is/hail/expr/ir/agg/ImputeTypeAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/ImputeTypeAggregator.scala index 22d63b43a7b..509105c9095 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/ImputeTypeAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/ImputeTypeAggregator.scala @@ -168,20 +168,20 @@ class ImputeTypeAggregator() extends StagedAggregator { type State = ImputeTypeState - def resultEmitType = ImputeTypeState.resultEmitType + override def resultEmitType = ImputeTypeState.resultEmitType - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { assert(init.length == 0) state.initialize(cb) } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { val Array(s) = seq state.seqOp(cb, s) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -190,7 +190,8 @@ class ImputeTypeAggregator() extends StagedAggregator { ): Unit = state.combOp(cb, other) - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = { + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = { val emitCodes = Array( state.getAnyNonMissing, state.getAllDefined, diff --git a/hail/hail/src/is/hail/expr/ir/agg/LinearRegressionAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/LinearRegressionAggregator.scala index 6f43d58ccb4..c6b4559ff09 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/LinearRegressionAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/LinearRegressionAggregator.scala @@ -134,7 +134,7 @@ class LinearRegressionAggregator() extends StagedAggregator { cb += Region.storeInt(stateType.loadField(state.off, 2), k0) } - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { val Array(kt, k0t) = init kt.toI(cb) .consume( @@ -254,7 +254,7 @@ class LinearRegressionAggregator() extends StagedAggregator { ) } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { val Array(y, x) = seq y.toI(cb) .consume( @@ -321,7 +321,7 @@ class LinearRegressionAggregator() extends StagedAggregator { ) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -330,7 +330,8 @@ class LinearRegressionAggregator() extends StagedAggregator { ): Unit = combOpF(state, other)(cb) - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = { + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = { val resAddr = cb.newLocal[Long]( "linear_regression_agg_res", Code.invokeScalaObject4[Region, Long, Long, Int, Long]( diff --git a/hail/hail/src/is/hail/expr/ir/agg/MonoidAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/MonoidAggregator.scala index c6162cb2ce7..58bacbdc7db 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/MonoidAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/MonoidAggregator.scala @@ -22,12 +22,12 @@ trait StagedMonoidSpec { class MonoidAggregator(monoid: StagedMonoidSpec) extends StagedAggregator { type State = PrimitiveRVAState val sType = SType.canonical(monoid.typ) - def resultEmitType = EmitType(sType, monoid.neutral.isDefined) + override def resultEmitType = EmitType(sType, monoid.neutral.isDefined) val initOpTypes: Seq[Type] = Array[Type]() val seqOpTypes: Seq[Type] = Array[Type](monoid.typ) - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { assert(init.length == 0) val stateRequired = state.vtypes.head.r.required val ev = state.fields(0) @@ -40,14 +40,14 @@ class MonoidAggregator(monoid: StagedMonoidSpec) extends StagedAggregator { } } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { val Array(elt) = seq val ev = state.fields(0) val update = cb.memoizeField(elt, "monoid_elt") combine(cb, ev, update) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -59,7 +59,8 @@ class MonoidAggregator(monoid: StagedMonoidSpec) extends StagedAggregator { combine(cb, ev1, ev2) } - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = state.fields(0).toI(cb) private def combine( @@ -81,7 +82,7 @@ class MonoidAggregator(monoid: StagedMonoidSpec) extends StagedAggregator { class ComparisonMonoid(val typ: Type, val functionName: String) extends StagedMonoidSpec { - def neutral: Option[Value[_]] = None + override def neutral: Option[Value[_]] = None private def cmp[T](v1: Code[T], v2: Code[T])(implicit tct: ClassTag[T]): Code[T] = Code.invokeStatic2[Math, T, T, T](functionName, v1, v2) @@ -89,7 +90,7 @@ class ComparisonMonoid(val typ: Type, val functionName: String) extends StagedMo private def nancmp[T](v1: Code[T], v2: Code[T])(implicit tct: ClassTag[T]): Code[T] = Code.invokeScalaObject2[T, T, T](UtilFunctions.getClass, "nan" + functionName, v1, v2) - def apply(cb: EmitCodeBuilder, v1: Value[_], v2: Value[_]): Value[_] = typ match { + override def apply(cb: EmitCodeBuilder, v1: Value[_], v2: Value[_]): Value[_] = typ match { case TInt32 => cb.memoize(cmp[Int](coerce(v1), coerce(v2))) case TInt64 => cb.memoize(cmp[Long](coerce(v1), coerce(v2))) case TFloat32 => cb.memoize(nancmp[Float](coerce(v1), coerce(v2))) @@ -100,13 +101,13 @@ class ComparisonMonoid(val typ: Type, val functionName: String) extends StagedMo class SumMonoid(val typ: Type) extends StagedMonoidSpec { - def neutral: Option[Value[_]] = Some(typ match { + override def neutral: Option[Value[_]] = Some(typ match { case TInt64 => const(0L) case TFloat64 => const(0.0d) case _ => throw new UnsupportedOperationException(s"can't sum over type $typ") }) - def apply(cb: EmitCodeBuilder, v1: Value[_], v2: Value[_]): Value[_] = typ match { + override def apply(cb: EmitCodeBuilder, v1: Value[_], v2: Value[_]): Value[_] = typ match { case TInt64 => cb.memoize(coerce[Long](v1) + coerce[Long](v2)) case TFloat64 => cb.memoize(coerce[Double](v1) + coerce[Double](v2)) case _ => throw new UnsupportedOperationException(s"can't sum over type $typ") @@ -115,13 +116,13 @@ class SumMonoid(val typ: Type) extends StagedMonoidSpec { class ProductMonoid(val typ: Type) extends StagedMonoidSpec { - def neutral: Option[Value[_]] = Some(typ match { + override def neutral: Option[Value[_]] = Some(typ match { case TInt64 => const(1L) case TFloat64 => const(1.0d) case _ => throw new UnsupportedOperationException(s"can't product over type $typ") }) - def apply(cb: EmitCodeBuilder, v1: Value[_], v2: Value[_]): Value[_] = typ match { + override def apply(cb: EmitCodeBuilder, v1: Value[_], v2: Value[_]): Value[_] = typ match { case TInt64 => cb.memoize(coerce[Long](v1) * coerce[Long](v2)) case TFloat64 => cb.memoize(coerce[Double](v1) * coerce[Double](v2)) case _ => throw new UnsupportedOperationException(s"can't product over type $typ") diff --git a/hail/hail/src/is/hail/expr/ir/agg/NDArraySumAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/NDArraySumAggregator.scala index 7d403580a45..912322c82bf 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/NDArraySumAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/NDArraySumAggregator.scala @@ -99,7 +99,8 @@ class NDArraySumAggregator(ndVTyp: VirtualTypeWithReq) extends StagedAggregator cb.invokeVoid(combOpMethod, cb.this_) } - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = state.get(cb).map(cb)(sv => sv.copyToRegion(cb, region, sv.st)) } diff --git a/hail/hail/src/is/hail/expr/ir/agg/PrevNonNullAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/PrevNonNullAggregator.scala index 5f124f46210..74e6fee5320 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/PrevNonNullAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/PrevNonNullAggregator.scala @@ -14,12 +14,12 @@ class PrevNonNullAggregator(typ: VirtualTypeWithReq) extends StagedAggregator { val initOpTypes: Seq[Type] = Array[Type]() val seqOpTypes: Seq[Type] = Array[Type](typ.t) - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { assert(init.length == 0) state.storeMissing(cb) } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { val Array(elt: EmitCode) = seq elt.toI(cb) .consume( @@ -29,7 +29,7 @@ class PrevNonNullAggregator(typ: VirtualTypeWithReq) extends StagedAggregator { ) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -43,6 +43,7 @@ class PrevNonNullAggregator(typ: VirtualTypeWithReq) extends StagedAggregator { sc => state.storeNonmissing(cb, sc), ) - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = state.get(cb).map(cb)(sv => sv.copyToRegion(cb, region, sv.st)) } diff --git a/hail/hail/src/is/hail/expr/ir/agg/ReservoirSampleAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/ReservoirSampleAggregator.scala index ecc38217718..d08f5d0b64c 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/ReservoirSampleAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/ReservoirSampleAggregator.scala @@ -33,10 +33,10 @@ class ReservoirSampleRVAS(val eltType: VirtualTypeWithReq, val kb: EmitClassBuil private val garbageOffset: Code[Long] => Code[Long] = storageType.loadField(_, 2) private val builderStateOffset: Code[Long] => Code[Long] = storageType.loadField(_, 3) - def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = + override def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = cb += region.getNewRegion(regionSize) - def createState(cb: EmitCodeBuilder): Unit = { + override def createState(cb: EmitCodeBuilder): Unit = { cb.assign(rand, Code.newInstance[java.util.Random]()) cb.if_(region.isNull, cb.assign(r, Region.stagedCreate(regionSize, kb.pool()))) } @@ -70,14 +70,14 @@ class ReservoirSampleRVAS(val eltType: VirtualTypeWithReq, val kb: EmitClassBuil ) } - def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { + override def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { (cb: EmitCodeBuilder, ob: Value[OutputBuffer]) => cb += ob.writeInt(maxSize) cb += ob.writeLong(seenSoFar) builder.serialize(codec)(cb, ob) } - def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { + override def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { (cb: EmitCodeBuilder, ib: Value[InputBuffer]) => cb.assign(maxSize, ib.readInt()) cb.assign(seenSoFar, ib.readLong()) @@ -245,7 +245,7 @@ class ReservoirSampleRVAS(val eltType: VirtualTypeWithReq, val kb: EmitClassBuil builder.loadElement(cb, idx).toI(cb) } - def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = { + override def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = { cb.assign(maxSize, Region.loadInt(maxSizeOffset(src))) cb.assign(seenSoFar, Region.loadLong(elementsSeenOffset(src))) cb.assign(garbage, Region.loadLong(garbageOffset(src))) @@ -262,8 +262,11 @@ class ReservoirSampleAggregator(typ: VirtualTypeWithReq) extends StagedAggregato val initOpTypes: Seq[Type] = Array(TInt32) val seqOpTypes: Seq[Type] = Array(typ.t) - protected def _initOp(cb: EmitCodeBuilder, state: ReservoirSampleRVAS, init: Array[EmitCode]) - : Unit = { + override protected def _initOp( + cb: EmitCodeBuilder, + state: ReservoirSampleRVAS, + init: Array[EmitCode], + ): Unit = { assert(init.length == 1) val Array(sizeTriplet) = init sizeTriplet.toI(cb) @@ -274,13 +277,16 @@ class ReservoirSampleAggregator(typ: VirtualTypeWithReq) extends StagedAggregato ) } - protected def _seqOp(cb: EmitCodeBuilder, state: ReservoirSampleRVAS, seq: Array[EmitCode]) - : Unit = { + override protected def _seqOp( + cb: EmitCodeBuilder, + state: ReservoirSampleRVAS, + seq: Array[EmitCode], + ): Unit = { val Array(elt: EmitCode) = seq state.seqOp(cb, elt) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -288,7 +294,8 @@ class ReservoirSampleAggregator(typ: VirtualTypeWithReq) extends StagedAggregato other: ReservoirSampleRVAS, ): Unit = state.combine(cb, other) - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = // deepCopy is handled by state.resultArray IEmitCode.present(cb, state.resultArray(cb, region, resultPType)) } diff --git a/hail/hail/src/is/hail/expr/ir/agg/TakeAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/TakeAggregator.scala index 00295f2efad..16e8746b34b 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/TakeAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/TakeAggregator.scala @@ -25,9 +25,10 @@ class TakeRVAS(val eltType: VirtualTypeWithReq, val kb: EmitClassBuilder[_]) private val maxSizeOffset: Code[Long] => Code[Long] = storageType.loadField(_, 0) private val builderStateOffset: Code[Long] => Code[Long] = storageType.loadField(_, 1) - def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = cb += region.getNewRegion(regionSize) + override def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = + cb += region.getNewRegion(regionSize) - def createState(cb: EmitCodeBuilder): Unit = + override def createState(cb: EmitCodeBuilder): Unit = cb.if_(region.isNull, cb.assign(r, Region.stagedCreate(regionSize, kb.pool()))) override def load( @@ -55,13 +56,13 @@ class TakeRVAS(val eltType: VirtualTypeWithReq, val kb: EmitClassBuilder[_]) ) } - def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { + override def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { (cb: EmitCodeBuilder, ob: Value[OutputBuffer]) => cb += ob.writeInt(maxSize) builder.serialize(codec)(cb, ob) } - def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { + override def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { (cb: EmitCodeBuilder, ib: Value[InputBuffer]) => cb.assign(maxSize, ib.readInt()) builder.deserialize(codec)(cb, ib) @@ -97,7 +98,7 @@ class TakeRVAS(val eltType: VirtualTypeWithReq, val kb: EmitClassBuilder[_]) builder.loadElement(cb, idx).toI(cb) } - def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = { + override def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = { cb.assign(maxSize, Region.loadInt(maxSizeOffset(src))) builder.copyFrom(cb, builderStateOffset(src)) } @@ -112,7 +113,7 @@ class TakeAggregator(typ: VirtualTypeWithReq) extends StagedAggregator { val initOpTypes: Seq[Type] = Array(TInt32) val seqOpTypes: Seq[Type] = Array(typ.t) - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { assert(init.length == 1) val Array(sizeTriplet) = init sizeTriplet.toI(cb) @@ -123,12 +124,12 @@ class TakeAggregator(typ: VirtualTypeWithReq) extends StagedAggregator { ) } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { val Array(elt: EmitCode) = seq state.seqOp(cb, elt) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -136,7 +137,8 @@ class TakeAggregator(typ: VirtualTypeWithReq) extends StagedAggregator { other: TakeRVAS, ): Unit = state.combine(cb, other) - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = // deepCopy is handled by state.resultArray IEmitCode.present(cb, state.resultArray(cb, region, resultPType)) } diff --git a/hail/hail/src/is/hail/expr/ir/agg/TakeByAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/TakeByAggregator.scala index 3fa6237ba57..2395c8f9c32 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/TakeByAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/TakeByAggregator.scala @@ -100,9 +100,10 @@ class TakeByRVAS( } } - def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = cb += region.getNewRegion(regionSize) + override def newState(cb: EmitCodeBuilder, off: Value[Long]): Unit = + cb += region.getNewRegion(regionSize) - def createState(cb: EmitCodeBuilder): Unit = + override def createState(cb: EmitCodeBuilder): Unit = cb.if_( region.isNull, { cb.assign(r, Region.stagedCreate(regionSize, kb.pool())) @@ -193,7 +194,7 @@ class TakeByRVAS( }) } - def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = { + override def copyFrom(cb: EmitCodeBuilder, src: Value[Long]): Unit = { maybeGCCode( cb, { cb => @@ -205,7 +206,7 @@ class TakeByRVAS( )({ cb => cb.assign(maxGarbage, Region.loadInt(storageType.fieldOffset(src, 4))) }) } - def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { + override def serialize(codec: BufferSpec): (EmitCodeBuilder, Value[OutputBuffer]) => Unit = { (cb: EmitCodeBuilder, ob: Value[OutputBuffer]) => maybeGCCode( cb, @@ -218,7 +219,7 @@ class TakeByRVAS( )(cb => cb += ob.writeInt(maxGarbage), runBefore = true) } - def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { + override def deserialize(codec: BufferSpec): (EmitCodeBuilder, Value[InputBuffer]) => Unit = { (cb: EmitCodeBuilder, ib: Value[InputBuffer]) => maybeGCCode( cb, @@ -676,7 +677,7 @@ class TakeByAggregator(valueType: VirtualTypeWithReq, keyType: VirtualTypeWithRe val initOpTypes: Seq[Type] = Array(TInt32) val seqOpTypes: Seq[Type] = Array(valueType.t, keyType.t) - protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { + override protected def _initOp(cb: EmitCodeBuilder, state: State, init: Array[EmitCode]): Unit = { assert(init.length == 1) val Array(sizeTriplet) = init sizeTriplet.toI(cb) @@ -687,12 +688,12 @@ class TakeByAggregator(valueType: VirtualTypeWithReq, keyType: VirtualTypeWithRe ) } - protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { + override protected def _seqOp(cb: EmitCodeBuilder, state: State, seq: Array[EmitCode]): Unit = { val Array(value: EmitCode, key: EmitCode) = seq state.seqOp(cb, value, key) } - protected def _combOp( + override protected def _combOp( ctx: ExecuteContext, cb: EmitCodeBuilder, region: Value[Region], @@ -700,7 +701,8 @@ class TakeByAggregator(valueType: VirtualTypeWithReq, keyType: VirtualTypeWithRe other: TakeByRVAS, ): Unit = state.combine(cb, other) - protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]): IEmitCode = + override protected def _result(cb: EmitCodeBuilder, state: State, region: Value[Region]) + : IEmitCode = // state.result does a deep copy IEmitCode.present( cb, diff --git a/hail/hail/src/is/hail/expr/ir/functions/ApproxCDFFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/ApproxCDFFunctions.scala index 657ca2a1a9a..a09d4ceaef9 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/ApproxCDFFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/ApproxCDFFunctions.scala @@ -51,7 +51,7 @@ object ApproxCDFFunctions extends RegistryFunctions { unwrapReturn(cb, r, SBaseStructPointer(statePType), row).asBaseStruct } - def registerAll(): Unit = { + override def registerAll(): Unit = { registerSCode3( "approxCDFCombine", TInt32, diff --git a/hail/hail/src/is/hail/expr/ir/functions/ArrayFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/ArrayFunctions.scala index 1123e18e4bb..3ede0b5c9a0 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/ArrayFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/ArrayFunctions.scala @@ -120,7 +120,7 @@ object ArrayFunctions extends RegistryFunctions { foldIR(ToStream(a), one)((product, v) => ApplyBinaryPrimOp(Multiply(), product, v)) } - def registerAll(): Unit = { + override def registerAll(): Unit = { registerIR1("isEmpty", TArray(tv("T")), TBoolean)((_, a, _) => isEmpty(a)) registerIR2("extend", TArray(tv("T")), TArray(tv("T")), TArray(tv("T")))((_, a, b, _) => diff --git a/hail/hail/src/is/hail/expr/ir/functions/CallFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/CallFunctions.scala index 6920d0b760a..aab47deda2e 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/CallFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/CallFunctions.scala @@ -12,7 +12,7 @@ import is.hail.variant._ import scala.reflect.classTag object CallFunctions extends RegistryFunctions { - def registerAll(): Unit = { + override def registerAll(): Unit = { registerWrappedScalaFunction1("Call", TString, TCall, (rt: Type, st: SType) => SCanonicalCall)( Call.getClass, "parse", diff --git a/hail/hail/src/is/hail/expr/ir/functions/DictFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/DictFunctions.scala index 5771280f4f1..142efca444d 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/DictFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/DictFunctions.scala @@ -49,7 +49,7 @@ object DictFunctions extends RegistryFunctions { val tdict = TDict(tv("key"), tv("value")) - def registerAll(): Unit = { + override def registerAll(): Unit = { registerIR1("isEmpty", tdict, TBoolean)((_, d, _) => ArrayFunctions.isEmpty(CastToArray(d))) registerIR2("contains", tdict, tv("key"), TBoolean)((_, a, b, _) => contains(a, b)) diff --git a/hail/hail/src/is/hail/expr/ir/functions/Functions.scala b/hail/hail/src/is/hail/expr/ir/functions/Functions.scala index 287866330e0..6351769d822 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/Functions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/Functions.scala @@ -1269,7 +1269,7 @@ abstract class UnseededMissingnessObliviousJVMFunction( args: SValue* ): SValue - def apply( + override def apply( r: EmitRegion, returnType: SType, typeParameters: Seq[Type], diff --git a/hail/hail/src/is/hail/expr/ir/functions/GenotypeFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/GenotypeFunctions.scala index 272686ac2f7..48f5e114c85 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/GenotypeFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/GenotypeFunctions.scala @@ -9,7 +9,7 @@ import is.hail.types.virtual.{TArray, TFloat64, TInt32, Type} object GenotypeFunctions extends RegistryFunctions { - def registerAll(): Unit = { + override def registerAll(): Unit = { registerSCode1("gqFromPL", TArray(tv("N", "int32")), TInt32, (_: Type, _: SType) => SInt32) { case (_, cb, _, pl: SIndexableValue, errorID) => val m = cb.newLocal[Int]("m", 99) diff --git a/hail/hail/src/is/hail/expr/ir/functions/IntervalFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/IntervalFunctions.scala index bcf277ce170..91bc0e9d1da 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/IntervalFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/IntervalFunctions.scala @@ -306,7 +306,7 @@ object IntervalFunctions extends RegistryFunctions { BinarySearch.equalRange(cb, array, compare, ltNeedle, gtNeedle, 0, array.loadLength) } - def registerAll(): Unit = { + override def registerAll(): Unit = { registerIEmitCode4( "Interval", tv("T"), diff --git a/hail/hail/src/is/hail/expr/ir/functions/LocusFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/LocusFunctions.scala index 7f607977aa7..fd799a7be4c 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/LocusFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/LocusFunctions.scala @@ -118,7 +118,7 @@ object LocusFunctions extends RegistryFunctions { ))) } - def registerAll(): Unit = { + override def registerAll(): Unit = { val locusClass = Locus.getClass registerSCode1( diff --git a/hail/hail/src/is/hail/expr/ir/functions/MathFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/MathFunctions.scala index 517b50e431b..0bc7236727f 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/MathFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/MathFunctions.scala @@ -111,7 +111,7 @@ object MathFunctions extends RegistryFunctions { val mathPackageClass: Class[_] = Class.forName("scala.math.package$") - def registerAll(): Unit = { + override def registerAll(): Unit = { val thisClass = getClass val statsPackageClass = Class.forName("is.hail.stats.package$") val jMathClass = classOf[java.lang.Math] diff --git a/hail/hail/src/is/hail/expr/ir/functions/RandomSeededFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/RandomSeededFunctions.scala index a5036a39b90..a9ae2605e7d 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/RandomSeededFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/RandomSeededFunctions.scala @@ -111,7 +111,7 @@ object RandomSeededFunctions extends RegistryFunctions { ) } - def registerAll(): Unit = { + override def registerAll(): Unit = { registerSCode3( "rand_unif", TRNGState, diff --git a/hail/hail/src/is/hail/expr/ir/functions/ReferenceGenomeFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/ReferenceGenomeFunctions.scala index b51016e90d2..682a226a17a 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/ReferenceGenomeFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/ReferenceGenomeFunctions.scala @@ -14,7 +14,7 @@ object ReferenceGenomeFunctions extends RegistryFunctions { def rgCode(mb: EmitMethodBuilder[_], rg: String): Code[ReferenceGenome] = mb.getReferenceGenome(rg) - def registerAll(): Unit = { + override def registerAll(): Unit = { registerSCode1t( "isValidContig", Array(LocusFunctions.tlocus("R")), diff --git a/hail/hail/src/is/hail/expr/ir/functions/RelationalFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/RelationalFunctions.scala index 03a7070783b..b4e19fc6dae 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/RelationalFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/RelationalFunctions.scala @@ -48,7 +48,7 @@ case class WrappedMatrixToTableFunction( function.typ(mType) // MatrixType RVDTypes will go away } - def execute(ctx: ExecuteContext, tv: TableValue): TableValue = + override def execute(ctx: ExecuteContext, tv: TableValue): TableValue = function.execute(ctx, tv.toMatrixValue(colKey, colsFieldName, entriesFieldName)) override def preservesPartitionCounts: Boolean = function.preservesPartitionCounts @@ -82,13 +82,13 @@ case class WrappedMatrixToValueFunction( colKey: IndexedSeq[String], ) extends TableToValueFunction { - def typ(childType: TableType): Type = + override def typ(childType: TableType): Type = function.typ(MatrixType.fromTableType(childType, colsFieldName, entriesFieldName, colKey)) - def unionRequiredness(childType: RTable, resultType: TypeWithRequiredness): Unit = + override def unionRequiredness(childType: RTable, resultType: TypeWithRequiredness): Unit = function.unionRequiredness(childType, resultType) - def execute(ctx: ExecuteContext, tv: TableValue): Any = + override def execute(ctx: ExecuteContext, tv: TableValue): Any = function.execute(ctx, tv.toMatrixValue(colKey, colsFieldName, entriesFieldName)) } diff --git a/hail/hail/src/is/hail/expr/ir/functions/SetFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/SetFunctions.scala index 9446ddc7ede..453e0b72d40 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/SetFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/SetFunctions.scala @@ -20,7 +20,7 @@ object SetFunctions extends RegistryFunctions { ) } - def registerAll(): Unit = { + override def registerAll(): Unit = { registerIR1("toSet", TArray(tv("T")), TSet(tv("T")))((_, a, _) => ToSet(ToStream(a))) registerIR1("isEmpty", TSet(tv("T")), TBoolean) { (_, s, _) => diff --git a/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala index 5b1fbe3acaf..8f118f75753 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala @@ -283,7 +283,7 @@ object StringFunctions extends RegistryFunctions { .parse(timeStr) .getLong(ChronoField.INSTANT_SECONDS) - def registerAll(): Unit = { + override def registerAll(): Unit = { val thisClass = getClass registerSCode1("length", TString, TInt32, (_: Type, _: SType) => SInt32) { diff --git a/hail/hail/src/is/hail/expr/ir/functions/TableCalculateNewPartitions.scala b/hail/hail/src/is/hail/expr/ir/functions/TableCalculateNewPartitions.scala index 9c502d21ccc..9eb74b027e0 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/TableCalculateNewPartitions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/TableCalculateNewPartitions.scala @@ -10,9 +10,10 @@ import is.hail.utils._ case class TableCalculateNewPartitions( nPartitions: Int ) extends TableToValueFunction { - def typ(childType: TableType): Type = TArray(TInterval(childType.keyType)) + override def typ(childType: TableType): Type = TArray(TInterval(childType.keyType)) - def unionRequiredness(childType: types.RTable, resultType: types.TypeWithRequiredness): Unit = { + override def unionRequiredness(childType: types.RTable, resultType: types.TypeWithRequiredness) + : Unit = { val rinterval = types.tcoerce[types.RInterval]( types.tcoerce[types.RIterable](resultType).elementType ) @@ -24,7 +25,7 @@ case class TableCalculateNewPartitions( } } - def execute(ctx: ExecuteContext, tv: TableValue): Any = { + override def execute(ctx: ExecuteContext, tv: TableValue): Any = { val rvd = tv.rvd if (rvd.typ.key.isEmpty) FastSeq() diff --git a/hail/hail/src/is/hail/expr/ir/functions/UtilFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/UtilFunctions.scala index b089f9deb42..8abadd59b41 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/UtilFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/UtilFunctions.scala @@ -201,7 +201,7 @@ object UtilFunctions extends RegistryFunctions { ) } - def registerAll(): Unit = { + override def registerAll(): Unit = { val thisClass = getClass registerSCode4( diff --git a/hail/hail/src/is/hail/expr/ir/lowering/LowerBlockMatrixIR.scala b/hail/hail/src/is/hail/expr/ir/lowering/LowerBlockMatrixIR.scala index 644d3fc3226..5de628750d3 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/LowerBlockMatrixIR.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/LowerBlockMatrixIR.scala @@ -83,19 +83,19 @@ abstract class BlockMatrixStage(val broadcastVals: IndexedSeq[Ref], val ctxType: val outer = this val newCtxType = TStruct("old" -> ctxType, "new" -> newTyp) new BlockMatrixStage(broadcastVals, newCtxType) { - def blockContext(idx: (Int, Int)): IR = + override def blockContext(idx: (Int, Int)): IR = makestruct("old" -> outer.blockContext(idx), "new" -> newCtx(idx)) - def blockBody(ctxRef: Ref): IR = bindIR(GetField(ctxRef, "old"))(outer.blockBody) + override def blockBody(ctxRef: Ref): IR = bindIR(GetField(ctxRef, "old"))(outer.blockBody) } } def mapBody(f: (IR, IR) => IR): BlockMatrixStage = { val outer = this new BlockMatrixStage(broadcastVals, outer.ctxType) { - def blockContext(idx: (Int, Int)): IR = outer.blockContext(idx) + override def blockContext(idx: (Int, Int)): IR = outer.blockContext(idx) - def blockBody(ctxRef: Ref): IR = f(ctxRef, outer.blockBody(ctxRef)) + override def blockBody(ctxRef: Ref): IR = f(ctxRef, outer.blockBody(ctxRef)) } } @@ -107,7 +107,7 @@ abstract class BlockMatrixStage(val broadcastVals: IndexedSeq[Ref], val ctxType: val outer = this val ctxType = TArray(TArray(TTuple(TTuple(TInt64, TInt64), outer.ctxType))) new BlockMatrixStage(outer.broadcastVals, ctxType) { - def blockContext(idx: (Int, Int)): IR = { + override def blockContext(idx: (Int, Int)): IR = { val i = idx._1 val j = idx._2 MakeArray(rowBlocks(i).map { ii => @@ -125,7 +125,7 @@ abstract class BlockMatrixStage(val broadcastVals: IndexedSeq[Ref], val ctxType: }: _*) } - def blockBody(ctxRef: Ref): IR = { + override def blockBody(ctxRef: Ref): IR = { NDArrayConcat( ToArray(mapIR(ToStream(ctxRef)) { ctxRows => NDArrayConcat( @@ -401,7 +401,7 @@ case class DenseContexts(sparsity: MatrixSparsity.Dense, dynamic: DynamicDenseCo DenseContexts(ib, sparsity, newContexts) } - def withNewSparsity(ib: IRBuilder, newSparsity: MatrixSparsity): BMSContexts = { + override def withNewSparsity(ib: IRBuilder, newSparsity: MatrixSparsity): BMSContexts = { require(newSparsity.nRows == sparsity.nRows && newSparsity.nCols == sparsity.nCols) newSparsity match { case sparse: MatrixSparsity.Sparse => @@ -413,7 +413,7 @@ case class DenseContexts(sparsity: MatrixSparsity.Dense, dynamic: DynamicDenseCo } } - def groupedByCol(ib: IRBuilder): DenseContexts = { + override def groupedByCol(ib: IRBuilder): DenseContexts = { val groupedContexts = ToArray(mapIR(rangeIR(nCols)) { col => sliceArrayIR(contexts, col * nRows, (col + 1) * nRows) }) @@ -476,7 +476,7 @@ case class DenseContexts(sparsity: MatrixSparsity.Dense, dynamic: DynamicDenseCo DenseContexts(ib, MatrixSparsity.Dense(rowDeps.length, colDeps.length), newContexts) } - def collect(makeBlock: (Ref, Ref, Ref) => IR): IR = { + override def collect(makeBlock: (Ref, Ref, Ref) => IR): IR = { NDArrayConcat( ToArray(mapIR(rangeIR(nCols)) { j => val colBlocks = mapIR(rangeIR(nRows)) { i => @@ -518,7 +518,7 @@ case class SparseContexts( override def map(ib: IRBuilder)(f: (IR, IR, IR, IR) => IR): SparseContexts = new SparseContexts(sparsity, dynamic.map(ib)(f)) - def withNewSparsity(ib: IRBuilder, newSparsity: MatrixSparsity): BMSContexts = { + override def withNewSparsity(ib: IRBuilder, newSparsity: MatrixSparsity): BMSContexts = { require(newSparsity.nRows == sparsity.nRows && newSparsity.nCols == sparsity.nCols) newSparsity match { case sparse: MatrixSparsity.Sparse => @@ -1474,7 +1474,7 @@ object LowerBlockMatrixIR { val right = lower(rightIR) val newCtxType = TArray(TTuple(left.ctxType, right.ctxType)) new BlockMatrixStage(left.broadcastVals ++ right.broadcastVals, newCtxType) { - def blockContext(idx: (Int, Int)): IR = { + override def blockContext(idx: (Int, Int)): IR = { val (i, j) = idx MakeArray( Array.tabulate[Option[IR]](leftIR.typ.nColBlocks) { k => @@ -1489,7 +1489,7 @@ object LowerBlockMatrixIR { ) } - def blockBody(ctxRef: Ref): IR = { + override def blockBody(ctxRef: Ref): IR = { val tupleNDArrayStream = ToStream(ctxRef) val streamElementName = freshName() val streamElementRef = diff --git a/hail/hail/src/is/hail/expr/ir/lowering/LowerDistributedSort.scala b/hail/hail/src/is/hail/expr/ir/lowering/LowerDistributedSort.scala index 92330bd8e1a..62c4dd70800 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/LowerDistributedSort.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/LowerDistributedSort.scala @@ -102,15 +102,17 @@ object LowerDistributedSort extends Logging { override def isDistinctlyKeyed: Boolean = false // FIXME: No default value - def rowRequiredness(ctx: ExecuteContext, requestedType: TableType): VirtualTypeWithReq = + override def rowRequiredness(ctx: ExecuteContext, requestedType: TableType) + : VirtualTypeWithReq = VirtualTypeWithReq.subset(requestedType.rowType, rt.rowType) - def globalRequiredness(ctx: ExecuteContext, requestedType: TableType): VirtualTypeWithReq = + override def globalRequiredness(ctx: ExecuteContext, requestedType: TableType) + : VirtualTypeWithReq = VirtualTypeWithReq.subset(requestedType.globalType, rt.globalType) override def toJValue: JValue = JString("LocalSortReader") - def renderShort(): String = "LocalSortReader" + override def renderShort(): String = "LocalSortReader" override def defaultRender(): String = "LocalSortReader" @@ -982,15 +984,16 @@ case class DistributionSortReader( override def isDistinctlyKeyed: Boolean = false // FIXME: No default value - def rowRequiredness(ctx: ExecuteContext, requestedType: TableType): VirtualTypeWithReq = + override def rowRequiredness(ctx: ExecuteContext, requestedType: TableType): VirtualTypeWithReq = VirtualTypeWithReq.subset(requestedType.rowType, rt.rowType) - def globalRequiredness(ctx: ExecuteContext, requestedType: TableType): VirtualTypeWithReq = + override def globalRequiredness(ctx: ExecuteContext, requestedType: TableType) + : VirtualTypeWithReq = VirtualTypeWithReq.subset(requestedType.globalType, rt.globalType) override def toJValue: JValue = JString("DistributionSortReader") - def renderShort(): String = "DistributionSortReader" + override def renderShort(): String = "DistributionSortReader" override def defaultRender(): String = "DistributionSortReader" diff --git a/hail/hail/src/is/hail/expr/ir/lowering/LoweringPass.scala b/hail/hail/src/is/hail/expr/ir/lowering/LoweringPass.scala index c238e529b00..9fadd301880 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/LoweringPass.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/LoweringPass.scala @@ -61,7 +61,7 @@ case object LowerMatrixToTablePass extends LoweringPass { val after: Invariant = NoMatrixIR val context: String = "LowerMatrixToTable" - def transform(ctx: ExecuteContext, ir: BaseIR): BaseIR = ir match { + override def transform(ctx: ExecuteContext, ir: BaseIR): BaseIR = ir match { case x: IR => LowerMatrixIR(ctx, x) case x: TableIR => LowerMatrixIR(ctx, x) case x: MatrixIR => LowerMatrixIR(ctx, x) @@ -74,7 +74,7 @@ case object LiftRelationalValuesToRelationalLets extends LoweringPass { val after: Invariant = NoMatrixIR val context: String = "LiftRelationalValuesToRelationalLets" - def transform(ctx: ExecuteContext, ir: BaseIR): BaseIR = LiftRelationalValues(ir) + override def transform(ctx: ExecuteContext, ir: BaseIR): BaseIR = LiftRelationalValues(ir) } case object LegacyInterpretNonCompilablePass extends LoweringPass { @@ -82,7 +82,8 @@ case object LegacyInterpretNonCompilablePass extends LoweringPass { val after: Invariant = NoMatrixIR and NoRelationalLets and CompilableValueIRs val context: String = "InterpretNonCompilable" - def transform(ctx: ExecuteContext, ir: BaseIR): BaseIR = LowerOrInterpretNonCompilable(ctx, ir) + override def transform(ctx: ExecuteContext, ir: BaseIR): BaseIR = + LowerOrInterpretNonCompilable(ctx, ir) } case object LowerOrInterpretNonCompilablePass extends LoweringPass { @@ -90,7 +91,8 @@ case object LowerOrInterpretNonCompilablePass extends LoweringPass { val after: Invariant = CompilableIR val context: String = "LowerOrInterpretNonCompilable" - def transform(ctx: ExecuteContext, ir: BaseIR): BaseIR = LowerOrInterpretNonCompilable(ctx, ir) + override def transform(ctx: ExecuteContext, ir: BaseIR): BaseIR = + LowerOrInterpretNonCompilable(ctx, ir) } case class LowerToDistributedArrayPass(t: DArrayLowering.Type) extends LoweringPass { @@ -98,7 +100,8 @@ case class LowerToDistributedArrayPass(t: DArrayLowering.Type) extends LoweringP val after: Invariant = CompilableIR val context: String = "LowerToDistributedArray" - def transform(ctx: ExecuteContext, ir: BaseIR): BaseIR = LowerToCDA(ir.asInstanceOf[IR], t, ctx) + override def transform(ctx: ExecuteContext, ir: BaseIR): BaseIR = + LowerToCDA(ir.asInstanceOf[IR], t, ctx) } case object InlineApplyIR extends LoweringPass { @@ -123,7 +126,7 @@ case object LowerArrayAggsToRunAggsPass extends LoweringPass { val after: Invariant = EmittableIR val context: String = "LowerArrayAggsToRunAggs" - def transform(ctx: ExecuteContext, ir: BaseIR): BaseIR = + override def transform(ctx: ExecuteContext, ir: BaseIR): BaseIR = ctx.time { val x = ir.noSharing(ctx) val r = Requiredness(x, ctx) diff --git a/hail/hail/src/is/hail/expr/ir/lowering/RVDToTableStage.scala b/hail/hail/src/is/hail/expr/ir/lowering/RVDToTableStage.scala index 0dc13ec6598..c2a20863657 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/RVDToTableStage.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/RVDToTableStage.scala @@ -81,15 +81,16 @@ case class RVDTableReader(rvd: RVD, globals: IR, rt: RTable) extends TableReader override def isDistinctlyKeyed: Boolean = false - def rowRequiredness(ctx: ExecuteContext, requestedType: TableType): VirtualTypeWithReq = + override def rowRequiredness(ctx: ExecuteContext, requestedType: TableType): VirtualTypeWithReq = VirtualTypeWithReq.subset(requestedType.rowType, rt.rowType) - def globalRequiredness(ctx: ExecuteContext, requestedType: TableType): VirtualTypeWithReq = + override def globalRequiredness(ctx: ExecuteContext, requestedType: TableType) + : VirtualTypeWithReq = VirtualTypeWithReq.subset(requestedType.globalType, rt.globalType) override def toJValue: JValue = JString("RVDTableReader") - def renderShort(): String = "RVDTableReader" + override def renderShort(): String = "RVDTableReader" override def defaultRender(): String = "RVDTableReader" diff --git a/hail/hail/src/is/hail/expr/ir/orderings/BinaryOrdering.scala b/hail/hail/src/is/hail/expr/ir/orderings/BinaryOrdering.scala index 6436b8f67d7..c4220eeb0ec 100644 --- a/hail/hail/src/is/hail/expr/ir/orderings/BinaryOrdering.scala +++ b/hail/hail/src/is/hail/expr/ir/orderings/BinaryOrdering.scala @@ -13,7 +13,7 @@ object BinaryOrdering { val type1: SBinary = t1 val type2: SBinary = t2 - def _compareNonnull(cb: EmitCodeBuilder, x: SValue, y: SValue): Value[Int] = { + override def _compareNonnull(cb: EmitCodeBuilder, x: SValue, y: SValue): Value[Int] = { val xv: SBinaryValue = x.asBinary val yv: SBinaryValue = y.asBinary val xlen = xv.loadLength(cb) diff --git a/hail/hail/src/is/hail/expr/ir/streams/EmitStream.scala b/hail/hail/src/is/hail/expr/ir/streams/EmitStream.scala index a089441a318..a29a7964dc9 100644 --- a/hail/hail/src/is/hail/expr/ir/streams/EmitStream.scala +++ b/hail/hail/src/is/hail/expr/ir/streams/EmitStream.scala @@ -422,7 +422,7 @@ object EmitStream { val element: EmitCode = EmitCode.fromI(mb)(cb => container.loadElement(cb, idx)) - def close(cb: EmitCodeBuilder): Unit = {} + override def close(cb: EmitCodeBuilder): Unit = {} } ) @@ -669,7 +669,7 @@ object EmitStream { val element: EmitCode = eltField.load - def close(cb: EmitCodeBuilder): Unit = {} + override def close(cb: EmitCodeBuilder): Unit = {} } ), ) @@ -793,7 +793,7 @@ object EmitStream { val element: EmitCode = EmitCode.present(mb, new SInt32Value(curr)) - def close(cb: EmitCodeBuilder): Unit = {} + override def close(cb: EmitCodeBuilder): Unit = {} } SStreamValue(producer) @@ -864,7 +864,7 @@ object EmitStream { val element: EmitCode = EmitCode.present(mb, new SInt32Value(curr)) - def close(cb: EmitCodeBuilder): Unit = {} + override def close(cb: EmitCodeBuilder): Unit = {} } SStreamValue(producer) @@ -940,7 +940,7 @@ object EmitStream { val element: EmitCode = EmitCode.present(mb, new SInt32Value(curr)) - def close(cb: EmitCodeBuilder): Unit = {} + override def close(cb: EmitCodeBuilder): Unit = {} } SStreamValue(producer) @@ -1074,7 +1074,7 @@ object EmitStream { val element: EmitCode = elementField - def close(cb: EmitCodeBuilder): Unit = { + override def close(cb: EmitCodeBuilder): Unit = { childProducer.close(cb) if (requiresMemoryManagementPerElement) cb += childProducer.elementRegion.invalidate() @@ -1339,7 +1339,7 @@ object EmitStream { val element: EmitCode = bodyResult - def close(cb: EmitCodeBuilder): Unit = childProducer.close(cb) + override def close(cb: EmitCodeBuilder): Unit = childProducer.close(cb) } mb.implementLabel(childProducer.LendOfStream)(cb => cb.goto(producer.LendOfStream)) @@ -1725,7 +1725,7 @@ object EmitStream { } val element: EmitCode = innerProducer.element - def close(cb: EmitCodeBuilder): Unit = { + override def close(cb: EmitCodeBuilder): Unit = { cb.if_( innerUnclosed, { cb.assign(innerUnclosed, false) @@ -2935,7 +2935,7 @@ object EmitStream { val element: EmitCode = bodyCode - def close(cb: EmitCodeBuilder): Unit = + override def close(cb: EmitCodeBuilder): Unit = producers.foreach(_.close(cb)) } @@ -3028,7 +3028,7 @@ object EmitStream { val element: EmitCode = bodyCode - def close(cb: EmitCodeBuilder): Unit = + override def close(cb: EmitCodeBuilder): Unit = producers.foreach(_.close(cb)) } @@ -3107,7 +3107,7 @@ object EmitStream { val element: EmitCode = bodyCode - def close(cb: EmitCodeBuilder): Unit = + override def close(cb: EmitCodeBuilder): Unit = producers.foreach(_.close(cb)) } @@ -3854,7 +3854,7 @@ object EmitStream { val elementField = mb.newEmitField(elementType) val producer = new StreamProducer { - def method: EmitMethodBuilder[_] = mb + override def method: EmitMethodBuilder[_] = mb val length: Option[EmitCodeBuilder => Code[Int]] = None @@ -3865,7 +3865,7 @@ object EmitStream { val requiresMemoryManagementPerElement: Boolean = childProducer.requiresMemoryManagementPerElement - def initialize(cb: EmitCodeBuilder, outerRegion: Value[Region]): Unit = { + override def initialize(cb: EmitCodeBuilder, outerRegion: Value[Region]): Unit = { cb.assign(queueSize, emit(maxQueueSize, cb).getOrAssert(cb).asInt32.value) cb.assign( queue, @@ -3955,7 +3955,7 @@ object EmitStream { cb.goto(childProducer.LproduceElement) } - def close(cb: EmitCodeBuilder): Unit = childProducer.close(cb) + override def close(cb: EmitCodeBuilder): Unit = childProducer.close(cb) } mb.implementLabel(childProducer.LendOfStream)(cb => cb.goto(producer.LendOfStream)) diff --git a/hail/hail/src/is/hail/expr/ir/streams/StreamUtils.scala b/hail/hail/src/is/hail/expr/ir/streams/StreamUtils.scala index 80bf7dd0774..2ccc07dcf78 100644 --- a/hail/hail/src/is/hail/expr/ir/streams/StreamUtils.scala +++ b/hail/hail/src/is/hail/expr/ir/streams/StreamUtils.scala @@ -254,7 +254,7 @@ object StreamUtils { def implInit(cb: EmitCodeBuilder, outerRegion: Value[Region]): Unit - final def initialize(cb: EmitCodeBuilder, outerRegion: Value[Region]): Unit = { + final override def initialize(cb: EmitCodeBuilder, outerRegion: Value[Region]): Unit = { implInit(cb, outerRegion) cb.assign(bracket, Code.newArray[Int](k)) cb.assign(heads, Code.newArray[Long](k)) @@ -265,7 +265,7 @@ object StreamUtils { def implClose(cb: EmitCodeBuilder): Unit - final def close(cb: EmitCodeBuilder): Unit = { + final override def close(cb: EmitCodeBuilder): Unit = { implClose(cb) cb.assign(bracket, Code._null) cb.assign(heads, Code._null) diff --git a/hail/hail/src/is/hail/io/BufferSpecs.scala b/hail/hail/src/is/hail/io/BufferSpecs.scala index 76561cfcafb..7930caf17b8 100644 --- a/hail/hail/src/is/hail/io/BufferSpecs.scala +++ b/hail/hail/src/is/hail/io/BufferSpecs.scala @@ -89,35 +89,35 @@ trait BufferSpec extends Spec { } final case class LEB128BufferSpec(child: BufferSpec) extends BufferSpec { - def buildInputBuffer(in: InputStream): InputBuffer = + override def buildInputBuffer(in: InputStream): InputBuffer = new LEB128InputBuffer(child.buildInputBuffer(in)) - def buildOutputBuffer(out: OutputStream): OutputBuffer = + override def buildOutputBuffer(out: OutputStream): OutputBuffer = new LEB128OutputBuffer(child.buildOutputBuffer(out)) - def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBuffer] = + override def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBuffer] = Code.newInstance[LEB128InputBuffer, InputBuffer](child.buildCodeInputBuffer(in)) - def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBuffer] = + override def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBuffer] = Code.newInstance[LEB128OutputBuffer, OutputBuffer](child.buildCodeOutputBuffer(out)) } final case class BlockingBufferSpec(blockSize: Int, child: BlockBufferSpec) extends BufferSpec { require(blockSize <= (1 << 16)) - def buildInputBuffer(in: InputStream): InputBuffer = + override def buildInputBuffer(in: InputStream): InputBuffer = new BlockingInputBuffer(blockSize, child.buildInputBuffer(in)) - def buildOutputBuffer(out: OutputStream): OutputBuffer = + override def buildOutputBuffer(out: OutputStream): OutputBuffer = new BlockingOutputBuffer(blockSize, child.buildOutputBuffer(out)) - def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBuffer] = + override def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBuffer] = Code.newInstance[BlockingInputBuffer, Int, InputBlockBuffer]( blockSize, child.buildCodeInputBuffer(in), ) - def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBuffer] = + override def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBuffer] = Code.newInstance[BlockingOutputBuffer, Int, OutputBlockBuffer]( blockSize, child.buildCodeOutputBuffer(out), @@ -147,20 +147,20 @@ abstract class LZ4BlockBufferSpecCommon extends BlockBufferSpec { def child: BlockBufferSpec - def buildInputBuffer(in: InputStream): InputBlockBuffer = + override def buildInputBuffer(in: InputStream): InputBlockBuffer = new LZ4InputBlockBuffer(lz4, blockSize, child.buildInputBuffer(in)) - def buildOutputBuffer(out: OutputStream): OutputBlockBuffer = + override def buildOutputBuffer(out: OutputStream): OutputBlockBuffer = new LZ4OutputBlockBuffer(lz4, blockSize, child.buildOutputBuffer(out)) - def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBlockBuffer] = + override def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBlockBuffer] = Code.newInstance[LZ4InputBlockBuffer, LZ4, Int, InputBlockBuffer]( stagedlz4, blockSize, child.buildCodeInputBuffer(in), ) - def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBlockBuffer] = + override def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBlockBuffer] = Code.newInstance[LZ4OutputBlockBuffer, LZ4, Int, OutputBlockBuffer]( stagedlz4, blockSize, @@ -170,16 +170,16 @@ abstract class LZ4BlockBufferSpecCommon extends BlockBufferSpec { final case class LZ4HCBlockBufferSpec(blockSize: Int, child: BlockBufferSpec) extends LZ4BlockBufferSpecCommon { - def lz4 = LZ4.hc - def stagedlz4: Code[LZ4] = Code.invokeScalaObject0[LZ4](LZ4.getClass, "hc") - def typeName = "LZ4HCBlockBufferSpec" + override def lz4 = LZ4.hc + override def stagedlz4: Code[LZ4] = Code.invokeScalaObject0[LZ4](LZ4.getClass, "hc") + override def typeName = "LZ4HCBlockBufferSpec" } final case class LZ4FastBlockBufferSpec(blockSize: Int, child: BlockBufferSpec) extends LZ4BlockBufferSpecCommon { - def lz4 = LZ4.fast - def stagedlz4: Code[LZ4] = Code.invokeScalaObject0[LZ4](LZ4.getClass, "fast") - def typeName = "LZ4FastBlockBufferSpec" + override def lz4 = LZ4.fast + override def stagedlz4: Code[LZ4] = Code.invokeScalaObject0[LZ4](LZ4.getClass, "fast") + override def typeName = "LZ4FastBlockBufferSpec" } final case class LZ4SizeBasedBlockBufferSpec( @@ -196,10 +196,10 @@ final case class LZ4SizeBasedBlockBufferSpec( def stagedlz4: Code[LZ4] = Code.invokeScalaObject0[LZ4](LZ4.getClass, "fast") def typeName = "LZ4SizeBasedBlockBufferSpec" - def buildInputBuffer(in: InputStream): InputBlockBuffer = + override def buildInputBuffer(in: InputStream): InputBlockBuffer = new LZ4SizeBasedCompressingInputBlockBuffer(lz4, blockSize, child.buildInputBuffer(in)) - def buildOutputBuffer(out: OutputStream): OutputBlockBuffer = + override def buildOutputBuffer(out: OutputStream): OutputBlockBuffer = new LZ4SizeBasedCompressingOutputBlockBuffer( lz4, blockSize, @@ -207,14 +207,14 @@ final case class LZ4SizeBasedBlockBufferSpec( child.buildOutputBuffer(out), ) - def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBlockBuffer] = + override def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBlockBuffer] = Code.newInstance[LZ4SizeBasedCompressingInputBlockBuffer, LZ4, Int, InputBlockBuffer]( stagedlz4, blockSize, child.buildCodeInputBuffer(in), ) - def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBlockBuffer] = + override def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBlockBuffer] = Code.newInstance[LZ4SizeBasedCompressingOutputBlockBuffer, LZ4, Int, Int, OutputBlockBuffer]( stagedlz4, blockSize, @@ -227,19 +227,19 @@ final case class ZstdBlockBufferSpec(blockSize: Int, child: BlockBufferSpec) extends BlockBufferSpec { require(blockSize <= (1 << 16)) - def buildInputBuffer(in: InputStream): InputBlockBuffer = + override def buildInputBuffer(in: InputStream): InputBlockBuffer = new ZstdInputBlockBuffer(blockSize, child.buildInputBuffer(in)) - def buildOutputBuffer(out: OutputStream): OutputBlockBuffer = + override def buildOutputBuffer(out: OutputStream): OutputBlockBuffer = new ZstdOutputBlockBuffer(blockSize, child.buildOutputBuffer(out)) - def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBlockBuffer] = + override def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBlockBuffer] = Code.newInstance[ZstdInputBlockBuffer, Int, InputBlockBuffer]( blockSize, child.buildCodeInputBuffer(in), ) - def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBlockBuffer] = + override def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBlockBuffer] = Code.newInstance[ZstdOutputBlockBuffer, Int, OutputBlockBuffer]( blockSize, child.buildCodeOutputBuffer(out), @@ -253,19 +253,19 @@ final case class ZstdSizedBasedBlockBufferSpec( ) extends BlockBufferSpec { require(blockSize <= (1 << 16)) - def buildInputBuffer(in: InputStream): InputBlockBuffer = + override def buildInputBuffer(in: InputStream): InputBlockBuffer = new ZstdSizedBasedInputBlockBuffer(blockSize, child.buildInputBuffer(in)) - def buildOutputBuffer(out: OutputStream): OutputBlockBuffer = + override def buildOutputBuffer(out: OutputStream): OutputBlockBuffer = new ZstdSizedBasedOutputBlockBuffer(blockSize, minCompressionSize, child.buildOutputBuffer(out)) - def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBlockBuffer] = + override def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBlockBuffer] = Code.newInstance[ZstdSizedBasedInputBlockBuffer, Int, InputBlockBuffer]( blockSize, child.buildCodeInputBuffer(in), ) - def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBlockBuffer] = + override def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBlockBuffer] = Code.newInstance[ZstdSizedBasedOutputBlockBuffer, Int, Int, OutputBlockBuffer]( blockSize, minCompressionSize, @@ -278,14 +278,15 @@ object StreamBlockBufferSpec { } final class StreamBlockBufferSpec extends BlockBufferSpec { - def buildInputBuffer(in: InputStream): InputBlockBuffer = new StreamBlockInputBuffer(in) + override def buildInputBuffer(in: InputStream): InputBlockBuffer = new StreamBlockInputBuffer(in) - def buildOutputBuffer(out: OutputStream): OutputBlockBuffer = new StreamBlockOutputBuffer(out) + override def buildOutputBuffer(out: OutputStream): OutputBlockBuffer = + new StreamBlockOutputBuffer(out) - def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBlockBuffer] = + override def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBlockBuffer] = Code.newInstance[StreamBlockInputBuffer, InputStream](in) - def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBlockBuffer] = + override def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBlockBuffer] = Code.newInstance[StreamBlockOutputBuffer, OutputStream](out) override def equals(other: Any): Boolean = other.isInstanceOf[StreamBlockBufferSpec] @@ -296,10 +297,10 @@ final class StreamBufferSpec extends BufferSpec { override def buildOutputBuffer(out: OutputStream): OutputBuffer = new StreamOutputBuffer(out) - def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBuffer] = + override def buildCodeInputBuffer(in: Code[InputStream]): Code[InputBuffer] = Code.newInstance[StreamInputBuffer, InputStream](in) - def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBuffer] = + override def buildCodeOutputBuffer(out: Code[OutputStream]): Code[OutputBuffer] = Code.newInstance[StreamOutputBuffer, OutputStream](out) override def equals(other: Any): Boolean = other.isInstanceOf[StreamBufferSpec] diff --git a/hail/hail/src/is/hail/io/ByteArrayReader.scala b/hail/hail/src/is/hail/io/ByteArrayReader.scala index a48a2189035..d5eece2237f 100644 --- a/hail/hail/src/is/hail/io/ByteArrayReader.scala +++ b/hail/hail/src/is/hail/io/ByteArrayReader.scala @@ -27,7 +27,7 @@ class ByteArrayReader(val arr: Array[Byte]) extends AbstractBinaryReader { def seek(pos: Int): Unit = position = pos - def skipBytes(bytes: Long): Long = { + override def skipBytes(bytes: Long): Long = { require(bytes < Integer.MAX_VALUE) position += bytes.toInt if (position < length) diff --git a/hail/hail/src/is/hail/io/Decoder.scala b/hail/hail/src/is/hail/io/Decoder.scala index 25a5d9452fa..0ad0d158b7e 100644 --- a/hail/hail/src/is/hail/io/Decoder.scala +++ b/hail/hail/src/is/hail/io/Decoder.scala @@ -9,8 +9,6 @@ import is.hail.utils.RestartableByteArrayInputStream import java.io._ trait Decoder extends Closeable { - def close(): Unit - def ptype: PType def readRegionValue(region: Region): Long @@ -26,17 +24,17 @@ final class CompiledDecoder( theHailClassLoader: HailClassLoader, f: (HailClassLoader) => DecoderAsmFunction, ) extends Decoder { - def close(): Unit = + override def close(): Unit = in.close() - def readByte(): Byte = in.readByte() + override def readByte(): Byte = in.readByte() private[this] val compiled = f(theHailClassLoader) - def readRegionValue(r: Region): Long = + override def readRegionValue(r: Region): Long = compiled(r, in) - def seek(offset: Long): Unit = in.seek(offset) + override def seek(offset: Long): Unit = in.seek(offset) } final class ByteArrayDecoder( diff --git a/hail/hail/src/is/hail/io/DoubleInputBuffer.scala b/hail/hail/src/is/hail/io/DoubleInputBuffer.scala index 2770606d7d5..11a5406d22e 100644 --- a/hail/hail/src/is/hail/io/DoubleInputBuffer.scala +++ b/hail/hail/src/is/hail/io/DoubleInputBuffer.scala @@ -10,7 +10,7 @@ final class DoubleInputBuffer(in: InputStream, bufSize: Int) extends Closeable { private var end: Int = 0 private var off: Int = 0 - def close(): Unit = + override def close(): Unit = in.close() def readDoubles(to: Array[Double]): Unit = readDoubles(to, 0, to.length) @@ -45,7 +45,7 @@ final class DoubleOutputBuffer(out: OutputStream, bufSize: Int) extends Closeabl private val buf: Array[Byte] = new Array[Byte](bufSize) private var off: Int = 0 - def close(): Unit = { + override def close(): Unit = { flush() out.close() } diff --git a/hail/hail/src/is/hail/io/Encoder.scala b/hail/hail/src/is/hail/io/Encoder.scala index 6e0a43dd0a4..6d63b27c4a5 100644 --- a/hail/hail/src/is/hail/io/Encoder.scala +++ b/hail/hail/src/is/hail/io/Encoder.scala @@ -10,7 +10,7 @@ import java.io._ trait Encoder extends Closeable { def flush(): Unit - def close(): Unit + override def close(): Unit def writeRegionValue(region: Region, offset: Long): Unit @@ -28,10 +28,10 @@ final class CompiledEncoder( private[this] var partitionRegion: Region = _ private[this] val compiled = f(theHailClassLoader) - def flush(): Unit = + override def flush(): Unit = out.flush() - def close(): Unit = { + override def close(): Unit = { compiled.asInstanceOf[FunctionWithPartitionRegion].setPool(null) compiled.asInstanceOf[FunctionWithPartitionRegion].addPartitionRegion(null) if (partitionRegion != null) partitionRegion.close() @@ -45,7 +45,7 @@ final class CompiledEncoder( poolSet = true } - def writeRegionValue(r: Region, offset: Long): Unit = { + override def writeRegionValue(r: Region, offset: Long): Unit = { setScratchPool(r.pool) writeRegionValue(offset) } @@ -55,10 +55,10 @@ final class CompiledEncoder( compiled(offset, out) } - def writeByte(b: Byte): Unit = + override def writeByte(b: Byte): Unit = out.writeByte(b) - def indexOffset(): Long = out.indexOffset() + override def indexOffset(): Long = out.indexOffset() } final class ByteArrayEncoder( @@ -68,7 +68,7 @@ final class ByteArrayEncoder( private[this] val baos = new ByteArrayOutputStream() private[this] val enc = makeEnc(baos, theHailClassLoader) - def close(): Unit = { + override def close(): Unit = { enc.close() baos.close() } diff --git a/hail/hail/src/is/hail/io/HadoopFSDataBinaryReader.scala b/hail/hail/src/is/hail/io/HadoopFSDataBinaryReader.scala index 0a556280f1f..1fdfcc0c972 100644 --- a/hail/hail/src/is/hail/io/HadoopFSDataBinaryReader.scala +++ b/hail/hail/src/is/hail/io/HadoopFSDataBinaryReader.scala @@ -10,11 +10,11 @@ class HadoopFSDataBinaryReader(fis: SeekableDataInputStream) override def read(byteArray: Array[Byte], hasRead: Int, toRead: Int): Int = fis.read(byteArray, hasRead, toRead) - def close(): Unit = fis.close() + override def close(): Unit = fis.close() def seek(pos: Long): Unit = fis.seek(pos) - def skipBytes(bytes: Long): Long = fis.skip(bytes) + override def skipBytes(bytes: Long): Long = fis.skip(bytes) def getPosition: Long = fis.getPosition } diff --git a/hail/hail/src/is/hail/io/IndexBTree.scala b/hail/hail/src/is/hail/io/IndexBTree.scala index b45babfc995..92dfc50dc4e 100644 --- a/hail/hail/src/is/hail/io/IndexBTree.scala +++ b/hail/hail/src/is/hail/io/IndexBTree.scala @@ -108,7 +108,7 @@ class IndexBTree(indexFileName: String, fs: FS, branchingFactor: Int = 1024) ext ) } - def close(): Unit = is.close() + override def close(): Unit = is.close() def calcDepth(): Int = IndexBTree.calcDepth(fs.getFileSize(indexFileName) / 8, branchingFactor) diff --git a/hail/hail/src/is/hail/io/IndexedBinaryBlockReader.scala b/hail/hail/src/is/hail/io/IndexedBinaryBlockReader.scala index beef7d2c48d..bbff8bcb57f 100644 --- a/hail/hail/src/is/hail/io/IndexedBinaryBlockReader.scala +++ b/hail/hail/src/is/hail/io/IndexedBinaryBlockReader.scala @@ -44,18 +44,18 @@ abstract class IndexedBinaryBlockReader[T](job: Configuration, split: FileSplit) ) } - def createKey(): LongWritable = new LongWritable() + override def createKey(): LongWritable = new LongWritable() - def createValue(): T + override def createValue(): T - def getPos: Long = pos + override def getPos: Long = pos - def getProgress: Float = + override def getProgress: Float = if (partitionStart == end) 0.0f else Math.min(1.0f, (pos - partitionStart) / (end - partitionStart).toFloat) - def close() = bfis.close() + override def close() = bfis.close() } diff --git a/hail/hail/src/is/hail/io/IndexedBinaryInputFormat.scala b/hail/hail/src/is/hail/io/IndexedBinaryInputFormat.scala index c07d6608111..b5da9a743c1 100644 --- a/hail/hail/src/is/hail/io/IndexedBinaryInputFormat.scala +++ b/hail/hail/src/is/hail/io/IndexedBinaryInputFormat.scala @@ -5,6 +5,6 @@ import org.apache.hadoop.mapred._ abstract class IndexedBinaryInputFormat[T] extends FileInputFormat[LongWritable, T] { - def getRecordReader(split: InputSplit, job: JobConf, reporter: Reporter) + override def getRecordReader(split: InputSplit, job: JobConf, reporter: Reporter) : RecordReader[LongWritable, T] } diff --git a/hail/hail/src/is/hail/io/InputBuffers.scala b/hail/hail/src/is/hail/io/InputBuffers.scala index e6d6c62af8b..80c95bae189 100644 --- a/hail/hail/src/is/hail/io/InputBuffers.scala +++ b/hail/hail/src/is/hail/io/InputBuffers.scala @@ -11,7 +11,7 @@ import java.util.function.Supplier import com.github.luben.zstd.{Zstd, ZstdDecompressCtx} trait InputBuffer extends Closeable { - def close(): Unit + override def close(): Unit def seek(offset: Long): Unit @@ -59,7 +59,7 @@ trait InputBuffer extends Closeable { } trait InputBlockBuffer extends Spec with Closeable { - def close(): Unit + override def close(): Unit def seek(offset: Long): Unit @@ -84,11 +84,11 @@ trait InputBlockBuffer extends Spec with Closeable { final class StreamInputBuffer(in: InputStream) extends InputBuffer { private[this] val buff = new Array[Byte](8) - def close(): Unit = in.close() + override def close(): Unit = in.close() - def seek(offset: Long) = in.asInstanceOf[ByteTrackingInputStream].seek(offset) + override def seek(offset: Long) = in.asInstanceOf[ByteTrackingInputStream].seek(offset) - def readByte(): Byte = { + override def readByte(): Byte = { in.readFully(buff, 0, 1) Memory.loadByte(buff, 0) } @@ -96,63 +96,63 @@ final class StreamInputBuffer(in: InputStream) extends InputBuffer { override def read(buf: Array[Byte], toOff: Int, n: Int): Unit = in.readFully(buf, toOff, n) - def readInt(): Int = { + override def readInt(): Int = { in.readFully(buff, 0, 4) Memory.loadInt(buff, 0) } - def readLong(): Long = { + override def readLong(): Long = { in.readFully(buff) Memory.loadLong(buff, 0) } - def readFloat(): Float = { + override def readFloat(): Float = { in.readFully(buff, 0, 4) Memory.loadFloat(buff, 0) } - def readDouble(): Double = { + override def readDouble(): Double = { in.readFully(buff) Memory.loadDouble(buff, 0) } - def readBytes(toRegion: Region, toOff: Long, n: Int): Unit = + override def readBytes(toRegion: Region, toOff: Long, n: Int): Unit = Region.storeBytes(toOff, readBytesArray(n)) - def readBytesArray(n: Int): Array[Byte] = + override def readBytesArray(n: Int): Array[Byte] = Array.tabulate(n)(_ => readByte()) - def skipByte(): Unit = { + override def skipByte(): Unit = { val bytesRead = in.skip(1) assert(bytesRead == 1L) } - def skipInt(): Unit = { + override def skipInt(): Unit = { val bytesRead = in.skip(4) assert(bytesRead == 4L) } - def skipLong(): Unit = { + override def skipLong(): Unit = { val bytesRead = in.skip(8) assert(bytesRead == 8L) } - def skipFloat(): Unit = { + override def skipFloat(): Unit = { val bytesRead = in.skip(4) assert(bytesRead == 4L) } - def skipDouble(): Unit = { + override def skipDouble(): Unit = { val bytesRead = in.skip(8) assert(bytesRead == 8L) } - def skipBytes(n: Int): Unit = { + override def skipBytes(n: Int): Unit = { val bytesRead = in.skip(n.toLong) assert(bytesRead == n) } - def readDoubles(to: Array[Double], off: Int, n: Int): Unit = { + override def readDoubles(to: Array[Double], off: Int, n: Int): Unit = { var i = 0 while (i < n) { to(off + i) = readDouble() @@ -162,55 +162,55 @@ final class StreamInputBuffer(in: InputStream) extends InputBuffer { } final class MemoryInputBuffer(mb: MemoryBuffer) extends InputBuffer { - def close(): Unit = {} + override def close(): Unit = {} - def seek(offset: Long) = ??? + override def seek(offset: Long) = ??? - def readByte(): Byte = mb.readByte() + override def readByte(): Byte = mb.readByte() - def readInt(): Int = mb.readInt() + override def readInt(): Int = mb.readInt() - def readLong(): Long = mb.readLong() + override def readLong(): Long = mb.readLong() - def readFloat(): Float = mb.readFloat() + override def readFloat(): Float = mb.readFloat() - def readDouble(): Double = mb.readDouble() + override def readDouble(): Double = mb.readDouble() - def readBytes(toRegion: Region, toOff: Long, n: Int): Unit = mb.readBytes(toOff, n) + override def readBytes(toRegion: Region, toOff: Long, n: Int): Unit = mb.readBytes(toOff, n) - def readBytesArray(n: Int): Array[Byte] = { + override def readBytesArray(n: Int): Array[Byte] = { val arr = new Array[Byte](n) mb.readBytesArray(arr, n) arr } - def skipByte(): Unit = mb.skipByte() + override def skipByte(): Unit = mb.skipByte() - def skipInt(): Unit = mb.skipInt() + override def skipInt(): Unit = mb.skipInt() - def skipLong(): Unit = mb.skipLong() + override def skipLong(): Unit = mb.skipLong() - def skipFloat(): Unit = mb.skipFloat() + override def skipFloat(): Unit = mb.skipFloat() - def skipDouble(): Unit = mb.skipDouble() + override def skipDouble(): Unit = mb.skipDouble() - def skipBytes(n: Int): Unit = mb.skipBytes(n) + override def skipBytes(n: Int): Unit = mb.skipBytes(n) - def readDoubles(to: Array[Double], off: Int, n: Int): Unit = ??? + override def readDoubles(to: Array[Double], off: Int, n: Int): Unit = ??? } final class LEB128InputBuffer(in: InputBuffer) extends InputBuffer { - def close(): Unit = + override def close(): Unit = in.close() - def seek(offset: Long): Unit = in.seek(offset) + override def seek(offset: Long): Unit = in.seek(offset) - def readByte(): Byte = + override def readByte(): Byte = in.readByte() override def read(buf: Array[Byte], toOff: Int, n: Int) = in.read(buf, toOff, n) - def readInt(): Int = { + override def readInt(): Int = { var b: Byte = readByte() var x: Int = b & 0x7f var shift: Int = 7 @@ -222,7 +222,7 @@ final class LEB128InputBuffer(in: InputBuffer) extends InputBuffer { x } - def readLong(): Long = { + override def readLong(): Long = { var b: Byte = readByte() var x: Long = b & 0x7fL var shift: Int = 7 @@ -234,35 +234,37 @@ final class LEB128InputBuffer(in: InputBuffer) extends InputBuffer { x } - def readFloat(): Float = in.readFloat() + override def readFloat(): Float = in.readFloat() - def readDouble(): Double = in.readDouble() + override def readDouble(): Double = in.readDouble() - def readBytes(toRegion: Region, toOff: Long, n: Int): Unit = in.readBytes(toRegion, toOff, n) + override def readBytes(toRegion: Region, toOff: Long, n: Int): Unit = + in.readBytes(toRegion, toOff, n) - def readBytesArray(n: Int): Array[Byte] = in.readBytesArray(n) + override def readBytesArray(n: Int): Array[Byte] = in.readBytesArray(n) - def skipByte(): Unit = in.skipByte() + override def skipByte(): Unit = in.skipByte() - def skipInt(): Unit = { + override def skipInt(): Unit = { var b: Byte = readByte() while ((b & 0x80) != 0) b = readByte() } - def skipLong(): Unit = { + override def skipLong(): Unit = { var b: Byte = readByte() while ((b & 0x80) != 0) b = readByte() } - def skipFloat(): Unit = in.skipFloat() + override def skipFloat(): Unit = in.skipFloat() - def skipDouble(): Unit = in.skipDouble() + override def skipDouble(): Unit = in.skipDouble() - def skipBytes(n: Int): Unit = in.skipBytes(n) + override def skipBytes(n: Int): Unit = in.skipBytes(n) - def readDoubles(to: Array[Double], toOff: Int, n: Int): Unit = in.readDoubles(to, toOff, n) + override def readDoubles(to: Array[Double], toOff: Int, n: Int): Unit = + in.readDoubles(to, toOff, n) } final class TracingInputBuffer( @@ -272,11 +274,11 @@ final class TracingInputBuffer( private[this] val logfile = new FileOutputStream(filename, true) logger.info(s"tracing to $filename") - def close(): Unit = in.close() + override def close(): Unit = in.close() - def seek(offset: Long): Unit = ??? + override def seek(offset: Long): Unit = ??? - def readByte(): Byte = { + override def readByte(): Byte = { val x = in.readByte() logfile.write(x.toInt) x @@ -290,53 +292,53 @@ final class TracingInputBuffer( } } - def readInt(): Int = { + override def readInt(): Int = { val bytes = readBytesArray(4) Memory.loadInt(bytes, 0) } - def readLong(): Long = { + override def readLong(): Long = { val bytes = readBytesArray(8) Memory.loadLong(bytes, 0) } - def readFloat(): Float = { + override def readFloat(): Float = { val bytes = readBytesArray(4) Memory.loadFloat(bytes, 0) } - def readDouble(): Double = { + override def readDouble(): Double = { val bytes = readBytesArray(8) Memory.loadDouble(bytes, 0) } - def readBytes(toRegion: Region, toOff: Long, n: Int): Unit = + override def readBytes(toRegion: Region, toOff: Long, n: Int): Unit = Region.storeBytes(toOff, readBytesArray(n)) - def readBytesArray(n: Int): Array[Byte] = + override def readBytesArray(n: Int): Array[Byte] = Array.tabulate(n)(_ => readByte()) override def skipBoolean(): Unit = skipByte() - def skipByte(): Unit = + override def skipByte(): Unit = skipBytes(1) - def skipInt(): Unit = + override def skipInt(): Unit = skipBytes(4) - def skipLong(): Unit = + override def skipLong(): Unit = skipBytes(8) - def skipFloat(): Unit = + override def skipFloat(): Unit = skipBytes(4) - def skipDouble(): Unit = + override def skipDouble(): Unit = skipBytes(8) - @inline def skipBytes(n: Int): Unit = + @inline override def skipBytes(n: Int): Unit = for (_ <- 0 until n) readByte() - def readDoubles(to: Array[Double], off: Int, n: Int): Unit = { + override def readDoubles(to: Array[Double], off: Int, n: Int): Unit = { var i = 0 while (i < n) { to(off + i) = readDouble() @@ -363,52 +365,52 @@ final class BlockingInputBuffer(blockSize: Int, in: InputBlockBuffer) extends In assert(off + n <= end) } - def close(): Unit = + override def close(): Unit = in.close() - def seek(offset: Long): Unit = { + override def seek(offset: Long): Unit = { in.seek(offset) end = in.readBlock(buf) off = (offset & 0xffff).asInstanceOf[Int] assert(off <= end) } - def readByte(): Byte = { + override def readByte(): Byte = { ensure(1) val b = Memory.loadByte(buf, off.toLong) off += 1 b } - def readInt(): Int = { + override def readInt(): Int = { ensure(4) val i = Memory.loadInt(buf, off.toLong) off += 4 i } - def readLong(): Long = { + override def readLong(): Long = { ensure(8) val l = Memory.loadLong(buf, off.toLong) off += 8 l } - def readFloat(): Float = { + override def readFloat(): Float = { ensure(4) val f = Memory.loadFloat(buf, off.toLong) off += 4 f } - def readDouble(): Double = { + override def readDouble(): Double = { ensure(8) val d = Memory.loadDouble(buf, off.toLong) off += 8 d } - def readBytes(toRegion: Region, toOff0: Long, n0: Int): Unit = { + override def readBytes(toRegion: Region, toOff0: Long, n0: Int): Unit = { assert(n0 >= 0) var toOff = toOff0 var n = n0 @@ -445,38 +447,38 @@ final class BlockingInputBuffer(blockSize: Int, in: InputBlockBuffer) extends In } } - def readBytesArray(n: Int): Array[Byte] = { + override def readBytesArray(n: Int): Array[Byte] = { val arr = new Array[Byte](n) read(arr, 0, n) arr } - def skipByte(): Unit = { + override def skipByte(): Unit = { ensure(1) off += 1 } - def skipInt(): Unit = { + override def skipInt(): Unit = { ensure(4) off += 4 } - def skipLong(): Unit = { + override def skipLong(): Unit = { ensure(8) off += 8 } - def skipFloat(): Unit = { + override def skipFloat(): Unit = { ensure(4) off += 4 } - def skipDouble(): Unit = { + override def skipDouble(): Unit = { ensure(8) off += 8 } - def skipBytes(n0: Int): Unit = { + override def skipBytes(n0: Int): Unit = { var n = n0 if (off + n > end) { n -= (end - off) @@ -488,7 +490,7 @@ final class BlockingInputBuffer(blockSize: Int, in: InputBlockBuffer) extends In } } - def readDoubles(to: Array[Double], toOff0: Int, n0: Int): Unit = { + override def readDoubles(to: Array[Double], toOff0: Int, n0: Int): Unit = { assert(toOff0 >= 0) assert(n0 >= 0) assert(toOff0 <= to.length - n0) @@ -513,13 +515,14 @@ final class BlockingInputBuffer(blockSize: Int, in: InputBlockBuffer) extends In final class StreamBlockInputBuffer(in: InputStream) extends InputBlockBuffer { private[this] val lenBuf = new Array[Byte](4) - def close(): Unit = + override def close(): Unit = in.close() // this takes a virtual offset and will seek the underlying stream to offset >> 16 - def seek(offset: Long): Unit = in.asInstanceOf[ByteTrackingInputStream].seek(offset >> 16) + override def seek(offset: Long): Unit = + in.asInstanceOf[ByteTrackingInputStream].seek(offset >> 16) - def readBlock(buf: Array[Byte]): Int = { + override def readBlock(buf: Array[Byte]): Int = { in.readFully(lenBuf, 0, 4) val len = Memory.loadInt(lenBuf, 0) assert(len >= 0) @@ -533,10 +536,10 @@ final class LZ4InputBlockBuffer(lz4: LZ4, blockSize: Int, in: InputBlockBuffer) extends InputBlockBuffer { private[this] val comp = new Array[Byte](4 + lz4.maxCompressedLength(blockSize)) - def close(): Unit = + override def close(): Unit = in.close() - def seek(offset: Long): Unit = in.seek(offset) + override def seek(offset: Long): Unit = in.seek(offset) override def skipBytesReadRemainder(n0: Int, buf: Array[Byte]): Int = { var n = n0 @@ -557,7 +560,7 @@ final class LZ4InputBlockBuffer(lz4: LZ4, blockSize: Int, in: InputBlockBuffer) return -n } - def readBlock(buf: Array[Byte]): Int = { + override def readBlock(buf: Array[Byte]): Int = { val blockLen = in.readBlock(comp) val result = if (blockLen == -1) { -1 @@ -576,12 +579,12 @@ final class LZ4SizeBasedCompressingInputBlockBuffer(lz4: LZ4, blockSize: Int, in private[this] val comp = new Array[Byte](8 + lz4.maxCompressedLength(blockSize)) private[this] var lim = 0 - def close(): Unit = + override def close(): Unit = in.close() - def seek(offset: Long): Unit = in.seek(offset) + override def seek(offset: Long): Unit = in.seek(offset) - def readBlock(buf: Array[Byte]): Int = { + override def readBlock(buf: Array[Byte]): Int = { val blockLen = in.readBlock(comp) val result = if (blockLen == -1) { -1 @@ -606,7 +609,7 @@ final class LZ4SizeBasedCompressingInputBlockBuffer(lz4: LZ4, blockSize: Int, in object ZstdDecompressLib { val instance = ThreadLocal.withInitial(new Supplier[ZstdDecompressCtx]() { - def get: ZstdDecompressCtx = new ZstdDecompressCtx() + override def get: ZstdDecompressCtx = new ZstdDecompressCtx() }) } @@ -614,12 +617,12 @@ final class ZstdInputBlockBuffer(blockSize: Int, in: InputBlockBuffer) extends I private[this] val zstd = ZstdDecompressLib.instance.get private[this] val comp = new Array[Byte](4 + Zstd.compressBound(blockSize.toLong).toInt) - def close(): Unit = + override def close(): Unit = in.close() - def seek(offset: Long): Unit = in.seek(offset) + override def seek(offset: Long): Unit = in.seek(offset) - def readBlock(buf: Array[Byte]): Int = { + override def readBlock(buf: Array[Byte]): Int = { val blockLen = in.readBlock(comp) if (blockLen == -1) { blockLen @@ -637,12 +640,12 @@ final class ZstdSizedBasedInputBlockBuffer(blockSize: Int, in: InputBlockBuffer) private[this] val zstd = ZstdDecompressLib.instance.get private[this] val comp = new Array[Byte](4 + Zstd.compressBound(blockSize.toLong).toInt) - def close(): Unit = + override def close(): Unit = in.close() - def seek(offset: Long): Unit = in.seek(offset) + override def seek(offset: Long): Unit = in.seek(offset) - def readBlock(buf: Array[Byte]): Int = { + override def readBlock(buf: Array[Byte]): Int = { val blockLen = in.readBlock(comp) if (blockLen == -1) { blockLen diff --git a/hail/hail/src/is/hail/io/OutputBuffers.scala b/hail/hail/src/is/hail/io/OutputBuffers.scala index 876e34fd4fa..193e022abcb 100644 --- a/hail/hail/src/is/hail/io/OutputBuffers.scala +++ b/hail/hail/src/is/hail/io/OutputBuffers.scala @@ -13,7 +13,7 @@ import com.github.luben.zstd.{Zstd, ZstdCompressCtx} trait OutputBuffer extends Closeable { def flush(): Unit - def close(): Unit + override def close(): Unit def indexOffset(): Long @@ -64,7 +64,7 @@ final class StreamOutputBuffer(out: OutputStream) extends OutputBuffer { override def close(): Unit = out.close() - def indexOffset(): Long = out.asInstanceOf[ByteTrackingOutputStream].bytesWritten + override def indexOffset(): Long = out.asInstanceOf[ByteTrackingOutputStream].bytesWritten override def writeByte(b: Byte): Unit = out.write(b.toInt) @@ -73,28 +73,28 @@ final class StreamOutputBuffer(out: OutputStream) extends OutputBuffer { out.write(buf, 0, 4) } - def writeLong(l: Long): Unit = { + override def writeLong(l: Long): Unit = { Memory.storeLong(buf, 0, l) out.write(buf, 0, 8) } - def writeFloat(f: Float): Unit = { + override def writeFloat(f: Float): Unit = { Memory.storeFloat(buf, 0, f) out.write(buf, 0, 4) } - def writeDouble(d: Double): Unit = { + override def writeDouble(d: Double): Unit = { Memory.storeDouble(buf, 0, d) out.write(buf, 0, 8) } - def writeBytes(region: Region, off: Long, n: Int): Unit = + override def writeBytes(region: Region, off: Long, n: Int): Unit = out.write(Region.loadBytes(off, n)) - def writeBytes(addr: Long, n: Int): Unit = + override def writeBytes(addr: Long, n: Int): Unit = out.write(Region.loadBytes(addr, n)) - def writeDoubles(from: Array[Double], fromOff: Int, n: Int): Unit = { + override def writeDoubles(from: Array[Double], fromOff: Int, n: Int): Unit = { var i = 0 while (i < n) { writeDouble(from(fromOff + i)) @@ -104,40 +104,40 @@ final class StreamOutputBuffer(out: OutputStream) extends OutputBuffer { } final class MemoryOutputBuffer(mb: MemoryBuffer) extends OutputBuffer { - def flush(): Unit = {} + override def flush(): Unit = {} - def close(): Unit = {} + override def close(): Unit = {} - def indexOffset(): Long = ??? + override def indexOffset(): Long = ??? - def writeByte(b: Byte): Unit = mb.writeByte(b) + override def writeByte(b: Byte): Unit = mb.writeByte(b) - def writeInt(i: Int): Unit = mb.writeInt(i) + override def writeInt(i: Int): Unit = mb.writeInt(i) - def writeLong(l: Long): Unit = mb.writeLong(l) + override def writeLong(l: Long): Unit = mb.writeLong(l) - def writeFloat(f: Float): Unit = mb.writeFloat(f) + override def writeFloat(f: Float): Unit = mb.writeFloat(f) - def writeDouble(d: Double): Unit = mb.writeDouble(d) + override def writeDouble(d: Double): Unit = mb.writeDouble(d) - def writeBytes(region: Region, off: Long, n: Int): Unit = mb.writeBytes(off, n) + override def writeBytes(region: Region, off: Long, n: Int): Unit = mb.writeBytes(off, n) - def writeBytes(addr: Long, n: Int): Unit = mb.writeBytes(addr, n) + override def writeBytes(addr: Long, n: Int): Unit = mb.writeBytes(addr, n) - def writeDoubles(from: Array[Double], fromOff: Int, n: Int): Unit = ??? + override def writeDoubles(from: Array[Double], fromOff: Int, n: Int): Unit = ??? } final class LEB128OutputBuffer(out: OutputBuffer) extends OutputBuffer { - def flush(): Unit = out.flush() + override def flush(): Unit = out.flush() - def close(): Unit = + override def close(): Unit = out.close() - def indexOffset(): Long = out.indexOffset() + override def indexOffset(): Long = out.indexOffset() - def writeByte(b: Byte): Unit = out.writeByte(b) + override def writeByte(b: Byte): Unit = out.writeByte(b) - def writeInt(i: Int): Unit = { + override def writeInt(i: Int): Unit = { var j = i do { var b = j & 0x7f @@ -148,7 +148,7 @@ final class LEB128OutputBuffer(out: OutputBuffer) extends OutputBuffer { } while (j != 0) } - def writeLong(l: Long): Unit = { + override def writeLong(l: Long): Unit = { var j = l do { var b = j & 0x7f @@ -159,15 +159,15 @@ final class LEB128OutputBuffer(out: OutputBuffer) extends OutputBuffer { } while (j != 0) } - def writeFloat(f: Float): Unit = out.writeFloat(f) + override def writeFloat(f: Float): Unit = out.writeFloat(f) - def writeDouble(d: Double): Unit = out.writeDouble(d) + override def writeDouble(d: Double): Unit = out.writeDouble(d) - def writeBytes(region: Region, off: Long, n: Int): Unit = out.writeBytes(region, off, n) + override def writeBytes(region: Region, off: Long, n: Int): Unit = out.writeBytes(region, off, n) - def writeBytes(addr: Long, n: Int): Unit = out.writeBytes(addr, n) + override def writeBytes(addr: Long, n: Int): Unit = out.writeBytes(addr, n) - def writeDoubles(from: Array[Double], fromOff: Int, n: Int): Unit = + override def writeDoubles(from: Array[Double], fromOff: Int, n: Int): Unit = out.writeDoubles(from, fromOff, n) } @@ -175,7 +175,7 @@ final class BlockingOutputBuffer(blockSize: Int, out: OutputBlockBuffer) extends private val buf: Array[Byte] = new Array[Byte](blockSize) private var off: Int = 0 - def indexOffset(): Long = { + override def indexOffset(): Long = { if (off == blockSize) writeBlock() (out.getPos() << 16) | off @@ -186,54 +186,54 @@ final class BlockingOutputBuffer(blockSize: Int, out: OutputBlockBuffer) extends off = 0 } - def flush(): Unit = { + override def flush(): Unit = { writeBlock() out.flush() } - def close(): Unit = { + override def close(): Unit = { flush() out.close() } - def writeByte(b: Byte): Unit = { + override def writeByte(b: Byte): Unit = { if (off + 1 > buf.length) writeBlock() Memory.storeByte(buf, off.toLong, b) off += 1 } - def writeInt(i: Int): Unit = { + override def writeInt(i: Int): Unit = { if (off + 4 > buf.length) writeBlock() Memory.storeInt(buf, off.toLong, i) off += 4 } - def writeLong(l: Long): Unit = { + override def writeLong(l: Long): Unit = { if (off + 8 > buf.length) writeBlock() Memory.storeLong(buf, off.toLong, l) off += 8 } - def writeFloat(f: Float): Unit = { + override def writeFloat(f: Float): Unit = { if (off + 4 > buf.length) writeBlock() Memory.storeFloat(buf, off.toLong, f) off += 4 } - def writeDouble(d: Double): Unit = { + override def writeDouble(d: Double): Unit = { if (off + 8 > buf.length) writeBlock() Memory.storeDouble(buf, off.toLong, d) off += 8 } - def writeBytes(fromRegion: Region, fromOff0: Long, n0: Int) = writeBytes(fromOff0, n0) + override def writeBytes(fromRegion: Region, fromOff0: Long, n0: Int) = writeBytes(fromOff0, n0) - def writeBytes(addr0: Long, n0: Int): Unit = { + override def writeBytes(addr0: Long, n0: Int): Unit = { assert(n0 >= 0) var addr = addr0 var n = n0 @@ -251,7 +251,7 @@ final class BlockingOutputBuffer(blockSize: Int, out: OutputBlockBuffer) extends off += n } - def writeDoubles(from: Array[Double], fromOff0: Int, n0: Int): Unit = { + override def writeDoubles(from: Array[Double], fromOff0: Int, n0: Int): Unit = { assert(n0 >= 0) assert(fromOff0 >= 0) assert(fromOff0 <= from.length - n0) @@ -274,38 +274,38 @@ final class BlockingOutputBuffer(blockSize: Int, out: OutputBlockBuffer) extends final class StreamBlockOutputBuffer(out: OutputStream) extends OutputBlockBuffer { private val lenBuf = new Array[Byte](4) - def flush(): Unit = + override def flush(): Unit = out.flush() - def close(): Unit = + override def close(): Unit = out.close() - def writeBlock(buf: Array[Byte], len: Int): Unit = { + override def writeBlock(buf: Array[Byte], len: Int): Unit = { Memory.storeInt(lenBuf, 0, len) out.write(lenBuf, 0, 4) out.write(buf, 0, len) } - def getPos(): Long = out.asInstanceOf[ByteTrackingOutputStream].bytesWritten + override def getPos(): Long = out.asInstanceOf[ByteTrackingOutputStream].bytesWritten } final class LZ4OutputBlockBuffer(lz4: LZ4, blockSize: Int, out: OutputBlockBuffer) extends OutputBlockBuffer { private val comp = new Array[Byte](4 + lz4.maxCompressedLength(blockSize)) - def flush(): Unit = + override def flush(): Unit = out.flush() - def close(): Unit = + override def close(): Unit = out.close() - def writeBlock(buf: Array[Byte], decompLen: Int): Unit = { + override def writeBlock(buf: Array[Byte], decompLen: Int): Unit = { val compLen = lz4.compress(comp, 4, buf, decompLen) Memory.storeInt(comp, 0, decompLen) // decompLen out.writeBlock(comp, compLen + 4) } - def getPos(): Long = out.getPos() + override def getPos(): Long = out.getPos() } final class LZ4SizeBasedCompressingOutputBlockBuffer( @@ -316,13 +316,13 @@ final class LZ4SizeBasedCompressingOutputBlockBuffer( ) extends OutputBlockBuffer { private val comp = new Array[Byte](8 + lz4.maxCompressedLength(blockSize)) - def flush(): Unit = + override def flush(): Unit = out.flush() - def close(): Unit = + override def close(): Unit = out.close() - def writeBlock(buf: Array[Byte], decompLen: Int): Unit = { + override def writeBlock(buf: Array[Byte], decompLen: Int): Unit = { if (decompLen < minCompressionSize) { System.arraycopy(buf, 0, comp, 4, decompLen) Memory.storeInt(comp, 0, 0) // uncompressed @@ -335,12 +335,12 @@ final class LZ4SizeBasedCompressingOutputBlockBuffer( } } - def getPos(): Long = out.getPos() + override def getPos(): Long = out.getPos() } object ZstdCompressLib { val instance = ThreadLocal.withInitial(new Supplier[ZstdCompressCtx]() { - def get = { + override def get = { val zstd = new ZstdCompressCtx() zstd.setLevel(Zstd.defaultCompressionLevel()) zstd.setChecksum(false) @@ -354,17 +354,17 @@ final class ZstdOutputBlockBuffer(blockSize: Int, out: OutputBlockBuffer) private[this] val zstd = ZstdCompressLib.instance.get private[this] val comp = new Array[Byte](4 + Zstd.compressBound(blockSize.toLong).toInt) - def flush(): Unit = out.flush() + override def flush(): Unit = out.flush() - def close(): Unit = out.close() + override def close(): Unit = out.close() - def writeBlock(buf: Array[Byte], decompLen: Int): Unit = { + override def writeBlock(buf: Array[Byte], decompLen: Int): Unit = { val compLen = zstd.compressByteArray(comp, 4, comp.length - 4, buf, 0, decompLen) Memory.storeInt(comp, 0, decompLen.toInt) out.writeBlock(comp, compLen.toInt + 4) } - def getPos(): Long = out.getPos() + override def getPos(): Long = out.getPos() } final class ZstdSizedBasedOutputBlockBuffer( @@ -375,11 +375,11 @@ final class ZstdSizedBasedOutputBlockBuffer( private[this] val zstd = ZstdCompressLib.instance.get private[this] val comp = new Array[Byte](4 + Zstd.compressBound(blockSize.toLong).toInt) - def flush(): Unit = out.flush() + override def flush(): Unit = out.flush() - def close(): Unit = out.close() + override def close(): Unit = out.close() - def writeBlock(buf: Array[Byte], decompLen: Int): Unit = { + override def writeBlock(buf: Array[Byte], decompLen: Int): Unit = { val compLen = if (decompLen < minCompressionSize) { System.arraycopy(buf, 0, comp, 4, decompLen) Memory.storeInt(comp, 0, 0) @@ -393,5 +393,5 @@ final class ZstdSizedBasedOutputBlockBuffer( out.writeBlock(comp, compLen + 4) } - def getPos(): Long = out.getPos() + override def getPos(): Long = out.getPos() } diff --git a/hail/hail/src/is/hail/io/RichContextRDDRegionValue.scala b/hail/hail/src/is/hail/io/RichContextRDDRegionValue.scala index 07c5b1dde55..0156dbabc03 100644 --- a/hail/hail/src/is/hail/io/RichContextRDDRegionValue.scala +++ b/hail/hail/src/is/hail/io/RichContextRDDRegionValue.scala @@ -204,7 +204,7 @@ class RichContextRDDLong(val crdd: ContextRDD[Long]) extends AnyVal { new Iterator[Long]() { private[this] var cleared: Boolean = false - def hasNext: Boolean = { + override def hasNext: Boolean = { if (!cleared) { cleared = true producerCtx.region.clear() @@ -212,7 +212,7 @@ class RichContextRDDLong(val crdd: ContextRDD[Long]) extends AnyVal { it.hasNext } - def next(): Long = { + override def next(): Long = { if (!cleared) { producerCtx.region.clear() } @@ -264,7 +264,7 @@ class RichContextRDDRegionValue(val crdd: ContextRDD[RegionValue]) extends AnyVa new Iterator[RegionValue]() { private[this] var cleared: Boolean = false - def hasNext: Boolean = { + override def hasNext: Boolean = { if (!cleared) { cleared = true producerCtx.region.clear() @@ -272,7 +272,7 @@ class RichContextRDDRegionValue(val crdd: ContextRDD[RegionValue]) extends AnyVa it.hasNext } - def next(): RegionValue = { + override def next(): RegionValue = { if (!cleared) { producerCtx.region.clear() } @@ -297,7 +297,7 @@ class RichContextRDDRegionValue(val crdd: ContextRDD[RegionValue]) extends AnyVa new Iterator[RegionValue]() { private[this] var cleared: Boolean = false - def hasNext: Boolean = { + override def hasNext: Boolean = { if (!cleared) { cleared = true ctx.region.clear() @@ -305,7 +305,7 @@ class RichContextRDDRegionValue(val crdd: ContextRDD[RegionValue]) extends AnyVa it.hasNext } - def next(): RegionValue = { + override def next(): RegionValue = { if (!cleared) { ctx.region.clear() } diff --git a/hail/hail/src/is/hail/io/TypedCodecSpec.scala b/hail/hail/src/is/hail/io/TypedCodecSpec.scala index 87fdc32c573..16e6b1faf32 100644 --- a/hail/hail/src/is/hail/io/TypedCodecSpec.scala +++ b/hail/hail/src/is/hail/io/TypedCodecSpec.scala @@ -17,19 +17,20 @@ object TypedCodecSpec { final case class TypedCodecSpec(_eType: EType, _vType: Type, _bufferSpec: BufferSpec) extends AbstractTypedCodecSpec { - def encodedType: EType = _eType - def encodedVirtualType: Type = _vType + override def encodedType: EType = _eType + override def encodedVirtualType: Type = _vType - def buildEncoder(ctx: ExecuteContext, t: PType): (OutputStream, HailClassLoader) => Encoder = { + override def buildEncoder(ctx: ExecuteContext, t: PType) + : (OutputStream, HailClassLoader) => Encoder = { val bufferToEncoder = encodedType.buildEncoder(ctx, t) (out: OutputStream, theHailClassLoader: HailClassLoader) => bufferToEncoder(_bufferSpec.buildOutputBuffer(out), theHailClassLoader) } - def decodedPType(requestedType: Type): PType = + override def decodedPType(requestedType: Type): PType = encodedType.decodedPType(requestedType) - def buildDecoder(ctx: ExecuteContext, requestedType: Type) + override def buildDecoder(ctx: ExecuteContext, requestedType: Type) : (PType, (InputStream, HailClassLoader) => Decoder) = { val (rt, bufferToDecoder) = encodedType.buildDecoder(ctx, requestedType) ( @@ -45,9 +46,9 @@ final case class TypedCodecSpec(_eType: EType, _vType: Type, _bufferSpec: Buffer pType -> makeDec } - def buildCodeInputBuffer(is: Code[InputStream]): Code[InputBuffer] = + override def buildCodeInputBuffer(is: Code[InputStream]): Code[InputBuffer] = _bufferSpec.buildCodeInputBuffer(is) - def buildCodeOutputBuffer(os: Code[OutputStream]): Code[OutputBuffer] = + override def buildCodeOutputBuffer(os: Code[OutputStream]): Code[OutputBuffer] = _bufferSpec.buildCodeOutputBuffer(os) } diff --git a/hail/hail/src/is/hail/io/avro/AvroPartitionReader.scala b/hail/hail/src/is/hail/io/avro/AvroPartitionReader.scala index 4d3de648d5c..b0d80b23012 100644 --- a/hail/hail/src/is/hail/io/avro/AvroPartitionReader.scala +++ b/hail/hail/src/is/hail/io/avro/AvroPartitionReader.scala @@ -25,7 +25,7 @@ import org.apache.avro.io.DatumReader import org.json4s.{Extraction, JValue} case class AvroPartitionReader(schema: Schema, uidFieldName: String) extends PartitionReader { - def contextType: Type = TStruct("partitionPath" -> TString, "partitionIndex" -> TInt64) + override def contextType: Type = TStruct("partitionPath" -> TString, "partitionIndex" -> TInt64) def fullRowTypeWithoutUIDs: TStruct = AvroReader.schemaToType(schema) @@ -78,7 +78,7 @@ case class AvroPartitionReader(schema: Schema, uidFieldName: String) extends Par override def method: EmitMethodBuilder[_] = cb.emb val length: Option[EmitCodeBuilder => Code[Int]] = None - def initialize(cb: EmitCodeBuilder, outerRegion: Value[Region]): Unit = { + override def initialize(cb: EmitCodeBuilder, outerRegion: Value[Region]): Unit = { val mb = cb.emb val codeSchema = cb.newLocal("schema", mb.getObject(schema)) cb.assign(record, Code.newInstance[GenericData.Record, Schema](codeSchema)) @@ -121,14 +121,14 @@ case class AvroPartitionReader(schema: Schema, uidFieldName: String) extends Par } } - def close(cb: EmitCodeBuilder): Unit = cb += it.invoke[Unit]("close") + override def close(cb: EmitCodeBuilder): Unit = cb += it.invoke[Unit]("close") } SStreamValue(producer) } } - def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) + override def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) } object AvroReader { diff --git a/hail/hail/src/is/hail/io/avro/AvroTableReader.scala b/hail/hail/src/is/hail/io/avro/AvroTableReader.scala index f82b1dacc8f..5098bb3d39f 100644 --- a/hail/hail/src/is/hail/io/avro/AvroTableReader.scala +++ b/hail/hail/src/is/hail/io/avro/AvroTableReader.scala @@ -39,9 +39,9 @@ class AvroTableReader( RVDPartitioner.unkeyed(stateManager, paths.length) } - def pathsUsed: Seq[String] = paths + override def pathsUsed: Seq[String] = paths - def partitionCounts: Option[IndexedSeq[Long]] = None + override def partitionCounts: Option[IndexedSeq[Long]] = None override def uidType = TTuple(TInt64, TInt64) @@ -66,7 +66,7 @@ class AvroTableReader( : VirtualTypeWithReq = VirtualTypeWithReq(PCanonicalStruct(required = true)) - def renderShort(): String = defaultRender() + override def renderShort(): String = defaultRender() override def lower(ctx: ExecuteContext, requestedType: TableType): TableStage = { val globals = MakeStruct(FastSeq()) diff --git a/hail/hail/src/is/hail/io/bgen/BgenSettings.scala b/hail/hail/src/is/hail/io/bgen/BgenSettings.scala index c22a2fea6de..e97da857b26 100644 --- a/hail/hail/src/is/hail/io/bgen/BgenSettings.scala +++ b/hail/hail/src/is/hail/io/bgen/BgenSettings.scala @@ -115,7 +115,7 @@ object BgenSettings { )) new AbstractIndexSpec { - def relPath = fatal("relPath called for bgen index spec") + override def relPath = fatal("relPath called for bgen index spec") val leafCodec = TypedCodecSpec(leafEType, leafVType, bufferSpec) val internalNodeCodec = TypedCodecSpec(internalNodeEType, internalNodeVType, bufferSpec) val keyType = keyVType diff --git a/hail/hail/src/is/hail/io/bgen/LoadBgen.scala b/hail/hail/src/is/hail/io/bgen/LoadBgen.scala index 47cc3c32965..18f2b06cc7b 100644 --- a/hail/hail/src/is/hail/io/bgen/LoadBgen.scala +++ b/hail/hail/src/is/hail/io/bgen/LoadBgen.scala @@ -510,19 +510,19 @@ class MatrixBGENReader( filePartitionInfo: IndexedSeq[FilePartitionInfo], variants: Option[String], ) extends MatrixHybridReader { - def pathsUsed: Seq[String] = filePartitionInfo.map(_.metadata.path) + override def pathsUsed: Seq[String] = filePartitionInfo.map(_.metadata.path) lazy val nVariants: Long = filePartitionInfo.map(_.metadata.nVariants).sum - def rowUIDType = TTuple(TInt64, TInt64) + override def rowUIDType = TTuple(TInt64, TInt64) - def colUIDType = TInt64 + override def colUIDType = TInt64 private val nSamples = sampleIds.length - def columnCount: Option[Int] = Some(nSamples) + override def columnCount: Option[Int] = Some(nSamples) - def partitionCounts: Option[IndexedSeq[Long]] = None + override def partitionCounts: Option[IndexedSeq[Long]] = None private var _settings: BgenSettings = _ @@ -574,7 +574,7 @@ class MatrixBGENReader( override def toJValue: JValue = params.toJValue - def renderShort(): String = defaultRender() + override def renderShort(): String = defaultRender() override def hashCode(): Int = params.hashCode() @@ -672,12 +672,12 @@ case class BgenPartitionReaderWithVariantFilter( lazy val uidType = TTuple(TInt64, TInt64) lazy val fullRowType: TStruct = MatrixBGENReader.fullTableType(rg).rowType - def rowRequiredness(requestedType: TStruct): RStruct = + override def rowRequiredness(requestedType: TStruct): RStruct = StagedBGENReader.rowRequiredness(requestedType) - def uidFieldName: String = TableReader.uidFieldName + override def uidFieldName: String = TableReader.uidFieldName - def emitStream( + override def emitStream( ctx: ExecuteContext, cb: EmitCodeBuilder, mb: EmitMethodBuilder[_], @@ -830,7 +830,7 @@ case class BgenPartitionReaderWithVariantFilter( } } - def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) + override def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) } case class BgenPartitionReader(fileMetadata: Array[BgenFileMetadata], rg: Option[String]) @@ -846,12 +846,12 @@ case class BgenPartitionReader(fileMetadata: Array[BgenFileMetadata], rg: Option lazy val fullRowType: TStruct = MatrixBGENReader.fullTableType(rg).rowType - def rowRequiredness(requestedType: TStruct): RStruct = + override def rowRequiredness(requestedType: TStruct): RStruct = StagedBGENReader.rowRequiredness(requestedType) - def uidFieldName: String = TableReader.uidFieldName + override def uidFieldName: String = TableReader.uidFieldName - def emitStream( + override def emitStream( ctx: ExecuteContext, cb: EmitCodeBuilder, mb: EmitMethodBuilder[_], @@ -973,5 +973,5 @@ case class BgenPartitionReader(fileMetadata: Array[BgenFileMetadata], rg: Option } } - def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) + override def toJValue: JValue = Extraction.decompose(this)(PartitionReader.formats) } diff --git a/hail/hail/src/is/hail/io/fs/AzureStorageFS.scala b/hail/hail/src/is/hail/io/fs/AzureStorageFS.scala index 046171318bd..3bf3efebc7c 100644 --- a/hail/hail/src/is/hail/io/fs/AzureStorageFS.scala +++ b/hail/hail/src/is/hail/io/fs/AzureStorageFS.scala @@ -151,9 +151,9 @@ class AzureStorageFS(val credential: AzureCloudCredentials) extends FS { case _: IllegalArgumentException => false } - def getConfiguration(): Unit = () + override def getConfiguration(): Unit = () - def setConfiguration(config: Any): Unit = {} + override def setConfiguration(config: Any): Unit = {} // ABS errors if you attempt credentialed access for a public container, // so we try once with credentials, if that fails use anonymous access for @@ -190,7 +190,7 @@ class AzureStorageFS(val credential: AzureCloudCredentials) extends FS { getServiceClient(url).getBlobContainerClient(url.container) } - def openNoCompression(url: URL): SeekableDataInputStream = handlePublicAccessError(url) { + override def openNoCompression(url: URL): SeekableDataInputStream = handlePublicAccessError(url) { val blobSize = getBlobClient(url).getProperties.getBlobSize val is: SeekableInputStream = new FSSeekableInputStream { @@ -238,7 +238,7 @@ class AzureStorageFS(val credential: AzureCloudCredentials) extends FS { new WrappedSeekableDataInputStream(is) } - def createNoCompression(url: URL): PositionedDataOutputStream = retryTransientErrors { + override def createNoCompression(url: URL): PositionedDataOutputStream = retryTransientErrors { val blockBlobClient = getBlobClient(url).getBlockBlobClient val os: PositionedOutputStream = new FSPositionedOutputStream(4 * 1024 * 1024) { @@ -268,7 +268,7 @@ class AzureStorageFS(val credential: AzureCloudCredentials) extends FS { new WrappedPositionedDataOutputStream(os) } - def delete(url: URL, recursive: Boolean): Unit = retryTransientErrors { + override def delete(url: URL, recursive: Boolean): Unit = retryTransientErrors { val blobClient: BlobClient = getBlobClient(url) if (recursive) { @@ -294,7 +294,7 @@ class AzureStorageFS(val credential: AzureCloudCredentials) extends FS { } } - def listDirectory(url: URL): Array[FileListEntry] = handlePublicAccessError(url) { + override def listDirectory(url: URL): Array[FileListEntry] = handlePublicAccessError(url) { val blobContainerClient: BlobContainerClient = getContainerClient(url) val statList: ArrayBuffer[FileListEntry] = ArrayBuffer() @@ -307,7 +307,7 @@ class AzureStorageFS(val credential: AzureCloudCredentials) extends FS { statList.toArray } - def glob(url: URL): Array[FileListEntry] = handlePublicAccessError(url) { + override def glob(url: URL): Array[FileListEntry] = handlePublicAccessError(url) { globWithPrefix(prefix = url.withPath(""), path = dropTrailingSlash(url.path)) } @@ -351,7 +351,7 @@ class AzureStorageFS(val credential: AzureCloudCredentials) extends FS { Some(getBlobClient(url).getProperties.getETag) } - def makeQualified(filename: String): String = { + override def makeQualified(filename: String): String = { parseUrl(filename): Unit filename } diff --git a/hail/hail/src/is/hail/io/fs/FS.scala b/hail/hail/src/is/hail/io/fs/FS.scala index fa1ff6871c8..ffa6e92bb4d 100644 --- a/hail/hail/src/is/hail/io/fs/FS.scala +++ b/hail/hail/src/is/hail/io/fs/FS.scala @@ -18,14 +18,14 @@ import org.apache.hadoop class WrappedSeekableDataInputStream(is: SeekableInputStream) extends DataInputStream(is) with Seekable { - def getPosition: Long = is.getPosition + override def getPosition: Long = is.getPosition - def seek(pos: Long): Unit = is.seek(pos) + override def seek(pos: Long): Unit = is.seek(pos) } class WrappedPositionedDataOutputStream(os: PositionedOutputStream) extends DataOutputStream(os) with Positioned { - def getPosition: Long = os.getPosition + override def getPosition: Long = os.getPosition } class WrappedPositionOutputStream(os: OutputStream) extends OutputStream with Positioned { @@ -44,7 +44,7 @@ class WrappedPositionOutputStream(os: OutputStream) extends OutputStream with Po override def close(): Unit = os.close() - def getPosition: Long = count + override def getPosition: Long = count } abstract class FSURL[URL <: FSURL[URL]] { @@ -74,15 +74,15 @@ class BlobStorageFileStatus( size: Long, ) extends FileStatus { // NB: it is called getPath but it *must* return the URL *with* the scheme. - def getPath: String = + override def getPath: String = dropTrailingSlash( actualUrl ) // getPath is a backwards compatible method: in the past, Hail dropped trailing slashes - def getActualUrl: String = actualUrl - def getModificationTime: java.lang.Long = modificationTime - def getLen: Long = size - def isSymlink: Boolean = false - def getOwner: String = null + override def getActualUrl: String = actualUrl + override def getModificationTime: java.lang.Long = modificationTime + override def getLen: Long = size + override def isSymlink: Boolean = false + override def getOwner: String = null } class BlobStorageFileListEntry( @@ -95,8 +95,8 @@ class BlobStorageFileListEntry( modificationTime, size, ) with FileListEntry { - def isDirectory: Boolean = isDir - def isFile: Boolean = !isDir + override def isDirectory: Boolean = isDir + override def isFile: Boolean = !isDir override def isFileOrFileAndDirectory = isFile override def toString: String = s"BSFLE($actualUrl $modificationTime $size $isDir)" @@ -110,15 +110,16 @@ trait CompressionCodec { object GZipCompressionCodec extends CompressionCodec { // java.util.zip.GZIPInputStream does not support concatenated files/multiple blocks - def makeInputStream(is: InputStream): InputStream = new GzipCompressorInputStream(is, true) + override def makeInputStream(is: InputStream): InputStream = + new GzipCompressorInputStream(is, true) - def makeOutputStream(os: OutputStream): OutputStream = new GZIPOutputStream(os) + override def makeOutputStream(os: OutputStream): OutputStream = new GZIPOutputStream(os) } object BGZipCompressionCodec extends CompressionCodec { - def makeInputStream(is: InputStream): InputStream = new BGzipInputStream(is) + override def makeInputStream(is: InputStream): InputStream = new BGzipInputStream(is) - def makeOutputStream(os: OutputStream): OutputStream = new BGzipOutputStream(os) + override def makeOutputStream(os: OutputStream): OutputStream = new BGzipOutputStream(os) } class FileAndDirectoryException(message: String) extends RuntimeException(message) @@ -203,7 +204,7 @@ abstract class FSSeekableInputStream extends InputStream with Seekable { protected def physicalSeek(newPos: Long): Unit - def seek(newPos: Long): Unit = { + override def seek(newPos: Long): Unit = { eof = false val distance = newPos - pos val bufferSeekPosition = bb.position() + distance @@ -218,7 +219,7 @@ abstract class FSSeekableInputStream extends InputStream with Seekable { pos = newPos } - def getPosition: Long = pos + override def getPosition: Long = pos } abstract class FSPositionedOutputStream(val capacity: Int) extends OutputStream with Positioned { @@ -226,9 +227,9 @@ abstract class FSPositionedOutputStream(val capacity: Int) extends OutputStream protected[this] val bb: ByteBuffer = ByteBuffer.allocate(capacity) protected[this] var pos: Long = 0 - def flush(): Unit + override def flush(): Unit - def write(i: Int): Unit = { + override def write(i: Int): Unit = { if (bb.remaining() == 0) flush() bb.put(i.toByte) @@ -249,7 +250,7 @@ abstract class FSPositionedOutputStream(val capacity: Int) extends OutputStream } } - def getPosition: Long = pos + override def getPosition: Long = pos } trait FS extends Serializable with Logging { diff --git a/hail/hail/src/is/hail/io/fs/GoogleStorageFS.scala b/hail/hail/src/is/hail/io/fs/GoogleStorageFS.scala index 4c5f33f251d..c4e3cac4310 100644 --- a/hail/hail/src/is/hail/io/fs/GoogleStorageFS.scala +++ b/hail/hail/src/is/hail/io/fs/GoogleStorageFS.scala @@ -144,10 +144,10 @@ class GoogleStorageFS( override def validUrl(filename: String): Boolean = filename.startsWith("gs://") - def getConfiguration(): Option[RequesterPaysConfig] = + override def getConfiguration(): Option[RequesterPaysConfig] = requesterPaysConfig - def setConfiguration(config: Any): Unit = + override def setConfiguration(config: Any): Unit = requesterPaysConfig = config.asInstanceOf[Option[RequesterPaysConfig]] private[this] def requesterPaysOptions[T](bucket: String, makeUserProjectOption: String => T) @@ -186,7 +186,7 @@ class GoogleStorageFS( .getService } - def openNoCompression(url: URL): SeekableDataInputStream = retryTransientErrors { + override def openNoCompression(url: URL): SeekableDataInputStream = retryTransientErrors { val is: SeekableInputStream = new FSSeekableInputStream { private[this] var reader: ReadChannel = null private[this] var options: Option[Seq[BlobSourceOption]] = None @@ -248,7 +248,7 @@ class GoogleStorageFS( new WrappedSeekableDataInputStream(is) } - def createNoCompression(url: URL): PositionedDataOutputStream = retryTransientErrors { + override def createNoCompression(url: URL): PositionedDataOutputStream = retryTransientErrors { logger.info(f"createNoCompression: $url") val blobId = BlobId.of(url.bucket, url.path) @@ -376,7 +376,7 @@ class GoogleStorageFS( if (deleteSource) storage.delete(srcId): Unit } - def delete(url: URL, recursive: Boolean): Unit = + override def delete(url: URL, recursive: Boolean): Unit = retryTransientErrors[Unit] { if (recursive) { var page = retryTransientErrors { @@ -418,7 +418,7 @@ class GoogleStorageFS( } } - def glob(url: URL): Array[FileListEntry] = retryTransientErrors { + override def glob(url: URL): Array[FileListEntry] = retryTransientErrors { globWithPrefix(url.withPath(""), path = dropTrailingSlash(url.path)) } @@ -500,7 +500,7 @@ class GoogleStorageFS( url.bucket, ) - def makeQualified(filename: String): String = { + override def makeQualified(filename: String): String = { if (!filename.startsWith("gs://")) throw new IllegalArgumentException(s"Invalid path, expected gs://bucket/path $filename") filename diff --git a/hail/hail/src/is/hail/io/fs/HadoopFS.scala b/hail/hail/src/is/hail/io/fs/HadoopFS.scala index 0241f4840f7..e8532341652 100644 --- a/hail/hail/src/is/hail/io/fs/HadoopFS.scala +++ b/hail/hail/src/is/hail/io/fs/HadoopFS.scala @@ -13,21 +13,21 @@ import org.apache.hadoop.fs.{EtagSource, FSDataInputStream, FSDataOutputStream, class HadoopFileListEntry(fs: hadoop.fs.FileStatus) extends FileListEntry { val normalizedPath = fs.getPath - def getPath: String = fs.getPath.toString + override def getPath: String = fs.getPath.toString - def getActualUrl: String = fs.getPath.toString + override def getActualUrl: String = fs.getPath.toString - def getModificationTime: java.lang.Long = fs.getModificationTime + override def getModificationTime: java.lang.Long = fs.getModificationTime - def getLen: Long = fs.getLen + override def getLen: Long = fs.getLen - def isDirectory: Boolean = fs.isDirectory + override def isDirectory: Boolean = fs.isDirectory - def isFile: Boolean = fs.isFile + override def isFile: Boolean = fs.isFile - def isSymlink: Boolean = fs.isSymlink + override def isSymlink: Boolean = fs.isSymlink - def getOwner: String = fs.getOwner + override def getOwner: String = fs.getOwner } object HadoopFS { @@ -47,7 +47,7 @@ object HadoopFS { closed = true } - def getPosition: Long = os.getPos + override def getPosition: Long = os.getPos } def toSeekableInputStream(is: FSDataInputStream): SeekableInputStream = @@ -66,9 +66,9 @@ object HadoopFS { closed = true } - def seek(pos: Long): Unit = is.seek(pos) + override def seek(pos: Long): Unit = is.seek(pos) - def getPosition: Long = is.getPos + override def getPosition: Long = is.getPos } } @@ -93,19 +93,19 @@ class HadoopFS(private[this] var conf: SerializableHadoopConfiguration) extends override def validUrl(filename: String): Boolean = Try(getFileSystem(filename)).isSuccess - def getConfiguration(): SerializableHadoopConfiguration = conf + override def getConfiguration(): SerializableHadoopConfiguration = conf - def setConfiguration(_conf: Any): Unit = + override def setConfiguration(_conf: Any): Unit = conf = _conf.asInstanceOf[SerializableHadoopConfiguration] - def createNoCompression(url: URL): PositionedDataOutputStream = { + override def createNoCompression(url: URL): PositionedDataOutputStream = { val os = url.hadoopFs.create(url.hadoopPath) new WrappedPositionedDataOutputStream( HadoopFS.toPositionedOutputStream(os) ) } - def openNoCompression(url: URL): SeekableDataInputStream = { + override def openNoCompression(url: URL): SeekableDataInputStream = { val is = try url.hadoopFs.open(url.hadoopPath) @@ -126,7 +126,7 @@ class HadoopFS(private[this] var conf: SerializableHadoopConfiguration) extends def getFileSystem(filename: String): hadoop.fs.FileSystem = new hadoop.fs.Path(filename).getFileSystem(conf.value) - def listDirectory(url: URL): Array[FileListEntry] = { + override def listDirectory(url: URL): Array[FileListEntry] = { val statuses = url.hadoopFs.globStatus(url.hadoopPath) if (statuses == null) { throw new FileNotFoundException(url.toString) @@ -147,7 +147,7 @@ class HadoopFS(private[this] var conf: SerializableHadoopConfiguration) extends def rmtree(dirname: String): Unit = getFileSystem(dirname).delete(new hadoop.fs.Path(dirname), true) - def delete(url: URL, recursive: Boolean): Unit = + override def delete(url: URL, recursive: Boolean): Unit = url.hadoopFs.delete(url.hadoopPath, recursive): Unit override def globAll(filenames: Iterable[String]): Array[FileListEntry] = { @@ -159,7 +159,7 @@ class HadoopFS(private[this] var conf: SerializableHadoopConfiguration) extends }.toArray } - def glob(url: URL): Array[FileListEntry] = { + override def glob(url: URL): Array[FileListEntry] = { var files = url.hadoopFs.globStatus(url.hadoopPath) if (files == null) files = Array.empty @@ -177,7 +177,7 @@ class HadoopFS(private[this] var conf: SerializableHadoopConfiguration) extends fle } - def fileListEntry(url: URL): FileListEntry = + override def fileListEntry(url: URL): FileListEntry = new HadoopFileListEntry(url.hadoopFs.getFileStatus(url.hadoopPath)) override def eTag(url: URL): Option[String] = @@ -186,7 +186,7 @@ class HadoopFS(private[this] var conf: SerializableHadoopConfiguration) extends else None - def makeQualified(path: String): String = { + override def makeQualified(path: String): String = { val ppath = new hadoop.fs.Path(path) val pathFS = ppath.getFileSystem(conf.value) pathFS.makeQualified(ppath).toString diff --git a/hail/hail/src/is/hail/io/fs/RouterFS.scala b/hail/hail/src/is/hail/io/fs/RouterFS.scala index 39104f43769..9f1ac503105 100644 --- a/hail/hail/src/is/hail/io/fs/RouterFS.scala +++ b/hail/hail/src/is/hail/io/fs/RouterFS.scala @@ -91,28 +91,28 @@ class RouterFS(fss: IndexedSeq[FS]) extends FS { override def createNoCompression(url: RouterFSURL): PositionedDataOutputStream = url.fs.createNoCompression(url.url) - def openNoCompression(url: URL): SeekableDataInputStream = + override def openNoCompression(url: URL): SeekableDataInputStream = url.fs.openNoCompression(url.url) override def mkDir(url: URL): Unit = url.fs.mkDir(url.url) - def delete(url: URL, recursive: Boolean) = url.fs.delete(url.url, recursive) + override def delete(url: URL, recursive: Boolean) = url.fs.delete(url.url, recursive) - def listDirectory(url: URL): Array[FileListEntry] = url.fs.listDirectory(url.url) + override def listDirectory(url: URL): Array[FileListEntry] = url.fs.listDirectory(url.url) - def glob(url: URL): Array[FileListEntry] = url.fs.glob(url.url) + override def glob(url: URL): Array[FileListEntry] = url.fs.glob(url.url) - def fileStatus(url: URL): FileStatus = url.fs.fileStatus(url.url) + override def fileStatus(url: URL): FileStatus = url.fs.fileStatus(url.url) - def fileListEntry(url: URL): FileListEntry = url.fs.fileListEntry(url.url) + override def fileListEntry(url: URL): FileListEntry = url.fs.fileListEntry(url.url) override def eTag(url: URL): Option[String] = url.fs.eTag(url.url) - def makeQualified(path: String): String = lookupFS(path).makeQualified(path) + override def makeQualified(path: String): String = lookupFS(path).makeQualified(path) - def getConfiguration(): Any = fss.map(_.getConfiguration()) + override def getConfiguration(): Any = fss.map(_.getConfiguration()) - def setConfiguration(config: Any): Unit = + override def setConfiguration(config: Any): Unit = fss.zip(config.asInstanceOf[IndexedSeq[_]]).foreach { case (fs: FS, config: Any) => fs.setConfiguration(config) } diff --git a/hail/hail/src/is/hail/io/gen/ExportGen.scala b/hail/hail/src/is/hail/io/gen/ExportGen.scala index bd62fffe52e..cd56f750350 100644 --- a/hail/hail/src/is/hail/io/gen/ExportGen.scala +++ b/hail/hail/src/is/hail/io/gen/ExportGen.scala @@ -60,7 +60,7 @@ class GenAnnotationView(rowType: PStruct) extends View { private var cachedVarid: String = _ private var cachedRsid: String = _ - def set(offset: Long): Unit = { + override def set(offset: Long): Unit = { assert(rowType.isFieldDefined(offset, varidIdx)) assert(rowType.isFieldDefined(offset, rsidIdx)) this.rsidOffset = rowType.loadField(offset, rsidIdx) diff --git a/hail/hail/src/is/hail/io/hadoop/ByteArrayOutputFormat.scala b/hail/hail/src/is/hail/io/hadoop/ByteArrayOutputFormat.scala index a24e4f6d283..7016b73cf9f 100644 --- a/hail/hail/src/is/hail/io/hadoop/ByteArrayOutputFormat.scala +++ b/hail/hail/src/is/hail/io/hadoop/ByteArrayOutputFormat.scala @@ -12,11 +12,11 @@ class ByteArrayOutputFormat extends FileOutputFormat[NullWritable, BytesOnlyWrit class ByteArrayRecordWriter(out: DataOutputStream) extends RecordWriter[NullWritable, BytesOnlyWritable] { - def write(key: NullWritable, value: BytesOnlyWritable): Unit = + override def write(key: NullWritable, value: BytesOnlyWritable): Unit = if (value != null) value.write(out) - def close(reporter: Reporter): Unit = + override def close(reporter: Reporter): Unit = out.close() } diff --git a/hail/hail/src/is/hail/io/index/IndexReader.scala b/hail/hail/src/is/hail/io/index/IndexReader.scala index 593171afba2..439ea4e216e 100644 --- a/hail/hail/src/is/hail/io/index/IndexReader.scala +++ b/hail/hail/src/is/hail/io/index/IndexReader.scala @@ -250,7 +250,7 @@ class IndexReader( var localPos = 0 var leafNode: LeafNode = _ - def next(): LeafChild = { + override def next(): LeafChild = { assert(hasNext) if (leafNode == null || localPos >= leafNode.children.length) { @@ -265,7 +265,7 @@ class IndexReader( child } - def hasNext: Boolean = pos < end + override def hasNext: Boolean = pos < end def seek(key: Annotation): Unit = { val newPos = lowerBound(key) @@ -281,7 +281,7 @@ class IndexReader( def iterateUntil(key: Annotation): Iterator[LeafChild] = iterator(0, lowerBound(key)) - def close(): Unit = { + override def close(): Unit = { leafDecoder.close() internalDecoder.close() logger.info(s"Index reader cache queries: ${cacheHits + cacheMisses}") diff --git a/hail/hail/src/is/hail/io/index/IndexWriter.scala b/hail/hail/src/is/hail/io/index/IndexWriter.scala index 3cb5a37d78a..4b737331be2 100644 --- a/hail/hail/src/is/hail/io/index/IndexWriter.scala +++ b/hail/hail/src/is/hail/io/index/IndexWriter.scala @@ -121,7 +121,7 @@ class IndexWriter( def trackedOS(): ByteTrackingOutputStream = comp.trackedOS() - def close(): Unit = { + override def close(): Unit = { region.close() comp.close() } diff --git a/hail/hail/src/is/hail/io/plink/LoadPlink.scala b/hail/hail/src/is/hail/io/plink/LoadPlink.scala index 14649b640a8..429a28e3baa 100644 --- a/hail/hail/src/is/hail/io/plink/LoadPlink.scala +++ b/hail/hail/src/is/hail/io/plink/LoadPlink.scala @@ -367,16 +367,16 @@ class MatrixPLINKReader( partitioner: RVDPartitioner, ) extends MatrixHybridReader { - def rowUIDType = TInt64 - def colUIDType = TInt64 + override def rowUIDType = TInt64 + override def colUIDType = TInt64 - def pathsUsed: Seq[String] = FastSeq(params.bed, params.bim, params.fam) + override def pathsUsed: Seq[String] = FastSeq(params.bed, params.bim, params.fam) def nSamples: Int = sampleInfo.length val columnCount: Option[Int] = Some(nSamples) - def partitionCounts: Option[IndexedSeq[Long]] = None + override def partitionCounts: Option[IndexedSeq[Long]] = None val globals = Row(sampleInfo.zipWithIndex.map { case (s, idx) => Row((0 until s.length).map(s.apply) :+ idx.toLong: _*) @@ -582,7 +582,7 @@ class MatrixPLINKReader( decomposeWithName(params, "MatrixPLINKReader") } - def renderShort(): String = defaultRender() + override def renderShort(): String = defaultRender() override def hashCode(): Int = params.hashCode() diff --git a/hail/hail/src/is/hail/io/tabix/TabixReader.scala b/hail/hail/src/is/hail/io/tabix/TabixReader.scala index 891e90616b6..8beac442497 100644 --- a/hail/hail/src/is/hail/io/tabix/TabixReader.scala +++ b/hail/hail/src/is/hail/io/tabix/TabixReader.scala @@ -26,8 +26,7 @@ class Tabix( ) case class TbiPair(var _1: Long, var _2: Long) extends java.lang.Comparable[TbiPair] { - @Override - def compareTo(other: TbiPair): Int = TbiOrd.compare(this, other) + override def compareTo(other: TbiPair): Int = TbiOrd.compare(this, other) } object TbiPair { @@ -35,7 +34,7 @@ object TbiPair { } object TbiOrd extends Ordering[TbiPair] { - def compare(u: TbiPair, v: TbiPair): Int = if (u._1 == v._1) { + override def compare(u: TbiPair, v: TbiPair): Int = if (u._1 == v._1) { 0 } else if (less64(u._1, v._1)) { -1 diff --git a/hail/hail/src/is/hail/io/vcf/LoadVCF.scala b/hail/hail/src/is/hail/io/vcf/LoadVCF.scala index 285fabb75b1..d39a1f41b8a 100644 --- a/hail/hail/src/is/hail/io/vcf/LoadVCF.scala +++ b/hail/hail/src/is/hail/io/vcf/LoadVCF.scala @@ -1522,7 +1522,7 @@ object LoadVCF extends Logging { val abf = new MissingArrayBuilder[Float] val abd = new MissingArrayBuilder[Double] - def hasNext: Boolean = { + override def hasNext: Boolean = { while (!present && it.hasNext) { val lwc = it.next() val line = lwc.value @@ -1583,7 +1583,7 @@ object LoadVCF extends Logging { present } - def next(): Long = { + override def next(): Long = { // call hasNext to advance if necessary if (!hasNext) throw new java.util.NoSuchElementException() @@ -1688,9 +1688,9 @@ class PartitionedVCFRDD( val contigRemappingBc = if (reverseContigMapping.size != 0) sparkContext.broadcast(reverseContigMapping) else null - protected def getPartitions: Array[Partition] = _partitions + override protected def getPartitions: Array[Partition] = _partitions - def compute(split: Partition, context: TaskContext): Iterator[WithContext[String]] = { + override def compute(split: Partition, context: TaskContext): Iterator[WithContext[String]] = { val p = split.asInstanceOf[PartitionedVCFPartition] val chromToQuery = if (contigRemappingBc != null) @@ -1715,9 +1715,9 @@ class PartitionedVCFRDD( private var l = lines.next() private var curIdx: Long = lines.getCurIdx() - def hasNext: Boolean = l != null + override def hasNext: Boolean = l != null - def next(): WithContext[String] = { + override def next(): WithContext[String] = { assert(l != null) val n = l l = lines.next() @@ -1917,8 +1917,8 @@ class MatrixVCFReader( LoadVCF.warnDuplicates(sampleIDs) - def rowUIDType = TTuple(TInt64, TInt64) - def colUIDType = TInt64 + override def rowUIDType = TTuple(TInt64, TInt64) + override def colUIDType = TInt64 val (infoPType, rowValuePType, formatPType) = header.getPTypes( params.arrayElementsRequired, @@ -1926,7 +1926,7 @@ class MatrixVCFReader( params.callFields, ) - def fullMatrixTypeWithoutUIDs: MatrixType = MatrixType( + override def fullMatrixTypeWithoutUIDs: MatrixType = MatrixType( globalType = TStruct.empty, colType = TStruct("s" -> TString), colKey = Array("s"), @@ -1958,13 +1958,13 @@ class MatrixVCFReader( fullType.key, ) - def pathsUsed: Seq[String] = params.files + override def pathsUsed: Seq[String] = params.files def nCols: Int = sampleIDs.length val columnCount: Option[Int] = Some(nCols) - def partitionCounts: Option[IndexedSeq[Long]] = None + override def partitionCounts: Option[IndexedSeq[Long]] = None def partitioner(sm: HailStateManager): Option[RVDPartitioner] = params.partitionsJSON.map { partitionsJSON => @@ -2140,7 +2140,7 @@ class MatrixVCFReader( decomposeWithName(params, "MatrixVCFReader") } - def renderShort(): String = defaultRender() + override def renderShort(): String = defaultRender() override def hashCode(): Int = params.hashCode() @@ -2189,7 +2189,7 @@ case class GVCFPartitionReader( lazy val fullRowType: TStruct = fullRowPType.virtualType - def rowRequiredness(requestedType: TStruct): RStruct = + override def rowRequiredness(requestedType: TStruct): RStruct = VirtualTypeWithReq(tcoerce[PStruct](fullRowPType.subsetTo(requestedType))).r.asInstanceOf[ RStruct ] @@ -2199,7 +2199,7 @@ case class GVCFPartitionReader( decomposeWithName(this, "MatrixVCFReader") } - def emitStream( + override def emitStream( ctx: ExecuteContext, cb: EmitCodeBuilder, mb: EmitMethodBuilder[_], diff --git a/hail/hail/src/is/hail/io/vcf/TabixReadVCFIterator.scala b/hail/hail/src/is/hail/io/vcf/TabixReadVCFIterator.scala index ad5cf64e400..c29a749d3bb 100644 --- a/hail/hail/src/is/hail/io/vcf/TabixReadVCFIterator.scala +++ b/hail/hail/src/is/hail/io/vcf/TabixReadVCFIterator.scala @@ -48,9 +48,9 @@ class TabixReadVCFIterator( private[this] var l = lines.next() private[this] var curIdx: Long = lines.getCurIdx() private[this] val inner = new Iterator[GenericLine] { - def hasNext: Boolean = l != null + override def hasNext: Boolean = l != null - def next(): GenericLine = { + override def next(): GenericLine = { assert(l != null) try { val n = l @@ -85,11 +85,11 @@ class TabixReadVCFIterator( start <= pos && pos <= end } - def hasNext: Boolean = inner.hasNext + override def hasNext: Boolean = inner.hasNext - def next(): GenericLine = inner.next() + override def next(): GenericLine = inner.next() - def close(): Unit = lines.close() + override def close(): Unit = lines.close() } } diff --git a/hail/hail/src/is/hail/linalg/BLAS.scala b/hail/hail/src/is/hail/linalg/BLAS.scala index bee4988b60f..a9776f0d2fd 100644 --- a/hail/hail/src/is/hail/linalg/BLAS.scala +++ b/hail/hail/src/is/hail/linalg/BLAS.scala @@ -11,7 +11,7 @@ import com.sun.jna.ptr.{DoubleByReference, FloatByReference, IntByReference} object BLAS extends Logging { private[this] val libraryInstance = ThreadLocal.withInitial(new Supplier[BLASLibrary]() { - def get() = { + override def get() = { val standard = Native.load("blas", classOf[BLASLibrary]).asInstanceOf[BLASLibrary] verificationTest(standard) match { diff --git a/hail/hail/src/is/hail/linalg/BlockMatrix.scala b/hail/hail/src/is/hail/linalg/BlockMatrix.scala index 512672608a0..7b0eefdeedc 100644 --- a/hail/hail/src/is/hail/linalg/BlockMatrix.scala +++ b/hail/hail/src/is/hail/linalg/BlockMatrix.scala @@ -48,7 +48,7 @@ class CollectMatricesRDD(@transient var bms: IndexedSeq[BlockMatrix]) private val nBlocks = bms.map(_.blocks.getNumPartitions) private val firstPartition = nBlocks.scan(0)(_ + _).init - protected def getPartitions: Array[Partition] = + override protected def getPartitions: Array[Partition] = bms.iterator.zipWithIndex.map { case (bm, i) => CollectMatricesRDDPartition( i, @@ -65,7 +65,7 @@ class CollectMatricesRDD(@transient var bms: IndexedSeq[BlockMatrix]) bms.zipWithIndex.map { case (bm, i) => val n = nBlocks(i) new NarrowDependency(bm.blocks) { - def getParents(j: Int): Seq[Int] = + override def getParents(j: Int): Seq[Int] = if (j == i) 0 until n else @@ -73,7 +73,7 @@ class CollectMatricesRDD(@transient var bms: IndexedSeq[BlockMatrix]) } } - def compute(split: Partition, context: TaskContext): Iterator[BDM[Double]] = { + override def compute(split: Partition, context: TaskContext): Iterator[BDM[Double]] = { val p = split.asInstanceOf[CollectMatricesRDDPartition] val m = BDM.zeros[Double](p.nRows, p.nCols) val prev = parent[((Int, Int), BDM[Double])](p.index) @@ -117,11 +117,13 @@ object BlockMatrix { new RDD[((Int, Int), BDM[Double])](SparkBackend.sparkContext, Nil) { override val partitioner = Some(gp) - protected def getPartitions: Array[Partition] = Array.tabulate(gp.numPartitions)(pi => - new Partition { def index: Int = pi } - ) + override protected def getPartitions: Array[Partition] = + Array.tabulate(gp.numPartitions)(pi => + new Partition { override def index: Int = pi } + ) - def compute(split: Partition, context: TaskContext): Iterator[((Int, Int), BDM[Double])] = + override def compute(split: Partition, context: TaskContext) + : Iterator[((Int, Int), BDM[Double])] = Iterator.single(piBlock(gp, split.index)) }, gp.blockSize, @@ -1159,12 +1161,12 @@ class BlockMatrix( val newBlocks = blocks.zipPartitions(that.blocks, preservesPartitioning = true) { (thisIter, thatIter) => new Iterator[((Int, Int), BDM[Double])] { - def hasNext: Boolean = { + override def hasNext: Boolean = { assert(thisIter.hasNext == thatIter.hasNext) thisIter.hasNext } - def next(): ((Int, Int), BDM[Double]) = { + override def next(): ((Int, Int), BDM[Double]) = { val ((i1, j1), lm1) = thisIter.next() val ((i2, j2), lm2) = thatIter.next() assertCompatibleLocalMatrix(lm1) @@ -1212,12 +1214,12 @@ class BlockMatrix( val newBlocks = blocks.zipPartitions(that.blocks, preservesPartitioning = true) { (thisIter, thatIter) => new Iterator[((Int, Int), BDM[Double])] { - def hasNext: Boolean = { + override def hasNext: Boolean = { assert(thisIter.hasNext == thatIter.hasNext) thisIter.hasNext } - def next(): ((Int, Int), BDM[Double]) = { + override def next(): ((Int, Int), BDM[Double]) = { val ((i1, j1), lm1) = thisIter.next() val ((i2, j2), lm2) = thatIter.next() assertCompatibleLocalMatrix(lm1) @@ -1276,14 +1278,14 @@ class BlockMatrix( blocks.zipPartitions(bm2.blocks, bm3.blocks, bm4.blocks, preservesPartitioning = true) { (it1, it2, it3, it4) => new Iterator[((Int, Int), BDM[Double])] { - def hasNext: Boolean = { + override def hasNext: Boolean = { assert(it1.hasNext == it2.hasNext) assert(it1.hasNext == it3.hasNext) assert(it1.hasNext == it4.hasNext) it1.hasNext } - def next(): ((Int, Int), BDM[Double]) = { + override def next(): ((Int, Int), BDM[Double]) = { val ((i1, j1), lm1) = it1.next() val ((i2, j2), lm2) = it2.next() val ((i3, j3), lm3) = it3.next() @@ -1380,12 +1382,12 @@ class BlockMatrix( val newBlocks = blocks.zipPartitions(that.blocks, preservesPartitioning = true) { (thisIter, thatIter) => new Iterator[((Int, Int), BDM[Double])] { - def hasNext: Boolean = { + override def hasNext: Boolean = { assert(thisIter.hasNext == thatIter.hasNext) thisIter.hasNext } - def next(): ((Int, Int), BDM[Double]) = { + override def next(): ((Int, Int), BDM[Double]) = { val ((i1, j1), lm1) = thisIter.next() val ((i2, j2), lm2) = thatIter.next() assert(i1 == i2, s"$i1 $i2") @@ -1697,7 +1699,7 @@ private class BlockMatrixFilterRDD(bm: BlockMatrix, keepRows: Array[Long], keepC s"Finished constructing block matrix filter RDD. Total time ${(System.nanoTime() - t0).toDouble / 1000000000}" ) - protected def getPartitions: Array[Partition] = + override protected def getPartitions: Array[Partition] = Array.tabulate(newGP.numPartitions) { partitionIndex => val blockIndex = newGP.partitionToBlock(partitionIndex) BlockMatrixFilterRDDPartition( @@ -1709,7 +1711,7 @@ private class BlockMatrixFilterRDD(bm: BlockMatrix, keepRows: Array[Long], keepC override def getDependencies: Seq[Dependency[_]] = Array[Dependency[_]]( new NarrowDependency(bm.blocks) { - def getParents(partitionId: Int): Seq[Int] = { + override def getParents(partitionId: Int): Seq[Int] = { val blockForPartition = newGP.partitionToBlock(partitionId) val blockParents = blockParentMap(blockForPartition) val partitionParents = @@ -1719,7 +1721,8 @@ private class BlockMatrixFilterRDD(bm: BlockMatrix, keepRows: Array[Long], keepC } ) - def compute(split: Partition, context: TaskContext): Iterator[((Int, Int), BDM[Double])] = { + override def compute(split: Partition, context: TaskContext) + : Iterator[((Int, Int), BDM[Double])] = { val part = split.asInstanceOf[BlockMatrixFilterRDDPartition] val blockForPartition = newGP.partitionToBlock(split.index) @@ -1831,7 +1834,7 @@ private class BlockMatrixFilterColsRDD(bm: BlockMatrix, keep: Array[Long]) private val newGP = tempDenseGP.copy(partitionIndexToBlockIndex = newGPMaybeBlocks) - protected def getPartitions: Array[Partition] = + override protected def getPartitions: Array[Partition] = Array.tabulate(newGP.numPartitions) { partitionIndex: Int => val blockIndex = newGP.partitionToBlock(partitionIndex) BlockMatrixFilterOneDimRDDPartition( @@ -1842,7 +1845,7 @@ private class BlockMatrixFilterColsRDD(bm: BlockMatrix, keep: Array[Long]) override def getDependencies: Seq[Dependency[_]] = Array[Dependency[_]]( new NarrowDependency(bm.blocks) { - def getParents(partitionId: Int): Seq[Int] = { + override def getParents(partitionId: Int): Seq[Int] = { val blockForPartition = newGP.partitionToBlock(partitionId) val blockParents = blockParentMap(blockForPartition) val partitionParents = @@ -1852,7 +1855,8 @@ private class BlockMatrixFilterColsRDD(bm: BlockMatrix, keep: Array[Long]) } ) - def compute(split: Partition, context: TaskContext): Iterator[((Int, Int), BDM[Double])] = { + override def compute(split: Partition, context: TaskContext) + : Iterator[((Int, Int), BDM[Double])] = { val blockIndex = newGP.partitionToBlock(split.index) val (blockRow, newBlockCol) = newGP.blockCoordinates(blockIndex) val (blockNRows, newBlockNCols) = newGP.blockDims(blockIndex) @@ -1934,7 +1938,7 @@ private class BlockMatrixFilterRowsRDD(bm: BlockMatrix, keep: Array[Long]) private val newGP = tempDenseGP.copy(partitionIndexToBlockIndex = newGPMaybeBlocks) - protected def getPartitions: Array[Partition] = + override protected def getPartitions: Array[Partition] = Array.tabulate(newGP.numPartitions) { partitionIndex: Int => val blockIndex = newGP.partitionToBlock(partitionIndex) BlockMatrixFilterOneDimRDDPartition( @@ -1945,7 +1949,7 @@ private class BlockMatrixFilterRowsRDD(bm: BlockMatrix, keep: Array[Long]) override def getDependencies: Seq[Dependency[_]] = Array[Dependency[_]]( new NarrowDependency(bm.blocks) { - def getParents(partitionId: Int): Seq[Int] = { + override def getParents(partitionId: Int): Seq[Int] = { val blockForPartition = newGP.partitionToBlock(partitionId) val blockParents = blockParentMap(blockForPartition) val partitionParents = @@ -1955,7 +1959,8 @@ private class BlockMatrixFilterRowsRDD(bm: BlockMatrix, keep: Array[Long]) } ) - def compute(split: Partition, context: TaskContext): Iterator[((Int, Int), BDM[Double])] = { + override def compute(split: Partition, context: TaskContext) + : Iterator[((Int, Int), BDM[Double])] = { val blockIndex = newGP.partitionToBlock(split.index) val (newBlockRow, blockCol) = newGP.blockCoordinates(blockIndex) val (newBlockNRows, blockNCols) = newGP.blockDims(blockIndex) @@ -2005,7 +2010,7 @@ private class BlockMatrixTransposeRDD(bm: BlockMatrix) override def getDependencies: Seq[Dependency[_]] = Array[Dependency[_]]( new NarrowDependency(bm.blocks) { - def getParents(partitionId: Int): Seq[Int] = { + override def getParents(partitionId: Int): Seq[Int] = { val parent = transposedPartitionIndicesToParentPartitions(partitionId) val (oldI, oldJ) = bm.gp.partCoordinates(parent) val (newI, newJ) = newGP.partCoordinates(partitionId) @@ -2015,11 +2020,12 @@ private class BlockMatrixTransposeRDD(bm: BlockMatrix) } ) - def compute(split: Partition, context: TaskContext): Iterator[((Int, Int), BDM[Double])] = + override def compute(split: Partition, context: TaskContext) + : Iterator[((Int, Int), BDM[Double])] = bm.blocks.iterator(split.asInstanceOf[BlockMatrixTransposeRDDPartition].prevPartition, context) .map { case ((j, i), lm) => ((i, j), lm.t) } - protected def getPartitions: Array[Partition] = + override protected def getPartitions: Array[Partition] = Array.tabulate(newGP.numPartitions) { pi => BlockMatrixTransposeRDDPartition( pi, @@ -2052,24 +2058,25 @@ private class BlockMatrixUnionOpRDD( override def getDependencies: Seq[Dependency[_]] = Array[Dependency[_]]( new NarrowDependency(l.blocks) { - def getParents(partitionId: Int): Seq[Int] = + override def getParents(partitionId: Int): Seq[Int] = Array(lGP.blockToPartition(gp.partitionToBlock(partitionId))).filter(_ >= 0) }, new NarrowDependency(r.blocks) { - def getParents(partitionId: Int): Seq[Int] = + override def getParents(partitionId: Int): Seq[Int] = Array(rGP.blockToPartition(gp.partitionToBlock(partitionId))).filter(_ >= 0) }, ) - def compute(split: Partition, context: TaskContext): Iterator[((Int, Int), BDM[Double])] = { + override def compute(split: Partition, context: TaskContext) + : Iterator[((Int, Int), BDM[Double])] = { val (i, j) = gp.partCoordinates(split.index) val lm = op(block(l, lParts, lGP, context, i, j) -> block(r, rParts, rGP, context, i, j)) Iterator.single(((i, j), lm)) } - protected def getPartitions: Array[Partition] = Array.tabulate(gp.numPartitions)(pi => - new Partition { def index: Int = pi } + override protected def getPartitions: Array[Partition] = Array.tabulate(gp.numPartitions)(pi => + new Partition { override def index: Int = pi } ) @transient override val partitioner: Option[Partitioner] = Some(gp) @@ -2101,13 +2108,13 @@ private class BlockMatrixMultiplyRDD(l: BlockMatrix, r: BlockMatrix) override def getDependencies: Seq[Dependency[_]] = Array[Dependency[_]]( new NarrowDependency(l.blocks) { - def getParents(partitionId: Int): Seq[Int] = { + override def getParents(partitionId: Int): Seq[Int] = { val i = gp.blockBlockRow(partitionId) (0 until nProducts).map(k => lGP.coordinatesPart(i, k)).filter(_ >= 0).to(ArraySeq) } }, new NarrowDependency(r.blocks) { - def getParents(partitionId: Int): Seq[Int] = { + override def getParents(partitionId: Int): Seq[Int] = { val j = gp.blockBlockCol(partitionId) (0 until nProducts).map(k => rGP.coordinatesPart(k, j)).filter(_ >= 0).to(ArraySeq) } @@ -2143,7 +2150,8 @@ private class BlockMatrixMultiplyRDD(l: BlockMatrix, r: BlockMatrix) ) } - def compute(split: Partition, context: TaskContext): Iterator[((Int, Int), BDM[Double])] = { + override def compute(split: Partition, context: TaskContext) + : Iterator[((Int, Int), BDM[Double])] = { val (i, j) = gp.blockCoordinates(split.index) val (blockNRows, blockNCols) = gp.blockDims(split.index) val product = BDM.zeros[Double](blockNRows, blockNCols) @@ -2159,8 +2167,8 @@ private class BlockMatrixMultiplyRDD(l: BlockMatrix, r: BlockMatrix) Iterator.single(((i, j), product)) } - protected def getPartitions: Array[Partition] = Array.tabulate(gp.numPartitions)(pi => - new Partition { def index: Int = pi } + override protected def getPartitions: Array[Partition] = Array.tabulate(gp.numPartitions)(pi => + new Partition { override def index: Int = pi } ) @transient override val partitioner: Option[Partitioner] = Some(gp) @@ -2258,12 +2266,12 @@ class WriteBlocksRDD( override def getDependencies: Seq[Dependency[_]] = Array[Dependency[_]]( new NarrowDependency(crdd.rdd) { - def getParents(partitionId: Int): Seq[Int] = + override def getParents(partitionId: Int): Seq[Int] = partitions(partitionId).asInstanceOf[WriteBlocksRDDPartition].range } ) - protected def getPartitions: Array[Partition] = { + override protected def getPartitions: Array[Partition] = { val nRows = parentPartStarts.last assert(nRows == gp.nRows) val nBlockRows = gp.nBlockRows @@ -2304,7 +2312,7 @@ class WriteBlocksRDD( parts } - def compute(split: Partition, context: TaskContext): Iterator[(Int, String)] = { + override def compute(split: Partition, context: TaskContext): Iterator[(Int, String)] = { val blockRow = split.index val nRowsInBlock = gp.blockRowNRows(blockRow) val ctx = TaskContext.get() diff --git a/hail/hail/src/is/hail/linalg/LAPACK.scala b/hail/hail/src/is/hail/linalg/LAPACK.scala index 2b1a1095a90..ffa9f63fbda 100644 --- a/hail/hail/src/is/hail/linalg/LAPACK.scala +++ b/hail/hail/src/is/hail/linalg/LAPACK.scala @@ -19,7 +19,7 @@ class UnderscoreFunctionMapper extends FunctionMapper { // see: https://software.intel.com/content/www/us/en/develop/documentation/mkl-linux-developer-guide/top/language-specific-usage-options/mixed-language-programming-with-the-intel-math-kernel-library/calling-lapack-blas-and-cblas-routines-from-c-c-language-environments.html object LAPACK extends Logging { private[this] val libraryInstance = ThreadLocal.withInitial(new Supplier[LAPACKLibrary]() { - def get() = { + override def get() = { val mklEnvVar = "HAIL_MKL_PATH" val openblasEnvVar = "HAIL_OPENBLAS_PATH" val libraryName = if (sys.env.contains(mklEnvVar)) { diff --git a/hail/hail/src/is/hail/linalg/MatrixSparsity.scala b/hail/hail/src/is/hail/linalg/MatrixSparsity.scala index 9030f7a68f1..203bff1f5a6 100644 --- a/hail/hail/src/is/hail/linalg/MatrixSparsity.scala +++ b/hail/hail/src/is/hail/linalg/MatrixSparsity.scala @@ -126,7 +126,7 @@ object MatrixSparsity { } } - def newToOldPosNonSubset(newSparsity: Sparse): IndexedSeq[Integer] = { + override def newToOldPosNonSubset(newSparsity: Sparse): IndexedSeq[Integer] = { if (newSparsity.isEmpty) return ArraySeq.empty var cur = definedCoords.search(newSparsity.definedCoords.head)(Ordering.by(_.swap)).insertionPoint diff --git a/hail/hail/src/is/hail/linalg/RowMatrix.scala b/hail/hail/src/is/hail/linalg/RowMatrix.scala index a58ea3634d9..e0b02047d84 100644 --- a/hail/hail/src/is/hail/linalg/RowMatrix.scala +++ b/hail/hail/src/is/hail/linalg/RowMatrix.scala @@ -303,11 +303,12 @@ class ReadBlocksAsRowsRDD( private val nBlockCols = gp.nBlockCols private val blockSize = gp.blockSize - protected def getPartitions: Array[Partition] = Array.tabulate(partitionStarts.length - 1)(pi => - ReadBlocksAsRowsRDDPartition(pi, partitionStarts(pi), partitionStarts(pi + 1)) - ) + override protected def getPartitions: Array[Partition] = + Array.tabulate(partitionStarts.length - 1)(pi => + ReadBlocksAsRowsRDDPartition(pi, partitionStarts(pi), partitionStarts(pi + 1)) + ) - def compute(split: Partition, context: TaskContext): Iterator[(Long, Array[Double])] = { + override def compute(split: Partition, context: TaskContext): Iterator[(Long, Array[Double])] = { val ReadBlocksAsRowsRDDPartition(_, start, end) = split.asInstanceOf[ReadBlocksAsRowsRDDPartition] @@ -315,9 +316,9 @@ class ReadBlocksAsRowsRDD( var i = start new Iterator[(Long, Array[Double])] { - def hasNext: Boolean = i < end + override def hasNext: Boolean = i < end - def next(): (Long, Array[Double]) = { + override def next(): (Long, Array[Double]) = { if (i == start || i % blockSize == 0) { val blockRow = (i / blockSize).toInt val nRowsInBlock = gp.blockRowNRows(blockRow) diff --git a/hail/hail/src/is/hail/lir/Blocks.scala b/hail/hail/src/is/hail/lir/Blocks.scala index f7bee6bb757..0b3c2602fc4 100644 --- a/hail/hail/src/is/hail/lir/Blocks.scala +++ b/hail/hail/src/is/hail/lir/Blocks.scala @@ -11,9 +11,9 @@ class Blocks( def nBlocks: Int = blocks.length - def length: Int = blocks.length + override def length: Int = blocks.length - def apply(i: Int): Block = blocks(i) + override def apply(i: Int): Block = blocks(i) def index(block: Block): Int = blockIdx.get(block) } diff --git a/hail/hail/src/is/hail/lir/Locals.scala b/hail/hail/src/is/hail/lir/Locals.scala index 5e2ed36ee44..45e81c6f0f9 100644 --- a/hail/hail/src/is/hail/lir/Locals.scala +++ b/hail/hail/src/is/hail/lir/Locals.scala @@ -7,9 +7,9 @@ class Locals( def nLocals: Int = locals.length - def length: Int = locals.length + override def length: Int = locals.length - def apply(i: Int): Local = locals(i) + override def apply(i: Int): Local = locals(i) def getIndex(l: Local): Option[Int] = { l match { diff --git a/hail/hail/src/is/hail/lir/X.scala b/hail/hail/src/is/hail/lir/X.scala index b7b5965af9a..a610d5dfd8e 100644 --- a/hail/hail/src/is/hail/lir/X.scala +++ b/hail/hail/src/is/hail/lir/X.scala @@ -137,12 +137,12 @@ abstract class FieldRef { class Field private[lir] (classx: Classx[_], val name: String, val ti: TypeInfo[_]) extends FieldRef { - def owner: String = classx.name + override def owner: String = classx.name } class StaticField private[lir] (classx: Classx[_], val name: String, val ti: TypeInfo[_]) extends FieldRef { - def owner: String = classx.name + override def owner: String = classx.name } class FieldLit( @@ -176,11 +176,11 @@ class Method private[lir] ( def nParameters: Int = parameterTypeInfo.length + (!isStatic).toInt - def owner: String = classx.name + override def owner: String = classx.name - def desc = s"(${parameterTypeInfo.map(_.desc).mkString})${returnTypeInfo.desc}" + override def desc = s"(${parameterTypeInfo.map(_.desc).mkString})${returnTypeInfo.desc}" - def isInterface: Boolean = false + override def isInterface: Boolean = false private var _entry: Block = _ @@ -520,7 +520,7 @@ abstract class StmtX extends X { var prev: StmtX = _ var next: StmtX = _ - def remove(): Unit = { + override def remove(): Unit = { assert(parent != null) if (parent.first == this) parent.first = next @@ -602,7 +602,7 @@ abstract class ValueX extends X { def ti: TypeInfo[_] - def remove(): Unit = { + override def remove(): Unit = { var i = 0 while (parent.children(i) ne this) i += 1 @@ -626,14 +626,14 @@ class GotoX(var lineNumber: Int = 0) extends ControlX { def setL(newL: Block): Unit = setTarget(0, newL) - def targetArity(): Int = 1 + override def targetArity(): Int = 1 - def target(i: Int): Block = { + override def target(i: Int): Block = { assert(i == 0) _L } - def setTarget(i: Int, b: Block): Unit = { + override def setTarget(i: Int, b: Block): Unit = { assert(i == 0) if (_L != null) _L.removeUse(this, 0) @@ -655,9 +655,9 @@ class IfX(val op: Int, var lineNumber: Int = 0) extends ControlX { def setLfalse(newLfalse: Block): Unit = setTarget(1, newLfalse) - def targetArity(): Int = 2 + override def targetArity(): Int = 2 - def target(i: Int): Block = { + override def target(i: Int): Block = { if (i == 0) _Ltrue else { @@ -666,7 +666,7 @@ class IfX(val op: Int, var lineNumber: Int = 0) extends ControlX { } } - def setTarget(i: Int, b: Block): Unit = { + override def setTarget(i: Int, b: Block): Unit = { if (i == 0) { if (_Ltrue != null) _Ltrue.removeUse(this, 0) @@ -706,15 +706,15 @@ class SwitchX(var lineNumber: Int = 0) extends ControlX { if (block != null) block.addUse(this, i + 1) } - def targetArity(): Int = 1 + _Lcases.length + override def targetArity(): Int = 1 + _Lcases.length - def target(i: Int): Block = + override def target(i: Int): Block = if (i == 0) _Ldefault else _Lcases(i - 1) - def setTarget(i: Int, b: Block): Unit = { + override def setTarget(i: Int, b: Block): Unit = { if (i == 0) { if (_Ldefault != null) _Ldefault.removeUse(this, 0) @@ -739,19 +739,19 @@ class PutFieldX(val op: Int, val f: FieldRef, var lineNumber: Int = 0) extends S class IincX(var l: Local, val i: Int, var lineNumber: Int = 0) extends StmtX class ReturnX(var lineNumber: Int = 0) extends ControlX { - def targetArity(): Int = 0 + override def targetArity(): Int = 0 - def target(i: Int): Block = throw new IndexOutOfBoundsException() + override def target(i: Int): Block = throw new IndexOutOfBoundsException() - def setTarget(i: Int, b: Block): Unit = throw new IndexOutOfBoundsException() + override def setTarget(i: Int, b: Block): Unit = throw new IndexOutOfBoundsException() } class ThrowX(var lineNumber: Int = 0) extends ControlX { - def targetArity(): Int = 0 + override def targetArity(): Int = 0 - def target(i: Int): Block = throw new IndexOutOfBoundsException() + override def target(i: Int): Block = throw new IndexOutOfBoundsException() - def setTarget(i: Int, b: Block): Unit = throw new IndexOutOfBoundsException() + override def setTarget(i: Int, b: Block): Unit = throw new IndexOutOfBoundsException() } class StmtOpX(val op: Int, var lineNumber: Int = 0) extends StmtX @@ -761,7 +761,7 @@ class MethodStmtX(val op: Int, val method: MethodRef, var lineNumber: Int = 0) e class TypeInsnX(val op: Int, val ti: TypeInfo[_], var lineNumber: Int = 0) extends ValueX {} class InsnX(val op: Int, _ti: TypeInfo[_], var lineNumber: Int = 0) extends ValueX { - def ti: TypeInfo[_] = { + override def ti: TypeInfo[_] = { if (_ti != null) return _ti @@ -838,15 +838,15 @@ class InsnX(val op: Int, _ti: TypeInfo[_], var lineNumber: Int = 0) extends Valu } class LoadX(var l: Local, var lineNumber: Int = 0) extends ValueX { - def ti: TypeInfo[_] = l.ti + override def ti: TypeInfo[_] = l.ti } class GetFieldX(val op: Int, val f: FieldRef, var lineNumber: Int = 0) extends ValueX { - def ti: TypeInfo[_] = f.ti + override def ti: TypeInfo[_] = f.ti } class NewArrayX(val eti: TypeInfo[_], var lineNumber: Int = 0) extends ValueX { - def ti: TypeInfo[_] = arrayInfo(eti) + override def ti: TypeInfo[_] = arrayInfo(eti) } class NewInstanceX(val ti: TypeInfo[_], val ctor: MethodRef, var lineNumber: Int = 0) extends ValueX @@ -861,5 +861,5 @@ class LdcX(val a: Any, val ti: TypeInfo[_], var lineNumber: Int = 0) extends Val } class MethodX(val op: Int, val method: MethodRef, var lineNumber: Int = 0) extends ValueX { - def ti: TypeInfo[_] = method.returnTypeInfo + override def ti: TypeInfo[_] = method.returnTypeInfo } diff --git a/hail/hail/src/is/hail/methods/ForceCount.scala b/hail/hail/src/is/hail/methods/ForceCount.scala index 522be2bbec6..b6c433d4dcf 100644 --- a/hail/hail/src/is/hail/methods/ForceCount.scala +++ b/hail/hail/src/is/hail/methods/ForceCount.scala @@ -9,7 +9,7 @@ import is.hail.types.virtual.{MatrixType, TInt64, TableType, Type} case class ForceCountTable() extends TableToValueFunction { override def typ(childType: TableType): Type = TInt64 - def unionRequiredness(childType: RTable, resultType: TypeWithRequiredness): Unit = () + override def unionRequiredness(childType: RTable, resultType: TypeWithRequiredness): Unit = () override def execute(ctx: ExecuteContext, tv: TableValue): Any = tv.rvd.count() } @@ -17,7 +17,7 @@ case class ForceCountTable() extends TableToValueFunction { case class ForceCountMatrixTable() extends MatrixToValueFunction { override def typ(childType: MatrixType): Type = TInt64 - def unionRequiredness(childType: RTable, resultType: TypeWithRequiredness): Unit = () + override def unionRequiredness(childType: RTable, resultType: TypeWithRequiredness): Unit = () override def execute(ctx: ExecuteContext, mv: MatrixValue): Any = throw new UnsupportedOperationException diff --git a/hail/hail/src/is/hail/methods/IBD.scala b/hail/hail/src/is/hail/methods/IBD.scala index 792a0232840..8060accdf8d 100644 --- a/hail/hail/src/is/hail/methods/IBD.scala +++ b/hail/hail/src/is/hail/methods/IBD.scala @@ -388,12 +388,12 @@ case class IBD( } } - def preservesPartitionCounts: Boolean = false + override def preservesPartitionCounts: Boolean = false - def typ(childType: MatrixType): TableType = + override def typ(childType: MatrixType): TableType = TableType(IBD.ibdPType.virtualType, IBD.ibdKey, TStruct.empty) - def execute(ctx: ExecuteContext, input: MatrixValue): TableValue = { + override def execute(ctx: ExecuteContext, input: MatrixValue): TableValue = { input.requireUniqueSamples("ibd") val computeMaf = mafFieldName.map(IBD.generateComputeMaf(input, _)) val crdd = diff --git a/hail/hail/src/is/hail/methods/LinearRegression.scala b/hail/hail/src/is/hail/methods/LinearRegression.scala index eed287388d8..88136ead1ac 100644 --- a/hail/hail/src/is/hail/methods/LinearRegression.scala +++ b/hail/hail/src/is/hail/methods/LinearRegression.scala @@ -39,9 +39,9 @@ case class LinearRegressionRowsSingle( ) } - def preservesPartitionCounts: Boolean = true + override def preservesPartitionCounts: Boolean = true - def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { + override def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { val (y, cov, completeColIdx) = RegressionUtils.getPhenosCovCompleteSamples(mv, yFields.toArray, covFields.toArray) @@ -219,9 +219,9 @@ case class LinearRegressionRowsChained( ) } - def preservesPartitionCounts: Boolean = true + override def preservesPartitionCounts: Boolean = true - def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { + override def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { val localData = yFields.map(y => RegressionUtils.getPhenosCovCompleteSamples(mv, y.toArray, covFields.toArray) diff --git a/hail/hail/src/is/hail/methods/LocalLDPrune.scala b/hail/hail/src/is/hail/methods/LocalLDPrune.scala index 12f87044d13..73fb3950258 100644 --- a/hail/hail/src/is/hail/methods/LocalLDPrune.scala +++ b/hail/hail/src/is/hail/methods/LocalLDPrune.scala @@ -305,7 +305,7 @@ case class LocalLDPrune( globalType = TStruct.empty, ) - def preservesPartitionCounts: Boolean = false + override def preservesPartitionCounts: Boolean = false def makeStream(stream: IR, entriesFieldName: String, nCols: IR): StreamLocalLDPrune = { val newRow = mapIR(stream) { row => @@ -318,7 +318,7 @@ case class LocalLDPrune( StreamLocalLDPrune(newRow, r2Threshold, windowSize, maxQueueSize, nCols) } - def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { + override def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { val nSamples = mv.nCols val tableType = typ(mv.typ) val ts = TableExecuteIntermediate(mv.toTableValue).asTableStage(ctx).mapPartition(Some( diff --git a/hail/hail/src/is/hail/methods/LogisticRegression.scala b/hail/hail/src/is/hail/methods/LogisticRegression.scala index e792e1d8d89..429b195e07b 100644 --- a/hail/hail/src/is/hail/methods/LogisticRegression.scala +++ b/hail/hail/src/is/hail/methods/LogisticRegression.scala @@ -31,9 +31,9 @@ case class LogisticRegression( ) } - def preservesPartitionCounts: Boolean = true + override def preservesPartitionCounts: Boolean = true - def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { + override def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { val logRegTest = LogisticRegressionTest.tests(test) val tableType = typ(mv.typ) val newRVDType = tableType.canonicalRVDType diff --git a/hail/hail/src/is/hail/methods/MatrixExportEntriesByCol.scala b/hail/hail/src/is/hail/methods/MatrixExportEntriesByCol.scala index 58b00d092ab..f65e425e036 100644 --- a/hail/hail/src/is/hail/methods/MatrixExportEntriesByCol.scala +++ b/hail/hail/src/is/hail/methods/MatrixExportEntriesByCol.scala @@ -22,11 +22,11 @@ case class MatrixExportEntriesByCol( headerJsonInFile: Boolean, useStringKeyAsFileName: Boolean, ) extends MatrixToValueFunction with Logging { - def typ(childType: MatrixType): Type = TVoid + override def typ(childType: MatrixType): Type = TVoid - def unionRequiredness(childType: RTable, resultType: TypeWithRequiredness): Unit = () + override def unionRequiredness(childType: RTable, resultType: TypeWithRequiredness): Unit = () - def execute(ctx: ExecuteContext, mv: MatrixValue): Any = { + override def execute(ctx: ExecuteContext, mv: MatrixValue): Any = { val fs = ctx.fs fs.delete(path, recursive = true) // overwrite by default diff --git a/hail/hail/src/is/hail/methods/NPartitions.scala b/hail/hail/src/is/hail/methods/NPartitions.scala index 0f11da13751..e513411f4a8 100644 --- a/hail/hail/src/is/hail/methods/NPartitions.scala +++ b/hail/hail/src/is/hail/methods/NPartitions.scala @@ -9,7 +9,7 @@ import is.hail.types.virtual.{MatrixType, TInt32, TableType, Type} case class NPartitionsTable() extends TableToValueFunction { override def typ(childType: TableType): Type = TInt32 - def unionRequiredness(childType: RTable, resultType: TypeWithRequiredness): Unit = () + override def unionRequiredness(childType: RTable, resultType: TypeWithRequiredness): Unit = () override def execute(ctx: ExecuteContext, tv: TableValue): Any = tv.rvd.getNumPartitions } @@ -17,7 +17,7 @@ case class NPartitionsTable() extends TableToValueFunction { case class NPartitionsMatrixTable() extends MatrixToValueFunction { override def typ(childType: MatrixType): Type = TInt32 - def unionRequiredness(childType: RTable, resultType: TypeWithRequiredness): Unit = () + override def unionRequiredness(childType: RTable, resultType: TypeWithRequiredness): Unit = () override def execute(ctx: ExecuteContext, mv: MatrixValue): Any = mv.rvd.getNumPartitions diff --git a/hail/hail/src/is/hail/methods/Nirvana.scala b/hail/hail/src/is/hail/methods/Nirvana.scala index db5ed39cd4b..2ec18d641cf 100644 --- a/hail/hail/src/is/hail/methods/Nirvana.scala +++ b/hail/hail/src/is/hail/methods/Nirvana.scala @@ -504,8 +504,8 @@ case class Nirvana(config: String, blockSize: Int = 500000) extends TableToTable ) } - def preservesPartitionCounts: Boolean = false + override def preservesPartitionCounts: Boolean = false - def execute(ctx: ExecuteContext, tv: TableValue): TableValue = + override def execute(ctx: ExecuteContext, tv: TableValue): TableValue = Nirvana.annotate(ctx, tv, config, blockSize) } diff --git a/hail/hail/src/is/hail/methods/PCA.scala b/hail/hail/src/is/hail/methods/PCA.scala index 13a5a74486f..a853830a1ca 100644 --- a/hail/hail/src/is/hail/methods/PCA.scala +++ b/hail/hail/src/is/hail/methods/PCA.scala @@ -27,9 +27,9 @@ case class PCA(entryField: String, k: Int, computeLoadings: Boolean) ), ) - def preservesPartitionCounts: Boolean = false + override def preservesPartitionCounts: Boolean = false - def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { + override def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { if (k < 1) fatal(s"""requested invalid number of components: $k | Expect componenents >= 1""".stripMargin) diff --git a/hail/hail/src/is/hail/methods/PCRelate.scala b/hail/hail/src/is/hail/methods/PCRelate.scala index 32d1f1a88a1..c9136a46c3a 100644 --- a/hail/hail/src/is/hail/methods/PCRelate.scala +++ b/hail/hail/src/is/hail/methods/PCRelate.scala @@ -152,10 +152,10 @@ case class PCRelate( def storageLevel: StorageLevel = PCRelate.defaultStorageLevel - def typ(bmType: BlockMatrixType, auxType: Type): TableType = + override def typ(bmType: BlockMatrixType, auxType: Type): TableType = TableType(sig, keys, TStruct.empty) - def execute(ctx: ExecuteContext, g: M, value: Any): TableValue = { + override def execute(ctx: ExecuteContext, g: M, value: Any): TableValue = { val pcs = rowsToBDM(value.asInstanceOf[IndexedSeq[IndexedSeq[java.lang.Double]]]) assert(pcs.rows == g.nCols) val r = computeResult(ctx, g, pcs) diff --git a/hail/hail/src/is/hail/methods/PoissonRegression.scala b/hail/hail/src/is/hail/methods/PoissonRegression.scala index 02ba7579277..2fdb742c4fc 100644 --- a/hail/hail/src/is/hail/methods/PoissonRegression.scala +++ b/hail/hail/src/is/hail/methods/PoissonRegression.scala @@ -30,9 +30,9 @@ case class PoissonRegression( ) } - def preservesPartitionCounts: Boolean = true + override def preservesPartitionCounts: Boolean = true - def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { + override def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { val poisRegTest = PoissonRegressionTest.tests(test) val tableType = typ(mv.typ) val newRVDType = tableType.canonicalRVDType diff --git a/hail/hail/src/is/hail/methods/Skat.scala b/hail/hail/src/is/hail/methods/Skat.scala index 68381421020..622bba2adef 100644 --- a/hail/hail/src/is/hail/methods/Skat.scala +++ b/hail/hail/src/is/hail/methods/Skat.scala @@ -168,9 +168,9 @@ case class Skat( TableType(skatSchema, FastSeq("id"), TStruct.empty) } - def preservesPartitionCounts: Boolean = false + override def preservesPartitionCounts: Boolean = false - def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { + override def execute(ctx: ExecuteContext, mv: MatrixValue): TableValue = { if (maxSize <= 0 || maxSize > 46340) fatal(s"Maximum group size must be in [1, 46340], got $maxSize") diff --git a/hail/hail/src/is/hail/rvd/AbstractRVDSpec.scala b/hail/hail/src/is/hail/rvd/AbstractRVDSpec.scala index 17b0cedad3d..f714e86303a 100644 --- a/hail/hail/src/is/hail/rvd/AbstractRVDSpec.scala +++ b/hail/hail/src/is/hail/rvd/AbstractRVDSpec.scala @@ -308,15 +308,15 @@ case class IndexSpec2( _annotationType: Type, _offsetField: Option[String] = None, ) extends AbstractIndexSpec { - def relPath: String = _relPath + override def relPath: String = _relPath - def leafCodec: AbstractTypedCodecSpec = _leafCodec + override def leafCodec: AbstractTypedCodecSpec = _leafCodec - def internalNodeCodec: AbstractTypedCodecSpec = _internalNodeCodec + override def internalNodeCodec: AbstractTypedCodecSpec = _internalNodeCodec - def keyType: Type = _keyType + override def keyType: Type = _keyType - def annotationType: Type = _annotationType + override def annotationType: Type = _annotationType override def offsetField: Option[String] = _offsetField } @@ -462,11 +462,11 @@ case class IndexedRVDSpec2( require(codecSpec2.encodedType.required) - def typedCodecSpec: AbstractTypedCodecSpec = codecSpec2 + override def typedCodecSpec: AbstractTypedCodecSpec = codecSpec2 - def indexSpec: AbstractIndexSpec = _indexSpec + override def indexSpec: AbstractIndexSpec = _indexSpec - def partitioner(sm: HailStateManager): RVDPartitioner = { + override def partitioner(sm: HailStateManager): RVDPartitioner = { val keyType = codecSpec2.encodedVirtualType.asInstanceOf[TStruct].select(key)._1 val rangeBoundsType = TArray(TInterval(keyType)) new RVDPartitioner( @@ -480,9 +480,9 @@ case class IndexedRVDSpec2( ) } - def partFiles: IndexedSeq[String] = _partFiles + override def partFiles: IndexedSeq[String] = _partFiles - def key: IndexedSeq[String] = _key + override def key: IndexedSeq[String] = _key val attrs: Map[String, String] = _attrs @@ -597,7 +597,7 @@ case class OrderedRVDSpec2( require(codecSpec2.encodedType.required) - def partitioner(sm: HailStateManager): RVDPartitioner = { + override def partitioner(sm: HailStateManager): RVDPartitioner = { val keyType = codecSpec2.encodedVirtualType.asInstanceOf[TStruct].select(key)._1 val rangeBoundsType = TArray(TInterval(keyType)) new RVDPartitioner( @@ -611,11 +611,11 @@ case class OrderedRVDSpec2( ) } - def partFiles: IndexedSeq[String] = _partFiles + override def partFiles: IndexedSeq[String] = _partFiles - def key: IndexedSeq[String] = _key + override def key: IndexedSeq[String] = _key - def attrs: Map[String, String] = _attrs + override def attrs: Map[String, String] = _attrs - def typedCodecSpec: AbstractTypedCodecSpec = codecSpec2 + override def typedCodecSpec: AbstractTypedCodecSpec = codecSpec2 } diff --git a/hail/hail/src/is/hail/rvd/RVD.scala b/hail/hail/src/is/hail/rvd/RVD.scala index f7acbb04ddd..5ef20add708 100644 --- a/hail/hail/src/is/hail/rvd/RVD.scala +++ b/hail/hail/src/is/hail/rvd/RVD.scala @@ -186,9 +186,9 @@ class RVD( new Iterator[Long] { var first = true - def hasNext: Boolean = it.hasNext + override def hasNext: Boolean = it.hasNext - def next(): Long = { + override def next(): Long = { val ptr = it.next() if (first) @@ -1016,9 +1016,9 @@ class RVD( val sorted: RDD[((Int, Interval), Array[Byte])] = new ShuffledRDD( partitionKeyedIntervals, new Partitioner { - def getPartition(key: Any): Int = key.asInstanceOf[(Int, Interval)]._1 + override def getPartition(key: Any): Int = key.asInstanceOf[(Int, Interval)]._1 - def numPartitions: Int = nParts + override def numPartitions: Int = nParts }, ).setKeyOrdering(Ordering.by[(Int, Interval), Interval](_._2)(intervalOrd)) @@ -1188,7 +1188,7 @@ object RVD extends Logging { type CRDD = ContextRDD[Long] val unkeyedCoercer: RVDCoercer = new RVDCoercer(fullType) { - def _coerce(typ: RVDType, crdd: CRDD): RVD = { + override def _coerce(typ: RVDType, crdd: CRDD): RVD = { assert(typ.key.isEmpty) unkeyed(typ.rowType, crdd) } @@ -1198,7 +1198,7 @@ object RVD extends Logging { return unkeyedCoercer val emptyCoercer: RVDCoercer = new RVDCoercer(fullType) { - def _coerce(typ: RVDType, crdd: CRDD): RVD = empty(execCtx, typ) + override def _coerce(typ: RVDType, crdd: CRDD): RVD = empty(execCtx, typ) } val keyInfo = getKeyInfo(execCtx, fullType, partitionKey, keys) @@ -1243,7 +1243,7 @@ object RVD extends Logging { bounds, ) - def _coerce(typ: RVDType, crdd: CRDD): RVD = + override def _coerce(typ: RVDType, crdd: CRDD): RVD = RVD(typ, unfixedPartitioner, orderPartitions(crdd)) .repartition(execCtx, newPartitioner, shuffle = false) } @@ -1273,7 +1273,7 @@ object RVD extends Logging { pkBounds, ) - def _coerce(typ: RVDType, crdd: CRDD): RVD = { + override def _coerce(typ: RVDType, crdd: CRDD): RVD = { RVD( typ.copy(key = typ.key.take(partitionKey)), unfixedPartitioner, @@ -1292,7 +1292,7 @@ object RVD extends Logging { val newPartitioner = calculateKeyRanges(execCtx, this.fullType, keyInfo, keys.getNumPartitions, partitionKey) - def _coerce(typ: RVDType, crdd: CRDD): RVD = + override def _coerce(typ: RVDType, crdd: CRDD): RVD = RVD.unkeyed(typ.rowType, crdd) .repartition(execCtx, newPartitioner, shuffle = true, filter = false) } diff --git a/hail/hail/src/is/hail/rvd/RVDContext.scala b/hail/hail/src/is/hail/rvd/RVDContext.scala index a8b2faf10ec..d4ec9c9f5f8 100644 --- a/hail/hail/src/is/hail/rvd/RVDContext.scala +++ b/hail/hail/src/is/hail/rvd/RVDContext.scala @@ -43,7 +43,7 @@ class RVDContext(val partitionRegion: Region, val r: Region) extends AutoCloseab def rvb = theRvb // frees the memory associated with this context - def close(): Unit = { + override def close(): Unit = { var e: Exception = null children.foreach { child => try diff --git a/hail/hail/src/is/hail/rvd/RVDPartitioner.scala b/hail/hail/src/is/hail/rvd/RVDPartitioner.scala index c4adeaa36c0..6bd314dcf58 100644 --- a/hail/hail/src/is/hail/rvd/RVDPartitioner.scala +++ b/hail/hail/src/is/hail/rvd/RVDPartitioner.scala @@ -114,9 +114,9 @@ class RVDPartitioner( val selfBc = broadcast(sc) new Partitioner { - def numPartitions: Int = selfBc.value.numPartitions + override def numPartitions: Int = selfBc.value.numPartitions - def getPartition(key: Any): Int = selfBc.value.lowerBound(key) + override def getPartition(key: Any): Int = selfBc.value.lowerBound(key) } } diff --git a/hail/hail/src/is/hail/rvd/RVDType.scala b/hail/hail/src/is/hail/rvd/RVDType.scala index d940af030bf..42ef2dc15d3 100644 --- a/hail/hail/src/is/hail/rvd/RVDType.scala +++ b/hail/hail/src/is/hail/rvd/RVDType.scala @@ -96,7 +96,7 @@ final case class RVDType(rowType: PStruct, key: IndexedSeq[String]) extends Seri // Left is a point, right is an interval. // Returns -1 if point is below interval, 0 if it is inside, and 1 if it // is above, always considering missing greatest. - def compare(o1: Long, o2: Long): Int = { + override def compare(o1: Long, o2: Long): Int = { val leftDefined = t1.isFieldDefined(o1, f1) val rightDefined = t2.isFieldDefined(o2, f2) @@ -128,10 +128,10 @@ final case class RVDType(rowType: PStruct, key: IndexedSeq[String]) extends Seri val wrv = WritableRegionValue(sm, kType, region) val kRowOrdering = kRowOrd(sm) - def setFiniteValue(representative: RegionValue): Unit = + override def setFiniteValue(representative: RegionValue): Unit = wrv.setSelect(rowType, kFieldIdx, representative) - def compareFinite(rv: RegionValue): Int = + override def compareFinite(rv: RegionValue): Int = kRowOrdering.compare(wrv.value, rv) } @@ -188,7 +188,7 @@ object RVDType { val nFields = fields1.length new UnsafeOrdering { - def compare(o1: Long, o2: Long): Int = { + override def compare(o1: Long, o2: Long): Int = { var i = 0 var hasMissing = false while (i < nFields) { diff --git a/hail/hail/src/is/hail/services/JSONLogLayout.scala b/hail/hail/src/is/hail/services/JSONLogLayout.scala index c9984eec19d..3c40c53fe27 100644 --- a/hail/hail/src/is/hail/services/JSONLogLayout.scala +++ b/hail/hail/src/is/hail/services/JSONLogLayout.scala @@ -25,18 +25,19 @@ class DateFormatter { object JSONLogLayout { private val datefmt = ThreadLocal.withInitial(new Supplier[DateFormatter]() { - def get() = new DateFormatter() + override def get() = new DateFormatter() }) } class JSONLogLayout extends Layout { import JSONLogLayout._ - def ignoresThrowable(): Boolean = false + override def ignoresThrowable(): Boolean = false + @SuppressWarnings(Array("org.wartremover.contrib.warts.MissingOverride")) def activateOptions(): Unit = () - def format(event: LoggingEvent): String = { + override def format(event: LoggingEvent): String = { val threadName = event.getThreadName(); val timestamp = event.getTimeStamp(); val mdc = event.getProperties(); @@ -55,7 +56,7 @@ class JSONLogLayout extends Layout { val mdcFields = ArraySeq.newBuilder[JField] mdc.forEach(new BiConsumer[Any, Any]() { - def accept(key: Any, value: Any): Unit = + override def accept(key: Any, value: Any): Unit = mdcFields += JField(key.toString, JString(value.toString)) }) fields += JField("mdc", JObject(mdcFields.result(): _*)) diff --git a/hail/hail/src/is/hail/sparkextras/BlockedRDD.scala b/hail/hail/src/is/hail/sparkextras/BlockedRDD.scala index aef65f7c131..3d540ad9acb 100644 --- a/hail/hail/src/is/hail/sparkextras/BlockedRDD.scala +++ b/hail/hail/src/is/hail/sparkextras/BlockedRDD.scala @@ -39,7 +39,7 @@ class BlockedRDD[T]( override def getDependencies: Seq[Dependency[_]] = FastSeq(new NarrowDependency(prev) { - def getParents(id: Int): Seq[Int] = + override def getParents(id: Int): Seq[Int] = partitions(id).asInstanceOf[BlockedRDDPartition].range }) diff --git a/hail/hail/src/is/hail/sparkextras/ContextRDD.scala b/hail/hail/src/is/hail/sparkextras/ContextRDD.scala index 0084d094b45..db71452f7d7 100644 --- a/hail/hail/src/is/hail/sparkextras/ContextRDD.scala +++ b/hail/hail/src/is/hail/sparkextras/ContextRDD.scala @@ -31,9 +31,9 @@ abstract class Combiner[U] { class CommutativeAndAssociativeCombiner[U](zero: => U, combine: (U, U) => U) extends Combiner[U] { var state: U = zero - def combine(i: Int, value0: U): Unit = state = combine(state, value0) + override def combine(i: Int, value0: U): Unit = state = combine(state, value0) - def result(): U = state + override def result(): U = state } class AssociativeCombiner[U](zero: => U, combine: (U, U) => U) extends Combiner[U] with Logging { @@ -44,7 +44,7 @@ class AssociativeCombiner[U](zero: => U, combine: (U, U) => U) extends Combiner[ // U it holds. private val t = new java.util.TreeMap[Int, TreeValue]() - def combine(i: Int, value0: U): Unit = { + override def combine(i: Int, value0: U): Unit = { logger.info(s"at result $i, AssociativeCombiner contains ${t.size()} queued results") var value = value0 var end = i @@ -69,7 +69,7 @@ class AssociativeCombiner[U](zero: => U, combine: (U, U) => U) extends Combiner[ t.put(i, TreeValue(value, end)) } - def result(): U = { + override def result(): U = { // after 'result' returns, 't' owns no values. val n = t.size() if (n > 0) { @@ -295,13 +295,13 @@ class ContextRDD[T: ClassTag]( f: (RVDContext, T, U) => V ): ContextRDD[V] = czipPartitions(that, preservesPartitioning) { (ctx, l, r) => new Iterator[V] { - def hasNext = { + override def hasNext = { val lhn = l.hasNext val rhn = r.hasNext assert(lhn == rhn) lhn } - def next(): V = + override def next(): V = f(ctx, l.next(), r.next()) } } @@ -420,7 +420,7 @@ class ContextRDD[T: ClassTag]( } private class CRDDCoalescer(partEnds: Array[Int]) extends PartitionCoalescer with Serializable { - def coalesce(maxPartitions: Int, prev: RDD[_]): Array[PartitionGroup] = { + override def coalesce(maxPartitions: Int, prev: RDD[_]): Array[PartitionGroup] = { assert(maxPartitions == partEnds.length) val groups = Array.fill(maxPartitions)(new PartitionGroup()) val parts = prev.partitions diff --git a/hail/hail/src/is/hail/sparkextras/IndexReadRDD.scala b/hail/hail/src/is/hail/sparkextras/IndexReadRDD.scala index 94e00942a2f..bedb3f97ba3 100644 --- a/hail/hail/src/is/hail/sparkextras/IndexReadRDD.scala +++ b/hail/hail/src/is/hail/sparkextras/IndexReadRDD.scala @@ -16,7 +16,7 @@ class IndexReadRDD[T: ClassTag]( @transient val intervalBounds: Option[Array[Interval]], f: (IndexedFilePartition, TaskContext) => T, ) extends RDD[T](SparkBackend.sparkContext, Nil) { - def getPartitions: Array[Partition] = + override def getPartitions: Array[Partition] = Array.tabulate(partFiles.length) { i => IndexedFilePartition(i, partFiles(i), intervalBounds.map(_(i))) } diff --git a/hail/hail/src/is/hail/sparkextras/MapPartitionsWithValueRDD.scala b/hail/hail/src/is/hail/sparkextras/MapPartitionsWithValueRDD.scala index 8b04d55d2e4..8c80c9ad76c 100644 --- a/hail/hail/src/is/hail/sparkextras/MapPartitionsWithValueRDD.scala +++ b/hail/hail/src/is/hail/sparkextras/MapPartitionsWithValueRDD.scala @@ -10,7 +10,7 @@ case class MapPartitionsWithValueRDDPartition[V]( parentPartition: Partition, value: V, ) extends Partition { - def index: Int = parentPartition.index + override def index: Int = parentPartition.index } class MapPartitionsWithValueRDD[T: ClassTag, U: ClassTag, V]( diff --git a/hail/hail/src/is/hail/sparkextras/RepartitionedOrderedRDD2.scala b/hail/hail/src/is/hail/sparkextras/RepartitionedOrderedRDD2.scala index cfd1054d297..36b61a3bbeb 100644 --- a/hail/hail/src/is/hail/sparkextras/RepartitionedOrderedRDD2.scala +++ b/hail/hail/src/is/hail/sparkextras/RepartitionedOrderedRDD2.scala @@ -46,7 +46,7 @@ class RepartitionedOrderedRDD2 private ( val typ: RVDType = prev.typ val kOrd: ExtendedOrdering = PartitionBoundOrdering(sm, typ.kType.virtualType) - def getPartitions: Array[Partition] = { + override def getPartitions: Array[Partition] = { require(newRangeBounds.forall { i => typ.kType.virtualType.relaxedTypeCheck(i.start) && typ.kType.virtualType.relaxedTypeCheck( i.end @@ -109,7 +109,7 @@ class RepartitionedOrderedRDD2 private ( innerRegion.clear() } - def hasNext: Boolean = { + override def hasNext: Boolean = { if (pulled) return true @@ -126,7 +126,7 @@ class RepartitionedOrderedRDD2 private ( true } - def next(): Long = { + override def next(): Long = { // hasNext() must be called before next() to fill `current` if (!hasNext) throw new NoSuchElementException diff --git a/hail/hail/src/is/hail/stats/LeveneHaldane.scala b/hail/hail/src/is/hail/stats/LeveneHaldane.scala index 515b8085c08..e3dbb57a663 100644 --- a/hail/hail/src/is/hail/stats/LeveneHaldane.scala +++ b/hail/hail/src/is/hail/stats/LeveneHaldane.scala @@ -21,7 +21,7 @@ class LeveneHaldane( ) extends AbstractIntegerDistribution(rng) { // The probability mass function P(nAB), computing no more than necessary - def probability(nAB: Int): Double = + override def probability(nAB: Int): Double = if (nAB < 0 || nAB > nA || nAB % 2 != nA % 2) 0.0 else if (nAB >= mode) @@ -47,7 +47,7 @@ class LeveneHaldane( } // P(nAB <= n1) - def cumulativeProbability(n1: Int): Double = + override def cumulativeProbability(n1: Int): Double = cumulativeProbability(-1, n1) // P(nAB > n0) @@ -81,14 +81,15 @@ class LeveneHaldane( } def nB: Int = 2 * n - nA - def getNumericalMean: Double = 1.0 * nA * nB / (2 * n - 1) + override def getNumericalMean: Double = 1.0 * nA * nB / (2 * n - 1) - def getNumericalVariance: Double = + override def getNumericalVariance: Double = 1.0 * nA * nB / (2 * n - 1) * (1 + (nA - 1.0) * (nB - 1) / (2 * n - 3) - 1.0 * nA * nB / (2 * n - 1)) - def isSupportConnected: Boolean = true // interpreted as restricted to the even or odd integers, - def getSupportUpperBound: Int = nA - def getSupportLowerBound: Int = nA % 2 + override def isSupportConnected: Boolean = + true // interpreted as restricted to the even or odd integers, + override def getSupportUpperBound: Int = nA + override def getSupportLowerBound: Int = nA % 2 } object LeveneHaldane { diff --git a/hail/hail/src/is/hail/stats/LogisticRegressionModel.scala b/hail/hail/src/is/hail/stats/LogisticRegressionModel.scala index 93133f2d14d..56ad88aa118 100644 --- a/hail/hail/src/is/hail/stats/LogisticRegressionModel.scala +++ b/hail/hail/src/is/hail/stats/LogisticRegressionModel.scala @@ -61,7 +61,7 @@ object WaldTest extends GLMTest { ("fit", GLMFit.schema), ) - def test( + override def test( X: DenseMatrix[Double], y: DenseVector[Double], nullFit: GLMFit, @@ -102,7 +102,7 @@ case class WaldStats( z: DenseVector[Double], p: DenseVector[Double], ) extends GLMStats { - def addToRVB(rvb: RegionValueBuilder): Unit = { + override def addToRVB(rvb: RegionValueBuilder): Unit = { rvb.addDouble(b(-1)) rvb.addDouble(se(-1)) rvb.addDouble(z(-1)) @@ -118,7 +118,7 @@ object LikelihoodRatioTest extends GLMTest { ("fit", GLMFit.schema), ) - def test( + override def test( X: DenseMatrix[Double], y: DenseVector[Double], nullFit: GLMFit, @@ -149,7 +149,7 @@ object LikelihoodRatioTest extends GLMTest { } case class LikelihoodRatioStats(b: DenseVector[Double], chi2: Double, p: Double) extends GLMStats { - def addToRVB(rvb: RegionValueBuilder): Unit = { + override def addToRVB(rvb: RegionValueBuilder): Unit = { rvb.addDouble(b(-1)) rvb.addDouble(chi2) rvb.addDouble(p) @@ -164,7 +164,7 @@ object LogisticFirthTest extends GLMTest { ("fit", GLMFit.schema), ) - def test( + override def test( X: DenseMatrix[Double], y: DenseVector[Double], nullFit: GLMFit, @@ -201,7 +201,7 @@ object LogisticFirthTest extends GLMTest { } case class FirthStats(b: DenseVector[Double], chi2: Double, p: Double) extends GLMStats { - def addToRVB(rvb: RegionValueBuilder): Unit = { + override def addToRVB(rvb: RegionValueBuilder): Unit = { rvb.addDouble(b(-1)) rvb.addDouble(chi2) rvb.addDouble(p) @@ -214,7 +214,7 @@ object LogisticScoreTest extends GLMTest { ("p_value", TFloat64), ) - def test( + override def test( X: DenseMatrix[Double], y: DenseVector[Double], nullFit: GLMFit, @@ -264,7 +264,7 @@ object LogisticScoreTest extends GLMTest { } case class ScoreStats(chi2: Double, p: Double) extends GLMStats { - def addToRVB(rvb: RegionValueBuilder): Unit = { + override def addToRVB(rvb: RegionValueBuilder): Unit = { rvb.addDouble(chi2) rvb.addDouble(p) } @@ -283,7 +283,7 @@ class LogisticRegressionModel(X: DenseMatrix[Double], y: DenseVector[Double]) val n: Int = X.rows val m: Int = X.cols - def bInterceptOnly(): DenseVector[Double] = { + override def bInterceptOnly(): DenseVector[Double] = { require(m > 0) val b = DenseVector.zeros[Double](m) val avg = sum(y) / n @@ -291,7 +291,7 @@ class LogisticRegressionModel(X: DenseMatrix[Double], y: DenseVector[Double]) b } - def fit(optNullFit: Option[GLMFit] = None, maxIter: Int, tol: Double): GLMFit = { + override def fit(optNullFit: Option[GLMFit] = None, maxIter: Int, tol: Double): GLMFit = { val b = DenseVector.zeros[Double](m) val mu = DenseVector.zeros[Double](n) diff --git a/hail/hail/src/is/hail/stats/PoissonRegressionModel.scala b/hail/hail/src/is/hail/stats/PoissonRegressionModel.scala index 39e4a0dcc0c..1f9f3652549 100644 --- a/hail/hail/src/is/hail/stats/PoissonRegressionModel.scala +++ b/hail/hail/src/is/hail/stats/PoissonRegressionModel.scala @@ -15,7 +15,7 @@ object PoissonScoreTest extends GLMTest { ("p_value", TFloat64), ) - def test( + override def test( X: DenseMatrix[Double], y: DenseVector[Double], nullFit: GLMFit, @@ -71,7 +71,7 @@ class PoissonRegressionModel(X: DenseMatrix[Double], y: DenseVector[Double]) val n: Int = X.rows val m: Int = X.cols - def bInterceptOnly(): DenseVector[Double] = { + override def bInterceptOnly(): DenseVector[Double] = { require(m > 0) val b = DenseVector.zeros[Double](m) val avg = sum(y) / n @@ -79,7 +79,7 @@ class PoissonRegressionModel(X: DenseMatrix[Double], y: DenseVector[Double]) b } - def fit(optNullFit: Option[GLMFit] = None, maxIter: Int, tol: Double): GLMFit = { + override def fit(optNullFit: Option[GLMFit] = None, maxIter: Int, tol: Double): GLMFit = { val b = DenseVector.zeros[Double](m) val mu = DenseVector.zeros[Double](n) diff --git a/hail/hail/src/is/hail/stats/eigSymD.scala b/hail/hail/src/is/hail/stats/eigSymD.scala index 2367330d9f3..fff342fd541 100644 --- a/hail/hail/src/is/hail/stats/eigSymD.scala +++ b/hail/hail/src/is/hail/stats/eigSymD.scala @@ -17,7 +17,7 @@ object eigSymD extends UFunc { type DenseEigSymD = EigSymD[DenseVector[Double], DenseMatrix[Double]] implicit object eigSymD_DM_Impl extends Impl[DenseMatrix[Double], DenseEigSymD] { - def apply(X: DenseMatrix[Double]): DenseEigSymD = + override def apply(X: DenseMatrix[Double]): DenseEigSymD = doeigSymD(X, rightEigenvectors = true) match { case (ev, Some(rev)) => EigSymD(ev, rev) case _ => throw new RuntimeException("Shouldn't be here!") @@ -27,7 +27,7 @@ object eigSymD extends UFunc { object justEigenvalues extends UFunc { implicit object eigSymD_DM_Impl extends Impl[DenseMatrix[Double], DenseVector[Double]] { - def apply(X: DenseMatrix[Double]): DenseVector[Double] = + override def apply(X: DenseMatrix[Double]): DenseVector[Double] = doeigSymD(X, rightEigenvectors = false)._1 } @@ -84,7 +84,7 @@ object eigSymR extends UFunc { type DenseeigSymR = EigSymR[DenseVector[Double], DenseMatrix[Double]] implicit object eigSymR_DM_Impl extends Impl[DenseMatrix[Double], DenseeigSymR] { - def apply(X: DenseMatrix[Double]): DenseeigSymR = + override def apply(X: DenseMatrix[Double]): DenseeigSymR = doeigSymR(X, rightEigenvectors = true) match { case (ev, Some(rev)) => EigSymR(ev, rev) case _ => throw new RuntimeException("Shouldn't be here!") @@ -94,7 +94,7 @@ object eigSymR extends UFunc { object justEigenvalues extends UFunc { implicit object eigSymR_DM_Impl extends Impl[DenseMatrix[Double], DenseVector[Double]] { - def apply(X: DenseMatrix[Double]): DenseVector[Double] = + override def apply(X: DenseMatrix[Double]): DenseVector[Double] = doeigSymR(X, rightEigenvectors = false)._1 } diff --git a/hail/hail/src/is/hail/types/Box.scala b/hail/hail/src/is/hail/types/Box.scala index 3c516dd19df..96dc7165fe6 100644 --- a/hail/hail/src/is/hail/types/Box.scala +++ b/hail/hail/src/is/hail/types/Box.scala @@ -4,7 +4,7 @@ import java.util.function._ final case class Box[T]( b: ThreadLocal[Option[T]] = ThreadLocal.withInitial( - new Supplier[Option[T]] { def get = None } + new Supplier[Option[T]] { override def get = None } ), matchCond: (T, T) => Boolean = { (a: T, b: T) => a == b }, ) { diff --git a/hail/hail/src/is/hail/types/TypeWithRequiredness.scala b/hail/hail/src/is/hail/types/TypeWithRequiredness.scala index 07aefe4c367..ef64a885911 100644 --- a/hail/hail/src/is/hail/types/TypeWithRequiredness.scala +++ b/hail/hail/src/is/hail/types/TypeWithRequiredness.scala @@ -295,22 +295,24 @@ object RPrimitive { final case class RPrimitive() extends TypeWithRequiredness { val children: IndexedSeq[TypeWithRequiredness] = RPrimitive.children - def _unionLiteral(a: Annotation): Unit = () - def _matchesPType(pt: PType): Boolean = RPrimitive.typeSupported(pt.virtualType) - def _unionPType(pType: PType): Unit = assert(RPrimitive.typeSupported(pType.virtualType)) - def _unionEmitType(emitType: EmitType) = assert(RPrimitive.typeSupported(emitType.virtualType)) + override def _unionLiteral(a: Annotation): Unit = () + override def _matchesPType(pt: PType): Boolean = RPrimitive.typeSupported(pt.virtualType) + override def _unionPType(pType: PType): Unit = assert(RPrimitive.typeSupported(pType.virtualType)) - def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RPrimitive = { + override def _unionEmitType(emitType: EmitType) = + assert(RPrimitive.typeSupported(emitType.virtualType)) + + override def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RPrimitive = { assert(newChildren.isEmpty) RPrimitive() } - def canonicalPType(t: Type): PType = { + override def canonicalPType(t: Type): PType = { assert(RPrimitive.typeSupported(t)) PType.canonical(t, required) } - def _toString: String = "RPrimitive" + override def _toString: String = "RPrimitive" } abstract class RContainer extends TypeWithRequiredness { @@ -361,26 +363,26 @@ object RIterable { case class RIterable(elementType: TypeWithRequiredness, eltRequired: Boolean) extends RContainer { override val children: IndexedSeq[TypeWithRequiredness] = FastSeq(elementType) - def _unionLiteral(a: Annotation): Unit = + override def _unionLiteral(a: Annotation): Unit = a.asInstanceOf[Iterable[_]].foreach(elt => elementType.unionLiteral(elt)) - def _matchesPType(pt: PType): Boolean = + override def _matchesPType(pt: PType): Boolean = elementType.matchesPType(tcoerce[PIterable](pt).elementType) - def _unionPType(pType: PType): Unit = + override def _unionPType(pType: PType): Unit = elementType.fromPType(pType.asInstanceOf[PIterable].elementType) - def _unionEmitType(emitType: EmitType): Unit = + override def _unionEmitType(emitType: EmitType): Unit = elementType.fromEmitType(emitType.st.asInstanceOf[SIndexablePointer].elementEmitType) - def _toString: String = s"RIterable[${elementType.toString}]" + override def _toString: String = s"RIterable[${elementType.toString}]" - def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RIterable = { + override def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RIterable = { val IndexedSeq(newElt: TypeWithRequiredness) = newChildren RIterable(newElt) } - def canonicalPType(t: Type): PType = { + override def canonicalPType(t: Type): PType = { val elt = elementType.canonicalPType(tcoerce[TIterable](t).elementType) t match { case _: TArray => PCanonicalArray(elt, required = required) @@ -397,11 +399,11 @@ case class RIterable(elementType: TypeWithRequiredness, eltRequired: Boolean) ex } case class RNDArray(elementType: TypeWithRequiredness) extends RContainer { - def eltRequired = true + override def eltRequired = true override val children: IndexedSeq[TypeWithRequiredness] = FastSeq(elementType) - def _unionLiteral(a: Annotation): Unit = { + override def _unionLiteral(a: Annotation): Unit = { val data = a.asInstanceOf[NDArray].getRowMajorElements() data.foreach { elt => if (elt != null) @@ -409,65 +411,65 @@ case class RNDArray(elementType: TypeWithRequiredness) extends RContainer { } } - def _matchesPType(pt: PType): Boolean = + override def _matchesPType(pt: PType): Boolean = elementType.matchesPType(tcoerce[PNDArray](pt).elementType) - def _unionPType(pType: PType): Unit = + override def _unionPType(pType: PType): Unit = elementType.fromPType(pType.asInstanceOf[PNDArray].elementType) - def _unionEmitType(emitType: EmitType): Unit = + override def _unionEmitType(emitType: EmitType): Unit = elementType.fromEmitType(emitType.st.asInstanceOf[SNDArray].elementEmitType) - def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RNDArray = { + override def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RNDArray = { val IndexedSeq(newElt: TypeWithRequiredness) = newChildren RNDArray(newElt) } - def canonicalPType(t: Type): PType = { + override def canonicalPType(t: Type): PType = { val tnd = tcoerce[TNDArray](t) PCanonicalNDArray(elementType.canonicalPType(tnd.elementType), tnd.nDims, required = required) } - def _toString: String = s"RNDArray[${elementType.toString}]" + override def _toString: String = s"RNDArray[${elementType.toString}]" } case class RInterval(startType: TypeWithRequiredness, endType: TypeWithRequiredness) extends TypeWithRequiredness { val children: IndexedSeq[TypeWithRequiredness] = FastSeq(startType, endType) - def _unionLiteral(a: Annotation): Unit = { + override def _unionLiteral(a: Annotation): Unit = { startType.unionLiteral(a.asInstanceOf[Interval].start) endType.unionLiteral(a.asInstanceOf[Interval].end) } - def _matchesPType(pt: PType): Boolean = + override def _matchesPType(pt: PType): Boolean = startType.matchesPType(tcoerce[PInterval](pt).pointType) && endType.matchesPType(tcoerce[PInterval](pt).pointType) - def _unionPType(pType: PType): Unit = { + override def _unionPType(pType: PType): Unit = { startType.fromPType(pType.asInstanceOf[PInterval].pointType) endType.fromPType(pType.asInstanceOf[PInterval].pointType) } - def _unionEmitType(emitType: EmitType): Unit = { + override def _unionEmitType(emitType: EmitType): Unit = { val sInterval = emitType.st.asInstanceOf[SInterval] startType.fromEmitType(sInterval.pointEmitType) endType.fromEmitType(sInterval.pointEmitType) } - def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RInterval = { + override def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RInterval = { val IndexedSeq(newStart: TypeWithRequiredness, newEnd: TypeWithRequiredness) = newChildren RInterval(newStart, newEnd) } - def canonicalPType(t: Type): PType = t match { + override def canonicalPType(t: Type): PType = t match { case TInterval(pointType) => val unified = startType.deepCopy().asInstanceOf[TypeWithRequiredness] unified.unionFrom(endType) PCanonicalInterval(unified.canonicalPType(pointType), required = required) } - def _toString: String = s"RInterval[${startType.toString}, ${endType.toString}]" + override def _toString: String = s"RInterval[${startType.toString}, ${endType.toString}]" } case class RField(name: String, typ: TypeWithRequiredness, index: Int) @@ -477,7 +479,7 @@ sealed abstract class RBaseStruct extends TypeWithRequiredness { def size: Int = fields.length val children: IndexedSeq[TypeWithRequiredness] = fields.map(_.typ) - def _unionLiteral(a: Annotation): Unit = a match { + override def _unionLiteral(a: Annotation): Unit = a match { case a: Row => children.lazyZip(a.toSeq).foreach((r, f) => r.unionLiteral(f)) case (k, v) => // needed to handle the elements of a Map/Dict @@ -486,13 +488,13 @@ sealed abstract class RBaseStruct extends TypeWithRequiredness { children(1).unionLiteral(v) } - def _matchesPType(pt: PType): Boolean = + override def _matchesPType(pt: PType): Boolean = tcoerce[PBaseStruct](pt).fields.forall(f => children(f.index).matchesPType(f.typ)) - def _unionPType(pType: PType): Unit = + override def _unionPType(pType: PType): Unit = pType.asInstanceOf[PBaseStruct].fields.foreach(f => children(f.index).fromPType(f.typ)) - def _unionEmitType(emitType: EmitType): Unit = + override def _unionEmitType(emitType: EmitType): Unit = emitType.st.asInstanceOf[SBaseStruct].fieldEmitTypes.zipWithIndex.foreach { case (et, idx) => children(idx).fromEmitType(et) } @@ -502,7 +504,7 @@ sealed abstract class RBaseStruct extends TypeWithRequiredness { fields.lazyZip(other.fields).foreach((fd1, fd2) => fd1.typ.unionFrom(fd2.typ)) } - def canonicalPType(t: Type): PType = t match { + override def canonicalPType(t: Type): PType = t match { case ts: TStruct => PCanonicalStruct( required = required, @@ -529,7 +531,7 @@ case class RStruct(fields: IndexedSeq[RField]) extends RBaseStruct { def fieldOption(name: String): Option[TypeWithRequiredness] = fieldType.get(name) def hasField(name: String): Boolean = fieldType.contains(name) - def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RStruct = { + override def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RStruct = { assert(newChildren.length == fields.length) RStruct.fromNamesAndTypes(Array.tabulate(fields.length)(i => fields(i).name -> tcoerce[TypeWithRequiredness](newChildren(i)) @@ -539,7 +541,7 @@ case class RStruct(fields: IndexedSeq[RField]) extends RBaseStruct { def select(newFields: Array[String]): RStruct = RStruct(Array.tabulate(newFields.length)(i => RField(newFields(i), field(newFields(i)), i))) - def _toString: String = + override def _toString: String = s"RStruct[${fields.map(f => s"${f.name}: ${f.typ.toString}").mkString(",")}]" } @@ -552,14 +554,14 @@ case class RTuple(fields: IndexedSeq[RField]) extends RBaseStruct { val fieldType: collection.Map[String, TypeWithRequiredness] = toMapFast(fields)(_.name, _.typ) def field(idx: Int): TypeWithRequiredness = fieldType(idx.toString) - def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RTuple = { + override def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RTuple = { assert(newChildren.length == fields.length) RTuple(fields.lazyZip(newChildren).map { (f, c) => RField(f.name, tcoerce[TypeWithRequiredness](c), f.index) }) } - def _toString: String = + override def _toString: String = s"RTuple[${fields.map(f => s"${f.index}: ${f.typ.toString}").mkString(",")}]" } @@ -658,7 +660,7 @@ case class RTable( def changeKey(key: IndexedSeq[String]): RTable = RTable(rowFields, globalFields, key) - def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RTable = { + override def copy(newChildren: IndexedSeq[BaseTypeWithRequiredness]): RTable = { assert(newChildren.length == rowFields.length + globalFields.length) val newRowFields = rowFields.lazyZip(newChildren.take(rowFields.length)).map { case ((n, _), r: TypeWithRequiredness) => n -> r diff --git a/hail/hail/src/is/hail/types/encoded/EArray.scala b/hail/hail/src/is/hail/types/encoded/EArray.scala index 99b18b22bab..61095abb90e 100644 --- a/hail/hail/src/is/hail/types/encoded/EArray.scala +++ b/hail/hail/src/is/hail/types/encoded/EArray.scala @@ -13,7 +13,7 @@ import is.hail.utils._ final case class EArray(val elementType: EType, override val required: Boolean = false) extends EContainer { - def _decodedSType(requestedType: Type): SType = { + override def _decodedSType(requestedType: Type): SType = { val elementPType = elementType.decodedPType(requestedType.asInstanceOf[TContainer].elementType) requestedType match { case _: TSet => @@ -193,7 +193,7 @@ final case class EArray(val elementType: EType, override val required: Boolean = new SIndexablePointerValue(st, array, len, cb.memoize(arrayType.firstElementOffset(array, len))) } - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = { + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = { val skip = elementType.buildSkip(cb.emb.ecb) val len = cb.newLocal[Int]("len", in.readInt()) val i = cb.newLocal[Int]("i") @@ -212,8 +212,8 @@ final case class EArray(val elementType: EType, override val required: Boolean = } } - def _asIdent = s"array_of_${elementType.asIdent}" - def _toPretty = s"EArray[$elementType]" + override def _asIdent = s"array_of_${elementType.asIdent}" + override def _toPretty = s"EArray[$elementType]" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false): Unit = { sb ++= "EArray[" @@ -221,5 +221,5 @@ final case class EArray(val elementType: EType, override val required: Boolean = sb += ']' } - def setRequired(newRequired: Boolean): EArray = EArray(elementType, newRequired) + override def setRequired(newRequired: Boolean): EArray = EArray(elementType, newRequired) } diff --git a/hail/hail/src/is/hail/types/encoded/EBaseStruct.scala b/hail/hail/src/is/hail/types/encoded/EBaseStruct.scala index 40b4602f903..ee6fdacf115 100644 --- a/hail/hail/src/is/hail/types/encoded/EBaseStruct.scala +++ b/hail/hail/src/is/hail/types/encoded/EBaseStruct.scala @@ -55,7 +55,7 @@ final case class EBaseStruct(fields: IndexedSeq[EField], override val required: ) } - def _decodedSType(requestedType: Type): SType = requestedType match { + override def _decodedSType(requestedType: Type): SType = requestedType match { case t: TInterval => val structPType = decodedPType(t.structRepresentation).asInstanceOf[PStruct] val pointType = structPType.field("start").typ @@ -198,7 +198,7 @@ final case class EBaseStruct(fields: IndexedSeq[EField], override val required: } } - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = { + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = { val mbytes = cb.newLocal[Long]("mbytes", r.allocate(const(1L), const(nMissingBytes.toLong))) cb += in.readBytes(r, mbytes, nMissingBytes) fields.foreach { f => @@ -210,7 +210,7 @@ final case class EBaseStruct(fields: IndexedSeq[EField], override val required: } } - def _asIdent: String = { + override def _asIdent: String = { val sb = new StringBuilder sb ++= "struct_of_" types.foreachBetween(sb ++= _.asIdent)(sb ++= "AND") @@ -218,7 +218,7 @@ final case class EBaseStruct(fields: IndexedSeq[EField], override val required: sb.result() } - def _toPretty: String = { + override def _toPretty: String = { val sb = new StringBuilder _pretty(sb, 0, compact = true) sb.result() @@ -237,5 +237,5 @@ final case class EBaseStruct(fields: IndexedSeq[EField], override val required: sb += '\n' ++= " " * indent += '}': Unit } - def setRequired(newRequired: Boolean): EBaseStruct = EBaseStruct(fields, newRequired) + override def setRequired(newRequired: Boolean): EBaseStruct = EBaseStruct(fields, newRequired) } diff --git a/hail/hail/src/is/hail/types/encoded/EBinary.scala b/hail/hail/src/is/hail/types/encoded/EBinary.scala index e82016973be..7dc1c31479a 100644 --- a/hail/hail/src/is/hail/types/encoded/EBinary.scala +++ b/hail/hail/src/is/hail/types/encoded/EBinary.scala @@ -61,18 +61,18 @@ class EBinary(override val required: Boolean) extends EType { } } - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = cb += in.skipBytes(in.readInt()) - def _decodedSType(requestedType: Type): SType = requestedType match { + override def _decodedSType(requestedType: Type): SType = requestedType match { case TBinary => SBinaryPointer(PCanonicalBinary(false)) case TString => SStringPointer(PCanonicalString(false)) } - def _asIdent = "binary" - def _toPretty = "EBinary" + override def _asIdent = "binary" + override def _toPretty = "EBinary" - def setRequired(newRequired: Boolean): EBinary = EBinary(newRequired) + override def setRequired(newRequired: Boolean): EBinary = EBinary(newRequired) } object EBinary { diff --git a/hail/hail/src/is/hail/types/encoded/EBlockMatrixNDArray.scala b/hail/hail/src/is/hail/types/encoded/EBlockMatrixNDArray.scala index be1fe202e80..5b065a056a2 100644 --- a/hail/hail/src/is/hail/types/encoded/EBlockMatrixNDArray.scala +++ b/hail/hail/src/is/hail/types/encoded/EBlockMatrixNDArray.scala @@ -18,10 +18,10 @@ final case class EBlockMatrixNDArray( ) extends EType { type DecodedPType = PCanonicalNDArray - def setRequired(newRequired: Boolean): EBlockMatrixNDArray = + override def setRequired(newRequired: Boolean): EBlockMatrixNDArray = EBlockMatrixNDArray(elementType, newRequired) - def _decodedSType(requestedType: Type): SType = { + override def _decodedSType(requestedType: Type): SType = { val elementPType = elementType.decodedPType(requestedType.asInstanceOf[TNDArray].elementType) SNDArrayPointer(PCanonicalNDArray(elementPType, 2, false)) } @@ -108,7 +108,7 @@ final case class EBlockMatrixNDArray( tFinisher(cb) } - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = { + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = { val skip = elementType.buildSkip(cb.emb.ecb) val len = cb.newLocal[Int]("len", in.readInt() * in.readInt()) @@ -120,7 +120,7 @@ final case class EBlockMatrixNDArray( override def _asIdent: String = s"bm_ndarray_${if (encodeRowMajor) "row" else "column"}_major_of_${elementType.asIdent}" - def _toPretty = s"ENDArray[$elementType]" + override def _toPretty = s"ENDArray[$elementType]" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false): Unit = { sb ++= "ENDArray[" diff --git a/hail/hail/src/is/hail/types/encoded/EBoolean.scala b/hail/hail/src/is/hail/types/encoded/EBoolean.scala index 744f9e7938d..9853311c837 100644 --- a/hail/hail/src/is/hail/types/encoded/EBoolean.scala +++ b/hail/hail/src/is/hail/types/encoded/EBoolean.scala @@ -25,16 +25,16 @@ class EBoolean(override val required: Boolean) extends EType { ): SValue = new SBooleanValue(cb.memoize(in.readBoolean())) - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = cb += in.skipBoolean() - def _decodedSType(requestedType: Type): SType = SBoolean + override def _decodedSType(requestedType: Type): SType = SBoolean - def _asIdent = "bool" + override def _asIdent = "bool" - def _toPretty = "EBoolean" + override def _toPretty = "EBoolean" - def setRequired(newRequired: Boolean): EBoolean = EBoolean(newRequired) + override def setRequired(newRequired: Boolean): EBoolean = EBoolean(newRequired) } object EBoolean { diff --git a/hail/hail/src/is/hail/types/encoded/EDictAsUnsortedArrayOfPairs.scala b/hail/hail/src/is/hail/types/encoded/EDictAsUnsortedArrayOfPairs.scala index a6e77e60812..58de49ad9cd 100644 --- a/hail/hail/src/is/hail/types/encoded/EDictAsUnsortedArrayOfPairs.scala +++ b/hail/hail/src/is/hail/types/encoded/EDictAsUnsortedArrayOfPairs.scala @@ -18,7 +18,7 @@ final case class EDictAsUnsortedArrayOfPairs( private[this] val arrayRepr = EArray(elementType, required) - def _decodedSType(requestedType: Type): SType = { + override def _decodedSType(requestedType: Type): SType = { val elementPType = elementType.decodedPType(requestedType.asInstanceOf[TContainer].elementType) requestedType match { case _: TDict => @@ -27,13 +27,17 @@ final case class EDictAsUnsortedArrayOfPairs( } } - def _buildEncoder(cb: EmitCodeBuilder, v: SValue, out: Value[OutputBuffer]): Unit = + override def _buildEncoder(cb: EmitCodeBuilder, v: SValue, out: Value[OutputBuffer]): Unit = // Anything we have to encode from a region should already be sorted so we don't // have to do anything else arrayRepr._buildEncoder(cb, v, out) - def _buildDecoder(cb: EmitCodeBuilder, t: Type, region: Value[Region], in: Value[InputBuffer]) - : SValue = { + override def _buildDecoder( + cb: EmitCodeBuilder, + t: Type, + region: Value[Region], + in: Value[InputBuffer], + ): SValue = { val tmpRegion = cb.memoize(Region.stagedCreate(Region.REGULAR, region.getPool()), "tmp_region") val arrayDecoder = arrayRepr.buildDecoder(t, cb.emb.ecb) @@ -64,12 +68,12 @@ final case class EDictAsUnsortedArrayOfPairs( ret } - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = arrayRepr._buildSkip(cb, r, in) - def _asIdent = s"dict_of_${elementType.asIdent}" - def _toPretty = s"EDictAsUnsortedArrayOfPairs[$elementType]" + override def _asIdent = s"dict_of_${elementType.asIdent}" + override def _toPretty = s"EDictAsUnsortedArrayOfPairs[$elementType]" - def setRequired(newRequired: Boolean): EType = + override def setRequired(newRequired: Boolean): EType = EDictAsUnsortedArrayOfPairs(elementType, newRequired) } diff --git a/hail/hail/src/is/hail/types/encoded/EFloat32.scala b/hail/hail/src/is/hail/types/encoded/EFloat32.scala index 3442bb5840a..c67ff0ce65d 100644 --- a/hail/hail/src/is/hail/types/encoded/EFloat32.scala +++ b/hail/hail/src/is/hail/types/encoded/EFloat32.scala @@ -25,16 +25,16 @@ class EFloat32(override val required: Boolean) extends EType { ): SValue = new SFloat32Value(cb.memoize(in.readFloat())) - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = cb += in.skipFloat() - def _decodedSType(requestedType: Type): SType = SFloat32 + override def _decodedSType(requestedType: Type): SType = SFloat32 - def _asIdent = "float32" + override def _asIdent = "float32" - def _toPretty = "EFloat32" + override def _toPretty = "EFloat32" - def setRequired(newRequired: Boolean): EFloat32 = EFloat32(newRequired) + override def setRequired(newRequired: Boolean): EFloat32 = EFloat32(newRequired) } object EFloat32 { diff --git a/hail/hail/src/is/hail/types/encoded/EFloat64.scala b/hail/hail/src/is/hail/types/encoded/EFloat64.scala index 05aabe2a1bd..a065051cb36 100644 --- a/hail/hail/src/is/hail/types/encoded/EFloat64.scala +++ b/hail/hail/src/is/hail/types/encoded/EFloat64.scala @@ -25,16 +25,16 @@ class EFloat64(override val required: Boolean) extends EType { ): SValue = new SFloat64Value(cb.memoize(in.readDouble())) - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = cb += in.skipDouble() - def _decodedSType(requestedType: Type): SType = SFloat64 + override def _decodedSType(requestedType: Type): SType = SFloat64 - def _asIdent = "float64" + override def _asIdent = "float64" - def _toPretty = "EFloat64" + override def _toPretty = "EFloat64" - def setRequired(newRequired: Boolean): EFloat64 = EFloat64(newRequired) + override def setRequired(newRequired: Boolean): EFloat64 = EFloat64(newRequired) } object EFloat64 { diff --git a/hail/hail/src/is/hail/types/encoded/EInt32.scala b/hail/hail/src/is/hail/types/encoded/EInt32.scala index b559261b162..94b5fdc1477 100644 --- a/hail/hail/src/is/hail/types/encoded/EInt32.scala +++ b/hail/hail/src/is/hail/types/encoded/EInt32.scala @@ -45,19 +45,19 @@ class EInt32(override val required: Boolean) extends EType { in: Value[InputBuffer], ): Code[Int] = in.readInt() - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = cb += in.skipInt() - def _decodedSType(requestedType: Type): SType = requestedType match { + override def _decodedSType(requestedType: Type): SType = requestedType match { case TCall => SCanonicalCall case _ => SInt32 } - def _asIdent = "int32" + override def _asIdent = "int32" - def _toPretty = "EInt32" + override def _toPretty = "EInt32" - def setRequired(newRequired: Boolean): EInt32 = EInt32(newRequired) + override def setRequired(newRequired: Boolean): EInt32 = EInt32(newRequired) } object EInt32 { diff --git a/hail/hail/src/is/hail/types/encoded/EInt64.scala b/hail/hail/src/is/hail/types/encoded/EInt64.scala index b88517bcc02..f62f4ab2974 100644 --- a/hail/hail/src/is/hail/types/encoded/EInt64.scala +++ b/hail/hail/src/is/hail/types/encoded/EInt64.scala @@ -25,16 +25,16 @@ class EInt64(override val required: Boolean) extends EType { ): SValue = new SInt64Value(cb.memoize(in.readLong())) - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = cb += in.skipLong() - def _decodedSType(requestedType: Type): SType = SInt64 + override def _decodedSType(requestedType: Type): SType = SInt64 - def _asIdent = "int64" + override def _asIdent = "int64" - def _toPretty = "EInt64" + override def _toPretty = "EInt64" - def setRequired(newRequired: Boolean): EInt64 = EInt64(newRequired) + override def setRequired(newRequired: Boolean): EInt64 = EInt64(newRequired) } object EInt64 { diff --git a/hail/hail/src/is/hail/types/encoded/ENDArrayColumnMajor.scala b/hail/hail/src/is/hail/types/encoded/ENDArrayColumnMajor.scala index d6b505c44f4..101a7914525 100644 --- a/hail/hail/src/is/hail/types/encoded/ENDArrayColumnMajor.scala +++ b/hail/hail/src/is/hail/types/encoded/ENDArrayColumnMajor.scala @@ -62,7 +62,7 @@ case class ENDArrayColumnMajor(elementType: EType, nDims: Int, required: Boolean pndFinisher(cb) } - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = { + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = { val skip = elementType.buildSkip(cb.emb.ecb) val numElements = cb.newLocal[Long]( @@ -73,7 +73,7 @@ case class ENDArrayColumnMajor(elementType: EType, nDims: Int, required: Boolean cb.for_(cb.assign(i, 0L), i < numElements, cb.assign(i, i + 1L), skip(cb, r, in)) } - def _decodedSType(requestedType: Type): SType = { + override def _decodedSType(requestedType: Type): SType = { val requestedTNDArray = requestedType.asInstanceOf[TNDArray] val elementPType = elementType.decodedPType(requestedTNDArray.elementType) SNDArrayPointer(PCanonicalNDArray(elementPType, requestedTNDArray.nDims, false)) diff --git a/hail/hail/src/is/hail/types/encoded/ERNGState.scala b/hail/hail/src/is/hail/types/encoded/ERNGState.scala index 71b7d70c8f6..907af812c81 100644 --- a/hail/hail/src/is/hail/types/encoded/ERNGState.scala +++ b/hail/hail/src/is/hail/types/encoded/ERNGState.scala @@ -54,7 +54,7 @@ final case class ERNGState(override val required: Boolean, staticInfo: Option[SR ) } - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = staticInfo match { case Some(staticInfo) => for (_ <- 0 until (4 + staticInfo.numWordsInLastBlock)) @@ -67,11 +67,11 @@ final case class ERNGState(override val required: Boolean, staticInfo: Option[SR cb += in.skipInt() } - def _decodedSType(requestedType: Type): SRNGState = SRNGState(staticInfo) + override def _decodedSType(requestedType: Type): SRNGState = SRNGState(staticInfo) - def _asIdent = "rngstate" + override def _asIdent = "rngstate" - def _toPretty = "ERNGState" + override def _toPretty = "ERNGState" - def setRequired(newRequired: Boolean): ERNGState = ERNGState(newRequired, staticInfo) + override def setRequired(newRequired: Boolean): ERNGState = ERNGState(newRequired, staticInfo) } diff --git a/hail/hail/src/is/hail/types/encoded/EStructOfArrays.scala b/hail/hail/src/is/hail/types/encoded/EStructOfArrays.scala index 50a828445b6..852f1f52083 100644 --- a/hail/hail/src/is/hail/types/encoded/EStructOfArrays.scala +++ b/hail/hail/src/is/hail/types/encoded/EStructOfArrays.scala @@ -72,14 +72,18 @@ final case class EStructOfArrays( ) } - def _decodedSType(requestedType: Type): SType = { + override def _decodedSType(requestedType: Type): SType = { require(requestedType.isInstanceOf[TArray]) val elementPType = elementType.decodedPType(requestedType.asInstanceOf[TContainer].elementType) SIndexablePointer(PCanonicalArray(elementPType, required = false)) } - def _buildDecoder(cb: EmitCodeBuilder, t: Type, region: Value[Region], in: Value[InputBuffer]) - : SValue = { + override def _buildDecoder( + cb: EmitCodeBuilder, + t: Type, + region: Value[Region], + in: Value[InputBuffer], + ): SValue = { val st = tcoerce[SIndexablePointer](decodedSType(t)) val pt = tcoerce[PCanonicalArray](st.pType) val ept = tcoerce[PBaseStruct](pt.elementType) @@ -157,26 +161,27 @@ final case class EStructOfArrays( ) } - def _buildEncoder(cb: EmitCodeBuilder, v: SValue, out: Value[OutputBuffer]): Unit = v match { - case sv: SIndexablePointerValue => - val pArray = sv.st.pType.asInstanceOf[PCanonicalArrayBackedContainer].arrayRep - val r: Value[Region] = // scratch region - cb.memoize(cb.emb.ecb.pool().invoke[Region]("getRegion")) - cb += out.writeInt(sv.length) - if (!elementType.required) { - val nMissingBytes = cb.memoize(pArray.nMissingBytes(sv.length)) - cb += out.writeBytes(sv.a + pArray.missingBytesOffset, nMissingBytes) - } + override def _buildEncoder(cb: EmitCodeBuilder, v: SValue, out: Value[OutputBuffer]): Unit = + v match { + case sv: SIndexablePointerValue => + val pArray = sv.st.pType.asInstanceOf[PCanonicalArrayBackedContainer].arrayRep + val r: Value[Region] = // scratch region + cb.memoize(cb.emb.ecb.pool().invoke[Region]("getRegion")) + cb += out.writeInt(sv.length) + if (!elementType.required) { + val nMissingBytes = cb.memoize(pArray.nMissingBytes(sv.length)) + cb += out.writeBytes(sv.a + pArray.missingBytesOffset, nMissingBytes) + } - fields.foreach { field => - val pFieldType = tcoerce[PBaseStruct](pArray.elementType).fieldByName(field.name).typ - require(EStructOfArrays.supportsFieldType(pFieldType.virtualType)) - val arrayType = PCanonicalArray(pFieldType, required = false) - transposeAndWriteField(cb, field, arrayType, sv, r, out) - } + fields.foreach { field => + val pFieldType = tcoerce[PBaseStruct](pArray.elementType).fieldByName(field.name).typ + require(EStructOfArrays.supportsFieldType(pFieldType.virtualType)) + val arrayType = PCanonicalArray(pFieldType, required = false) + transposeAndWriteField(cb, field, arrayType, sv, r, out) + } - cb += r.invalidate() - } + cb += r.invalidate() + } private[this] def transposeAndWriteField( cb: EmitCodeBuilder, @@ -217,7 +222,7 @@ final case class EStructOfArrays( arrayEncoder(cb, arrayValue, out) } - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = { + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = { val length = cb.memoize(in.readInt()) val nMissingBytes = cb.memoize(UnsafeUtils.packBitsToBytes(length)) // valid for all top level arrays @@ -227,12 +232,12 @@ final case class EStructOfArrays( fields.foreach(field => field.typ.buildSkip(cb.emb.ecb)(cb, r, in)) } - def setRequired(newRequired: Boolean): EStructOfArrays = + override def setRequired(newRequired: Boolean): EStructOfArrays = EStructOfArrays(fields, required = newRequired, structRequired = structRequired) - def _asIdent: String = s"struct_of_arrays_from_${elementType.asIdent}" + override def _asIdent: String = s"struct_of_arrays_from_${elementType.asIdent}" - def _toPretty: String = { + override def _toPretty: String = { val sb = new StringBuilder _pretty(sb, 0, compact = true) sb.result() diff --git a/hail/hail/src/is/hail/types/encoded/EType.scala b/hail/hail/src/is/hail/types/encoded/EType.scala index 0b002136582..7653c481203 100644 --- a/hail/hail/src/is/hail/types/encoded/EType.scala +++ b/hail/hail/src/is/hail/types/encoded/EType.scala @@ -167,7 +167,7 @@ abstract class EType extends BaseType with Serializable with Requiredness { def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit - final def pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = { + final override def pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = { if (required) sb.append("+") _pretty(sb, indent, compact) diff --git a/hail/hail/src/is/hail/types/encoded/EUnsortedSet.scala b/hail/hail/src/is/hail/types/encoded/EUnsortedSet.scala index a8c85430c78..3e09b8fea88 100644 --- a/hail/hail/src/is/hail/types/encoded/EUnsortedSet.scala +++ b/hail/hail/src/is/hail/types/encoded/EUnsortedSet.scala @@ -14,7 +14,7 @@ final case class EUnsortedSet(val elementType: EType, override val required: Boo extends EContainer { private[this] val arrayRepr = EArray(elementType, required) - def _decodedSType(requestedType: Type): SType = { + override def _decodedSType(requestedType: Type): SType = { val elementPType = elementType.decodedPType(requestedType.asInstanceOf[TContainer].elementType) requestedType match { case _: TSet => @@ -22,13 +22,17 @@ final case class EUnsortedSet(val elementType: EType, override val required: Boo } } - def _buildEncoder(cb: EmitCodeBuilder, v: SValue, out: Value[OutputBuffer]): Unit = + override def _buildEncoder(cb: EmitCodeBuilder, v: SValue, out: Value[OutputBuffer]): Unit = // Anything we have to encode from a region should already be sorted so we don't // have to do anything else arrayRepr._buildEncoder(cb, v, out) - def _buildDecoder(cb: EmitCodeBuilder, t: Type, region: Value[Region], in: Value[InputBuffer]) - : SValue = { + override def _buildDecoder( + cb: EmitCodeBuilder, + t: Type, + region: Value[Region], + in: Value[InputBuffer], + ): SValue = { val tmpRegion = cb.memoize(Region.stagedCreate(Region.REGULAR, region.getPool()), "tmp_region") val arrayDecoder = arrayRepr.buildDecoder(t, cb.emb.ecb) @@ -55,10 +59,10 @@ final case class EUnsortedSet(val elementType: EType, override val required: Boo ret } - def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = + override def _buildSkip(cb: EmitCodeBuilder, r: Value[Region], in: Value[InputBuffer]): Unit = arrayRepr._buildSkip(cb, r, in) - def _asIdent = s"set_of_${elementType.asIdent}" - def _toPretty = s"EUnsortedSet[$elementType]" - def setRequired(newRequired: Boolean): EType = EUnsortedSet(elementType, newRequired) + override def _asIdent = s"set_of_${elementType.asIdent}" + override def _toPretty = s"EUnsortedSet[$elementType]" + override def setRequired(newRequired: Boolean): EType = EUnsortedSet(elementType, newRequired) } diff --git a/hail/hail/src/is/hail/types/physical/PArrayBackedContainer.scala b/hail/hail/src/is/hail/types/physical/PArrayBackedContainer.scala index d7a053d34f5..2afee77ed19 100644 --- a/hail/hail/src/is/hail/types/physical/PArrayBackedContainer.scala +++ b/hail/hail/src/is/hail/types/physical/PArrayBackedContainer.scala @@ -96,10 +96,10 @@ trait PArrayBackedContainer extends PContainer { override def clearMissingBits(aoff: Long, length: Int) = arrayRep.clearMissingBits(aoff, length) - def contentsByteSize(length: Int): Long = + override def contentsByteSize(length: Int): Long = arrayRep.contentsByteSize(length) - def contentsByteSize(length: Code[Int]): Code[Long] = + override def contentsByteSize(length: Code[Int]): Code[Long] = arrayRep.contentsByteSize(length) override def initialize(aoff: Long, length: Int, setMissing: Boolean = false) = @@ -215,5 +215,5 @@ trait PArrayBackedContainer extends PContainer { } trait PCanonicalArrayBackedContainer extends PArrayBackedContainer { - def arrayRep: PCanonicalArray + override def arrayRep: PCanonicalArray } diff --git a/hail/hail/src/is/hail/types/physical/PBaseStruct.scala b/hail/hail/src/is/hail/types/physical/PBaseStruct.scala index a84dc6b4c01..7286ea3de4e 100644 --- a/hail/hail/src/is/hail/types/physical/PBaseStruct.scala +++ b/hail/hail/src/is/hail/types/physical/PBaseStruct.scala @@ -53,7 +53,7 @@ abstract class PBaseStruct extends PType { def identBase: String - def _asIdent: String = { + override def _asIdent: String = { val sb = new StringBuilder sb ++= identBase sb ++= "_of_" @@ -79,7 +79,7 @@ abstract class PBaseStruct extends PType { types.zip(right.types).map { case (l, r) => l.unsafeOrdering(sm, r) } new UnsafeOrdering { - def compare(o1: Long, o2: Long): Int = { + override def compare(o1: Long, o2: Long): Int = { var i = 0 while (i < types.length) { val leftDefined = isFieldDefined(o1, i) diff --git a/hail/hail/src/is/hail/types/physical/PBinary.scala b/hail/hail/src/is/hail/types/physical/PBinary.scala index accb05d25ae..d7e92db3528 100644 --- a/hail/hail/src/is/hail/types/physical/PBinary.scala +++ b/hail/hail/src/is/hail/types/physical/PBinary.scala @@ -10,7 +10,7 @@ abstract class PBinary extends PType { lazy val virtualType: TBinary.type = TBinary override def unsafeOrdering(sm: HailStateManager): UnsafeOrdering = new UnsafeOrdering { - def compare(o1: Long, o2: Long): Int = { + override def compare(o1: Long, o2: Long): Int = { val l1 = loadLength(o1) val l2 = loadLength(o2) diff --git a/hail/hail/src/is/hail/types/physical/PBoolean.scala b/hail/hail/src/is/hail/types/physical/PBoolean.scala index 2d233a6dad2..df516c00a64 100644 --- a/hail/hail/src/is/hail/types/physical/PBoolean.scala +++ b/hail/hail/src/is/hail/types/physical/PBoolean.scala @@ -15,21 +15,21 @@ case object PBooleanRequired extends PBoolean(true) class PBoolean(override val required: Boolean) extends PType with PPrimitive { lazy val virtualType: TBoolean.type = TBoolean - def _asIdent = "bool" + override def _asIdent = "bool" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = sb ++= "PBoolean" override def unsafeOrdering(sm: HailStateManager): UnsafeOrdering = new UnsafeOrdering { - def compare(o1: Long, o2: Long): Int = + override def compare(o1: Long, o2: Long): Int = java.lang.Boolean.compare(Region.loadBoolean(o1), Region.loadBoolean(o2)) } override def byteSize: Long = 1 - def sType: SBoolean.type = SBoolean + override def sType: SBoolean.type = SBoolean - def storePrimitiveAtAddress(cb: EmitCodeBuilder, addr: Code[Long], value: SValue): Unit = + override def storePrimitiveAtAddress(cb: EmitCodeBuilder, addr: Code[Long], value: SValue): Unit = cb += Region.storeBoolean(addr, value.asBoolean.value) override def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SBooleanValue = diff --git a/hail/hail/src/is/hail/types/physical/PCanonicalArray.scala b/hail/hail/src/is/hail/types/physical/PCanonicalArray.scala index 7e9b1a17d94..ea32428c78a 100644 --- a/hail/hail/src/is/hail/types/physical/PCanonicalArray.scala +++ b/hail/hail/src/is/hail/types/physical/PCanonicalArray.scala @@ -15,9 +15,9 @@ final case class PCanonicalArray(elementType: PType, required: Boolean = false) extends PArray with PCanonicalArrayBackedContainer { assert(elementType.isRealizable) - def arrayRep = this + override def arrayRep = this - def _asIdent = s"array_of_${elementType.asIdent}" + override def _asIdent = s"array_of_${elementType.asIdent}" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false): Unit = { sb ++= "PCArray[" @@ -55,7 +55,7 @@ final case class PCanonicalArray(elementType: PType, required: Boolean = false) override val byteSize: Long = 8 - def setRequired(required: Boolean): PCanonicalArray = + override def setRequired(required: Boolean): PCanonicalArray = if (required == this.required) this else PCanonicalArray(elementType, required) override def loadLength(aoff: Long): Int = @@ -219,16 +219,16 @@ final case class PCanonicalArray(elementType: PType, required: Boolean = false) length, ) - def hasNext: Boolean = i != length - def isDefined: Boolean = isElementDefined(aoff, i) + override def hasNext: Boolean = i != length + override def isDefined: Boolean = isElementDefined(aoff, i) - def value: Long = + override def value: Long = firstElementOffset + i * elementByteSize - def iterate(): Unit = i += 1 + override def iterate(): Unit = i += 1 } - def elementIterator(aoff: Long, length: Int): Iterator = new Iterator(aoff, length) + override def elementIterator(aoff: Long, length: Int): Iterator = new Iterator(aoff, length) override def allocate(region: Region, length: Int): Long = region.allocate(contentsAlignment, contentsByteSize(length)) diff --git a/hail/hail/src/is/hail/types/physical/PCanonicalBinary.scala b/hail/hail/src/is/hail/types/physical/PCanonicalBinary.scala index 88889112c89..72121e420b5 100644 --- a/hail/hail/src/is/hail/types/physical/PCanonicalBinary.scala +++ b/hail/hail/src/is/hail/types/physical/PCanonicalBinary.scala @@ -13,7 +13,7 @@ case object PCanonicalBinaryOptional extends PCanonicalBinary(false) case object PCanonicalBinaryRequired extends PCanonicalBinary(true) class PCanonicalBinary(val required: Boolean) extends PBinary { - def _asIdent = "binary" + override def _asIdent = "binary" override def copiedType: PType = this @@ -53,48 +53,48 @@ class PCanonicalBinary(val required: Boolean) extends PBinary { override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = sb ++= "PCBinary" - def contentAlignment: Long = 4 + override def contentAlignment: Long = 4 - def lengthHeaderBytes: Long = 4 + override def lengthHeaderBytes: Long = 4 - def contentByteSize(length: Int): Long = 4L + length + override def contentByteSize(length: Int): Long = 4L + length - def contentByteSize(length: Code[Int]): Code[Long] = (const(4) + length).toL + override def contentByteSize(length: Code[Int]): Code[Long] = (const(4) + length).toL - def allocate(region: Region, length: Int): Long = + override def allocate(region: Region, length: Int): Long = region.allocate(contentAlignment, contentByteSize(length)) - def allocate(region: Code[Region], length: Code[Int]): Code[Long] = + override def allocate(region: Code[Region], length: Code[Int]): Code[Long] = region.allocate(const(contentAlignment), contentByteSize(length)) - def loadLength(boff: Long): Int = Region.loadInt(boff) + override def loadLength(boff: Long): Int = Region.loadInt(boff) - def loadLength(boff: Code[Long]): Code[Int] = Region.loadInt(boff) + override def loadLength(boff: Code[Long]): Code[Int] = Region.loadInt(boff) - def loadBytes(bAddress: Code[Long], length: Code[Int]): Code[Array[Byte]] = + override def loadBytes(bAddress: Code[Long], length: Code[Int]): Code[Array[Byte]] = Region.loadBytes(this.bytesAddress(bAddress), length) - def loadBytes(bAddress: Code[Long]): Code[Array[Byte]] = + override def loadBytes(bAddress: Code[Long]): Code[Array[Byte]] = Code.memoize(bAddress, "pcbin_load_bytes_addr") { bAddress => loadBytes(bAddress, this.loadLength(bAddress)) } - def loadBytes(bAddress: Long, length: Int): Array[Byte] = + override def loadBytes(bAddress: Long, length: Int): Array[Byte] = Region.loadBytes(this.bytesAddress(bAddress), length) - def loadBytes(bAddress: Long): Array[Byte] = + override def loadBytes(bAddress: Long): Array[Byte] = this.loadBytes(bAddress, this.loadLength(bAddress)) - def storeLength(boff: Long, len: Int): Unit = Region.storeInt(boff, len) + override def storeLength(boff: Long, len: Int): Unit = Region.storeInt(boff, len) - def storeLength(cb: EmitCodeBuilder, boff: Code[Long], len: Code[Int]): Unit = + override def storeLength(cb: EmitCodeBuilder, boff: Code[Long], len: Code[Int]): Unit = cb += Region.storeInt(boff, len) - def bytesAddress(boff: Long): Long = boff + lengthHeaderBytes + override def bytesAddress(boff: Long): Long = boff + lengthHeaderBytes - def bytesAddress(boff: Code[Long]): Code[Long] = boff + lengthHeaderBytes + override def bytesAddress(boff: Code[Long]): Code[Long] = boff + lengthHeaderBytes - def store(addr: Long, bytes: Array[Byte]): Unit = { + override def store(addr: Long, bytes: Array[Byte]): Unit = { Region.storeInt(addr, bytes.length) Region.storeBytes(bytesAddress(addr), bytes) } @@ -163,12 +163,12 @@ class PCanonicalBinary(val required: Boolean) extends PBinary { } } - def sType: SBinaryPointer = SBinaryPointer(setRequired(false)) + override def sType: SBinaryPointer = SBinaryPointer(setRequired(false)) - def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SBinaryPointerValue = + override def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SBinaryPointerValue = new SBinaryPointerValue(sType, cb.memoize(addr)) - def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) + override def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) : Value[Long] = { value.st match { case SBinaryPointer(PCanonicalBinary(_)) => @@ -191,7 +191,7 @@ class PCanonicalBinary(val required: Boolean) extends PBinary { } } - def storeAtAddress( + override def storeAtAddress( cb: EmitCodeBuilder, addr: Code[Long], region: Value[Region], @@ -200,7 +200,7 @@ class PCanonicalBinary(val required: Boolean) extends PBinary { ): Unit = cb += Region.storeAddress(addr, store(cb, region, value, deepCopy)) - def unstagedStoreAtAddress( + override def unstagedStoreAtAddress( sm: HailStateManager, addr: Long, region: Region, @@ -212,10 +212,10 @@ class PCanonicalBinary(val required: Boolean) extends PBinary { Region.storeAddress(addr, copyFromAddress(sm, region, srcArray, srcAddress, deepCopy)) } - def setRequired(required: Boolean): PCanonicalBinary = + override def setRequired(required: Boolean): PCanonicalBinary = if (required == this.required) this else PCanonicalBinary(required) - def loadFromNested(addr: Code[Long]): Code[Long] = Region.loadAddress(addr) + override def loadFromNested(addr: Code[Long]): Code[Long] = Region.loadAddress(addr) override def unstagedLoadFromNested(addr: Long): Long = Region.loadAddress(addr) diff --git a/hail/hail/src/is/hail/types/physical/PCanonicalCall.scala b/hail/hail/src/is/hail/types/physical/PCanonicalCall.scala index c8c5bb49b05..88cbf0be3cd 100644 --- a/hail/hail/src/is/hail/types/physical/PCanonicalCall.scala +++ b/hail/hail/src/is/hail/types/physical/PCanonicalCall.scala @@ -10,7 +10,7 @@ import is.hail.types.physical.stypes.interfaces.SCall import is.hail.utils._ final case class PCanonicalCall(required: Boolean = false) extends PCall { - def _asIdent = "call" + override def _asIdent = "call" override def copiedType: PType = this @@ -19,13 +19,13 @@ final case class PCanonicalCall(required: Boolean = false) extends PCall { val representation: PInt32 = PInt32(required) - def byteSize: Long = representation.byteSize + override def byteSize: Long = representation.byteSize override def alignment: Long = representation.alignment override def unsafeOrdering(sm: HailStateManager): UnsafeOrdering = representation.unsafeOrdering(sm) // this was a terrible idea - def setRequired(required: Boolean) = + override def setRequired(required: Boolean) = if (required == this.required) this else PCanonicalCall(required) override def unstagedStoreAtAddress( @@ -62,12 +62,12 @@ final case class PCanonicalCall(required: Boolean = false) extends PCall { representation._copyFromAddress(sm, region, pt.representation, srcAddress, deepCopy) } - def sType: SCall = SCanonicalCall + override def sType: SCall = SCanonicalCall - def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SCanonicalCallValue = + override def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SCanonicalCallValue = new SCanonicalCallValue(cb.memoize(Region.loadInt(addr))) - def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) + override def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) : Value[Long] = { value.st match { case SCanonicalCall => @@ -77,7 +77,7 @@ final case class PCanonicalCall(required: Boolean = false) extends PCall { } } - def storeAtAddress( + override def storeAtAddress( cb: EmitCodeBuilder, addr: Code[Long], region: Value[Region], @@ -86,7 +86,7 @@ final case class PCanonicalCall(required: Boolean = false) extends PCall { ): Unit = cb += Region.storeInt(addr, value.asCall.canonicalCall(cb)) - def loadFromNested(addr: Code[Long]): Code[Long] = representation.loadFromNested(addr) + override def loadFromNested(addr: Code[Long]): Code[Long] = representation.loadFromNested(addr) override def unstagedLoadFromNested(addr: Long): Long = representation.unstagedLoadFromNested(addr) diff --git a/hail/hail/src/is/hail/types/physical/PCanonicalDict.scala b/hail/hail/src/is/hail/types/physical/PCanonicalDict.scala index 90c2c5e4f3d..908784f7e44 100644 --- a/hail/hail/src/is/hail/types/physical/PCanonicalDict.scala +++ b/hail/hail/src/is/hail/types/physical/PCanonicalDict.scala @@ -23,10 +23,10 @@ final case class PCanonicalDict(keyType: PType, valueType: PType, required: Bool val arrayRep: PCanonicalArray = PCanonicalArray(elementType, required) - def setRequired(required: Boolean) = + override def setRequired(required: Boolean) = if (required == this.required) this else PCanonicalDict(keyType, valueType, required) - def _asIdent = s"dict_of_${keyType.asIdent}AND${valueType.asIdent}" + override def _asIdent = s"dict_of_${keyType.asIdent}AND${valueType.asIdent}" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false): Unit = { sb ++= "PCDict[" diff --git a/hail/hail/src/is/hail/types/physical/PCanonicalLocus.scala b/hail/hail/src/is/hail/types/physical/PCanonicalLocus.scala index 64f1fe78313..6d7249bbf6f 100644 --- a/hail/hail/src/is/hail/types/physical/PCanonicalLocus.scala +++ b/hail/hail/src/is/hail/types/physical/PCanonicalLocus.scala @@ -27,19 +27,19 @@ object PCanonicalLocus { final case class PCanonicalLocus(rgName: String, required: Boolean = false) extends PLocus { - def byteSize: Long = representation.byteSize + override def byteSize: Long = representation.byteSize override def alignment: Long = representation.alignment override def copiedType: PType = this - def rg: String = rgName + override def rg: String = rgName - def _asIdent = s"locus_$rgName" + override def _asIdent = s"locus_$rgName" override def _pretty(sb: StringBuilder, indent: Call, compact: Boolean): Unit = sb ++= "PCLocus(" ++= rgName += ')': Unit - def setRequired(required: Boolean): PCanonicalLocus = + override def setRequired(required: Boolean): PCanonicalLocus = if (required == this.required) this else PCanonicalLocus(this.rgName, required) val representation: PCanonicalStruct = PCanonicalLocus.representation(required) @@ -49,13 +49,14 @@ final case class PCanonicalLocus(rgName: String, required: Boolean = false) exte private[physical] def contigAddr(address: Long): Long = representation.loadField(address, 0) - def contig(address: Long): String = contigType.loadString(contigAddr(address)) - def position(address: Long): Int = Region.loadInt(representation.fieldOffset(address, 1)) + override def contig(address: Long): String = contigType.loadString(contigAddr(address)) + override def position(address: Long): Int = Region.loadInt(representation.fieldOffset(address, 1)) lazy val contigType: PCanonicalString = representation.field("contig").typ.asInstanceOf[PCanonicalString] - def position(off: Code[Long]): Code[Int] = Region.loadInt(representation.loadField(off, 1)) + override def position(off: Code[Long]): Code[Int] = + Region.loadInt(representation.loadField(off, 1)) lazy val positionType: PInt32 = representation.field("position").typ.asInstanceOf[PInt32] @@ -65,7 +66,7 @@ final case class PCanonicalLocus(rgName: String, required: Boolean = false) exte val binaryOrd = representation.fieldType("contig").unsafeOrdering(sm) new UnsafeOrdering { - def compare(o1: Long, o2: Long): Int = { + override def compare(o1: Long, o2: Long): Int = { val cOff1 = representation.loadField(o1, 0) val cOff2 = representation.loadField(o2, 0) @@ -115,14 +116,15 @@ final case class PCanonicalLocus(rgName: String, required: Boolean = false) exte representation._copyFromAddress(sm, region, pt.representation, srcAddress, deepCopy) } - def sType: SCanonicalLocusPointer = SCanonicalLocusPointer(setRequired(false)) + override def sType: SCanonicalLocusPointer = SCanonicalLocusPointer(setRequired(false)) - def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SCanonicalLocusPointerValue = { + override def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]) + : SCanonicalLocusPointerValue = { val a = cb.memoize(addr) new SCanonicalLocusPointerValue(sType, a, cb.memoize(contigAddr(a)), cb.memoize(position(a))) } - def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) + override def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) : Value[Long] = { value.st match { case SCanonicalLocusPointer(pt) => @@ -139,7 +141,7 @@ final case class PCanonicalLocus(rgName: String, required: Boolean = false) exte } } - def storeAtAddress( + override def storeAtAddress( cb: EmitCodeBuilder, addr: Code[Long], region: Value[Region], @@ -173,7 +175,7 @@ final case class PCanonicalLocus(rgName: String, required: Boolean = false) exte } } - def loadFromNested(addr: Code[Long]): Code[Long] = addr + override def loadFromNested(addr: Code[Long]): Code[Long] = addr override def unstagedLoadFromNested(addr: Long): Long = addr diff --git a/hail/hail/src/is/hail/types/physical/PCanonicalNDArray.scala b/hail/hail/src/is/hail/types/physical/PCanonicalNDArray.scala index 8f81ebe4fff..984a9219445 100644 --- a/hail/hail/src/is/hail/types/physical/PCanonicalNDArray.scala +++ b/hail/hail/src/is/hail/types/physical/PCanonicalNDArray.scala @@ -39,7 +39,7 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo lazy val strideType: PCanonicalTuple = shapeType - def loadShape(ndAddr: Long, idx: Int): Long = { + override def loadShape(ndAddr: Long, idx: Int): Long = { val shapeTupleAddr = representation.loadField(ndAddr, 0) Region.loadLong(shapeType.loadField(shapeTupleAddr, idx)) } @@ -49,8 +49,11 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo Region.loadLong(strideType.loadField(shapeTupleAddr, idx)) } - def loadShapes(cb: EmitCodeBuilder, addr: Value[Long], settables: IndexedSeq[Settable[Long]]) - : Unit = { + override def loadShapes( + cb: EmitCodeBuilder, + addr: Value[Long], + settables: IndexedSeq[Settable[Long]], + ): Unit = { assert(settables.length == nDims, s"got ${settables.length} settables, expect $nDims dims") val shapeTuple = shapeType.loadCheapSCode(cb, representation.loadField(addr, "shape")) (0 until nDims).foreach { dimIdx => @@ -58,8 +61,11 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo } } - def loadStrides(cb: EmitCodeBuilder, addr: Value[Long], settables: IndexedSeq[Settable[Long]]) - : Unit = { + override def loadStrides( + cb: EmitCodeBuilder, + addr: Value[Long], + settables: IndexedSeq[Settable[Long]], + ): Unit = { assert(settables.length == nDims) val strideTuple = strideType.loadCheapSCode(cb, representation.loadField(addr, "strides")) (0 until nDims).foreach { dimIdx => @@ -85,14 +91,16 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo override def unsafeOrdering(sm: HailStateManager): UnsafeOrdering = representation.unsafeOrdering(sm) - def numElements(shape: IndexedSeq[Value[Long]]): Code[Long] = + override def numElements(shape: IndexedSeq[Value[Long]]): Code[Long] = shape.foldLeft(1L: Code[Long])(_ * _) def numElements(shape: IndexedSeq[Long]): Long = shape.foldLeft(1L)(_ * _) - def makeColumnMajorStrides(sourceShapeArray: IndexedSeq[Value[Long]], cb: EmitCodeBuilder) - : IndexedSeq[Value[Long]] = { + override def makeColumnMajorStrides( + sourceShapeArray: IndexedSeq[Value[Long]], + cb: EmitCodeBuilder, + ): IndexedSeq[Value[Long]] = { val strides = new Array[Value[Long]](nDims) for (i <- 0 until nDims) if (i == 0) strides(i) = const(elementType.byteSize) @@ -102,7 +110,7 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo strides } - def makeRowMajorStrides(sourceShapeArray: IndexedSeq[Value[Long]], cb: EmitCodeBuilder) + override def makeRowMajorStrides(sourceShapeArray: IndexedSeq[Value[Long]], cb: EmitCodeBuilder) : IndexedSeq[Value[Long]] = { val strides = new Array[Value[Long]](nDims) for (i <- (nDims - 1) to 0 by -1) @@ -113,7 +121,7 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo strides } - def getElementAddress(indices: IndexedSeq[Long], nd: Long): Long = { + override def getElementAddress(indices: IndexedSeq[Long], nd: Long): Long = { var bytesAway = 0L indices.zipWithIndex.foreach { case (requestedIndex: Long, strideIndex: Int) => bytesAway += requestedIndex * loadStride(nd, strideIndex) @@ -168,8 +176,11 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo address } - def loadElement(cb: EmitCodeBuilder, indices: IndexedSeq[Value[Long]], ndAddress: Value[Long]) - : SValue = { + override def loadElement( + cb: EmitCodeBuilder, + indices: IndexedSeq[Value[Long]], + ndAddress: Value[Long], + ): SValue = { val off = getElementAddress(cb, indices, ndAddress) elementType.loadCheapSCode(cb, elementType.loadFromNested(off)) } @@ -220,7 +231,7 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo region, ) - def constructByCopyingArray( + override def constructByCopyingArray( shape: IndexedSeq[Value[Long]], strides: IndexedSeq[Value[Long]], dataCode: SIndexableValue, @@ -293,7 +304,7 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo .coerceToShape(cb, newShape) } - def constructDataFunction( + override def constructDataFunction( shape: IndexedSeq[Value[Long]], strides: IndexedSeq[Value[Long]], cb: EmitCodeBuilder, @@ -371,7 +382,7 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo ) } - def _copyFromAddress( + override def _copyFromAddress( sm: HailStateManager, region: Region, srcPType: PType, @@ -395,10 +406,10 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo private def deepRenameNDArray(t: TNDArray) = PCanonicalNDArray(this.elementType.deepRename(t.elementType), this.nDims, this.required) - def setRequired(required: Boolean): PCanonicalNDArray = + override def setRequired(required: Boolean): PCanonicalNDArray = if (required == this.required) this else PCanonicalNDArray(elementType, nDims, required) - def unstagedStoreAtAddress( + override def unstagedStoreAtAddress( sm: HailStateManager, destAddress: Long, region: Region, @@ -463,9 +474,9 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo } } - def sType: SNDArrayPointer = SNDArrayPointer(setRequired(false)) + override def sType: SNDArrayPointer = SNDArrayPointer(setRequired(false)) - def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SNDArrayPointerValue = { + override def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SNDArrayPointerValue = { val a = cb.memoize(addr) val shapeTuple = shapeType.loadCheapSCode(cb, representation.loadField(a, "shape")) val shape = @@ -478,14 +489,14 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo new SNDArrayPointerValue(sType, a, shape, strides, firstDataAddress) } - def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) + override def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) : Value[Long] = { val addr = cb.memoize(this.representation.allocate(region)) storeAtAddress(cb, addr, region, value, deepCopy) addr } - def storeAtAddress( + override def storeAtAddress( cb: EmitCodeBuilder, addr: Code[Long], region: Value[Region], @@ -548,7 +559,7 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo override def dataFirstElementPointer(ndAddr: Code[Long]): Code[Long] = Region.loadAddress(representation.loadField(ndAddr, "data")) - def loadFromNested(addr: Code[Long]): Code[Long] = addr + override def loadFromNested(addr: Code[Long]): Code[Long] = addr override def unstagedLoadFromNested(addr: Long): Long = addr diff --git a/hail/hail/src/is/hail/types/physical/PCanonicalSet.scala b/hail/hail/src/is/hail/types/physical/PCanonicalSet.scala index f5f516672ab..1a16d0c0a79 100644 --- a/hail/hail/src/is/hail/types/physical/PCanonicalSet.scala +++ b/hail/hail/src/is/hail/types/physical/PCanonicalSet.scala @@ -19,10 +19,10 @@ final case class PCanonicalSet(elementType: PType, required: Boolean = false) extends PSet with PCanonicalArrayBackedContainer { val arrayRep = PCanonicalArray(elementType, required) - def setRequired(required: Boolean) = + override def setRequired(required: Boolean) = if (required == this.required) this else PCanonicalSet(elementType, required) - def _asIdent = s"set_of_${elementType.asIdent}" + override def _asIdent = s"set_of_${elementType.asIdent}" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false): Unit = { sb ++= "PCSet[" diff --git a/hail/hail/src/is/hail/types/physical/PCanonicalString.scala b/hail/hail/src/is/hail/types/physical/PCanonicalString.scala index ebce82c4403..4c425ff8f56 100644 --- a/hail/hail/src/is/hail/types/physical/PCanonicalString.scala +++ b/hail/hail/src/is/hail/types/physical/PCanonicalString.scala @@ -12,7 +12,7 @@ case object PCanonicalStringOptional extends PCanonicalString(false) case object PCanonicalStringRequired extends PCanonicalString(true) class PCanonicalString(val required: Boolean) extends PString { - def _asIdent = "string" + override def _asIdent = "string" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = sb ++= "PCString" @@ -40,26 +40,26 @@ class PCanonicalString(val required: Boolean) extends PString { override def containsPointers: Boolean = true - def loadLength(boff: Long): Int = + override def loadLength(boff: Long): Int = this.binaryRepresentation.loadLength(boff) - def loadLength(boff: Code[Long]): Code[Int] = + override def loadLength(boff: Code[Long]): Code[Int] = this.binaryRepresentation.loadLength(boff) - def loadString(bAddress: Long): String = + override def loadString(bAddress: Long): String = new String(this.binaryRepresentation.loadBytes(bAddress)) - def loadString(bAddress: Code[Long]): Code[String] = + override def loadString(bAddress: Code[Long]): Code[String] = Code.newInstance[String, Array[Byte]](this.binaryRepresentation.loadBytes(bAddress)) - def allocateAndStoreString(region: Region, str: String): Long = { + override def allocateAndStoreString(region: Region, str: String): Long = { val byteRep = str.getBytes() val dstAddrss = this.binaryRepresentation.allocate(region, byteRep.length) this.binaryRepresentation.store(dstAddrss, byteRep) dstAddrss } - def allocateAndStoreString(cb: EmitCodeBuilder, region: Value[Region], str: Code[String]) + override def allocateAndStoreString(cb: EmitCodeBuilder, region: Value[Region], str: Code[String]) : Value[Long] = { val dstAddress = cb.newField[Long]("pcanonical_string_alloc_dst_address") val byteRep = cb.newField[Array[Byte]]("pcanonical_string_alloc_byte_rep") @@ -86,15 +86,15 @@ class PCanonicalString(val required: Boolean) extends PString { deepCopy, ) - def setRequired(required: Boolean): PCanonicalString = + override def setRequired(required: Boolean): PCanonicalString = if (required == this.required) this else PCanonicalString(required) - def sType: SStringPointer = SStringPointer(setRequired(false)) + override def sType: SStringPointer = SStringPointer(setRequired(false)) - def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SStringPointerValue = + override def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SStringPointerValue = new SStringPointerValue(sType, cb.memoize(addr)) - def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) + override def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) : Value[Long] = { value.st match { case SStringPointer(t) if t.equalModuloRequired(this) && !deepCopy => @@ -104,7 +104,7 @@ class PCanonicalString(val required: Boolean) extends PString { } } - def storeAtAddress( + override def storeAtAddress( cb: EmitCodeBuilder, addr: Code[Long], region: Value[Region], @@ -113,7 +113,8 @@ class PCanonicalString(val required: Boolean) extends PString { ): Unit = cb += Region.storeAddress(addr, store(cb, region, value, deepCopy)) - def loadFromNested(addr: Code[Long]): Code[Long] = binaryRepresentation.loadFromNested(addr) + override def loadFromNested(addr: Code[Long]): Code[Long] = + binaryRepresentation.loadFromNested(addr) override def unstagedLoadFromNested(addr: Long): Long = binaryRepresentation.unstagedLoadFromNested(addr) diff --git a/hail/hail/src/is/hail/types/physical/PCanonicalTuple.scala b/hail/hail/src/is/hail/types/physical/PCanonicalTuple.scala index f824d96338c..efffa544e46 100644 --- a/hail/hail/src/is/hail/types/physical/PCanonicalTuple.scala +++ b/hail/hail/src/is/hail/types/physical/PCanonicalTuple.scala @@ -20,7 +20,7 @@ final case class PCanonicalTuple( tf.index -> idx }.toMap - def setRequired(required: Boolean) = + override def setRequired(required: Boolean) = if (required == this.required) this else PCanonicalTuple(_types, required) override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = { @@ -44,7 +44,7 @@ final case class PCanonicalTuple( this.required, ) - def copiedType: PType = { + override def copiedType: PType = { val copiedTypes = types.map(_.copiedType) if (types.indices.forall(i => types(i).eq(copiedTypes(i)))) this diff --git a/hail/hail/src/is/hail/types/physical/PDict.scala b/hail/hail/src/is/hail/types/physical/PDict.scala index c6cc0996ffd..2aba8dd609f 100644 --- a/hail/hail/src/is/hail/types/physical/PDict.scala +++ b/hail/hail/src/is/hail/types/physical/PDict.scala @@ -9,7 +9,7 @@ abstract class PDict extends PContainer { val keyType: PType val valueType: PType - def sType: SContainer + override def sType: SContainer - def elementType: PStruct + override def elementType: PStruct } diff --git a/hail/hail/src/is/hail/types/physical/PFloat32.scala b/hail/hail/src/is/hail/types/physical/PFloat32.scala index bc8c07558c3..21bc8c8dae2 100644 --- a/hail/hail/src/is/hail/types/physical/PFloat32.scala +++ b/hail/hail/src/is/hail/types/physical/PFloat32.scala @@ -16,13 +16,13 @@ class PFloat32(override val required: Boolean) extends PNumeric with PPrimitive override type NType = PFloat32 - def _asIdent = "float32" + override def _asIdent = "float32" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = sb ++= "PFloat32" override def unsafeOrdering(sm: HailStateManager): UnsafeOrdering = new UnsafeOrdering { - def compare(o1: Long, o2: Long): Int = + override def compare(o1: Long, o2: Long): Int = java.lang.Float.compare(Region.loadFloat(o1), Region.loadFloat(o2)) } @@ -38,7 +38,7 @@ class PFloat32(override val required: Boolean) extends PNumeric with PPrimitive override def sType: SType = SFloat32 - def storePrimitiveAtAddress(cb: EmitCodeBuilder, addr: Code[Long], value: SValue): Unit = + override def storePrimitiveAtAddress(cb: EmitCodeBuilder, addr: Code[Long], value: SValue): Unit = cb.append(Region.storeFloat(addr, value.asFloat.value)) override def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SFloat32Value = diff --git a/hail/hail/src/is/hail/types/physical/PFloat64.scala b/hail/hail/src/is/hail/types/physical/PFloat64.scala index 04dd6186fcf..3e6454f1407 100644 --- a/hail/hail/src/is/hail/types/physical/PFloat64.scala +++ b/hail/hail/src/is/hail/types/physical/PFloat64.scala @@ -17,13 +17,13 @@ class PFloat64(override val required: Boolean) extends PNumeric with PPrimitive override type NType = PFloat64 - def _asIdent = "float64" + override def _asIdent = "float64" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = sb ++= "PFloat64" override def unsafeOrdering(sm: HailStateManager): UnsafeOrdering = new UnsafeOrdering { - def compare(o1: Long, o2: Long): Int = + override def compare(o1: Long, o2: Long): Int = java.lang.Double.compare(Region.loadDouble(o1), Region.loadDouble(o2)) } @@ -39,7 +39,7 @@ class PFloat64(override val required: Boolean) extends PNumeric with PPrimitive override def sType: SType = SFloat64 - def storePrimitiveAtAddress(cb: EmitCodeBuilder, addr: Code[Long], value: SValue): Unit = + override def storePrimitiveAtAddress(cb: EmitCodeBuilder, addr: Code[Long], value: SValue): Unit = cb.append(Region.storeDouble(addr, value.asDouble.value)) override def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SFloat64Value = diff --git a/hail/hail/src/is/hail/types/physical/PInt32.scala b/hail/hail/src/is/hail/types/physical/PInt32.scala index 6ed69161613..de80955df40 100644 --- a/hail/hail/src/is/hail/types/physical/PInt32.scala +++ b/hail/hail/src/is/hail/types/physical/PInt32.scala @@ -13,7 +13,7 @@ case object PInt32Required extends PInt32(true) class PInt32(override val required: Boolean) extends PNumeric with PPrimitive { lazy val virtualType: TInt32.type = TInt32 - def _asIdent = "int32" + override def _asIdent = "int32" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = sb ++= "PInt32" @@ -21,7 +21,7 @@ class PInt32(override val required: Boolean) extends PNumeric with PPrimitive { override type NType = PInt32 override def unsafeOrdering(sm: HailStateManager): UnsafeOrdering = new UnsafeOrdering { - def compare(o1: Long, o2: Long): Int = + override def compare(o1: Long, o2: Long): Int = Integer.compare(Region.loadInt(o1), Region.loadInt(o2)) } @@ -37,7 +37,7 @@ class PInt32(override val required: Boolean) extends PNumeric with PPrimitive { override def sType: SType = SInt32 - def storePrimitiveAtAddress(cb: EmitCodeBuilder, addr: Code[Long], value: SValue): Unit = + override def storePrimitiveAtAddress(cb: EmitCodeBuilder, addr: Code[Long], value: SValue): Unit = cb.append(Region.storeInt(addr, value.asInt.value)) override def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SInt32Value = diff --git a/hail/hail/src/is/hail/types/physical/PInt64.scala b/hail/hail/src/is/hail/types/physical/PInt64.scala index 7b90f36ba57..9020c6b6ba4 100644 --- a/hail/hail/src/is/hail/types/physical/PInt64.scala +++ b/hail/hail/src/is/hail/types/physical/PInt64.scala @@ -14,7 +14,7 @@ case object PInt64Required extends PInt64(true) class PInt64(override val required: Boolean) extends PNumeric with PPrimitive { lazy val virtualType: TInt64.type = TInt64 - def _asIdent = "int64" + override def _asIdent = "int64" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = sb ++= "PInt64" @@ -22,7 +22,7 @@ class PInt64(override val required: Boolean) extends PNumeric with PPrimitive { override type NType = PInt64 override def unsafeOrdering(sm: HailStateManager): UnsafeOrdering = new UnsafeOrdering { - def compare(o1: Long, o2: Long): Int = + override def compare(o1: Long, o2: Long): Int = java.lang.Long.compare(Region.loadLong(o1), Region.loadLong(o2)) } @@ -38,7 +38,7 @@ class PInt64(override val required: Boolean) extends PNumeric with PPrimitive { override def sType: SType = SInt64 - def storePrimitiveAtAddress(cb: EmitCodeBuilder, addr: Code[Long], value: SValue): Unit = + override def storePrimitiveAtAddress(cb: EmitCodeBuilder, addr: Code[Long], value: SValue): Unit = cb.append(Region.storeLong(addr, value.asLong.value)) override def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SInt64Value = diff --git a/hail/hail/src/is/hail/types/physical/PInterval.scala b/hail/hail/src/is/hail/types/physical/PInterval.scala index 0e9a7b1a927..401ad762861 100644 --- a/hail/hail/src/is/hail/types/physical/PInterval.scala +++ b/hail/hail/src/is/hail/types/physical/PInterval.scala @@ -15,7 +15,7 @@ abstract class PInterval extends PType { new UnsafeOrdering { private val pOrd = pointType.unsafeOrdering(sm) - def compare(o1: Long, o2: Long): Int = { + override def compare(o1: Long, o2: Long): Int = { val sdef1 = startDefined(o1) if (sdef1 == startDefined(o2)) { val cmp = pOrd.compare(loadStart(o1), loadStart(o2)) @@ -47,7 +47,7 @@ abstract class PInterval extends PType { new UnsafeOrdering { private val pOrd = pointType.unsafeOrdering(sm) - def compare(o1: Long, o2: Long): Int = { + override def compare(o1: Long, o2: Long): Int = { val edef1 = endDefined(o1) if (edef1 == endDefined(o2)) { val cmp = pOrd.compare(loadEnd(o1), loadEnd(o2)) diff --git a/hail/hail/src/is/hail/types/physical/PPrimitive.scala b/hail/hail/src/is/hail/types/physical/PPrimitive.scala index 5ad5b64a874..d7b7324fc56 100644 --- a/hail/hail/src/is/hail/types/physical/PPrimitive.scala +++ b/hail/hail/src/is/hail/types/physical/PPrimitive.scala @@ -8,13 +8,11 @@ import is.hail.types.physical.stypes.SValue import is.hail.utils._ trait PPrimitive extends PType { - def byteSize: Long - def _construct(mb: EmitMethodBuilder[_], region: Value[Region], pc: SValue): SValue = pc override def containsPointers: Boolean = false - def _copyFromAddress( + override def _copyFromAddress( sm: HailStateManager, region: Region, srcPType: PType, @@ -30,7 +28,7 @@ trait PPrimitive extends PType { addr } - def unstagedStoreAtAddress( + override def unstagedStoreAtAddress( sm: HailStateManager, addr: Long, region: Region, @@ -42,7 +40,7 @@ trait PPrimitive extends PType { Region.copyFrom(srcAddress, addr, byteSize) } - def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) + override def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) : Value[Long] = { val newAddr = cb.memoize(region.allocate(alignment, byteSize)) storeAtAddress(cb, newAddr, region, value, deepCopy) @@ -67,7 +65,7 @@ trait PPrimitive extends PType { addr } - def setRequired(required: Boolean): PPrimitive = { + override def setRequired(required: Boolean): PPrimitive = { if (required == this.required) this else @@ -80,7 +78,7 @@ trait PPrimitive extends PType { } } - def loadFromNested(addr: Code[Long]): Code[Long] = addr + override def loadFromNested(addr: Code[Long]): Code[Long] = addr override def unstagedLoadFromNested(addr: Long): Long = addr diff --git a/hail/hail/src/is/hail/types/physical/PStruct.scala b/hail/hail/src/is/hail/types/physical/PStruct.scala index 23d7eb4d8f9..0422aca8171 100644 --- a/hail/hail/src/is/hail/types/physical/PStruct.scala +++ b/hail/hail/src/is/hail/types/physical/PStruct.scala @@ -2,7 +2,7 @@ package is.hail.types.physical import is.hail.asm4s.{Code, Value} import is.hail.expr.ir.EmitCodeBuilder -import is.hail.types.physical.stypes.interfaces.{SBaseStruct, SBaseStructValue} +import is.hail.types.physical.stypes.interfaces.SBaseStruct import is.hail.types.virtual.{Field, TStruct} import scala.collection.compat._ @@ -35,7 +35,7 @@ trait PStruct extends PBaseStruct { def rename(m: Map[String, String]): PStruct - def identBase: String = "struct" + override def identBase: String = "struct" final def selectFields(names: Seq[String]): PCanonicalStruct = PCanonicalStruct(required, names.map(f => f -> field(f).typ): _*) @@ -61,6 +61,4 @@ trait PStruct extends PBaseStruct { def setFieldMissing(cb: EmitCodeBuilder, offset: Code[Long], fieldName: String): Unit def insertFields(fieldsToInsert: IterableOnce[(String, PType)]): PStruct - - def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SBaseStructValue } diff --git a/hail/hail/src/is/hail/types/physical/PSubsetStruct.scala b/hail/hail/src/is/hail/types/physical/PSubsetStruct.scala index 14695382838..adb382b11bc 100644 --- a/hail/hail/src/is/hail/types/physical/PSubsetStruct.scala +++ b/hail/hail/src/is/hail/types/physical/PSubsetStruct.scala @@ -79,7 +79,7 @@ final case class PSubsetStruct(ps: PStruct, _fieldNames: IndexedSeq[String]) override def fieldOffset(structAddress: Code[Long], fieldIdx: Int): Code[Long] = ps.fieldOffset(structAddress, idxMap(fieldIdx)) - def loadField(structAddress: Code[Long], fieldName: String): Code[Long] = + override def loadField(structAddress: Code[Long], fieldName: String): Code[Long] = ps.loadField(structAddress, fieldName) override def loadField(structAddress: Long, fieldIdx: Int): Long = @@ -104,7 +104,7 @@ final case class PSubsetStruct(ps: PStruct, _fieldNames: IndexedSeq[String]) override def setFieldPresent(cb: EmitCodeBuilder, structAddress: Code[Long], fieldIdx: Int) : Unit = ??? - def insertFields(fieldsToInsert: IterableOnce[(String, PType)]): PSubsetStruct = ??? + override def insertFields(fieldsToInsert: IterableOnce[(String, PType)]): PSubsetStruct = ??? override def initialize(structAddress: Long, setMissing: Boolean): Unit = ps.initialize(structAddress, setMissing) @@ -113,10 +113,10 @@ final case class PSubsetStruct(ps: PStruct, _fieldNames: IndexedSeq[String]) : Unit = ps.stagedInitialize(cb, structAddress, setMissing) - def allocate(region: Region): Long = + override def allocate(region: Region): Long = ps.allocate(region) - def allocate(region: Code[Region]): Code[Long] = + override def allocate(region: Code[Region]): Code[Long] = ps.allocate(region) override def setRequired(required: Boolean): PType = @@ -140,14 +140,14 @@ final case class PSubsetStruct(ps: PStruct, _fieldNames: IndexedSeq[String]) ): Long = throw new UnsupportedOperationException - def sType: SBaseStruct = + override def sType: SBaseStruct = SStructView.subset(_fieldNames, ps.sType) - def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) + override def store(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) : Value[Long] = throw new UnsupportedOperationException - def storeAtAddress( + override def storeAtAddress( cb: EmitCodeBuilder, addr: Code[Long], region: Value[Region], @@ -156,10 +156,10 @@ final case class PSubsetStruct(ps: PStruct, _fieldNames: IndexedSeq[String]) ): Unit = throw new UnsupportedOperationException - def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SBaseStructValue = + override def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SBaseStructValue = throw new UnsupportedOperationException - def unstagedStoreAtAddress( + override def unstagedStoreAtAddress( sm: HailStateManager, addr: Long, region: Region, @@ -169,7 +169,7 @@ final case class PSubsetStruct(ps: PStruct, _fieldNames: IndexedSeq[String]) ): Unit = throw new UnsupportedOperationException - def loadFromNested(addr: Code[Long]): Code[Long] = addr + override def loadFromNested(addr: Code[Long]): Code[Long] = addr override def unstagedLoadFromNested(addr: Long): Long = addr diff --git a/hail/hail/src/is/hail/types/physical/PTuple.scala b/hail/hail/src/is/hail/types/physical/PTuple.scala index 58310f736fa..9e8fa2c19e2 100644 --- a/hail/hail/src/is/hail/types/physical/PTuple.scala +++ b/hail/hail/src/is/hail/types/physical/PTuple.scala @@ -16,5 +16,5 @@ trait PTuple extends PBaseStruct { lazy val nFields: Int = fields.size - def identBase: String = "tuple" + override def identBase: String = "tuple" } diff --git a/hail/hail/src/is/hail/types/physical/PUnrealizable.scala b/hail/hail/src/is/hail/types/physical/PUnrealizable.scala index 0219ea824af..f484c7cc483 100644 --- a/hail/hail/src/is/hail/types/physical/PUnrealizable.scala +++ b/hail/hail/src/is/hail/types/physical/PUnrealizable.scala @@ -14,7 +14,7 @@ trait PUnrealizable extends PType { override def alignment: Long = unsupported - protected[physical] def _copyFromAddress( + override protected[physical] def _copyFromAddress( sm: HailStateManager, region: Region, srcPType: PType, @@ -32,7 +32,7 @@ trait PUnrealizable extends PType { ): Long = unsupported - def unstagedStoreAtAddress( + override def unstagedStoreAtAddress( sm: HailStateManager, addr: Long, region: Region, diff --git a/hail/hail/src/is/hail/types/physical/PVoid.scala b/hail/hail/src/is/hail/types/physical/PVoid.scala index 0200863f9c0..d74ce38952b 100644 --- a/hail/hail/src/is/hail/types/physical/PVoid.scala +++ b/hail/hail/src/is/hail/types/physical/PVoid.scala @@ -11,21 +11,21 @@ case object PVoid extends PType with PUnrealizable { override def sType: SType = SVoid - def virtualType: Type = TVoid + override def virtualType: Type = TVoid override val required = true - def _asIdent = "void" + override def _asIdent = "void" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = sb ++= "PVoid" - def setRequired(required: Boolean) = PVoid + override def setRequired(required: Boolean) = PVoid override def unsafeOrdering(sm: HailStateManager): UnsafeOrdering = throw new NotImplementedError() - def loadFromNested(addr: Code[Long]): Code[Long] = throw new NotImplementedError() + override def loadFromNested(addr: Code[Long]): Code[Long] = throw new NotImplementedError() override def unstagedLoadFromNested(addr: Long): Long = throw new NotImplementedError() } diff --git a/hail/hail/src/is/hail/types/physical/StoredSTypePType.scala b/hail/hail/src/is/hail/types/physical/StoredSTypePType.scala index 8e47c4ae588..04d1aa7918c 100644 --- a/hail/hail/src/is/hail/types/physical/StoredSTypePType.scala +++ b/hail/hail/src/is/hail/types/physical/StoredSTypePType.scala @@ -117,7 +117,7 @@ case class StoredSTypePType(sType: SType, required: Boolean) extends PType { override def deepRename(t: Type): PType = StoredSTypePType(sType.castRename(t), required) - def byteSize: Long = ct.byteSize + override def byteSize: Long = ct.byteSize override def alignment: Long = ct.alignment @@ -137,12 +137,13 @@ case class StoredSTypePType(sType: SType, required: Boolean) extends PType { override def unsafeOrdering(sm: HailStateManager): UnsafeOrdering = unsupportedCanonicalMethod - def unstagedLoadFromNested(addr: Long): Long = unsupportedCanonicalMethod + override def unstagedLoadFromNested(addr: Long): Long = unsupportedCanonicalMethod - def unstagedStoreJavaObject(sm: HailStateManager, annotation: Annotation, region: Region): Long = + override def unstagedStoreJavaObject(sm: HailStateManager, annotation: Annotation, region: Region) + : Long = unsupportedCanonicalMethod - def unstagedStoreJavaObjectAtAddress( + override def unstagedStoreJavaObjectAtAddress( sm: HailStateManager, addr: Long, annotation: Annotation, diff --git a/hail/hail/src/is/hail/types/physical/stypes/SingleCodeSCode.scala b/hail/hail/src/is/hail/types/physical/stypes/SingleCodeSCode.scala index 334929a0b86..6035c35fae4 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/SingleCodeSCode.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/SingleCodeSCode.scala @@ -46,72 +46,97 @@ sealed trait SingleCodeType { } case object Int32SingleCodeType extends SingleCodeType { - def ti: TypeInfo[_] = IntInfo + override def ti: TypeInfo[_] = IntInfo override def loadedSType: SType = SInt32 - def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = new SInt32Value(coerce[Int](c)) + override def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = + new SInt32Value(coerce[Int](c)) - def virtualType: Type = TInt32 + override def virtualType: Type = TInt32 - def coerceSCode(cb: EmitCodeBuilder, pc: SValue, region: Value[Region], deepCopy: Boolean) - : SingleCodeSCode = + override def coerceSCode( + cb: EmitCodeBuilder, + pc: SValue, + region: Value[Region], + deepCopy: Boolean, + ): SingleCodeSCode = SingleCodeSCode(this, pc.asInt.value) } case object Int64SingleCodeType extends SingleCodeType { - def ti: TypeInfo[_] = LongInfo + override def ti: TypeInfo[_] = LongInfo override def loadedSType: SType = SInt64 - def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = new SInt64Value(coerce[Long](c)) + override def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = + new SInt64Value(coerce[Long](c)) - def virtualType: Type = TInt64 + override def virtualType: Type = TInt64 - def coerceSCode(cb: EmitCodeBuilder, pc: SValue, region: Value[Region], deepCopy: Boolean) - : SingleCodeSCode = + override def coerceSCode( + cb: EmitCodeBuilder, + pc: SValue, + region: Value[Region], + deepCopy: Boolean, + ): SingleCodeSCode = SingleCodeSCode(this, pc.asLong.value) } case object Float32SingleCodeType extends SingleCodeType { - def ti: TypeInfo[_] = FloatInfo + override def ti: TypeInfo[_] = FloatInfo override def loadedSType: SType = SFloat32 - def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = new SFloat32Value(coerce[Float](c)) + override def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = + new SFloat32Value(coerce[Float](c)) - def virtualType: Type = TFloat32 + override def virtualType: Type = TFloat32 - def coerceSCode(cb: EmitCodeBuilder, pc: SValue, region: Value[Region], deepCopy: Boolean) - : SingleCodeSCode = + override def coerceSCode( + cb: EmitCodeBuilder, + pc: SValue, + region: Value[Region], + deepCopy: Boolean, + ): SingleCodeSCode = SingleCodeSCode(this, pc.asFloat.value) } case object Float64SingleCodeType extends SingleCodeType { - def ti: TypeInfo[_] = DoubleInfo + override def ti: TypeInfo[_] = DoubleInfo override def loadedSType: SType = SFloat64 - def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = new SFloat64Value(coerce[Double](c)) + override def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = + new SFloat64Value(coerce[Double](c)) - def virtualType: Type = TFloat64 + override def virtualType: Type = TFloat64 - def coerceSCode(cb: EmitCodeBuilder, pc: SValue, region: Value[Region], deepCopy: Boolean) - : SingleCodeSCode = + override def coerceSCode( + cb: EmitCodeBuilder, + pc: SValue, + region: Value[Region], + deepCopy: Boolean, + ): SingleCodeSCode = SingleCodeSCode(this, pc.asDouble.value) } case object BooleanSingleCodeType extends SingleCodeType { - def ti: TypeInfo[_] = BooleanInfo + override def ti: TypeInfo[_] = BooleanInfo override def loadedSType: SType = SBoolean - def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = new SBooleanValue(coerce[Boolean](c)) + override def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = + new SBooleanValue(coerce[Boolean](c)) - def virtualType: Type = TBoolean + override def virtualType: Type = TBoolean - def coerceSCode(cb: EmitCodeBuilder, pc: SValue, region: Value[Region], deepCopy: Boolean) - : SingleCodeSCode = + override def coerceSCode( + cb: EmitCodeBuilder, + pc: SValue, + region: Value[Region], + deepCopy: Boolean, + ): SingleCodeSCode = SingleCodeSCode(this, pc.asBoolean.value) } @@ -124,33 +149,41 @@ case class StreamSingleCodeType( override def loadedSType: SType = SStream(EmitType(eltType.sType, true)) - def virtualType: Type = TStream(eltType.virtualType) + override def virtualType: Type = TStream(eltType.virtualType) - def ti: TypeInfo[_] = classInfo[NoBoxLongIterator] + override def ti: TypeInfo[_] = classInfo[NoBoxLongIterator] - def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = + override def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = new SStreamConcrete( SStreamIteratorLong(eltRequired, eltType, requiresMemoryManagementPerElement), coerce[NoBoxLongIterator](c), ) - def coerceSCode(cb: EmitCodeBuilder, pc: SValue, region: Value[Region], deepCopy: Boolean) - : SingleCodeSCode = + override def coerceSCode( + cb: EmitCodeBuilder, + pc: SValue, + region: Value[Region], + deepCopy: Boolean, + ): SingleCodeSCode = throw new UnsupportedOperationException } case class PTypeReferenceSingleCodeType(pt: PType) extends SingleCodeType { - def ti: TypeInfo[_] = LongInfo + override def ti: TypeInfo[_] = LongInfo override def loadedSType: SType = pt.sType - def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = + override def loadToSValue(cb: EmitCodeBuilder, c: Value[_]): SValue = cb.memoizeField(pt.loadCheapSCode(cb, coerce[Long](c)), "PTypeReferenceSingleCodeType") - def virtualType: Type = pt.virtualType + override def virtualType: Type = pt.virtualType - def coerceSCode(cb: EmitCodeBuilder, pc: SValue, region: Value[Region], deepCopy: Boolean) - : SingleCodeSCode = + override def coerceSCode( + cb: EmitCodeBuilder, + pc: SValue, + region: Value[Region], + deepCopy: Boolean, + ): SingleCodeSCode = SingleCodeSCode(this, pt.store(cb, region, pc, deepCopy = deepCopy)) } diff --git a/hail/hail/src/is/hail/types/physical/stypes/concrete/SCanonicalCall.scala b/hail/hail/src/is/hail/types/physical/stypes/concrete/SCanonicalCall.scala index 5f0266619d9..ca1cb9e8ec6 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/concrete/SCanonicalCall.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/concrete/SCanonicalCall.scala @@ -12,8 +12,12 @@ import is.hail.utils._ import is.hail.variant._ case object SCanonicalCall extends SCall { - def _coerceOrCopy(cb: EmitCodeBuilder, region: Value[Region], value: SValue, deepCopy: Boolean) - : SValue = + override def _coerceOrCopy( + cb: EmitCodeBuilder, + region: Value[Region], + value: SValue, + deepCopy: Boolean, + ): SValue = value.st match { case SCanonicalCall => value } @@ -22,25 +26,25 @@ case object SCanonicalCall extends SCall { override def castRename(t: Type): SType = this - def settableTupleTypes(): IndexedSeq[TypeInfo[_]] = FastSeq(IntInfo) + override def settableTupleTypes(): IndexedSeq[TypeInfo[_]] = FastSeq(IntInfo) - def fromSettables(settables: IndexedSeq[Settable[_]]): SCanonicalCallSettable = { + override def fromSettables(settables: IndexedSeq[Settable[_]]): SCanonicalCallSettable = { val IndexedSeq(call: Settable[Int @unchecked]) = settables assert(call.ti == IntInfo) new SCanonicalCallSettable(call) } - def fromValues(values: IndexedSeq[Value[_]]): SCanonicalCallValue = { + override def fromValues(values: IndexedSeq[Value[_]]): SCanonicalCallValue = { val IndexedSeq(call: Value[Int @unchecked]) = values assert(call.ti == IntInfo) new SCanonicalCallValue(call) } - def storageType(): PType = PCanonicalCall(false) + override def storageType(): PType = PCanonicalCall(false) - def copiedType: SType = this + override def copiedType: SType = this - def containsPointers: Boolean = false + override def containsPointers: Boolean = false def constructFromIntRepr(cb: EmitCodeBuilder, c: Code[Int]): SCanonicalCallValue = new SCanonicalCallValue(cb.memoize(c)) @@ -59,7 +63,7 @@ class SCanonicalCallValue(val call: Value[Int]) extends SCallValue { new SCanonicalCallValue(repr) } - def containsAllele(cb: EmitCodeBuilder, allele: Value[Int]): Value[Boolean] = + override def containsAllele(cb: EmitCodeBuilder, allele: Value[Int]): Value[Boolean] = cb.memoize[Boolean]( Code.invokeScalaObject2[Int, Int, Boolean]( Call.getClass, diff --git a/hail/hail/src/is/hail/types/physical/stypes/concrete/SIndexablePointer.scala b/hail/hail/src/is/hail/types/physical/stypes/concrete/SIndexablePointer.scala index d3b74359cf2..be02f1ab2bf 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/concrete/SIndexablePointer.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/concrete/SIndexablePointer.scala @@ -134,9 +134,9 @@ final class SIndexablePointerSettable( override val length: Settable[Int], override val elementsAddress: Settable[Long], ) extends SIndexablePointerValue(st, a, length, elementsAddress) with SSettable { - def settableTuple(): IndexedSeq[Settable[_]] = FastSeq(a, length, elementsAddress) + override def settableTuple(): IndexedSeq[Settable[_]] = FastSeq(a, length, elementsAddress) - def store(cb: EmitCodeBuilder, v: SValue): Unit = v match { + override def store(cb: EmitCodeBuilder, v: SValue): Unit = v match { case v: SIndexablePointerValue => cb.assign(a, v.a) cb.assign(length, v.length) diff --git a/hail/hail/src/is/hail/types/physical/stypes/concrete/SNDArrayPointer.scala b/hail/hail/src/is/hail/types/physical/stypes/concrete/SNDArrayPointer.scala index b356a5454b0..21238903607 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/concrete/SNDArrayPointer.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/concrete/SNDArrayPointer.scala @@ -153,10 +153,10 @@ final class SNDArrayPointerSettable( override val firstDataAddress: Settable[Long], ) extends SNDArrayPointerValue(st, a, shape.map(SizeValueDyn.apply), strides, firstDataAddress) with SNDArraySettable { - def settableTuple(): IndexedSeq[Settable[_]] = + override def settableTuple(): IndexedSeq[Settable[_]] = FastSeq(a) ++ shape ++ strides ++ FastSeq(firstDataAddress) - def store(cb: EmitCodeBuilder, v: SValue): Unit = v match { + override def store(cb: EmitCodeBuilder, v: SValue): Unit = v match { case v: SNDArrayPointerValue => cb.assign(a, v.a) shape.lazyZip(v.shapes).foreach(cb.assign(_, _)) diff --git a/hail/hail/src/is/hail/types/physical/stypes/concrete/SNDArraySlice.scala b/hail/hail/src/is/hail/types/physical/stypes/concrete/SNDArraySlice.scala index 1a3d37d616f..bd5eb4cdc71 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/concrete/SNDArraySlice.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/concrete/SNDArraySlice.scala @@ -86,7 +86,8 @@ class SNDArraySliceValue( override def loadElement(indices: IndexedSeq[Value[Long]], cb: EmitCodeBuilder): SValue = pt.elementType.loadCheapSCode(cb, loadElementAddress(indices, cb)) - def coerceToShape(cb: EmitCodeBuilder, otherShape: IndexedSeq[SizeValue]): SNDArrayValue = { + override def coerceToShape(cb: EmitCodeBuilder, otherShape: IndexedSeq[SizeValue]) + : SNDArrayValue = { cb.if_(!hasShape(cb, otherShape), cb._fatal("incompatible shapes")) new SNDArraySliceValue(st, otherShape, strides, firstDataAddress) } diff --git a/hail/hail/src/is/hail/types/physical/stypes/concrete/SRNGState.scala b/hail/hail/src/is/hail/types/physical/stypes/concrete/SRNGState.scala index b65926a3e29..c7000330bc0 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/concrete/SRNGState.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/concrete/SRNGState.scala @@ -21,7 +21,7 @@ final case class SRNGStateStaticInfo( } final case class SRNGState(staticInfo: Option[SRNGStateStaticInfo]) extends SType { - def virtualType: Type = TRNGState + override def virtualType: Type = TRNGState override protected[stypes] def _typeWithRequiredness: TypeWithRequiredness = RPrimitive() @@ -130,7 +130,7 @@ class SCanonicalRNGStateValue( override def sizeToStoreInBytes(cb: EmitCodeBuilder): SInt64Value = new SInt64Value(4 * 8 + 4 * 8 + 4 + 4 + 4) - def splitStatic(cb: EmitCodeBuilder, idx: Long): SCanonicalRNGStateValue = { + override def splitStatic(cb: EmitCodeBuilder, idx: Long): SCanonicalRNGStateValue = { cb.if_(hasStaticSplit, cb._fatal("RNGState received two static splits")) val x = Array.ofDim[Long](4) x(0) = cb.emb.ctx.rngNonce @@ -149,7 +149,7 @@ class SCanonicalRNGStateValue( ) } - def splitDyn(cb: EmitCodeBuilder, idx: Value[Long]): SCanonicalRNGStateValue = { + override def splitDyn(cb: EmitCodeBuilder, idx: Value[Long]): SCanonicalRNGStateValue = { val newRunningSum = Array.tabulate[Settable[Long]](4)(i => cb.newLocal[Long](s"splitDyn_x$i", runningSum(i))) val newLastDynBlock = @@ -180,7 +180,7 @@ class SCanonicalRNGStateValue( hasStaticSplit, newNumDynBlocks) } - def rand(cb: EmitCodeBuilder): IndexedSeq[Value[Long]] = { + override def rand(cb: EmitCodeBuilder): IndexedSeq[Value[Long]] = { cb.if_(!hasStaticSplit, cb._fatal("RNGState never received static split")) val x = Array.tabulate[Settable[Long]](4)(i => cb.newLocal[Long](s"rand_x$i", runningSum(i))) val key = Threefry.defaultKey @@ -204,7 +204,7 @@ class SCanonicalRNGStateValue( x } - def copyIntoEngine(cb: EmitCodeBuilder, tf: Value[ThreefryRandomEngine]): Unit = { + override def copyIntoEngine(cb: EmitCodeBuilder, tf: Value[ThreefryRandomEngine]): Unit = { cb.if_(!hasStaticSplit, cb._fatal("RNGState never received static split")) val x = Array.tabulate[Settable[Long]](4)(i => cb.newLocal[Long](s"cie_x$i", runningSum(i))) val finalTweak = @@ -282,7 +282,7 @@ class SRNGStateStaticSizeValue( override def sizeToStoreInBytes(cb: EmitCodeBuilder): SInt64Value = new SInt64Value(4 * 8 + staticInfo.numWordsInLastBlock * 8) - def splitStatic(cb: EmitCodeBuilder, idx: Long): SRNGStateStaticSizeValue = { + override def splitStatic(cb: EmitCodeBuilder, idx: Long): SRNGStateStaticSizeValue = { assert(!staticInfo.hasStaticSplit) val x = Array.ofDim[Long](4) x(0) = cb.emb.ctx.rngNonce @@ -302,7 +302,7 @@ class SRNGStateStaticSizeValue( ) } - def splitDyn(cb: EmitCodeBuilder, idx: Value[Long]): SRNGStateStaticSizeValue = { + override def splitDyn(cb: EmitCodeBuilder, idx: Value[Long]): SRNGStateStaticSizeValue = { if (staticInfo.numWordsInLastBlock < 4) { return new SRNGStateStaticSizeValue( st = SRNGState(Some(SRNGStateStaticInfo( @@ -331,7 +331,7 @@ class SRNGStateStaticSizeValue( ) } - def rand(cb: EmitCodeBuilder): IndexedSeq[Value[Long]] = { + override def rand(cb: EmitCodeBuilder): IndexedSeq[Value[Long]] = { assert(staticInfo.hasStaticSplit) val x = Array.tabulate[Settable[Long]](4)(i => cb.newLocal[Long](s"rand_x$i", runningSum(i))) val key = Threefry.defaultKey @@ -346,7 +346,7 @@ class SRNGStateStaticSizeValue( x } - def copyIntoEngine(cb: EmitCodeBuilder, tf: Value[ThreefryRandomEngine]): Unit = { + override def copyIntoEngine(cb: EmitCodeBuilder, tf: Value[ThreefryRandomEngine]): Unit = { val x = Array.tabulate[Settable[Long]](4)(i => cb.newLocal[Long](s"cie_x$i", runningSum(i))) for (i <- lastDynBlock.indices) cb.assign(x(i), x(i) ^ lastDynBlock(i)) val finalTweak = if (staticInfo.numWordsInLastBlock < 4) { diff --git a/hail/hail/src/is/hail/types/physical/stypes/concrete/SStringPointer.scala b/hail/hail/src/is/hail/types/physical/stypes/concrete/SStringPointer.scala index 82f27e81aa1..3e49198ea4c 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/concrete/SStringPointer.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/concrete/SStringPointer.scala @@ -58,13 +58,13 @@ class SStringPointerValue(val st: SStringPointer, val a: Value[Long]) extends SS def binaryRepr: SBinaryPointerValue = new SBinaryPointerValue(SBinaryPointer(pt.binaryRepresentation), a) - def loadLength(cb: EmitCodeBuilder): Value[Int] = + override def loadLength(cb: EmitCodeBuilder): Value[Int] = cb.memoize(pt.loadLength(a)) - def loadString(cb: EmitCodeBuilder): Value[String] = + override def loadString(cb: EmitCodeBuilder): Value[String] = cb.memoize(pt.loadString(a)) - def toBytes(cb: EmitCodeBuilder): SBinaryPointerValue = + override def toBytes(cb: EmitCodeBuilder): SBinaryPointerValue = binaryRepr override def sizeToStoreInBytes(cb: EmitCodeBuilder): SInt64Value = diff --git a/hail/hail/src/is/hail/types/physical/stypes/concrete/SStructView.scala b/hail/hail/src/is/hail/types/physical/stypes/concrete/SStructView.scala index 7a970e49c8b..aec31c20010 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/concrete/SStructView.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/concrete/SStructView.scala @@ -106,7 +106,7 @@ final class SStructView( ct } - def storageType(): PType = { + override def storageType(): PType = { val pt = PCanonicalStruct( required = false, args = rename.fieldNames.zip(fieldEmitTypes.map(_.copiedType.storageType)): _*, diff --git a/hail/hail/src/is/hail/types/physical/stypes/concrete/SUnreachable.scala b/hail/hail/src/is/hail/types/physical/stypes/concrete/SUnreachable.scala index a584abc8195..349e4e65b56 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/concrete/SUnreachable.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/concrete/SUnreachable.scala @@ -169,7 +169,8 @@ case object SUnreachableCall extends SUnreachable with SCall { class SUnreachableCallValue extends SUnreachableValue with SCallValue { override def unphase(cb: EmitCodeBuilder): SCallValue = this - def containsAllele(cb: EmitCodeBuilder, allele: Value[Int]): Value[Boolean] = const(false) + override def containsAllele(cb: EmitCodeBuilder, allele: Value[Int]): Value[Boolean] = + const(false) override def forEachAllele(cb: EmitCodeBuilder)(alleleCode: Value[Int] => Unit): Unit = {} diff --git a/hail/hail/src/is/hail/types/physical/stypes/interfaces/SBaseStruct.scala b/hail/hail/src/is/hail/types/physical/stypes/interfaces/SBaseStruct.scala index 9f3cc00b0bd..2543e5ab93e 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/interfaces/SBaseStruct.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/interfaces/SBaseStruct.scala @@ -33,7 +33,7 @@ object SBaseStruct { } trait SBaseStruct extends SType { - def virtualType: TBaseStruct + override def virtualType: TBaseStruct def size: Int @@ -42,7 +42,7 @@ trait SBaseStruct extends SType { def fieldIdx(fieldName: String): Int - def _typeWithRequiredness: TypeWithRequiredness = { + override def _typeWithRequiredness: TypeWithRequiredness = { virtualType match { case ts: TStruct => RStruct.fromNamesAndTypes(ts.fieldNames.zip(fieldEmitTypes).map { case (name, et) => (name, et.typeWithRequiredness.r) @@ -57,7 +57,7 @@ trait SBaseStruct extends SType { trait SBaseStructSettable extends SBaseStructValue with SSettable trait SBaseStructValue extends SValue { - def st: SBaseStruct + override def st: SBaseStruct def isFieldMissing(cb: EmitCodeBuilder, fieldIdx: Int): Value[Boolean] diff --git a/hail/hail/src/is/hail/types/physical/stypes/interfaces/SContainer.scala b/hail/hail/src/is/hail/types/physical/stypes/interfaces/SContainer.scala index 61de1ee8f14..2c61d1ff483 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/interfaces/SContainer.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/interfaces/SContainer.scala @@ -17,7 +17,7 @@ trait SContainer extends SType { } trait SIndexableValue extends SValue { - def st: SContainer + override def st: SContainer def loadLength: Value[Int] diff --git a/hail/hail/src/is/hail/types/physical/stypes/interfaces/SInterval.scala b/hail/hail/src/is/hail/types/physical/stypes/interfaces/SInterval.scala index fb67fbe0827..7711311a6c5 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/interfaces/SInterval.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/interfaces/SInterval.scala @@ -18,7 +18,7 @@ trait SInterval extends SType { } trait SIntervalValue extends SValue { - def st: SInterval + override def st: SInterval def includesStart: Value[Boolean] diff --git a/hail/hail/src/is/hail/types/physical/stypes/interfaces/SNDArray.scala b/hail/hail/src/is/hail/types/physical/stypes/interfaces/SNDArray.scala index ec5ccf2ac7c..1872c69d8ee 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/interfaces/SNDArray.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/interfaces/SNDArray.scala @@ -1243,7 +1243,7 @@ object SizeValueStatic { } final class SizeValueDyn(val v: Value[Long]) extends SizeValue { - def get: Code[Long] = v.get + override def get: Code[Long] = v.get override def equals(other: Any): Boolean = other match { case SizeValueDyn(v2) => v eq v2 @@ -1252,7 +1252,7 @@ final class SizeValueDyn(val v: Value[Long]) extends SizeValue { } final class SizeValueStatic(val v: Long) extends SizeValue { - def get: Code[Long] = const(v) + override def get: Code[Long] = const(v) override def equals(other: Any): Boolean = other match { case SizeValueStatic(v2) => v == v2 @@ -1261,7 +1261,7 @@ final class SizeValueStatic(val v: Long) extends SizeValue { } trait SNDArrayValue extends SValue { - def st: SNDArray + override def st: SNDArray def pt: PNDArray diff --git a/hail/hail/src/is/hail/types/physical/stypes/interfaces/SStream.scala b/hail/hail/src/is/hail/types/physical/stypes/interfaces/SStream.scala index ac27349918b..9890312ac12 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/interfaces/SStream.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/interfaces/SStream.scala @@ -95,7 +95,7 @@ object SStreamValue { } trait SStreamValue extends SUnrealizableValue { - def st: SStream + override def st: SStream def getProducer(mb: EmitMethodBuilder[_]): StreamProducer @@ -160,7 +160,7 @@ case class SStreamControlFlow(st: SimpleSStream, producer: StreamProducer) exten producer } - def valueTuple: IndexedSeq[Value[_]] = throw new NotImplementedError() + override def valueTuple: IndexedSeq[Value[_]] = throw new NotImplementedError() override def defineUnusedLabels(mb: EmitMethodBuilder[_]): Unit = { (producer.LendOfStream.isImplemented, producer.LproduceElementDone.isImplemented) match { diff --git a/hail/hail/src/is/hail/types/physical/stypes/primitives/SPrimitive.scala b/hail/hail/src/is/hail/types/physical/stypes/primitives/SPrimitive.scala index aa05e7763d2..3ab9a75d964 100644 --- a/hail/hail/src/is/hail/types/physical/stypes/primitives/SPrimitive.scala +++ b/hail/hail/src/is/hail/types/physical/stypes/primitives/SPrimitive.scala @@ -11,7 +11,7 @@ trait SPrimitive extends SType { override def copiedType: SType = this - def containsPointers: Boolean = false + override def containsPointers: Boolean = false } abstract class SPrimitiveValue extends SValue { diff --git a/hail/hail/src/is/hail/types/virtual/MatrixType.scala b/hail/hail/src/is/hail/types/virtual/MatrixType.scala index c2669d58ca9..8380486c904 100644 --- a/hail/hail/src/is/hail/types/virtual/MatrixType.scala +++ b/hail/hail/src/is/hail/types/virtual/MatrixType.scala @@ -154,7 +154,7 @@ case class MatrixType( "g" -> entryType, ) - def pretty(sb: StringBuilder, indent0: Int = 0, compact: Boolean = false): Unit = { + override def pretty(sb: StringBuilder, indent0: Int = 0, compact: Boolean = false): Unit = { val space: String = if (compact) "" else " " val newline: String = if (compact) "" else "\n" diff --git a/hail/hail/src/is/hail/types/virtual/TArray.scala b/hail/hail/src/is/hail/types/virtual/TArray.scala index c037eaa8c10..332804275cb 100644 --- a/hail/hail/src/is/hail/types/virtual/TArray.scala +++ b/hail/hail/src/is/hail/types/virtual/TArray.scala @@ -12,7 +12,7 @@ final case class TArray(elementType: Type) extends TContainer { sb += '>' } - def _toPretty = s"Array[$elementType]" + override def _toPretty = s"Array[$elementType]" override def canCompare(other: Type): Boolean = other match { case TArray(otherType) => elementType.canCompare(otherType) @@ -32,7 +32,7 @@ final case class TArray(elementType: Type) extends TContainer { sb += ']' } - def _typeCheck(a: Any): Boolean = a.isInstanceOf[IndexedSeq[_]] && + override def _typeCheck(a: Any): Boolean = a.isInstanceOf[IndexedSeq[_]] && a.asInstanceOf[IndexedSeq[_]].forall(elementType.typeCheck) override def _showStr(a: Annotation): String = @@ -42,7 +42,7 @@ final case class TArray(elementType: Type) extends TContainer { override def str(a: Annotation): String = JsonMethods.compact(export(a)) - def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = + override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = ExtendedOrdering.iterableOrdering(elementType.ordering(sm), missingEqual) override def valueSubsetter(subtype: Type): Any => Any = { diff --git a/hail/hail/src/is/hail/types/virtual/TBaseStruct.scala b/hail/hail/src/is/hail/types/virtual/TBaseStruct.scala index 3fd9f17f90b..a243a8f4bd8 100644 --- a/hail/hail/src/is/hail/types/virtual/TBaseStruct.scala +++ b/hail/hail/src/is/hail/types/virtual/TBaseStruct.scala @@ -37,7 +37,7 @@ abstract class TBaseStruct extends Type { def size: Int - def _toPretty: String = { + override def _toPretty: String = { val sb = new StringBuilder _pretty(sb, 0, compact = true) sb.result() diff --git a/hail/hail/src/is/hail/types/virtual/TBinary.scala b/hail/hail/src/is/hail/types/virtual/TBinary.scala index d904f538c41..a6c4cbfcb09 100644 --- a/hail/hail/src/is/hail/types/virtual/TBinary.scala +++ b/hail/hail/src/is/hail/types/virtual/TBinary.scala @@ -4,11 +4,11 @@ import is.hail.annotations._ import is.hail.backend.HailStateManager case object TBinary extends Type { - def _toPretty = "Binary" + override def _toPretty = "Binary" - def _typeCheck(a: Any): Boolean = a.isInstanceOf[Array[Byte]] + override def _typeCheck(a: Any): Boolean = a.isInstanceOf[Array[Byte]] - def mkOrdering(sm: HailStateManager, _missingEqual: Boolean = true): ExtendedOrdering = + override def mkOrdering(sm: HailStateManager, _missingEqual: Boolean = true): ExtendedOrdering = ExtendedOrdering.iterableOrdering(new ExtendedOrdering { val missingEqual = _missingEqual diff --git a/hail/hail/src/is/hail/types/virtual/TBoolean.scala b/hail/hail/src/is/hail/types/virtual/TBoolean.scala index 4035d46402f..a4309630fe8 100644 --- a/hail/hail/src/is/hail/types/virtual/TBoolean.scala +++ b/hail/hail/src/is/hail/types/virtual/TBoolean.scala @@ -4,12 +4,12 @@ import is.hail.annotations._ import is.hail.backend.HailStateManager case object TBoolean extends Type { - def _toPretty = "Boolean" + override def _toPretty = "Boolean" override def pyString(sb: StringBuilder): Unit = sb ++= "bool" - def _typeCheck(a: Any): Boolean = a.isInstanceOf[Boolean] + override def _typeCheck(a: Any): Boolean = a.isInstanceOf[Boolean] override def _showStr(a: Annotation): String = if (a.asInstanceOf[Boolean]) "True" else "False" diff --git a/hail/hail/src/is/hail/types/virtual/TCall.scala b/hail/hail/src/is/hail/types/virtual/TCall.scala index 39cfe195e93..d68b581ef84 100644 --- a/hail/hail/src/is/hail/types/virtual/TCall.scala +++ b/hail/hail/src/is/hail/types/virtual/TCall.scala @@ -5,14 +5,14 @@ import is.hail.backend.HailStateManager import is.hail.variant.Call case object TCall extends Type { - def _toPretty = "Call" + override def _toPretty = "Call" override def pyString(sb: StringBuilder): Unit = sb ++= "call" val representation: Type = TInt32 - def _typeCheck(a: Any): Boolean = a.isInstanceOf[Int] + override def _typeCheck(a: Any): Boolean = a.isInstanceOf[Int] override def str(a: Annotation): String = if (a == null) "NA" else Call.toString(a.asInstanceOf[Call]) diff --git a/hail/hail/src/is/hail/types/virtual/TDict.scala b/hail/hail/src/is/hail/types/virtual/TDict.scala index a67fb9f5eea..6f0549e02eb 100644 --- a/hail/hail/src/is/hail/types/virtual/TDict.scala +++ b/hail/hail/src/is/hail/types/virtual/TDict.scala @@ -27,7 +27,7 @@ final case class TDict(keyType: Type, valueType: Type) extends TContainer { override def subst() = TDict(keyType.subst(), valueType.subst()) - def _toPretty = s"Dict[$keyType, $valueType]" + override def _toPretty = s"Dict[$keyType, $valueType]" override def pyString(sb: StringBuilder): Unit = { sb ++= "dict<" @@ -45,7 +45,7 @@ final case class TDict(keyType: Type, valueType: Type) extends TContainer { sb += ']' } - def _typeCheck(a: Any): Boolean = a == null || (a.isInstanceOf[Map[_, _]] && + override def _typeCheck(a: Any): Boolean = a == null || (a.isInstanceOf[Map[_, _]] && a.asInstanceOf[Map[_, _]].forall { case (k, v) => keyType.typeCheck(k) && valueType.typeCheck(v) }) diff --git a/hail/hail/src/is/hail/types/virtual/TFloat32.scala b/hail/hail/src/is/hail/types/virtual/TFloat32.scala index 532a1cb7ad0..383d3fa75d1 100644 --- a/hail/hail/src/is/hail/types/virtual/TFloat32.scala +++ b/hail/hail/src/is/hail/types/virtual/TFloat32.scala @@ -5,12 +5,12 @@ import is.hail.backend.HailStateManager import is.hail.utils._ case object TFloat32 extends TNumeric { - def _toPretty = "Float32" + override def _toPretty = "Float32" override def pyString(sb: StringBuilder): Unit = sb ++= "float32" - def _typeCheck(a: Any): Boolean = a.isInstanceOf[Float] + override def _typeCheck(a: Any): Boolean = a.isInstanceOf[Float] override def _showStr(a: Annotation): String = "%.02e".format(a.asInstanceOf[Float]) diff --git a/hail/hail/src/is/hail/types/virtual/TFloat64.scala b/hail/hail/src/is/hail/types/virtual/TFloat64.scala index 9646fd853b0..5fdc15d3b96 100644 --- a/hail/hail/src/is/hail/types/virtual/TFloat64.scala +++ b/hail/hail/src/is/hail/types/virtual/TFloat64.scala @@ -10,7 +10,7 @@ case object TFloat64 extends TNumeric { override def pyString(sb: StringBuilder): Unit = sb ++= "float64" - def _typeCheck(a: Any): Boolean = a.isInstanceOf[Double] + override def _typeCheck(a: Any): Boolean = a.isInstanceOf[Double] override def _showStr(a: Annotation): String = "%.02e".format(a.asInstanceOf[Double]) diff --git a/hail/hail/src/is/hail/types/virtual/TInt32.scala b/hail/hail/src/is/hail/types/virtual/TInt32.scala index 26bea25deaf..1c1b6095a88 100644 --- a/hail/hail/src/is/hail/types/virtual/TInt32.scala +++ b/hail/hail/src/is/hail/types/virtual/TInt32.scala @@ -4,12 +4,12 @@ import is.hail.annotations._ import is.hail.backend.HailStateManager case object TInt32 extends TIntegral { - def _toPretty = "Int32" + override def _toPretty = "Int32" override def pyString(sb: StringBuilder): Unit = sb ++= "int32" - def _typeCheck(a: Any): Boolean = a.isInstanceOf[Int] + override def _typeCheck(a: Any): Boolean = a.isInstanceOf[Int] override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = ExtendedOrdering.extendToNull(implicitly[Ordering[Int]], missingEqual) diff --git a/hail/hail/src/is/hail/types/virtual/TInt64.scala b/hail/hail/src/is/hail/types/virtual/TInt64.scala index 93f88f000b8..41079d78258 100644 --- a/hail/hail/src/is/hail/types/virtual/TInt64.scala +++ b/hail/hail/src/is/hail/types/virtual/TInt64.scala @@ -4,12 +4,12 @@ import is.hail.annotations._ import is.hail.backend.HailStateManager case object TInt64 extends TIntegral { - def _toPretty = "Int64" + override def _toPretty = "Int64" override def pyString(sb: StringBuilder): Unit = sb ++= "int64" - def _typeCheck(a: Any): Boolean = a.isInstanceOf[Long] + override def _typeCheck(a: Any): Boolean = a.isInstanceOf[Long] override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = ExtendedOrdering.extendToNull(implicitly[Ordering[Long]], missingEqual) diff --git a/hail/hail/src/is/hail/types/virtual/TInterval.scala b/hail/hail/src/is/hail/types/virtual/TInterval.scala index be1031e3b46..1d3b94e7ef8 100644 --- a/hail/hail/src/is/hail/types/virtual/TInterval.scala +++ b/hail/hail/src/is/hail/types/virtual/TInterval.scala @@ -8,7 +8,7 @@ case class TInterval(pointType: Type) extends Type { override def children = FastSeq(pointType) - def _toPretty = s"""Interval[$pointType]""" + override def _toPretty = s"""Interval[$pointType]""" override def pyString(sb: StringBuilder): Unit = { sb ++= "interval<" @@ -22,7 +22,7 @@ case class TInterval(pointType: Type) extends Type { sb += ']' } - def _typeCheck(a: Any): Boolean = a.isInstanceOf[Interval] && { + override def _typeCheck(a: Any): Boolean = a.isInstanceOf[Interval] && { val i = a.asInstanceOf[Interval] pointType.typeCheck(i.start) && pointType.typeCheck(i.end) } diff --git a/hail/hail/src/is/hail/types/virtual/TLocus.scala b/hail/hail/src/is/hail/types/virtual/TLocus.scala index cee504eac00..0002981cccc 100644 --- a/hail/hail/src/is/hail/types/virtual/TLocus.scala +++ b/hail/hail/src/is/hail/types/virtual/TLocus.scala @@ -21,14 +21,14 @@ object TLocus { case class TLocus(rgName: String) extends Type { - def _toPretty = s"Locus($rgName)" + override def _toPretty = s"Locus($rgName)" def rg: String = rgName override def pyString(sb: StringBuilder): Unit = sb ++= "locus<" ++= prettyIdentifier(rgName) += '>': Unit - def _typeCheck(a: Any): Boolean = a.isInstanceOf[Locus] + override def _typeCheck(a: Any): Boolean = a.isInstanceOf[Locus] override def mkOrdering(sm: HailStateManager, missingEqual: Boolean = true): ExtendedOrdering = sm.referenceGenomes(rgName).extendedLocusOrdering diff --git a/hail/hail/src/is/hail/types/virtual/TNDArray.scala b/hail/hail/src/is/hail/types/virtual/TNDArray.scala index eb769c310a1..1221fc12d4d 100644 --- a/hail/hail/src/is/hail/types/virtual/TNDArray.scala +++ b/hail/hail/src/is/hail/types/virtual/TNDArray.scala @@ -46,7 +46,7 @@ final case class TNDArray(elementType: Type, nDimsBase: NatBase) extends Type { sb += '>' } - def _toPretty = s"NDArray[$elementType,$nDims]" + override def _toPretty = s"NDArray[$elementType,$nDims]" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false): Unit = { sb ++= "NDArray[" @@ -105,7 +105,7 @@ final case class TNDArray(elementType: Type, nDimsBase: NatBase) extends Type { override def subst(): TNDArray = TNDArray(elementType.subst(), nDimsBase.subst()) - def _typeCheck(a: Annotation): Boolean = a match { + override def _typeCheck(a: Annotation): Boolean = a match { case nd: NDArray => nd.forall(e => elementType.typeCheck(e)) case _ => false } diff --git a/hail/hail/src/is/hail/types/virtual/TRNGState.scala b/hail/hail/src/is/hail/types/virtual/TRNGState.scala index b9b25283658..3ce9fada0a8 100644 --- a/hail/hail/src/is/hail/types/virtual/TRNGState.scala +++ b/hail/hail/src/is/hail/types/virtual/TRNGState.scala @@ -8,9 +8,9 @@ case object TRNGState extends Type { override def pyString(sb: StringBuilder): Unit = sb ++= "rng_state" - def _typeCheck(a: Any): Boolean = ??? + override def _typeCheck(a: Any): Boolean = ??? - def mkOrdering(sm: HailStateManager, missingEqual: Boolean) + override def mkOrdering(sm: HailStateManager, missingEqual: Boolean) : is.hail.annotations.ExtendedOrdering = ??? override def isIsomorphicTo(t: Type): Boolean = diff --git a/hail/hail/src/is/hail/types/virtual/TSet.scala b/hail/hail/src/is/hail/types/virtual/TSet.scala index 98a2282fbc0..f0ed6561167 100644 --- a/hail/hail/src/is/hail/types/virtual/TSet.scala +++ b/hail/hail/src/is/hail/types/virtual/TSet.scala @@ -6,7 +6,7 @@ import is.hail.backend.HailStateManager import org.json4s.jackson.JsonMethods final case class TSet(elementType: Type) extends TContainer { - def _toPretty = s"Set[$elementType]" + override def _toPretty = s"Set[$elementType]" override def pyString(sb: StringBuilder): Unit = { sb ++= "set<" @@ -26,7 +26,7 @@ final case class TSet(elementType: Type) extends TContainer { override def subst() = TSet(elementType.subst()) - def _typeCheck(a: Any): Boolean = + override def _typeCheck(a: Any): Boolean = a.isInstanceOf[Set[_]] && a.asInstanceOf[Set[_]].forall(elementType.typeCheck) override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false): Unit = { diff --git a/hail/hail/src/is/hail/types/virtual/TStream.scala b/hail/hail/src/is/hail/types/virtual/TStream.scala index b5186a350a1..679215610cd 100644 --- a/hail/hail/src/is/hail/types/virtual/TStream.scala +++ b/hail/hail/src/is/hail/types/virtual/TStream.scala @@ -12,7 +12,7 @@ final case class TStream(elementType: Type) extends TIterable { sb += '>' } - def _toPretty = s"Stream[$elementType]" + override def _toPretty = s"Stream[$elementType]" override def canCompare(other: Type): Boolean = throw new UnsupportedOperationException("Stream comparison is currently undefined.") @@ -30,7 +30,7 @@ final case class TStream(elementType: Type) extends TIterable { sb += ']' } - def _typeCheck(a: Any): Boolean = a.isInstanceOf[IndexedSeq[_]] && + override def _typeCheck(a: Any): Boolean = a.isInstanceOf[IndexedSeq[_]] && a.asInstanceOf[IndexedSeq[_]].forall(elementType.typeCheck) override def str(a: Annotation): String = JsonMethods.compact(export(a)) diff --git a/hail/hail/src/is/hail/types/virtual/TString.scala b/hail/hail/src/is/hail/types/virtual/TString.scala index 6f7e99211d0..1204beeb444 100644 --- a/hail/hail/src/is/hail/types/virtual/TString.scala +++ b/hail/hail/src/is/hail/types/virtual/TString.scala @@ -4,14 +4,14 @@ import is.hail.annotations._ import is.hail.backend.HailStateManager case object TString extends Type { - def _toPretty = "String" + override def _toPretty = "String" override def pyString(sb: StringBuilder): Unit = sb ++= "str" override def _showStr(a: Annotation): String = "\"" + a.asInstanceOf[String] + "\"" - def _typeCheck(a: Any): Boolean = a.isInstanceOf[String] + override def _typeCheck(a: Any): Boolean = a.isInstanceOf[String] override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = ExtendedOrdering.extendToNull(implicitly[Ordering[String]], missingEqual) diff --git a/hail/hail/src/is/hail/types/virtual/TStruct.scala b/hail/hail/src/is/hail/types/virtual/TStruct.scala index 15ead3ff969..dfb7b065c5e 100644 --- a/hail/hail/src/is/hail/types/virtual/TStruct.scala +++ b/hail/hail/src/is/hail/types/virtual/TStruct.scala @@ -65,7 +65,7 @@ final case class TStruct(fields: IndexedSeq[Field]) extends TBaseStruct { } } - def size: Int = fields.length + override def size: Int = fields.length override def truncate(newSize: Int): TStruct = TStruct(fields.take(newSize)) diff --git a/hail/hail/src/is/hail/types/virtual/TTuple.scala b/hail/hail/src/is/hail/types/virtual/TTuple.scala index 8c4917430d7..d9281fe32ab 100644 --- a/hail/hail/src/is/hail/types/virtual/TTuple.scala +++ b/hail/hail/src/is/hail/types/virtual/TTuple.scala @@ -34,7 +34,7 @@ final case class TTuple(_types: IndexedSeq[TupleField]) extends TBaseStruct { override lazy val _isCanonical: Boolean = _types.indices.forall(i => i == _types(i).index) - def size: Int = types.length + override def size: Int = types.length override def truncate(newSize: Int): TTuple = TTuple(_types.take(newSize)) diff --git a/hail/hail/src/is/hail/types/virtual/TVariable.scala b/hail/hail/src/is/hail/types/virtual/TVariable.scala index fa647f19c8e..f65a2092675 100644 --- a/hail/hail/src/is/hail/types/virtual/TVariable.scala +++ b/hail/hail/src/is/hail/types/virtual/TVariable.scala @@ -53,7 +53,7 @@ final case class TVariable(name: String, cond: String = null) extends Type { override def isRealizable = false - def _typeCheck(a: Any): Boolean = + override def _typeCheck(a: Any): Boolean = throw new RuntimeException("TVariable is not realizable") override def unify(concrete: Type): Boolean = diff --git a/hail/hail/src/is/hail/types/virtual/TableType.scala b/hail/hail/src/is/hail/types/virtual/TableType.scala index 4129f8c7978..017682b70b4 100644 --- a/hail/hail/src/is/hail/types/virtual/TableType.scala +++ b/hail/hail/src/is/hail/types/virtual/TableType.scala @@ -55,7 +55,7 @@ case class TableType(rowType: TStruct, key: IndexedSeq[String], globalType: TStr lazy val valueType: TStruct = TableType.valueType(rowType, key) def valueFieldIdx: Array[Int] = canonicalRVDType.valueFieldIdx - def pretty(sb: StringBuilder, indent0: Int = 0, compact: Boolean = false): Unit = { + override def pretty(sb: StringBuilder, indent0: Int = 0, compact: Boolean = false): Unit = { val space: String = if (compact) "" else " " val padding: String = if (compact) "" else " " * indent0 diff --git a/hail/hail/src/is/hail/types/virtual/Type.scala b/hail/hail/src/is/hail/types/virtual/Type.scala index d5423df108a..429a74ff08e 100644 --- a/hail/hail/src/is/hail/types/virtual/Type.scala +++ b/hail/hail/src/is/hail/types/virtual/Type.scala @@ -53,7 +53,7 @@ abstract class Type extends VType with Serializable { else (this, identity[Annotation]) - final def pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = + final override def pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = _pretty(sb, indent, compact) def _toPretty: String diff --git a/hail/hail/src/is/hail/utils/AbsoluteFuzzyComparable.scala b/hail/hail/src/is/hail/utils/AbsoluteFuzzyComparable.scala index 3709b4f9b0e..818b6838995 100644 --- a/hail/hail/src/is/hail/utils/AbsoluteFuzzyComparable.scala +++ b/hail/hail/src/is/hail/utils/AbsoluteFuzzyComparable.scala @@ -11,13 +11,13 @@ object AbsoluteFuzzyComparable { afc.absoluteEq(tolerance, x, y) implicit object afcDoubles extends AbsoluteFuzzyComparable[Double] { - def absoluteEq(tolerance: Double, x: Double, y: Double) = Math.abs(x - y) <= tolerance + override def absoluteEq(tolerance: Double, x: Double, y: Double) = Math.abs(x - y) <= tolerance } implicit def afcMaps[K, V](implicit vRFC: AbsoluteFuzzyComparable[V]) : AbsoluteFuzzyComparable[Map[K, V]] = new AbsoluteFuzzyComparable[Map[K, V]] { - def absoluteEq(tolerance: Double, x: Map[K, V], y: Map[K, V]) = + override def absoluteEq(tolerance: Double, x: Map[K, V], y: Map[K, V]) = x.keySet == y.keySet && x.keys.forall(k => vRFC.absoluteEq(tolerance, x(k), y(k))) } } diff --git a/hail/hail/src/is/hail/utils/BufferedAggregatorIterator.scala b/hail/hail/src/is/hail/utils/BufferedAggregatorIterator.scala index 7190686b520..ad45cd74769 100644 --- a/hail/hail/src/is/hail/utils/BufferedAggregatorIterator.scala +++ b/hail/hail/src/is/hail/utils/BufferedAggregatorIterator.scala @@ -31,10 +31,10 @@ class BufferedAggregatorIterator[T, V, U, K]( } else false } - def hasNext: Boolean = + override def hasNext: Boolean = fb.isValid || buffer.size() > 0 - def next(): (K, U) = { + override def next(): (K, U) = { if (!hasNext) throw new NoSuchElementException while (fb.isValid) { diff --git a/hail/hail/src/is/hail/utils/Cache.scala b/hail/hail/src/is/hail/utils/Cache.scala index f0216a50323..15e45d0dac5 100644 --- a/hail/hail/src/is/hail/utils/Cache.scala +++ b/hail/hail/src/is/hail/utils/Cache.scala @@ -66,5 +66,5 @@ class LongToRegionValueCache(capacity: Int) extends Closeable { m.clear() } - def close(): Unit = free() + override def close(): Unit = free() } diff --git a/hail/hail/src/is/hail/utils/FlipbookIterator.scala b/hail/hail/src/is/hail/utils/FlipbookIterator.scala index 8da883b73e6..762feaf0b34 100644 --- a/hail/hail/src/is/hail/utils/FlipbookIterator.scala +++ b/hail/hail/src/is/hail/utils/FlipbookIterator.scala @@ -31,8 +31,8 @@ abstract class StateMachine[A] { object StateMachine { def terminal[A]: StateMachine[A] = new StateMachine[A] { val isValid = false - def value: A = ??? - def advance(): Unit = {} + override def value: A = ??? + override def advance(): Unit = {} } } @@ -45,10 +45,10 @@ class StagingIterator[A] private (sm: StateMachine[A]) extends FlipbookIterator[ private var isConsumed: Boolean = false // FlipbookIterator interface - def isValid: Boolean = sm.isValid - def value: A = { assert(isValid && !isConsumed); sm.value } + override def isValid: Boolean = sm.isValid + override def value: A = { assert(isValid && !isConsumed); sm.value } - def advance(): Unit = { + override def advance(): Unit = { assert(isValid) isConsumed = false sm.advance() @@ -67,21 +67,21 @@ class StagingIterator[A] private (sm: StateMachine[A]) extends FlipbookIterator[ // (Buffered)Iterator interface, not intended to be used directly, only for // passing a StagingIterator where an Iterator is expected - def head: A = { stage(); sm.value } + override def head: A = { stage(); sm.value } - def hasNext: Boolean = + override def hasNext: Boolean = if (isValid) { stage() isValid } else false - def next(): A = { + override def next(): A = { stage() assert(isValid) consume() } - def toStagingIterator: StagingIterator[A] = this + override def toStagingIterator: StagingIterator[A] = this } object FlipbookIterator { @@ -95,7 +95,7 @@ object FlipbookIterator { ord: (A, A) => Int, ): FlipbookIterator[collection.IndexedSeq[(A, Int)]] = { object TmpOrd extends Ordering[(A, Int)] { - def compare(x: (A, Int), y: (A, Int)): Int = ord(y._1, x._1) + override def compare(x: (A, Int), y: (A, Int)): Int = ord(y._1, x._1) } val sm = new StateMachine[collection.IndexedSeq[(A, Int)]] { val q: mutable.PriorityQueue[(A, Int)] = new mutable.PriorityQueue()(TmpOrd) @@ -109,7 +109,7 @@ object FlipbookIterator { i += 1 } - def advance(): Unit = { + override def advance(): Unit = { var i = 0 while (i < value.length) { val j = value(i)._2 @@ -158,10 +158,10 @@ abstract class FlipbookIterator[A] extends BufferedIterator[A] { self => override def filter(pred: A => Boolean): FlipbookIterator[A] = FlipbookIterator( new StateMachine[A] { - def value = self.value - def isValid = self.isValid + override def value = self.value + override def isValid = self.isValid - def advance(): Unit = + override def advance(): Unit = do self.advance() while (self.isValid && !pred(self.value)) while (self.isValid && !pred(self.value)) self.advance() @@ -174,9 +174,9 @@ abstract class FlipbookIterator[A] extends BufferedIterator[A] { self => new StateMachine[B] { var value: B = _ if (self.isValid) value = f(self.value) - def isValid = self.isValid + override def isValid = self.isValid - def advance(): Unit = { + override def advance(): Unit = { self.advance() if (self.isValid) value = f(self.value) } @@ -191,10 +191,10 @@ abstract class FlipbookIterator[A] extends BufferedIterator[A] { self => var it: FlipbookIterator[B] = _ if (self.isValid) it = f(self.value).toIterator.toFlipbookIterator findNextValid() - def value: B = it.value - def isValid = self.isValid + override def value: B = it.value + override def isValid = self.isValid - def advance(): Unit = { + override def advance(): Unit = { it.advance() findNextValid() } @@ -209,23 +209,23 @@ abstract class FlipbookIterator[A] extends BufferedIterator[A] { self => private[this] trait ValidityCachingStateMachine extends StateMachine[A] { private[this] var _isValid: Boolean = _ - final def isValid = _isValid + final override def isValid = _isValid final def refreshValidity(): Unit = _isValid = calculateValidity def calculateValidity: Boolean - def value: A - def advance(): Unit + override def value: A + override def advance(): Unit refreshValidity() } def staircased(ord: OrderingView[A]): StagingIterator[FlipbookIterator[A]] = { ord.setBottom() val stepSM = new ValidityCachingStateMachine { - def value: A = self.value - def calculateValidity: Boolean = self.isValid && ord.isEquivalent(self.value) - def advance() = { + override def value: A = self.value + override def calculateValidity: Boolean = self.isValid && ord.isEquivalent(self.value) + override def advance() = { self.advance() refreshValidity() } @@ -234,7 +234,7 @@ abstract class FlipbookIterator[A] extends BufferedIterator[A] { self => val sm = new StateMachine[FlipbookIterator[A]] { var isValid: Boolean = true val value: FlipbookIterator[A] = stepIterator - def advance() = { + override def advance() = { stepIterator.exhaust() if (self.isValid) { ord.setValue(self.value) @@ -276,7 +276,7 @@ abstract class FlipbookIterator[A] extends BufferedIterator[A] { self => val sm = new StateMachine[Muple[A, B]] { val value = Muple(leftDefault, rightDefault) var isValid = true - def advance(): Unit = { + override def advance(): Unit = { left.stage() right.stage() val c = { @@ -342,7 +342,7 @@ abstract class FlipbookIterator[A] extends BufferedIterator[A] { self => } } - def advance(): Unit = { + override def advance(): Unit = { left.advance() setValue() } @@ -440,7 +440,7 @@ abstract class FlipbookIterator[A] extends BufferedIterator[A] { self => class MergeStateMachine extends StateMachine[A] { var value: A = _ var isValid = true - def advance(): Unit = { + override def advance(): Unit = { left.stage() right.stage() val c = { @@ -476,7 +476,7 @@ abstract class FlipbookIterator[A] extends BufferedIterator[A] { self => // head, next, and hasNext are not meant to be used directly, only to enable // FlipbookIterator to be used where an Iterator is expected. - def head: A - def next(): A - def hasNext: Boolean + override def head: A + override def next(): A + override def hasNext: Boolean } diff --git a/hail/hail/src/is/hail/utils/MultiArray2.scala b/hail/hail/src/is/hail/utils/MultiArray2.scala index c72c9114071..737f526d8ae 100644 --- a/hail/hail/src/is/hail/utils/MultiArray2.scala +++ b/hail/hail/src/is/hail/utils/MultiArray2.scala @@ -17,23 +17,23 @@ class MultiArray2[@specialized(Int, Long, Float, Double, Boolean) T]( class Row(val i: Int) extends IndexedSeq[T] { require(i >= 0 && i < n1) - def apply(j: Int): T = { + override def apply(j: Int): T = { if (j < 0 || j >= length) throw new ArrayIndexOutOfBoundsException a(i * n2 + j) } - def length: Int = n2 + override def length: Int = n2 } class Column(val j: Int) extends IndexedSeq[T] { require(j >= 0 && j < n2) - def apply(i: Int): T = { + override def apply(i: Int): T = { if (i < 0 || i >= length) throw new ArrayIndexOutOfBoundsException a(i * n2 + j) } - def length: Int = n1 + override def length: Int = n1 } def row(i: Int) = new Row(i) @@ -73,7 +73,7 @@ class MultiArray2[@specialized(Int, Long, Float, Double, Boolean) T]( new MultiArray2(n1, n2, a.zip(other.a)) } - def iterator: Iterator[T] = a.iterator + override def iterator: Iterator[T] = a.iterator } object MultiArray2 { diff --git a/hail/hail/src/is/hail/utils/NumericImplicits.scala b/hail/hail/src/is/hail/utils/NumericImplicits.scala index 047cacab411..d71acfaeab4 100644 --- a/hail/hail/src/is/hail/utils/NumericImplicits.scala +++ b/hail/hail/src/is/hail/utils/NumericImplicits.scala @@ -30,22 +30,22 @@ trait NumericImplicits { } implicit object subBVectorSVector extends OpSub.Impl2[BVector[Double], SVector, BVector[Double]] { - def apply(a: BVector[Double], b: SVector): BVector[Double] = a - toBVector(b) + override def apply(a: BVector[Double], b: SVector): BVector[Double] = a - toBVector(b) } implicit object subBVectorIndexedRow extends OpSub.Impl2[BVector[Double], IndexedRow, IndexedRow] { - def apply(a: BVector[Double], b: IndexedRow): IndexedRow = + override def apply(a: BVector[Double], b: IndexedRow): IndexedRow = IndexedRow(b.index, a - toBVector(b.vector)) } implicit object addBVectorSVector extends OpAdd.Impl2[BVector[Double], SVector, BVector[Double]] { - def apply(a: BVector[Double], b: SVector): BVector[Double] = a + toBVector(b) + override def apply(a: BVector[Double], b: SVector): BVector[Double] = a + toBVector(b) } implicit object addBVectorIndexedRow extends OpAdd.Impl2[BVector[Double], IndexedRow, IndexedRow] { - def apply(a: BVector[Double], b: IndexedRow): IndexedRow = + override def apply(a: BVector[Double], b: IndexedRow): IndexedRow = IndexedRow(b.index, a + toBVector(b.vector)) } diff --git a/hail/hail/src/is/hail/utils/NumericPair.scala b/hail/hail/src/is/hail/utils/NumericPair.scala index ebb0876e9c5..335382cdb4f 100644 --- a/hail/hail/src/is/hail/utils/NumericPair.scala +++ b/hail/hail/src/is/hail/utils/NumericPair.scala @@ -13,36 +13,36 @@ trait NumericPairImplicits { implicit object IntPair extends NumericPair[Int, java.lang.Integer] { - def box(t: Int): java.lang.Integer = Int.box(t) + override def box(t: Int): java.lang.Integer = Int.box(t) - def unbox(u: java.lang.Integer): Int = Int.unbox(u) + override def unbox(u: java.lang.Integer): Int = Int.unbox(u) val numeric: scala.math.Numeric[Int] = implicitly[Numeric[Int]] } implicit object LongPair extends NumericPair[Long, java.lang.Long] { - def box(t: Long): java.lang.Long = Long.box(t) + override def box(t: Long): java.lang.Long = Long.box(t) - def unbox(u: java.lang.Long): Long = Long.unbox(u) + override def unbox(u: java.lang.Long): Long = Long.unbox(u) val numeric: scala.math.Numeric[Long] = implicitly[Numeric[Long]] } implicit object FloatPair extends NumericPair[Float, java.lang.Float] { - def box(t: Float): java.lang.Float = Float.box(t) + override def box(t: Float): java.lang.Float = Float.box(t) - def unbox(u: java.lang.Float): Float = Float.unbox(u) + override def unbox(u: java.lang.Float): Float = Float.unbox(u) val numeric: scala.math.Numeric[Float] = implicitly[Numeric[Float]] } implicit object DoublePair extends NumericPair[Double, java.lang.Double] { - def box(t: Double): java.lang.Double = Double.box(t) + override def box(t: Double): java.lang.Double = Double.box(t) - def unbox(u: java.lang.Double): Double = Double.unbox(u) + override def unbox(u: java.lang.Double): Double = Double.unbox(u) val numeric: scala.math.Numeric[Double] = implicitly[Numeric[Double]] } diff --git a/hail/hail/src/is/hail/utils/RelativeFuzzyComparable.scala b/hail/hail/src/is/hail/utils/RelativeFuzzyComparable.scala index 5cc572e6b2c..234fdc8ad52 100644 --- a/hail/hail/src/is/hail/utils/RelativeFuzzyComparable.scala +++ b/hail/hail/src/is/hail/utils/RelativeFuzzyComparable.scala @@ -11,13 +11,13 @@ object RelativeFuzzyComparable { afc.relativeEq(tolerance, x, y) implicit object rfcDoubles extends RelativeFuzzyComparable[Double] { - def relativeEq(tolerance: Double, x: Double, y: Double) = D_==(x, y, tolerance) + override def relativeEq(tolerance: Double, x: Double, y: Double) = D_==(x, y, tolerance) } implicit def rfcMaps[K, V](implicit vRFC: RelativeFuzzyComparable[V]) : RelativeFuzzyComparable[Map[K, V]] = new RelativeFuzzyComparable[Map[K, V]] { - def relativeEq(tolerance: Double, x: Map[K, V], y: Map[K, V]) = + override def relativeEq(tolerance: Double, x: Map[K, V], y: Map[K, V]) = x.keySet == y.keySet && x.keys.forall(k => vRFC.relativeEq(tolerance, x(k), y(k))) } } diff --git a/hail/hail/src/is/hail/utils/SpillingCollectIterator.scala b/hail/hail/src/is/hail/utils/SpillingCollectIterator.scala index 07bc110528d..07b48263b1f 100644 --- a/hail/hail/src/is/hail/utils/SpillingCollectIterator.scala +++ b/hail/hail/src/is/hail/utils/SpillingCollectIterator.scala @@ -70,7 +70,7 @@ class SpillingCollectIterator[T: ClassTag] private ( } } - def hasNext: Boolean = { + override def hasNext: Boolean = { if (it == null || !it.hasNext) { if (i >= files.length) { it = null @@ -100,7 +100,7 @@ class SpillingCollectIterator[T: ClassTag] private ( it.hasNext } - def next(): T = { + override def next(): T = { hasNext: Unit it.next() } diff --git a/hail/hail/src/is/hail/utils/package.scala b/hail/hail/src/is/hail/utils/package.scala index dfd2e0bf005..3621bc1ebd9 100644 --- a/hail/hail/src/is/hail/utils/package.scala +++ b/hail/hail/src/is/hail/utils/package.scala @@ -342,9 +342,9 @@ package object utils def rowIterator(r: Row): Iterator[Any] = new Iterator[Any] { var idx: Int = 0 - def hasNext: Boolean = idx < r.size + override def hasNext: Boolean = idx < r.size - def next(): Any = { + override def next(): Any = { val a = r(idx) idx += 1 a @@ -439,7 +439,7 @@ package object utils def dictionaryOrdering[T](ords: Ordering[T]*): Ordering[T] = { new Ordering[T] { - def compare(x: T, y: T): Int = { + override def compare(x: T, y: T): Int = { var i = 0 while (i < ords.size) { val v = ords(i).compare(x, y) diff --git a/hail/hail/src/is/hail/utils/richUtils/ByteTrackingOutputStream.scala b/hail/hail/src/is/hail/utils/richUtils/ByteTrackingOutputStream.scala index d8d89d4c99c..a98e0ad7716 100644 --- a/hail/hail/src/is/hail/utils/richUtils/ByteTrackingOutputStream.scala +++ b/hail/hail/src/is/hail/utils/richUtils/ByteTrackingOutputStream.scala @@ -5,7 +5,7 @@ import java.io.OutputStream class ByteTrackingOutputStream(base: OutputStream) extends OutputStream { var bytesWritten = 0L - def write(c: Int): Unit = { + override def write(c: Int): Unit = { bytesWritten += 1 base.write(c) } diff --git a/hail/hail/src/is/hail/utils/richUtils/RichContextRDD.scala b/hail/hail/src/is/hail/utils/richUtils/RichContextRDD.scala index ecfae69f4c6..d7b53bae4c5 100644 --- a/hail/hail/src/is/hail/utils/richUtils/RichContextRDD.scala +++ b/hail/hail/src/is/hail/utils/richUtils/RichContextRDD.scala @@ -69,7 +69,7 @@ class RichContextRDD[T: ClassTag](crdd: ContextRDD[T]) { new Iterator[T]() { private[this] var cleared: Boolean = false - def hasNext: Boolean = { + override def hasNext: Boolean = { if (!cleared) { cleared = true ctx.region.clear() @@ -77,7 +77,7 @@ class RichContextRDD[T: ClassTag](crdd: ContextRDD[T]) { it.hasNext } - def next(): T = { + override def next(): T = { if (!cleared) { ctx.region.clear() } diff --git a/hail/hail/src/is/hail/utils/richUtils/RichIndexedRowMatrix.scala b/hail/hail/src/is/hail/utils/richUtils/RichIndexedRowMatrix.scala index 65cd8f16dca..bd51518844c 100644 --- a/hail/hail/src/is/hail/utils/richUtils/RichIndexedRowMatrix.scala +++ b/hail/hail/src/is/hail/utils/richUtils/RichIndexedRowMatrix.scala @@ -85,7 +85,8 @@ private class EmptyPartitionIsAZeroMatrixRDD(blocks: RDD[((Int, Int), BDM[Double case Some(p: GridPartitioner) => p } - def compute(split: Partition, context: TaskContext): Iterator[((Int, Int), BDM[Double])] = { + override def compute(split: Partition, context: TaskContext) + : Iterator[((Int, Int), BDM[Double])] = { val p = split.asInstanceOf[BlockPartition] val it = blocks.iterator(split, context) if (it.hasNext) @@ -94,7 +95,7 @@ private class EmptyPartitionIsAZeroMatrixRDD(blocks: RDD[((Int, Int), BDM[Double Iterator.single(p.blockCoordinates -> BDM.zeros[Double](p.blockDims._1, p.blockDims._2)) } - protected def getPartitions: Array[Partition] = + override protected def getPartitions: Array[Partition] = blocks.partitions.map { p => new BlockPartition(p.index, gp.blockCoordinates(p.index), gp.blockDims(p.index)) }.toArray diff --git a/hail/hail/src/is/hail/utils/richUtils/RichIterable.scala b/hail/hail/src/is/hail/utils/richUtils/RichIterable.scala index ab8f956d3d6..1b73c43646f 100644 --- a/hail/hail/src/is/hail/utils/richUtils/RichIterable.scala +++ b/hail/hail/src/is/hail/utils/richUtils/RichIterable.scala @@ -30,40 +30,40 @@ class RichIterable[T](val i: Iterable[T]) extends Serializable { i.iterator.foreachBetween(f)(g) def intersperse[S >: T](sep: S): Iterable[S] = new Iterable[S] { - def iterator = i.iterator.intersperse(sep) + override def iterator = i.iterator.intersperse(sep) } def intersperse[S >: T](start: S, sep: S, end: S): Iterable[S] = new Iterable[S] { - def iterator = i.iterator.intersperse(start, sep, end) + override def iterator = i.iterator.intersperse(start, sep, end) } def +:[S >: T](elt: S): Iterable[S] = new Iterable[S] { - def iterator = Iterator.single(elt) ++ i + override def iterator = Iterator.single(elt) ++ i } def lazyMapWith[T2, S](i2: Iterable[T2], f: (T, T2) => S): Iterable[S] = new Iterable[S] with Serializable { - def iterator: Iterator[S] = new Iterator[S] { + override def iterator: Iterator[S] = new Iterator[S] { val it: Iterator[T] = i.iterator val it2: Iterator[T2] = i2.iterator - def hasNext: Boolean = it.hasNext && it2.hasNext + override def hasNext: Boolean = it.hasNext && it2.hasNext - def next(): S = f(it.next(), it2.next()) + override def next(): S = f(it.next(), it2.next()) } } def lazyMapWith2[T2, T3, S](i2: Iterable[T2], i3: Iterable[T3], f: (T, T2, T3) => S) : Iterable[S] = new Iterable[S] with Serializable { - def iterator: Iterator[S] = new Iterator[S] { + override def iterator: Iterator[S] = new Iterator[S] { val it: Iterator[T] = i.iterator val it2: Iterator[T2] = i2.iterator val it3: Iterator[T3] = i3.iterator - def hasNext: Boolean = it.hasNext && it2.hasNext && it3.hasNext + override def hasNext: Boolean = it.hasNext && it2.hasNext && it3.hasNext - def next(): S = f(it.next(), it2.next(), it3.next()) + override def next(): S = f(it.next(), it2.next(), it3.next()) } } @@ -89,12 +89,12 @@ class RichIterable[T](val i: Iterable[T]) extends Serializable { } def truncatable(delim: String = ", ", toTake: Int = 10): Truncatable = new Truncatable { - def truncate: String = if (i.size > toTake) + override def truncate: String = if (i.size > toTake) i.take(toTake).mkString(delim) + delim + "..." else i.mkString(delim) - def strings: (String, String) = (truncate, i.mkString(delim)) + override def strings: (String, String) = (truncate, i.mkString(delim)) } def counter(): Map[T, Int] = { diff --git a/hail/hail/src/is/hail/utils/richUtils/RichIterator.scala b/hail/hail/src/is/hail/utils/richUtils/RichIterator.scala index c910abb604c..48fe4090361 100644 --- a/hail/hail/src/is/hail/utils/richUtils/RichIterator.scala +++ b/hail/hail/src/is/hail/utils/richUtils/RichIterator.scala @@ -24,9 +24,9 @@ class RichIterator[T](val it: Iterator[T]) extends AnyVal { val bit = it.buffered StagingIterator( new StateMachine[T] { - def value: T = bit.head - def isValid = bit.hasNext - def advance(): Unit = bit.next(): Unit + override def value: T = bit.head + override def isValid = bit.hasNext + override def advance(): Unit = bit.next(): Unit } ) } @@ -45,9 +45,9 @@ class RichIterator[T](val it: Iterator[T]) extends AnyVal { def intersperse[S >: T](sep: S): Iterator[S] = new Iterator[S] { var nextIsSep = false - def hasNext = it.hasNext + override def hasNext = it.hasNext - def next() = { + override def next() = { val n = if (nextIsSep) sep else it.next() nextIsSep = !nextIsSep n @@ -56,9 +56,9 @@ class RichIterator[T](val it: Iterator[T]) extends AnyVal { def intersperse[S >: T](start: S, sep: S, end: S): Iterator[S] = new Iterator[S] { var state = 0 - def hasNext = state != 4 + override def hasNext = state != 4 - def next() = { + override def next() = { state match { case 0 => state = if (it.hasNext) 1 else 3 @@ -117,9 +117,9 @@ class RichIterator[T](val it: Iterator[T]) extends AnyVal { new Iterator[Iterator[T]] { var prev: Iterator[T] = null - def hasNext: Boolean = it.hasNext + override def hasNext: Boolean = it.hasNext - def next(): Iterator[T] = { + override def next(): Iterator[T] = { if (!hasNext) throw new NoSuchElementException("next on empty iterator") @@ -129,9 +129,9 @@ class RichIterator[T](val it: Iterator[T]) extends AnyVal { prev = new Iterator[T] { var i = 0 - def hasNext: Boolean = it.hasNext && i < groupSize + override def hasNext: Boolean = it.hasNext && i < groupSize - def next(): T = { + override def next(): T = { if (!hasNext) throw new NoSuchElementException("next on empty iterator") i += 1 diff --git a/hail/hail/src/is/hail/utils/richUtils/RichRDD.scala b/hail/hail/src/is/hail/utils/richUtils/RichRDD.scala index 74d01f4d4ae..576c2830ef3 100644 --- a/hail/hail/src/is/hail/utils/richUtils/RichRDD.scala +++ b/hail/hail/src/is/hail/utils/richUtils/RichRDD.scala @@ -161,14 +161,14 @@ class RichRDD[T](val r: RDD[T]) extends AnyVal { new RDD[T]( r.sparkContext, FastSeq(new NarrowDependency[T](r) { - def getParents(partitionId: Int): Seq[Int] = FastSeq(keep(partitionId)) + override def getParents(partitionId: Int): Seq[Int] = FastSeq(keep(partitionId)) }), ) { - def getPartitions: Array[Partition] = keep.indices.map { i => + override def getPartitions: Array[Partition] = keep.indices.map { i => SubsetRDDPartition(i, parentPartitions(keep(i))) }.toArray - def compute(split: Partition, context: TaskContext): Iterator[T] = + override def compute(split: Partition, context: TaskContext): Iterator[T] = r.compute(split.asInstanceOf[SubsetRDDPartition].parentPartition, context) @transient override val partitioner: Option[Partitioner] = newPartitioner @@ -193,17 +193,17 @@ class RichRDD[T](val r: RDD[T]) extends AnyVal { new RDD[T]( r.sparkContext, FastSeq(new NarrowDependency[T](r) { - def getParents(partitionId: Int): Seq[Int] = newToOldPI.get(partitionId) match { + override def getParents(partitionId: Int): Seq[Int] = newToOldPI.get(partitionId) match { case Some(oldPI) => Array(oldPI) case None => Array.empty[Int] } }), ) { - def getPartitions: Array[Partition] = Array.tabulate(newNPartitions) { i => + override def getPartitions: Array[Partition] = Array.tabulate(newNPartitions) { i => SupersetRDDPartition(i, newToOldPI.get(i).map(parentPartitions)) } - def compute(split: Partition, context: TaskContext): Iterator[T] = + override def compute(split: Partition, context: TaskContext): Iterator[T] = split.asInstanceOf[SupersetRDDPartition].maybeParentPartition match { case Some(part) => r.compute(part, context) case None => newPIPartition(split.index) diff --git a/hail/hail/src/is/hail/utils/richUtils/RichString.scala b/hail/hail/src/is/hail/utils/richUtils/RichString.scala index e871170e64f..5af8606c9b3 100644 --- a/hail/hail/src/is/hail/utils/richUtils/RichString.scala +++ b/hail/hail/src/is/hail/utils/richUtils/RichString.scala @@ -4,9 +4,9 @@ import is.hail.utils.Truncatable class RichString(val str: String) extends AnyVal { def truncatable(toTake: Int = 60): Truncatable = new Truncatable { - def truncate: String = if (str.length > toTake - 3) str.take(toTake) + "..." else str + override def truncate: String = if (str.length > toTake - 3) str.take(toTake) + "..." else str - def strings: (String, String) = (truncate, str) + override def strings: (String, String) = (truncate, str) } def equalsCaseInsensitive(other: String): Boolean = diff --git a/hail/hail/src/is/hail/variant/Call.scala b/hail/hail/src/is/hail/variant/Call.scala index 1faede6d86f..eae747fd6c1 100644 --- a/hail/hail/src/is/hail/variant/Call.scala +++ b/hail/hail/src/is/hail/variant/Call.scala @@ -444,9 +444,9 @@ object Call extends Serializable { } new IndexedSeq[Int] with Serializable { - def length: Int = nAlleles + override def length: Int = nAlleles - def apply(idx: Int): Int = { + override def apply(idx: Int): Int = { if (idx < 0 || idx >= nAlleles) throw new ArrayIndexOutOfBoundsException(idx) diff --git a/hail/hail/src/is/hail/variant/ReferenceGenome.scala b/hail/hail/src/is/hail/variant/ReferenceGenome.scala index 2b390ca6977..11ad7c133ea 100644 --- a/hail/hail/src/is/hail/variant/ReferenceGenome.scala +++ b/hail/hail/src/is/hail/variant/ReferenceGenome.scala @@ -108,7 +108,8 @@ case class ReferenceGenome( val locusOrdering = { val localContigsIndex = contigsIndex new Ordering[Locus] { - def compare(x: Locus, y: Locus): Int = ReferenceGenome.compare(localContigsIndex, x, y) + override def compare(x: Locus, y: Locus): Int = + ReferenceGenome.compare(localContigsIndex, x, y) } } diff --git a/hail/hail/src/is/hail/variant/RegionValueVariant.scala b/hail/hail/src/is/hail/variant/RegionValueVariant.scala index fe223effb09..2702a2bb0cd 100644 --- a/hail/hail/src/is/hail/variant/RegionValueVariant.scala +++ b/hail/hail/src/is/hail/variant/RegionValueVariant.scala @@ -17,7 +17,7 @@ class RegionValueVariant(rowType: PStruct) extends View { private var cachedAlleles: Array[String] = null private var cachedLocus: Locus = null - def set(address: Long): Unit = { + override def set(address: Long): Unit = { if (!rowType.isFieldDefined(address, locusIdx)) fatal(s"The row field 'locus' cannot have missing values.") if (!rowType.isFieldDefined(address, allelesIdx)) diff --git a/hail/hail/test/src/is/hail/backend/ServiceBackendSuite.scala b/hail/hail/test/src/is/hail/backend/ServiceBackendSuite.scala index e4a54c45a9d..e95015f28dc 100644 --- a/hail/hail/test/src/is/hail/backend/ServiceBackendSuite.scala +++ b/hail/hail/test/src/is/hail/backend/ServiceBackendSuite.scala @@ -241,6 +241,8 @@ class ServiceBackendSuite extends HailSuite with IdiomaticMockito with OptionVal regions = Array("lunar1"), ) + // no idea why this trigger the missing override rule + @SuppressWarnings(Array("org.wartremover.contrib.warts.MissingOverride")) val batchClient = mock[BatchClient] diff --git a/hail/hail/test/src/is/hail/expr/ir/EmitStreamSuite.scala b/hail/hail/test/src/is/hail/expr/ir/EmitStreamSuite.scala index 917cac3350d..67ac11bd135 100644 --- a/hail/hail/test/src/is/hail/expr/ir/EmitStreamSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/EmitStreamSuite.scala @@ -120,7 +120,7 @@ class EmitStreamSuite extends HailSuite { var _eltRegion: Region = _ var eos: Boolean = _ - def init(outerRegion: Region, eltRegion: Region): Unit = _eltRegion = eltRegion + override def init(outerRegion: Region, eltRegion: Region): Unit = _eltRegion = eltRegion override def next(): Long = if (eos || !it.hasNext) { diff --git a/hail/hail/test/src/is/hail/expr/ir/FunctionSuite.scala b/hail/hail/test/src/is/hail/expr/ir/FunctionSuite.scala index 6581a4c99a0..0eca3f1279b 100644 --- a/hail/hail/test/src/is/hail/expr/ir/FunctionSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/FunctionSuite.scala @@ -25,7 +25,7 @@ class ScalaTestCompanion { } object TestRegisterFunctions extends RegistryFunctions { - def registerAll(): Unit = { + override def registerAll(): Unit = { registerIR1("addone", TInt32, TInt32)((_, a, _) => ApplyBinaryPrimOp(Add(), a, I32(1))) registerJavaStaticFunction("compare", Array(TInt32, TInt32), TInt32, null)( classOf[java.lang.Integer], diff --git a/hail/hail/test/src/is/hail/expr/ir/MissingArrayBuilderSuite.scala b/hail/hail/test/src/is/hail/expr/ir/MissingArrayBuilderSuite.scala index fa81fa53108..77a8b76b0bb 100644 --- a/hail/hail/test/src/is/hail/expr/ir/MissingArrayBuilderSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/MissingArrayBuilderSuite.scala @@ -11,7 +11,7 @@ import org.testng.annotations.{DataProvider, Test} class MissingArrayBuilderSuite extends TestNGSuite { def ordering[T <: AnyVal](f: (T, T) => Boolean): AsmFunction2[T, T, Boolean] = new AsmFunction2[T, T, Boolean] { - def apply(i: T, j: T): Boolean = f(i, j) + override def apply(i: T, j: T): Boolean = f(i, j) } def addToArrayBuilder[B <: MissingArrayBuilder, T]( diff --git a/hail/hail/test/src/is/hail/expr/ir/OrderingSuite.scala b/hail/hail/test/src/is/hail/expr/ir/OrderingSuite.scala index 459e4337c85..04fbc108720 100644 --- a/hail/hail/test/src/is/hail/expr/ir/OrderingSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/OrderingSuite.scala @@ -453,7 +453,7 @@ class OrderingSuite extends HailSuite with ScalaCheckDrivenPropertyChecks { val compareGen = for { elt <- arbitrary[Type] - set: Set[Annotation] <- genNonMissingT(ctx, TSet(elt)) + set <- genNonMissingT[Set[Annotation]](ctx, TSet(elt)) v <- genNonMissing(ctx, elt) } yield (elt, set, v) @@ -509,7 +509,7 @@ class OrderingSuite extends HailSuite with ScalaCheckDrivenPropertyChecks { val compareGen = for { tdict <- arbitrary[TDict] - dict: Map[Annotation, Annotation] <- genNonMissingT(ctx, tdict, innerRequired = false) + dict <- genNonMissingT[Map[Annotation, Annotation]](ctx, tdict, innerRequired = false) key <- genNonMissing(ctx, tdict.keyType, innerRequired = false) } yield (tdict, dict, key) diff --git a/hail/hail/test/src/is/hail/expr/ir/PruneSuite.scala b/hail/hail/test/src/is/hail/expr/ir/PruneSuite.scala index becf2444242..5d43809f7e6 100644 --- a/hail/hail/test/src/is/hail/expr/ir/PruneSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/PruneSuite.scala @@ -202,25 +202,25 @@ class PruneSuite extends HailSuite { false, false, new MatrixReader { - def pathsUsed: IndexedSeq[String] = FastSeq() + override def pathsUsed: IndexedSeq[String] = FastSeq() override def columnCount: Option[Int] = None - def partitionCounts: Option[IndexedSeq[Long]] = None + override def partitionCounts: Option[IndexedSeq[Long]] = None - def rowUIDType = TTuple(TInt64, TInt64) - def colUIDType = TTuple(TInt64, TInt64) + override def rowUIDType = TTuple(TInt64, TInt64) + override def colUIDType = TTuple(TInt64, TInt64) - def fullMatrixTypeWithoutUIDs: MatrixType = mat.typ + override def fullMatrixTypeWithoutUIDs: MatrixType = mat.typ - def lower( + override def lower( ctx: ExecuteContext, requestedType: MatrixType, dropCols: Boolean, dropRows: Boolean, ): TableIR = ??? - def toJValue: JValue = ??? + override def toJValue: JValue = ??? override def renderShort(): String = "mr" }, diff --git a/hail/hail/test/src/is/hail/expr/ir/agg/StagedBlockLinkedListSuite.scala b/hail/hail/test/src/is/hail/expr/ir/agg/StagedBlockLinkedListSuite.scala index 67db8f6a79b..26d83a71835 100644 --- a/hail/hail/test/src/is/hail/expr/ir/agg/StagedBlockLinkedListSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/agg/StagedBlockLinkedListSuite.scala @@ -127,8 +127,8 @@ class StagedBlockLinkedListSuite extends HailSuite { private var ptr = 0L - def clear(): Unit = ptr = initF(region) - def addOne(e: E): this.type = { pushF(region, ptr, e); this } + override def clear(): Unit = ptr = initF(region) + override def addOne(e: E): this.type = { pushF(region, ptr, e); this } def ++=(other: BlockLinkedList[E]): this.type = { appendF(region, ptr, other); this } def toIndexedSeq: IndexedSeq[E] = materializeF(region, ptr) diff --git a/hail/hail/test/src/is/hail/utils/FlipbookIteratorSuite.scala b/hail/hail/test/src/is/hail/utils/FlipbookIteratorSuite.scala index eca7c01483c..a8149015829 100644 --- a/hail/hail/test/src/is/hail/utils/FlipbookIteratorSuite.scala +++ b/hail/hail/test/src/is/hail/utils/FlipbookIteratorSuite.scala @@ -34,10 +34,10 @@ class FlipbookIteratorSuite extends HailSuite { def boxOrdView[A: Ordering]: OrderingView[Box[A]] = new OrderingView[Box[A]] { var value: A = _ - def setFiniteValue(a: Box[A]): Unit = + override def setFiniteValue(a: Box[A]): Unit = value = a.value - def compareFinite(a: Box[A]): Int = + override def compareFinite(a: Box[A]): Int = implicitly[Ordering[A]].compare(value, a.value) } @@ -48,18 +48,18 @@ class FlipbookIteratorSuite extends HailSuite { override def knownSize: Int = buf.size - def clear(): Unit = buf.clear() + override def clear(): Unit = buf.clear() - def addOne(x: Box[A]) = { + override def addOne(x: Box[A]) = { buf += x.value this } - def iterator: Iterator[Box[A]] = new Iterator[Box[A]] { + override def iterator: Iterator[Box[A]] = new Iterator[Box[A]] { var i = 0 - def hasNext = i < buf.size + override def hasNext = i < buf.size - def next() = { + override def next() = { box.value = buf(i) i += 1 box @@ -79,7 +79,7 @@ class FlipbookIteratorSuite extends HailSuite { val sm = new StateMachine[Box[A]] { val value: Box[A] = Box[A] var isValid = true - def advance(): Unit = + override def advance(): Unit = if (it.hasNext) value.value = it.next() else From 9b5e325bc1bc88f6f870915632c4da2029d83f13 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 3 Feb 2026 13:49:31 -0500 Subject: [PATCH 26/51] [batch] ensure worker VMs are ipv4 only (#15256) ## Change Description Updates worker VMs to only use ipv4 network stacks. This: - Makes more sense given that they are now behind an ipv4 only cloudNAT - Will hopefully prevent a race condition causing some docker accesses to fail Brief description and justification of what this PR is doing. ## Security Assessment - This change potentially impacts the Hail Batch instance as deployed by Broad Institute in GCP ### Impact Rating - This change has a low security impact ### Impact Description Unlikely to have any security impact. At the margins it's an improvement because it reduces the number of interfaces that the VM opens, and forces the expected "ipv4 interface behind a cloud NAT" networking model. ### Appsec Review - [x] Required: The impact has been assessed and approved by appsec --- batch/batch/cloud/gcp/driver/create_instance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/batch/batch/cloud/gcp/driver/create_instance.py b/batch/batch/cloud/gcp/driver/create_instance.py index 82f4f454b55..452e2f24c16 100644 --- a/batch/batch/cloud/gcp/driver/create_instance.py +++ b/batch/batch/cloud/gcp/driver/create_instance.py @@ -129,6 +129,7 @@ def scheduling() -> dict: 'network': 'global/networks/default', 'subnetwork': f'regions/{region}/subnetworks/default', 'networkTier': 'PREMIUM', + 'stackType': 'IPV4_ONLY', } ], 'scheduling': scheduling(), From 9432b611c2e62d927b2a76edbd2b4fa5dc68d79f Mon Sep 17 00:00:00 2001 From: Patrick Schultz Date: Wed, 4 Feb 2026 10:20:15 -0500 Subject: [PATCH 27/51] [compiler] delete EmitRegion (#15248) ## Change Description Deletes the vestigial `EmitRegion` class. Once upon a time, it was used to package together a MethodBuilder, and the Region argument to the method (which was usually the first argument). But now it confuses more than it helps, like in the common case where we also pass a CodeBuilder, which itself has a reference to a MethodBuilder. ## Security Assessment - This change cannot impact the Hail Batch instance as deployed by Broad Institute in GCP --- .../src/is/hail/expr/ir/ArraySorter.scala | 7 +- hail/hail/src/is/hail/expr/ir/Emit.scala | 20 ++-- .../is/hail/expr/ir/agg/AppendOnlyBTree.scala | 5 +- .../expr/ir/agg/CollectAsSetAggregator.scala | 6 +- .../expr/ir/agg/DownsampleAggregator.scala | 6 +- .../hail/expr/ir/agg/GroupedAggregator.scala | 10 +- .../ir/functions/ApproxCDFFunctions.scala | 6 +- .../expr/ir/functions/ArrayFunctions.scala | 2 +- .../is/hail/expr/ir/functions/Functions.scala | 91 ++++++++++--------- .../expr/ir/functions/LocusFunctions.scala | 41 +++++---- .../expr/ir/functions/MathFunctions.scala | 16 ++-- .../expr/ir/functions/NDArrayFunctions.scala | 36 +++----- .../ir/functions/RandomSeededFunctions.scala | 14 +-- .../functions/ReferenceGenomeFunctions.scala | 23 +++-- .../expr/ir/functions/StringFunctions.scala | 16 ++-- .../expr/ir/functions/UtilFunctions.scala | 10 +- .../is/hail/io/bgen/StagedBGENReader.scala | 22 ++--- .../encoded/EDictAsUnsortedArrayOfPairs.scala | 4 +- .../is/hail/types/encoded/EUnsortedSet.scala | 4 +- .../is/hail/expr/ir/StagedBTreeSuite.scala | 2 +- .../hail/expr/ir/TakeByAggregatorSuite.scala | 3 +- 21 files changed, 171 insertions(+), 173 deletions(-) diff --git a/hail/hail/src/is/hail/expr/ir/ArraySorter.scala b/hail/hail/src/is/hail/expr/ir/ArraySorter.scala index 3267f0913d6..8d676780ab6 100644 --- a/hail/hail/src/is/hail/expr/ir/ArraySorter.scala +++ b/hail/hail/src/is/hail/expr/ir/ArraySorter.scala @@ -7,9 +7,8 @@ import is.hail.types.physical.stypes.interfaces.SIndexableValue import is.hail.types.virtual.{TArray, TDict, TSet, Type} import is.hail.utils.FastSeq -class ArraySorter(r: EmitRegion, array: StagedArrayBuilder) { +class ArraySorter(mb: EmitMethodBuilder[_], r: Value[Region], array: StagedArrayBuilder) { val ti: TypeInfo[_] = array.elt.ti - val mb: EmitMethodBuilder[_] = r.mb private[this] var prunedMissing: Boolean = false @@ -191,8 +190,8 @@ class ArraySorter(r: EmitRegion, array: StagedArrayBuilder) { array.elt.loadedSType.storageType().setRequired(this.prunedMissing || array.eltRequired) ) - arrayType.constructFromElements(cb, r.region, len, deepCopy = false) { (cb, idx) => - array.loadFromIndex(cb, r.region, idx) + arrayType.constructFromElements(cb, r, len, deepCopy = false) { (cb, idx) => + array.loadFromIndex(cb, r, idx) } case td: TDict => PCanonicalDict.coerceArrayCode(toRegion(cb, TArray(td.elementType))) diff --git a/hail/hail/src/is/hail/expr/ir/Emit.scala b/hail/hail/src/is/hail/expr/ir/Emit.scala index a2dd8288548..bd5eb117e68 100644 --- a/hail/hail/src/is/hail/expr/ir/Emit.scala +++ b/hail/hail/src/is/hail/expr/ir/Emit.scala @@ -234,14 +234,6 @@ case class AggContainer( } } -object EmitRegion { - def default(mb: EmitMethodBuilder[_]): EmitRegion = EmitRegion(mb, mb.getCodeParam[Region](1)) -} - -case class EmitRegion(mb: EmitMethodBuilder[_], region: Value[Region]) { - def baseRegion: Value[Region] = mb.getCodeParam[Region](1) -} - object EmitValue { def apply(missing: Option[Value[Boolean]], v: SValue): EmitValue = new EmitValue( @@ -1424,7 +1416,7 @@ class Emit[C](val ctx: EmitContext, val cb: EmitClassBuilder[C]) { val vab = new StagedArrayBuilder(cb, sct, producer.element.required, 0) StreamUtils.writeToArrayBuilder(cb, producer, vab, region) - val sorter = new ArraySorter(EmitRegion(mb, region), vab) + val sorter = new ArraySorter(mb, region, vab) sorter.sort( cb, region, @@ -1524,7 +1516,7 @@ class Emit[C](val ctx: EmitContext, val cb: EmitClassBuilder[C]) { val vab = new StagedArrayBuilder(cb, sct, producer.element.required, 0) StreamUtils.writeToArrayBuilder(cb, producer, vab, region) - val sorter = new ArraySorter(EmitRegion(mb, region), vab) + val sorter = new ArraySorter(mb, region, vab) def lessThan(cb: EmitCodeBuilder, region: Value[Region], l: Value[_], r: Value[_]) : Value[Boolean] = @@ -1550,7 +1542,7 @@ class Emit[C](val ctx: EmitContext, val cb: EmitClassBuilder[C]) { val vab = new StagedArrayBuilder(cb, sct, producer.element.required, 0) StreamUtils.writeToArrayBuilder(cb, producer, vab, region) - val sorter = new ArraySorter(EmitRegion(mb, region), vab) + val sorter = new ArraySorter(mb, region, vab) def lessThan(cb: EmitCodeBuilder, region: Value[Region], l: Value[_], r: Value[_]) : Value[Boolean] = { @@ -1590,7 +1582,7 @@ class Emit[C](val ctx: EmitContext, val cb: EmitClassBuilder[C]) { val sct = SingleCodeType.fromSType(producer.element.st) val sortedElts = new StagedArrayBuilder(cb, sct, producer.element.required, 16) StreamUtils.writeToArrayBuilder(cb, producer, sortedElts, region) - val sorter = new ArraySorter(EmitRegion(mb, region), sortedElts) + val sorter = new ArraySorter(mb, region, sortedElts) def lt(cb: EmitCodeBuilder, region: Value[Region], l: Value[_], r: Value[_]) : Value[Boolean] = { @@ -2787,7 +2779,7 @@ class Emit[C](val ctx: EmitContext, val cb: EmitClassBuilder[C]) { assert(impl.unify(Array.empty[Type], x.argTypes, rt)) val newState = EmitCode.present(mb, state.asRNGState.splitStatic(cb, staticUID)) impl.applyI( - EmitRegion(cb.emb, region), + region, cb, impl.computeReturnEmitType(x.typ, newState.emitType +: codeArgs.map(_.emitType)).st, Seq[Type](), @@ -3542,7 +3534,7 @@ class Emit[C](val ctx: EmitContext, val cb: EmitClassBuilder[C]) { val unified = impl.unify(typeArgs, args.map(_.typ), rt) assert(unified) val retType = impl.computeReturnEmitType(x.typ, codeArgs.map(_.emitType)) - impl.apply(EmitRegion(mb, region), retType.st, typeArgs, errorID, codeArgs: _*) + impl.apply(mb, region, retType.st, typeArgs, errorID, codeArgs: _*) case WritePartition(stream, pctx, writer) => val ctxCode = emit(pctx) diff --git a/hail/hail/src/is/hail/expr/ir/agg/AppendOnlyBTree.scala b/hail/hail/src/is/hail/expr/ir/agg/AppendOnlyBTree.scala index a1411542107..243227cf1e7 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/AppendOnlyBTree.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/AppendOnlyBTree.scala @@ -18,7 +18,7 @@ trait BTreeKey { def copy(cb: EmitCodeBuilder, src: Code[Long], dest: Code[Long]): Unit - def deepCopy(cb: EmitCodeBuilder, er: EmitRegion, src: Code[Long], dest: Code[Long]): Unit + def deepCopy(cb: EmitCodeBuilder, r: Value[Region], src: Code[Long], dest: Code[Long]): Unit def compKeys(cb: EmitCodeBuilder, k1: EmitValue, k2: EmitValue): Value[Int] @@ -430,7 +430,6 @@ class AppendOnlyBTree( val destNode = f.getCodeParam[Long](1) val srcNode = f.getCodeParam[Long](2) - val er = EmitRegion(cb.emb, region) val newNode = cb.newLocal[Long]("new_node") def copyChild(i: Int): Unit = { @@ -448,7 +447,7 @@ class AppendOnlyBTree( (0 until maxElements).foreach { i => cb.if_( hasKey(cb, srcNode, i), { - key.deepCopy(cb, er, destNode, srcNode) + key.deepCopy(cb, region, destNode, srcNode) cb.if_( !isLeaf(cb, srcNode), { copyChild(i) diff --git a/hail/hail/src/is/hail/expr/ir/agg/CollectAsSetAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/CollectAsSetAggregator.scala index 45b7b11e7f5..f018dbebe37 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/CollectAsSetAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/CollectAsSetAggregator.scala @@ -3,9 +3,7 @@ package is.hail.expr.ir.agg import is.hail.annotations.Region import is.hail.asm4s._ import is.hail.backend.ExecuteContext -import is.hail.expr.ir.{ - EmitClassBuilder, EmitCode, EmitCodeBuilder, EmitRegion, EmitValue, IEmitCode, -} +import is.hail.expr.ir.{EmitClassBuilder, EmitCode, EmitCodeBuilder, EmitValue, IEmitCode} import is.hail.expr.ir.orderings.CodeOrdering import is.hail.io._ import is.hail.types.VirtualTypeWithReq @@ -50,7 +48,7 @@ class TypedKey(typ: PType, kb: EmitClassBuilder[_], region: Value[Region]) exten override def copy(cb: EmitCodeBuilder, src: Code[Long], dest: Code[Long]): Unit = cb += Region.copyFrom(src, dest, storageType.byteSize) - override def deepCopy(cb: EmitCodeBuilder, er: EmitRegion, dest: Code[Long], src: Code[Long]) + override def deepCopy(cb: EmitCodeBuilder, r: Value[Region], dest: Code[Long], src: Code[Long]) : Unit = storageType.storeAtAddress( cb, diff --git a/hail/hail/src/is/hail/expr/ir/agg/DownsampleAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/DownsampleAggregator.scala index b6dbe9cdcfe..e1873b8deb9 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/DownsampleAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/DownsampleAggregator.scala @@ -4,7 +4,7 @@ import is.hail.annotations.Region import is.hail.asm4s._ import is.hail.backend.ExecuteContext import is.hail.expr.ir.{ - EmitClassBuilder, EmitCode, EmitCodeBuilder, EmitRegion, EmitValue, IEmitCode, ParamType, + EmitClassBuilder, EmitCode, EmitCodeBuilder, EmitValue, IEmitCode, ParamType, } import is.hail.expr.ir.orderings.CodeOrdering import is.hail.io.{BufferSpec, InputBuffer, OutputBuffer} @@ -38,7 +38,7 @@ class DownsampleBTreeKey(binType: PBaseStruct, pointType: PBaseStruct, kb: EmitC override def copy(cb: EmitCodeBuilder, src: Code[Long], dest: Code[Long]): Unit = cb += Region.copyFrom(src, dest, storageType.byteSize) - override def deepCopy(cb: EmitCodeBuilder, er: EmitRegion, srcc: Code[Long], dest: Code[Long]) + override def deepCopy(cb: EmitCodeBuilder, r: Value[Region], srcc: Code[Long], dest: Code[Long]) : Unit = { val src = cb.newLocal[Long]("dsa_deep_copy_src", srcc) cb.if_( @@ -48,7 +48,7 @@ class DownsampleBTreeKey(binType: PBaseStruct, pointType: PBaseStruct, kb: EmitC storageType.storeAtAddress( cb, dest, - er.region, + r, storageType.loadCheapSCode(cb, src), deepCopy = true, ) diff --git a/hail/hail/src/is/hail/expr/ir/agg/GroupedAggregator.scala b/hail/hail/src/is/hail/expr/ir/agg/GroupedAggregator.scala index 45ec86a4abf..67d4f1783c4 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/GroupedAggregator.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/GroupedAggregator.scala @@ -4,7 +4,7 @@ import is.hail.annotations.Region import is.hail.asm4s._ import is.hail.backend.ExecuteContext import is.hail.expr.ir.{ - EmitClassBuilder, EmitCode, EmitCodeBuilder, EmitRegion, EmitValue, IEmitCode, ParamType, + EmitClassBuilder, EmitCode, EmitCodeBuilder, EmitValue, IEmitCode, ParamType, } import is.hail.expr.ir.orderings.CodeOrdering import is.hail.io._ @@ -107,8 +107,12 @@ class GroupedBTreeKey( deepCopy = false, ) - override def deepCopy(cb: EmitCodeBuilder, er: EmitRegion, dest: Code[Long], srcCode: Code[Long]) - : Unit = { + override def deepCopy( + cb: EmitCodeBuilder, + r: Value[Region], + dest: Code[Long], + srcCode: Code[Long], + ): Unit = { val src = cb.newLocal("ga_deep_copy_src", srcCode) storageType.storeAtAddress( cb, diff --git a/hail/hail/src/is/hail/expr/ir/functions/ApproxCDFFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/ApproxCDFFunctions.scala index a09d4ceaef9..508596de703 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/ApproxCDFFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/ApproxCDFFunctions.scala @@ -61,12 +61,12 @@ object ApproxCDFFunctions extends RegistryFunctions { (_, _, _, _) => SBaseStructPointer(statePType), ) { case (r, cb, _, k: SInt32Value, left: SBaseStructValue, right: SBaseStructValue, _) => - val leftState = makeStateManager(cb, r.region, k.value, left) - val rightState = makeStateManager(cb, r.region, k.value, right) + val leftState = makeStateManager(cb, r, k.value, left) + val rightState = makeStateManager(cb, r, k.value, right) cb += leftState.invoke[ApproxCDFStateManager, Unit]("combOp", rightState) - fromStateManager(cb, r.region, leftState) + fromStateManager(cb, r, leftState) } } } diff --git a/hail/hail/src/is/hail/expr/ir/functions/ArrayFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/ArrayFunctions.scala index 3ede0b5c9a0..ec316e162fb 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/ArrayFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/ArrayFunctions.scala @@ -342,7 +342,7 @@ object ArrayFunctions extends RegistryFunctions { ) val pt = rt.pType.asInstanceOf[PCanonicalArray] val (push, finish) = - pt.constructFromIndicesUnsafe(cb, er.region, len.value, deepCopy = false) + pt.constructFromIndicesUnsafe(cb, er, len.value, deepCopy = false) indices.forEachDefined(cb) { case (cb, pos, idx: SInt32Value) => cb.if_( idx.value < 0 || idx.value >= len.value, diff --git a/hail/hail/src/is/hail/expr/ir/functions/Functions.scala b/hail/hail/src/is/hail/expr/ir/functions/Functions.scala index 6351769d822..665146bc422 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/Functions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/Functions.scala @@ -465,13 +465,13 @@ abstract class RegistryFunctions { calculateReturnType: (Type, Seq[SType]) => SType, typeParameters: Array[Type] = Array.empty, )( - impl: (EmitRegion, EmitCodeBuilder, Seq[Type], SType, Array[SValue], Value[Int]) => SValue + impl: (Value[Region], EmitCodeBuilder, Seq[Type], SType, Array[SValue], Value[Int]) => SValue ): Unit = { IRFunctionRegistry.addJVMFunction( new UnseededMissingnessObliviousJVMFunction(name, typeParameters, valueParameterTypes, returnType, calculateReturnType) { override def apply( - r: EmitRegion, + r: Value[Region], cb: EmitCodeBuilder, returnSType: SType, typeParameters: Seq[Type], @@ -490,13 +490,13 @@ abstract class RegistryFunctions { calculateReturnType: (Type, Seq[SType]) => SType, typeParameters: Array[Type] = Array.empty, )( - impl: (EmitRegion, EmitCodeBuilder, SType, Array[Type], Array[SValue]) => Value[_] + impl: (Value[Region], EmitCodeBuilder, SType, Array[Type], Array[SValue]) => Value[_] ): Unit = { IRFunctionRegistry.addJVMFunction( new UnseededMissingnessObliviousJVMFunction(name, typeParameters, valueParameterTypes, returnType, calculateReturnType) { override def apply( - r: EmitRegion, + r: Value[Region], cb: EmitCodeBuilder, returnSType: SType, typeParameters: Seq[Type], @@ -518,20 +518,21 @@ abstract class RegistryFunctions { calculateReturnType: (Type, Seq[EmitType]) => EmitType, typeParameters: Array[Type] = Array.empty, )( - impl: (EmitRegion, SType, Value[Int], Array[EmitCode]) => EmitCode + impl: (EmitMethodBuilder[_], Value[Region], SType, Value[Int], Array[EmitCode]) => EmitCode ): Unit = { IRFunctionRegistry.addJVMFunction( new UnseededMissingnessAwareJVMFunction(name, typeParameters, valueParameterTypes, returnType, calculateReturnType) { override def apply( - r: EmitRegion, + mb: EmitMethodBuilder[_], + region: Value[Region], rpt: SType, typeParameters: Seq[Type], errorID: Value[Int], args: EmitCode* ): EmitCode = { assert(unify(typeParameters, args.map(_.st.virtualType), rpt.virtualType)) - impl(r, rpt, errorID, args.toArray) + impl(mb, region, rpt, errorID, args.toArray) } } ) @@ -566,13 +567,14 @@ abstract class RegistryFunctions { res } override def apply( - r: EmitRegion, + mb: EmitMethodBuilder[_], + region: Value[Region], rpt: SType, typeParameters: Seq[Type], errorID: Value[Int], args: EmitCode* ): EmitCode = - EmitCode.fromI(r.mb)(cb => apply(cb, r.region, rpt, typeParameters, errorID, args: _*)) + EmitCode.fromI(mb)(cb => apply(cb, region, rpt, typeParameters, errorID, args: _*)) } ) } @@ -646,16 +648,12 @@ abstract class RegistryFunctions { registerSCode(name, valueParameterTypes, returnType, calculateReturnType) { case (r, cb, _, rt, args, _) => val cts = valueParameterTypes.map(ct(_).runtimeClass) - try - unwrapReturn( - cb, - r.region, - rt, - Code.invokeScalaObject(cls, method, cts, args.map(a => wrap(cb, r.region, a).get))( - ct(returnType) - ), + try { + val ret = Code.invokeScalaObject(cls, method, cts, args.map(a => wrap(cb, r, a).get))( + ct(returnType) ) - catch { + unwrapReturn(cb, r, rt, ret) + } catch { case e: Throwable => throw new RuntimeException(s"error while registering function $name", e) } @@ -754,7 +752,7 @@ abstract class RegistryFunctions { rt: Type, pt: (Type, SType) => SType, )( - impl: (EmitRegion, EmitCodeBuilder, SType, SValue, Value[Int]) => SValue + impl: (Value[Region], EmitCodeBuilder, SType, SValue, Value[Int]) => SValue ): Unit = registerSCode(name, Array(mt1), rt, unwrappedApply(pt)) { case (r, cb, _, rt, Array(a1), errorID) => impl(r, cb, rt, a1, errorID) @@ -767,7 +765,7 @@ abstract class RegistryFunctions { rt: Type, pt: (Type, SType) => SType, )( - impl: (EmitRegion, EmitCodeBuilder, Seq[Type], SType, SValue, Value[Int]) => SValue + impl: (Value[Region], EmitCodeBuilder, Seq[Type], SType, SValue, Value[Int]) => SValue ): Unit = registerSCode(name, Array(mt1), rt, unwrappedApply(pt), typeParameters = typeParams) { case (r, cb, typeParams, rt, Array(a1), errorID) => impl(r, cb, typeParams, rt, a1, errorID) @@ -780,7 +778,7 @@ abstract class RegistryFunctions { rt: Type, pt: (Type, SType, SType) => SType, )( - impl: (EmitRegion, EmitCodeBuilder, SType, SValue, SValue, Value[Int]) => SValue + impl: (Value[Region], EmitCodeBuilder, SType, SValue, SValue, Value[Int]) => SValue ): Unit = registerSCode(name, Array(mt1, mt2), rt, unwrappedApply(pt)) { case (r, cb, _, rt, Array(a1, a2), errorID) => impl(r, cb, rt, a1, a2, errorID) @@ -794,7 +792,7 @@ abstract class RegistryFunctions { rt: Type, pt: (Type, SType, SType) => SType, )( - impl: (EmitRegion, EmitCodeBuilder, Seq[Type], SType, SValue, SValue, Value[Int]) => SValue + impl: (Value[Region], EmitCodeBuilder, Seq[Type], SType, SValue, SValue, Value[Int]) => SValue ): Unit = registerSCode(name, Array(mt1, mt2), rt, unwrappedApply(pt), typeParameters = typeParams) { case (r, cb, typeParams, rt, Array(a1, a2), errorID) => @@ -809,7 +807,7 @@ abstract class RegistryFunctions { rt: Type, pt: (Type, SType, SType, SType) => SType, )( - impl: (EmitRegion, EmitCodeBuilder, SType, SValue, SValue, SValue, Value[Int]) => SValue + impl: (Value[Region], EmitCodeBuilder, SType, SValue, SValue, SValue, Value[Int]) => SValue ): Unit = registerSCode(name, Array(mt1, mt2, mt3), rt, unwrappedApply(pt)) { case (r, cb, _, rt, Array(a1, a2, a3), errorID) => impl(r, cb, rt, a1, a2, a3, errorID) @@ -825,7 +823,7 @@ abstract class RegistryFunctions { pt: (Type, SType, SType, SType) => SType, )( impl: ( - EmitRegion, + Value[Region], EmitCodeBuilder, Seq[Type], SType, @@ -849,7 +847,16 @@ abstract class RegistryFunctions { rt: Type, pt: (Type, SType, SType, SType, SType) => SType, )( - impl: (EmitRegion, EmitCodeBuilder, SType, SValue, SValue, SValue, SValue, Value[Int]) => SValue + impl: ( + Value[Region], + EmitCodeBuilder, + SType, + SValue, + SValue, + SValue, + SValue, + Value[Int], + ) => SValue ): Unit = registerSCode(name, Array(mt1, mt2, mt3, mt4), rt, unwrappedApply(pt)) { case (r, cb, _, rt, Array(a1, a2, a3, a4), errorID) => @@ -867,7 +874,7 @@ abstract class RegistryFunctions { pt: (Type, SType, SType, SType, SType) => SType, )( impl: ( - EmitRegion, + Value[Region], EmitCodeBuilder, Seq[Type], SType, @@ -894,7 +901,7 @@ abstract class RegistryFunctions { pt: (Type, SType, SType, SType, SType, SType) => SType, )( impl: ( - EmitRegion, + Value[Region], EmitCodeBuilder, SType, SValue, @@ -922,7 +929,7 @@ abstract class RegistryFunctions { pt: (Type, SType, SType, SType, SType, SType, SType) => SType, )( impl: ( - EmitRegion, + Value[Region], EmitCodeBuilder, SType, SValue, @@ -952,7 +959,7 @@ abstract class RegistryFunctions { pt: (Type, SType, SType, SType, SType, SType, SType, SType) => SType, )( impl: ( - EmitRegion, + Value[Region], EmitCodeBuilder, SType, SValue, @@ -976,7 +983,7 @@ abstract class RegistryFunctions { rt: Type, pt: (Type, SType) => SType, )( - impl: (EmitCodeBuilder, EmitRegion, SType, SValue) => Value[_] + impl: (EmitCodeBuilder, Value[Region], SType, SValue) => Value[_] ): Unit = registerCode(name, Array(mt1), rt, unwrappedApply(pt)) { case (r, cb, rt, _, Array(a1)) => impl(cb, r, rt, a1) @@ -989,7 +996,7 @@ abstract class RegistryFunctions { rt: Type, pt: (Type, SType, SType) => SType, )( - impl: (EmitCodeBuilder, EmitRegion, SType, SValue, SValue) => Value[_] + impl: (EmitCodeBuilder, Value[Region], SType, SValue, SValue) => Value[_] ): Unit = registerCode(name, Array(mt1, mt2), rt, unwrappedApply(pt)) { case (r, cb, rt, _, Array(a1, a2)) => impl(cb, r, rt, a1, a2) @@ -1133,10 +1140,10 @@ abstract class RegistryFunctions { rt: Type, pt: (Type, EmitType, EmitType) => EmitType, )( - impl: (EmitRegion, SType, Value[Int], EmitCode, EmitCode) => EmitCode + impl: (EmitMethodBuilder[_], Value[Region], SType, Value[Int], EmitCode, EmitCode) => EmitCode ): Unit = registerEmitCode(name, Array(mt1, mt2), rt, unwrappedApply(pt)) { - case (r, rt, errorID, Array(a1, a2)) => impl(r, rt, errorID, a1, a2) + case (mb, r, rt, errorID, Array(a1, a2)) => impl(mb, r, rt, errorID, a1, a2) } def registerIR1( @@ -1206,7 +1213,8 @@ sealed abstract class JVMFunction { def computeReturnEmitType(returnType: Type, valueParameterTypes: Seq[EmitType]): EmitType def apply( - mb: EmitRegion, + mb: EmitMethodBuilder[_], + region: Value[Region], returnType: SType, typeParameters: Seq[Type], errorID: Value[Int], @@ -1261,7 +1269,7 @@ abstract class UnseededMissingnessObliviousJVMFunction( ) def apply( - r: EmitRegion, + r: Value[Region], cb: EmitCodeBuilder, returnSType: SType, typeParameters: Seq[Type], @@ -1270,20 +1278,21 @@ abstract class UnseededMissingnessObliviousJVMFunction( ): SValue override def apply( - r: EmitRegion, + mb: EmitMethodBuilder[_], + region: Value[Region], returnType: SType, typeParameters: Seq[Type], errorID: Value[Int], args: EmitCode* ): EmitCode = - EmitCode.fromI(r.mb)(cb => + EmitCode.fromI(mb)(cb => IEmitCode.multiMapEmitCodes(cb, args.toFastSeq) { args => - apply(r, cb, returnType, typeParameters, errorID, args: _*) + apply(region, cb, returnType, typeParameters, errorID, args: _*) } ) def applyI( - r: EmitRegion, + r: Value[Region], cb: EmitCodeBuilder, returnType: SType, typeParameters: Seq[Type], @@ -1305,12 +1314,12 @@ abstract class UnseededMissingnessObliviousJVMFunction( ) methodbuilder.emitSCode(cb => apply( - EmitRegion.default(methodbuilder), + methodbuilder.getCodeParam[Region](1), cb, rpt, typeParameters, methodbuilder.getCodeParam[Int](2), - (0 until args.length).map(i => methodbuilder.getSCodeParam(i + 3)): _* + args.indices.map(i => methodbuilder.getSCodeParam(i + 3)): _* ) ) methodbuilder diff --git a/hail/hail/src/is/hail/expr/ir/functions/LocusFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/LocusFunctions.scala index fd799a7be4c..5b0b8d65f80 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/LocusFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/LocusFunctions.scala @@ -160,8 +160,8 @@ object LocusFunctions extends RegistryFunctions { SCanonicalLocusPointer(PCanonicalLocus(tl.asInstanceOf[TLocus].rg)), ) { case ( - r: EmitRegion, - cb: EmitCodeBuilder, + r, + cb, rt: SCanonicalLocusPointer, inputLocus: SLocusValue, basePairsToAdd: SInt32Value, @@ -182,7 +182,7 @@ object LocusFunctions extends RegistryFunctions { )), ), ) - rt.pType.constructFromContigAndPosition(cb, r.region, contig, newPos) + rt.pType.constructFromContigAndPosition(cb, r, contig, newPos) } registerSCode2( @@ -215,10 +215,10 @@ object LocusFunctions extends RegistryFunctions { VariantMethods.getClass, "minRep", locus.getLocusObj(cb), - Code.checkcast[IndexedSeq[String]](svalueToJavaValue(cb, r.region, alleles)), + Code.checkcast[IndexedSeq[String]](svalueToJavaValue(cb, r, alleles)), ) - emitVariant(cb, r.region, variantTuple, rt) + emitVariant(cb, r, variantTuple, rt) } registerSCode2( @@ -234,8 +234,8 @@ object LocusFunctions extends RegistryFunctions { ).sType, ) { case ( - r: EmitRegion, - cb: EmitCodeBuilder, + r, + cb, SBaseStructPointer(rt: PCanonicalTuple), grouped: SIndexableValue, radiusVal: SFloat64Value, @@ -276,7 +276,7 @@ object LocusFunctions extends RegistryFunctions { ): IEmitCode = { val (pushElement, finish) = - arrayType.constructFromFunctions(cb, r.region, totalLen, deepCopy = false) + arrayType.constructFromFunctions(cb, r, totalLen, deepCopy = false) val offset = cb.newLocal[Int]("locuswindows_offset", 0) val lastCoord = cb.newLocal[Double]("locuswindows_coord") @@ -350,7 +350,7 @@ object LocusFunctions extends RegistryFunctions { rt.constructFromFields( cb, - r.region, + r, FastSeq[EmitCode]( EmitCode.fromI(cb.emb)(cb => addIdxWithCondition(cb) { case (cb, i, idx, coords) => @@ -389,12 +389,12 @@ object LocusFunctions extends RegistryFunctions { val slocus = str.loadString(cb) emitLocus( cb, - r.region, + r, Code.invokeScalaObject2[String, ReferenceGenome, Locus]( locusClass, "parse", slocus, - rgCode(r.mb, rt.rg), + rgCode(cb.emb, rt.rg), ), rt, ) @@ -409,14 +409,14 @@ object LocusFunctions extends RegistryFunctions { PCanonicalLocus(returnType.asInstanceOf[TLocus].rg).sType, ) { case (r, cb, SCanonicalLocusPointer(rt: PCanonicalLocus), contig, pos, _) => - cb += rgCode(r.mb, rt.rg).invoke[String, Int, Unit]( + cb += rgCode(cb.emb, rt.rg).invoke[String, Int, Unit]( "checkLocus", contig.asString.loadString(cb), pos.asInt.value, ) rt.constructFromContigAndPosition( cb, - r.region, + r, contig.asString.loadString(cb), pos.asInt.value, ) @@ -443,9 +443,9 @@ object LocusFunctions extends RegistryFunctions { VariantMethods.getClass, "parse", svar, - rgCode(r.mb, plocus.rg), + rgCode(cb.emb, plocus.rg), ) - emitVariant(cb, r.region, variant, rt) + emitVariant(cb, r, variant, rt) } registerIEmitCode2( @@ -586,21 +586,24 @@ object LocusFunctions extends RegistryFunctions { case (r, cb, SCanonicalLocusPointer(rt: PCanonicalLocus), globalPos, _) => val locus = cb.newLocal[Locus]( "global_pos_locus", - rgCode(r.mb, rt.rg).invoke[Long, Locus]("globalPosToLocus", globalPos.asLong.value), + rgCode(cb.emb, rt.rg).invoke[Long, Locus]("globalPosToLocus", globalPos.asLong.value), ) rt.constructFromContigAndPosition( cb, - r.region, + r, locus.invoke[String]("contig"), locus.invoke[Int]("position"), ) } registerSCode1("locusToGlobalPos", tlocus("T"), TInt64, (_: Type, _: SType) => SInt64) { - case (r, cb, _, locus: SLocusValue, _) => + case (_, cb, _, locus: SLocusValue, _) => val locusObject = locus.getLocusObj(cb) val globalPos = - cb.memoize(rgCode(r.mb, locus.st.rg).invoke[Locus, Long]("locusToGlobalPos", locusObject)) + cb.memoize(rgCode(cb.emb, locus.st.rg).invoke[Locus, Long]( + "locusToGlobalPos", + locusObject, + )) primitive(globalPos) } diff --git a/hail/hail/src/is/hail/expr/ir/functions/MathFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/MathFunctions.scala index 0bc7236727f..a9c5095c48e 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/MathFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/MathFunctions.scala @@ -341,9 +341,9 @@ object MathFunctions extends RegistryFunctions { statsPackageClass, "pgenchisq", x.value, - Code.checkcast[IndexedSeq[Double]](svalueToJavaValue(cb, r.region, w)), - Code.checkcast[IndexedSeq[Int]](svalueToJavaValue(cb, r.region, k)), - Code.checkcast[IndexedSeq[Double]](svalueToJavaValue(cb, r.region, lam)), + Code.checkcast[IndexedSeq[Double]](svalueToJavaValue(cb, r, w)), + Code.checkcast[IndexedSeq[Int]](svalueToJavaValue(cb, r, k)), + Code.checkcast[IndexedSeq[Double]](svalueToJavaValue(cb, r, lam)), sigma.value, maxIterations.value, minAccuracy.value, @@ -352,7 +352,7 @@ object MathFunctions extends RegistryFunctions { DaviesAlgorithm.pType.constructFromFields( cb, - r.region, + r, FastSeq( EmitValue.present(primitive(cb.memoize(res.invoke[Double]("value")))), EmitValue.present(primitive(cb.memoize(res.invoke[Int]("nIterations")))), @@ -439,7 +439,7 @@ object MathFunctions extends RegistryFunctions { fetStruct.constructFromFields( cb, - r.region, + r, FastSeq( EmitValue.present(primitive(cb.memoize(res(0)))), EmitValue.present(primitive(cb.memoize(res(1)))), @@ -473,7 +473,7 @@ object MathFunctions extends RegistryFunctions { chisqStruct.constructFromFields( cb, - r.region, + r, FastSeq( EmitValue.present(primitive(cb.memoize(res(0)))), EmitValue.present(primitive(cb.memoize(res(1)))), @@ -518,7 +518,7 @@ object MathFunctions extends RegistryFunctions { chisqStruct.constructFromFields( cb, - r.region, + r, FastSeq( EmitValue.present(primitive(cb.memoize(res(0)))), EmitValue.present(primitive(cb.memoize(res(1)))), @@ -560,7 +560,7 @@ object MathFunctions extends RegistryFunctions { hweStruct.constructFromFields( cb, - r.region, + r, FastSeq( EmitValue.present(primitive(cb.memoize(res(0)))), EmitValue.present(primitive(cb.memoize(res(1)))), diff --git a/hail/hail/src/is/hail/expr/ir/functions/NDArrayFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/NDArrayFunctions.scala index 46a159154ef..c6608115720 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/NDArrayFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/NDArrayFunctions.scala @@ -212,9 +212,9 @@ object NDArrayFunctions extends RegistryFunctions { TNDArray(TFloat64, Nat(2)), (t, p1, p2) => PCanonicalNDArray(PFloat64Required, 2, true).sType, ) { - case (er, cb, SNDArrayPointer(pt), apc, bpc, errorID) => + case (r, cb, SNDArrayPointer(pt), apc, bpc, errorID) => val (resPCode, info) = - linear_solve(apc.asNDArray, bpc.asNDArray, pt, cb, er.region, errorID) + linear_solve(apc.asNDArray, bpc.asNDArray, pt, cb, r, errorID) cb.if_( info cne 0, cb._fatalWithError( @@ -287,14 +287,14 @@ object NDArrayFunctions extends RegistryFunctions { TNDArray(TFloat64, Nat(2)), (t, p1, p2, p3) => PCanonicalNDArray(PFloat64Required, 2, true).sType, ) { - case (er, cb, SNDArrayPointer(pt), apc, bpc, lower, errorID) => + case (r, cb, SNDArrayPointer(pt), apc, bpc, lower, errorID) => val (resPCode, info) = linear_triangular_solve( apc.asNDArray, bpc.asNDArray, lower.asBoolean, pt, cb, - er.region, + r, errorID, ) cb.if_( @@ -317,7 +317,7 @@ object NDArrayFunctions extends RegistryFunctions { (_, _, _, _) => PCanonicalNDArray(PFloat64Required, 2, true).sType, ) { case ( - er, + r, cb, rst: SNDArrayPointer, block: SNDArrayValue, @@ -325,7 +325,7 @@ object NDArrayFunctions extends RegistryFunctions { upper: SInt64Value, _, ) => - val newBlock = rst.coerceOrCopy(cb, er.region, block, deepCopy = false).asInstanceOf[ + val newBlock = rst.coerceOrCopy(cb, r, block, deepCopy = false).asInstanceOf[ SNDArrayPointerValue ] val IndexedSeq(nRows, nCols) = newBlock.shapes @@ -351,13 +351,11 @@ object NDArrayFunctions extends RegistryFunctions { cb.assign(j, j + 1L) }, // block(i to i, 0 until j) := 0.0 - newBlock.slice(cb, i, (null, j)).coiterateMutate(cb, er.region) { _ => - primitive(0.0d) - }, + newBlock.slice(cb, i, (null, j)).coiterateMutate(cb, r)(_ => primitive(0.0d)), ) // block(iRight until nRows, ::) := 0.0 - newBlock.slice(cb, (iRight, null), ColonIndex).coiterateMutate(cb, er.region) { _ => + newBlock.slice(cb, (iRight, null), ColonIndex).coiterateMutate(cb, r) { _ => primitive(0.0d) } }, @@ -369,7 +367,7 @@ object NDArrayFunctions extends RegistryFunctions { cb.assign(iRight, (nCols.get - upper.value).min(nRows.get)) // block(0 util iLeft, ::) := 0.0 - newBlock.slice(cb, (null, iLeft), ColonIndex).coiterateMutate(cb, er.region) { _ => + newBlock.slice(cb, (null, iLeft), ColonIndex).coiterateMutate(cb, r) { _ => primitive(0.0d) } @@ -383,9 +381,7 @@ object NDArrayFunctions extends RegistryFunctions { cb.assign(j, j + 1) }, // block(i to i, j to nCols) := 0.0 - newBlock.slice(cb, i, (j, null)).coiterateMutate(cb, er.region) { _ => - primitive(0.0d) - }, + newBlock.slice(cb, i, (j, null)).coiterateMutate(cb, r)(_ => primitive(0.0d)), ) }, ) @@ -402,7 +398,7 @@ object NDArrayFunctions extends RegistryFunctions { (_, _, _, _) => PCanonicalNDArray(PFloat64Required, 2, true).sType, ) { case ( - er, + r, cb, rst: SNDArrayPointer, block: SNDArrayValue, @@ -410,7 +406,7 @@ object NDArrayFunctions extends RegistryFunctions { stops: SIndexableValue, _, ) => - val newBlock = rst.coerceOrCopy(cb, er.region, block, deepCopy = false).asInstanceOf[ + val newBlock = rst.coerceOrCopy(cb, r, block, deepCopy = false).asInstanceOf[ SNDArrayPointerValue ] val row = cb.newLocal[Long]("rowIdx") @@ -421,12 +417,8 @@ object NDArrayFunctions extends RegistryFunctions { cb.assign(row, row + 1L), { val start = starts.loadElement(cb, row.toI).getOrAssert(cb).asInt64.value val stop = stops.loadElement(cb, row.toI).getOrAssert(cb).asInt64.value - newBlock.slice(cb, row, (null, start)).coiterateMutate(cb, er.region) { _ => - primitive(0.0d) - } - newBlock.slice(cb, row, (stop, null)).coiterateMutate(cb, er.region) { _ => - primitive(0.0d) - } + newBlock.slice(cb, row, (null, start)).coiterateMutate(cb, r)(_ => primitive(0.0d)) + newBlock.slice(cb, row, (stop, null)).coiterateMutate(cb, r)(_ => primitive(0.0d)) }, ) diff --git a/hail/hail/src/is/hail/expr/ir/functions/RandomSeededFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/RandomSeededFunctions.scala index a9ae2605e7d..1055b35de39 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/RandomSeededFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/RandomSeededFunctions.scala @@ -156,11 +156,11 @@ object RandomSeededFunctions extends RegistryFunctions { val result = rt.pType.constructUninitialized( FastSeq(SizeValueDyn(nRows.value), SizeValueDyn(nCols.value)), cb, - r.region, + r, ) val rng = cb.emb.getThreefryRNG() rngState.copyIntoEngine(cb, rng) - result.coiterateMutate(cb, r.region) { _ => + result.coiterateMutate(cb, r) { _ => primitive(cb.memoize(rng.invoke[Double, Double, Double]( "runif", min.asDouble.value, @@ -236,11 +236,11 @@ object RandomSeededFunctions extends RegistryFunctions { val result = rt.pType.constructUninitialized( FastSeq(SizeValueDyn(nRows.value), SizeValueDyn(nCols.value)), cb, - r.region, + r, ) val rng = cb.emb.getThreefryRNG() rngState.copyIntoEngine(cb, rng) - result.coiterateMutate(cb, r.region) { _ => + result.coiterateMutate(cb, r) { _ => primitive(cb.memoize(rng.invoke[Double, Double, Double]( "rnorm", mean.asDouble.value, @@ -314,7 +314,7 @@ object RandomSeededFunctions extends RegistryFunctions { ) => val rng = cb.emb.getThreefryRNG() rngState.copyIntoEngine(cb, rng) - rt.constructFromElements(cb, r.region, n.value, deepCopy = false) { case (cb, _) => + rt.constructFromElements(cb, r, n.value, deepCopy = false) { case (cb, _) => IEmitCode.present( cb, primitive(cb.memoize(rng.invoke[Double, Double]("rpois", lambda.value))), @@ -436,7 +436,7 @@ object RandomSeededFunctions extends RegistryFunctions { val rng = cb.emb.getThreefryRNG() rngState.copyIntoEngine(cb, rng) val (push, finish) = PCanonicalArray(PInt32(required = true)) - .constructFromFunctions(cb, r.region, colors.loadLength, deepCopy = false) + .constructFromFunctions(cb, r, colors.loadLength, deepCopy = false) cb.if_( colors.hasMissingValues(cb), cb._fatal("rand_multi_hyper: colors may not contain missing values"), @@ -556,7 +556,7 @@ object RandomSeededFunctions extends RegistryFunctions { val arrayRt = rt.asInstanceOf[SIndexablePointer] val (push, finish) = arrayRt.pType.asInstanceOf[PCanonicalArray].constructFromFunctions( cb, - r.region, + r, resultSize, false, ) diff --git a/hail/hail/src/is/hail/expr/ir/functions/ReferenceGenomeFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/ReferenceGenomeFunctions.scala index 682a226a17a..7fda87413fc 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/ReferenceGenomeFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/ReferenceGenomeFunctions.scala @@ -22,12 +22,12 @@ object ReferenceGenomeFunctions extends RegistryFunctions { TBoolean, (_: Type, _: SType) => SBoolean, ) { - case (r, cb, Seq(tlocus: TLocus), _, contig, _) => + case (_, cb, Seq(tlocus: TLocus), _, contig, _) => val scontig = contig.asString.loadString(cb) - primitive(cb.memoize(rgCode(r.mb, tlocus.asInstanceOf[TLocus].rg).invoke[String, Boolean]( - "isValidContig", - scontig, - ))) + primitive(cb.memoize( + rgCode(cb.emb, tlocus.rg) + .invoke[String, Boolean]("isValidContig", scontig) + )) } registerSCode2t( @@ -38,9 +38,9 @@ object ReferenceGenomeFunctions extends RegistryFunctions { TBoolean, (_: Type, _: SType, _: SType) => SBoolean, ) { - case (r, cb, Seq(tlocus: TLocus), _, contig, pos, _) => + case (_, cb, Seq(tlocus: TLocus), _, contig, pos, _) => val scontig = contig.asString.loadString(cb) - primitive(cb.memoize(rgCode(r.mb, tlocus.rg).invoke[String, Int, Boolean]( + primitive(cb.memoize(rgCode(cb.emb, tlocus.rg).invoke[String, Int, Boolean]( "isValidLocus", scontig, pos.asInt.value, @@ -61,7 +61,7 @@ object ReferenceGenomeFunctions extends RegistryFunctions { val scontig = contig.asString.loadString(cb) unwrapReturn( cb, - r.region, + r, st, rgCode(cb.emb, typeParam.rg).invoke[String, Int, Int, Int, String]( "getSequence", @@ -80,9 +80,12 @@ object ReferenceGenomeFunctions extends RegistryFunctions { TInt32, (_: Type, _: SType) => SInt32, ) { - case (r, cb, Seq(tlocus: TLocus), _, contig, _) => + case (_, cb, Seq(tlocus: TLocus), _, contig, _) => val scontig = contig.asString.loadString(cb) - primitive(cb.memoize(rgCode(r.mb, tlocus.rg).invoke[String, Int]("contigLength", scontig))) + primitive(cb.memoize(rgCode(cb.emb, tlocus.rg).invoke[String, Int]( + "contigLength", + scontig, + ))) } registerIR( diff --git a/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala index 8f118f75753..a21a88aa2c3 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala @@ -287,7 +287,7 @@ object StringFunctions extends RegistryFunctions { val thisClass = getClass registerSCode1("length", TString, TInt32, (_: Type, _: SType) => SInt32) { - case (_: EmitRegion, cb, _, s: SStringValue, _) => + case (_, cb, _, s: SStringValue, _) => primitive(cb.memoize(s.loadString(cb).invoke[Int]("length"))) } @@ -299,7 +299,7 @@ object StringFunctions extends RegistryFunctions { TString, (_: Type, _: SType, _: SType, _: SType) => SJavaString, ) { - case (_: EmitRegion, cb, st: SJavaString.type, s, start, end, _) => + case (_, cb, st: SJavaString.type, s, start, end, _) => val str = s.asString.loadString(cb).invoke[Int, Int, String]( "substring", start.asInt.value, @@ -354,7 +354,7 @@ object StringFunctions extends RegistryFunctions { registerSCode1("str", tv("T"), TString, (_: Type, _: SType) => SJavaString) { case (r, cb, st: SJavaString.type, a, _) => - val annotation = svalueToJavaValue(cb, r.region, a) + val annotation = svalueToJavaValue(cb, r, a) val str = cb.emb.getType(a.st.virtualType).invoke[Any, String]("str", annotation) st.construct(cb, str) } @@ -715,8 +715,8 @@ object StringFunctions extends RegistryFunctions { { case (_: Type, _: EmitType, _: EmitType) => EmitType(SInt32, false) }, - ) { case (r: EmitRegion, _, _, e1: EmitCode, e2: EmitCode) => - EmitCode.fromI(r.mb) { cb => + ) { case (mb, _, _, _, e1, e2) => + EmitCode.fromI(mb) { cb => e1.toI(cb).flatMap(cb) { case sc1: SStringValue => e2.toI(cb).flatMap(cb) { case sc2: SStringValue => val n = cb.newLocal("hamming_n", 0) @@ -785,7 +785,7 @@ object StringFunctions extends RegistryFunctions { TTuple(tv("T")), (rType: Type, _: Seq[SType]) => SType.canonical(rType), typeParameters = Array(tv("T")), - ) { case (er, cb, _, resultType, Array(s: SStringValue), _) => + ) { case (r, cb, _, resultType, Array(s: SStringValue), _) => val warnCtx = cb.emb.genFieldThisRef[mutable.HashSet[String]]("parse_json_context") cb.if_(warnCtx.load().isNull, cb.assign(warnCtx, Code.newInstance[mutable.HashSet[String]]())) @@ -793,11 +793,11 @@ object StringFunctions extends RegistryFunctions { JSONAnnotationImpex.getClass, "irImportAnnotation", s.loadString(cb), - er.mb.ecb.getType(resultType.virtualType.asInstanceOf[TTuple].types(0)), + cb.emb.ecb.getType(resultType.virtualType.asInstanceOf[TTuple].types(0)), warnCtx, ) - unwrapReturn(cb, er.region, resultType, row) + unwrapReturn(cb, r, resultType, row) } } } diff --git a/hail/hail/src/is/hail/expr/ir/functions/UtilFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/UtilFunctions.scala index 8abadd59b41..e6b63f8a53c 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/UtilFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/UtilFunctions.scala @@ -215,14 +215,14 @@ object UtilFunctions extends RegistryFunctions { case (_: Type, _: SType, _: SType, _: SType, _: SType) => SBoolean }, ) { - case (er, cb, _, l, r, tol, abs, _) => + case (region, cb, _, l, r, tol, abs, _) => assert( l.st.virtualType == r.st.virtualType, s"\n lt=${l.st.virtualType}\n rt=${r.st.virtualType}", ) - val lb = svalueToJavaValue(cb, er.region, l) - val rb = svalueToJavaValue(cb, er.region, r) - primitive(cb.memoize(er.mb.getType(l.st.virtualType).invoke[ + val lb = svalueToJavaValue(cb, region, l) + val rb = svalueToJavaValue(cb, region, r) + primitive(cb.memoize(cb.emb.getType(l.st.virtualType).invoke[ Any, Any, Double, @@ -517,7 +517,7 @@ object UtilFunctions extends RegistryFunctions { (_: Type, _: SType, _: SType) => SJavaString, ) { case (r, cb, st: SJavaString.type, format, args, _) => - val javaObjArgs = Code.checkcast[Row](svalueToJavaValue(cb, r.region, args)) + val javaObjArgs = Code.checkcast[Row](svalueToJavaValue(cb, r, args)) val formatted = Code.invokeScalaObject2[String, Row, String]( thisClass, "format", diff --git a/hail/hail/src/is/hail/io/bgen/StagedBGENReader.scala b/hail/hail/src/is/hail/io/bgen/StagedBGENReader.scala index 2b42b0f9d7c..0a76ab436f8 100644 --- a/hail/hail/src/is/hail/io/bgen/StagedBGENReader.scala +++ b/hail/hail/src/is/hail/io/bgen/StagedBGENReader.scala @@ -619,7 +619,7 @@ object BGENFunctions extends RegistryFunctions { Array(TVariable("locusType")), ) { case ( - er, + r, cb, Seq(locType), _, @@ -634,7 +634,7 @@ object BGENFunctions extends RegistryFunctions { val path = _path.asString.loadString(cb) val idxPath = _idxPath.asString.loadString(cb) val recoding = - cb.memoize(coerce[Map[String, String]](svalueToJavaValue(cb, er.region, _recoding))) + cb.memoize(coerce[Map[String, String]](svalueToJavaValue(cb, r, _recoding))) val skipInvalidLoci = _skipInvalidLoci.asBoolean.value val bufferSize = _bufferSize.asInt.value @@ -712,11 +712,11 @@ object BGENFunctions extends RegistryFunctions { BufferSpec.wireSpec, ) - def dumpBuffer(cb: EmitCodeBuilder) = { - val sorter = new ArraySorter(er, buffer) + def dumpBuffer() = { + val sorter = new ArraySorter(cb.emb, r, buffer) sorter.sort( cb, - er.region, + r, { case (cb, _, l, r) => val lv = bufferSct.loadToSValue(cb, l).asBaseStruct.subset("locus", "alleles") val rv = bufferSct.loadToSValue(cb, r).asBaseStruct.subset("locus", "alleles") @@ -757,7 +757,7 @@ object BGENFunctions extends RegistryFunctions { nRead < nVariants, { StagedBGENReader.decodeRow( cb, - er.region, + r, cbfis, nSamples, fileIdx, @@ -771,8 +771,8 @@ object BGENFunctions extends RegistryFunctions { // do nothing if missing (invalid locus) }, { case row: SBaseStructValue => - cb.if_(currSize ceq bufferSize, dumpBuffer(cb)) - buffer.add(cb, bufferSct.coerceSCode(cb, row, er.region, false).code) + cb.if_(currSize ceq bufferSize, dumpBuffer()) + buffer.add(cb, bufferSct.coerceSCode(cb, row, r, false).code) cb.assign(currSize, currSize + 1) cb.assign(nWritten, nWritten + 1) }, @@ -780,7 +780,7 @@ object BGENFunctions extends RegistryFunctions { cb.assign(nRead, nRead + 1) }, ) - cb.if_(currSize > 0, dumpBuffer(cb)) + cb.if_(currSize > 0, dumpBuffer()) val ecb = cb.emb.genEmitClass[Unit]("buffer_stream") ecb.cb.addInterface(typeInfo[NoBoxLongIterator].iname) @@ -906,7 +906,7 @@ object BGENFunctions extends RegistryFunctions { ) val nAdded = cb.newLocal[Long]("nAdded", 0) - mergedStream.memoryManagedConsume(er.region, cb) { cb => + mergedStream.memoryManagedConsume(r, cb) { cb => val row = mergedStream.element.toI(cb).getOrAssert(cb).asBaseStruct val key = row.subset("locus", "alleles") val offset = row.loadField(cb, "offset").getOrAssert(cb).asInt64.value @@ -915,7 +915,7 @@ object BGENFunctions extends RegistryFunctions { cb, IEmitCode.present(cb, key), offset, - IEmitCode.present(cb, SStackStruct.constructFromArgs(cb, er.region, TStruct())), + IEmitCode.present(cb, SStackStruct.constructFromArgs(cb, r, TStruct())), ) } cb.if_( diff --git a/hail/hail/src/is/hail/types/encoded/EDictAsUnsortedArrayOfPairs.scala b/hail/hail/src/is/hail/types/encoded/EDictAsUnsortedArrayOfPairs.scala index 58de49ad9cd..76ede0064fc 100644 --- a/hail/hail/src/is/hail/types/encoded/EDictAsUnsortedArrayOfPairs.scala +++ b/hail/hail/src/is/hail/types/encoded/EDictAsUnsortedArrayOfPairs.scala @@ -2,7 +2,7 @@ package is.hail.types.encoded import is.hail.annotations._ import is.hail.asm4s._ -import is.hail.expr.ir.{ArraySorter, EmitCodeBuilder, EmitRegion, StagedArrayBuilder} +import is.hail.expr.ir.{ArraySorter, EmitCodeBuilder, StagedArrayBuilder} import is.hail.io.{InputBuffer, OutputBuffer} import is.hail.types.physical._ import is.hail.types.physical.stypes.{SType, SValue, SingleCodeType} @@ -50,7 +50,7 @@ final case class EDictAsUnsortedArrayOfPairs( ab.add(cb, ab.elt.coerceSCode(cb, res, region, false).code) } - val sorter = new ArraySorter(EmitRegion(cb.emb, region), ab) + val sorter = new ArraySorter(cb.emb, region, ab) def lessThan(cb: EmitCodeBuilder, region: Value[Region], l: Value[_], r: Value[_]) : Value[Boolean] = { val lk = cb.memoize(sct.loadToSValue(cb, l).asBaseStruct.loadField(cb, 0)) diff --git a/hail/hail/src/is/hail/types/encoded/EUnsortedSet.scala b/hail/hail/src/is/hail/types/encoded/EUnsortedSet.scala index 3e09b8fea88..e036a8ddef2 100644 --- a/hail/hail/src/is/hail/types/encoded/EUnsortedSet.scala +++ b/hail/hail/src/is/hail/types/encoded/EUnsortedSet.scala @@ -2,7 +2,7 @@ package is.hail.types.encoded import is.hail.annotations._ import is.hail.asm4s._ -import is.hail.expr.ir.{ArraySorter, EmitCodeBuilder, EmitRegion, StagedArrayBuilder} +import is.hail.expr.ir.{ArraySorter, EmitCodeBuilder, StagedArrayBuilder} import is.hail.io.{InputBuffer, OutputBuffer} import is.hail.types.physical._ import is.hail.types.physical.stypes.{SType, SValue, SingleCodeType} @@ -45,7 +45,7 @@ final case class EUnsortedSet(val elementType: EType, override val required: Boo ab.add(cb, ab.elt.coerceSCode(cb, res, region, false).code) } - val sorter = new ArraySorter(EmitRegion(cb.emb, region), ab) + val sorter = new ArraySorter(cb.emb, region, ab) def lessThan(cb: EmitCodeBuilder, region: Value[Region], l: Value[_], r: Value[_]) : Value[Boolean] = cb.emb.ecb.getOrdering(sct.loadedSType, sct.loadedSType) diff --git a/hail/hail/test/src/is/hail/expr/ir/StagedBTreeSuite.scala b/hail/hail/test/src/is/hail/expr/ir/StagedBTreeSuite.scala index d95cbf5cbf0..7d54317a01d 100644 --- a/hail/hail/test/src/is/hail/expr/ir/StagedBTreeSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/StagedBTreeSuite.scala @@ -48,7 +48,7 @@ class TestBTreeKey(mb: EmitMethodBuilder[_]) extends BTreeKey { override def copy(cb: EmitCodeBuilder, src: Code[Long], dest: Code[Long]): Unit = cb += Region.copyFrom(src, dest, storageType.byteSize) - override def deepCopy(cb: EmitCodeBuilder, er: EmitRegion, src: Code[Long], dest: Code[Long]) + override def deepCopy(cb: EmitCodeBuilder, r: Value[Region], src: Code[Long], dest: Code[Long]) : Unit = copy(cb, src, dest) diff --git a/hail/hail/test/src/is/hail/expr/ir/TakeByAggregatorSuite.scala b/hail/hail/test/src/is/hail/expr/ir/TakeByAggregatorSuite.scala index 629297bb6d8..6fc96486876 100644 --- a/hail/hail/test/src/is/hail/expr/ir/TakeByAggregatorSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/TakeByAggregatorSuite.scala @@ -106,8 +106,7 @@ class TakeByAggregatorSuite extends HailSuite { new TakeByRVAS(VirtualTypeWithReq(PInt32Required), VirtualTypeWithReq(PInt32Required), kb) val ab = new agg.StagedArrayBuilder(PInt32Required, kb, argR) val rt = PCanonicalArray(tba.valueType) - val er = new EmitRegion(fb.apply_method, argR) - val rng = er.mb.newRNG(0) + val rng = fb.apply_method.newRNG(0) fb.emitWithBuilder { cb => tba.createState(cb) From a702d83c05bcbff670a760da4e36052f4345030a Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Thu, 5 Feb 2026 11:22:09 -0500 Subject: [PATCH 28/51] [query] delete unused function registry from python client (#15257) This change does not affect the broad-managed batch service in gcp. --- hail/python/hail/__init__.py | 1 - hail/python/hail/ir/__init__.py | 2 - hail/python/hail/ir/register_functions.py | 1385 --------------------- hail/python/hail/methods/impex.py | 15 +- 4 files changed, 13 insertions(+), 1390 deletions(-) delete mode 100644 hail/python/hail/ir/register_functions.py diff --git a/hail/python/hail/__init__.py b/hail/python/hail/__init__.py index c2caa3dc3f8..6cb0e32ee85 100644 --- a/hail/python/hail/__init__.py +++ b/hail/python/hail/__init__.py @@ -175,7 +175,6 @@ def version() -> str: __all__.extend([x for x in expr.__all__ if not hasattr(builtins, x)]) del builtins -ir.register_functions() ir.register_aggregators() import warnings diff --git a/hail/python/hail/ir/__init__.py b/hail/python/hail/ir/__init__.py index 1c4f9f874f3..727870730e7 100644 --- a/hail/python/hail/ir/__init__.py +++ b/hail/python/hail/ir/__init__.py @@ -211,7 +211,6 @@ MatrixWriter, ) from .register_aggregators import register_aggregators -from .register_functions import register_functions from .renderer import ( CSERenderer, ParensRenderer, @@ -528,7 +527,6 @@ 'finalize_randomness', 'make_filter_and_replace', 'register_aggregators', - 'register_functions', 'subst', 'tensor_shape_to_matrix_shape', 'toArray', diff --git a/hail/python/hail/ir/register_functions.py b/hail/python/hail/ir/register_functions.py deleted file mode 100644 index 12c019b563e..00000000000 --- a/hail/python/hail/ir/register_functions.py +++ /dev/null @@ -1,1385 +0,0 @@ -from hail.expr.nat import NatVariable -from hail.expr.types import dtype, tarray, tfloat32, tfloat64, tint32, tint64, tndarray, tvariable - -from .ir import register_function, register_seeded_function - -vcf_header_type_str = ( - "struct{sampleIDs: array, " - "infoFields: array, " - "formatFields: array, " - "filterAttrs: dict>, " - "infoAttrs: dict>, " - "formatAttrs: dict>, " - "infoFlagFields: array}" -) - - -def register_functions(): - locusVar = tvariable("R", "locus") - - register_function("isValidContig", (dtype("str"),), dtype("bool"), (locusVar,)) - register_function( - "isValidLocus", - ( - dtype("str"), - dtype("int32"), - ), - dtype("bool"), - (locusVar,), - ) - register_function("contigLength", (dtype("str"),), dtype("int32"), (locusVar,)) - register_function( - "getReferenceSequenceFromValidLocus", - ( - dtype("str"), - dtype("int32"), - dtype("int32"), - dtype("int32"), - ), - dtype("str"), - (locusVar,), - ) - register_function( - "getReferenceSequence", - ( - dtype("str"), - dtype("int32"), - dtype("int32"), - dtype("int32"), - ), - dtype("str"), - (locusVar,), - ) - - register_function("parse_json", (dtype("str"),), dtype("tuple(?T)"), (dtype("?T"),)) - - register_function("flatten", (dtype("array>"),), dtype("array")) - register_function( - "difference", - ( - dtype("set"), - dtype("set"), - ), - dtype("set"), - ) - register_function("median", (dtype("set"),), dtype("?T")) - register_function("median", (dtype("array"),), dtype("?T")) - register_function("uniqueMinIndex", (dtype("array"),), dtype("int32")) - register_function("mean", (dtype("array"),), dtype("float64")) - register_function("toFloat32", (dtype("?T:numeric"),), dtype("float32")) - register_function("uniqueMaxIndex", (dtype("array"),), dtype("int32")) - register_function("toSet", (dtype("array"),), dtype("set")) - - def array_floating_point_divide(arg_type, ret_type): - register_function( - "div", - ( - arg_type, - tarray(arg_type), - ), - tarray(ret_type), - ) - register_function("div", (tarray(arg_type), arg_type), tarray(ret_type)) - register_function("div", (tarray(arg_type), tarray(arg_type)), tarray(ret_type)) - - array_floating_point_divide(tint32, tfloat32) - array_floating_point_divide(tint64, tfloat32) - array_floating_point_divide(tfloat32, tfloat32) - array_floating_point_divide(tfloat64, tfloat64) - - def ndarray_floating_point_divide(arg_type, ret_type): - register_function( - "div", - ( - arg_type, - tndarray(arg_type, NatVariable()), - ), - tndarray(ret_type, NatVariable()), - ) - register_function("div", (tndarray(arg_type, NatVariable()), arg_type), tndarray(ret_type, NatVariable())) - register_function( - "div", - (tndarray(arg_type, NatVariable()), tndarray(arg_type, NatVariable())), - tndarray(ret_type, NatVariable()), - ) - - ndarray_floating_point_divide(tint32, tfloat32) - ndarray_floating_point_divide(tint64, tfloat32) - ndarray_floating_point_divide(tfloat32, tfloat32) - ndarray_floating_point_divide(tfloat64, tfloat64) - register_function("values", (dtype("dict"),), dtype("array")) - register_function( - "sliceRight", - ( - dtype("str"), - dtype("int32"), - ), - dtype("str"), - ) - register_function( - "get", - ( - dtype("dict"), - dtype("?key"), - ), - dtype("?value"), - ) - register_function( - "get", - ( - dtype("dict"), - dtype("?key"), - dtype("?value"), - ), - dtype("?value"), - ) - register_function("max", (dtype("array"),), dtype("?T")) - register_function("nanmax", (dtype("array"),), dtype("?T")) - register_function( - "max", - ( - dtype("?T"), - dtype("?T"), - ), - dtype("?T"), - ) - register_function( - "nanmax", - ( - dtype("?T"), - dtype("?T"), - ), - dtype("?T"), - ) - register_function( - "max_ignore_missing", - ( - dtype("?T"), - dtype("?T"), - ), - dtype("?T"), - ) - register_function( - "nanmax_ignore_missing", - ( - dtype("?T"), - dtype("?T"), - ), - dtype("?T"), - ) - register_function("product", (dtype("array"),), dtype("?T")) - register_function("toInt32", (dtype("?T:numeric"),), dtype("int32")) - register_function( - "extend", - ( - dtype("array"), - dtype("array"), - ), - dtype("array"), - ) - register_function("argmin", (dtype("array"),), dtype("int32")) - register_function("toFloat64", (dtype("?T:numeric"),), dtype("float64")) - register_function("sort", (dtype("array"),), dtype("array")) - register_function( - "sort", - ( - dtype("array"), - dtype("bool"), - ), - dtype("array"), - ) - register_function( - "isSubset", - ( - dtype("set"), - dtype("set"), - ), - dtype("bool"), - ) - register_function( - "slice", - ( - dtype("str"), - dtype("int32"), - dtype("int32"), - ), - dtype("str"), - ) - register_function( - "add", - ( - dtype("array"), - dtype("array"), - ), - dtype("array"), - ) - register_function( - "add", - ( - dtype("array"), - dtype("?T"), - ), - dtype("array"), - ) - register_function( - "add", - ( - dtype("?T:numeric"), - dtype("array"), - ), - dtype("array"), - ) - register_function( - "add", - ( - dtype("ndarray"), - dtype("ndarray"), - ), - dtype("ndarray"), - ) - register_function( - "add", - ( - dtype("ndarray"), - dtype("?T"), - ), - dtype("ndarray"), - ) - register_function( - "add", - ( - dtype("?T:numeric"), - dtype("ndarray"), - ), - dtype("ndarray"), - ) - register_function( - "pow", - ( - dtype("array"), - dtype("array"), - ), - dtype("array"), - ) - register_function( - "pow", - ( - dtype("array"), - dtype("?T"), - ), - dtype("array"), - ) - register_function( - "pow", - ( - dtype("?T:numeric"), - dtype("array"), - ), - dtype("array"), - ) - register_function( - "pow", - ( - dtype("ndarray"), - dtype("ndarray"), - ), - dtype("ndarray"), - ) - register_function( - "pow", - ( - dtype("ndarray"), - dtype("?T"), - ), - dtype("ndarray"), - ) - register_function( - "pow", - ( - dtype("?T:numeric"), - dtype("ndarray"), - ), - dtype("ndarray"), - ) - register_function( - "append", - ( - dtype("array"), - dtype("?T"), - ), - dtype("array"), - ) - register_function( - "sliceLeft", - ( - dtype("str"), - dtype("int32"), - ), - dtype("str"), - ) - register_function( - "remove", - ( - dtype("set"), - dtype("?T"), - ), - dtype("set"), - ) - register_function( - "index", - ( - dtype("str"), - dtype("int32"), - ), - dtype("str"), - ) - register_function("indexArray", (dtype("array"), dtype("int32"), dtype("str")), dtype("?T")) - register_function( - "index", - ( - dtype("dict"), - dtype("?key"), - ), - dtype("?value"), - ) - register_function("dictToArray", (dtype("dict"),), dtype("array")) - register_function( - "mod", - ( - dtype("array"), - dtype("array"), - ), - dtype("array"), - ) - register_function( - "mod", - ( - dtype("array"), - dtype("?T"), - ), - dtype("array"), - ) - register_function( - "mod", - ( - dtype("?T:numeric"), - dtype("array"), - ), - dtype("array"), - ) - register_function( - "mod", - ( - dtype("ndarray"), - dtype("ndarray"), - ), - dtype("ndarray"), - ) - register_function( - "mod", - ( - dtype("ndarray"), - dtype("?T"), - ), - dtype("ndarray"), - ) - register_function( - "mod", - ( - dtype("?T:numeric"), - dtype("ndarray"), - ), - dtype("ndarray"), - ) - register_function("dict", (dtype("array"),), dtype("dict")) - register_function("dict", (dtype("set"),), dtype("dict")) - register_function("keys", (dtype("dict"),), dtype("array")) - register_function("min", (dtype("array"),), dtype("?T")) - register_function("nanmin", (dtype("array"),), dtype("?T")) - register_function( - "min", - ( - dtype("?T"), - dtype("?T"), - ), - dtype("?T"), - ) - register_function( - "nanmin", - ( - dtype("?T"), - dtype("?T"), - ), - dtype("?T"), - ) - register_function( - "min_ignore_missing", - ( - dtype("?T"), - dtype("?T"), - ), - dtype("?T"), - ) - register_function( - "nanmin_ignore_missing", - ( - dtype("?T"), - dtype("?T"), - ), - dtype("?T"), - ) - register_function("sum", (dtype("array"),), dtype("?T")) - register_function("toInt64", (dtype("?T:numeric"),), dtype("int64")) - register_function( - "contains", - ( - dtype("dict"), - dtype("?key"), - ), - dtype("bool"), - ) - register_function( - "contains", - ( - dtype("array"), - dtype("?T"), - ), - dtype("bool"), - ) - register_function( - "contains", - ( - dtype("set"), - dtype("?T"), - ), - dtype("bool"), - ) - register_function( - "-", - ( - dtype("array"), - dtype("?T"), - ), - dtype("array"), - ) - register_function( - "-", - ( - dtype("array"), - dtype("array"), - ), - dtype("array"), - ) - register_function( - "-", - ( - dtype("?T:numeric"), - dtype("array"), - ), - dtype("array"), - ) - register_function( - "-", - ( - dtype("ndarray"), - dtype("ndarray"), - ), - dtype("ndarray"), - ) - register_function( - "-", - ( - dtype("ndarray"), - dtype("?T"), - ), - dtype("ndarray"), - ) - register_function( - "-", - ( - dtype("?T:numeric"), - dtype("ndarray"), - ), - dtype("ndarray"), - ) - register_function("addone", (dtype("int32"),), dtype("int32")) - register_function("isEmpty", (dtype("dict"),), dtype("bool")) - register_function("isEmpty", (dtype("array"),), dtype("bool")) - register_function("isEmpty", (dtype("set"),), dtype("bool")) - register_function( - "union", - ( - dtype("set"), - dtype("set"), - ), - dtype("set"), - ) - register_function( - "mul", - ( - dtype("array"), - dtype("array"), - ), - dtype("array"), - ) - register_function( - "mul", - ( - dtype("array"), - dtype("?T"), - ), - dtype("array"), - ) - register_function( - "mul", - ( - dtype("?T:numeric"), - dtype("array"), - ), - dtype("array"), - ) - register_function( - "mul", - ( - dtype("ndarray"), - dtype("ndarray"), - ), - dtype("ndarray"), - ) - register_function( - "mul", - ( - dtype("ndarray"), - dtype("?T"), - ), - dtype("ndarray"), - ) - register_function( - "mul", - ( - dtype("?T:numeric"), - dtype("ndarray"), - ), - dtype("ndarray"), - ) - register_function( - "intersection", - ( - dtype("set"), - dtype("set"), - ), - dtype("set"), - ) - register_function( - "add", - ( - dtype("set"), - dtype("?T"), - ), - dtype("set"), - ) - register_function("argmax", (dtype("array"),), dtype("int32")) - register_function( - "floordiv", - ( - dtype("array"), - dtype("array"), - ), - dtype("array"), - ) - register_function( - "floordiv", - ( - dtype("array"), - dtype("?T"), - ), - dtype("array"), - ) - register_function( - "floordiv", - ( - dtype("?T:numeric"), - dtype("array"), - ), - dtype("array"), - ) - register_function( - "floordiv", - ( - dtype("ndarray"), - dtype("ndarray"), - ), - dtype("ndarray"), - ) - register_function( - "floordiv", - ( - dtype("ndarray"), - dtype("?T"), - ), - dtype("ndarray"), - ) - register_function( - "floordiv", - ( - dtype("?T:numeric"), - dtype("ndarray"), - ), - dtype("ndarray"), - ) - register_function("keySet", (dtype("dict"),), dtype("set")) - register_function("qnorm", (dtype("float64"),), dtype("float64")) - register_function( - "oneHotAlleles", - ( - dtype("call"), - dtype("int32"), - ), - dtype("array"), - ) - register_function( - "dpois", - ( - dtype("float64"), - dtype("float64"), - dtype("bool"), - ), - dtype("float64"), - ) - register_function( - "dpois", - ( - dtype("float64"), - dtype("float64"), - ), - dtype("float64"), - ) - register_function("ploidy", (dtype("call"),), dtype("int32")) - register_function( - "lor", - ( - dtype("bool"), - dtype("bool"), - ), - dtype("bool"), - ) - register_function( - "ppois", - ( - dtype("float64"), - dtype("float64"), - dtype("bool"), - dtype("bool"), - ), - dtype("float64"), - ) - register_function( - "ppois", - ( - dtype("float64"), - dtype("float64"), - ), - dtype("float64"), - ) - register_function("log10", (dtype("float64"),), dtype("float64")) - register_function("isHet", (dtype("call"),), dtype("bool")) - register_function("add_on_contig", (dtype("?T:locus"), dtype("int32")), dtype("?T:locus")) - register_function("contig_idx", (dtype("?T:locus")), dtype("int32")) - register_function("isAutosomalOrPseudoAutosomal", (dtype("?T:locus"),), dtype("bool")) - register_function( - "testCodeUnification", - ( - dtype("?x:numeric"), - dtype("?x:int32"), - ), - dtype("?x"), - ) - register_seeded_function("rand_pois", (dtype("float64"),), dtype("float64")) - register_seeded_function( - "rand_pois", - ( - dtype("int32"), - dtype("float64"), - ), - dtype("array"), - ) - register_function("toFloat32", (dtype("str"),), dtype("float32")) - register_function("toFloat32", (dtype("bool"),), dtype("float32")) - register_function("isAutosomal", (dtype("?T:locus"),), dtype("bool")) - register_function("isPhased", (dtype("call"),), dtype("bool")) - register_function("isHomVar", (dtype("call"),), dtype("bool")) - register_function( - "corr", - ( - dtype("array"), - dtype("array"), - ), - dtype("float64"), - ) - register_function( - "log", - ( - dtype("float64"), - dtype("float64"), - ), - dtype("float64"), - ) - register_function("log", (dtype("float64"),), dtype("float64")) - register_function("foobar2", (), dtype("int32")) - register_function( - "approxEqual", - ( - dtype("float64"), - dtype("float64"), - dtype("float64"), - dtype("bool"), - dtype("bool"), - ), - dtype("bool"), - ) - register_function("includesEnd", (dtype("interval"),), dtype("bool")) - register_function("position", (dtype("?T:locus"),), dtype("int32")) - register_seeded_function( - "rand_unif", - ( - dtype("float64"), - dtype("float64"), - ), - dtype("float64"), - ) - register_seeded_function("rand_int32", (dtype("int32")), dtype("int32")) - register_seeded_function("rand_int64", (dtype("int64"),), dtype("int64")) - register_function("showStr", (dtype("?T"), dtype("int32")), dtype("str")) - register_function("str", (dtype("?T"),), dtype("str")) - register_function( - "valuesSimilar", - ( - dtype("?T"), - dtype("?T"), - dtype('float64'), - dtype('bool'), - ), - dtype("bool"), - ) - register_function( - "replace", - ( - dtype("str"), - dtype("str"), - dtype("str"), - ), - dtype("str"), - ) - register_function("exp", (dtype("float64"),), dtype("float64")) - register_function( - "land", - ( - dtype("bool"), - dtype("bool"), - ), - dtype("bool"), - ) - register_function( - "compare", - ( - dtype("int32"), - dtype("int32"), - ), - dtype("int32"), - ) - register_function("triangle", (dtype("int32"),), dtype("int32")) - register_function( - "Interval", - ( - dtype("?T"), - dtype("?T"), - dtype("bool"), - dtype("bool"), - ), - dtype("interval"), - ) - register_function("contig", (dtype("?T:locus"),), dtype("str")) - register_function("Call", (dtype("bool"),), dtype("call")) - register_function("Call", (dtype("str"),), dtype("call")) - register_function( - "Call", - ( - dtype("int32"), - dtype("bool"), - ), - dtype("call"), - ) - register_function( - "Call", - ( - dtype("int32"), - dtype("int32"), - dtype("bool"), - ), - dtype("call"), - ) - register_function( - "Call", - ( - dtype("array"), - dtype("bool"), - ), - dtype("call"), - ) - register_function( - "qchisqtail", - ( - dtype("float64"), - dtype("float64"), - ), - dtype("float64"), - ) - register_function( - "binomTest", - ( - dtype("int32"), - dtype("int32"), - dtype("float64"), - dtype("int32"), - ), - dtype("float64"), - ) - register_function( - "qpois", - ( - dtype("float64"), - dtype("float64"), - ), - dtype("int32"), - ) - register_function( - "qpois", - ( - dtype("float64"), - dtype("float64"), - dtype("bool"), - dtype("bool"), - ), - dtype("int32"), - ) - register_function( - "dgamma", - ( - dtype("float64"), - dtype("float64"), - dtype("float64"), - dtype("bool"), - ), - dtype("float64"), - ) - register_function( - "pgamma", - ( - dtype("float64"), - dtype("float64"), - dtype("float64"), - dtype("bool"), - dtype("bool"), - ), - dtype("float64"), - ) - register_function( - "qgamma", - ( - dtype("float64"), - dtype("float64"), - dtype("float64"), - dtype("bool"), - dtype("bool"), - ), - dtype("float64"), - ) - register_function("is_finite", (dtype("float32"),), dtype("bool")) - register_function("is_finite", (dtype("float64"),), dtype("bool")) - register_function("inYPar", (dtype("?T:locus"),), dtype("bool")) - register_function( - "contingency_table_test", - ( - dtype("int32"), - dtype("int32"), - dtype("int32"), - dtype("int32"), - dtype("int32"), - ), - dtype("struct{p_value: float64, odds_ratio: float64}"), - ) - register_function("toInt32", (dtype("bool"),), dtype("int32")) - register_function("toInt32", (dtype("str"),), dtype("int32")) - register_function("foobar1", (), dtype("int32")) - register_function("toFloat64", (dtype("str"),), dtype("float64")) - register_function("toFloat64", (dtype("bool"),), dtype("float64")) - register_function( - "dbeta", - ( - dtype("float64"), - dtype("float64"), - dtype("float64"), - ), - dtype("float64"), - ) - register_function("Locus", (dtype("str"),), dtype("?T:locus")) - register_function( - "Locus", - ( - dtype("str"), - dtype("int32"), - ), - dtype("?T:locus"), - ) - register_function("LocusAlleles", (dtype("str"),), dtype("struct{locus: ?T, alleles: array}")) - register_function( - "LocusInterval", - ( - dtype("str"), - dtype("bool"), - ), - dtype("interval"), - ) - register_function( - "LocusInterval", - ( - dtype("str"), - dtype("int32"), - dtype("int32"), - dtype("bool"), - dtype("bool"), - dtype("bool"), - ), - dtype("interval"), - ) - register_function("globalPosToLocus", (dtype("int64"),), dtype("?T:locus")) - register_function("locusToGlobalPos", (dtype("?T:locus"),), dtype("int64")) - register_function( - "liftoverLocus", - ( - dtype("?T:locus"), - dtype('float64'), - ), - dtype("struct{result:?U:locus,is_negative_strand:bool}"), - ) - register_function( - "liftoverLocusInterval", - ( - dtype("interval"), - dtype('float64'), - ), - dtype("struct{result:interval,is_negative_strand:bool}"), - ) - register_function( - "min_rep", - ( - dtype("?T:locus"), - dtype("array"), - ), - dtype("struct{locus: ?T, alleles: array}"), - ) - register_function( - "locus_windows_per_contig", - ( - dtype("array>"), - dtype("float64"), - ), - dtype("tuple(array, array)"), - ) - register_function("toBoolean", (dtype("str"),), dtype("bool")) - register_seeded_function("rand_bool", (dtype("float64"),), dtype("bool")) - register_function( - "pchisqtail", - ( - dtype("float64"), - dtype("float64"), - ), - dtype("float64"), - ) - register_seeded_function("rand_cat", (dtype("array"),), dtype("int32")) - register_function("inYNonPar", (dtype("?T:locus"),), dtype("bool")) - register_function( - "concat", - ( - dtype("str"), - dtype("str"), - ), - dtype("str"), - ) - register_function( - "pow", - ( - dtype("float32"), - dtype("float32"), - ), - dtype("float64"), - ) - register_function( - "pow", - ( - dtype("int32"), - dtype("int32"), - ), - dtype("float64"), - ) - register_function( - "pow", - ( - dtype("int64"), - dtype("int64"), - ), - dtype("float64"), - ) - register_function( - "pow", - ( - dtype("float64"), - dtype("float64"), - ), - dtype("float64"), - ) - register_function("length", (dtype("str"),), dtype("int32")) - register_function( - "slice", - ( - dtype("str"), - dtype("int32"), - dtype("int32"), - ), - dtype("str"), - ) - register_function( - "split", - ( - dtype("str"), - dtype("str"), - dtype("int32"), - ), - dtype("array"), - ) - register_function( - "split", - ( - dtype("str"), - dtype("str"), - ), - dtype("array"), - ) - register_function( - "splitQuotedChar", - ( - dtype("str"), - dtype("str"), - dtype("array"), - dtype("str"), - ), - dtype("array"), - ) - register_function( - "splitQuotedRegex", - ( - dtype("str"), - dtype("str"), - dtype("array"), - dtype("str"), - ), - dtype("array"), - ) - register_function( - "splitChar", - ( - dtype("str"), - dtype("str"), - dtype("array"), - ), - dtype("array"), - ) - register_function( - "splitRegex", - ( - dtype("str"), - dtype("str"), - dtype("array"), - ), - dtype("array"), - ) - register_seeded_function( - "rand_gamma", - ( - dtype("float64"), - dtype("float64"), - ), - dtype("float64"), - ) - register_function("UnphasedDiploidGtIndexCall", (dtype("int32"),), dtype("call")) - register_function( - "lgt_to_gt", - ( - dtype("call"), - dtype("array"), - ), - dtype("call"), - ) - register_function( - "local_to_global_g", - (dtype("array"), dtype("array"), dtype("int32"), dtype("?T")), - dtype("array"), - ) - register_function( - "local_to_global_a_r", - (dtype("array"), dtype("array"), dtype("int32"), dtype("?T"), dtype("bool")), - dtype("array"), - ) - register_function( - "index", - ( - dtype("call"), - dtype("int32"), - ), - dtype("int32"), - ) - register_function("sign", (dtype("int64"),), dtype("int64")) - register_function("sign", (dtype("float64"),), dtype("float64")) - register_function("sign", (dtype("float32"),), dtype("float32")) - register_function("sign", (dtype("int32"),), dtype("int32")) - register_function("unphasedDiploidGtIndex", (dtype("call"),), dtype("int32")) - register_function("gamma", (dtype("float64"),), dtype("float64")) - register_function( - "mod", - ( - dtype("float64"), - dtype("float64"), - ), - dtype("float64"), - ) - register_function( - "mod", - ( - dtype("int64"), - dtype("int64"), - ), - dtype("int64"), - ) - register_function( - "mod", - ( - dtype("float32"), - dtype("float32"), - ), - dtype("float32"), - ) - register_function( - "mod", - ( - dtype("int32"), - dtype("int32"), - ), - dtype("int32"), - ) - register_function( - "fisher_exact_test", - ( - dtype("int32"), - dtype("int32"), - dtype("int32"), - dtype("int32"), - ), - dtype("struct{p_value: float64, odds_ratio: float64, ci_95_lower: float64, ci_95_upper: float64}"), - ) - register_function("floor", (dtype("float64"),), dtype("float64")) - register_function("floor", (dtype("float32"),), dtype("float32")) - register_function("isNonRef", (dtype("call"),), dtype("bool")) - register_function("includesStart", (dtype("interval"),), dtype("bool")) - register_function("isHetNonRef", (dtype("call"),), dtype("bool")) - register_function( - "hardy_weinberg_test", - (dtype("int32"), dtype("int32"), dtype("int32"), dtype("bool")), - dtype("struct{het_freq_hwe: float64, p_value: float64}"), - ) - register_function("haplotype_freq_em", (dtype("array"),), dtype("array")) - register_function("nNonRefAlleles", (dtype("call"),), dtype("int32")) - register_function("abs", (dtype("float64"),), dtype("float64")) - register_function("abs", (dtype("float32"),), dtype("float32")) - register_function("abs", (dtype("int64"),), dtype("int64")) - register_function("abs", (dtype("int32"),), dtype("int32")) - register_function( - "endswith", - ( - dtype("str"), - dtype("str"), - ), - dtype("bool"), - ) - register_function("sqrt", (dtype("float64"),), dtype("float64")) - register_function("isnan", (dtype("float32"),), dtype("bool")) - register_function("isnan", (dtype("float64"),), dtype("bool")) - register_function("lower", (dtype("str"),), dtype("str")) - register_seeded_function( - "rand_beta", - ( - dtype("float64"), - dtype("float64"), - ), - dtype("float64"), - ) - register_seeded_function( - "rand_beta", - ( - dtype("float64"), - dtype("float64"), - dtype("float64"), - dtype("float64"), - ), - dtype("float64"), - ) - register_function("toInt64", (dtype("bool"),), dtype("int64")) - register_function("toInt64", (dtype("str"),), dtype("int64")) - register_function("testCodeUnification2", (dtype("?x"),), dtype("?x")) - register_function( - "contains", - ( - dtype("str"), - dtype("str"), - ), - dtype("bool"), - ) - register_function( - "contains", - ( - dtype("interval"), - dtype("?T"), - ), - dtype("bool"), - ) - register_function("entropy", (dtype("str"),), dtype("float64")) - register_function( - "filtering_allele_frequency", - ( - dtype("int32"), - dtype("int32"), - dtype("float64"), - ), - dtype("float64"), - ) - register_function("gqFromPL", (dtype("array"),), dtype("int32")) - register_function( - "startswith", - ( - dtype("str"), - dtype("str"), - ), - dtype("bool"), - ) - register_function("ceil", (dtype("float32"),), dtype("float32")) - register_function("ceil", (dtype("float64"),), dtype("float64")) - register_function("json", (dtype("?T"),), dtype("str")) - register_function("strip", (dtype("str"),), dtype("str")) - register_function( - "firstMatchIn", - ( - dtype("str"), - dtype("str"), - ), - dtype("array"), - ) - register_function("isEmpty", (dtype("interval"),), dtype("bool")) - register_function( - "~", - ( - dtype("str"), - dtype("str"), - ), - dtype("bool"), - ) - register_function( - "mkString", - ( - dtype("set"), - dtype("str"), - ), - dtype("str"), - ) - register_function( - "mkString", - ( - dtype("array"), - dtype("str"), - ), - dtype("str"), - ) - register_function("dosage", (dtype("array"),), dtype("float64")) - register_function("upper", (dtype("str"),), dtype("str")) - register_function( - "overlaps", - ( - dtype("interval"), - dtype("interval"), - ), - dtype("bool"), - ) - register_function( - "downcode", - ( - dtype("call"), - dtype("int32"), - ), - dtype("call"), - ) - register_function("inXPar", (dtype("?T:locus"),), dtype("bool")) - register_function( - "format", - ( - dtype("str"), - dtype("?T:tuple"), - ), - dtype("str"), - ) - register_function("pnorm", (dtype("float64"),), dtype("float64")) - register_function("is_infinite", (dtype("float32"),), dtype("bool")) - register_function("is_infinite", (dtype("float64"),), dtype("bool")) - register_function("isHetRef", (dtype("call"),), dtype("bool")) - register_function("isMitochondrial", (dtype("?T:locus"),), dtype("bool")) - register_function( - "hamming", - ( - dtype("str"), - dtype("str"), - ), - dtype("int32"), - ) - register_function("end", (dtype("interval"),), dtype("?T")) - register_function("start", (dtype("interval"),), dtype("?T")) - register_function("inXNonPar", (dtype("?T:locus"),), dtype("bool")) - register_function("escapeString", (dtype("str"),), dtype("str")) - register_function("isHomRef", (dtype("call"),), dtype("bool")) - register_seeded_function( - "rand_norm", - ( - dtype("float64"), - dtype("float64"), - ), - dtype("float64"), - ) - register_function( - "chi_squared_test", - ( - dtype("int32"), - dtype("int32"), - dtype("int32"), - dtype("int32"), - ), - dtype("struct{p_value: float64, odds_ratio: float64}"), - ) - register_function("strftime", (dtype("str"), dtype("int64"), dtype("str")), dtype("str")) - register_function("strptime", (dtype("str"), dtype("str"), dtype("str")), dtype("int64")) - register_function( - "index_bgen", - (dtype("str"), dtype("str"), dtype("dict"), dtype('bool'), dtype("int32")), - dtype("int64"), - (dtype("?T"),), - ) - register_function( - "getVCFHeader", - (dtype("str"), dtype("str"), dtype("str"), dtype("str")), - dtype(vcf_header_type_str), - ) diff --git a/hail/python/hail/methods/impex.py b/hail/python/hail/methods/impex.py index 8043e82bcd5..d710f12a6b1 100644 --- a/hail/python/hail/methods/impex.py +++ b/hail/python/hail/methods/impex.py @@ -3235,10 +3235,21 @@ def index_bgen( info(f"indexed {n} sites in {path} at {idx_path}") +vcf_header_type_str = ( + "struct{" + "sampleIDs: array, " + "infoFields: array, " + "formatFields: array, " + "filterAttrs: dict>, " + "infoAttrs: dict>, " + "formatAttrs: dict>, " + "infoFlagFields: array" + "}" +) + + @typecheck(path=expr_str, filter=nullable(expr_str), find=nullable(expr_str), replace=nullable(expr_str)) def get_vcf_header_info(path, filter=None, find=None, replace=None): - from hail.ir.register_functions import vcf_header_type_str - return hl.expr.functions._func( "getVCFHeader", hl.dtype(vcf_header_type_str), From 9ec65536323d68660f5309ed9614226b3d740eab Mon Sep 17 00:00:00 2001 From: kush-chandra Date: Thu, 5 Feb 2026 12:26:50 -0500 Subject: [PATCH 29/51] [batch] Refactor js to remove unsafe-inline CSP (#15191) ## Change Description Fixes https://github.com/hail-is/hail-security/issues/63 There's a lot of copypaste to move inline script into their own files. I also had to remove inline event handlers. ## Security Assessment - This change potentially impacts the Hail Batch instance as deployed by Broad Institute in GCP ### Impact Rating Delete all except the correct answer: - This change has a medium security impact ### Impact Description Replace this content with a description of the impact of the change: - Removes the unsafe-inline policy, so any remaining inline scripts will fail to run. Tested in remote server. ### Appsec Review - [x] Required: The impact has been assessed and approved by appsec --- auth/MANIFEST.in | 1 + auth/auth/auth.py | 20 +- auth/auth/static/js/account-creating.js | 5 + auth/auth/static/js/roles.js | 73 ++++ auth/auth/static/js/user.js | 23 ++ auth/auth/static/js/users.js | 35 ++ auth/auth/templates/account-creating.html | 8 +- auth/auth/templates/roles.html | 90 +--- auth/auth/templates/user.html | 29 +- auth/auth/templates/users.html | 36 +- batch/batch/driver/main.py | 3 + batch/batch/driver/static/js/quotas.js | 6 + batch/batch/driver/templates/quotas.html | 9 +- batch/batch/front_end/front_end.py | 21 +- batch/batch/front_end/static/js/job.js | 11 + .../batch/front_end/static/js/table_search.js | 386 ++++++++++++++++++ batch/batch/front_end/templates/batch.html | 2 +- batch/batch/front_end/templates/batches.html | 2 +- batch/batch/front_end/templates/job.html | 18 +- .../front_end/templates/table_search.html | 385 +---------------- ci/ci/templates/batch.html | 2 +- ci/ci/templates/dev-deploy-table.html | 6 +- ci/ci/templates/index.html | 7 +- ci/ci/templates/job-table.html | 4 +- ci/ci/templates/pr-table.html | 2 +- ci/ci/templates/pr.html | 2 +- devbin/dev_proxy.py | 4 +- monitoring/monitoring/templates/billing.html | 6 +- .../web_common/static/focus_on_keyup.js | 2 + .../web_common/static/js/swagger/index.js | 31 ++ web_common/web_common/static/search_bar.js | 7 + .../web_common/templates/swagger/index.html | 34 +- web_common/web_common/web_common.py | 2 +- 33 files changed, 686 insertions(+), 586 deletions(-) create mode 100644 auth/auth/static/js/account-creating.js create mode 100644 auth/auth/static/js/roles.js create mode 100644 auth/auth/static/js/user.js create mode 100644 auth/auth/static/js/users.js create mode 100644 batch/batch/driver/static/js/quotas.js create mode 100644 batch/batch/front_end/static/js/job.js create mode 100644 batch/batch/front_end/static/js/table_search.js create mode 100644 web_common/web_common/static/js/swagger/index.js diff --git a/auth/MANIFEST.in b/auth/MANIFEST.in index 16688ad841a..fe8d7b8c4df 100644 --- a/auth/MANIFEST.in +++ b/auth/MANIFEST.in @@ -1 +1,2 @@ recursive-include auth/templates * +recursive-include auth/static * diff --git a/auth/auth/auth.py b/auth/auth/auth.py index f88a0dabc1b..719399f551c 100644 --- a/auth/auth/auth.py +++ b/auth/auth/auth.py @@ -9,6 +9,7 @@ from urllib.parse import urlparse import aiohttp_session +import jinja2 import kubernetes_asyncio.client import kubernetes_asyncio.client.rest import kubernetes_asyncio.config @@ -67,6 +68,7 @@ CLOUD = get_global_config()['cloud'] DEFAULT_NAMESPACE = os.environ['HAIL_DEFAULT_NAMESPACE'] +AUTH_ROOT = os.path.dirname(os.path.abspath(__file__)) is_test_deployment = DEFAULT_NAMESPACE != 'default' @@ -241,6 +243,22 @@ async def openapi(request): return await render_template('auth', request, None, 'openapi.yaml', page_context) +@routes.get('/auth/static/js/{filename}') +@web_security_headers +async def fetch_js_file(request): + filename = request.match_info['filename'] + if filename.endswith('.js'): + page_context = {'base_path': deploy_config.base_path('auth')} + try: + response = await render_template('auth', request, None, f'js/{filename}', page_context) + response.content_type = 'application/javascript' + return response + except jinja2.exceptions.TemplateNotFound as error: + raise web.HTTPNotFound() from error + else: + raise web.HTTPNotFound() + + @routes.get('') @routes.get('/') @web_security_headers @@ -1239,7 +1257,7 @@ def run(): ] ) - setup_aiohttp_jinja2(app, 'auth') + setup_aiohttp_jinja2(app, 'auth', jinja2.FileSystemLoader(f'{AUTH_ROOT}/static/')) setup_aiohttp_session(app) setup_common_static_routes(routes) diff --git a/auth/auth/static/js/account-creating.js b/auth/auth/static/js/account-creating.js new file mode 100644 index 00000000000..ea518d93414 --- /dev/null +++ b/auth/auth/static/js/account-creating.js @@ -0,0 +1,5 @@ +let protocol = location.protocol.replace("http", "ws") +let sock = new WebSocket(protocol + location.host + "{{ base_path }}/creating/wait"); +sock.onmessage = function (_event) { + window.location.reload() +} diff --git a/auth/auth/static/js/roles.js b/auth/auth/static/js/roles.js new file mode 100644 index 00000000000..36a5cd63577 --- /dev/null +++ b/auth/auth/static/js/roles.js @@ -0,0 +1,73 @@ +const csrfToken = document.head.querySelector('meta[name="csrf"]')?.getAttribute('value') || ''; + +document.getElementsByName('remove-role-button').forEach(button => { + button.addEventListener('click', (_e) => { removeRole(button.dataset.user, button.dataset.role); }) +}); + +document.getElementsByName('add-role-button').forEach(button => { + button.addEventListener('click', (_e) => { addRole(button.dataset['username-input-id'], button.dataset.role); }) +}); + +async function removeRole(username, roleName) { + if (!confirm(`Are you sure you want to remove the '${roleName}' role from user '${username}'?`)) { + return; + } + + try { + const response = await fetch("{{ base_path }}/api/v1alpha/system_roles/${username}", { + method: 'PATCH', + headers: { + 'Content-Type': 'application/json', + 'X-CSRF-Token': csrfToken + }, + body: JSON.stringify({ + role_removal: roleName + }) + }); + + if (response.ok) { + // Refresh the page to show updated data + window.location.reload(); + } else { + const errorData = await response.json().catch(() => ({})); + alert(`Failed to remove role: ${errorData.message || response.statusText}`); + } + } catch (error) { + console.error('Error removing role:', error); + alert('Failed to remove role. Please try again.'); + } +} + +async function addRole(usernameInputId, roleName) { + const username = document.getElementById(usernameInputId).value.trim(); + + if (!username) { + alert('Please enter a username'); + return; + } + + try { + const response = await fetch("{{ base_path }}/api/v1alpha/system_roles/${username}", { + method: 'PATCH', + headers: { + 'Content-Type': 'application/json', + 'X-CSRF-Token': csrfToken + }, + body: JSON.stringify({ + role_addition: roleName + }) + }); + + if (response.ok) { + // Clear the input and refresh the page + document.getElementById(usernameInputId).value = ''; + window.location.reload(); + } else { + const errorData = await response.json().catch(() => ({})); + alert(`Failed to add role: ${errorData.message || response.statusText}`); + } + } catch (error) { + console.error('Error adding role:', error); + alert('Failed to add role. Please try again.'); + } +} diff --git a/auth/auth/static/js/user.js b/auth/auth/static/js/user.js new file mode 100644 index 00000000000..2d9298b9d1a --- /dev/null +++ b/auth/auth/static/js/user.js @@ -0,0 +1,23 @@ +const tosCheckbox = document.getElementById('tosCheckbox'); +tosCheckbox.addEventListener('load', (_e) => { updateButtons(); }) +tosCheckbox.addEventListener('change', (_e) => { updateButtons(); }) + +function updateButtons() { + const checkbox = document.getElementById('tosCheckbox'); + const signupButton = document.getElementById('signupButton'); + const loginButton = document.getElementById('loginButton'); + + signupButton.disabled = !checkbox.checked; + loginButton.disabled = !checkbox.checked; + + // Update button styles based on state + [signupButton, loginButton].forEach(button => { + if (button.disabled) { + button.classList.add('opacity-50', 'cursor-not-allowed'); + button.classList.remove('hover:bg-blue-700'); + } else { + button.classList.remove('opacity-50', 'cursor-not-allowed'); + button.classList.add('hover:bg-blue-700'); + } + }); +} diff --git a/auth/auth/static/js/users.js b/auth/auth/static/js/users.js new file mode 100644 index 00000000000..ebee6818611 --- /dev/null +++ b/auth/auth/static/js/users.js @@ -0,0 +1,35 @@ +const showActiveCheckbox = document.getElementById('show-active'); +showActiveCheckbox.addEventListener('change', (_e) => { filterTable(); }) +const showDevelopersCheckbox = document.getElementById('show-developers'); +showDevelopersCheckbox.addEventListener('change', (_e) => { filterTable(); }) + +const reactivateUserForm = document.getElementById('reactivate-user-form'); +reactivateUserForm.addEventListener('submit', (_e) => { validateReactivateForm(); }) + +function filterTable() { + const showActive = document.getElementById('show-active').checked; + const showDevelopers = document.getElementById('show-developers').checked; + + const rows = document.querySelectorAll('#users-table tbody tr'); + + rows.forEach(row => { + const state = row.querySelector('[data-state]').dataset.state.trim(); + const isDeveloper = row.querySelector('[data-developer]').dataset.developer.trim(); + + let show = true; + if (showActive && state !== 'active') show = false; + if (showDevelopers && isDeveloper !== '1') show = false; + + row.style.display = show ? '' : 'none'; + }); +} +function validateReactivateForm() { + const userId = document.getElementById('inactive-id').value.trim(); + const username = document.getElementById('inactive-username').value.trim(); + + if (!userId && !username) { + alert('Please provide either a User ID or a Username.'); + return false; + } + return true; +} diff --git a/auth/auth/templates/account-creating.html b/auth/auth/templates/account-creating.html index e4fad1f0d4a..45dd8308aa7 100644 --- a/auth/auth/templates/account-creating.html +++ b/auth/auth/templates/account-creating.html @@ -5,11 +5,7 @@ Creating account for {{ username }} - {% endblock %} diff --git a/auth/auth/templates/roles.html b/auth/auth/templates/roles.html index 676cb81664c..da460aa4695 100644 --- a/auth/auth/templates/roles.html +++ b/auth/auth/templates/roles.html @@ -1,8 +1,11 @@ {% extends "layout.html" %} {% block title %}Roles{% endblock %} +{% block head %} + +{% endblock %} {% block content %} -{% if all_system_role_assignments.keys().__len__() > 0 %} +{% if all_system_role_assignments.keys().__len__() >= 0 %}

System role assignments

@@ -28,8 +31,10 @@

System role assignments

    {% for user in all_system_role_assignments[role]['users'] %}
  • -
-{% endif %} - - + +{% endif %} {% endblock %} diff --git a/auth/auth/templates/user.html b/auth/auth/templates/user.html index cb0d98a911f..d1222c71f17 100644 --- a/auth/auth/templates/user.html +++ b/auth/auth/templates/user.html @@ -32,10 +32,10 @@

Log in to continue

{% else %}

You must sign up or log in to continue

{% endif %} - +
- + {% endif %}

diff --git a/auth/auth/templates/users.html b/auth/auth/templates/users.html index 88fc7e7383e..7e2baf60006 100644 --- a/auth/auth/templates/users.html +++ b/auth/auth/templates/users.html @@ -37,7 +37,7 @@

Create User

Reactivate User

-
+
User ID @@ -93,11 +93,11 @@

Invalidate All User Sessions

Users

@@ -130,33 +130,5 @@

Users

- + {% endblock %} diff --git a/batch/batch/driver/main.py b/batch/batch/driver/main.py index 0d79e33ac6b..c2fbc9fff7a 100644 --- a/batch/batch/driver/main.py +++ b/batch/batch/driver/main.py @@ -93,6 +93,8 @@ log.info(f'REFRESH_INTERVAL_IN_SECONDS {REFRESH_INTERVAL_IN_SECONDS}') +DRIVER_ROOT = os.path.dirname(os.path.abspath(__file__)) + routes = web.RouteTableDef() deploy_config = get_deploy_config() @@ -1789,6 +1791,7 @@ def run(): setup_aiohttp_jinja2(app, 'batch.driver') setup_common_static_routes(routes) + routes.static('/batch_driver/static/js', f'{DRIVER_ROOT}/static/js') app.add_routes(routes) app.router.add_get("/metrics", server_stats) diff --git a/batch/batch/driver/static/js/quotas.js b/batch/batch/driver/static/js/quotas.js new file mode 100644 index 00000000000..6fd91163110 --- /dev/null +++ b/batch/batch/driver/static/js/quotas.js @@ -0,0 +1,6 @@ +if (Object.hasOwn(window, "Plotly")) { + if (document.getElementById('plotly-timeseries') !== undefined) { + var graphJson = JSON.parse(document.getElementById('plotly-timeseries').dataset.graph); + window.Plotly.plot('plotly-timeseries', graphJson, {}); + } +} diff --git a/batch/batch/driver/templates/quotas.html b/batch/batch/driver/templates/quotas.html index 84fbb1264b2..ee3185d3ed7 100644 --- a/batch/batch/driver/templates/quotas.html +++ b/batch/batch/driver/templates/quotas.html @@ -4,13 +4,10 @@

Quotas by Region

{% if plot_json is not none %} +
-
- -
+ +