Migrate PyTorch multislice tests (#220)

Author: jonb377

dags/multipod/configs/pytorch_config.py

@@ -0,0 +1,56 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dags.vm_resource import TpuVersion, Zone, DockerImage, ClusterName
+from dags.multipod.configs import gke_config
+from xlml.apis import task
+from typing import List
+
+
+# TODO(jonbolin): Refactor this to cluster definition
+CLUSTER_CONFIG = {
+    ClusterName.V4_8_MULTISLICE_CLUSTER: {
+        'tpu_version': TpuVersion.V4,
+        'tpu_cores': 8,
+        'tpu_zone': Zone.US_CENTRAL2_B.value,
+    },
+    ClusterName.V4_16_MULTISLICE_CLUSTER: {
+        'tpu_version': TpuVersion.V4,
+        'tpu_cores': 16,
+        'tpu_zone': Zone.US_CENTRAL2_B.value,
+    },
+}
+
+
+def get_nightly_pytorch_config(
+    test_name: str,
+    test_owner: str,
+    run_commands: List[str],
+    cluster: ClusterName,
+    num_slices: int,
+) -> task.XpkTask:
+  cmds = (
+      'git clone https://github.com/pytorch/xla /pytorch/xla',
+      *run_commands,
+  )
+  return gke_config.get_gke_config(
+      cluster_name=cluster.value,
+      test_name=test_name,
+      run_model_cmds=cmds,
+      num_slices=num_slices,
+      docker_image=DockerImage.PYTORCH_NIGHTLY.value,
+      test_owner=test_owner,
+      time_out_in_min=60,
+      **CLUSTER_CONFIG[cluster],
+  )

dags/multipod/pytorch.py

@@ -0,0 +1,69 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+A DAG to run PyTorch multislice tests
+"""
+import datetime
+from airflow import models
+from dags import composer_env, test_owner
+from dags.vm_resource import TpuVersion, Zone, DockerImage, ClusterName
+from dags.multipod.configs import pytorch_config
+from xlml.apis import metric_config
+
+# Run once a day at 10 am UTC (3 am PST)
+SCHEDULED_TIME = "0 10 * * *" if composer_env.is_prod_env() else None
+
+with models.DAG(
+    dag_id="pytorch_multislice",
+    schedule=SCHEDULED_TIME,
+    tags=["multipod_team", "pytorch", "nightly"],
+    start_date=datetime.datetime(2024, 3, 1),
+    catchup=False,
+    concurrency=2,
+) as dag:
+  v4_8 = ClusterName.V4_8_MULTISLICE_CLUSTER
+  v4_16 = ClusterName.V4_16_MULTISLICE_CLUSTER
+
+  for num_slices, cluster in [(1, v4_8), (2, v4_8), (1, v4_16)]:
+    ici_chips = 4 if cluster == v4_8 else 8
+    run_cmds = (
+        (
+            "python /pytorch/xla/test/spmd/test_sharding_strategies.py "
+            f"--ici_fsdp_parallelism {ici_chips} "
+            f"--dcn_data_parallelism {num_slices}"
+        ),
+    )
+    pytorch_config.get_nightly_pytorch_config(
+        test_name="shardings",
+        test_owner=test_owner.JON_B,
+        run_commands=run_cmds,
+        cluster=cluster,
+        num_slices=num_slices,
+    ).run()
+
+  pytorch_config.get_nightly_pytorch_config(
+      test_name="checkpoint",
+      test_owner=test_owner.JON_B,
+      run_commands=(
+          f"export CHKPT_PATH={metric_config.SshEnvVars.GCS_OUTPUT.value}",
+          "pip install gcsfs",
+          (
+              "python /pytorch/xla/test/spmd/test_xla_distributed_checkpoint.py "
+              "EndToEndCheckpointTest.test_multihost_checkpoint"
+          ),
+      ),
+      cluster=v4_16,
+      num_slices=2,
+  ).run()

dags/vm_resource.py

@@ -137,6 +137,7 @@ class DockerImage(enum.Enum):
   """Common docker images."""
 
   XPK_JAX_TEST = "gcr.io/cloud-ml-auto-solutions/xpk_jax_test:latest"
+  PYTORCH_NIGHTLY = f"us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:nightly_3.10_tpuvm_{datetime.datetime.today().strftime('%Y%m%d')}"
   MAXTEXT_TPU_JAX_STABLE = f"gcr.io/tpu-prod-env-multipod/maxtext_jax_stable:{datetime.datetime.today().strftime('%Y-%m-%d')}"
   MAXTEXT_TPU_JAX_NIGHTLY = f"gcr.io/tpu-prod-env-multipod/maxtext_jax_nightly:{datetime.datetime.today().strftime('%Y-%m-%d')}"
   MAXTEXT_GPU_JAX_STABLE = f"gcr.io/tpu-prod-env-multipod/maxtext_gpu_jax_stable:{datetime.datetime.today().strftime('%Y-%m-%d')}"