Changes to include all commits

DAGWorks-Inc · Sep 16, 2023 · 1ef7d11 · 1ef7d11
1 parent 7a5df66
commit 1ef7d11
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 17 deletions.
diff --git a/contrib/docs/compile_docs.py b/contrib/docs/compile_docs.py
@@ -90,22 +90,31 @@ def dataflows_with_commits(dataflows: list[dict]) -> list[dict]:
                 "dag.png",
             ]:
                 continue
-            log_output = subprocess.check_output(
-                [
-                    "git",
-                    "log",
-                    "-1",
-                    "--format='%H %cd'",
-                    "--",
-                    os.path.join(dataflow["path"], file),
-                ],
-                universal_newlines=True,
-            ).strip()
+            log_output = (
+                subprocess.check_output(
+                    [
+                        "git",
+                        "log",
+                        # "-1",
+                        "--format='%H %cd'",
+                        "--",
+                        os.path.join(dataflow["path"], file),
+                    ],
+                    universal_newlines=True,
+                )
+                .strip()
+                .splitlines()
+            )
 
             # Split the output to get commit hash and commit date
             # print(log_output)
-            commit_hash, commit_date = log_output.strip("'").split(" ", 1)
-            dataflow[file] = {"commit": commit_hash, "timestamp": commit_date}
+            commit_hashes = []
+            commit_dates = []
+            for log in log_output:
+                commit_hash, commit_date = log.strip("'").split(" ", 1)
+                commit_hashes.append(commit_hash)
+                commit_dates.append(commit_date)
+            dataflow[file] = {"commit": commit_hashes, "timestamp": commit_dates}
             # print(dataflow)
     return dataflows
 
@@ -231,7 +240,10 @@ def user_dataflows(dataflows_with_everyting: Collect[list[dict]]) -> dict[str, l
             commit_path = df_path.replace("docs", "src/pages/commits")
             os.makedirs(commit_path, exist_ok=True)
             with open(os.path.join(commit_path, "commit.md"), "w") as f:
-                f.write(f"[commit::{single_df['__init__.py']['commit']}]")
+                for commit, ts in zip(
+                    single_df["__init__.py"]["commit"], single_df["__init__.py"]["timestamp"]
+                ):
+                    f.write(f"[commit::{commit}][ts::{ts}]\n")
 
     return result
 

diff --git a/hamilton/dataflows/__init__.py b/hamilton/dataflows/__init__.py
@@ -79,6 +79,20 @@ def resolve_latest(branch: str = "main") -> str:
     return commit_sha
 
 
+def resolve_dataflow_commit(user: str, dataflow: str) -> str:
+    url = f"https://dagworks-inc.github.io/hamilton/commits/users/{user}/{dataflow}/commit/"
+    response = requests.get(url)
+    if response.status_code != 200:
+        raise ValueError(
+            f"Failed to resolve latest commit for {user}/{dataflow}:\n{response.status_code}\n{response.text}"
+        )
+    chunk_index = response.text.find("[commit::")
+    print(chunk_index)
+    commit = response.text[chunk_index + len("[commit::") :]
+    commit_sha = commit[: commit.find("]")]
+    return commit_sha
+
+
 def pull_user_module(user: str, dataflow: str, version: str = "latest", overwrite: bool = False):
     """Pulls a dataflow
 
@@ -91,7 +105,7 @@ def pull_user_module(user: str, dataflow: str, version: str = "latest", overwrit
     """
     print(f"pulling dataflow {user}/{dataflow} with version {version}")
     if version == "latest":
-        version = resolve_latest()
+        version = resolve_dataflow_commit(user, dataflow)
     h_files = ["__init__.py", "requirements.txt", "README.md", "valid_configs.jsonl"]
 
     local_file_path = USER_PATH.format(commit_ish=version, user=user, dataflow=dataflow)
@@ -151,7 +165,7 @@ def inspect(user: str, dataflow: str, version: str = "latest") -> dict:
     - commit hash
     """
     if version == "latest":
-        version = resolve_latest()
+        version = resolve_dataflow_commit(user, dataflow)
 
     local_file_path = USER_PATH.format(commit_ish=version, user=user, dataflow=dataflow)
     if not os.path.exists(local_file_path):
@@ -423,4 +437,4 @@ def init():
 
     pprint.pprint(inspect_module(_module))
     print(install_dependencies(_user, _dataflow, _version))
-    print(resolve_latest())
+    print(resolve_dataflow_commit(_user, _dataflow))