Updating Cloud integration to new directory structure (#10762)

- Closes #10749

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_File.enso

@@ -74,9 +74,22 @@ type Enso_File
     private Value (enso_path : Enso_Path)
 
     ## ICON folder
-       Represents the root folder of the current users.
+       Represents the current user's home directory.
+    home : Enso_File
+    home -> Enso_File =
+        Enso_File.root /  "Users" / Enso_User.current.name
+
+
+    ## ICON folder
+       Represents the root folder of the organization.
+
+       ? Organization Root Directory
+
+         The organization root directory cannot be directly written into.
+         You should put your files and projects in a subdirectory dedicated to a
+         given user or team in the `Users` or `Teams` subdirectories within it.
     root : Enso_File
-    root = Enso_File.Value (Enso_Path.root_for Enso_User.current.organization_name)
+    root = Enso_File.Value Enso_Path.root
 
     ## ICON folder
        Represents the current working directory.
@@ -88,7 +101,7 @@ type Enso_File
        directory.
     current_working_directory : Enso_File
     current_working_directory =
-        Enso_File.cloud_project_parent_directory . if_nothing Enso_File.root
+        Enso_File.cloud_project_parent_directory . if_nothing Enso_File.home
 
     ## PRIVATE
        The parent directory containing the currently open project if in the
@@ -404,10 +417,6 @@ type Enso_File
     / self (name : Text) -> Enso_File ! Not_Found =
         Enso_File.Value (self.enso_path.resolve name)
 
-    ## PRIVATE
-    is_current_user_root self -> Boolean =
-        self.enso_path.is_root && self.enso_path.organization_name == Enso_User.current.organization_name
-
     ## PRIVATE
        Returns the text representation of the file descriptor.
     to_text : Text

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Enso_Path.enso

@@ -17,22 +17,19 @@ from project.Data.Text.Extensions import all
    This is a helper for handling `enso://` paths.
 type Enso_Path
     ## PRIVATE
-    private Value (organization_name : Text) (path_segments : Vector Text)
+    private Value (path_segments : Vector Text)
 
     ## PRIVATE
     parse (path : Text) -> Enso_Path =
         if path.starts_with Enso_Path.protocol_prefix . not then Error.throw (Illegal_Argument.Error "Invalid path - it should start with `enso://`.") else
             raw_segments = path.drop Enso_Path.protocol_prefix.length . split Enso_Path.delimiter
             if raw_segments.is_empty then Error.throw (Illegal_Argument.Error "Invalid path - it should contain at least one segment.") else
-                organization_name = raw_segments.first
-                if organization_name.is_empty then Error.throw (Illegal_Argument.Error "Invalid path - organization name cannot be empty.") else
-                    segments = raw_segments.drop 1 . filter s-> s.is_empty.not
-                    normalized = normalize segments
-                    Enso_Path.Value organization_name normalized
+                segments = raw_segments.filter s-> s.is_empty.not
+                normalized = normalize segments
+                Enso_Path.Value normalized
 
     ## PRIVATE
-    root_for (organization_name : Text) =
-        Enso_Path.Value organization_name []
+    root = Enso_Path.Value []
 
     ## PRIVATE
     is_root self -> Boolean =
@@ -41,23 +38,22 @@ type Enso_Path
     ## PRIVATE
     parent self -> Enso_Path =
         if self.is_root then Error.throw (Illegal_Argument.Error "Cannot get parent of the root directory.") else
-            Enso_Path.Value self.organization_name (self.path_segments.drop (..Last 1))
+            Enso_Path.Value (self.path_segments.drop (..Last 1))
 
     ## PRIVATE
     resolve self (subpath : Text) -> Enso_Path =
         new_segments = subpath.split Enso_Path.delimiter . filter (p-> p.is_empty.not)
         normalized_segments = normalize (self.path_segments + new_segments)
-        Enso_Path.Value self.organization_name normalized_segments
+        Enso_Path.Value normalized_segments
 
     ## PRIVATE
     is_descendant_of self (other : Enso_Path) -> Boolean =
-        if self.organization_name != other.organization_name then False else
-            if self.path_segments.length < other.path_segments.length then False else
-                (self.path_segments.take other.path_segments.length) == other.path_segments
+        if self.path_segments.length < other.path_segments.length then False else
+            (self.path_segments.take other.path_segments.length) == other.path_segments
 
     ## PRIVATE
     to_text self -> Text =
-        Enso_Path.protocol_prefix + self.organization_name + Enso_Path.delimiter + self.path_segments.join Enso_Path.delimiter
+        Enso_Path.protocol_prefix + self.path_segments.join Enso_Path.delimiter
 
     ## PRIVATE
     delimiter = "/"

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Existing_Enso_Asset.enso

@@ -130,7 +130,6 @@ type Asset_Cache
 ## PRIVATE
    Returns the cached reference or fetches it from the cloud.
 fetch_asset_reference (file : Enso_File) -> Existing_Enso_Asset ! File_Error =
-    # TODO remove workaround for bug https://github.com/enso-org/cloud-v2/issues/1173
-    path = if file.enso_path.is_root then file.enso_path.to_text + "/" else file.enso_path.to_text
+    path = file.enso_path.to_text
     Utils.get_cached (Asset_Cache.asset_key file) cache_duration=Cloud_Caching_Settings.get_file_cache_ttl <|
         Existing_Enso_Asset.resolve_path path if_not_found=(Error.throw (File_Error.Not_Found file))

test/AWS_Tests/data/credentials-with-secrets.datalink

@@ -7,11 +7,11 @@
     "subType": "access_key",
     "accessKeyId": {
       "type": "secret",
-      "secretPath": "enso://USERNAME/datalink-secret-AWS-keyid"
+      "secretPath": "enso://Users/USERNAME/datalink-secret-AWS-keyid"
     },
     "secretAccessKey": {
       "type": "secret",
-      "secretPath": "enso://USERNAME/datalink-secret-AWS-secretkey"
+      "secretPath": "enso://Users/USERNAME/datalink-secret-AWS-secretkey"
     }
   }
 }

test/AWS_Tests/src/S3_Spec.enso

@@ -54,7 +54,7 @@ delete_afterwards file ~action =
    it.
 replace_username_in_data_link base_file =
     content = Data_Link.read_raw_config base_file
-    new_content = content.replace "USERNAME" Enso_User.current.organization_name
+    new_content = content.replace "USERNAME" Enso_User.current.name
     temp_file = File.create_temporary_file prefix=base_file.name suffix=base_file.extension
     Data_Link.write_raw_config temp_file new_content replace_existing=True . if_not_error temp_file
 

test/Base_Tests/src/Network/Enso_Cloud/Cloud_Tests_Setup.enso

@@ -184,7 +184,7 @@ type Temporary_Directory
         was_initialized = Ref.new False
         Temporary_Directory.Value was_initialized <|
             directory_name = "test-run-"+name+"-"+Temporary_Directory.timestamp_text
-            test_root = (Enso_File.root / directory_name).create_directory
+            test_root = (Enso_File.home / directory_name).create_directory
             test_root.if_not_error <|
                 was_initialized.put True
                 with_initializer test_root . if_not_error test_root

test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso

@@ -1,5 +1,6 @@
 from Standard.Base import all
 import Standard.Base.Enso_Cloud.Cloud_Caching_Settings
+import Standard.Base.Enso_Cloud.Errors.Enso_Cloud_Error
 import Standard.Base.Errors.Common.Forbidden_Operation
 import Standard.Base.Errors.Common.Not_Found
 import Standard.Base.Errors.File_Error.File_Error
@@ -25,8 +26,8 @@ add_specs suite_builder setup:Cloud_Tests_Setup = suite_builder.group "Enso Clou
         Panic.rethrow <| "Hello Another!".write (sub / "another.txt")
     group_builder.teardown test_root.cleanup
 
-    group_builder.specify "should be able to list the root directory" <|
-        assets = Enso_File.root.list
+    group_builder.specify "should be able to list a directory" <|
+        assets = Enso_File.home.list
         # We don't a priori know the contents, so we can only check very generic properties
         assets . should_be_a Vector
         assets.each f-> f.should_be_a Enso_File
@@ -36,7 +37,7 @@ add_specs suite_builder setup:Cloud_Tests_Setup = suite_builder.group "Enso Clou
 
     group_builder.specify "should allow to create and delete a directory" <|
         my_name = "my_test_dir-" + (Random.uuid.take 5)
-        my_dir = (Enso_File.root / my_name).create_directory
+        my_dir = (test_root.get / my_name).create_directory
         my_dir.should_succeed
         delete_on_fail caught_panic =
             my_dir.delete
@@ -45,19 +46,19 @@ add_specs suite_builder setup:Cloud_Tests_Setup = suite_builder.group "Enso Clou
             my_dir.is_directory . should_be_true
             my_dir.exists . should_be_true
             my_dir.name . should_equal my_name
-            Enso_File.root.list . should_contain my_dir
+            test_root.get.list . should_contain my_dir
 
         my_dir.delete . should_succeed
 
         Test.with_retries <|
-            Enso_File.root.list . should_not_contain my_dir
+            test_root.get.list . should_not_contain my_dir
             my_dir.exists . should_be_false
 
     group_builder.specify "should set the current working directory by environment variable" <|
         # If nothing set, defaults to root:
-        Enso_File.current_working_directory . should_equal Enso_File.root
+        Enso_File.current_working_directory . should_equal Enso_File.home
 
-        subdir = (Enso_File.root / ("my_test_CWD-" + Random.uuid.take 5)).create_directory
+        subdir = (test_root.get / ("my_test_CWD-" + Random.uuid.take 5)).create_directory
         subdir.should_succeed
         cleanup =
             Enso_User.flush_caches
@@ -70,7 +71,7 @@ add_specs suite_builder setup:Cloud_Tests_Setup = suite_builder.group "Enso Clou
                 Enso_File.current_working_directory . should_equal subdir
 
         # It should be back to default afterwards:
-        Enso_File.current_working_directory . should_equal Enso_File.root
+        Enso_File.current_working_directory . should_equal Enso_File.home
 
     group_builder.specify "should allow to find a file by name" <|
         f = test_root.get / "test_file.json"
@@ -96,9 +97,9 @@ add_specs suite_builder setup:Cloud_Tests_Setup = suite_builder.group "Enso Clou
                     f.size.should_equal 16
 
     group_builder.specify "should be able to find a file by path" <|
-        File.new "enso://"+Enso_User.current.organization_name+"/" . should_equal Enso_File.root
-        File.new "enso://"+Enso_User.current.organization_name+"/test_file.json" . should_equal (Enso_File.root / "test_file.json")
-        File.new "enso://"+Enso_User.current.organization_name+"/abc/" . should_equal (Enso_File.root / "abc")
+        File.new "enso://" . should_equal Enso_File.root
+        File.new "enso://test_file.json" . should_equal (Enso_File.root / "test_file.json")
+        File.new "enso://abc/" . should_equal (Enso_File.root / "abc")
 
     group_builder.specify "should fail to read nonexistent files" <|
         f = Enso_File.root / "nonexistent_file.json"
@@ -108,8 +109,9 @@ add_specs suite_builder setup:Cloud_Tests_Setup = suite_builder.group "Enso Clou
         r.should_fail_with File_Error
         r.catch.should_be_a File_Error.Not_Found
 
-    group_builder.specify "should not allow to delete the root directory" <|
+    group_builder.specify "should not allow to delete the root or home directory" <|
         Enso_File.root.delete . should_fail_with Illegal_Argument
+        Enso_File.home.delete . should_fail_with Any
 
     # See Inter_Backend_File_Operations_Spec for copy/move tests
     group_builder.specify "should be able to write a file using with_output_stream" <|
@@ -239,7 +241,7 @@ add_specs suite_builder setup:Cloud_Tests_Setup = suite_builder.group "Enso Clou
         nested_file.last_modified_time . should_be_a Date_Time
 
         Enso_File.root.parent . should_equal Nothing
-        Enso_File.root.path . should_equal ("enso://"+Enso_User.current.organization_name+"/")
+        Enso_File.root.path . should_equal "enso://"
 
         dir.path . should_contain "enso://"
         dir.path . should_contain "/test-directory"
@@ -275,7 +277,7 @@ add_specs suite_builder setup:Cloud_Tests_Setup = suite_builder.group "Enso Clou
         dir.last_modified_time.should_fail_with Illegal_Argument
 
     group_builder.specify "should be able to read other file metadata" pending="TODO needs further design" <|
-        nested_file = Enso_File.root / "test-directory" / "another.txt"
+        nested_file = test_root.get / "test-directory" / "another.txt"
 
         nested_file.is_absolute.should_be_true
         nested_file.absolute . should_equal nested_file

test/Base_Tests/src/Network/Enso_Cloud/Secrets_Spec.enso

@@ -56,11 +56,11 @@ add_specs suite_builder setup:Cloud_Tests_Setup =
                     fetched_secret = Enso_Secret.get name
                     fetched_secret . should_equal created_secret
 
-                    path_secret = Enso_Secret.get "enso://"+Enso_User.current.organization_name+"/"+name
+                    path_secret = Enso_Secret.get "enso://Users/"+Enso_User.current.name+"/"+name
                     path_secret . should_equal created_secret
 
         group_builder.specify "does not allow both parent and path in Enso_Secret.get" <| setup.with_prepared_environment <|
-            Enso_Secret.get "enso://"+Enso_User.current.organization_name+"/SOME-SECRET" parent=Enso_File.root . should_fail_with Illegal_Argument
+            Enso_Secret.get "enso://Users/foo/SOME-SECRET" parent=Enso_File.home . should_fail_with Illegal_Argument
 
         group_builder.specify "should fail to create a secret if it already exists" <| setup.with_prepared_environment <|
             name = "my_test_secret-3-"+Temporary_Directory.timestamp_text

tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/AssetStore.java

@@ -1,14 +1,32 @@
 package org.enso.shttp.cloud_mock;
 
+import java.util.Arrays;
+import java.util.Deque;
 import java.util.LinkedList;
 import java.util.List;
 
 public class AssetStore {
-  static final String ROOT_DIRECTORY_ID = "directory-27xJM00p8jWoL2qByTo6tQfciWC";
+  static final String HOME_DIRECTORY_ID = "directory-27xJM00p8jWoL2qByTo6tQfciWC";
+  static final String USERS_DIRECTORY_ID = "directory-27xJM00p8jWoL2qByTo6tQfciBB";
+  static final String ROOT_DIRECTORY_ID = "directory-27xJM00p8jWoL2qByTo6tQfciAA";
   private final List<Secret> secrets = new LinkedList<>();
 
+  final Directory rootDirectory;
+  final Directory homeDirectory;
+
+  public AssetStore() {
+    rootDirectory = new Directory(ROOT_DIRECTORY_ID, "", null, new LinkedList<>());
+    Directory usersDirectory =
+        new Directory(USERS_DIRECTORY_ID, "Users", rootDirectory.id, new LinkedList<>());
+    homeDirectory =
+        new Directory(HOME_DIRECTORY_ID, "My test User 1", usersDirectory.id, new LinkedList<>());
+
+    rootDirectory.children.add(usersDirectory);
+    usersDirectory.children.add(homeDirectory);
+  }
+
   String createSecret(String parentDirectoryId, String title, String value) {
-    if (!parentDirectoryId.equals(ROOT_DIRECTORY_ID)) {
+    if (!parentDirectoryId.equals(HOME_DIRECTORY_ID)) {
       throw new IllegalArgumentException(
           "In Cloud Mock secrets can only be created in the root directory");
     }
@@ -39,30 +57,71 @@ Secret findSecretById(String id) {
   }
 
   List<Secret> listAssets(String parentDirectoryId) {
-    if (!parentDirectoryId.equals(ROOT_DIRECTORY_ID)) {
+    if (!parentDirectoryId.equals(HOME_DIRECTORY_ID)) {
       throw new IllegalArgumentException(
           "In Cloud Mock secrets can only be listed in the root directory");
     }
 
     return List.copyOf(secrets);
   }
 
-  Secret findAssetInRootByTitle(String subPath) {
-    return secrets.stream()
-        .filter(
-            secret ->
-                secret.title.equals(subPath) && secret.parentDirectoryId.equals(ROOT_DIRECTORY_ID))
-        .findFirst()
-        .orElse(null);
-  }
-
   record Secret(String id, String title, String value, String parentDirectoryId) {
     Asset asAsset() {
       return new Asset(id, title, parentDirectoryId);
     }
   }
 
+  Asset resolvePath(String[] path) {
+    Deque<String> pathSegments = new LinkedList<>(Arrays.asList(path));
+    Directory currentDirectory = rootDirectory;
+    Asset currentAsset = currentDirectory.asAsset();
+
+    while (!pathSegments.isEmpty()) {
+      String nextSegment = pathSegments.poll();
+      if (currentDirectory == null) {
+        throw new IllegalArgumentException(
+            "The path references a subdirectory of an asset that is not a directory");
+      }
+
+      var nextDirectory =
+          currentDirectory.children.stream()
+              .filter(directory -> directory.title.equals(nextSegment))
+              .findFirst()
+              .orElse(null);
+      if (nextDirectory != null) {
+        // Enter the subdirectory
+        currentDirectory = nextDirectory;
+        currentAsset = currentDirectory.asAsset();
+      } else {
+        // Otherwise, start looking for secrets
+        final var currentDirectoryId = currentDirectory.id;
+        var nextSecret =
+            secrets.stream()
+                .filter(
+                    secret ->
+                        secret.title.equals(nextSegment)
+                            && secret.parentDirectoryId.equals(currentDirectoryId))
+                .findFirst()
+                .orElse(null);
+        if (nextSecret != null) {
+          // Found a secret, mark it for return.
+          currentAsset = nextSecret.asAsset();
+          // But if further path segments are encountered - we will crash.
+          currentDirectory = null;
+        } else {
+          return null;
+        }
+      }
+    }
+
+    return currentAsset;
+  }
+
   public record Asset(String id, String title, String parentId) {}
 
-  final Asset rootDirectory = new Asset(ROOT_DIRECTORY_ID, "", null);
+  record Directory(String id, String title, String parentId, LinkedList<Directory> children) {
+    Asset asAsset() {
+      return new Asset(id, title, parentId);
+    }
+  }
 }

tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/DirectoriesHandler.java

@@ -8,7 +8,7 @@
 /**
  * Lists assets in a directory.
  *
- * <p>In the mock, only root directory can be listed, and it may only contain secrets.
+ * <p>In the mock, only user's home directory can be listed, and it may only contain secrets.
  */
 public class DirectoriesHandler implements CloudHandler {
 
@@ -37,7 +37,7 @@ public void handleCloudAPI(CloudExchange exchange) throws IOException {
   }
 
   private void listDirectory(String parentId, CloudExchange exchange) throws IOException {
-    final String effectiveParentId = parentId.isEmpty() ? AssetStore.ROOT_DIRECTORY_ID : parentId;
+    final String effectiveParentId = parentId.isEmpty() ? AssetStore.HOME_DIRECTORY_ID : parentId;
     ListDirectoryResponse response =
         new ListDirectoryResponse(
             assetStore.listAssets(effectiveParentId).stream()