Introduce Value_Type.Null, fix edge cases for read_many

distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso

@@ -4,10 +4,12 @@ import project.Data.Read.Many_Files_List.Many_Files_List
 import project.Data.Read.Return_As.Return_As
 import project.Data.Text.Encoding.Encoding
 import project.Data.Text.Text
+import project.Data.Vector.No_Wrap
 import project.Data.Vector.Vector
 import project.Enso_Cloud.Data_Link.Data_Link
 import project.Enso_Cloud.Data_Link_Helpers
 import project.Error.Error
+import project.Errors.Common.Failed_To_Load
 import project.Errors.Common.Missing_Argument
 import project.Errors.File_Error.File_Error
 import project.Errors.Illegal_Argument.Illegal_Argument
@@ -144,9 +146,11 @@ read path=(Missing_Argument.throw "path") format=Auto_Detect (on_problems : Prob
 read_many : Many_Files_List -> File_Format -> Return_As -> Problem_Behavior -> Any ! File_Error
 read_many (paths : Many_Files_List = Missing_Argument.throw "paths") format=Auto_Detect return=..As_Merged_Table (on_problems : Problem_Behavior = ..Report_Warning) =
     return_as = Return_As.resolve return
-    loaded_objects = paths.paths_to_load.map on_problems=on_problems path->
-        Data.read path format on_problems
-    return_as.make_return paths loaded_objects on_problems
+    if paths.paths_to_load.contains Nothing then Error.throw (Illegal_Argument.Error "The list of paths to load should not contain Nothing. Use `filter` with `..Not_Nothing` to filter out missing entries before calling `read_many`.") else
+        loaded_objects = paths.paths_to_load.map on_problems=No_Wrap.Value path->
+            Data.read path format on_problems . catch Any error->
+                Failed_To_Load.Warning path error
+        return_as.make_return paths loaded_objects on_problems
 
 ## ALIAS load text, open text
    GROUP Input

distribution/lib/Standard/Base/0.0.0-dev/src/Data/Read/Many_Files_List.enso

@@ -7,6 +7,15 @@ import project.Data.Vector.Vector
    used in `Data.read_many`.
 type Many_Files_List
     ## PRIVATE
+       Arguments:
+       - original_value: The original value that represents a list of files.
+         Some return modes may use it as it contains more information than just
+         list of files (e.g. it can be a source table).
+       - paths_to_load: A vector of paths to load. This is the fallback that can
+         be used by any return mode if it does not recognize the original value.
+         The vector is expected to contain values that can be passed into
+         `Data.read` (so it can be Text, File, URI or any other kind of file -
+         e.g. S3_File).
     Value original_value paths_to_load:Vector
 
     ## PRIVATE

distribution/lib/Standard/Base/0.0.0-dev/src/Data/Read/Return_As.enso

@@ -3,10 +3,12 @@ import project.Data.Read.Many_Files_List.Many_Files_List
 import project.Data.Text.Text
 import project.Data.Vector.Vector
 import project.Error.Error
+import project.Errors.Common.Failed_To_Load
 import project.Errors.Common.Type_Error
 import project.Errors.Illegal_Argument.Illegal_Argument
 import project.Errors.Problem_Behavior.Problem_Behavior
 import project.Function.Function
+import project.Meta
 import project.Metadata.Display
 import project.Metadata.Widget
 import project.Nothing.Nothing
@@ -33,7 +35,7 @@ type Return_As
     to_display_text self -> Text = self.underlying.to_display_text
 
     ## PRIVATE
-    make_return self (input : Many_Files_List) (objects : Vector Any) (on_problems : Problem_Behavior) =
+    make_return self (input : Many_Files_List) (objects : Vector (Any | Failed_To_Load)) (on_problems : Problem_Behavior) =
         self.underlying.make_return input objects on_problems
 
     ## PRIVATE
@@ -77,9 +79,17 @@ type Return_As_Base
         Panic.catch Type_Error (value:Return_As_Base) _->Nothing
 
     ## PRIVATE
-    make_return self (input : Many_Files_List) (objects : Vector Any) (on_problems : Problem_Behavior) =
-        _ = [input, on_problems]
-        objects
+    make_return self (input : Many_Files_List) (objects : Vector (Any | Failed_To_Load)) (on_problems : Problem_Behavior) =
+        _ = input
+        replace_with_nothing_and_propagate objects on_problems
+
+## PRIVATE
+   A helper method that takes a Vector and replaces `Failed_To_Load` with `Nothing`, raising them as warnings.
+replace_with_nothing_and_propagate (vector : Vector (Any | Failed_To_Load)) (on_problems : Problem_Behavior) =
+    failed = vector.filter o-> o.is_a Failed_To_Load
+    if failed.is_empty then vector else
+        on_problems.attach_problems_before failed <|
+            vector.map o-> if o.is_a Failed_To_Load then Nothing else o
 
 ## PRIVATE
 Return_As.from (that : Return_As_Base) =

distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Common.enso

@@ -607,3 +607,13 @@ type Floating_Point_Equality
             "(Error  (location = "+location+"))"
         Floating_Point_Equality.Used_As_Dictionary_Key value ->
             "(Used_As_Dictionary_Key (value = "+value.to_text+"))"
+
+## A warning indicating that a file failed to be loaded.
+type Failed_To_Load
+    ## PRIVATE
+    Warning path cause
+
+    ## PRIVATE
+       Create a human-readable version of the error.
+    to_display_text : Text
+    to_display_text self = "Failed to load file at path: "+self.path.to_display_text+": "+self.cause.to_display_text

distribution/lib/Standard/Database/0.0.0-dev/src/DB_Column.enso

@@ -595,12 +595,13 @@ type DB_Column
          cause a hard error, the value may be truncated or wrap-around etc.
     + : DB_Column | Any -> DB_Column
     + self other =
-        op = case Value_Type_Helpers.resolve_addition_kind self other of
-            Value_Type_Helpers.Addition_Kind.Numeric_Add -> "ADD_NUMBER"
-            Value_Type_Helpers.Addition_Kind.Text_Concat -> "ADD_TEXT"
-        op.if_not_error <|
-            new_name = self.naming_helper.binary_operation_name "+" self other
-            self.make_binary_op op other new_name
+        new_name = self.naming_helper.binary_operation_name "+" self other
+        case Value_Type_Helpers.resolve_addition_kind self other of
+            Value_Type_Helpers.Addition_Kind.Numeric_Add ->
+                self.make_binary_op "ADD_NUMBER" other new_name
+            Value_Type_Helpers.Addition_Kind.Text_Concat ->
+                self.make_binary_op "ADD_TEXT" other new_name
+            Nothing -> self.const Nothing
 
     ## ALIAS minus, subtract, time difference
        GROUP Standard.Base.Operators
@@ -630,6 +631,7 @@ type DB_Column
                 self.make_binary_op "-" other
             Value_Type_Helpers.Subtraction_Kind.Date_Time_Difference ->
                 Error.throw (Unsupported_Database_Operation.Error "Subtracting date/time value")
+            Nothing -> self.const Nothing
 
     ## ALIAS multiply, product, times
        GROUP Standard.Base.Operators

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Type_Mapping.enso

@@ -64,6 +64,8 @@ type Postgres_Type_Mapping
                    but we may also consider using the standard SQL `bit(n)` and `bit varying(n)` types.
                    See: https://www.postgresql.org/docs/current/datatype-bit.html
                 SQL_Type.Value Types.BINARY "bytea" precision=max_precision
+            Value_Type.Null ->
+                Error.throw (Unsupported_Database_Type.Error "Null" "Postgres")
             Value_Type.Mixed ->
                 Error.throw (Unsupported_Database_Type.Error "Mixed" "Postgres")
             Value_Type.Unsupported_Data_Type type_name underlying_type ->
@@ -146,7 +148,7 @@ type Postgres_Type_Mapping
 simple_types_map = Dictionary.from_vector <|
     ints = [[Types.SMALLINT, Value_Type.Integer Bits.Bits_16], [Types.BIGINT, Value_Type.Integer Bits.Bits_64], [Types.INTEGER, Value_Type.Integer Bits.Bits_32]]
     floats = [[Types.DOUBLE, Value_Type.Float Bits.Bits_64], [Types.REAL, Value_Type.Float Bits.Bits_32]]
-    other = [[Types.DATE, Value_Type.Date], [Types.TIME, Value_Type.Time]]
+    other = [[Types.DATE, Value_Type.Date], [Types.TIME, Value_Type.Time], [Types.NULL, Value_Type.Null]]
     ints + floats + other
 
 ## PRIVATE

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso

@@ -192,14 +192,12 @@ type SQLite_Dialect
     adapt_unified_column : Internal_Column -> Value_Type -> (SQL_Expression -> SQL_Type_Reference) -> Internal_Column
     adapt_unified_column self column approximate_result_type infer_result_type_from_database_callback =
         _ = infer_result_type_from_database_callback
-        # TODO [RW] This may be revisited with #6281.
-        case approximate_result_type of
-            Nothing -> column
-            _ ->
-                sql_type = self.get_type_mapping.value_type_to_sql approximate_result_type Problem_Behavior.Ignore
-                new_expression = self.make_cast_expression column sql_type
-                new_sql_type_reference = SQL_Type_Reference.from_constant sql_type
-                Internal_Column.Value column.name new_sql_type_reference new_expression
+        needs_cast = approximate_result_type != Value_Type.Null
+        if needs_cast.not then column else
+            sql_type = self.get_type_mapping.value_type_to_sql approximate_result_type Problem_Behavior.Ignore
+            new_expression = self.make_cast_expression column sql_type
+            new_sql_type_reference = SQL_Type_Reference.from_constant sql_type
+            Internal_Column.Value column.name new_sql_type_reference new_expression
 
     ## PRIVATE
        Add an extra cast to adjust the output type of certain operations with
@@ -636,6 +634,9 @@ make_custom_cast column target_value_type type_mapping =
     result = Ref.new Nothing
     column_type =
         type_mapping.sql_type_to_value_type column.sql_type_reference.get
+    if target_value_type == Value_Type.Null then
+        result.put column.expression
+
     if target_value_type.is_text && (column_type == Value_Type.Boolean) then
         expr = SQL_Expression.Operation "IIF" [column.expression, SQL_Expression.Literal "'true'", SQL_Expression.Literal "'false'"]
         result.put expr

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso

@@ -60,6 +60,7 @@ type SQLite_Type_Mapping
             Value_Type.Date -> unsupported_date_time
             Value_Type.Date_Time _ -> unsupported_date_time
             Value_Type.Binary _ _ -> SQLite_Types.blob
+            Value_Type.Null -> SQLite_Types.null
             Value_Type.Mixed ->
                 ## The best we could do would be to store mixed values and
                    report invalid type or coerce the values to Text. Both seem
@@ -151,7 +152,7 @@ simple_types_map = Dictionary.from_vector <|
     numerics = [Types.DECIMAL, Types.NUMERIC] . map x-> [x, default_float]
     strings = [Types.CHAR, Types.VARCHAR] . map x-> [x, default_text]
     blobs = [Types.BINARY, Types.BLOB, Types.CLOB] . map x-> [x, Value_Type.Binary]
-    special_types = [[Types.BOOLEAN, Value_Type.Boolean]]
+    special_types = [[Types.BOOLEAN, Value_Type.Boolean], [Types.NULL, Value_Type.Null]]
     ints + floats + numerics + strings + blobs + special_types
 
 ## PRIVATE
@@ -246,6 +247,9 @@ type SQLite_Types
        The artificial 6th affinity that is used to distinguish boolean columns.
     boolean = SQL_Type.Value Types.BOOLEAN "BOOLEAN"
 
+    ## PRIVATE
+    null = SQL_Type.Value Types.NULL "NULL"
+
 ## PRIVATE
 default_text = Value_Type.Char size=Nothing variable_length=True
 

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Upload/Helpers/SQL_Helpers.enso

@@ -2,6 +2,7 @@ private
 
 from Standard.Base import all
 import Standard.Base.Data.Vector.No_Wrap
+import Standard.Base.Errors.Illegal_Argument.Illegal_Argument
 
 import project.Column_Description.Column_Description
 import project.Connection.Connection.Connection
@@ -10,6 +11,7 @@ import project.Internal.IR.Query.Query
 import project.Internal.IR.SQL_Expression.SQL_Expression
 import project.SQL_Query.SQL_Query
 import project.SQL_Statement.SQL_Statement
+from project.Errors import Unsupported_Database_Type
 
 ## PRIVATE
 make_batched_insert_template : Connection -> Text -> Vector (Vector Text) -> SQL_Query
@@ -32,6 +34,9 @@ prepare_create_table_statement connection table_name columns primary_key tempora
     type_mapping = connection.dialect.get_type_mapping
     column_descriptors = columns.map on_problems=No_Wrap.Value def->
         sql_type = type_mapping.value_type_to_sql def.value_type on_problems
+            . catch Unsupported_Database_Type error->
+                Error.throw (Illegal_Argument.Error "Definition for column ["+def.name+"] is invalid: "+error.to_display_text)
+
         sql_type_text = type_mapping.sql_type_to_text sql_type
         Create_Column_Descriptor.Value def.name sql_type_text def.constraints
     connection.dialect.generate_sql <|

distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Type_Mapping.enso

@@ -62,6 +62,7 @@ type SQLServer_Type_Mapping
                         True  -> SQL_Type.Value Types.BINARY "VARBINARY" size
                         False -> SQL_Type.Value Types.BINARY "BINARY" size
             Value_Type.Mixed -> Error.throw (Unsupported_Database_Type.Error "Mixed" "SQLServer")
+            Value_Type.Null -> Error.throw (Unsupported_Database_Type.Error "Null" "SQLServer")
             Value_Type.Unsupported_Data_Type type_name underlying_type ->
                 underlying_type.if_nothing <| Error.throw <| Illegal_Argument.Error <|
                     "An unsupported SQL type ["+type_name.to_text+"] cannot be converted into an SQL type because it did not contain the SQL metadata needed to reconstruct it."
@@ -101,6 +102,7 @@ type SQLServer_Type_Mapping
                 "varbinary" -> Value_Type.Binary size=sql_type.precision variable_length=True
                 "binary" -> Value_Type.Binary size=sql_type.precision variable_length=False
                 _ -> on_unknown_type sql_type
+            Types.NULL -> Value_Type.Null
             _ -> case sql_type.name of
                 "datetimeoffset" -> Value_Type.Date_Time with_timezone=True
                 _ -> on_unknown_type sql_type

distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Type_Mapping.enso

@@ -63,6 +63,7 @@ type Snowflake_Type_Mapping
                     True  -> SQL_Type.Value Types.BINARY "binary"
                     False -> SQL_Type.Value Types.BINARY "binary" size
             Value_Type.Mixed -> Error.throw (Unsupported_Database_Type.Error "Mixed" "Snowflake")
+            Value_Type.Null -> Error.throw (Unsupported_Database_Type.Error "Null" "Snowflake")
             Value_Type.Unsupported_Data_Type type_name underlying_type ->
                 underlying_type.if_nothing <| Error.throw <| Illegal_Argument.Error <|
                     "An unsupported SQL type ["+type_name.to_text+"] cannot be converted into an SQL type because it did not contain the SQL metadata needed to reconstruct it."
@@ -159,7 +160,7 @@ type Snowflake_Type_Mapping
 ## PRIVATE
 simple_types_map = Dictionary.from_vector <|
     floats = [[Types.DOUBLE, Value_Type.Float Bits.Bits_64], [Types.REAL, Value_Type.Float Bits.Bits_64]]
-    other = [[Types.DATE, Value_Type.Date], [Types.TIME, Value_Type.Time], [Types.BOOLEAN, Value_Type.Boolean]]
+    other = [[Types.DATE, Value_Type.Date], [Types.TIME, Value_Type.Time], [Types.BOOLEAN, Value_Type.Boolean], [Types.NULL, Value_Type.Null]]
     floats + other
 
 ## PRIVATE

distribution/lib/Standard/Table/0.0.0-dev/src/Column.enso

@@ -536,6 +536,7 @@ type Column
                         run_binary_op self fn other new_name
                     _ ->
                         run_vectorized_binary_op self Java_Storage.Maps.SUB other
+            Nothing -> self.const Nothing
 
     ## ALIAS multiply, product, times
        GROUP Standard.Base.Operators

distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Cast_Helpers.enso

@@ -13,7 +13,7 @@ polyglot java import org.enso.table.data.column.storage.type.StorageType
    Checks if one type can be cast into another and returns a dataflow error
    explaining the situation if not.
 check_cast_compatibility source_type target_type ~action =
-    are_compatible = if (target_type == Value_Type.Mixed) || (source_type == Value_Type.Mixed) || target_type.is_text || (source_type == target_type) then True else
+    are_compatible = if (target_type == Value_Type.Mixed) || (source_type == Value_Type.Mixed) || (source_type == Value_Type.Null) || target_type.is_text || (source_type == target_type) then True else
         if source_type.is_text && is_a_valid_parse_target target_type then Error.throw (Illegal_Argument.Error "To parse a text column into "+target_type.to_display_text+" type, `parse` should be used instead of `cast`.") else
             if source_type == Value_Type.Boolean then target_type.is_numeric else
                 if source_type.is_numeric then target_type.is_numeric else

distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Read_Many_As_Merged_Table_Strategy.enso

@@ -1,4 +1,5 @@
 from Standard.Base import all
+import Standard.Base.Errors.Common.Failed_To_Load
 import Standard.Base.Errors.Illegal_Argument.Illegal_Argument
 
 import project.Columns_To_Keep.Columns_To_Keep
@@ -39,11 +40,18 @@ Read_Many_As_Merged_Table_Strategy.from (that : Vector) =
                 (No_Rows.Warning "The "+path.to_display_text+" loaded as an empty array, so it is not included in the `As_Merged_Table` result of `read_many`.")
     Read_Many_As_Merged_Table_Strategy.Value callback
 
+## PRIVATE
+Read_Many_As_Merged_Table_Strategy.from (that : Failed_To_Load) =
+    callback path _ _ on_problems =
+        on_problems.attach_problem_after Read_Many_As_Table_Result.No_Data <|
+            (No_Rows.Warning "The "+path.to_display_text+" failed to load, so it is not included in the `As_Merged_Table` result of `read_many`. Caused by: "+that.cause.to_display_text)
+    Read_Many_As_Merged_Table_Strategy.Value callback
+
 ## PRIVATE
    The fallback strategy for converting a generic object into a table.
    Custom data types may implement a conversion to override this strategy, like above.
 Read_Many_As_Merged_Table_Strategy.from (that : Any) =
-    callback path _ _ on_problems = 
+    callback path _ _ on_problems =
         Read_Many_As_Table_Result.Table metadata=Nothing data=(_interpret_as_table that path on_problems)
     Read_Many_As_Merged_Table_Strategy.Value callback
 

distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Read_Many_Helpers.enso

@@ -2,7 +2,9 @@ private
 
 from Standard.Base import all
 import Standard.Base.Data.Read.Many_Files_List.Many_Files_List
+import Standard.Base.Errors.Common.Failed_To_Load
 import Standard.Base.Errors.Illegal_Argument.Illegal_Argument
+from Standard.Base.Data.Read.Return_As import replace_with_nothing_and_propagate
 
 import project.Column.Column
 import project.Errors.Invalid_Value_Type
@@ -30,15 +32,18 @@ ensure_column_type_valid_to_be_files_list (column : Column) ~action =
         Value_Type.Mixed -> True
         # Columns containing paths as Text will be Char
         Value_Type.Char _ _ -> True
+        # Empty columns are also valid
+        Value_Type.Null -> True
         _ -> False
     if is_expected_type then action else
         Error.throw (Invalid_Value_Type.Column "Text or Mixed" column.value_type column.name)
 
-make_return (return_shape : Return_As_Table) (input : Many_Files_List) (objects : Vector Any) (on_problems : Problem_Behavior) -> Table =
+make_return (return_shape : Return_As_Table) (input : Many_Files_List) (objects : Vector (Any | Failed_To_Load)) (on_problems : Problem_Behavior) -> Table =
     base_table = _input_as_table input
     case return_shape of
         Return_As_Table.With_New_Column ->
-            _add_objects_column base_table objects
+            replaced = replace_with_nothing_and_propagate objects on_problems
+            _add_objects_column base_table replaced
         Return_As_Table.As_Merged_Table columns_to_keep match_columns ->
             tables = input.paths_to_load.zip objects path-> object->
                 strategy = Read_Many_As_Merged_Table_Strategy.from object

distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Storage.enso

@@ -21,6 +21,7 @@ polyglot java import org.enso.table.data.column.storage.type.DateTimeType
 polyglot java import org.enso.table.data.column.storage.type.DateType
 polyglot java import org.enso.table.data.column.storage.type.FloatType
 polyglot java import org.enso.table.data.column.storage.type.IntegerType
+polyglot java import org.enso.table.data.column.storage.type.NullType
 polyglot java import org.enso.table.data.column.storage.type.StorageType
 polyglot java import org.enso.table.data.column.storage.type.TextType
 polyglot java import org.enso.table.data.column.storage.type.TimeOfDayType
@@ -46,6 +47,7 @@ to_value_type storage_type = case storage_type of
     _ : TimeOfDayType -> Value_Type.Time
     _ : BigDecimalType -> Value_Type.Decimal
     _ : BigIntegerType -> Value_Type.Decimal scale=0
+    _ : NullType -> Value_Type.Null
     _ : AnyObjectType -> Value_Type.Mixed
 
 ## PRIVATE
@@ -70,6 +72,7 @@ closest_storage_type value_type = case value_type of
     Value_Type.Decimal _ scale ->
         is_integer = scale.is_nothing.not && scale <= 0
         if is_integer then BigIntegerType.INSTANCE else BigDecimalType.INSTANCE
+    Value_Type.Null -> NullType.INSTANCE
     _ ->
         Error.throw (Illegal_Argument.Error "Columns of type "+value_type.to_display_text+" are currently not supported in the in-memory backend.")