Add DB_Table.offset for SQLServer

CHANGELOG.md

@@ -47,12 +47,14 @@
   - In `Delimited` format, the `keep_invalid_rows` setting has been renamed to
     `on_invalid_rows`. The default behaviour was also changed to add any extra
     columns instead of discarding them.
+- [Added DB_Table.Offset for SQLServer][12206]
 
 [11926]: https://github.com/enso-org/enso/pull/11926
 [12031]: https://github.com/enso-org/enso/pull/12031
 [12071]: https://github.com/enso-org/enso/pull/12071
 [12092]: https://github.com/enso-org/enso/pull/12092
 [12231]: https://github.com/enso-org/enso/pull/12231
+[12206]: https://github.com/enso-org/enso/pull/12206
 
 #### Enso Language & Runtime
 

distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Internal/Redshift_Dialect.enso

@@ -76,8 +76,10 @@ type Redshift_Dialect
        Generates SQL modifier for limiting the number of rows and its position in the query
     get_limit_sql_modifier : Integer -> Any
     get_limit_sql_modifier self limit =
-        SQL_Builder.code (" LIMIT " + limit.to_text)
-
+        case limit of
+            Nothing -> ""
+            _ : Integer -> SQL_Builder.code (" LIMIT " + limit.to_text)
+
     ## PRIVATE
        Returns an ordering of SQL_Part's that determine what order each part gets written in for the final SQL output
        If you add extensions using Context_Extension you have to provide an order for them here

distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso

@@ -57,6 +57,7 @@ import project.Internal.Aggregate_Helper
 import project.Internal.Base_Generator
 import project.Internal.Common.Database_Join_Helper
 import project.Internal.Common.Lookup_Query_Helper
+import project.Internal.Common.Offset_Helpers
 import project.Internal.Common.Row_Number_Helpers
 import project.Internal.DB_Data_Link_Helpers
 import project.Internal.Helpers
@@ -905,20 +906,10 @@ type DB_Table
     add_row_number self (name:Text="Row") (from:Integer=0) (step:Integer=1) (group_by:(Vector | Text | Integer | Regex)=[]) (order_by:(Vector | Text)=[]) (on_problems:Problem_Behavior=..Report_Warning) =
         Feature.Add_Row_Number.if_supported_else_throw self.connection.dialect "add_row_number" <|
             problem_builder = Problem_Builder.new error_on_missing_columns=True
-            grouping_columns = self.columns_helper.select_columns_helper group_by Case_Sensitivity.Default True problem_builder
-            grouping_columns.each column->
-                if column.value_type.is_floating_point then
-                    problem_builder.report_other_warning (Floating_Point_Equality.Error column.name)
+            grouping_columns = _resolve_grouping_columns self.columns_helper group_by problem_builder
             ordering = Table_Helpers.resolve_order_by self.columns order_by problem_builder
             problem_builder.attach_problems_before on_problems <|
-                order_descriptors = case ordering.is_empty of
-                    False -> ordering.map element->
-                        column = element.column
-                        associated_selector = element.associated_selector
-                        self.connection.dialect.prepare_order_descriptor column associated_selector.direction text_ordering=Nothing
-                    True -> case self.default_ordering of
-                        Nothing -> Error.throw (Illegal_Argument.Error "The table has no existing ordering (e.g. from a `sort` operation or primary key). `add_row_number` requires an ordering in database.")
-                        descriptors -> descriptors
+                order_descriptors = _resolve_order_descriptors self.connection.dialect ordering self.default_ordering "add_row_number"
                 grouping_expressions = (grouping_columns.map _.as_internal).map .expression
 
                 new_expr = Row_Number_Helpers.make_row_number from step order_descriptors grouping_expressions
@@ -3194,9 +3185,26 @@ type DB_Table
     @default (self-> Widget_Helpers.make_fill_default_value_selector2)
     @group_by Widget_Helpers.make_column_name_multi_selector
     @order_by Widget_Helpers.make_order_by_selector
-    offset self columns:(Vector (Integer | Text | Regex | By_Type))=(Missing_Argument.throw "columns") n:Integer=-1 fill_with:Fill_With=..Nothing (group_by:(Vector | Text | Integer | Regex)=[]) (order_by:(Vector | Text)=[]) (set_mode:Set_Mode=..Add) (on_problems:Problem_Behavior=..Report_Warning) -> Table =
-        _ = [columns, n, fill_with, group_by, order_by, set_mode, on_problems]
-        Error.throw (Unsupported_Database_Operation.Error "offset")
+    offset self columns:(Vector (Integer | Text | Regex | By_Type))=(Missing_Argument.throw "columns") n:Integer=-1 fill_with:Fill_With=..Nothing (group_by:(Vector | Text | Integer | Regex)=[]) (order_by:(Vector | Text)=[]) (set_mode:Set_Mode=..Add) (on_problems:Problem_Behavior=..Report_Warning) -> DB_Table =
+        Feature.Offset.if_supported_else_throw self.connection.dialect "offset" <|
+            problem_builder = Problem_Builder.new error_on_missing_columns=True
+            grouping_columns = _resolve_grouping_columns self.columns_helper group_by problem_builder
+            ordering = Table_Helpers.resolve_order_by self.columns order_by problem_builder
+            resolved_columns = self.columns_helper.select_columns_helper columns Case_Sensitivity.Default True problem_builder
+            problem_builder.attach_problems_before on_problems <| if columns.is_empty then self else
+                order_descriptors = _resolve_order_descriptors self.connection.dialect ordering self.default_ordering "offset"
+                grouping_expressions = (grouping_columns.map _.as_internal).map .expression
+
+                offset_expr c:DB_Column -> SQL_Expression =
+                    Offset_Helpers.make_offset n (c.as_internal) order_descriptors grouping_expressions fill_with
+                new_column_name c:DB_Column -> Text = 
+                    if set_mode==Set_Mode.Add then self.column_naming_helper.function_name "offset" [c, n, fill_with] else c.name
+
+                new_columns = resolved_columns.map c->(Internal_Column.Value (new_column_name c) c.sql_type_reference (offset_expr c))
+
+                updated_table = new_columns.fold self t-> col->
+                    t.set (self.make_column col)
+                updated_table.as_subquery
 
 ## PRIVATE
 
@@ -3287,6 +3295,25 @@ make_literal_table connection column_vectors column_names alias =
 
             DB_Table.new alias connection internal_columns context
 
+## PRIVATE
+private _resolve_grouping_columns columns_helper:Table_Column_Helper group_by:Vector problem_builder:Problem_Builder -> Vector =
+    grouping_columns = columns_helper.select_columns_helper group_by Case_Sensitivity.Default True problem_builder
+    grouping_columns.each column->
+        if column.value_type.is_floating_point then
+            problem_builder.report_other_warning (Floating_Point_Equality.Error column.name)
+    grouping_columns
+
+## PRIVATE
+private _resolve_order_descriptors dialect ordering default_ordering operation_name:Text =
+    case ordering.is_empty of
+        False -> ordering.map element->
+            column = element.column
+            associated_selector = element.associated_selector
+            dialect.prepare_order_descriptor column associated_selector.direction text_ordering=Nothing
+        True -> case default_ordering of
+            Nothing -> Error.throw (Illegal_Argument.Error "The table has no existing ordering (e.g. from a `sort` operation or primary key). `"+operation_name+"` requires an ordering in database.")
+            descriptors -> descriptors
+
 ## PRIVATE
 Many_Files_List.from (that : DB_Table) =
     _ = that

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso

@@ -100,6 +100,9 @@ type SQL_Generator
                     wrapped_binder ++ "." ++ "x"
                 False ->
                     self.generate_expression dialect standalone_expression
+        SQL_Expression.List values ->
+            parsed_values = values.map (self.generate_expression dialect)
+            SQL_Builder.join ", " parsed_values
         query : Query -> self.generate_sub_query dialect query
         descriptor : Order_Descriptor -> self.generate_order dialect descriptor
 
@@ -194,9 +197,7 @@ type SQL_Generator
         orders = ctx.orders.map (self.generate_order dialect)
         order_part = (SQL_Builder.join ", " orders) . prefix_if_present " ORDER BY "
 
-        limit_part = case ctx.limit of
-                Nothing -> ""
-                _ : Integer -> dialect.get_limit_sql_modifier ctx.limit
+        limit_part = dialect.get_limit_sql_modifier ctx.limit
 
         extensions = ctx.extensions.map extension->
             part = extension.run_generator (gen_exprs extension.expressions)

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso

@@ -0,0 +1,20 @@
+from Standard.Base import all
+from Standard.Database.Errors import Unsupported_Database_Operation
+from Standard.Table import Fill_With
+
+import project.Internal.IR.Operation_Metadata
+import project.Internal.IR.Order_Descriptor.Order_Descriptor
+import project.Internal.IR.SQL_Expression.SQL_Expression
+import project.Internal.IR.Internal_Column.Internal_Column
+
+## PRIVATE
+make_offset (n : Integer) (colName: Internal_Column) (order_descriptors : Vector Order_Descriptor) (grouping_expressions : Vector SQL_Expression) fill_with:Fill_With -> SQL_Expression =
+    lead_lag = if n<0 then "LAG" else "LEAD"
+    case fill_with of
+        Fill_With.Nothing -> SQL_Expression.Operation lead_lag [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.List grouping_expressions, SQL_Expression.List order_descriptors]
+        Fill_With.Closest_Value -> 
+            order_descriptors_for_row_number = if n<0 then order_descriptors else order_descriptors.map o->o.reverse
+            params = [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.List grouping_expressions, SQL_Expression.List order_descriptors, SQL_Expression.List order_descriptors_for_row_number]
+            SQL_Expression.Operation lead_lag+"_CLOSEST" params
+        Fill_With.Wrap_Around -> Error.throw (Unsupported_Database_Operation.Error "offset with Fill_With.Wrap_Around")
+

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/Order_Descriptor.enso

@@ -8,6 +8,16 @@ type Order_Descriptor
     ## PRIVATE
     Value (expression : SQL_Expression) (direction : Sort_Direction) (nulls_order : Nothing | Nulls_Order = Nothing) (collation : Nothing | Text = Nothing)
 
+    reverse self -> Order_Descriptor =
+        new_direction = case self.direction of
+            Sort_Direction.Ascending -> Sort_Direction.Descending
+            Sort_Direction.Descending -> Sort_Direction.Ascending
+        new_nulls_order = case self.nulls_order of
+            Nothing -> Nothing
+            Nulls_Order.First -> Nulls_Order.Last
+            Nulls_Order.Last -> Nulls_Order.First
+        Order_Descriptor.Value self.expression new_direction new_nulls_order self.collation
+
     ## PRIVATE
        A bottom-up, depth-first traversal of this IR node and its children. Each
        node is passed to the provided function, and the return value of the

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso

@@ -103,6 +103,16 @@ type SQL_Expression
          enclosing `Let` value.
     Let_Ref (name : Text) (binder : Text) (standalone_expression : SQL_Expression)
 
+    ## PRIVATE
+       The internal representation of an  list of SQL expressions which will be
+       separated by commas in a list in the final SQL query.
+
+       NOTE - Only valid in certain locations so should be used carefully
+
+       Arguments:
+       - value: the values that should be built into a comma separated list.
+    List values:Vector
+
     ## PRIVATE
        A bottom-up, depth-first traversal of this IR node and its children. Each
        node is passed to the provided function, and the return value of the
@@ -121,4 +131,6 @@ type SQL_Expression
                 SQL_Expression.Let name binder (rec bindee) (rec body)
             SQL_Expression.Let_Ref name binder standalone_expression ->
                 SQL_Expression.Let_Ref name binder (rec standalone_expression)
+            SQL_Expression.List values ->
+                SQL_Expression.List (values.map rec)
         f new_sql_expression

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso

@@ -87,7 +87,9 @@ type Postgres_Dialect
        Generates SQL modifier for limiting the number of rows and its position in the query
     get_limit_sql_modifier : Integer -> Any
     get_limit_sql_modifier self limit =
-        SQL_Builder.code (" LIMIT " + limit.to_text)
+        case limit of
+            Nothing -> ""
+            _ : Integer -> SQL_Builder.code (" LIMIT " + limit.to_text)
 
     ## PRIVATE
        Returns an ordering of SQL_Part's that determine what order each part gets written in for the final SQL output

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso

@@ -82,7 +82,9 @@ type SQLite_Dialect
        Generates SQL modifier for limiting the number of rows and its position in the query
     get_limit_sql_modifier : Integer -> Any
     get_limit_sql_modifier self limit =
-        SQL_Builder.code (" LIMIT " + limit.to_text)
+        case limit of
+            Nothing -> ""
+            _ : Integer -> SQL_Builder.code (" LIMIT " + limit.to_text)
 
     ## PRIVATE
        Returns an ordering of SQL_Part's that determine what order each part gets written in for the final SQL output

distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso

@@ -92,7 +92,11 @@ type SQLServer_Dialect
        Generates SQL modifier for limiting the number of rows and its position in the query
     get_limit_sql_modifier : Integer -> Any
     get_limit_sql_modifier self limit =
-        SQL_Builder.code (" TOP " + limit.to_text + " ")
+        case limit of
+            Nothing -> 
+                ## SQLServer doesn't allow order by in sub queries unless we add in the below. We should consider removing order bys from subqueries in the future.
+                "TOP 100 PERCENT "
+            _ : Integer -> SQL_Builder.code (" TOP " + limit.to_text + " ")
 
     ## PRIVATE
        Returns an ordering of SQL_Part's that determine what order each part gets written in for the final SQL output
@@ -292,6 +296,7 @@ type SQLServer_Dialect
             Feature.Join -> True
             Feature.Union -> True
             Feature.Aggregate -> True
+            Feature.Offset -> True
             _ -> False
 
     ## PRIVATE
@@ -449,6 +454,11 @@ private _generate_expression dialect base_gen expr expression_kind:Expression_Ki
             converted_expr
 
         pair final_expr null_checks_result
+    SQL_Expression.List values ->
+        parsed_args_and_null_checks = values.map (v-> _generate_expression dialect base_gen v expression_kind materialize_null_check)
+        final_expr = SQL_Builder.join ", " (parsed_args_and_null_checks.map p->p.first)
+        null_checks = parsed_args_and_null_checks.map .second  . flatten
+        pair final_expr null_checks
     query : Query -> pair (base_gen.generate_sub_query dialect query) []
     descriptor : Order_Descriptor -> pair (base_gen.generate_order dialect descriptor) []
 
@@ -503,7 +513,8 @@ make_dialect_operations =
     date_ops = [["year", Base_Generator.make_function "year"], make_datepart "quarter", ["month", Base_Generator.make_function "month"], make_datepart "week" "iso_week", ["day", Base_Generator.make_function "day"], make_datepart "hour", make_datepart "minute", make_datepart "day_of_year" "dayofyear", make_day_of_week, make_datepart "second", make_datepart "millisecond", make_extract_microsecond, ["date_add", make_date_add], ["date_diff", make_date_diff], ["date_trunc_to_day", make_date_trunc_to_day]]
     special_overrides = [is_empty, ["IIF", _make_iif]]
     other = [["RUNTIME_ERROR", make_runtime_error_op]]
-    my_mappings = text + counts + arith_extensions + bool + stats + date_ops + special_overrides + other
+    windows = [["LEAD", _make_lead_lag "LEAD"], ["LAG", _make_lead_lag "LAG"], ["LEAD_CLOSEST", _make_lead_lag_closest_value "LEAD"], ["LAG_CLOSEST", _make_lead_lag_closest_value "LAG"]]
+    my_mappings = text + counts + arith_extensions + bool + stats + date_ops + special_overrides + other + windows
     base = Base_Generator.base_dialect_operations . extend_with my_mappings
     Base_Generator.Dialect_Operations.Value (base.operations_dict.remove "IS_IN")
 
@@ -617,6 +628,37 @@ left = Base_Generator.lift_binary_op "LEFT" str-> n->
 right = Base_Generator.lift_binary_op "RIGHT" str-> n->
     SQL_Builder.code "right(" ++ str ++ ", CAST(" ++ n ++ " AS INT))"
 
+_build_partition_sql grouping:SQL_Builder ordering:SQL_Builder -> SQL_Builder =
+    group_part = if grouping.is_empty then "" else
+        SQL_Builder.code "PARTITION BY " ++ grouping
+    SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ ordering ++ ")"
+
+## PRIVATE
+_build_lead_lag_sql lead_lag:Text n:SQL_Builder colName:SQL_Builder grouping:SQL_Builder ordering:SQL_Builder -> SQL_Builder =
+    partition_sql = _build_partition_sql grouping ordering
+    SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) " ++ partition_sql ++ ")"
+
+## PRIVATE
+_make_lead_lag lead_lag:Text arguments:Vector -> SQL_Builder = if arguments.length != 4 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else
+    n = arguments.at 0
+    colName = arguments.at 1
+    grouping = arguments.at 2
+    ordering = arguments.at 3
+    _build_lead_lag_sql lead_lag n colName grouping ordering
+
+## PRIVATE
+_make_lead_lag_closest_value lead_lag:Text arguments:Vector -> SQL_Builder = if arguments.length != 5 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else
+    n = arguments.at 0
+    colName = arguments.at 1
+    grouping = arguments.at 2
+    ordering_for_lead_lag = arguments.at 3
+    ordering_for_row_number = arguments.at 4
+
+    lead_lag_sql = _build_lead_lag_sql lead_lag n colName grouping ordering_for_lead_lag     
+    partition_sql_for_row_number = _build_partition_sql grouping ordering_for_row_number
+    fill_sql = SQL_Builder.code "FIRST_VALUE(" ++ colName ++ ") " ++ partition_sql_for_row_number
+    SQL_Builder.code "CASE WHEN ROW_NUMBER() " ++ partition_sql_for_row_number ++ " <= " ++ n ++ " THEN " ++ fill_sql ++ " ELSE " ++ lead_lag_sql ++ " END"
+
 ## PRIVATE
 make_order_descriptor internal_column sort_direction text_ordering =
     nulls = Nothing

distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso

@@ -95,7 +95,9 @@ type Snowflake_Dialect
        Generates SQL modifier for limiting the number of rows and its position in the query
     get_limit_sql_modifier : Integer -> Any
     get_limit_sql_modifier self limit =
-        SQL_Builder.code (" LIMIT " + limit.to_text)
+        case limit of
+            Nothing -> ""
+            _ : Integer -> SQL_Builder.code (" LIMIT " + limit.to_text)
 
     ## PRIVATE
        Returns an ordering of SQL_Part's that determine what order each part gets written in for the final SQL output

distribution/lib/Standard/Table/0.0.0-dev/src/Table.enso

@@ -146,12 +146,13 @@ type Table
     new : Vector (Vector | Column) -> Table
     new columns =
         invalid_input_shape =
-            Error.throw (Illegal_Argument.Error "Each column must be represented by a pair whose first element is the column name and the second element is a vector of elements that will constitute that column, or an existing column. Got: "+columns.to_text)
+            Error.throw (Illegal_Argument.Error "Each column must be represented by a vector whose first element is the column name, second element is a vector of elements that will constitute that column (or an existing column) and an optional third element that is the Value_Type for the column. Got: "+columns.to_text)
         cols = columns.map on_problems=No_Wrap.Value c->
             case c of
-                v : Vector ->
-                    if v.length != 2 then invalid_input_shape else
-                        Column.from_vector (v.at 0) (v.at 1) . java_column
+                v : Vector -> case v.length of
+                    2 -> Column.from_vector (v.at 0) (v.at 1) . java_column
+                    3 -> Column.from_vector (v.at 0) (v.at 1) (v.at 2) . java_column
+                    _ -> invalid_input_shape
                 col : Column -> col.java_column
                 _ -> invalid_input_shape
         Panic.recover Illegal_Argument <|