Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SNOW-1738538 Update test expectations to textproto ast for better stability and readability #2464

Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 46 additions & 1 deletion src/snowflake/snowpark/_internal/ast_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@
# Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved.
#
import ast
import base64
import datetime
import decimal
import inspect
import logging
import os
import platform
import re
import sys
import typing
from functools import reduce
Expand All @@ -18,6 +20,7 @@

import dateutil
from dateutil.tz import tzlocal
from google.protobuf.text_format import MessageToString, Parse

import snowflake.snowpark
import snowflake.snowpark._internal.proto.ast_pb2 as proto
Expand All @@ -40,7 +43,7 @@
ColumnOrName,
ColumnOrSqlExpr,
)
from snowflake.snowpark._internal.utils import str_to_enum
from snowflake.snowpark._internal.utils import TempObjectType, str_to_enum
from snowflake.snowpark.types import DataType, StructType

# This flag causes an explicit error to be raised if any Snowpark object instance is missing an AST or field, when this
Expand Down Expand Up @@ -1325,3 +1328,45 @@ def build_expr_from_dict_str_str(
t = ast_dict.add()
t._1 = k
t._2 = v


def reset_snowpark_temp_ids(textproto: str) -> str:
id_lookup = {}
cnt = 0
for prefix, object_type, orig_id in re.findall(
rf"\"(SNOWPARK_TEMP_)({'|'.join([e.value for e in TempObjectType])})_(.*)\"",
textproto,
):
if orig_id not in id_lookup:
cnt = cnt + 1
id_lookup[orig_id] = f"{cnt:010x}"
sub_id = id_lookup[orig_id]
textproto = textproto.replace(
f"{prefix}{object_type}_{orig_id}", f"{prefix}{object_type}_{sub_id}"
)
return textproto


def base64_str_to_request(base64_str: str) -> proto.Request:
message = proto.Request()
message.ParseFromString(base64.b64decode(base64_str.strip()))
return message


def base64_str_to_textproto(base64_str: str) -> str:
request = base64_str_to_request(base64_str)

# Force a fixed python version to avoid unnecessary diffs
request.client_language.python_language.version.major = 3
request.client_language.python_language.version.minor = 9
request.client_language.python_language.version.patch = 1
request.client_language.python_language.version.label = "final"

message = MessageToString(request)
message = reset_snowpark_temp_ids(message)
sfc-gh-evandenberg marked this conversation as resolved.
Show resolved Hide resolved
return message


def textproto_to_request(textproto_str) -> proto.Request:
request = Parse(textproto_str, proto.Request())
return request
4 changes: 2 additions & 2 deletions src/snowflake/snowpark/_internal/proto/ast_pb2.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: proto/ast.proto
# Protobuf Python Version: 5.27.0
# Protobuf Python Version: 5.28.2
"""Generated protocol buffer code."""
from google.protobuf import (
descriptor as _descriptor,
Expand All @@ -16,7 +16,7 @@
from google.protobuf.internal import builder as _builder

_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC, 5, 27, 0, "", "proto/ast.proto"
_runtime_version.Domain.PUBLIC, 5, 28, 2, "", "proto/ast.proto"
)
# @@protoc_insertion_point(imports)

Expand Down
4 changes: 2 additions & 2 deletions src/snowflake/snowpark/_internal/proto/scalapb_pb2.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: scalapb/scalapb.proto
# Protobuf Python Version: 5.27.0
# Protobuf Python Version: 5.28.2
"""Generated protocol buffer code."""
from google.protobuf import (
descriptor as _descriptor,
Expand All @@ -16,7 +16,7 @@
from google.protobuf.internal import builder as _builder

_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC, 5, 27, 0, "", "scalapb/scalapb.proto"
_runtime_version.Domain.PUBLIC, 5, 28, 2, "", "scalapb/scalapb.proto"
)
# @@protoc_insertion_point(imports)

Expand Down
Loading
Loading