-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
5 changed files
with
168 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
target/classes | ||
target/maven-status | ||
target/maven-archiver | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
# JDBC Data Reader for Snowpark | ||
|
||
See[ medium article for more details.](https://medium.com/@orellabac/ingest-external-data-into-snowflake-with-snowpark-and-jdbc-eb487b61078c) | ||
|
||
**Building** | ||
|
||
``` | ||
mvn clean package | ||
``` | ||
|
||
The target file will be at: `target/snowpark-jdbc-reader-0.0.1.jar` | ||
|
||
For convience there is a prebuilt-jar you can use. | ||
|
||
**Deployment** | ||
|
||
upload the jar to snowflake, you can use the [SnowSight UI to do that easily ](https://docs.snowflake.com/en/user-guide/data-load-local-file-system-stage-ui) | ||
|
||
and create an UDF like this: | ||
|
||
``` | ||
CREATE OR REPLACE SECRET external_database_cred | ||
TYPE = password | ||
USERNAME = 'serveradmin' | ||
PASSWORD = 'xxxxxxxxx'; | ||
CREATE OR REPLACE EXTERNAL ACCESS INTEGRATION external_database_network_rule_ext_int | ||
ALLOWED_NETWORK_RULES = (external_database_network_rule) | ||
ALLOWED_AUTHENTICATION_SECRETS = (external_database_cred) | ||
ENABLED = true; | ||
CREATE OR REPLACE FUNCTION READ_JDBC(OPTION OBJECT, query STRING) | ||
RETURNS TABLE(data OBJECT) | ||
LANGUAGE JAVA | ||
RUNTIME_VERSION='11' | ||
IMPORTS = ('@mystage/snowpark-jdbc-reader-0.0.1.jar' ) -- Add the jar for any jdbc driver you need in this section | ||
EXTERNAL_ACCESS_INTEGRATIONS = (external_database_network_rule_ext_int) | ||
SECRETS = ('cred' = external_database_cred ) | ||
HANDLER = 'JdbcDataReader'; | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
<?xml version="1.0" encoding="UTF-8"?> | ||
|
||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | ||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | ||
<modelVersion>4.0.0</modelVersion> | ||
|
||
<groupId>com.snowflake</groupId> | ||
<artifactId>snowpark-jdbc-reader</artifactId> | ||
<version>0.0.1</version> | ||
|
||
<name>Simple JDBC reader</name> | ||
|
||
<url>https://medium.com/@orellabac/ingest-external-data-into-snowflake-with-snowpark-and-jdbc-eb487b61078c</url> | ||
|
||
<properties> | ||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> | ||
<maven.compiler.source>11</maven.compiler.source> | ||
<maven.compiler.target>11</maven.compiler.target> | ||
</properties> | ||
|
||
<dependencies> | ||
<dependency> | ||
<groupId>com.snowflake</groupId> | ||
<artifactId>snowpark</artifactId> | ||
<version>1.9.0</version> | ||
<scope>provided</scope> | ||
</dependency> | ||
<dependency> | ||
<groupId>junit</groupId> | ||
<artifactId>junit</artifactId> | ||
<version>4.11</version> | ||
<scope>test</scope> | ||
</dependency> | ||
</dependencies> | ||
<build> | ||
</build> | ||
</project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,86 @@ | ||
import java.io.PrintWriter; | ||
import java.io.StringWriter; | ||
import java.sql.*; | ||
import java.util.*; | ||
import java.util.stream.Stream; | ||
import com.snowflake.snowpark_java.types.SnowflakeSecrets; | ||
|
||
public class JdbcDataReader { | ||
|
||
public static class OutputRow { | ||
public Map<String, String> data; | ||
|
||
public OutputRow(Map<String, String> data) { | ||
this.data = data; | ||
} | ||
} | ||
|
||
public static Class getOutputClass() { | ||
return OutputRow.class; | ||
} | ||
|
||
public Stream<OutputRow> process(Map<String, String> jdbcConfig, String query) { | ||
String jdbcUrl = jdbcConfig.get("url"); | ||
Properties properties = new Properties(); | ||
|
||
if ("true".equals(jdbcConfig.get("use_secrets"))) | ||
{ | ||
SnowflakeSecrets sfSecrets = SnowflakeSecrets.newInstance(); | ||
jdbcConfig.remove("use_secrets"); | ||
var secret = sfSecrets.getUsernamePassword("cred"); | ||
properties.setProperty("username", secret.getUsername()); | ||
properties.setProperty("password", secret.getPassword()); | ||
} | ||
|
||
try { | ||
properties.putAll(jdbcConfig); | ||
// Load the JDBC driver | ||
Class.forName(jdbcConfig.get("driver")); | ||
// Create a connection to the database | ||
Connection connection = DriverManager.getConnection(jdbcUrl, properties); | ||
// Create a statement for executing SQL queries | ||
Statement statement = connection.createStatement(); | ||
// Execute the query | ||
ResultSet resultSet = statement.executeQuery(query); | ||
// Get metadata about the result set | ||
ResultSetMetaData metaData = resultSet.getMetaData(); | ||
// Create a list of column names | ||
List<String> columnNames = new ArrayList<>(); | ||
int columnCount = metaData.getColumnCount(); | ||
for (int i = 1; i <= columnCount; i++) { | ||
columnNames.add(metaData.getColumnName(i)); | ||
} | ||
// Convert the ResultSet to a Stream of OutputRow objects | ||
Stream<OutputRow> resultStream = Stream.generate(() -> { | ||
try { | ||
if (resultSet.next()) { | ||
Map<String, String> rowMap = new HashMap<>(); | ||
for (String columnName : columnNames) { | ||
String columnValue = resultSet.getString(columnName); | ||
rowMap.put(columnName, columnValue); | ||
} | ||
return new OutputRow(rowMap); | ||
} else { | ||
// Close resources | ||
resultSet.close(); | ||
statement.close(); | ||
connection.close(); | ||
return null; | ||
} | ||
} catch (SQLException e) { | ||
e.printStackTrace(); | ||
return null; | ||
} | ||
}).takeWhile(Objects::nonNull); | ||
return resultStream; | ||
} catch (Exception e) { | ||
StringWriter sw = new StringWriter(); | ||
PrintWriter pw = new PrintWriter(sw); | ||
e.printStackTrace(pw); | ||
String stackTrace = sw.toString(); | ||
Map<String, String> rowMap = new HashMap<>(); | ||
rowMap.put("ERROR",e.getMessage() + "\n" + stackTrace); | ||
return Stream.of(new OutputRow(rowMap)); | ||
} | ||
} | ||
} |
Binary file not shown.