Merge pull request #20 from MaryamZi/add-samples

Add samples

data_integration/README.md

@@ -0,0 +1,5 @@
+## Data integration samples
+
+Consists of samples on 
+- [data transformation and mapping](./transformation/)
+- [database operations](./database/)

data_integration/database/README.md

@@ -0,0 +1,5 @@
+## Database samples
+
+Implemented in two ways
+1. using [the Ballerina persistence layer](./db_persistence/) 
+2. using [the Ballerina SQL/MySQL clients](./db_sql_client/) (slightly modified)

data_integration/database/db_persistence/.gitignore

@@ -0,0 +1,3 @@
+target
+Config.toml
+Dependencies.toml

data_integration/database/db_persistence/Ballerina.toml

@@ -0,0 +1,18 @@
+[package]
+org = "integration_samples"
+name = "db_persistence"
+version = "0.1.0"
+distribution = "2201.6.0"
+
+[build-options]
+observabilityIncluded = true
+
+[persist]
+datastore = "mysql"
+module = "db_persistence.store"
+
+[[platform.java11.dependency]]
+groupId = "io.ballerina.stdlib"
+artifactId = "persist.sql-native"
+version = "1.0.0"
+

data_integration/database/db_persistence/functions.bal

@@ -0,0 +1,44 @@
+import db_persistence.store;
+
+import ballerina/persist;
+
+final store:Client cl = check new;
+
+function selectAll() returns store:Book[]|persist:Error {
+    stream<store:Book, persist:Error?> bookStr = cl->/books();
+    return from store:Book book in bookStr select book;
+}
+
+function selectFilteringByKeyId(int id) returns store:Book|persist:Error {
+    return cl->/books/[id]();
+}
+
+function selectFilteringByAuthor(string author) returns store:Book[]|persist:Error {
+    stream<store:Book, persist:Error?> bookStr = cl->/books();
+    return from store:Book book in bookStr where book.author == author select book;
+}
+
+function insert(store:Book book) returns int|persist:Error {
+    int[] ids = check cl->/books.post([book]);
+    return ids[0];
+}
+
+function update(function (store:Book) returns store:BookUpdate? fn) returns persist:Error? {
+    stream<store:Book, persist:Error?> bookStr = cl->/books();
+    [int, store:BookUpdate][] itemsToUpdate = [];
+
+    check bookStr.forEach(function (store:Book book) {
+        store:BookUpdate? update = fn(book);
+        if update !is () {
+            itemsToUpdate.push([book.id, update]);
+        }
+    });
+
+    foreach [int, store:BookUpdate] [id, bookUpdate] in itemsToUpdate {
+        _ = check cl->/books/[id].put(bookUpdate);
+    }
+}
+
+function delete(int id) returns persist:Error? {
+    _ = check cl->/books/[id].delete();
+}

data_integration/database/db_persistence/generated/store/persist_client.bal

@@ -0,0 +1,92 @@
+// AUTO-GENERATED FILE. DO NOT MODIFY.
+
+// This file is an auto-generated file by Ballerina persistence layer for model.
+// It should not be modified by hand.
+
+import ballerina/persist;
+import ballerina/jballerina.java;
+import ballerinax/mysql;
+import ballerinax/mysql.driver as _;
+import ballerinax/persist.sql as psql;
+
+const BOOK = "books";
+
+public isolated client class Client {
+    *persist:AbstractPersistClient;
+
+    private final mysql:Client dbClient;
+
+    private final map<psql:SQLClient> persistClients;
+
+    private final record {|psql:SQLMetadata...;|} & readonly metadata = {
+        [BOOK] : {
+            entityName: "Book",
+            tableName: "Book",
+            fieldMetadata: {
+                id: {columnName: "id"},
+                title: {columnName: "title"},
+                author: {columnName: "author"},
+                isbn: {columnName: "isbn"},
+                price: {columnName: "price"}
+            },
+            keyFields: ["id"]
+        }
+    };
+
+    public isolated function init() returns persist:Error? {
+        mysql:Client|error dbClient = new (host = host, user = user, password = password, database = database, port = port, options = connectionOptions);
+        if dbClient is error {
+            return <persist:Error>error(dbClient.message());
+        }
+        self.dbClient = dbClient;
+        self.persistClients = {[BOOK] : check new (dbClient, self.metadata.get(BOOK))};
+    }
+
+    isolated resource function get books(BookTargetType targetType = <>) returns stream<targetType, persist:Error?> = @java:Method {
+        'class: "io.ballerina.stdlib.persist.sql.datastore.MySQLProcessor",
+        name: "query"
+    } external;
+
+    isolated resource function get books/[int id](BookTargetType targetType = <>) returns targetType|persist:Error = @java:Method {
+        'class: "io.ballerina.stdlib.persist.sql.datastore.MySQLProcessor",
+        name: "queryOne"
+    } external;
+
+    isolated resource function post books(BookInsert[] data) returns int[]|persist:Error {
+        psql:SQLClient sqlClient;
+        lock {
+            sqlClient = self.persistClients.get(BOOK);
+        }
+        _ = check sqlClient.runBatchInsertQuery(data);
+        return from BookInsert inserted in data
+            select inserted.id;
+    }
+
+    isolated resource function put books/[int id](BookUpdate value) returns Book|persist:Error {
+        psql:SQLClient sqlClient;
+        lock {
+            sqlClient = self.persistClients.get(BOOK);
+        }
+        _ = check sqlClient.runUpdateQuery(id, value);
+        return self->/books/[id].get();
+    }
+
+    isolated resource function delete books/[int id]() returns Book|persist:Error {
+        Book result = check self->/books/[id].get();
+        psql:SQLClient sqlClient;
+        lock {
+            sqlClient = self.persistClients.get(BOOK);
+        }
+        _ = check sqlClient.runDeleteQuery(id);
+        return result;
+    }
+
+    public isolated function close() returns persist:Error? {
+        error? result = self.dbClient.close();
+        if result is error {
+            return <persist:Error>error(result.message());
+        }
+        return result;
+    }
+}
+

data_integration/database/db_persistence/generated/store/persist_db_config.bal

@@ -0,0 +1,14 @@
+// AUTO-GENERATED FILE. DO NOT MODIFY.
+
+// This file is an auto-generated file by Ballerina persistence layer.
+// It should not be modified by hand.
+
+import ballerinax/mysql;
+
+configurable int port = ?;
+configurable string host = ?;
+configurable string user = ?;
+configurable string database = ?;
+configurable string password = ?;
+configurable mysql:Options & readonly connectionOptions = {};
+

data_integration/database/db_persistence/generated/store/persist_types.bal

@@ -0,0 +1,32 @@
+// AUTO-GENERATED FILE. DO NOT MODIFY.
+
+// This file is an auto-generated file by Ballerina persistence layer for model.
+// It should not be modified by hand.
+
+public type Book record {|
+    readonly int id;
+    string title;
+    string author;
+    string isbn;
+    int price;
+|};
+
+public type BookOptionalized record {|
+    int id?;
+    string title?;
+    string author?;
+    string isbn?;
+    int price?;
+|};
+
+public type BookTargetType typedesc<BookOptionalized>;
+
+public type BookInsert Book;
+
+public type BookUpdate record {|
+    string title?;
+    string author?;
+    string isbn?;
+    int price?;
+|};
+

data_integration/database/db_persistence/generated/store/script.sql

@@ -0,0 +1,15 @@
+-- AUTO-GENERATED FILE.
+
+-- This file is an auto-generated file by Ballerina persistence layer for model.
+-- Please verify the generated scripts and execute them against the target DB server.
+
+DROP TABLE IF EXISTS `Book`;
+
+CREATE TABLE `Book` (
+	`id` INT NOT NULL,
+	`title` VARCHAR(191) NOT NULL,
+	`author` VARCHAR(191) NOT NULL,
+	`isbn` VARCHAR(191) NOT NULL,
+	`price` INT NOT NULL,
+	PRIMARY KEY(`id`)
+);

data_integration/database/db_persistence/main.bal

@@ -0,0 +1,44 @@
+import db_persistence.store;
+
+import ballerina/io;
+import ballerina/persist;
+
+public function main() returns error? {
+    check clearTableAndAddData();
+
+    store:Book[] allBooks = check selectAll();
+    io:println(allBooks);
+
+    store:Book book3 = check selectFilteringByKeyId(3);
+    io:println(book3);
+
+    store:Book[] filteredBooks = check selectFilteringByAuthor("Charles Dickens");
+    io:println(filteredBooks);
+
+    int id = check insert({
+        id: 4,
+        title: "Great Expectations",
+        author: "Charles Dickens",
+        price: 1432,
+        isbn: "9781503275188"
+    });
+    io:println(id);
+}
+
+function clearTableAndAddData() returns error? {
+    stream<store:Book, persist:Error?> str = cl->/books();
+    check from store:Book {id} in str
+    do {
+        _ = check cl->/books/[id].delete;
+    };
+
+    store:Book[] books = [
+        {id: 1, title: "A Christmas Carol", author: "Charles Dickens", isbn: "9781503212831", price: 1843},
+        {id: 2, title: "Oliver Twist", author: "Charles Dickens", isbn: "9780141439747", price: 1838},
+        {id: 3, title: "Heidi", author: "Johanna Spyri", isbn: "9780517189672", price: 1811}
+    ];
+
+    foreach store:Book book in books {
+        _ = check insert(book);
+    }
+}

data_integration/database/db_persistence/persist/model.bal

@@ -0,0 +1,9 @@
+import ballerina/persist as _;
+
+public type Book record {|
+    readonly int id;
+    string title;
+    string author;
+    string isbn;
+    int price;
+|};

data_integration/database/db_persistence/tests/tests.bal

@@ -0,0 +1,107 @@
+import db_persistence.store;
+
+import ballerina/persist;
+import ballerina/test;
+
+@test:BeforeSuite
+function addToTable() returns error? {
+    check cleanDb();
+    // Can alternatively be added via the script.
+    // Auto increment support is not yet available in the persistence layer,
+    // but is planned for a future release.
+    store:Book[] books = [
+        {id: 1, title: "A Christmas Carol", author: "Charles Dickens", isbn: "9781503212831", price: 1843},
+        {id: 2, title: "Oliver Twist", author: "Charles Dickens", isbn: "9780141439747", price: 1838},
+        {id: 3, title: "Heidi", author: "Johanna Spyri", isbn: "9780517189672", price: 1811}
+    ];
+
+    foreach store:Book book in books {
+        _ = check insert(book);
+    }
+}
+
+@test:Config
+function testSelectAll() returns error? {
+    store:Book[] allBooks = check selectAll();
+    store:Book[] expectedBooks = [
+        {id: 1, title: "A Christmas Carol", author: "Charles Dickens", isbn: "9781503212831", price: 1843},
+        {id: 2, title: "Oliver Twist", author: "Charles Dickens", isbn: "9780141439747", price: 1838},
+        {id: 3, title: "Heidi", author: "Johanna Spyri", isbn: "9780517189672", price: 1811}
+    ];
+    test:assertEquals(allBooks, expectedBooks);
+}
+
+@test:Config
+function testSelectFilteringByKeyId() returns error? {
+    store:Book book3 = check selectFilteringByKeyId(3);
+    test:assertEquals(book3, {id: 3, title: "Heidi", author: "Johanna Spyri", isbn: "9780517189672", price: 1811});
+
+    store:Book|persist:Error book10 = selectFilteringByKeyId(10);
+    test:assertTrue(book10 is persist:Error);
+}
+
+@test:Config
+function testSelectFilteringByAuthor() returns error? {
+    store:Book[] stephenKingBooks = check selectFilteringByAuthor("Charles Dickens");
+    test:assertEquals(stephenKingBooks.length(), 2);
+    test:assertEquals(stephenKingBooks, [
+        {id: 1, title: "A Christmas Carol", author: "Charles Dickens", isbn: "9781503212831", price: 1843},
+        {id: 2, title: "Oliver Twist", author: "Charles Dickens", isbn: "9780141439747", price: 1838}
+    ]);
+}
+
+@test:Config {
+    dependsOn: [testSelectAll, testSelectFilteringByAuthor]
+}
+function testInsert() returns error? {
+    int id = check insert({
+        id: 4,
+        title: "Great Expectations",
+        author: "Charles Dickens",
+        price: 1432,
+        isbn: "9781503275188"
+    });
+    test:assertEquals(id, 4);
+}
+
+@test:Config {
+    dependsOn: [testInsert, testSelectFilteringByKeyId, testSelectFilteringByAuthor]
+}
+function testUpdate() returns error? {
+    _ = check update(book => book.price < 1840 ? 
+            let store:Book {title, author, isbn, price} = book in
+                {title, author, isbn, price: price + 10} : 
+            ());
+
+    test:assertEquals(check selectAll(), [
+        {id: 1, title: "A Christmas Carol", author: "Charles Dickens", isbn: "9781503212831", price: 1843},
+        {id: 2, title: "Oliver Twist", author: "Charles Dickens", isbn: "9780141439747", price: 1848},
+        {id: 3, title: "Heidi", author: "Johanna Spyri", isbn: "9780517189672", price: 1821},
+        {id: 4, title: "Great Expectations", author: "Charles Dickens", isbn: "9781503275188", price: 1442}
+    ]);
+}
+
+@test:Config {
+    dependsOn: [testUpdate]
+}
+function testDelete() returns error? {
+    check delete(2);
+    store:Book|persist:Error book2 = selectFilteringByKeyId(2);
+    test:assertTrue(book2 is persist:Error);
+    persist:Error err = <persist:Error> book2;
+    test:assertEquals(err.message(), "A record does not exist for 'Book' for key 2.");
+
+    persist:Error? delError = delete(12);
+    test:assertTrue(delError is persist:Error);
+    err = <persist:Error> delError;
+    test:assertEquals(err.message(), "A record does not exist for 'Book' for key 12.");
+}
+
+@test:AfterSuite
+function cleanDb() returns error? {
+    stream<store:Book, persist:Error?> books = cl->/books();
+    check from store:Book {id} in books
+    do {
+        _ = check cl->/books/[id].delete;
+    };
+}