-
Notifications
You must be signed in to change notification settings - Fork 0
/
etl_Transformation_bp_hbase_lookup.xmi
20 lines (20 loc) · 8.34 KB
/
etl_Transformation_bp_hbase_lookup.xmi
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
<?xml version="1.0" encoding="ASCII"?>
<etl:Transformation xmi:version="2.0" xmlns:xmi="http://www.omg.org/XMI" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:etl="http://www.neoflex.ru/meta/etl" name="bp_hbase_lookup" label="bp_hbase_lookup" jsonView="{"cells":[{"type":"devs.transformation.ExpressionSource","size":{"width":80,"height":80},"inPorts":[],"outPorts":["out"],"ports":{"groups":{"in":{"position":{"name":"left"},"attrs":{".port-body":{"fill":"white","magnet":"passive","stroke":"#000","r":10},".port-label":{"text":"","visibility":"hidden","fill":"#000"},"circle":{"r":7,"stroke":"#4d4d4dde","stroke-width":1}},"label":{"position":{"name":"left","args":{"y":10}}}},"out":{"position":{"name":"right"},"label":{"position":{"name":"right","args":{"y":10}}},"attrs":{".port-body":{"fill":"#4d4d4dde","stroke":"#000","r":10,"magnet":true},".port-label":{"text":"","visibility":"hidden","fill":"#000"},"circle":{"r":7,"stroke":"#4d4d4dde","stroke-width":1}}}},"items":[{"id":"out","group":"out","attrs":{".port-label":{"text":"out"}}}]},"position":{"x":60,"y":210},"angle":0,"id":"d7cc60e6-2bd8-4f64-bf6f-eff53e51fcf0","z":1,"name":"Expression_0","attrs":{".label":{"text":"Expression_0"}}},{"type":"devs.transformation.SparkSQL","size":{"width":80,"height":80},"inPorts":["sqlin"],"outPorts":["out"],"ports":{"groups":{"in":{"position":{"name":"left"},"attrs":{".port-body":{"fill":"white","magnet":"passive","stroke":"#000","r":10},".port-label":{"text":"","visibility":"hidden","fill":"#000"},"circle":{"r":7,"stroke":"#4d4d4dde","stroke-width":1}},"label":{"position":{"name":"left","args":{"y":10}}}},"out":{"position":{"name":"right"},"label":{"position":{"name":"right","args":{"y":10}}},"attrs":{".port-body":{"fill":"#4d4d4dde","stroke":"#000","r":10,"magnet":true},".port-label":{"text":"","visibility":"hidden","fill":"#000"},"circle":{"r":7,"stroke":"#4d4d4dde","stroke-width":1}}}},"items":[{"id":"out","group":"out","attrs":{".port-label":{"text":"out"}}},{"id":"0","multiple":true,"group":"in","attrs":{".port-label":{"text":"seq","visibility":"visible"},"circle":{"fill":"white","r":7,"stroke":"#4d4d4dde","stroke-width":1}}}]},"position":{"x":270,"y":210},"angle":0,"id":"0ddd83be-245f-4531-b789-7a90dbfb7986","z":2,"name":"Spark_SQL_1","attrs":{".label":{"text":"Spark_SQL_1"}}},{"type":"devs.transformation.LocalTarget","size":{"width":80,"height":80},"inPorts":["in"],"outPorts":[],"ports":{"groups":{"in":{"position":{"name":"left"},"attrs":{".port-body":{"fill":"white","magnet":"passive","stroke":"#000","r":10},".port-label":{"text":"","visibility":"hidden","fill":"#000"},"circle":{"r":7,"stroke":"#4d4d4dde","stroke-width":1}},"label":{"position":{"name":"left","args":{"y":10}}}},"out":{"position":{"name":"right"},"label":{"position":{"name":"right","args":{"y":10}}},"attrs":{".port-body":{"fill":"#4d4d4dde","stroke":"#000","r":10,"magnet":true},".port-label":{"text":"","visibility":"hidden","fill":"#000"},"circle":{"r":7,"stroke":"#4d4d4dde","stroke-width":1}}}},"items":[{"id":"in","group":"in","attrs":{".port-label":{"text":"in"}}}]},"position":{"x":480,"y":210},"angle":0,"id":"636d1f9a-f351-421a-86d2-26f8b0985672","z":3,"name":"Local_3","attrs":{".label":{"text":"Local_3"}}},{"type":"devs.transformation.Flow","source":{"id":"d7cc60e6-2bd8-4f64-bf6f-eff53e51fcf0","port":"out"},"target":{"id":"0ddd83be-245f-4531-b789-7a90dbfb7986","port":"0"},"id":"603b1000-47fd-42d4-b9c8-89c9cb99ae16","z":4,"attrs":{}},{"type":"devs.transformation.Flow","source":{"id":"0ddd83be-245f-4531-b789-7a90dbfb7986","port":"out"},"target":{"id":"636d1f9a-f351-421a-86d2-26f8b0985672","port":"in"},"id":"9efae34f-2b43-446c-bf47-2269b5862893","z":5,"attrs":{}}]}">
<targets xsi:type="etl:LocalTarget" name="Local_3" label="Local_3" localFileName="/tmp/blueprint/hbase_lookup_result" saveMode="OVERWRITE">
<inputPort name="Local_3in"/>
</targets>
<transformationSteps xsi:type="etl:SparkSQL" name="Spark_SQL_1" label="Spark_SQL_1" statement="with seqlk as (
 select 
 s.id as id, 
 HBASE_LOOKUP('address', '0:cust_id;0:address;0:main', LONG2BINARY(s.id)) as arr
 from seq s
)
select 
 lk.id,
 BINARY2LONG(lk.arr[0]) as cust_id,
 BINARY2STRING(lk.arr[1]) as address,
 BINARY2BOOLEAN(lk.arr[2]) as main
from seqlk lk
">
<outputPort name="Spark_SQL_1out"/>
<sqlPorts name="seq" alias="seq">
<fields name="id" dataTypeDomain="LONG"/>
</sqlPorts>
</transformationSteps>
<sources xsi:type="etl:ExpressionSource" name="Expression_0" label="Expression_0" expression="(10L to 21L).map(id=>Map("id"->id.asInstanceOf[AnyRef]))
">
<outputPort name="Expression_0out">
<fields name="id" dataTypeDomain="LONG"/>
</outputPort>
</sources>
<transitions name="2610928e-09f3-4787-ac46-e56359be85cc" start="//@sources.0/@outputPort" finish="//@transformationSteps.0/@sqlPorts.0"/>
<transitions name="71320cc8-a61d-4fe0-be65-017524f6e7b2" start="//@transformationSteps.0/@outputPort" finish="//@targets.0/@inputPort"/>
<auditInfo changeUser="admin" changeDateTime="2019-06-20T12:06:45.153+0300" createUser="admin" createDateTime="2019-05-21T18:31:54.667+0300"/>
</etl:Transformation>