HPCC-33278 Fix //version race condition in parquetType.ecl

Signed-off-by: Jack Del Vecchio <jack.delvecchio@lexisnexisrisk.com>

Author: jackdelv

testing/regress/ecl/key/parquetTypes.xml

@@ -1,8 +1,3 @@
-<Dataset name='SetTest'>
- <Row><testid>1100</testid><testname>empty</testname><s0></s0><s1></s1><s2></s2><s3></s3><s4></s4><s5></s5><s6></s6><s7></s7><s8></s8><s9></s9><s10></s10></Row>
- <Row><testid>1101</testid><testname>single</testname><s0><Item>true</Item></s0><s1><Item>1</Item></s1><s2><Item>1</Item></s2><s3><Item>1.0</Item></s3><s4><Item>1</Item></s4><s5><Item>a</Item></s5><s6><Item>a</Item></s6><s7><Item>A</Item></s7><s8><Item>a</Item></s8><s9><Item>a</Item></s9><s10><Item>FFFF</Item></s10></Row>
- <Row><testid>1102</testid><testname>multiple</testname><s0><Item>true</Item><Item>false</Item></s0><s1><Item>1</Item><Item>2</Item></s1><s2><Item>1</Item><Item>2</Item></s2><s3><Item>1.0</Item><Item>2.0</Item></s3><s4><Item>1</Item><Item>2</Item></s4><s5><Item>a</Item><Item>b</Item></s5><s6><Item>a</Item><Item>b</Item></s6><s7><Item>A</Item><Item>B</Item></s7><s8><Item>a</Item><Item>b</Item></s8><s9><Item>a</Item><Item>b</Item></s9><s10><Item>0000</Item><Item>FFFF</Item></s10></Row>
-</Dataset>
 <Dataset name='BooleanTest'>
  <Row><BooleanTest>Pass</BooleanTest></Row>
 </Dataset>
@@ -36,3 +31,8 @@
 <Dataset name='UnicodeTest'>
  <Row><UnicodeTest>Pass</UnicodeTest></Row>
 </Dataset>
+<Dataset name='SetTest'>
+ <Row><testid>1100</testid><testname>empty</testname><s0></s0><s1></s1><s2></s2><s3></s3><s4></s4><s5></s5><s6></s6><s7></s7><s8></s8><s9></s9><s10></s10></Row>
+ <Row><testid>1101</testid><testname>single</testname><s0><Item>true</Item></s0><s1><Item>1</Item></s1><s2><Item>1</Item></s2><s3><Item>1.0</Item></s3><s4><Item>1</Item></s4><s5><Item>a</Item></s5><s6><Item>a</Item></s6><s7><Item>A</Item></s7><s8><Item>a</Item></s8><s9><Item>a</Item></s9><s10><Item>FFFF</Item></s10></Row>
+ <Row><testid>1102</testid><testname>multiple</testname><s0><Item>true</Item><Item>false</Item></s0><s1><Item>1</Item><Item>2</Item></s1><s2><Item>1</Item><Item>2</Item></s2><s3><Item>1.0</Item><Item>2.0</Item></s3><s4><Item>1</Item><Item>2</Item></s4><s5><Item>a</Item><Item>b</Item></s5><s6><Item>a</Item><Item>b</Item></s6><s7><Item>A</Item><Item>B</Item></s7><s8><Item>a</Item><Item>b</Item></s8><s9><Item>a</Item><Item>b</Item></s9><s10><Item>0000</Item><Item>FFFF</Item></s10></Row>
+</Dataset>

testing/regress/ecl/parquetTypes.ecl

@@ -21,13 +21,14 @@
 //version compressionType='LZ4'
 //version compressionType='ZSTD'
 
+import STD.File AS FileServices;
 import ^ as root;
 compressionType := #IFDEFINED(root.compressionType, 'UNCOMPRESSED');
 
 IMPORT Std;
 IMPORT Parquet;
 
-dropzoneDirectory := Std.File.GetDefaultDropZone() + '/regress/parquet';
+dropzoneDirectory := Std.File.GetDefaultDropZone() + '/regress/parquet/' + WORKUNIT + '-';
 
 // Covers data types supported by ECL and Arrow
 
@@ -40,9 +41,7 @@ booleanDatasetOut := DATASET([
     {001, 'aab', FALSE}
 ], booleanRecord);
 
-ParquetIO.Write(booleanDatasetOut, dropzoneDirectory + '/BooleanTest.parquet', TRUE, compressionType);
-
-booleanDatasetIn := ParquetIO.Read(booleanRecord, dropzoneDirectory + '/BooleanTest.parquet');
+booleanDatasetIn := ParquetIO.Read(booleanRecord, dropzoneDirectory + 'BooleanTest.parquet');
 
 {UNSIGNED testid, STRING3 testname, BOOLEAN isEqual} booleanJoin (booleanDatasetOut a, booleanDatasetIn b) := TRANSFORM
     SELF.testid := a.testid;
@@ -63,9 +62,7 @@ integerDatasetOut := DATASET([
     {101, 'maxvalues', 127, 32767, 8388607, 2147483647, 549755813887, 140737488355327, 36028797018963967, 9223372036854775807}
 ], integerRecord);
 
-ParquetIO.Write(integerDatasetOut, dropzoneDirectory + '/IntegerTest.parquet', TRUE, compressionType);
-
-integerDatasetIn := ParquetIO.Read(integerRecord, dropzoneDirectory + '/IntegerTest.parquet');
+integerDatasetIn := ParquetIO.Read(integerRecord, dropzoneDirectory + 'IntegerTest.parquet');
 
 {UNSIGNED testid, STRING testname, BOOLEAN isEqual} integerJoin (integerDatasetOut a, integerDatasetIn b) := TRANSFORM
     SELF.testid := a.testid;
@@ -93,9 +90,7 @@ unsignedDatasetOut := DATASET([
     {201, 'maxvalues', 255, 65535, 16777215, 4294967295, 1099511627775, 281474976710655, 72057594037927935, 18446744073709551615}
 ], unsignedRecord);
 
-ParquetIO.Write(unsignedDatasetOut, dropzoneDirectory + '/UnsignedTest.parquet', TRUE, compressionType);
-
-unsignedDatasetIn := ParquetIO.Read(unsignedRecord, dropzoneDirectory + '/UnsignedTest.parquet');
+unsignedDatasetIn := ParquetIO.Read(unsignedRecord, dropzoneDirectory + 'UnsignedTest.parquet');
 
 {UNSIGNED testid, STRING testname, BOOLEAN isEqual} unsignedJoin (unsignedDatasetOut a, unsignedDatasetIn b) := TRANSFORM
     SELF.testid := a.testid;
@@ -127,9 +122,7 @@ realDatasetOut := DATASET([
     {305, 'negative', -9.87D, -234.853D}
 ], realRecord);
 
-ParquetIO.Write(realDatasetOut, dropzoneDirectory + '/RealTest.parquet', TRUE, compressionType);
-
-realDatasetIn := ParquetIO.Read(realRecord, dropzoneDirectory + '/RealTest.parquet');
+realDatasetIn := ParquetIO.Read(realRecord, dropzoneDirectory + 'RealTest.parquet');
 
 {UNSIGNED testid, STRING testname, BOOLEAN isEqual} realJoin (realDatasetOut a, realDatasetIn b) := TRANSFORM
     SELF.testid := a.testid;
@@ -152,9 +145,7 @@ decimalDatasetOut := DATASET([
     {402, 'abb', 0.00D}
 ], decimalRecord);
 
-ParquetIO.Write(decimalDatasetOut, dropzoneDirectory + '/DecimalTest.parquet', TRUE, compressionType);
-
-decimalDatasetIn := ParquetIO.Read(decimalRecord, dropzoneDirectory + '/DecimalTest.parquet');
+decimalDatasetIn := ParquetIO.Read(decimalRecord, dropzoneDirectory + 'DecimalTest.parquet');
 
 {UNSIGNED testid, STRING3 testname, BOOLEAN isEqual} decimalJoin (decimalDatasetOut a, decimalDatasetIn b) := TRANSFORM
     SELF.testid := a.testid;
@@ -180,9 +171,7 @@ stringDatasetOut := DATASET([
     {506, 'data1', (STRING)X'00FF00FF00FF00FF'}
 ], stringRecord);
 
-ParquetIO.Write(stringDatasetOut, dropzoneDirectory + '/StringTest.parquet', TRUE, compressionType);
-
-stringDatasetIn := ParquetIO.Read(stringRecord, dropzoneDirectory + '/StringTest.parquet');
+stringDatasetIn := ParquetIO.Read(stringRecord, dropzoneDirectory + 'StringTest.parquet');
 
 {STRING5 name, BOOLEAN value} stringJoin (stringDatasetOut a, stringDatasetIn b) := TRANSFORM
     SELF.name := a.name;
@@ -214,9 +203,7 @@ dataDatasetOut := DATASET([
     {604, 'neg', X'0000000000000000', REALToBinary(-2.71828), REALToLargeBinary(-2.71828)}
 ], dataRecord);
 
-ParquetIO.Write(dataDatasetOut, dropzoneDirectory + '/DataTest.parquet', TRUE, compressionType);
-
-dataDatasetIn := ParquetIO.Read(dataRecord, dropzoneDirectory + '/DataTest.parquet');
+dataDatasetIn := ParquetIO.Read(dataRecord, dropzoneDirectory + 'DataTest.parquet');
 
 {UNSIGNED testid, STRING5 name, BOOLEAN allEqual} dataJoin(dataDatasetOut a, dataDatasetIn b) := TRANSFORM
     SELF.testid := a.testid;
@@ -243,9 +230,7 @@ varStringDatasetOut := DATASET([
     {706, 'data1', (STRING)X'00FF00FF00FF00FF'}
 ], varstringRecord);
 
-ParquetIO.Write(varStringDatasetOut, dropzoneDirectory + '/VarStringTest.parquet', TRUE, compressionType);
-
-varStringDatasetIn := ParquetIO.Read(varstringRecord, dropzoneDirectory + '/VarStringTest.parquet');
+varStringDatasetIn := ParquetIO.Read(varstringRecord, dropzoneDirectory + 'VarStringTest.parquet');
 
 {UNSIGNED testid, STRING testname, BOOLEAN isEqual} varstringJoin (varStringDatasetOut a, varStringDatasetIn b) := TRANSFORM
     SELF.testid := a.testid;
@@ -270,9 +255,7 @@ qStringDatasetOut := DATASET([
     {805, 'data1', (STRING)X'00FF00FF00FF00FF'}
 ], qstringRecord);
 
-ParquetIO.Write(qStringDatasetOut, dropzoneDirectory + '/QStringTest.parquet', TRUE, compressionType);
-
-qStringDatasetIn := ParquetIO.Read(qstringRecord, dropzoneDirectory + '/QStringTest.parquet');
+qStringDatasetIn := ParquetIO.Read(qstringRecord, dropzoneDirectory + 'QStringTest.parquet');
 
 {UNSIGNED testid, STRING testname, BOOLEAN isEqual} qstringJoin (qStringDatasetOut a, qStringDatasetIn b) := TRANSFORM
     SELF.testid := a.testid;
@@ -294,9 +277,7 @@ utf8DatasetOut := DATASET([
     {902, 'extrachars1', U8'\316\221\316\222\316\223\316\224\316\225\316\226\316\227\316\230\316\231\316\232\316\233\316\234'}
 ], utf8Record);
 
-ParquetIO.Write(utf8DatasetOut, dropzoneDirectory + '/UTF8Test.parquet', TRUE, compressionType);
-
-utf8DatasetIn := ParquetIO.Read(utf8Record, dropzoneDirectory + '/UTF8Test.parquet');
+utf8DatasetIn := ParquetIO.Read(utf8Record, dropzoneDirectory + 'UTF8Test.parquet');
 
 {UNSIGNED testid, STRING testname, BOOLEAN isEqual} utf8Join (utf8DatasetOut a, utf8DatasetIn b) := TRANSFORM
     SELF.testid := a.testid;
@@ -319,9 +300,7 @@ unicodeDatasetOut := DATASET([
     {1003, 'adw', U'ᄠᄡᄢᄣᄤᄥᄦᄨᄩᄪᄫᄬᄭᄮᄯᆰᆱᆲᆳᆴᆵᆶᆷᆸᆹᆼᆽᇲᇳᇴᇵᇶᇷᇸ㈸㋄㋅㋆㋇㋈㋉㋊㋋㋌'}
 ], unicodeRecord);
 
-ParquetIO.Write(unicodeDatasetOut, dropzoneDirectory + '/UnicodeTest.parquet', TRUE, compressionType);
-
-unicodeDatasetIn := ParquetIO.Read(unicodeRecord, dropzoneDirectory + '/UnicodeTest.parquet');
+unicodeDatasetIn := ParquetIO.Read(unicodeRecord, dropzoneDirectory + 'UnicodeTest.parquet');
 
 {UNSIGNED testid, STRING testname, BOOLEAN isEqual} unicodeJoin(unicodeDatasetOut a, unicodeDatasetIn b) := TRANSFORM
     SELF.testid := a.testid;
@@ -343,22 +322,52 @@ setDatasetOut := DATASET([
     {1102, 'multiple', [TRUE, FALSE], [1, 2], [1, 2], [1.0, 2.0], [1.0, 2.0], ['a', 'b'], ['a', 'b'], ['a', 'b'], [U'a', U'b'], [U'a', U'b'], [X'0000', X'FFFF']}
 ], setRecord);
 
-ParquetIO.Write(setDatasetOut, dropzoneDirectory + '/SetTest.parquet', TRUE, compressionType);
-
-OUTPUT(ParquetIO.Read(setRecord, dropzoneDirectory + '/SetTest.parquet'), NAMED('SetTest'));
+setResult := ParquetIO.Read(setRecord, dropzoneDirectory + 'SetTest.parquet');
 
 // ======================== OUTPUT ========================
 
-PARALLEL(
-    OUTPUT(booleanResult, NAMED('BooleanTest'), OVERWRITE),
-    OUTPUT(integerResult, NAMED('IntegerTest'), OVERWRITE),
-    OUTPUT(unsignedResult, NAMED('UnsignedTest'), OVERWRITE),
-    OUTPUT(realResult, NAMED('RealTest'), OVERWRITE),
-    OUTPUT(decimalResult, NAMED('DecimalTest'), OVERWRITE),
-    OUTPUT(stringResult, NAMED('StringTest'), OVERWRITE),
-    OUTPUT(dataResult, NAMED('DataTest'), OVERWRITE),
-    OUTPUT(varStringResult, NAMED('VarStringTest'), OVERWRITE),
-    OUTPUT(qStringResult, NAMED('QStringTest'), OVERWRITE),
-    OUTPUT(utf8Result, NAMED('UTF8Test'), OVERWRITE),
-    OUTPUT(unicodeResult, NAMED('UnicodeTest'), OVERWRITE)
+SEQUENTIAL(
+    // Set up test files
+    PARALLEL(
+        ParquetIO.Write(booleanDatasetOut, dropzoneDirectory + 'BooleanTest.parquet', TRUE, compressionType),
+        ParquetIO.Write(integerDatasetOut, dropzoneDirectory + 'IntegerTest.parquet', TRUE, compressionType),
+        ParquetIO.Write(unsignedDatasetOut, dropzoneDirectory + 'UnsignedTest.parquet', TRUE, compressionType),
+        ParquetIO.Write(realDatasetOut, dropzoneDirectory + 'RealTest.parquet', TRUE, compressionType),
+        ParquetIO.Write(decimalDatasetOut, dropzoneDirectory + 'DecimalTest.parquet', TRUE, compressionType),
+        ParquetIO.Write(stringDatasetOut, dropzoneDirectory + 'StringTest.parquet', TRUE, compressionType),
+        ParquetIO.Write(dataDatasetOut, dropzoneDirectory + 'DataTest.parquet', TRUE, compressionType),
+        ParquetIO.Write(varStringDatasetOut, dropzoneDirectory + 'VarStringTest.parquet', TRUE, compressionType),
+        ParquetIO.Write(qStringDatasetOut, dropzoneDirectory + 'QStringTest.parquet', TRUE, compressionType),
+        ParquetIO.Write(utf8DatasetOut, dropzoneDirectory + 'UTF8Test.parquet', TRUE, compressionType),
+        ParquetIO.Write(unicodeDatasetOut, dropzoneDirectory + 'UnicodeTest.parquet', TRUE, compressionType),
+        ParquetIO.Write(setDatasetOut, dropzoneDirectory + 'SetTest.parquet', TRUE, compressionType)
+    ),
+    // Read and compare results
+    OUTPUT(booleanResult, NAMED('BooleanTest')),
+    OUTPUT(integerResult, NAMED('IntegerTest')),
+    OUTPUT(unsignedResult, NAMED('UnsignedTest')),
+    OUTPUT(realResult, NAMED('RealTest')),
+    OUTPUT(decimalResult, NAMED('DecimalTest')),
+    OUTPUT(stringResult, NAMED('StringTest')),
+    OUTPUT(dataResult, NAMED('DataTest')),
+    OUTPUT(varStringResult, NAMED('VarStringTest')),
+    OUTPUT(qStringResult, NAMED('QStringTest')),
+    OUTPUT(utf8Result, NAMED('UTF8Test')),
+    OUTPUT(unicodeResult, NAMED('UnicodeTest')),
+    OUTPUT(setResult, NAMED('SetTest')),
+    // Clean up temporary files
+    PARALLEL(
+        FileServices.DeleteExternalFile('.', dropzoneDirectory + 'BooleanTest.parquet'),
+        FileServices.DeleteExternalFile('.', dropzoneDirectory + 'IntegerTest.parquet'),
+        FileServices.DeleteExternalFile('.', dropzoneDirectory + 'UnsignedTest.parquet'),
+        FileServices.DeleteExternalFile('.', dropzoneDirectory + 'RealTest.parquet'),
+        FileServices.DeleteExternalFile('.', dropzoneDirectory + 'DecimalTest.parquet'),
+        FileServices.DeleteExternalFile('.', dropzoneDirectory + 'StringTest.parquet'),
+        FileServices.DeleteExternalFile('.', dropzoneDirectory + 'DataTest.parquet'),
+        FileServices.DeleteExternalFile('.', dropzoneDirectory + 'VarStringTest.parquet'),
+        FileServices.DeleteExternalFile('.', dropzoneDirectory + 'QStringTest.parquet'),
+        FileServices.DeleteExternalFile('.', dropzoneDirectory + 'UTF8Test.parquet'),
+        FileServices.DeleteExternalFile('.', dropzoneDirectory + 'UnicodeTest.parquet'),
+        FileServices.DeleteExternalFile('.', dropzoneDirectory + 'SetTest.parquet')
+    )
 );