Skip to content

Fix/remove bom #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
davidesner wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Fix/remove bom #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
705bbd3
FIX remove BOM
davidesner Sep 8, 2022
3a1d5ae
Github actions
davidesner Sep 8, 2022
f41ea7a
debug log
davidesner Sep 8, 2022
d1d8bea
check for col existence
davidesner Sep 8, 2022
05d5a24
DEBUG
davidesner Sep 8, 2022
8652372
DEBUG
davidesner Sep 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions .github/workflows/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
name: Deploy
on:
push:
tags:
- '[0-9]*.[0-9]*.[0-9]*'
jobs:
deploy:
runs-on: ubuntu-latest
env:
APP_IMAGE: keboola-component
KBC_DEVELOPERPORTAL_USERNAME: ${{ secrets.KBC_DEVELOPERPORTAL_USERNAME }}
KBC_DEVELOPERPORTAL_PASSWORD: ${{ secrets.KBC_DEVELOPERPORTAL_PASSWORD }}
KBC_DEVELOPERPORTAL_VENDOR: ${{ secrets.KBC_DEVELOPERPORTAL_VENDOR }}
KBC_DEVELOPERPORTAL_APP: ${{ secrets.KBC_DEVELOPERPORTAL_APP }}
steps:
- uses: actions/checkout@v2
- name: Build image
run: docker build -t ${APP_IMAGE} .
- name: Set tag env
run: echo "GITHUB_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
- name: Deploy
run: |
chmod +x ./deploy.sh
- name: Run scripts
run: |
./deploy.sh
23 changes: 23 additions & 0 deletions .github/workflows/pushtest.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
name: PushToTest
on: push
jobs:
pushtotest:
runs-on: ubuntu-latest
env:
APP_IMAGE: keboola-component
KBC_DEVELOPERPORTAL_USERNAME: ${{ secrets.KBC_DEVELOPERPORTAL_USERNAME }}
KBC_DEVELOPERPORTAL_PASSWORD: ${{ secrets.KBC_DEVELOPERPORTAL_PASSWORD }}
KBC_DEVELOPERPORTAL_VENDOR: ${{ secrets.KBC_DEVELOPERPORTAL_VENDOR }}
KBC_DEVELOPERPORTAL_APP: ${{ secrets.KBC_DEVELOPERPORTAL_APP }}
steps:
- uses: actions/checkout@v2
- name: Build image
run: docker build -t ${APP_IMAGE} .
- name: Push to ECR
run: |
docker pull quay.io/keboola/developer-portal-cli-v2:latest
export REPOSITORY=`docker run --rm -e KBC_DEVELOPERPORTAL_USERNAME -e KBC_DEVELOPERPORTAL_PASSWORD -e KBC_DEVELOPERPORTAL_URL quay.io/keboola/developer-portal-cli-v2:latest ecr:get-repository $KBC_DEVELOPERPORTAL_VENDOR $KBC_DEVELOPERPORTAL_APP`
docker tag $APP_IMAGE:latest $REPOSITORY:test
eval $(docker run --rm -e KBC_DEVELOPERPORTAL_USERNAME -e KBC_DEVELOPERPORTAL_PASSWORD -e KBC_DEVELOPERPORTAL_URL quay.io/keboola/developer-portal-cli-v2:latest ecr:get-login $KBC_DEVELOPERPORTAL_VENDOR $KBC_DEVELOPERPORTAL_APP)
docker push $REPOSITORY:test
docker pull quay.io/keboola/syrup-cli:latest
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
/target/
.idea
24 changes: 12 additions & 12 deletions src/main/java/keboola/adform/masterdata_extractor/Extractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,18 +45,18 @@ public List<MasterFile> downloadAndUnzip(List<MasterFile> fileList, String folde
return Collections.emptyList();
}
//download files
downloadFiles(fileList, folderPath);

List<String> rawFilePaths;
//unzip archives and delete data
List<MasterFile> exFiles = unzip(fileList, folderPath);
//delete zipFiles
try {
FileHandler.deleteFile(folderPath + File.separator + fileList.get(0).getPrefix());
} catch (IOException ex) {
logger.error(ex.getMessage(), ex);
}
return exFiles;
downloadFiles(fileList, "/data/out/files");

// List<String> rawFilePaths;
// //unzip archives and delete data
// List<MasterFile> exFiles = unzip(fileList, folderPath);
// //delete zipFiles
// try {
// FileHandler.deleteFile(folderPath + File.separator + fileList.get(0).getPrefix());
// } catch (IOException ex) {
// logger.error(ex.getMessage(), ex);
// }
return null;

}

Expand Down
171 changes: 87 additions & 84 deletions src/main/java/keboola/adform/masterdata_extractor/Runner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,7 @@
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;
import java.util.*;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand Down Expand Up @@ -73,7 +68,7 @@ public static void main(String[] args) {
System.exit(1);
}
boolean dataExtracted = false;
Extractor ex = new Extractor(config.getParams().getUser(), config.getParams().getPass(), config.getParams().getMdListId(), log);
Extractor ex = new Extractor(config.getParams().getUser(), config.getParams().getPass(), config.getParams().getMdListId(), log);
try {
//authenticate, get session token
ex.client.authenticate();
Expand Down Expand Up @@ -128,69 +123,69 @@ public static void main(String[] args) {
String resFileFolder = outTablesPath + File.separator + prefix.toLowerCase() + ".csv";
List<MasterFile> downloadedFiles = ex.downloadAndUnzip(filesSince, resFileFolder);

/*This should not happen, check anyway*/
if (downloadedFiles.isEmpty()) {
System.out.print("Error downloading files with prefix: " + prefix);
System.err.print("Error downloading files with prefix: " + prefix);
System.exit(1);
}

//merge downloaded files
String resFileName = prefix.toLowerCase();
System.out.println("Preparing sliced tables...");
String[] headerCols = null;
try {
headerCols = prepareSlicedTables(downloadedFiles, config.getParams().getSrcCharset());
} catch (Exception e) {
System.err.println("Error processing files." + e.getMessage());
System.exit(2);
}
// /*This should not happen, check anyway*/
// if (downloadedFiles.isEmpty()) {
// System.out.print("Error downloading files with prefix: " + prefix);
// System.err.print("Error downloading files with prefix: " + prefix);
// System.exit(1);
// }
//
// //merge downloaded files
// String resFileName = prefix.toLowerCase();
// System.out.println("Preparing sliced tables...");
// String[] headerCols = null;
// try {
// headerCols = prepareSlicedTables(downloadedFiles, config.getParams().getSrcCharset());
// } catch (Exception e) {
// System.err.println("Error processing files." + e.getMessage());
// System.exit(2);
// }
//
// /*Build manifest file*/
// try {
// String[] pkey = null;
// if (!config.getParams().getKeyMap().containsKey(prefix)) {
// pkey = new String[] {MD_PRIMARY_KEY};
// } else {
// pkey = config.getParams().getKeyMap().get(prefix);
// }
// buildManifestFile(resFileName, config.getParams().getBucket(), outTablesPath, headerCols, pkey, true);
// } catch (Exception ex1) {
// System.out.println("Error writing manifest file." + ex1.getMessage());
// System.err.println(ex1.getMessage());
// System.exit(2);
// }
// i++;
// dataExtracted = true;
// }
//
//
// if (config.hasMeta()) {
// System.out.println("Downloading meta files");
// List<MasterFile> filesSince = null;
// if (config.getParams().isAlwaysGetMeta()) {
// filesSince = fileList.getFilesByPrefix("meta");
// } else if (config.getParams().getDate_to() != null) {
// filesSince = fileList.getFilesSince(startInterval, config.getParams().getDate_to(), "meta");
// } else {
// filesSince = fileList.getFilesSince(startInterval, "meta");
// }
//
//
// List<MasterFile> metaFiles = ex.downloadAndUnzip(filesSince, dataPath);
//
// dataExtracted = processMetaDataFiles(metaFiles, config, dataPath, outTablesPath);

/*Build manifest file*/
try {
String[] pkey = null;
if (!config.getParams().getKeyMap().containsKey(prefix)) {
pkey = new String[] {MD_PRIMARY_KEY};
} else {
pkey = config.getParams().getKeyMap().get(prefix);
}
buildManifestFile(resFileName, config.getParams().getBucket(), outTablesPath, headerCols, pkey, true);
} catch (Exception ex1) {
System.out.println("Error writing manifest file." + ex1.getMessage());
System.err.println(ex1.getMessage());
System.exit(2);
}
i++;
dataExtracted = true;
}


if (config.hasMeta()) {
System.out.println("Downloading meta files");
List<MasterFile> filesSince = null;
if (config.getParams().isAlwaysGetMeta()) {
filesSince = fileList.getFilesByPrefix("meta");
} else if (config.getParams().getDate_to() != null) {
filesSince = fileList.getFilesSince(startInterval, config.getParams().getDate_to(), "meta");
} else {
filesSince = fileList.getFilesSince(startInterval, "meta");
}


List<MasterFile> metaFiles = ex.downloadAndUnzip(filesSince, dataPath);

dataExtracted = processMetaDataFiles(metaFiles, config, dataPath, outTablesPath);

}

if (dataExtracted && i > 0) {
System.out.println("Files extracted successfully..");
} else if (!dataExtracted) {
System.out.println("Proccess finished successfully but no meta files were extracted. Check configuration parameters.");
} else {
System.out.println("Proccess finished successfully but only metadata tables were extracted. Check configuration parameters.");
}
System.exit(0);
// if (dataExtracted && i > 0) {
// System.out.println("Files extracted successfully..");
// } else if (!dataExtracted) {
// System.out.println("Proccess finished successfully but no meta files were extracted. Check configuration parameters.");
// } else {
// System.out.println("Proccess finished successfully but only metadata tables were extracted. Check configuration parameters.");
// }
// System.exit(0);
} catch (ExtractorException ex1) {
System.out.print("Error extracting data.");
System.err.print(ex1.getMessage());
Expand All @@ -202,7 +197,7 @@ private static boolean processMetaDataFiles(List<MasterFile> metaFiles, KBCConfi
if (metaFiles.isEmpty()) {
log.warn("No new metadata were retrieved!");
return false;
}
}
/*Convert from JSON to csv*/
JsonToCsvConvertor conv = new JsonToCsvConvertor();
for (String metaF : config.getParams().getMetaFiles()) {
Expand Down Expand Up @@ -242,48 +237,56 @@ private static boolean processMetaDataFiles(List<MasterFile> metaFiles, KBCConfi
}
return true;
}
private static void buildManifestFile(String resFileName, String destination, String outPath, String [] cols, String [] pkey, boolean incremental) throws Exception {
ManifestFile.Builder builder = new ManifestFile.Builder(resFileName, destination + "." + resFileName)
.setIncrementalLoad(incremental).setDelimiter(String.valueOf(DEFAULT_SEPARATOR)).setEnclosure(String.valueOf(DEFAULT_ENCLOSURE))
.setColumns(cols);
if (pkey != null) {
builder.setPrimaryKey(pkey);
}
ManifestFile manFile = builder.build();
ManifestBuilder.buildManifestFile(manFile, outPath, resFileName + ".csv");

private static void buildManifestFile(String resFileName, String destination, String outPath, String[] cols, String[] pkey, boolean incremental) throws Exception {
// remove BOM because some files contain it and KBC is incapable of handling it
if (cols != null) {
System.out.println(cols[0]);
cols[0] = CsvUtils.removeUTF8BOM(cols[0]);
}

System.out.println(Arrays.toString(cols));
ManifestFile.Builder builder = new ManifestFile.Builder(resFileName, destination + "." + resFileName)
.setIncrementalLoad(incremental).setDelimiter(String.valueOf(DEFAULT_SEPARATOR)).setEnclosure(String.valueOf(DEFAULT_ENCLOSURE))
.setColumns(cols);
if (pkey != null) {
builder.setPrimaryKey(pkey);
}
ManifestFile manFile = builder.build();
ManifestBuilder.buildManifestFile(manFile, outPath, resFileName + ".csv");
}

private static String[] prepareSlicedTables(List<MasterFile> downloadedFiles, String charset) throws Exception {
List<File> files = new ArrayList<>();
for(MasterFile mf : downloadedFiles) {
files.add(new File(mf.getLocalAbsolutePath()));
}

// get colums
String[] headerCols = CsvUtils.readHeader(files.get(0),
DEFAULT_SEPARATOR, DEFAULT_ENCLOSURE, DEFAULT_ESCAPE_CHAR, false, false, Charset.forName(charset));
// remove headers and create results
for (File mFile : files) {
CsvUtils.removeHeaderFromCsv(mFile, Charset.forName(charset));
CsvUtils.removeHeaderFromCsv(mFile, Charset.forName(charset));
}
//in case some files did not contain any data
CsvUtils.deleteEmptyFiles(files);
CsvUtils.deleteEmptyFiles(files);
return headerCols;

}

private static void printEnvStats() {
// Get current size of heap in bytes
long heapSize = Runtime.getRuntime().totalMemory();
long heapSize = Runtime.getRuntime().totalMemory();

// Get maximum size of heap in bytes. The heap cannot grow beyond this size.// Any attempt will result in an OutOfMemoryException.
long heapMaxSize = Runtime.getRuntime().maxMemory();

// Get amount of free memory within the heap in bytes. This size will increase // after garbage collection and decrease as new objects are created.
long heapFreeSize = Runtime.getRuntime().freeMemory();

log.info("Initial Heap size (MB): " + heapSize/1000000);
log.info("Max Heap size (MB): " + heapMaxSize/1000000);
log.info("Initial free memory (MB): " + heapFreeSize/1000000);
log.info("Initial free memory (MB): " + heapFreeSize/1000000);
}
}
Loading