Skip to content

Commit 6c33ec3

Browse files
MarinaFedyMarinaFedy
authored
Merge pull request #951 from singnet/data-preset-tab
Data preset tab, fix and updates
2 parents 915ff6f + 52ed2ce commit 6c33ec3

File tree

9 files changed

+107
-123
lines changed

9 files changed

+107
-123
lines changed

‎src/Redux/actionCreators/ServiceTrainingActions.js

Lines changed: 27 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { startAppLoader, stopAppLoader } from "./LoaderActions";
44
import { getServiceClient } from "./SDKActions";
55
import { fetchAuthenticatedUser, updateMetamaskWallet } from "./UserActions";
66
import { modelStatus } from "../reducers/ServiceTrainingReducer";
7+
import { DatafactoryInstanceS3, DatasetS3Endpoints, TrainingInstanceS3 } from "../../config/DatasetS3Client";
78
// import { userActions } from ".";
89
export const SET_MODEL_DETAILS = "SET_MODEL_DETAILS";
910
export const SET_MODELS_LIST = "SET_MODELS_LIST";
@@ -191,49 +192,46 @@ const modelStatusByNumber = {
191192
};
192193

193194
export const publishDatasetForTraining = (fileBlob, name) => async (dispatch) => {
194-
const linkAndKeyDataset = await dispatch(
195-
publishDatasetToS3(
196-
fileBlob,
197-
name,
198-
"https://xim5yugo7g.execute-api.us-east-1.amazonaws.com/default",
199-
"S1kDjcub9k78JFAyrLPsfS0yQoQ4mgmmpeWKlIoVvYsk6JVq5v4HHKvKQgZ0VdI7"
200-
)
201-
);
202-
return linkAndKeyDataset;
195+
try {
196+
const linkAndKeyDataset = await dispatch(publishDatasetToS3(fileBlob, name, TrainingInstanceS3));
197+
return linkAndKeyDataset;
198+
} catch (error) {
199+
console.log("publishing Dataset For Training error: ", error);
200+
}
203201
};
204202

205203
export const publishDatasetForImproving = (fileBlob, name) => async (dispatch) => {
206-
const linkAndKeyDataset = await dispatch(
207-
publishDatasetToS3(
208-
fileBlob,
209-
name,
210-
"https://ozx0e68owf.execute-api.us-east-1.amazonaws.com",
211-
"IYE2sz0hUSGhWcyLQTwXS0AbiXKq4h1eW85MZSo6uDhtYfXI8dXisTzRyXaBCImH"
212-
)
213-
);
214-
return linkAndKeyDataset;
204+
try {
205+
const linkAndKeyDataset = await dispatch(publishDatasetToS3(fileBlob, name, DatafactoryInstanceS3));
206+
return linkAndKeyDataset;
207+
} catch (error) {
208+
console.log("publishing Dataset For Improving error: ", error);
209+
}
215210
};
216211

217-
export const publishDatasetToS3 = (fileBlob, name, baseUrl, authToken) => async (dispatch) => {
212+
export const publishDatasetToS3 = (fileBlob, name, S3Instance) => async (dispatch) => {
218213
const { email } = await dispatch(fetchAuthenticatedUser());
219214

220215
try {
216+
const baseUrl = S3Instance.getUri();
221217
const fileKey = name + "_" + email + "_" + Date.now();
222-
const url = `${baseUrl}/upload?key=${fileKey}`;
223-
224-
let instance = axios.create({
225-
headers: {
226-
Authorization: authToken,
227-
},
228-
});
229-
230-
const response = await instance.get(url);
218+
const response = await S3Instance.get(DatasetS3Endpoints.UPLOAD, { params: { key: fileKey } });
231219
await axios.put(response.data.uploadURL, fileBlob);
232220
return {
233221
url: `${baseUrl}/download?key=${fileKey}`,
234-
datasetKey: fileKey
222+
datasetKey: fileKey,
235223
};
236224
} catch (err) {
237225
throw new Error(err);
238226
}
239227
};
228+
229+
export const getDatasetSizeFromS3 = async (fileKey, S3instance) => {
230+
return S3instance.get(DatasetS3Endpoints.DOWNLOAD, { params: { key: fileKey, action: "getsize" } })
231+
.then((response) => {
232+
return response.data.fileSize;
233+
})
234+
.catch((error) => {
235+
console.error("mergeDatasets Error:", error);
236+
});
237+
};

‎src/Redux/reducers/DatasetReducer.js

Lines changed: 1 addition & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -22,43 +22,8 @@ const initialState = {
2222
size: 51234,
2323
tag: "Text",
2424
},
25-
{
26-
datasetKey: "data_instruct_llm_1000.zip_matrejukna@gufum.com_1733211676614",
27-
name: "DataSet 4: Training data for text translation",
28-
size: 51234,
29-
tag: "Text",
30-
},
31-
],
32-
recentDatasets: [
33-
{
34-
datasetKey: "data_instruct_llm_1000.zip_training@singularitynet.io_1732864358004",
35-
// link: "https://ozx0e68owf.execute-api.us-east-1.amazonaws.com/download?key=data_instruct_llm_1000.zip_training@singularitynet.io_1732864358004",
36-
name: "DataSet Recent 1: Training data for text translation",
37-
size: 51234,
38-
tag: "Text",
39-
},
40-
{
41-
datasetKey: "data_instruct_llm_1000.zip_training@singularitynet.io_1732864358004",
42-
// link: "https://ozx0e68owf.execute-api.us-east-1.amazonaws.com/download?key=data_instruct_llm_1000.zip_training@singularitynet.io_1732864358004",
43-
name: "DataSet Recent 2: Training data for text translation",
44-
size: 51234,
45-
tag: "Text",
46-
},
47-
{
48-
datasetKey: "data_instruct_llm_1000.zip_training@singularitynet.io_1732864358004",
49-
// link: "https://ozx0e68owf.execute-api.us-east-1.amazonaws.com/download?key=data_instruct_llm_1000.zip_training@singularitynet.io_1732864358004",
50-
name: "DataSet Recent 3: Training data for text translation",
51-
size: 51234,
52-
tag: "Text",
53-
},
54-
{
55-
datasetKey: "data_instruct_llm_1000.zip_training@singularitynet.io_1732864358004",
56-
// link: "https://ozx0e68owf.execute-api.us-east-1.amazonaws.com/download?key=data_instruct_llm_1000.zip_training@singularitynet.io_1732864358004",
57-
name: "DataSet Recent 4: Training data for text translation",
58-
size: 51234,
59-
tag: "Text",
60-
},
6125
],
26+
recentDatasets: [],
6227
};
6328

6429
const datasetReducer = (state = initialState, action) => {

‎src/components/ServiceDetails/DataPreset/DashboardModal/ButtonGroup.js

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,13 @@ import { useDispatch } from "react-redux";
66
import {
77
addRecentDataset,
88
improveDataset,
9-
setMainDataset, setMergeDataset,
9+
setMainDataset,
10+
setMergeDataset,
1011
} from "../../../../Redux/actionCreators/DatasetActions";
1112
import { startAppLoader, stopAppLoader } from "../../../../Redux/actionCreators/LoaderActions";
1213
import { LoaderContent } from "../../../../utility/constants/LoaderContent";
14+
import { DatafactoryInstanceS3 } from "../../../../config/DatasetS3Client";
15+
import { getDatasetSizeFromS3 } from "../../../../Redux/actionCreators/ServiceTrainingActions";
1316

1417
const ButtonsGroup = ({ classes, selectedParameters, isTableView, toggleTableView, dataset, index }) => {
1518
const dispatch = useDispatch();
@@ -19,17 +22,18 @@ const ButtonsGroup = ({ classes, selectedParameters, isTableView, toggleTableVie
1922
const isImproveButtonDisable = !selectedParameters?.size;
2023

2124
const getImprovedDataset = async () => {
22-
try{
25+
try {
2326
dispatch(startAppLoader(LoaderContent.IMPROVE_DATASET));
2427
const { data } = await dispatch(improveDataset(dataset.datasetKey, Array.from(selectedParameters.keys())));
28+
const size = await getDatasetSizeFromS3(data.dataset_key_new, DatafactoryInstanceS3);
2529
const improvedDataset = {
2630
additionalInfo: {
2731
analysis: data.analysis,
28-
datasaet_sample: data.dataset_sample
32+
datasaet_sample: data.dataset_sample,
2933
},
3034
datasetKey: data.dataset_key_new,
3135
name: dataset.name + "_improved_rate_" + data.analysis.overall_score,
32-
size: dataset.size,
36+
size,
3337
tag: dataset.tag,
3438
};
3539
await dispatch(addRecentDataset(improvedDataset));

‎src/components/ServiceDetails/DataPreset/DashboardModal/index.js

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -29,20 +29,9 @@ const DashboardModal = ({ classes, onClose, isShow, dataset, index }) => {
2929
toggleTableView={toggleTableView}
3030
index={index}
3131
/>
32-
{
33-
isTableView &&
34-
(dataset?.additionalInfo?.dataset_sample || dataset?.additionalInfo?.datasaet_sample) &&
35-
(
36-
<TableSamples
37-
tableData={
38-
// TODO replace to correct word
39-
dataset?.additionalInfo?.dataset_sample
40-
? dataset?.additionalInfo?.dataset_sample
41-
: dataset?.additionalInfo?.datasaet_sample
42-
}
43-
/>
44-
)
45-
}
32+
{isTableView && (dataset?.additionalInfo?.dataset_sample || dataset?.additionalInfo?.datasaet_sample) && (
33+
<TableSamples tableData={dataset?.additionalInfo?.dataset_sample} />
34+
)}
4635
<Graphs graphs={dataset?.additionalInfo?.analysis?.graphs} />
4736
</div>
4837
</SNETDialog>

‎src/components/ServiceDetails/DataPreset/DatasetTabs/index.js

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,18 +12,25 @@ const DatasetTabs = ({ setDatasetInfo }) => {
1212
setActiveTab(activeTab);
1313
};
1414

15-
const tabs = [
16-
{
17-
name: "Recent",
18-
activeIndex: 0,
19-
component: <DatasetsList setDatasetInfo={setDatasetInfo} datasets={recentDatasets} />,
20-
},
21-
{
22-
name: "Examples",
23-
activeIndex: 1,
24-
component: <DatasetsList setDatasetInfo={setDatasetInfo} datasets={exampleDatasets} />,
25-
},
26-
];
15+
const exampleTab = {
16+
name: "Examples",
17+
activeIndex: 0,
18+
component: <DatasetsList setDatasetInfo={setDatasetInfo} datasets={exampleDatasets} />,
19+
};
20+
const recentTab = {
21+
name: "Recent",
22+
activeIndex: 1,
23+
component: <DatasetsList setDatasetInfo={setDatasetInfo} datasets={recentDatasets} />,
24+
};
25+
26+
const tabs = [];
27+
if (recentDatasets.length === 0) {
28+
tabs.push({ ...exampleTab, activeIndex: 0 });
29+
tabs.push({ ...recentTab, activeIndex: 1 });
30+
} else {
31+
tabs.push({ ...recentTab, activeIndex: 0 });
32+
tabs.push({ ...exampleTab, activeIndex: 1 });
33+
}
2734

2835
return (
2936
<div>

‎src/components/ServiceDetails/DataPreset/DatasetUploader/index.js

Lines changed: 4 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,20 +15,6 @@ import { LoaderContent } from "../../../../utility/constants/LoaderContent";
1515
import { loaderActions } from "../../../../Redux/actionCreators";
1616

1717
const acceptedFileTypes = { "application/zip": [".zip"], "application/x-zip-compressed": [".zip"] };
18-
// const datasetParameters = [
19-
// { title: "Size", value: "11.2Mb" },
20-
// { title: "Format", value: "TXT" },
21-
// {
22-
// title: "Rate",
23-
// value: "5.5/10",
24-
// additionalInfo: [
25-
// { value: "11", title: "Outlier filtering1" },
26-
// { value: "11", title: "Outlier filtering2" },
27-
// { value: "11", title: "Outlier filtering3" },
28-
// { value: "11", title: "Outlier filtering4" },
29-
// ],
30-
// },
31-
// ];
3218

3319
const DatasetUploader = ({ classes, setDatasetInfo, datasetInfo, cleanDatasetInfo, index }) => {
3420
const dispatch = useDispatch();
@@ -50,7 +36,7 @@ const DatasetUploader = ({ classes, setDatasetInfo, datasetInfo, cleanDatasetInf
5036
setTrainingDataFileName(name);
5137
setTrainingDataFileSize(size);
5238
const { url, datasetKey } = await dispatch(publishDatasetForImproving(fileBlob, name));
53-
dispatch(setDatasetInfo({ link: url, name, size, datasetKey }));
39+
setDatasetInfo({ link: url, name, size, datasetKey });
5440
} catch (error) {
5541
console.log("error: ", error);
5642

@@ -134,7 +120,9 @@ const DatasetUploader = ({ classes, setDatasetInfo, datasetInfo, cleanDatasetInf
134120
/>
135121
</>
136122
)}
137-
{isDashbordOpen && <DashboardModal onClose={closeDashbordModal} isShow={isDashbordOpen} dataset={datasetInfo} index={index} />}
123+
{isDashbordOpen && (
124+
<DashboardModal onClose={closeDashbordModal} isShow={isDashbordOpen} dataset={datasetInfo} index={index} />
125+
)}
138126
</div>
139127
);
140128
};

‎src/components/ServiceDetails/DataPreset/DatasetsList/index.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ import { withStyles } from "@mui/styles";
22
import { useStyles } from "./styles";
33

44
const DatasetsList = ({ classes, setDatasetInfo, datasets }) => {
5-
const DatasetLine = ({dataset}) => {
5+
const DatasetLine = ({ dataset }) => {
66
return (
77
<div onClick={() => setDatasetInfo(dataset)} className={classes.datasetLine}>
88
<div className={classes.datasetName}>{dataset?.name}</div>
9-
<div className={classes.datasetTag}>{dataset?.tag}</div>
9+
<div className={classes.datasetTag}>{dataset?.tag ? dataset?.tag : "Text"}</div>
1010
</div>
1111
);
1212
};

‎src/components/ServiceDetails/DataPreset/index.js

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import StyledButton from "../../common/StyledButton";
1010
import clsx from "clsx";
1111
import { isEmpty } from "lodash";
1212
import { useLocation, useNavigate } from "react-router-dom";
13-
import { setCurrentModelDetails } from "../../../Redux/actionCreators/ServiceTrainingActions";
13+
import { getDatasetSizeFromS3, setCurrentModelDetails } from "../../../Redux/actionCreators/ServiceTrainingActions";
1414
import { useDispatch, useSelector } from "react-redux";
1515
import {
1616
addRecentDataset,
@@ -23,6 +23,7 @@ import {
2323
import { loaderActions } from "../../../Redux/actionCreators";
2424
import { LoaderContent } from "../../../utility/constants/LoaderContent";
2525
import { startAppLoader, stopAppLoader } from "../../../Redux/actionCreators/LoaderActions";
26+
import { DatafactoryInstanceS3 } from "../../../config/DatasetS3Client";
2627

2728
const DataPreset = ({ classes }) => {
2829
const navigate = useNavigate();
@@ -33,19 +34,20 @@ const DataPreset = ({ classes }) => {
3334
const recentDatasets = useSelector((state) => state.datasetReducer.recentDatasets);
3435

3536
useEffect(() => {
36-
!!mainDataset && !mainDataset?.additionalInfo && getStatistic(mainDataset?.datasetKey, setMainDataset);
37+
!!mainDataset && !mainDataset?.additionalInfo && getStatistic(mainDataset, setMainDataset);
3738
}, [mainDataset]);
3839

3940
useEffect(() => {
40-
!!mergeDataset && !mergeDataset?.additionalInfo && getStatistic(mergeDataset?.datasetKey, setMergeDataset);
41+
!!mergeDataset && !mergeDataset?.additionalInfo && getStatistic(mergeDataset, setMergeDataset);
4142
}, [mergeDataset]);
4243

43-
const getStatistic = async (datasetKey, setDataset) => {
44+
const getStatistic = async (dataset, setDataset) => {
4445
try {
4546
dispatch(loaderActions.startAppLoader(LoaderContent.GET_DATASET_STATISTIC));
47+
const datasetKey = dataset?.datasetKey;
4648
const { data } = await dispatch(getDatasetStatistic(datasetKey));
47-
const enrichedDataset = { ...mainDataset, additionalInfo: data };
48-
const actualDatasetIndexInRecent = recentDatasets.find(el => el.datasetKey === datasetKey);
49+
const enrichedDataset = { ...dataset, additionalInfo: data };
50+
const actualDatasetIndexInRecent = recentDatasets.find((el) => el.datasetKey === datasetKey);
4951
if (!actualDatasetIndexInRecent) {
5052
await dispatch(addRecentDataset(enrichedDataset));
5153
} else {
@@ -81,18 +83,19 @@ const DataPreset = ({ classes }) => {
8183
try {
8284
dispatch(startAppLoader(LoaderContent.MERGE_DATASETS));
8385
const mergedDatasets = await dispatch(validateMergeDatasets(mainDataset?.datasetKey, mergeDataset?.datasetKey));
86+
const size = await getDatasetSizeFromS3(mergedDatasets.dataset_key_merged, DatafactoryInstanceS3);
8487
const mergedDataset = {
8588
additionalInfo: mergedDatasets,
8689
datasetKey: mergedDatasets.dataset_key_merged,
8790
name: "merged_" + mainDataset?.name + "_" + mergeDataset?.name,
88-
size: mainDataset?.size + mergeDataset?.size,
89-
tag: mainDataset?.tag,
90-
}
91+
size,
92+
tag: mainDataset?.tag,
93+
};
9194
await dispatch(addRecentDataset(mergedDataset));
9295
await dispatch(setMainDataset(mergedDataset));
9396
await dispatch(setMergeDataset(null));
9497
} catch (error) {
95-
console.log("error onMergeDatasets:", error);
98+
console.log("error on merge datasets:", error);
9699
} finally {
97100
dispatch(stopAppLoader());
98101
}

‎src/config/DatasetS3Client.js

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import axios from "axios";
2+
3+
const DATAFACTORY_S3 = {
4+
baseUrl: "https://ozx0e68owf.execute-api.us-east-1.amazonaws.com",
5+
authToken: "IYE2sz0hUSGhWcyLQTwXS0AbiXKq4h1eW85MZSo6uDhtYfXI8dXisTzRyXaBCImH",
6+
};
7+
8+
const TRAINING_S3 = {
9+
baseUrl: "https://xim5yugo7g.execute-api.us-east-1.amazonaws.com/default",
10+
authToken: "S1kDjcub9k78JFAyrLPsfS0yQoQ4mgmmpeWKlIoVvYsk6JVq5v4HHKvKQgZ0VdI7",
11+
};
12+
13+
export const TrainingInstanceS3 = axios.create({
14+
baseURL: TRAINING_S3.baseUrl,
15+
headers: {
16+
Authorization: TRAINING_S3.authToken,
17+
},
18+
});
19+
20+
export const DatafactoryInstanceS3 = axios.create({
21+
baseURL: DATAFACTORY_S3.baseUrl,
22+
headers: {
23+
Authorization: DATAFACTORY_S3.authToken,
24+
},
25+
});
26+
27+
export const DatasetS3Endpoints = {
28+
UPLOAD: "/upload", // params: key
29+
DOWNLOAD: "/download", // params: key
30+
};

0 commit comments

Comments
 (0)