Skip to content

Commit eb2dda6

Browse files
100mi100mi
authored
Merge pull request #12 from factly/feat/metafacts
Metafacts support and validation
2 parents a7f0a26 + e7badfc commit eb2dda6

File tree

22 files changed

+709
-148
lines changed

22 files changed

+709
-148
lines changed

‎docker-compose.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ services:
306306
- minio
307307
ports:
308308
- 8000:8000
309-
image: factly/validly-server:0.4.1
309+
image: factly/validly-server:0.4.2
310310
command: >
311311
/bin/sh -c "
312312
uvicorn app.main:app --reload --host 0.0.0.0;
@@ -322,11 +322,13 @@ services:
322322
metafacts-server:
323323
ports:
324324
- 8005:8005
325-
image: factly/metafacts-server:latest
325+
image: factly/metafacts-server:0.2.2
326326
command: >
327327
/bin/sh -c "
328328
uvicorn app.main:app --reload --host 0.0.0.0 --port 8005;
329329
"
330+
env_file:
331+
- metafacts.env
330332
networks:
331333
- validly
332334

‎metafacts.env.examples

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Dataset source configurations
2+
S3_SOURCE_ACCESS_KEY= "minio"
3+
S3_SOURCE_SECRET_KEY= "password"
4+
S3_SOURCE_ENDPOINT_URL= "http://minio:9000"
5+
S3_SOURCE_RESOURCE= "S3"

‎studio/src/App.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import React from 'react';
2-
import { BrowserRouter as Router, Switch, Link, Route } from 'react-router-dom';
2+
import { BrowserRouter as Router, Switch, Route } from 'react-router-dom';
33
import routes from './config/routes';
44
import 'antd/dist/antd.css';
55
import BasicLayout from './layout/basic';

‎studio/src/actions/metafacts.js

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import {
2+
ADD_META_FILES,
3+
ADD_META_TABLE_DATA,
4+
SET_META_UPLOAD_BUTTON,
5+
} from '../constants/metafacts';
6+
import { SET_META_LOADING } from '../constants/metafacts';
7+
import { parseJSON } from '../utils/fetch';
8+
import { getFormData } from '../utils/form';
9+
10+
export const addFiles = (data) => ({
11+
type: ADD_META_FILES,
12+
payload: data,
13+
});
14+
15+
export const addMetaTableData = (data) => ({
16+
type: ADD_META_TABLE_DATA,
17+
payload: data,
18+
});
19+
20+
export const getMetaTableData = (files) => {
21+
return (dispatch) => {
22+
dispatch(setLoading(true));
23+
let data = new FormData();
24+
files.map((file) => data.append('csv_files', file));
25+
fetch(window.REACT_APP_METAFACTS_SERVER_URL + '/meta-data/files', {
26+
method: 'POST',
27+
body: data,
28+
})
29+
.then((response) => response.json())
30+
.then((metaTabledata) => {
31+
dispatch(addMetaTableData(metaTabledata));
32+
dispatch(setUploadButton(false));
33+
})
34+
.finally(() => dispatch(setLoading(false)));
35+
};
36+
};
37+
export const getMetaTableDataFromS3 = (data) => {
38+
return (dispatch) => {
39+
dispatch(setLoading(true));
40+
let metafactsdata = getFormData({
41+
...data,
42+
resource: 's3',
43+
s3_access_key: '',
44+
s3_secret_key: '',
45+
s3_endpoint_url: '',
46+
file_format: 'csv',
47+
});
48+
return fetch(window.REACT_APP_METAFACTS_SERVER_URL + '/meta-data/s3', {
49+
method: 'POST',
50+
body: metafactsdata,
51+
})
52+
.then(parseJSON)
53+
.then(({ status, ok, json: metaTabledata }) => {
54+
if (!ok) {
55+
const error = new Error();
56+
error.detail = metaTabledata.detail;
57+
error.status = status;
58+
throw error;
59+
}
60+
dispatch(addMetaTableData(metaTabledata));
61+
})
62+
.finally(() => dispatch(setLoading(false)));
63+
};
64+
};
65+
export const setLoading = (data) => ({
66+
type: SET_META_LOADING,
67+
payload: data,
68+
});
69+
70+
export const setUploadButton = (data) => ({
71+
type: SET_META_UPLOAD_BUTTON,
72+
payload: data,
73+
});

‎studio/src/actions/metafactsValidly.js

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import {
2+
ADD_METAFACTS_EXPECTATIONS,
3+
ADD_METAFACTS_FILES,
4+
ADD_METAFACTS_VALIDATIONS,
5+
SET_METAFACTS_LOADING,
6+
SET_METAFACTS_UPLOAD_BUTTON,
7+
UPLOADED_METAFACTS_FILES_VALIDITY,
8+
} from '../constants/metafactsvalidly';
9+
import { getValidationDataArray } from './validly';
10+
11+
export const getMetaFactsValidationData = (files) => {
12+
return (dispatch) => {
13+
dispatch(setLoading(true));
14+
let data = new FormData();
15+
files.map((file) => data.append('datasets', file));
16+
data.append('result_type', 'COMPLETE');
17+
fetch(window.REACT_APP_VALIDLY_SERVER_URL + `/expectations/metadata/file`, {
18+
method: 'POST',
19+
body: data,
20+
})
21+
.then((response) => response.json())
22+
.then((validationData) => {
23+
const validationDataArray = getValidationDataArray(validationData);
24+
const areUploadedfilesCompletelyValid = validationDataArray.every(
25+
(file) => file.is_datacompletely_valid,
26+
);
27+
dispatch(setFilesValidity(areUploadedfilesCompletelyValid));
28+
dispatch(setMetaFactsUploadButton(false));
29+
dispatch(addMetaFactsExpectationCard(new Array(files.length).fill(null)));
30+
dispatch(addValidationData(validationDataArray));
31+
})
32+
.finally(() => dispatch(setLoading(false)));
33+
};
34+
};
35+
export const getGoogleSheetsValidations = (data) => {
36+
return (dispatch) => {
37+
dispatch(setLoading(true));
38+
let myHeaders = new Headers();
39+
myHeaders.append('Content-Type', 'application/json');
40+
let raw = JSON.stringify(data);
41+
let requestOptions = {
42+
method: 'POST',
43+
headers: myHeaders,
44+
body: raw,
45+
redirect: 'follow',
46+
};
47+
return fetch('http://localhost:8000/expectations/metadata/gsheet', requestOptions)
48+
.then((response) => response.json())
49+
.then((validationData) => {
50+
const validationDataArray = getValidationDataArray(validationData);
51+
const areUploadedfilesCompletelyValid = validationDataArray.every(
52+
(file) => file.is_datacompletely_valid,
53+
);
54+
dispatch(setFilesValidity(areUploadedfilesCompletelyValid));
55+
dispatch(setMetaFactsUploadButton(false));
56+
dispatch(addMetaFactsExpectationCard(new Array(1).fill(null)));
57+
dispatch(addValidationData(validationDataArray));
58+
})
59+
.finally(() => dispatch(setLoading(false)));
60+
};
61+
};
62+
export const addMetaFactsFiles = (data) => ({
63+
type: ADD_METAFACTS_FILES,
64+
payload: data,
65+
});
66+
const addValidationData = (data) => ({
67+
type: ADD_METAFACTS_VALIDATIONS,
68+
payload: data,
69+
});
70+
const setFilesValidity = (data) => ({
71+
type: UPLOADED_METAFACTS_FILES_VALIDITY,
72+
payload: data,
73+
});
74+
75+
export const addMetaFactsExpectationCard = (data) => ({
76+
type: ADD_METAFACTS_EXPECTATIONS,
77+
payload: data,
78+
});
79+
80+
export const setMetaFactsUploadButton = (data) => ({
81+
type: SET_METAFACTS_UPLOAD_BUTTON,
82+
payload: data,
83+
});
84+
85+
const setLoading = (data) => ({
86+
type: SET_METAFACTS_LOADING,
87+
payload: data,
88+
});

‎studio/src/actions/validly.js

Lines changed: 30 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -12,40 +12,15 @@ export const getValidationData = (files) => {
1212
let data = new FormData();
1313
files.map((file) => data.append('datasets', file));
1414
data.append('result_type', 'COMPLETE');
15-
fetch(window.REACT_APP_VALIDLY_SERVER_URL + '/expectation/datasets/?format=json', {
15+
fetch(window.REACT_APP_VALIDLY_SERVER_URL + `/expectation/datasets/?format=json`, {
1616
method: 'POST',
1717
body: data,
1818
})
1919
.then((response) => response.json())
2020
.then((validationData) => {
21-
let validationDataArray = [];
22-
Object.keys(validationData).map((filename) => {
23-
let expectationsValidatedList = [];
24-
let is_data_completely_valid;
25-
Object.keys(validationData[filename]).map((Expectation, index, array) => {
26-
if (
27-
validationData[filename][Expectation]['success'] === false &&
28-
is_data_completely_valid === undefined
29-
) {
30-
is_data_completely_valid = false;
31-
}
32-
if (is_data_completely_valid === undefined && array.length - 1 === index) {
33-
is_data_completely_valid = true;
34-
}
35-
expectationsValidatedList.push({
36-
Expectation_name: Expectation,
37-
...validationData[filename][Expectation],
38-
});
39-
});
40-
validationDataArray.push({
41-
filename: filename,
42-
is_datacompletely_valid: is_data_completely_valid,
43-
expectations_Validated_List: expectationsValidatedList,
44-
});
45-
});
46-
const areUploadedfilesCompletelyValid = validationDataArray.reduce(
47-
(totalBool, currentFile) => currentFile.is_datacompletely_valid && totalBool,
48-
true,
21+
const validationDataArray = getValidationDataArray(validationData);
22+
const areUploadedfilesCompletelyValid = validationDataArray.every(
23+
(file) => file.is_datacompletely_valid,
4924
);
5025
dispatch(setFilesValidity(areUploadedfilesCompletelyValid));
5126
dispatch(setUploadButton(false));
@@ -55,6 +30,32 @@ export const getValidationData = (files) => {
5530
.finally(() => dispatch(setLoading(false)));
5631
};
5732
};
33+
export const getExpectationsValidatedList = (validationData) => {
34+
const expectationsValidatedList = [];
35+
Object.keys(validationData).forEach((Expectation) => {
36+
expectationsValidatedList.push({
37+
Expectation_name: Expectation,
38+
...validationData[Expectation],
39+
});
40+
});
41+
return expectationsValidatedList;
42+
};
43+
export const getValidationDataArray = (validationData) => {
44+
const validationDataArray = [];
45+
Object.keys(validationData).forEach((filename) => {
46+
const expectationsValidatedList = getExpectationsValidatedList(validationData[filename]);
47+
const is_datacompletely_valid = expectationsValidatedList.every(
48+
(expectation) => expectation.success,
49+
);
50+
validationDataArray.push({
51+
filename: filename,
52+
is_datacompletely_valid,
53+
expectations_Validated_List: expectationsValidatedList,
54+
});
55+
});
56+
return validationDataArray;
57+
};
58+
5859
export const addValidationData = (data) => ({
5960
type: ADD_VALIDATIONS,
6061
payload: data,

‎studio/src/components/GlobalNav/Sidebar.js

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,26 +8,23 @@ import { MenuUnfoldOutlined, MenuFoldOutlined } from '@ant-design/icons';
88
import proSettings from '../../config/layout';
99

1010
const { Sider } = Layout;
11-
11+
const routesObj = {
12+
'/factly': '0',
13+
'/expectation': '0',
14+
'/meta-data': '1',
15+
'/validly-metafacts': '2',
16+
'/docs': '3',
17+
};
1218
function Sidebar() {
1319
const navTheme = proSettings.navTheme;
1420
const dispatch = useDispatch();
1521
const location = useLocation();
1622
const pathSnippets = location.pathname.split('/').filter((i) => i);
17-
const { files } = useSelector((state) => {
18-
return state.validly;
19-
});
2023
const selectedmenu = () => {
21-
if (['.factly', 'expectation'].includes(pathSnippets[0]) || pathSnippets.length === 0) {
24+
if (pathSnippets.length === 0) {
2225
return ['0'];
2326
}
24-
if (['meta-data'].includes(pathSnippets[0])) {
25-
return ['1'];
26-
}
27-
if (['docs'].includes(pathSnippets[0])) {
28-
if (files.length !== 0) return ['2'];
29-
return ['1'];
30-
}
27+
return [routesObj['/' + pathSnippets[0]]] || [];
3128
};
3229
const { collapsed } = useSelector((state) => state.sidebar.sider);
3330
const onCollapse = (collapsed) => {
@@ -85,9 +82,9 @@ function Sidebar() {
8582
>
8683
{routes
8784
.filter((each) => {
88-
if (each.title === 'Metafacts') {
89-
return each.enableNavigation === true && files.length !== 0;
90-
}
85+
// if (each.title === 'Metafacts') {
86+
// return each.enableNavigation === true && files.length !== 0;
87+
// }
9188
return each.enableNavigation === true;
9289
})
9390
.map((route, index) => {

‎studio/src/components/expectationcard.js

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
import React from 'react';
22
import { Modal, Tag, Table, Button } from 'antd';
33
import { Card } from 'antd';
4-
import { InfoCircleFilled, DownloadOutlined } from '@ant-design/icons';
5-
import { Typography, Space } from 'antd';
4+
import { DownloadOutlined } from '@ant-design/icons';
5+
import { Typography } from 'antd';
66
import { CSVLink } from 'react-csv';
7-
const { Text, Link } = Typography;
7+
const { Link } = Typography;
88
function ExpectationCard(Expectation) {
9-
const { Expectation_name, success } = Expectation;
9+
const { Expectation_name } = Expectation;
1010
const [show, setShow] = React.useState(false);
1111
const isResultNested = () => {
1212
if (Expectation.results) {
@@ -87,12 +87,11 @@ function ExpectationCard(Expectation) {
8787
</p>
8888
<p>
8989
<span>
90-
{getPartial_unexepected_list().map((value, index) => {
91-
if (index > 5) {
92-
return;
93-
}
94-
return <Tag color="warning"> {value} </Tag>;
95-
})}
90+
{getPartial_unexepected_list()
91+
.slice(0, 5)
92+
.map((value) => (
93+
<Tag color="warning"> {value} </Tag>
94+
))}
9695
</span>
9796
{isMoreHidden && <Link onClick={() => setShow(true)}>...more</Link>}
9897
</p>

0 commit comments

Comments
 (0)