From b50ab552e1c6faaf44ae7151c961d155c2d5866a Mon Sep 17 00:00:00 2001 From: Sny Date: Thu, 2 Jan 2025 09:45:50 +0530 Subject: [PATCH] OpenConceptLab/ocl_issues#2023 | Matching component with row states and repo select --- .eslintrc.json | 5 +- package-lock.json | 20 +- package.json | 4 +- src/components/app/App.jsx | 2 + src/components/common/AutocompleteLoading.jsx | 25 + src/components/repos/Matching.jsx | 1204 +++++++++++++++++ src/components/repos/RepoConceptsMatch.jsx | 454 +++++-- src/components/repos/RepoListItem.jsx | 18 + .../repos/RepoSearchAutocomplete.jsx | 112 ++ src/components/search/SearchControls.jsx | 3 +- .../search/SearchHighlightsDialog.jsx | 98 ++ src/components/search/SearchResults.jsx | 41 +- src/components/search/TableResults.jsx | 50 +- 13 files changed, 1840 insertions(+), 196 deletions(-) create mode 100644 src/components/common/AutocompleteLoading.jsx create mode 100644 src/components/repos/Matching.jsx create mode 100644 src/components/repos/RepoListItem.jsx create mode 100644 src/components/repos/RepoSearchAutocomplete.jsx create mode 100644 src/components/search/SearchHighlightsDialog.jsx diff --git a/.eslintrc.json b/.eslintrc.json index af0cb813..609411c1 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -236,7 +236,10 @@ "sameas", "nbsp", "scroller", - "llm" + "llm", + "unmapped", + "unmap", + "mins" ], "skipIfMatch": [ "http://[^s]*", diff --git a/package-lock.json b/package-lock.json index f0f2cffc..7115cb82 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8457,11 +8457,11 @@ "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==" }, "cross-fetch": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", - "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.2.0.tgz", + "integrity": "sha512-Q+xVJLoGOeIMXZmbUK4HYk+69cQH6LudR0Vu/pRm2YlU/hDV9CiS0gKUMaWY5f2NeUH9C1nV3bsTlCo0FsTV1Q==", "requires": { - "node-fetch": "^2.6.12" + "node-fetch": "^2.7.0" } }, "cross-spawn": { @@ -13647,9 +13647,9 @@ } }, "node-fetch": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", - "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", "requires": { "whatwg-url": "^5.0.0" } @@ -15624,9 +15624,9 @@ "integrity": "sha512-6X1p/sU7hecmjDZMAwN+r3go9EVjofKhwkUbVlL8lXhBZecPv9XVCkZ/kBPYOr0Mv0Vl5+Ziwgexg9Kh7+NNXQ==" }, "react-window": { - "version": "1.8.10", - "resolved": "https://registry.npmjs.org/react-window/-/react-window-1.8.10.tgz", - "integrity": "sha512-Y0Cx+dnU6NLa5/EvoHukUD0BklJ8qITCtVEPY1C/nL8wwoZ0b5aEw8Ff1dOVHw7fCzMt55XfJDd8S8W8LCaUCg==", + "version": "1.8.11", + "resolved": "https://registry.npmjs.org/react-window/-/react-window-1.8.11.tgz", + "integrity": "sha512-+SRbUVT2scadgFSWx+R1P754xHPEqvcfSfVX10QYg6POOz+WNgkN48pS+BtZNIMGiL1HYrSEiCkwsMS15QogEQ==", "requires": { "@babel/runtime": "^7.0.0", "memoize-one": ">=3.1.1 <6" diff --git a/package.json b/package.json index f92391f3..15eb1032 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "react-password-strength-bar": "^0.3.5", "react-router-dom": "^5.3.4", "react-virtuoso": "^4.12.3", - "react-window": "^1.8.10", + "react-window": "^1.8.11", "stacktrace-js": "^2.0.2", "xlsx": "^0.18.5" }, @@ -51,7 +51,7 @@ "babel-eslint": "^10.1.0", "babel-loader": "^8.4.1", "copy-webpack-plugin": "^4.6.0", - "cross-fetch": "^3.1.8", + "cross-fetch": "^3.2.0", "css-loader": "^0.28.8", "eslint": "^4.19.1", "eslint-config-airbnb": "^16.1.0", diff --git a/src/components/app/App.jsx b/src/components/app/App.jsx index d61514fd..077c6fcf 100644 --- a/src/components/app/App.jsx +++ b/src/components/app/App.jsx @@ -26,6 +26,7 @@ import UserSettings from '../users/UserSettings'; import OrgHome from '../orgs/OrgHome'; import URLRegistry from '../url-registry/URLRegistry' import RepoConceptsMatch from '../repos/RepoConceptsMatch' +import Matching from '../repos/Matching' const AuthenticationRequiredRoute = ({component: Component, ...rest}) => ( { + diff --git a/src/components/common/AutocompleteLoading.jsx b/src/components/common/AutocompleteLoading.jsx new file mode 100644 index 00000000..09e2afed --- /dev/null +++ b/src/components/common/AutocompleteLoading.jsx @@ -0,0 +1,25 @@ +import React from 'react'; +import { Box, LinearProgress } from '@mui/material' +import { BLACK } from '../../common/colors' + +const AutocompleteLoading = ({ text }) => { + + return ( + + + { + text && +
+ Loading results for { text } +
+ } +
+ +
+ + + + ) +} + +export default AutocompleteLoading; diff --git a/src/components/repos/Matching.jsx b/src/components/repos/Matching.jsx new file mode 100644 index 00000000..663b53b1 --- /dev/null +++ b/src/components/repos/Matching.jsx @@ -0,0 +1,1204 @@ +import React from 'react' +import * as XLSX from 'xlsx'; +import moment from 'moment' + +import { TableVirtuoso } from 'react-virtuoso'; +import Paper from '@mui/material/Paper' +import Button from '@mui/material/Button'; +import Typography from '@mui/material/Typography'; +import Table from '@mui/material/Table'; +import TableBody from '@mui/material/TableBody'; +import TableCell from '@mui/material/TableCell'; +import TableContainer from '@mui/material/TableContainer'; +import TableHead from '@mui/material/TableHead'; +import TableRow from '@mui/material/TableRow'; +import Dialog from '@mui/material/Dialog'; +import DialogContent from '@mui/material/DialogContent'; +import DialogTitle from '@mui/material/DialogTitle'; +import DialogActions from '@mui/material/DialogActions'; +import List from '@mui/material/List'; +import ListItem from '@mui/material/ListItem'; +import ListItemText from '@mui/material/ListItemText'; +import ListItemAvatar from '@mui/material/ListItemAvatar'; +import Avatar from '@mui/material/Avatar'; +import TextField from '@mui/material/TextField'; +import IconButton from '@mui/material/IconButton' +import Card from '@mui/material/Card'; +import CardContent from '@mui/material/CardContent'; +import Box from '@mui/material/Box'; +import CircularProgress from '@mui/material/CircularProgress'; +import Tabs from '@mui/material/Tabs'; +import Tab from '@mui/material/Tab'; +import Autocomplete from '@mui/material/Autocomplete'; +import Chip from '@mui/material/Chip'; +import Menu from '@mui/material/Menu'; +import MenuItem from '@mui/material/MenuItem'; +import { styled } from '@mui/material/styles'; + +import JoinRightIcon from '@mui/icons-material/JoinRight'; +import DownIcon from '@mui/icons-material/ArrowDropDown'; +import UploadIcon from '@mui/icons-material/Upload'; +import MatchingIcon from '@mui/icons-material/DeviceHub'; +import EditIcon from '@mui/icons-material/EditOutlined'; +import SaveIcon from '@mui/icons-material/SaveOutlined'; +import ConfirmedIcon from '@mui/icons-material/CheckCircleOutlineOutlined'; +import DoubleArrowIcon from '@mui/icons-material/DoubleArrow'; +import AutoMatchIcon from '@mui/icons-material/ChecklistRtl'; +import MediumMatchIcon from '@mui/icons-material/Rule'; +import LowMatchIcon from '@mui/icons-material/DynamicForm'; +import NoMatchIcon from '@mui/icons-material/RemoveRoad'; +import DownloadIcon from '@mui/icons-material/Download'; +import CancelOutlinedIcon from '@mui/icons-material/CancelOutlined'; + +import orderBy from 'lodash/orderBy' +import filter from 'lodash/filter' +import map from 'lodash/map' +import forEach from 'lodash/forEach' +import snakeCase from 'lodash/snakeCase' +import startCase from 'lodash/startCase' +import values from 'lodash/values' +import find from 'lodash/find' +import without from 'lodash/without' +import has from 'lodash/has' +import compact from 'lodash/compact' +import chunk from 'lodash/chunk' +import get from 'lodash/get' +import countBy from 'lodash/countBy' +import sum from 'lodash/sum' +import omit from 'lodash/omit' +import reject from 'lodash/reject' +import uniq from 'lodash/uniq' +import uniqBy from 'lodash/uniqBy' + +import APIService from '../../services/APIService'; +import { highlightTexts } from '../../common/utils'; +import { WHITE, SURFACE_COLORS } from '../../common/colors'; + +import CloseIconButton from '../common/CloseIconButton'; +import SearchResults from '../search/SearchResults'; +import SearchHighlightsDialog from '../search/SearchHighlightsDialog' +import ConceptHome from '../concepts/ConceptHome' +import RepoSearchAutocomplete from './RepoSearchAutocomplete' + +const HEADERS = [ + {id: 'id', label: 'ID'}, + {id: 'name', label: 'Name'}, + {id: 'synonyms', label: 'Synonyms'}, + {id: 'concept_class', label: 'Concept Class'}, + {id: 'datatype', label: 'Datatype'}, + {id: 'same_as_map_codes', label: 'Same As Codes'}, + {id: 'other_map_codes', label: 'Concept Set'}, +] + +const ROW_STATES = ['readyForReview', 'mapped', 'unmapped'] +const MATCH_TYPES = { + very_high: { + label: 'Auto Match', + icon: , + color: 'primary', + }, + high: { + label: 'High Match', + icon: , + color: 'warning', + }, + low: { + label: 'Low Match', + icon: , + color: 'secondary', + }, + no_match: { + label: 'No Match', + icon: , + color: 'error', + }, +} + +const ALGOS = [ + {id: 'es', label: 'Generic Elastic Search Matching'}, + {id: 'llm', label: 'LLM Matching', disabled: true}, +] +const DECISION_TABS = ['map', 'candidates', 'search', 'propose'] +const UPDATED_COLOR = 'rgba(255, 167, 38, 0.1)' + +const formatMappings = item => { + let same_as_mappings = [] + let other_mappings = {} + forEach((item.mappings || []), mapping => { + let mapType = mapping.map_type + mapType = mapType.replace('_', '').replace('-', '').replace(' ', '').toLowerCase() + if(mapType === 'sameas') + same_as_mappings.push(mapping) + else { + other_mappings[mapType] = other_mappings[mapType] || [] + other_mappings[mapType].push(mapping) + } + }) + same_as_mappings = orderBy(same_as_mappings, ['cascade_target_source_name', 'to_concept_code', 'cascade_target_concept_name']) + other_mappings = orderBy(other_mappings, ['map_type', 'cascade_target_source_name', 'to_concept_code', 'cascade_target_concept_name']) + return ( + + { + same_as_mappings.length > 1 && + <> + { + map(same_as_mappings, (mapping, i) => ( + + + + {`${mapping.cascade_target_source_name}:${mapping.to_concept_code}`} + + + {mapping.cascade_target_concept_name} + + + } + sx={{ + marginTop: '2px', + marginBottom: '2px', + }} + /> + + )) + } + + } + { + map(other_mappings, (mappings, mapType) => ( + + { + map(mappings, (mapping, i) => ( + + + + {`${mapping.cascade_target_source_name}:${mapping.to_concept_code}`} + + + {mapping.cascade_target_concept_name} + + + } + sx={{ + marginTop: '2px', + marginBottom: '2px', + }} + /> + + )) + } + + )) + } + + ) +} + + +const HeaderAutocomplete = ({isUpdatedValue, helperText, ...rest}) => { + return ( + option?.label ? option.label : (find(HEADERS, {id: option})?.label || option)} + isOptionEqualToValue={(option, value) => option?.id === value?.id || option?.id === value || option === value} + sx={{ + '.MuiOutlinedInput-root': { + padding: '4px 10px' + }, + '.MuiInputBase-input': { + padding: '0 !important', + fontSize: '14px', + }, + '.MuiFormHelperText-root': { + margin: '0 !important', + padding: '2px 0 0 6px', + backgroundColor: isUpdatedValue ? UPDATED_COLOR : undefined + } + }} + renderInput={(params) => } + options={HEADERS} + {...rest} + /> + ) +} + + +const MatchSummaryCard = ({id, icon, label, count, loading, color, selected, onClick }) => { + const isSelected = id === selected + return ( +
+ + + + + + + {icon} + + {loading && ( + + )} + + + + + + +
+ ) +} + + +const TableCellAction = ({ isEditing, onEdit, onSave, sx }) => { + return ( + + { + isEditing ? + + + : + + + + } + + ) +} + +const VirtuosoTableComponents = { + Scroller: React.forwardRef((props, ref) => ( + + )), + Table: (props) => ( + + ), + TableHead: React.forwardRef((props, ref) => ), + TableRow, + TableBody: React.forwardRef((props, ref) => ), +}; + +const VisuallyHiddenInput = styled('input')({ + clip: 'rect(0 0 0 0)', + clipPath: 'inset(50%)', + height: 1, + overflow: 'hidden', + position: 'absolute', + bottom: 0, + left: 0, + whiteSpace: 'nowrap', + width: 1, +}); + + +const Matching = () => { + // project state + const [name, setName] = React.useState('') + const [file, setFile] = React.useState(false) + const [data, setData] = React.useState(false) + const [columns, setColumns] = React.useState([]) + const [rowStatuses, setRowStatuses] = React.useState({mapped: [], readyForReview: [], unmapped: []}) + const [matchTypes, setMatchTypes] = React.useState({very_high: 0, high: 0, low: 0, no_match: 0}) + const [matchedConcepts, setMatchedConcepts] = React.useState([]); + const [otherMatchedConcepts, setOtherMatchedConcepts] = React.useState([]); + const [algo, setAlgo] = React.useState('es') + const [decisions, setDecisions] = React.useState({}) + const [startMatchingAt, setStartMatchingAt] = React.useState(false) + const [endMatchingAt, setEndMatchingAt] = React.useState(false) + + const [row, setRow] = React.useState(false) + const [loadingMatches, setLoadingMatches] = React.useState(false) + const [edit, setEdit] = React.useState([]); + const [selectedRowStatus, setSelectedRowStatus] = React.useState('all') + const [selectedMatchBucket, setSelectedMatchBucket] = React.useState(false) + const [editName, setEditName] = React.useState(false) + const [decisionTab, setDecisionTab] = React.useState('map') + const [algoMenuAnchorEl, setAlgoMenuAnchorEl] = React.useState(null) + + const [matchDialog, setMatchDialog] = React.useState(false) + const [showHighlights, setShowHighlights] = React.useState(false) + const [showItem, setShowItem] = React.useState(false) + + // repo state + const [repo, setRepo] = React.useState(false) + const [conceptCache, setConceptCache] = React.useState({}) + + const rowIndex = row?.__index + + const getColumns = row => { + let _columns = [] + if(row) { + _columns = map(row, (value, key) => { + let width; + if(['id', 'code'].includes(key.toLowerCase())) + width = '60px' + if(['changed by', 'creator'].includes(key.toLowerCase())) + width = '75px' + else if(['class', 'concept class', 'datatype'].includes(key.toLowerCase())) + width = '100px' + return {label: key, dataKey: key, width: width, original: key } + }) + } + return _columns + } + + const updateColumn = (position, newValue) => { + setColumns(prev => { + prev[position].label = newValue + return prev + }) + } + + const updateRow = (index, columnKey, newValue) => { + setData(prevData => map(prevData, row => (row.__index === index ? {...row, [`${columnKey}__updated`]: newValue} : row))) + } + + + const handleFileUpload = event => { + const file = event.target.files[0]; + setFile(file) + const reader = new FileReader(); + reader.onload = (e) => { + const workbook = XLSX.read(e.target.result, { type: 'binary', raw: true, cellText: true, codepage: 65001 }); + const sheetName = workbook.SheetNames[0]; + const sheet = workbook.Sheets[sheetName]; + const jsonData = XLSX.utils.sheet_to_json(sheet, { raw: false, defval: '' }); + setData(map(jsonData, (data, index) => ({...data, __index: index}))); + setColumns(getColumns(jsonData[0])) + }; + reader.readAsBinaryString(file); + }; + + const onAlgoButtonClick = event => setAlgoMenuAnchorEl(algoMenuAnchorEl ? null : event.currentTarget) + + const onAlgoSelect = newAlgo => { + setAlgo(newAlgo) + setAlgoMenuAnchorEl(null) + } + + const fixedHeaderContent = () => { + const isEditing = edit?.includes(-1) + return columns?.length ? ( + + + { + map(columns, (column, position) => { + const isUpdatedValue = column.label !== column.original + const isValidColumn = !isEditing && isValidColumnValue(column.label) + return ( + + { + isEditing ? + updateColumn(position, value?.label || value)} + /> : + + {column.label} {isValidColumn ? : } + + } + + ) + }) + } + setEdit(without(edit, -1))} + onEdit={() => setEdit([...edit, -1])} + /> + + ) : null; + } + + const rowContent = (_index, _row) => { + const isEditing = edit?.includes(_row.__index) + const bgColor = _row.__index === row.__index ? SURFACE_COLORS.main : WHITE + const defaultMatchTypeColor = 'rgba(0, 0, 0, 0.05)' + const matchType = get(find(matchedConcepts, c => c.row.__index === _row.__index), 'results[0].search_meta.match_type') + const matchTypeColor = matchType ? MATCH_TYPES[matchType].color + '.main' : defaultMatchTypeColor + return ( + + + {_row.__index + 1} + + { + map(columns, column => { + const defaultValue = _row[column.dataKey] + const value = has(_row, column.dataKey + '__updated') ? _row[column.dataKey + '__updated'] : defaultValue + const isUpdatedValue = defaultValue !== value + return ( + onCSVRowSelect(_row)} + key={column.dataKey} + > + { + isEditing ? + updateRow(_row.__index, column.dataKey, event.target.value)} + sx={{'.MuiOutlinedInput-root': {padding: '4px 10px'}, '.MuiFormHelperText-root': {margin: 0, padding: '2px 0 0 10px', whiteSpace: 'pre-line', backgroundColor: isUpdatedValue ? UPDATED_COLOR : undefined}}} + /> : + {value} + } + + ) + }) + } + setEdit(without(edit, _row.__index))} + onEdit={() => setEdit([...edit, _row.__index])} + /> + + ); + } + + const getPayloadForMatching = (rows, _repo) => { + return { + rows: map(rows, row => prepareRow(row)), + target_repo_url: _repo.version_url || _repo.url, + target_repo: { + 'owner': _repo.owner, + 'owner_type': _repo.owner_type, + 'source_version': _repo.version || _repo.id, + 'source': _repo.short_code || _repo.id + }, + } + } + + + const getRowsResults = async (rows) => { + const CHUNK_SIZE = 300; // Number of rows per batch + const MAX_CONCURRENT_REQUESTS = 4; // Number of parallel API requests allowed + const rowChunks = chunk(rows, CHUNK_SIZE); + + // Function to process a single batch + const processBatch = async (_repo, rowBatch) => { + const payload = getPayloadForMatching(rowBatch, _repo) + + try { + const response = await APIService.concepts() + .appendToUrl('$match/') + .post(payload, null, null, { + includeSearchMeta: true, + }); + + return response.data || []; + } catch { + return []; + } + }; + + // Function to handle concurrency + const processWithConcurrency = async (_repo) => { + const queue = rowChunks.slice(); // Copy of all chunks to be processed + const activeRequests = new Set(); + + while (queue.length > 0 || activeRequests.size > 0) { + // Fill activeRequests up to MAX_CONCURRENT_REQUESTS + while (queue.length > 0 && activeRequests.size < MAX_CONCURRENT_REQUESTS) { + const rowBatch = queue.shift(); + const promise = processBatch(_repo, rowBatch).then((data) => { + let matchTypes = map(data, 'results.0.search_meta.match_type') + let counts = countBy(matchTypes) + setMatchTypes(prev => ({ + very_high: prev.very_high + (counts?.very_high || 0), + high: prev.high + (counts?.high || 0), + low: prev.low + (counts?.low || 0), + no_match: prev.no_match + (sum(values(omit(counts, ['very_high', 'high', 'low']))) || 0) + })); + setRowStatuses(prev => { + prev.readyForReview = [...prev.readyForReview, ...map(data, 'row.__index')] + return prev + }) + setMatchedConcepts(prev => [...prev, ...data]); + activeRequests.delete(promise); // Remove from active set after completion + }); + activeRequests.add(promise); + } + + // Wait for at least one request to complete before continuing + await Promise.race(activeRequests); + } + }; + + const response = await APIService.new().overrideURL('/$resolveReference/').post({url: repo.version_url || repo.url}) + let _repo = get(response.data, '0.result.id') ? response.data[0].result : repo + setRepo(_repo) + await processWithConcurrency(_repo); + setEndMatchingAt(moment()) + setLoadingMatches(false) + }; + + + const prepareRow = csvRow => { + let row = {} + forEach(csvRow, (value, key) => { + if((value === 0 || value) && !has(csvRow, key + '__updated')) { + key = find(columns, {original: key.replace('__updated', '')})?.label || key + let newValue = value + let newKey = key === '__index' ? key : snakeCase(key.toLowerCase()) + let isList = key === '__index' ? false : newValue.includes('\n') + + if(isList) + newValue = newValue.split('\n') + if(key.includes('__updated')) + newKey = key.replace('__updated', '') + if(newKey.includes('class')) + newKey = 'concept_class' + if(newKey === 'set_members') + newKey = 'other_map_codes' + if(newKey === 'same_as') + newKey = 'same_as_map_codes' + if(isList) + row[newKey] = [...(row[newKey] || []), ...newValue] + else + row[newKey] = newValue + } + }) + return row + } + + const isValidColumnValue = value => { + if(!value) + return false + if(value.toLowerCase().includes('class')) + return true + if(find(HEADERS, val => val.label.toLowerCase() === value.toLowerCase())) + return true + return false + } + + + const onGetCandidates = event => { + event.stopPropagation() + event.preventDefault() + setMatchDialog(true) + } + + const onGetCandidatesSubmit = event => { + event.stopPropagation() + event.preventDefault() + setStartMatchingAt(moment()) + setLoadingMatches(true) + getRowsResults(data) + setMatchDialog(false) + } + + const showMatchSummary = Boolean(data?.length && (loadingMatches || matchedConcepts?.length)) + const getMatchingDuration = () => { + let start = startMatchingAt + let end = endMatchingAt + if(!end) + end = moment() + if(!start) + return false + return `${moment.duration(end.diff(start)).as('minutes').toFixed(2)} mins`; + } + + const getCandidatesButtonLabel = () => { + const matchingDuration = getMatchingDuration() + if(loadingMatches) + return `Getting Candidates (${matchingDuration})` + if(matchedConcepts?.length) + return `Re-run? (last: ${matchingDuration})` + return 'Get Candidates' + } + + const onMatchTypeChange = bucket => { + setSelectedMatchBucket(prev => prev === bucket ? false : bucket) + } + + const getRows = () => { + let rows = data?.length ? [...data] : [] + if(selectedRowStatus !== 'all') + rows = filter(rows, r => rowStatuses[selectedRowStatus].includes(r.__index)) + if(selectedMatchBucket) { + let getIndex = concept => { + if(selectedMatchBucket === 'no_match') + return (!concept?.results?.length || !['very_high', 'high', 'low'].includes(concept.results[0].search_meta.match_type)) ? concept.row.__index : null + return (concept?.results?.length && concept.results[0].search_meta.match_type === selectedMatchBucket) ? concept.row.__index : null + } + const rowIndexes = map(matchedConcepts, getIndex) + rows = filter(rows, r => rowIndexes.includes(r.__index)) + } + return rows + } + + const getStatusFromConceptAndRowIndex = (concept, index) => { + if(isMappedInDecisions(concept, index)) + return 'Mapped' + if(rowStatuses.unmapped.includes(index)) + return 'Unmapped' + if(rowStatuses.readyForReview.includes(index)) + return 'Ready for Review' + return 'No Decision' + } + + const onDownloadClick = () => { + const rows = map(data, row => { + const index = row.__index + let newRow = {...row, 'Concept ID': [], 'Concept URL': [], 'Match Score': [], 'Match Type': [], 'Decision': []} + let matched = find(matchedConcepts, concept => concept.row.__index === index) + let otherMatched = find(otherMatchedConcepts, concept => concept.row.__index === index) + let matches = orderBy([...(matched?.results || []), ...(otherMatched?.results || [])], 'search_meta.search_score', 'desc') + if(matches?.length) { + forEach(matches, concept => { + newRow['Concept ID'].push(concept.id) + newRow['Concept URL'].push(concept.version_url) + newRow['Match Score'].push(concept.search_meta.search_score) + newRow['Match Type'].push(startCase(concept.search_meta.match_type)) + newRow['Decision'].push(getStatusFromConceptAndRowIndex(concept, index)) + // newRow = {...newRow, 'Concept ID': concept.id, 'Concept URL': concept.url, 'Match Score': concept.search_meta.search_score, 'Match Type': startCase(concept.search_meta.match_type), 'Decision': getStatusFromConceptAndRowIndex(concept, index)} + }) + } else { + newRow = {...newRow, 'Match Type': 'No Match'} + } + ['Concept ID', 'Concept URL', 'Match Score', 'Match Type', 'Decision'].forEach(col => { + newRow[col] = newRow[col].join('\n') + }) + newRow = {...newRow, 'Repo Version': repo.version || repo.id, 'Repo ID': repo.short_code || repo.id, 'Repo URL': repo.version_url || repo.url} + delete newRow.__index + return newRow + }) + const worksheet = XLSX.utils.json_to_sheet(rows); + const workbook = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(workbook, worksheet, "Dates"); + XLSX.writeFile(workbook, "Matched.csv", { compression: true }); + } + + const onCSVRowSelect = csvRow => { + if(edit?.length > 0) + return + + const matched = get(find(matchedConcepts, concept => concept.row.__index === csvRow.__index), 'results.0') + let url = matched?.version_url || matched?.url + if(url) + APIService + .new() + .overrideURL(url) + .get(null, null, {includeMappings: true, mappingBrief: true, mapTypes: 'SAME-AS,SAME AS,SAME_AS', verbose: true}) + .then(response => { + const res = {...response.data, search_meta: {...matched.search_meta}} + setConceptCache({...conceptCache, [url]: res}) + setTimeout(() => { + highlightTexts([res], null, true) + }, 100) + }) + setRow(csvRow) + setDecisionTab('map') + } + + const onCloseDecisions = () => { + setRow(false) + setShowHighlights(false) + } + + const onMap = (event, concepts, unmap=false) => { + event.preventDefault() + event.stopPropagation() + let newDecisions = {...(decisions[rowIndex] || {})} + const conceptURLs = concepts.map(c => c.version_url) + setRowStatuses(prev => { + prev.readyForReview = without(prev.readyForReview, rowIndex) + if(unmap) { + newDecisions.mapped = reject(newDecisions.mapped, c => conceptURLs.includes(c.version_url)) + if(newDecisions.mapped?.length === 0) { + prev.mapped = without(uniq(prev.mapped), rowIndex) + prev.unmapped = uniq([...prev.unmapped, rowIndex]) + } + } + else { + prev.mapped = uniq([...prev.mapped, rowIndex]) + prev.unmapped = without(uniq(prev.unmapped), rowIndex) + } + updateMatchTypeCounts(null, prev) + return prev + }) + setDecisions(prev => { + let _decisions = prev[rowIndex] || {} + _decisions.mapped = unmap ? newDecisions.mapped : uniqBy([...(_decisions.mapped || []), ...concepts], 'version_url') + return {...prev, [rowIndex]: _decisions} + }) + return false + } + + const isMappedInDecisions = (concept, index) => Boolean(find(decisions[index || rowIndex]?.mapped, {version_url: concept.version_url})) + + const onStateTabChange = newValue => { + setSelectedRowStatus(newValue) + updateMatchTypeCounts(newValue) + } + + const updateMatchTypeCounts = (newRowStatus, newRowStatuses) => { + let rowStatus = newRowStatus || selectedRowStatus + if(rowStatus === 'all') + return + let rows = filter(matchedConcepts, concept => (newRowStatuses || rowStatuses)[newRowStatus || selectedRowStatus].includes(concept.row.__index)); + let matchTypes = map(rows, 'results.0.search_meta.match_type') + let counts = countBy(matchTypes) + setMatchTypes({ + very_high: (counts?.very_high || 0), + high: (counts?.high || 0), + low: (counts?.low || 0), + no_match: sum(values(omit(counts, ['very_high', 'high', 'low']))) || 0 + }); + } + + const onDecisionTabChange = (event, newValue) => { + setShowItem(false) + setDecisionTab(newValue) + if(newValue === 'candidates') { + if(!find(otherMatchedConcepts, c => c.row.__index === rowIndex)) { + const payload = getPayloadForMatching([row], repo) + APIService.concepts() + .appendToUrl('$match/') + .post(payload, null, null, { + includeSearchMeta: true, + includeMappings: true, + mappingBrief: true, + mapTypes: 'SAME-AS,SAME AS,SAME_AS', + verbose: true, + limit: 4, + offset: 1 + }).then(response => { + setOtherMatchedConcepts([...otherMatchedConcepts, ...response.data]) + }); + } + } + } + + + const matchedResponse = find(matchedConcepts, concept => concept.row.__index === rowIndex) + const matchedConcept = get(matchedResponse, 'results.0') ? conceptCache[matchedResponse.results[0].version_url || matchedResponse.results[0].url] || matchedResponse.results[0] : null + const isSplitView = Boolean(matchedResponse?.row?.__index > -1 && rowIndex !== false) + const rows = getRows() + + const getTitle = () => { + let title = 'Mapping Project' + if(!editName && name) + title += ` - ${name}` + return title + } + + return ( +
+ + + + {getTitle()} + { + editName ? + setName(event.target.value || '')} + onBlur={() => setEditName(false)} + /> : + setEditName(true)} /> + } + + +
+ + + + { + showMatchSummary && !loadingMatches && + + } +
+ + { + (Boolean(rows?.length) || selectedMatchBucket || ROW_STATES.includes(selectedRowStatus)) && +
+ onStateTabChange(newValue)} + > + + { + ROW_STATES.map(state => { + let count = rowStatuses[state].length + return ( + + ) + }) + } + + { + showMatchSummary && +
+ onMatchTypeChange('very_high')} + {...MATCH_TYPES.very_high} + /> + onMatchTypeChange('high')} + {...MATCH_TYPES.high} + /> + onMatchTypeChange('low')} + {...MATCH_TYPES.low} + /> + onMatchTypeChange('no_match')} + {...MATCH_TYPES.no_match} + /> +
+ } + +
+ } + setMatchDialog(false)} + scroll='paper' + sx={{ + '& .MuiDialog-paper': { + borderRadius: '28px', + minWidth: '312px', + minHeight: '262px', + padding: 0 + } + }} + > + + Auto Match + + + setRepo(item)} /> + + + + + + + + +
+
+ Mapping Decisions + +
+
+ + { + DECISION_TABS.map(_tab => { + return ( + + ) + }) + } + +
+ { + ['map', 'candidates'].includes(decisionTab) && isSplitView && +
+
+ c.row.__index === rowIndex )?.results || []) : [])]), 'version_url'), 'search_meta.search_score', 'desc'), + total: 1 + }} + resource='concepts' + noPagination + noSorting + noToolbar + resultContainerStyle={{height: decisionTab === 'candidates' ? (showItem ? 'calc(100vh - 550px)' : 'calc(100vh - 200px)') : 'auto'}} + onShowItemSelect={item => { + setShowItem(item) + setTimeout(() => { + highlightTexts([item], null, false) + }, 100) + }} + selectedToShow={showItem} + extraColumns={[ + { + sortable: false, + id: 'mappings', + labelKey: 'mapping.same_as_mappings', + renderer: formatMappings, + }, + { + sortable: false, + id: 'search_meta.search_score', + labelKey: 'search.score', + renderer: (concept) => { + return { + event.preventDefault() + event.stopPropagation() + setShowHighlights(concept) + return false + }} + disabled={!concept?.search_meta?.search_score} + /> + }, + }, + { + sortable: false, + id: 'map-control', + labelKey: '', + renderer: concept => { + const isMapped = isMappedInDecisions(concept) + return ( + + )}, + }, + ]} + /> +
+
+ } +
+ setShowHighlights(false)} + highlight={showHighlights?.search_meta?.search_highlight || []} + score={parseFloat(showHighlights?.search_meta?.search_score || 0).toFixed(2)} + /> +
+ { + showItem?.id && + setShowItem(false)} nested /> + } +
+ + + {ALGOS.map(_algo => ( + onAlgoSelect(_algo.id)} + > + {_algo.label} + + ))} + +
+ ) +} + +export default Matching; diff --git a/src/components/repos/RepoConceptsMatch.jsx b/src/components/repos/RepoConceptsMatch.jsx index 407455a8..c2f0ae58 100644 --- a/src/components/repos/RepoConceptsMatch.jsx +++ b/src/components/repos/RepoConceptsMatch.jsx @@ -1,7 +1,7 @@ import React from 'react'; import * as XLSX from 'xlsx'; +import moment from 'moment' -import { useTranslation } from 'react-i18next' import { useLocation, useHistory, useParams } from 'react-router-dom'; import { TableVirtuoso } from 'react-virtuoso'; @@ -14,13 +14,11 @@ import TableCell from '@mui/material/TableCell'; import TableContainer from '@mui/material/TableContainer'; import TableHead from '@mui/material/TableHead'; import TableRow from '@mui/material/TableRow'; -import Dialog from '@mui/material/Dialog'; -import DialogContent from '@mui/material/DialogContent'; -import DialogTitle from '@mui/material/DialogTitle'; -import DialogActions from '@mui/material/DialogActions'; import List from '@mui/material/List'; import ListItem from '@mui/material/ListItem'; import ListItemText from '@mui/material/ListItemText'; +import ListItemAvatar from '@mui/material/ListItemAvatar'; +import Avatar from '@mui/material/Avatar'; import Menu from '@mui/material/Menu'; import MenuItem from '@mui/material/MenuItem'; import OutlinedInput from '@mui/material/OutlinedInput'; @@ -28,6 +26,10 @@ import TextField from '@mui/material/TextField'; import IconButton from '@mui/material/IconButton' import InputAdornment from '@mui/material/InputAdornment'; import FormControl, { useFormControl } from '@mui/material/FormControl'; +import Card from '@mui/material/Card'; +import CardContent from '@mui/material/CardContent'; +import Box from '@mui/material/Box'; +import CircularProgress from '@mui/material/CircularProgress'; import { styled } from '@mui/material/styles'; import JoinRightIcon from '@mui/icons-material/JoinRight'; import DownIcon from '@mui/icons-material/ArrowDropDown'; @@ -36,6 +38,11 @@ import MatchingIcon from '@mui/icons-material/DeviceHub'; import SearchIcon from '@mui/icons-material/Search'; import EditIcon from '@mui/icons-material/EditOutlined'; import SaveIcon from '@mui/icons-material/SaveOutlined'; +import DoubleArrowIcon from '@mui/icons-material/DoubleArrow'; +import AutoMatchIcon from '@mui/icons-material/ChecklistRtl'; +import MediumMatchIcon from '@mui/icons-material/Rule'; +import LowMatchIcon from '@mui/icons-material/DynamicForm'; +import NoMatchIcon from '@mui/icons-material/RemoveRoad'; import orderBy from 'lodash/orderBy' import filter from 'lodash/filter' @@ -43,12 +50,15 @@ import map from 'lodash/map' import forEach from 'lodash/forEach' import isEqual from 'lodash/isEqual' import snakeCase from 'lodash/snakeCase' -import startCase from 'lodash/startCase' import values from 'lodash/values' import find from 'lodash/find' import debounce from 'lodash/debounce' import without from 'lodash/without' import has from 'lodash/has' +import chunk from 'lodash/chunk' +import countBy from 'lodash/countBy' +import sum from 'lodash/sum' +import omit from 'lodash/omit' import APIService from '../../services/APIService'; import { dropVersion, toParentURI, toOwnerURI, highlightTexts } from '../../common/utils'; @@ -56,21 +66,74 @@ import { WHITE, SURFACE_COLORS } from '../../common/colors'; import CloseIconButton from '../common/CloseIconButton'; import LoaderDialog from '../common/LoaderDialog'; -import Link from '../common/Link' import Error40X from '../errors/Error40X'; import SearchResults from '../search/SearchResults'; import ConceptHome from '../concepts/ConceptHome' +import SearchHighlightsDialog from '../search/SearchHighlightsDialog' import RepoHeader from './RepoHeader'; const UPDATED_COLOR = 'rgba(255, 167, 38, 0.1)' +const CONFIRMED_COLOR = 'rgba(208, 226, 211, 0.5)' +const CONFIRMED_SELECTED_COLOR = 'rgba(208, 226, 211, 1)' const ALGOS = [ {id: 'es', label: 'Generic Elastic Search Matching'}, {id: 'llm', label: 'LLM Matching', disabled: true}, ] + +const MatchSummaryCard = ({id, icon, title, count, loading, color, selected, onClick }) => { + const isSelected = id === selected + return ( +
+ + + + + + + {icon} + + {loading && ( + + )} + + + + + + +
+ ) +} + const VisuallyHiddenInput = styled('input')({ clip: 'rect(0 0 0 0)', clipPath: 'inset(50%)', @@ -146,7 +209,6 @@ const TableCellAction = ({ isEditing, onEdit, onSave, sx }) => { } const RepoConceptsMatch = () => { - const { t } = useTranslation() const location = useLocation() const history = useHistory() const params = useParams() @@ -156,7 +218,18 @@ const RepoConceptsMatch = () => { const [data, setData] = React.useState(false); const [columns, setColumns] = React.useState([]); const [searchedRows, setSearchedRows] = React.useState(false); + const [matchedConcepts, setMatchedConcepts] = React.useState([]); + const [autoMatchCount, setAutoMatchCount] = React.useState(0); + const [highMatchCount, setHighMatchCount] = React.useState(0); + const [lowMatchCount, setLowMatchCount] = React.useState(0); + const [noMatchCount, setNoMatchCount] = React.useState(0); + const [loadingMatches, setLoadingMatches] = React.useState(false) + const [startMatchingAt, setStartMatchingAt] = React.useState(false) + const [endMatchingAt, setEndMatchingAt] = React.useState(false) + const [selectedMatchBucket, setSelectedMatchBucket] = React.useState(false) const [row, setRow] = React.useState(false) + const [rowIndex, setRowIndex] = React.useState(false) + const [decisions, setDecisions] = React.useState({}) const [confidence, setConfidence] = React.useState(false) const [conceptsResponse, setConceptsResponse] = React.useState(false) const [showItem, setShowItem] = React.useState(false) @@ -191,6 +264,15 @@ const RepoConceptsMatch = () => { reader.readAsBinaryString(file); }; + + const onGetCandidates = event => { + event.stopPropagation() + event.preventDefault() + setStartMatchingAt(moment()) + setLoadingMatches(true) + getRowsResults(searchedRows) + } + const getURL = () => ((toParentURI(location.pathname) + '/').replace('//', '/') + params.repoVersion + '/').replace('//', '/') const fetchRepo = () => { setLoading(true) @@ -272,30 +354,30 @@ const RepoConceptsMatch = () => { map(columns, (column, position) => { const isUpdatedValue = column.label !== column.original return ( - - { - isEditing ? - updateColumn(position, event.target.value)} - size='small' - fullWidth - defaultValue={column.label} - helperText={column.original} - sx={{'.MuiOutlinedInput-root': {padding: '4px 10px'}, '.MuiInputBase-input': {padding: 0}, '.MuiFormHelperText-root': {margin: 0, padding: '2px 0 0 10px', backgroundColor: isUpdatedValue ? UPDATED_COLOR : undefined}}} - /> : - {column.label} - } - - ) + + { + isEditing ? + updateColumn(position, event.target.value)} + size='small' + fullWidth + defaultValue={column.label} + helperText={column.original} + sx={{'.MuiOutlinedInput-root': {padding: '4px 10px'}, '.MuiInputBase-input': {padding: 0}, '.MuiFormHelperText-root': {margin: 0, padding: '2px 0 0 10px', backgroundColor: isUpdatedValue ? UPDATED_COLOR : undefined}}} + /> : + {column.label} + } + + ) }) } { const rowContent = (_index, _row) => { const isEditing = edit?.includes(_index) - const bgColor = isEqual(_row, row) ? SURFACE_COLORS.main : WHITE + const isConfirmed = Boolean(decisions[_index]?.length) + const bgColor = isEqual(_row, row) ? (isConfirmed ? CONFIRMED_SELECTED_COLOR : SURFACE_COLORS.main) : (isConfirmed ? CONFIRMED_COLOR : WHITE) return ( { - columns.map(column => { + map(columns, column => { const defaultValue = _row[column.dataKey] const value = has(_row, column.dataKey + '__updated') ? _row[column.dataKey + '__updated'] : defaultValue const isUpdatedValue = defaultValue !== value return ( - onCSVRowSelect(_row)} - key={column.dataKey} - > - { - isEditing ? - updateRow(_index, column.dataKey, event.target.value)} - sx={{'.MuiOutlinedInput-root': {padding: '4px 10px'}, '.MuiFormHelperText-root': {margin: 0, padding: '2px 0 0 10px', whiteSpace: 'pre-line', backgroundColor: isUpdatedValue ? UPDATED_COLOR : undefined}}} - /> : - {value} - } - + onCSVRowSelect(_row, _index)} + key={column.dataKey} + > + { + isEditing ? + updateRow(_index, column.dataKey, event.target.value)} + sx={{'.MuiOutlinedInput-root': {padding: '4px 10px'}, '.MuiFormHelperText-root': {margin: 0, padding: '2px 0 0 10px', whiteSpace: 'pre-line', backgroundColor: isUpdatedValue ? UPDATED_COLOR : undefined}}} + /> : + {value} + } + ) }) } @@ -357,13 +440,91 @@ const RepoConceptsMatch = () => { } - const onCSVRowSelect = csvRow => { + const getRowsResults = async (rows) => { + const CHUNK_SIZE = 100; // Number of rows per batch + const MAX_CONCURRENT_REQUESTS = 5; // Number of parallel API requests allowed + const rowChunks = chunk(rows, CHUNK_SIZE); + + // Function to process a single batch + const processBatch = async (rowBatch, chunkIndex) => { + const payload = { + rows: map(rowBatch, (row, i) => ({ ...prepareRow(row), index: i + chunkIndex * CHUNK_SIZE })), + target_repo_url: repo.version_url, + target_repo: { + 'owner': repo.owner, + 'owner_type': repo.owner_type, + 'source_version': repo.version || repo.id, + 'source': repo.short_code || repo.id + }, + }; + + try { + const response = await APIService.concepts() + .appendToUrl('$match/') + .post(payload, null, null, { + includeSearchMeta: true, + }); + + return response.data || []; + } catch { + return []; + } + }; + + // Function to handle concurrency + const processWithConcurrency = async () => { + const queue = rowChunks.slice(); // Copy of all chunks to be processed + const activeRequests = new Set(); + + while (queue.length > 0 || activeRequests.size > 0) { + // Fill activeRequests up to MAX_CONCURRENT_REQUESTS + while (queue.length > 0 && activeRequests.size < MAX_CONCURRENT_REQUESTS) { + const chunkIndex = rowChunks.length - queue.length; + const rowBatch = queue.shift(); + const promise = processBatch(rowBatch, chunkIndex).then((data) => { + let matchTypes = map(data, 'results.0.search_meta.match_type') + let counts = countBy(matchTypes) + setAutoMatchCount((prev) => prev + (counts?.very_high || 0)); + setHighMatchCount((prev) => prev + (counts?.high || 0)); + setLowMatchCount((prev) => prev + (counts?.low || 0)); + setNoMatchCount((prev) => prev + (sum(values(omit(counts, ['very_high', 'high', 'low']))) || 0)); + setMatchedConcepts((prev) => [...prev, ...data]); + activeRequests.delete(promise); // Remove from active set after completion + }); + activeRequests.add(promise); + } + + // Wait for at least one request to complete before continuing + await Promise.race(activeRequests); + } + }; + + await processWithConcurrency(); + setEndMatchingAt(moment()) + setLoadingMatches(false) + }; + + + + const onCSVRowSelect = (csvRow, index) => { if(edit?.length > 0) return setShowItem(false) setRow(csvRow) - let data = {row: {}, target_repo_url: repo.version_url}; + setRowIndex(index) + + let data = {rows: [prepareRow(csvRow)], target_repo_url: repo.version_url}; + APIService.concepts().appendToUrl('$match/').post(data, null, null, {includeSearchMeta: true, includeMappings: true, mappingBrief: true, mapTypes: 'SAME-AS,SAME AS,SAME_AS', verbose: true}).then(response => { + setConceptsResponse({data: response.data?.results || []}) + setTimeout(() => { + highlightTexts(response?.data || [], null, true) + }, 100) + }) + } + + const prepareRow = csvRow => { + let row = {} forEach(csvRow, (value, key) => { if(value && !has(csvRow, key + '__updated')) { key = find(columns, {original: key.replace('__updated', '')})?.label || key @@ -382,16 +543,18 @@ const RepoConceptsMatch = () => { if(newKey === 'same_as') newKey = 'same_as_map_codes' if(isList) - data.row[newKey] = [...(data.row[newKey] || []), ...newValue] + row[newKey] = [...(row[newKey] || []), ...newValue] else - data.row[newKey] = newValue + row[newKey] = newValue } }) - APIService.concepts().appendToUrl('$match/').post(data, null, null, {includeSearchMeta: true, includeMappings: true, mappingBrief: true, mapTypes: 'SAME-AS,SAME AS,SAME_AS', verbose: true}).then(response => { - setConceptsResponse(response) - setTimeout(() => { - highlightTexts(response?.data || [], null, true) - }, 100) + return row + } + + const onConceptSelect = item => { + setDecisions(prev => { + prev[rowIndex] = item + return prev }) } @@ -419,6 +582,7 @@ const RepoConceptsMatch = () => { const onCloseResults = () => { setRow(false) + setRowIndex(false) setConfidence(false) setConceptsResponse(false) setShowItem(false) @@ -511,6 +675,19 @@ const RepoConceptsMatch = () => { ) } + const showMatchSummary = Boolean(data?.length && (loadingMatches || matchedConcepts?.length)) + const getMatchingDuration = () => { + let start = startMatchingAt + let end = endMatchingAt + if(!end) + end = moment() + if(!start) + return false + return `${moment.duration(end.diff(start)).as('minutes').toFixed(2)} minutes`; + } + + const matchingDuration = getMatchingDuration() + return (
@@ -562,8 +739,67 @@ const RepoConceptsMatch = () => { } +
-
+ { + showMatchSummary && +
+ } + title="Auto Match" + count={autoMatchCount} + color='primary.main' + loading={loadingMatches} + id='very_high' + selected={selectedMatchBucket} + onClick={() => setSelectedMatchBucket('very_high')} + /> + } + title="High Match" + count={highMatchCount} + color='warning.main' + loading={loadingMatches} + id='high' + selected={selectedMatchBucket} + onClick={() => setSelectedMatchBucket('high')} + /> + } + title="Low Match" + count={lowMatchCount} + color='secondary.main' + loading={loadingMatches} + id='low' + selected={selectedMatchBucket} + onClick={() => setSelectedMatchBucket('low')} + /> + } + title="No Match" + color='error.main' + count={noMatchCount} + loading={loadingMatches} + id='no_match' + selected={selectedMatchBucket} + onClick={() => setSelectedMatchBucket('no_match')} + /> + { + Boolean(matchingDuration) && + {matchingDuration} + } +
+ } +
{ { highlightTexts([item], null, false) }, 100) }} + onSelect={onConceptSelect} + selected={decisions[rowIndex] || []} selectedToShow={showItem} extraColumns={[ { @@ -644,79 +883,12 @@ const RepoConceptsMatch = () => { { confidence?.search_meta?.search_confidence && - setConfidence(false)} - scroll='paper' - sx={{ - '& .MuiDialog-paper': { - backgroundColor: 'surface.n92', - borderRadius: '28px', - minWidth: '312px', - minHeight: '262px', - padding: 0 - } - }} - > - - {t('search.search_highlight')} - - - - { - map(confidence.search_meta.search_highlight, (values, key) => ( - - - - { - map(values, value => { - value = value.replaceAll('', '').replaceAll('', '').replaceAll(' ', ' ') - return ( - - )}) - } - - } - /> - - - )) - } - - - {confidence.search_meta.search_score} - - } - /> - - - - - setConfidence(false)} /> - - + highlight={confidence.search_meta.search_highlight} + score={confidence.search_meta.search_score} + /> } { + return ( + + } /> + + + ) +} + +export default RepoListItem; diff --git a/src/components/repos/RepoSearchAutocomplete.jsx b/src/components/repos/RepoSearchAutocomplete.jsx new file mode 100644 index 00000000..8075cb06 --- /dev/null +++ b/src/components/repos/RepoSearchAutocomplete.jsx @@ -0,0 +1,112 @@ +import React from 'react'; +import { TextField, CircularProgress, Divider } from '@mui/material'; +import Autocomplete from '@mui/material/Autocomplete'; +import { get, debounce, map } from 'lodash' +import APIService from '../../services/APIService'; +import AutocompleteLoading from '../common/AutocompleteLoading'; +import RepoListItem from './RepoListItem'; +import GroupHeader from '../common/GroupHeader' +import GroupItems from '../common/GroupItems' + + +const RepoSearchAutocomplete = ({onChange, label, id, required, minCharactersForSearch, size, suggested}) => { + const minLength = minCharactersForSearch || 2; + const [input, setInput] = React.useState('') + const [open, setOpen] = React.useState(false) + const [sources, setSources] = React.useState(map(suggested || [], instance => ({...instance, resultType: 'Suggested Sources'}))) + const [selected, setSelected] = React.useState(undefined) + const [loading, setLoading] = React.useState(false) + const isSearchable = input && input.length >= minLength; + const handleInputChange = debounce((event, value, reason) => { + setInput(value || '') + if(reason !== 'reset' && value && value.length >= minLength) + fetchSources(value) + else + setLoading(false) + }, 300) + + const handleChange = (event, id, item) => { + event.preventDefault() + event.stopPropagation() + setSelected(item) + onChange(id, item) + } + + const fetchSources = searchStr => { + setLoading(true) + setSources([]) + const query = {limit: 25, q: searchStr, includeSummary: true} + APIService.sources().get(null, null, query).then(response => { + const sources = response.data + setSources(map(sources, source => ({...source, resultType: 'Search Results'}))) + setLoading(false) + }) + } + + return ( + x} + openOnFocus + blurOnSelect + open={open} + onOpen={() => setOpen(true)} + onClose={() => setOpen(false)} + isOptionEqualToValue={(option, value) => option.url === get(value, 'url')} + value={selected || ''} + id={id || 'source'} + size={size || 'medium'} + options={sources} + loading={loading} + loadingText={ + loading ? + : + `Type atleast ${minLength} characters to search` + } + noOptionsText={(isSearchable && !loading) ? "No results" : 'Start typing...'} + getOptionLabel={option => option ? option.name : ''} + fullWidth + required={required} + onInputChange={handleInputChange} + onChange={(event, item) => handleChange(event, id || 'source', item)} + groupBy={option => option.resultType} + renderGroup={params => ( +
  • + {params.group} + {params.children} +
    • + )} + renderInput={ + params => ( + + {loading ? : null} + {params.InputProps.endAdornment} + + ), + }} + /> + ) + } + renderOption={ + (props, option) => ( + + + + + ) + } + /> + ); +} + +export default RepoSearchAutocomplete; diff --git a/src/components/search/SearchControls.jsx b/src/components/search/SearchControls.jsx index 0b10789c..a51486a7 100644 --- a/src/components/search/SearchControls.jsx +++ b/src/components/search/SearchControls.jsx @@ -5,7 +5,7 @@ import DownIcon from '@mui/icons-material/ArrowDropDown'; import DisplayMenu from './DisplayMenu'; import SortMenu from './SortMenu'; -const SearchControls = ({ disabled, onDisplayChange, display, order, orderBy, onOrderByChange, sortableFields, noCardDisplay}) => { +const SearchControls = ({ disabled, onDisplayChange, display, order, orderBy, onOrderByChange, sortableFields, noCardDisplay, extraControls}) => { const { t } = useTranslation() const [displayAnchorEl, setDisplayAnchorEl] = React.useState(null); const [sortAnchorEl, setSortAnchorEl] = React.useState(null); @@ -47,6 +47,7 @@ const SearchControls = ({ disabled, onDisplayChange, display, order, orderBy, on orderBy={orderBy} fields={sortableFields} /> + {extraControls}
    ) } diff --git a/src/components/search/SearchHighlightsDialog.jsx b/src/components/search/SearchHighlightsDialog.jsx new file mode 100644 index 00000000..0d72e7eb --- /dev/null +++ b/src/components/search/SearchHighlightsDialog.jsx @@ -0,0 +1,98 @@ +import React from 'react'; +import { useTranslation } from 'react-i18next' + +import Dialog from '@mui/material/Dialog' +import DialogTitle from '@mui/material/DialogTitle' +import DialogContent from '@mui/material/DialogContent' +import DialogActions from '@mui/material/DialogActions' +import Typography from '@mui/material/Typography' +import List from '@mui/material/List' +import ListItem from '@mui/material/ListItem' +import ListItemText from '@mui/material/ListItemText' + +import startCase from 'lodash/startCase' +import map from 'lodash/map' + +import Link from '../common/Link' + + +const SearchHighlightsDialog = ({onClose, highlight, score, open}) => { + const { t } = useTranslation() + return ( + + + {t('search.search_highlight')} + + + + { + map(highlight, (values, key) => ( + + + + { + map(values, value => { + value = value.replaceAll('', '').replaceAll('', '').replaceAll(' ', ' ') + return ( + + )}) + } + + } + /> + + + )) + } + + + {score} + + } + /> + + + + + + + + ) +} + +export default SearchHighlightsDialog diff --git a/src/components/search/SearchResults.jsx b/src/components/search/SearchResults.jsx index 44d33ac4..9c4628d7 100644 --- a/src/components/search/SearchResults.jsx +++ b/src/components/search/SearchResults.jsx @@ -71,8 +71,8 @@ const ResultsToolbar = props => { onOrderByChange={onOrderByChange} sortableFields={sortableFields} noCardDisplay={noCardDisplay} + extraControls={toolbarControl} /> - {toolbarControl} ); } @@ -177,9 +177,9 @@ const SearchResults = props => { const sortableFields = (props.nested ? SORT_ATTRS.nested[props.resource] : SORT_ATTRS.global[props.resource]) || SORT_ATTRS.common[props.resource] const resultsProps = { - handleClick: handleClick, + handleClick: props.onSelect ? handleClick : false, handleRowClick: handleRowClick, - handleSelectAllClick: handleSelectAllClick, + handleSelectAllClick: props.onSelect ? handleSelectAllClick : false, selected: selected, results: props.results, resource: props.resource, @@ -210,22 +210,25 @@ const SearchResults = props => { return ( - + { + !props.noToolbar && + + } { props.noResults ? : diff --git a/src/components/search/TableResults.jsx b/src/components/search/TableResults.jsx index afc031d8..791940d5 100644 --- a/src/components/search/TableResults.jsx +++ b/src/components/search/TableResults.jsx @@ -23,18 +23,21 @@ const EnhancedTableHead = props => { return ( - - 0 && numSelected < rowCount} - checked={rowCount > 0 && numSelected === rowCount} - onChange={onSelectAllClick} - inputProps={{ - 'aria-label': 'select all desserts', - }} - /> - + { + onSelectAllClick && + + 0 && numSelected < rowCount} + checked={rowCount > 0 && numSelected === rowCount} + onChange={onSelectAllClick} + inputProps={{ + 'aria-label': 'select all desserts', + }} + /> + + } {columns.map((headCell) => ( - handleClick(event, id)} style={{color: color}}> - - + { + handleClick && + handleClick(event, id)} style={{color: color}}> + + + } { columns.map((column, idx) => { const value = getValue(row, column)