laserninja commented on code in PR #12900: URL: https://github.com/apache/gravitino/pull/12900#discussion_r4272850839
########## web-v2/web/src/app/catalogs/rightContent/ImportSemanticModelDialog.js: ########## @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { useState } from 'react' +import { Alert, Button, Modal, Space, Typography, Upload, message } from 'antd' +import { UploadOutlined } from '@ant-design/icons' +import { importSemanticModelApi } from '@/lib/api/semanticModels' +import { useAppDispatch } from '@/lib/hooks/useStore' +import { fetchSemanticModels } from '@/lib/store/metalakes' +import { semanticModelErrorMessage } from '@/lib/utils/semanticModel' + +export default function ImportSemanticModelDialog({ open, setOpen, metalake, catalog, schema }) { + const dispatch = useAppDispatch() + const [file, setFile] = useState(null) + const [loading, setLoading] = useState(false) + const [error, setError] = useState('') + + const importDocument = async () => { + if (!file || loading) return + setLoading(true) + setError('') + try { + const document = await file.text() + if (!document.trim()) throw new Error('The selected file is empty.') + + const response = await importSemanticModelApi({ + metalake, + catalog, + schema, + document, + format: file.name.toLowerCase().endsWith('.json') ? 'json' : 'yaml' + }) + if (!response?.semanticModel) throw new Error('Import returned an empty response.') + await dispatch(fetchSemanticModels({ metalake, catalog, schema, init: true })) + message.success('Semantic Model imported') + setOpen(false) + } catch (err) { + setError(semanticModelErrorMessage(err, 'Failed to import the OSSIE document.')) + } finally { + setLoading(false) + } + } + + return ( + <Modal + title='Import OSSIE document' + open={open} + onCancel={() => setOpen(false)} + onOk={importDocument} + okText='Import' + confirmLoading={loading} + okButtonProps={{ disabled: !file, 'data-refer': 'import-semantic-model-submit' }} + cancelButtonProps={{ disabled: loading }} + closable={!loading} + maskClosable={!loading} + keyboard={!loading} + > + <Space direction='vertical' className='w-full' size='middle'> + <Typography.Paragraph className='mb-0'> + Select a standalone OSSIE YAML or JSON document to create a Semantic Model in this schema. + </Typography.Paragraph> + <Upload + accept='.yaml,.yml,.json' + maxCount={1} + disabled={loading} + fileList={file ? [file] : []} + beforeUpload={selected => { Review Comment: Implemented in 41c8e2b29. Selecting YAML or JSON now previews the parsed model name, destination, and dataset/field/relationship/metric counts. Parsing failures block import; stale file reads cannot replace a newer preview. Import still sends the original document unchanged and leaves complete validation to the server. Added parser, selection-race, and import tests plus browser verification. ########## web-v2/web/src/app/catalogs/rightContent/entitiesContent/SemanticModelDetailsPage.js: ########## @@ -0,0 +1,700 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +'use client' + +import { useMemo, useState } from 'react' +import { + Button, + Divider, + Flex, + Modal, + Popover, + Space, + Spin, + Table, + Tabs, + Tag, + Tooltip, + Typography, + message +} from 'antd' +import { useAntdColumnResize } from 'react-antd-column-resize' +import useResizeObserver from 'use-resize-observer' +import { useRouter, useSearchParams } from 'next/navigation' +import Tags from '@/components/CustomTags' +import GetOwner from '@/components/GetOwner' +import Icons from '@/components/Icons' +import PropertiesContent from '@/components/PropertiesContent' +import { useAppDispatch, useAppSelector } from '@/lib/hooks/useStore' +import { getSemanticModelDetails } from '@/lib/store/metalakes' +import { copyToClipboard } from '@/lib/utils' +import { formatToDateTime, isValidDate } from '@/lib/utils/date' +import { formatSemanticModelDefinition } from '@/lib/utils/semanticModel' +import SemanticModelAIContext from './SemanticModelAIContext' +import SemanticModelExport from './SemanticModelExport' +import SemanticModelExtensions from './SemanticModelExtensions' +import CreateSemanticModelDialog from '../CreateSemanticModelDialog' +import SemanticMetadataEditor from '../SemanticMetadataEditor' + +const { Title, Paragraph } = Typography + +const sourceFullName = source => { + if (!source) { + return '-' + } + + return [...(source.namespace || []), source.name].filter(Boolean).join('.') +} + +const renderExpression = expression => { + const dialects = expression?.dialects || [] + + if (!dialects.length) { + return <span>-</span> + } + + return ( + <div className='min-w-0 space-y-2'> + {dialects.map((dialect, index) => ( + <div key={`${dialect?.dialect}-${index}`}> + <Tag>{dialect?.dialect}</Tag> + <pre className='m-0 max-h-64 overflow-auto whitespace-pre-wrap break-words font-mono text-xs [overflow-wrap:anywhere]'> + {dialect?.expression} + </pre> + </div> + ))} + </div> + ) +} + +const topAlignedColumns = columns => + columns.map(column => ({ ...column, onCell: () => ({ style: { verticalAlign: 'top' } }) })) + +const renderColumns = columns => <span className='font-mono text-xs'>{(columns || []).join(', ') || '-'}</span> + +const hasMetadata = entity => !!entity.aiContext || entity.customExtensions?.length > 0 + +const renderMetadata = (entity, label) => ( + <> + <SemanticModelAIContext context={entity?.aiContext} label={`${label} AI Context`} /> + <SemanticModelExtensions extensions={entity?.customExtensions} label={`${label} Vendor Extensions`} /> + </> +) + +export default function SemanticModelDetailsPage() { + const searchParams = useSearchParams() + const router = useRouter() + const dispatch = useAppDispatch() + const metalake = searchParams.get('metalake') + const catalog = searchParams.get('catalog') + const schema = searchParams.get('schema') + const semanticModelName = searchParams.get('semanticModel') + const auth = useAppSelector(state => state.auth) + const { anthEnable } = auth + const store = useAppSelector(state => state.metalakes) + const semanticModel = store.activatedDetails + const [tabKey, setTabKey] = useState('Datasets') + const [openEdit, setOpenEdit] = useState(false) + const [metadataEdit, setMetadataEdit] = useState(null) + const [fieldContext, setFieldContext] = useState(null) + const { ref, width } = useResizeObserver() + + const definition = semanticModel?.definition + const properties = semanticModel?.properties + const createdAt = semanticModel?.audit?.createTime + const createdAtText = !createdAt || !isValidDate(createdAt) ? '-' : formatToDateTime(createdAt) + const lastModifiedAt = semanticModel?.audit?.lastModifiedTime + const lastModifiedText = !lastModifiedAt || !isValidDate(lastModifiedAt) ? '-' : formatToDateTime(lastModifiedAt) + const metadataObjectFullName = `${catalog}.${schema}.${semanticModelName}` + + const datasets = useMemo( + () => (definition?.datasets || []).map(dataset => ({ ...dataset, key: dataset.name })), + [definition] + ) + + const relationships = useMemo( + () => (definition?.relationships || []).map(relationship => ({ ...relationship, key: relationship.name })), + [definition] + ) + + const metrics = useMemo( + () => (definition?.metrics || []).map(metric => ({ ...metric, key: metric.name })), + [definition] + ) + + const definitionText = useMemo(() => formatSemanticModelDefinition(definition), [definition]) + + const tagContent = ( + <div> + <Tags readOnly={true} metadataObjectType={'semantic_model'} metadataObjectFullName={metadataObjectFullName} /> + </div> + ) + + const propertyContent = <PropertiesContent properties={properties} /> + + const onSaved = async updated => { + if (updated.name !== semanticModelName) { + const params = new URLSearchParams(searchParams.toString()) + params.set('semanticModel', updated.name) + router.replace(`/catalogs?${params.toString()}`) + } else { + await dispatch(getSemanticModelDetails({ metalake, catalog, schema, semanticModel: updated.name, init: true })) + } + } + + const onCopyDefinition = async () => { + try { + await copyToClipboard(definitionText) + message.success('Definition copied!') + } catch (err) { + console.error('Failed to copy definition: ', err) + message.error('Failed to copy definition') + } + } + + const datasetColumns = useMemo( + () => [ + { + title: 'Dataset Name', + dataIndex: 'name', + key: 'name', + width: 200, + ellipsis: true, + sorter: (a, b) => a.name.toLowerCase().localeCompare(b.name.toLowerCase()) + }, + { + title: 'Source', + dataIndex: 'source', + key: 'source', + width: 260, + ellipsis: true, + render: source => <span className='font-mono text-xs'>{sourceFullName(source)}</span> + }, + { + title: 'Primary Key', + dataIndex: 'primaryKey', + key: 'primaryKey', + width: 180, + ellipsis: true, + render: renderColumns + }, + { + title: 'Unique Keys', + dataIndex: 'uniqueKeys', + key: 'uniqueKeys', + width: 220, + render: keys => + keys?.length ? ( + <Space direction='vertical' size={4}> + {keys.map((columns, index) => ( + <Tag key={index}>{renderColumns(columns)}</Tag> + ))} + </Space> + ) : ( + <span>-</span> + ) + }, + { + title: 'Fields', + dataIndex: 'fields', + key: 'fields', + width: 80, + render: fields => <span>{(fields || []).length}</span> + }, + { + title: 'Description', + dataIndex: 'description', + key: 'description', + ellipsis: true, + render: description => <span>{description || '-'}</span> + }, + { + title: 'Actions', + key: 'actions', + width: 90, + render: (_, record) => ( + <Button + size='small' + aria-label={`Edit dataset ${record.name}`} + onClick={() => setMetadataEdit({ kind: 'dataset', name: record.name })} + > + Edit + </Button> + ) + } + ], + [] + ) + + const fieldColumns = useMemo( + () => [ + { + title: 'Field Name', + dataIndex: 'name', + key: 'name', + width: 220, + render: (name, field) => ( + <div className='break-words'> + <div>{name}</div> + {field.label && <Typography.Text type='secondary'>{field.label}</Typography.Text>} + <div> + {' '} + {field.aiContext && (typeof field.aiContext === 'string' || Object.keys(field.aiContext).length > 0) && ( + <Button + size='small' + type='link' + aria-label={`${field.name} AI Context`} + onClick={() => setFieldContext(field)} + > + AI Context + </Button> + )} + </div> + </div> + ) + }, + { + title: 'Data Type', + dataIndex: 'datatype', + key: 'datatype', + width: 140, + render: datatype => (datatype ? <Tag>{datatype}</Tag> : <span>-</span>) + }, + { + title: 'Time Dimension', + dataIndex: 'dimension', + key: 'dimension', + width: 140, + render: dimension => <span>{dimension?.isTime ? 'Yes' : 'No'}</span> + }, + { + title: 'Expression', + dataIndex: 'expression', + key: 'expression', + render: renderExpression + }, + { + title: 'Description', + dataIndex: 'description', + key: 'description', + ellipsis: true, + render: description => <span>{description || '-'}</span> + } + ], + [] + ) + + const relationshipColumns = useMemo( + () => [ + { + title: 'Relationship Name', + dataIndex: 'name', + key: 'name', + width: 220, + ellipsis: true, + sorter: (a, b) => a.name.toLowerCase().localeCompare(b.name.toLowerCase()) + }, + { title: 'From', dataIndex: 'from', key: 'from', width: 160, ellipsis: true }, + { + title: 'From Columns', + dataIndex: 'fromColumns', + key: 'fromColumns', + width: 200, + ellipsis: true, + render: renderColumns + }, + { title: 'To', dataIndex: 'to', key: 'to', width: 160, ellipsis: true }, + { + title: 'To Columns', + dataIndex: 'toColumns', + key: 'toColumns', + ellipsis: true, + render: renderColumns + }, + { + title: 'Actions', + key: 'actions', + width: 90, + render: (_, record) => ( + <Button + size='small' + aria-label={`Edit relationship ${record.name}`} + onClick={() => setMetadataEdit({ kind: 'relationship', name: record.name })} + > + Edit + </Button> + ) + } + ], + [] + ) + + const metricColumns = useMemo( + () => [ + { + title: 'Metric Name', + dataIndex: 'name', + key: 'name', + width: 200, + ellipsis: true, + sorter: (a, b) => a.name.toLowerCase().localeCompare(b.name.toLowerCase()) + }, + { + title: 'Data Type', + dataIndex: 'datatype', + key: 'datatype', + width: 140, + render: datatype => (datatype ? <Tag>{datatype}</Tag> : <span>-</span>) + }, + { + title: 'Expression', + dataIndex: 'expression', + key: 'expression', + render: renderExpression + }, + { + title: 'Description', + dataIndex: 'description', + key: 'description', + ellipsis: true, + render: description => <span>{description || '-'}</span> + }, + { + title: 'Actions', + key: 'actions', + width: 90, + render: (_, record) => ( + <Button + size='small' + aria-label={`Edit metric ${record.name}`} + onClick={() => setMetadataEdit({ kind: 'metric', name: record.name })} + > + Edit + </Button> + ) + } + ], + [] + ) + + const tabOptions = [ + { label: 'Datasets', key: 'Datasets' }, + { label: 'Relationships', key: 'Relationships' }, + { label: 'Metrics', key: 'Metrics' }, + { label: 'Definition', key: 'Definition' } + ] + + const { + resizableColumns: datasetResizableColumns, + components: datasetComponents, + tableWidth: datasetTableWidth + } = useAntdColumnResize(() => { + return { columns: topAlignedColumns(datasetColumns), minWidth: 100 } + }, [datasetColumns]) + + const { + resizableColumns: relationshipResizableColumns, + components: relationshipComponents, + tableWidth: relationshipTableWidth + } = useAntdColumnResize(() => { + return { columns: topAlignedColumns(relationshipColumns), minWidth: 100 } + }, [relationshipColumns]) + + const { + resizableColumns: metricResizableColumns, + components: metricComponents, + tableWidth: metricTableWidth + } = useAntdColumnResize(() => { + return { columns: topAlignedColumns(metricColumns), minWidth: 100 } + }, [metricColumns]) + + const expandedFieldsRow = dataset => ( + <div> + {renderMetadata(dataset, dataset.name)} + <Table + data-refer='semantic-model-fields-grid' + size='small' + rowKey={field => field.name} + dataSource={dataset.fields || []} + tableLayout='fixed' + scroll={{ x: 1000 }} + columns={topAlignedColumns([ + ...fieldColumns, + { + title: 'Actions', + key: 'actions', + width: 90, + render: (_, field) => ( + <Space direction='vertical' size={4}> + <Button + size='small' + aria-label={`Edit field ${field.name}`} + onClick={() => setMetadataEdit({ kind: 'field', name: field.name, datasetName: dataset.name })} + > + Edit + </Button> + </Space> + ) + } + ])} + pagination={false} + expandable={{ + expandedRowRender: field => ( + <SemanticModelExtensions extensions={field.customExtensions} label={`${field.name} Vendor Extensions`} /> + ), + rowExpandable: field => field.customExtensions?.length > 0 + }} + /> + </div> + ) + + return ( + <> + <Spin spinning={store.activatedDetailsLoading}> + <Flex className='mb-2' gap='small' align='flex-start' ref={ref}> + <div className='size-8'> + <Icons.iconify icon='mdi:graph-outline' className='my-icon-large' /> + </div> + <div className='grow-1 relative bottom-1'> + <Title level={3} style={{ marginBottom: '0.125rem' }}> + <Space> + <span + title={semanticModelName} + className='min-w-10 truncate' + style={{ maxWidth: `calc(${width}px - 90px)`, display: 'inherit' }} + > + {semanticModelName || ''} + </span> + <Tooltip title='Edit'> + <Button + type='text' + size='small' + aria-label='Edit Semantic Model' + icon={<Icons.Pencil className='size-4' />} + disabled={store.activatedDetailsLoading || !definition} + onClick={() => setOpenEdit(true)} + /> + </Tooltip> + </Space> + </Title> + <Paragraph + type='secondary' + className='truncate' + title={semanticModel?.comment} + style={{ marginBottom: 0, maxWidth: `calc(${width}px - 56px)` }} + > + {semanticModel?.comment} + </Paragraph> + </div> + </Flex> + <Space split={<Divider type='vertical' />} wrap={true} className='mb-2'> + {anthEnable && ( + <Space size={4}> + <Tooltip title='Owned' placement='top'> + <Icons.User className='size-4' color='grey' /> + </Tooltip> + <GetOwner + metalake={metalake} + metadataObjectType={'semantic_model'} + metadataObjectFullName={metadataObjectFullName} + /> + </Space> + )} + {semanticModel?.audit?.creator && ( + <Space size={4}> + <Tooltip title='Creator' placement='top'> + <Icons.UserPen className='size-4' color='grey' /> + </Tooltip> + <span>{semanticModel.audit.creator}</span> + </Space> + )} + <Space size={4}> + <Tooltip title='Created' placement='top'> + <span className='inline-flex'> + <Icons.iconify icon='mdi:clock-outline' className='size-4' props={{ color: 'grey' }} /> + </span> + </Tooltip> + <span>{createdAtText}</span> + </Space> + <Space size={4}> + <Tooltip title='Tags' placement='top'> + <Icons.Tags className='size-4' color='grey' /> + </Tooltip> + {store.currentEntityTags && store.currentEntityTags?.length > 0 ? ( + <Popover placement='bottom' title={<span>Tags</span>} content={tagContent}> + <a className='text-defaultPrimary'>{store.currentEntityTags?.length}</a> + </Popover> + ) : ( + <a className='text-defaultPrimary'>0</a> + )} + </Space> + <Space size={4}> + <Tooltip title='Properties' placement='top'> + <Icons.TableProperties className='size-4' color='grey' /> + </Tooltip> + {properties && Object.keys(properties).length > 0 ? ( + <Popover placement='bottom' title={<span>Properties</span>} content={propertyContent}> + <a className='text-defaultPrimary'>{Object.keys(properties).length}</a> + </Popover> + ) : ( + <a className='text-defaultPrimary'>0</a> + )} + </Space> + {semanticModel?.audit?.lastModifier && ( + <Space size={4} data-refer='semantic-model-last-modifier'> + <Tooltip title='Last modified by'> + <Icons.UserPen className='size-4' color='grey' /> + </Tooltip> + <span>{semanticModel.audit.lastModifier}</span> + </Space> + )} + {lastModifiedAt && ( + <Space size={4} data-refer='semantic-model-last-modified-time'> + <Tooltip title='Last modified'> + <Icons.History className='size-4' color='grey' /> + </Tooltip> + <span>{lastModifiedText}</span> + </Space> + )} + </Space> + </Spin> + <Flex justify='space-between' align='start' gap='small' wrap> + <Button className='mb-3' disabled={!definition} onClick={() => setMetadataEdit({ kind: 'model' })}> + Edit Model AI Context + </Button> + <SemanticModelExport + key={metadataObjectFullName} + metalake={metalake} + catalog={catalog} + schema={schema} + semanticModel={semanticModelName} + disabled={store.activatedDetailsLoading || !definition} + /> + </Flex> + {renderMetadata(definition, 'Model')} + <Tabs data-refer='details-tabs' activeKey={tabKey} onChange={setTabKey} items={tabOptions} /> + <Spin spinning={store.activatedDetailsLoading}> + {tabKey === 'Datasets' && ( + <Table + data-refer='semantic-model-datasets-grid' + size='small' + tableLayout='fixed' + scroll={{ x: datasetTableWidth, y: 'max(20rem, calc(100vh - 37rem))' }} + dataSource={datasets} + columns={datasetResizableColumns} + components={datasetComponents} + expandable={{ + expandedRowRender: expandedFieldsRow, + rowExpandable: dataset => (dataset.fields || []).length > 0 || hasMetadata(dataset) + }} + pagination={{ position: ['bottomCenter'], showSizeChanger: true }} + /> + )} + {tabKey === 'Relationships' && ( + <Table + data-refer='semantic-model-relationships-grid' + size='small' + tableLayout='fixed' + scroll={{ x: relationshipTableWidth, y: 'max(20rem, calc(100vh - 37rem))' }} + dataSource={relationships} + columns={relationshipResizableColumns} + components={relationshipComponents} + expandable={{ + expandedRowRender: relationship => renderMetadata(relationship, relationship.name), + rowExpandable: hasMetadata + }} + pagination={{ position: ['bottomCenter'], showSizeChanger: true }} + /> + )} + {tabKey === 'Metrics' && ( + <Table + data-refer='semantic-model-metrics-grid' + size='small' + tableLayout='fixed' + scroll={{ x: metricTableWidth, y: 'max(20rem, calc(100vh - 37rem))' }} + dataSource={metrics} Review Comment: Implemented in 41c8e2b29. Dataset details now filter fields by technical name or label, keep matches under their datasets, and automatically expand matching groups. Metrics filter by name. Both filters are case-insensitive and have explicit clear actions. Verified filtering and clearing in component tests and Chromium. ########## web-v2/web/src/app/catalogs/rightContent/entitiesContent/SemanticModelExtensions.js: ########## @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Collapse, Typography } from 'antd' + +export default function SemanticModelExtensions({ extensions, label }) { + if (!extensions?.length) return null + + return ( + <section aria-label={label} className='mb-3'> + <Typography.Title level={5}>{label}</Typography.Title> + <Collapse + size='small' Review Comment: Fixed in 41c8e2b29. AI Context and Vendor Extensions now share the same section component, heading weight, spacing, and header height. Nested labels use smaller text and the theme secondary-text color. Browser checks confirm identical peer heading typography and header heights. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
