diff --git a/requirements.txt b/requirements.txt index 85dcd6373a8..922ec732a82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,8 @@ setuptools>=50.0.0 tzdata wheel backports.zoneinfo==0.2.1 -kombu==5.2.4 -celery[redis]==5.2.7 +kombu==5.5.2 +celery[redis]==5.5.1 Django==4.2.18 mysqlclient==2.1.1 SQLAlchemy==1.2.11 @@ -12,4 +12,4 @@ pycryptodome==3.21.0 PyJWT==2.3.0 django-auth-ldap==1.2.17 jsonschema==3.2.0 -typing-extensions==4.3.0 +typing-extensions==4.12.2 diff --git a/specifyweb/frontend/js_src/lib/components/BatchEdit/index.tsx b/specifyweb/frontend/js_src/lib/components/BatchEdit/index.tsx index 48220f4716f..51b97b92dd6 100644 --- a/specifyweb/frontend/js_src/lib/components/BatchEdit/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/BatchEdit/index.tsx @@ -123,13 +123,16 @@ export function BatchEditFromQuery({ }) ); + const isDisabled = + queryFieldSpecs.some(containsSystemTables) || + queryFieldSpecs.some(hasHierarchyBaseTable) || + containsDisallowedTables(query); + return ( <> { loading( treeRanksPromise.then(async () => { @@ -205,5 +208,13 @@ const hasHierarchyBaseTable = (queryFieldSpec: QueryFieldSpec) => queryFieldSpec.baseTable.name.toLowerCase() as 'collection' ); +// Using tables.SpAuditLog here leads to an error in some cases where the tables data hasn't loaded correctly +const DISALLOWED_TABLES = ['spauditlog']; + +const containsDisallowedTables = (query: SpecifyResource) => + DISALLOWED_TABLES.some( + (tableName) => query.get('contextName').toLowerCase() === tableName + ); + // Error filters const filters = [containsFaultyNestedToMany, containsSystemTables]; diff --git a/specifyweb/frontend/js_src/lib/components/WbActions/index.tsx b/specifyweb/frontend/js_src/lib/components/WbActions/index.tsx index ac6afb9ca41..70e40984783 100644 --- a/specifyweb/frontend/js_src/lib/components/WbActions/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbActions/index.tsx @@ -58,6 +58,7 @@ export function WbActions({ useBooleanState(); const [operationCompleted, openOperationCompleted, closeOperationCompleted] = useBooleanState(); + const { mode, refreshInitiatorAborted, startUpload, triggerStatusComponent } = useWbActions({ datasetId: dataset.id, @@ -262,6 +263,10 @@ function useWbActions({ const refreshInitiatorAborted = React.useRef(false); const loading = React.useContext(LoadingContext); + /** + * NOTE: Only validate and upload use startUpload + * For rollback, we directly call the API inside the RollbackConfirmation component + */ const startUpload = (newMode: WbStatus): void => { workbench.validation.stopLiveValidation(); loading( diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx index 08732b46b1a..52bc7c93808 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx @@ -40,6 +40,7 @@ import type { MappingElementProps } from './LineComponents'; import { getMappingLineProps, MappingLineComponent } from './LineComponents'; import { columnOptionsAreDefault } from './linesGetter'; import { + BatchEditPrefsView, ChangeBaseTable, EmptyDataSetDialog, mappingOptionsMenu, @@ -102,17 +103,36 @@ export type MappingState = State< readonly autoMapperSuggestions?: RA; readonly openSelectElement?: SelectElementPosition; readonly validationResults: RA; + readonly batchEditPrefs?: BatchEditPrefs; } >; +export type ReadonlySpec = { + readonly mustMatch: boolean; + readonly columnOptions: boolean; + readonly batchEditPrefs: boolean; +}; + +export type BatchEditPrefs = { + readonly deferForNullCheck: boolean; + readonly deferForMatch: boolean; +}; + +export const DEFAULT_BATCH_EDIT_PREFS: BatchEditPrefs = { + deferForMatch: true, + deferForNullCheck: false, +} as const; + export const getDefaultMappingState = ({ changesMade, lines, mustMatchPreferences, + batchEditPrefs, }: { readonly changesMade: boolean; readonly lines: RA; readonly mustMatchPreferences: IR; + readonly batchEditPrefs?: BatchEditPrefs; }): MappingState => ({ type: 'MappingState', showHiddenFields: getCache('wbPlanViewUi', 'showHiddenFields') ?? false, @@ -124,6 +144,7 @@ export const getDefaultMappingState = ({ focusedLine: 0, changesMade, mustMatchPreferences, + batchEditPrefs, }); // REFACTOR: split component into smaller components @@ -133,12 +154,15 @@ export function Mapper(props: { readonly onChangeBaseTable: () => void; readonly onSave: ( lines: RA, - mustMatchPreferences: IR + mustMatchPreferences: IR, + batchEditPrefs?: BatchEditPrefs ) => Promise; // Initial values for the state: readonly changesMade: boolean; readonly lines: RA; readonly mustMatchPreferences: IR; + readonly readonlySpec?: ReadonlySpec; + readonly batchEditPrefs?: BatchEditPrefs; }): JSX.Element { const [state, dispatch] = React.useReducer( reducer, @@ -146,6 +170,7 @@ export function Mapper(props: { changesMade: props.changesMade, lines: props.lines, mustMatchPreferences: props.mustMatchPreferences, + batchEditPrefs: props.batchEditPrefs, }, getDefaultMappingState ); @@ -264,7 +289,13 @@ export function Mapper(props: { const validationResults = ignoreValidation ? [] : validate(); if (validationResults.length === 0) { unsetUnloadProtect(); - loading(props.onSave(state.lines, state.mustMatchPreferences)); + loading( + props.onSave( + state.lines, + state.mustMatchPreferences, + state.batchEditPrefs + ) + ); } else dispatch({ type: 'ValidationAction', @@ -294,6 +325,11 @@ export function Mapper(props: { mappingPathIsComplete(state.mappingView) && getMappedFieldsBind(state.mappingView).length === 0; + const disableSave = + props.readonlySpec === undefined + ? isReadOnly + : Object.values(props.readonlySpec).every(Boolean); + return ( - => - getMustMatchTables({ - baseTableName: props.baseTableName, - lines: state.lines, - mustMatchPreferences: state.mustMatchPreferences, - }) - } - onChange={(mustMatchPreferences): void => - dispatch({ - type: 'MustMatchPrefChangeAction', - mustMatchPreferences, - }) - } - onClose={(): void => { - /* - * Since setting table as must match causes all of its fields to - * be optional, we may have to rerun validation on - * mustMatchPreferences changes - */ - if ( - state.validationResults.length > 0 && - state.lines.some(({ mappingPath }) => - mappingPathIsComplete(mappingPath) - ) - ) + {typeof props.batchEditPrefs === 'object' ? ( + + + dispatch({ + type: 'ChangeBatchEditPrefs', + prefs, + }) + } + /> + + ) : null} + + => + getMustMatchTables({ + baseTableName: props.baseTableName, + lines: state.lines, + mustMatchPreferences: state.mustMatchPreferences, + }) + } + onChange={(mustMatchPreferences): void => dispatch({ - type: 'ValidationAction', - validationResults: validate(), - }); - }} - /> + type: 'ChangeMustMatchPrefAction', + mustMatchPreferences, + }) + } + onClose={(): void => { + /* + * Since setting table as must match causes all of its fields to + * be optional, we may have to rerun validation on + * mustMatchPreferences changes + */ + if ( + state.validationResults.length > 0 && + state.lines.some(({ mappingPath }) => + mappingPathIsComplete(mappingPath) + ) + ) + dispatch({ + type: 'ValidationAction', + validationResults: validate(), + }); + }} + /> + {!isReadOnly && ( {isReadOnly ? wbText.dataEditor() : commonText.cancel()} - {!isReadOnly && ( + {!disableSave && ( handleSave(false)} + // This is a bit complicated to resolve correctly. Each component should have its own validator.. + onClick={(): void => handleSave(isReadOnly)} > {commonText.save()} @@ -557,7 +617,7 @@ export function Mapper(props: { customSelectSubtype: 'simple', fieldsData: mappingOptionsMenu({ id: (suffix) => id(`column-options-${line}-${suffix}`), - isReadOnly, + isReadOnly: props.readonlySpec?.columnOptions ?? isReadOnly, columnOptions, onChangeMatchBehaviour: (matchBehavior) => dispatch({ diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/MapperComponents.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/MapperComponents.tsx index 80f0044c554..c7c5a7166ca 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/MapperComponents.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/MapperComponents.tsx @@ -1,8 +1,10 @@ import React from 'react'; +import type { LocalizedString } from 'typesafe-i18n'; import { useBooleanState } from '../../hooks/useBooleanState'; import { useCachedState } from '../../hooks/useCachedState'; import { useId } from '../../hooks/useId'; +import { batchEditText } from '../../localization/batchEdit'; import { commonText } from '../../localization/common'; import { schemaText } from '../../localization/schema'; import { wbPlanText } from '../../localization/wbPlan'; @@ -23,7 +25,11 @@ import type { MappingElementProps, } from './LineComponents'; import { MappingPathComponent } from './LineComponents'; -import type { MappingPath } from './Mapper'; +import { + type BatchEditPrefs, + type MappingPath, + DEFAULT_BATCH_EDIT_PREFS, +} from './Mapper'; import { getMappingLineData } from './navigator'; import { navigatorSpecs } from './navigatorSpecs'; import type { ColumnOptions, MatchBehaviors } from './uploadPlanParser'; @@ -495,3 +501,88 @@ export function MustMatch({ ); } + +export function BatchEditPrefsView({ + prefs, + onChange: handleChange, +}: { + readonly prefs: BatchEditPrefs; + readonly onChange: (prefs: BatchEditPrefs) => void; +}): JSX.Element { + // + const prefLocalization: RR< + keyof BatchEditPrefs, + { readonly title: LocalizedString; readonly description: LocalizedString } + > = { + deferForMatch: { + title: batchEditText.deferForMatch(), + description: batchEditText.deferForMatchDescription({ + default: DEFAULT_BATCH_EDIT_PREFS.deferForMatch, + }), + }, + deferForNullCheck: { + title: batchEditText.deferForNullCheck(), + description: batchEditText.deferForNullCheckDescription({ + default: DEFAULT_BATCH_EDIT_PREFS.deferForNullCheck, + }), + }, + }; + + const isReadOnly = React.useContext(ReadOnlyContext); + + const [isOpen, handleOpen, handleClose] = useBooleanState(false); + + const [localPrefs, setLocalPrefs] = React.useState(prefs); + + const isChanged = React.useMemo( + () => JSON.stringify(localPrefs) !== JSON.stringify(prefs), + [localPrefs, prefs] + ); + + const handleCommit = () => { + if (isChanged) handleChange(localPrefs); + handleClose(); + }; + + return ( + <> + + {batchEditText.batchEditPrefs()} + + {isOpen && ( + + {isChanged ? commonText.apply() : commonText.close()} + + } + className={{ container: dialogClassNames.narrowContainer }} + header={batchEditText.batchEditPrefs()} + onClose={handleCommit} + > +
+
    + {Object.entries(prefLocalization).map( + ([id, { title, description }]) => ( +
  • + + + setLocalPrefs({ ...localPrefs, [id]: isChecked }) + } + /> + {` ${title}`} + +
  • + ) + )} +
+
+
+ )} + + ); +} diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx index ece4f0c92d5..505ec512d1d 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx @@ -18,8 +18,8 @@ import { ProtectedAction } from '../Permissions/PermissionDenied'; import type { UploadResult } from '../WorkBench/resultsParser'; import { savePlan } from './helpers'; import { getLinesFromHeaders, getLinesFromUploadPlan } from './linesGetter'; -import type { MappingLine } from './Mapper'; -import { Mapper } from './Mapper'; +import type { MappingLine, ReadonlySpec } from './Mapper'; +import { DEFAULT_BATCH_EDIT_PREFS, Mapper } from './Mapper'; import { BaseTableSelection } from './State'; import type { UploadPlan } from './uploadPlanParser'; @@ -78,6 +78,7 @@ export type Dataset = DatasetBase & readonly uploadplan: UploadPlan | null; readonly visualorder: RA | null; readonly isupdate: boolean; + readonly rolledback: boolean; }; /** @@ -87,10 +88,12 @@ export function WbPlanView({ dataset, uploadPlan, headers, + readonlySpec, }: { readonly uploadPlan: UploadPlan | null; readonly headers: RA; readonly dataset: Dataset; + readonly readonlySpec?: ReadonlySpec; }): JSX.Element { useTitle(dataset.name); @@ -162,14 +165,25 @@ export function WbPlanView({ type: 'SelectBaseTable', }) } - onSave={async (lines, mustMatchPreferences): Promise => + onSave={async ( + lines, + mustMatchPreferences, + batchEditPrefs + ): Promise => savePlan({ dataset, baseTableName: state.baseTableName, lines, mustMatchPreferences, + batchEditPrefs, }).then(() => navigate(`/specify/workbench/${dataset.id}/`)) } + readonlySpec={readonlySpec} + // we add default values by simply passing in a pre-made prefs. If prefs is undefined (only classical workbench), we don't even show anything + batchEditPrefs={ + uploadPlan?.batchEditPrefs ?? + (dataset.isupdate ? DEFAULT_BATCH_EDIT_PREFS : undefined) + } /> ); } diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts index a540ff24c2b..2ce797c7144 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts @@ -17,6 +17,7 @@ import { AutoMapper } from './autoMapper'; import { renameNewlyCreatedHeaders } from './headerHelper'; import type { AutoMapperSuggestion, + BatchEditPrefs, MappingLine, MappingPath, SelectElementPosition, @@ -43,11 +44,13 @@ export async function savePlan({ baseTableName, lines, mustMatchPreferences, + batchEditPrefs, }: { readonly dataset: Dataset; readonly baseTableName: keyof Tables; readonly lines: RA; readonly mustMatchPreferences: IR; + readonly batchEditPrefs?: BatchEditPrefs; }): Promise { const renamedLines = renameNewlyCreatedHeaders( baseTableName, @@ -67,7 +70,8 @@ export async function savePlan({ const uploadPlan = uploadPlanBuilder( baseTableName, renamedLines, - getMustMatchTables({ baseTableName, lines, mustMatchPreferences }) + getMustMatchTables({ baseTableName, lines, mustMatchPreferences }), + batchEditPrefs ); const dataSetRequestUrl = `/api/workbench/dataset/${dataset.id}/`; diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/index.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/index.tsx index cbe523559c7..19db9757354 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/index.tsx @@ -15,21 +15,15 @@ import { f } from '../../utils/functools'; import { ReadOnlyContext } from '../Core/Contexts'; import { useMenuItem } from '../Header/MenuContext'; import { treeRanksPromise } from '../InitialContext/treeRanks'; -import { hasPermission } from '../Permissions/helpers'; import { NotFoundView } from '../Router/NotFoundView'; +import { resolveVariantFromDataset } from '../WbUtils/datasetVariants'; import type { Dataset } from './Wrapped'; import { WbPlanView } from './Wrapped'; const fetchTreeRanks = async (): Promise => treeRanksPromise.then(f.true); -/** - * Entrypoint React component for the workbench mapper - */ export function WbPlanViewWrapper(): JSX.Element | null { const { id = '' } = useParams(); - const [treeRanksLoaded = false] = useAsyncState(fetchTreeRanks, true); - useMenuItem('workBench'); - const [dataSet] = useAsyncState( React.useCallback(async () => { const dataSetId = f.parseInt(id); @@ -41,19 +35,41 @@ export function WbPlanViewWrapper(): JSX.Element | null { }, [id]), true ); - useErrorContext('dataSet', dataSet); + return dataSet === false ? ( + + ) : dataSet === undefined ? null : ( + + ); +} +/** + * Entrypoint React component for the workbench mapper + */ +function WbPlanViewSafe({ + dataSet, +}: { + readonly dataSet: Dataset; +}): JSX.Element | null { + const [treeRanksLoaded = false] = useAsyncState(fetchTreeRanks, true); + useMenuItem(dataSet.isupdate ? 'batchEdit' : 'workBench'); + useErrorContext('dataSet', dataSet); const isReadOnly = React.useContext(ReadOnlyContext) || - !hasPermission('/workbench/dataset', 'update') || + !resolveVariantFromDataset(dataSet).canEdit() || typeof dataSet !== 'object' || - dataSet.uploadresult?.success === true; + dataSet.uploadresult?.success === true || + // FEATURE: Remove this + dataSet.isupdate; - return dataSet === false ? ( - - ) : treeRanksLoaded && typeof dataSet === 'object' ? ( + return treeRanksLoaded && typeof dataSet === 'object' ? ( dataSet.columns[physicalCol] ) } - uploadPlan={dataSet.uploadplan} /> ) : null; diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingReducer.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingReducer.ts index 6191af73dba..c59e2bccd55 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingReducer.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingReducer.ts @@ -21,6 +21,7 @@ import { import { defaultColumnOptions, getLinesFromHeaders } from './linesGetter'; import type { AutoMapperSuggestion, + BatchEditPrefs, MappingLine, MappingPath, MappingState, @@ -120,8 +121,8 @@ type ValidationResultClickAction = Action< } >; -type MustMatchPrefChangeAction = Action< - 'MustMatchPrefChangeAction', +type ChangMustMatchPrefAction = Action< + 'ChangeMustMatchPrefAction', { readonly mustMatchPreferences: IR; } @@ -163,19 +164,27 @@ type ReRunAutoMapperAction = Action< } >; +type ChangeBatchEditPrefs = Action< + 'ChangeBatchEditPrefs', + { + readonly prefs: BatchEditPrefs; + } +>; + export type MappingActions = | AddNewHeaderAction | AutoMapperSuggestionSelectedAction | AutoMapperSuggestionsLoadedAction + | ChangeBatchEditPrefs | ChangeDefaultValueAction | ChangeMatchBehaviorAction | ChangeSelectElementValueAction + | ChangMustMatchPrefAction | ClearMappingLineAction | ClearValidationAction | CloseSelectElementAction | FocusLineAction | MappingViewMapAction - | MustMatchPrefChangeAction | OpenSelectElementAction | ReRunAutoMapperAction | ResetMappingsAction @@ -187,6 +196,7 @@ export type MappingActions = | ValidationResultClickAction; export const reducer = generateReducer({ + /* Workbench Actions (Shared with Batch-Edit) */ ToggleMappingViewAction: ({ state, action }) => ({ ...state, // REFACTOR: replace setState calls in reducers with useCachedState hooks @@ -345,7 +355,7 @@ export const reducer = generateReducer({ ...state, mappingView: mappingPath, }), - MustMatchPrefChangeAction: ({ state, action }) => ({ + ChangeMustMatchPrefAction: ({ state, action }) => ({ ...state, changesMade: true, mustMatchPreferences: action.mustMatchPreferences, @@ -401,4 +411,10 @@ export const reducer = generateReducer({ lines, }; }, + /* Batch-Edit Specific Actions */ + ChangeBatchEditPrefs: ({ state, action }) => ({ + ...state, + changesMade: true, + batchEditPrefs: action.prefs, + }), }); diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/modelHelpers.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/modelHelpers.ts index 36a4ca97c62..d062cc0f85d 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/modelHelpers.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/modelHelpers.ts @@ -148,3 +148,10 @@ export const isCircularRelationship = ( parentRelationship.otherSideName === relationship.name) || (relationship.relatedTable === parentRelationship.table && relationship.otherSideName === parentRelationship.name); + +export const isNestedToMany = ( + parentRelationship: Relationship, + relationship: Relationship +) => + relationshipIsToMany(relationship) && + relationshipIsToMany(parentRelationship); diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index 636526249f6..960b9a54fa3 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -6,7 +6,9 @@ import { strictGetTable } from '../DataModel/tables'; import type { Tables } from '../DataModel/types'; import { getTreeDefinitions, isTreeTable } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; +import type { BatchEditPrefs } from './Mapper'; import type { SplitMappingPath } from './mappingHelpers'; +import { valueIsTreeMeta } from './mappingHelpers'; import { getNameFromTreeDefinitionName, valueIsTreeDefinition, @@ -161,7 +163,11 @@ const toUploadable = ( mustMatchPreferences: RA, isRoot = false ): Uploadable => - isTreeTable(table.name) + /** + * NOTE: pathHasTreeMeta handles the case where a Batch Edit dataset has a mapping line with a (any rank) query + * A mapping line like that requires a uploadTable record rather than a treeRecord in the upload plan + */ + isTreeTable(table.name) && pathHasTreeMeta(lines) ? Object.fromEntries([ [ mustMatchPreferences.includes(table.name) @@ -179,13 +185,19 @@ const toUploadable = ( ] as const, ]); +const pathHasTreeMeta = (lines: RA) => + lines.some(({ mappingPath }) => + mappingPath.some((value) => valueIsTreeMeta(value)) + ); + /** * Build an upload plan from individual mapping lines */ export const uploadPlanBuilder = ( baseTableName: keyof Tables, lines: RA, - mustMatchPreferences: RR + mustMatchPreferences: RR, + batchEditPrefs?: BatchEditPrefs ): UploadPlan => ({ baseTableName: toLowerCase(baseTableName), uploadable: toUploadable( @@ -196,6 +208,7 @@ export const uploadPlanBuilder = ( .map(([tableName]) => tableName), true ), + batchEditPrefs, }); const indexMappings = ( diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts index 6126608713a..d6b4960370d 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts @@ -6,7 +6,7 @@ import type { Tables } from '../DataModel/types'; import { softFail } from '../Errors/Crash'; import { getTreeDefinitions } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; -import type { MappingPath } from './Mapper'; +import type { BatchEditPrefs, MappingPath } from './Mapper'; import type { SplitMappingPath } from './mappingHelpers'; import { formatTreeDefinition } from './mappingHelpers'; import { formatToManyIndex, formatTreeRank } from './mappingHelpers'; @@ -57,6 +57,7 @@ export type Uploadable = TreeRecordVariety | UploadTableVariety; export type UploadPlan = { readonly baseTableName: Lowercase; readonly uploadable: Uploadable; + readonly batchEditPrefs?: BatchEditPrefs; }; const parseColumnOptions = (matchingOptions: ColumnOptions): ColumnOptions => ({ diff --git a/specifyweb/frontend/js_src/lib/components/WbUtils/Navigation.tsx b/specifyweb/frontend/js_src/lib/components/WbUtils/Navigation.tsx index 33da8f77aae..028bc257c3e 100644 --- a/specifyweb/frontend/js_src/lib/components/WbUtils/Navigation.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbUtils/Navigation.tsx @@ -63,9 +63,14 @@ export function Navigation({ }, [totalCount]); const isDisabled = - !['newCells', 'searchResults', 'updatedCells', 'deletedCells'].includes( - name - ) && isReadOnly; + ![ + 'newCells', + 'searchResults', + 'updatedCells', + 'deletedCells', + 'matchedAndChangedCells', + ].includes(name) && isReadOnly; + return ( {wbText.dataSetUploadedLabel()} )} + {dataset.isupdate && dataset.rolledback && ( + + {batchEditText.cannotEditAfterRollback()} + + )} {getField(tables.WorkbenchTemplateMappingItem, 'metaData').label} diff --git a/specifyweb/frontend/js_src/lib/components/WorkBench/RecordSet.tsx b/specifyweb/frontend/js_src/lib/components/WorkBench/RecordSet.tsx index 251fe745425..32932037b88 100644 --- a/specifyweb/frontend/js_src/lib/components/WorkBench/RecordSet.tsx +++ b/specifyweb/frontend/js_src/lib/components/WorkBench/RecordSet.tsx @@ -31,8 +31,13 @@ export function CreateRecordSetButton({ }): JSX.Element { const [isOpen, handleOpen, handleClose] = useBooleanState(); const ButtonComponent = small ? Button.Small : Button.Info; + const wbVariant = isUpdate ? 'batch_edit' : 'workbench'; + return ( - + {queryText.createRecordSet({ diff --git a/specifyweb/frontend/js_src/lib/components/WorkBench/WbSpreadsheet.tsx b/specifyweb/frontend/js_src/lib/components/WorkBench/WbSpreadsheet.tsx index d9d57c0dabf..404bbddf63c 100644 --- a/specifyweb/frontend/js_src/lib/components/WorkBench/WbSpreadsheet.tsx +++ b/specifyweb/frontend/js_src/lib/components/WorkBench/WbSpreadsheet.tsx @@ -41,6 +41,7 @@ function WbSpreadsheetComponent({ workbench, mappings, isResultsOpen, + hasBatchEditRolledBack, checkDeletedFail, spreadsheetChanged, onClickDisambiguate: handleClickDisambiguate, @@ -53,6 +54,7 @@ function WbSpreadsheetComponent({ readonly workbench: Workbench; readonly mappings: WbMapping | undefined; readonly isResultsOpen: boolean; + readonly hasBatchEditRolledBack: boolean; readonly checkDeletedFail: (statusCode: number) => boolean; readonly spreadsheetChanged: () => void; readonly onClickDisambiguate: () => void; @@ -173,7 +175,7 @@ function WbSpreadsheetComponent({ }; React.useEffect(() => { - if (hot === undefined) return; + if (hot === undefined || hasBatchEditRolledBack) return; hot.batch(() => { (mappings === undefined ? Promise.resolve({}) diff --git a/specifyweb/frontend/js_src/lib/components/WorkBench/WbView.tsx b/specifyweb/frontend/js_src/lib/components/WorkBench/WbView.tsx index 29995e8ba89..c03ba4b5d2a 100644 --- a/specifyweb/frontend/js_src/lib/components/WorkBench/WbView.tsx +++ b/specifyweb/frontend/js_src/lib/components/WorkBench/WbView.tsx @@ -169,9 +169,17 @@ export function WbView({ const searchRef = React.useRef(null); + const hasBatchEditRolledBack = dataset.rolledback && dataset.isupdate; + return (
- {/* NOTE: Data Mapper temporarily disabled in #5413 */} - {!dataset.isupdate && (canUpdate || isMapped) ? ( + {canUpdate || isMapped ? ( {wbPlanText.dataMapper()} @@ -229,6 +236,7 @@ export function WbView({ checkDeletedFail={checkDeletedFail} data={data} dataset={dataset} + hasBatchEditRolledBack={hasBatchEditRolledBack} hot={hot} isResultsOpen={showResults} isUploaded={isUploaded} diff --git a/specifyweb/frontend/js_src/lib/components/WorkBench/hotProps.tsx b/specifyweb/frontend/js_src/lib/components/WorkBench/hotProps.tsx index d3a06cbbb2d..c156a0ed532 100644 --- a/specifyweb/frontend/js_src/lib/components/WorkBench/hotProps.tsx +++ b/specifyweb/frontend/js_src/lib/components/WorkBench/hotProps.tsx @@ -3,6 +3,7 @@ import ReactDOMServer from 'react-dom/server'; import { wbPlanText } from '../../localization/wbPlan'; import { icons } from '../Atoms/Icons'; +import { ReadOnlyContext } from '../Core/Contexts'; import { TableIcon } from '../Molecules/TableIcon'; import { userPreferences } from '../Preferences/userPreferences'; import type { Dataset } from '../WbPlanView/Wrapped'; @@ -26,6 +27,7 @@ export function useHotProps({ readonly mappings: WbMapping | undefined; readonly physicalColToMappingCol: (physicalCol: number) => number | undefined; }) { + const isReadOnly = React.useContext(ReadOnlyContext); const [autoWrapCol] = userPreferences.use( 'workBench', 'editor', @@ -46,12 +48,12 @@ export function useHotProps({ (_, physicalCol) => ({ // Get data from nth column for nth column data: physicalCol, - readOnly: [-1, undefined].includes( - physicalColToMappingCol(physicalCol) - ), + readOnly: + isReadOnly || + [-1, undefined].includes(physicalColToMappingCol(physicalCol)), }) ), - [dataset.columns.length] + [dataset.columns.length, isReadOnly] ); const [enterMovesPref] = userPreferences.use( diff --git a/specifyweb/frontend/js_src/lib/components/WorkBench/resultsParser.ts b/specifyweb/frontend/js_src/lib/components/WorkBench/resultsParser.ts index e6dc78fb01e..8c1dc83d106 100644 --- a/specifyweb/frontend/js_src/lib/components/WorkBench/resultsParser.ts +++ b/specifyweb/frontend/js_src/lib/components/WorkBench/resultsParser.ts @@ -154,12 +154,15 @@ type PropagatedFailure = State<'PropagatedFailure'>; type MatchedAndChanged = State<'MatchedAndChanged', Omit>; type RecordResultTypes = + | Deleted | Deleted | FailedBusinessRule | Matched | MatchedAndChanged + | MatchedAndChanged | MatchedMultiple | NoChange + | NoChange | NoMatch | NullRecord | ParseFailures @@ -269,6 +272,10 @@ export function resolveValidationMessage( return backEndText.fieldRequiredByUploadPlan(); else if (key === 'invalidTreeStructure') return backEndText.invalidTreeStructure(); + else if (key === 'scopeChangeError') return backEndText.scopeChangeDetected(); + else if (key === 'multipleTreeDefsInRow') + return backEndText.multipleTreeDefsInRow(); + else if (key === 'invalidCotype') return backEndText.invalidCotype(); else if (key === 'missingRequiredTreeParent') return backEndText.missingRequiredTreeParent({ names: formatConjunction((payload.names as RA) ?? []), diff --git a/specifyweb/frontend/js_src/lib/localization/backEnd.ts b/specifyweb/frontend/js_src/lib/localization/backEnd.ts index f90ef809b6f..a3ca264fb7c 100644 --- a/specifyweb/frontend/js_src/lib/localization/backEnd.ts +++ b/specifyweb/frontend/js_src/lib/localization/backEnd.ts @@ -687,4 +687,14 @@ export const backEndText = createDictionary({ 'uk-ua': 'Таблиця уже завантажена', 'de-ch': 'Datensatz bereits hochgeladen', }, + scopeChangeDetected: { + 'en-us': + 'Scope change detected in this row. It is recommended to delete this row from the dataset', + }, + multipleTreeDefsInRow: { + 'en-us': 'Multiple tree definitions in row', + }, + invalidCotype: { + 'en-us': 'Invalid type for selected tree rank(s)', + }, } as const); diff --git a/specifyweb/frontend/js_src/lib/localization/batchEdit.ts b/specifyweb/frontend/js_src/lib/localization/batchEdit.ts index d8a75b4fd45..8e699df85f4 100644 --- a/specifyweb/frontend/js_src/lib/localization/batchEdit.ts +++ b/specifyweb/frontend/js_src/lib/localization/batchEdit.ts @@ -10,6 +10,9 @@ export const batchEditText = createDictionary({ batchEdit: { 'en-us': 'Batch Edit', }, + batchEditPrefs: { + 'en-us': 'Batch Edit Preferences', + }, numberOfRecords: { 'en-us': 'Number of records selected from the query', }, @@ -38,11 +41,11 @@ export const batchEditText = createDictionary({ 'en-us': 'Use the query builder to make a new batch edit dataset', }, showRollback: { - 'en-us': 'Show revert button', + 'en-us': 'Show rollback button', }, showRollbackDescription: { 'en-us': - 'Revert is currently an experimental feature. This preference will hide the button', + 'Rollback in Batch Edit is an experimental feature. This preference will hide the button', }, commit: { 'en-us': 'Commit', @@ -88,4 +91,26 @@ export const batchEditText = createDictionary({ batchEditRecordSetName: { 'en-us': 'BE commit of "{dataSet:string}"', }, + deferForMatch: { + 'en-us': 'Use only visible fields for match', + }, + deferForMatchDescription: { + 'en-us': + 'If true, invisible database fields will not be used for matching. Default value is {default:boolean}', + }, + deferForNullCheck: { + 'en-us': 'Use only visible fields for empty record check', + }, + deferForNullCheckDescription: { + 'en-us': + 'If true, invisible database fields will not be used for determining whether the record is empty or not. Default value is {default: boolean}', + }, + batchEditDisabled: { + 'en-us': + 'Batch Edit is disabled for system tables and scoping hierarchy tables', + }, + cannotEditAfterRollback: { + 'en-us': + '(Batch Edit datasets cannot be edited after rollback - Read Only)', + }, } as const); diff --git a/specifyweb/specify/load_datamodel.py b/specifyweb/specify/load_datamodel.py index 37f0ce999e3..6adf8bd985d 100644 --- a/specifyweb/specify/load_datamodel.py +++ b/specifyweb/specify/load_datamodel.py @@ -352,6 +352,16 @@ def __init__( self.relatedModelName = relatedModelName self.otherSideName = otherSideName + """ + Remote to-ones are one-to-one from the remote side. + i.e: The foreign key exists on the other side of the relationship + + Backend equivalent of relationshipIsRemoteToOne() + See: https://github.com/specify/specify7/pull/6073#discussion_r1915397675 + """ + def is_remote_to_one(self): + return self.type == "one-to-one" and self.column == None + def make_table(tabledef: ElementTree.Element) -> Table: iddef = tabledef.find("id") diff --git a/specifyweb/stored_queries/batch_edit.py b/specifyweb/stored_queries/batch_edit.py index 84f5eb5e52d..d2243ab99b3 100644 --- a/specifyweb/stored_queries/batch_edit.py +++ b/specifyweb/stored_queries/batch_edit.py @@ -17,12 +17,14 @@ Union, Literal, ) + from specifyweb.permissions.permissions import has_target_permission from specifyweb.specify.filter_by_col import CONCRETE_HIERARCHY from specifyweb.specify.models import datamodel from specifyweb.specify.load_datamodel import Field, Relationship, Table +from specifyweb.specify.tree_views import TREE_INFORMATION, get_all_tree_information +from specifyweb.specify.tree_utils import SPECIFY_TREES from specifyweb.specify.datamodel import is_tree_table -from specifyweb.specify.tree_views import get_all_tree_information, TREE_INFORMATION from specifyweb.stored_queries.execution import execute from specifyweb.stored_queries.queryfield import QueryField, fields_from_json from specifyweb.stored_queries.queryfieldspec import ( @@ -45,7 +47,6 @@ from jsonschema import validate from django.db import transaction -from decimal import Decimal MaybeField = Callable[[QueryFieldSpec], Optional[Field]] @@ -57,7 +58,7 @@ # REFACTOR: Break this file into smaller pieaces # TODO: Play-around with localizing -BATCH_EDIT_NULL_RECORD_DESCRIPTION = "(Not included in the query results)" +BATCH_EDIT_NULL_RECORD_DESCRIPTION = "" # TODO: add backend support for making system tables readonly BATCH_EDIT_READONLY_TABLES = [*CONCRETE_HIERARCHY] @@ -410,7 +411,7 @@ def _recur_row_plan( rel_type = ( "to_many" - if node.type.endswith("to-many") or node.type == "zero-to-one" + if node.type.endswith("to-many") or node.type == "zero-to-one" or node.is_remote_to_one() else "to_one" ) @@ -536,6 +537,7 @@ def to_many_planner(self) -> "RowPlanMap": key: RowPlanMap( batch_edit_pack=( BatchEditPack( + # NOTE: Check if default needs to be 1 here as well? order=BatchEditFieldPack(value=0), id=EMPTY_FIELD, version=EMPTY_FIELD, @@ -543,7 +545,9 @@ def to_many_planner(self) -> "RowPlanMap": if value.batch_edit_pack.order.idx is not None # only use id if order field is not present else BatchEditPack( - id=BatchEditFieldPack(value=0), + # Default value is 1 to ensure at least one to-many is added to the dataset. + # Check _extend_id_order for how this is used + id=BatchEditFieldPack(value=1), order=EMPTY_FIELD, version=EMPTY_FIELD, ) @@ -712,6 +716,8 @@ def update_to_manys(self, to_many_planner: RowPlanMap) -> RowPlanMap: } return RowPlanMap(batch_edit_pack=EMPTY_PACK, to_one=to_one, to_many=to_many) + # Responsible for extending a to-many relationship to include all to-many records in the same row + # Example: Consider a CO with 3 determinations. This function ensures all 3 determinations get added to the same row of the dataset @staticmethod def _extend_id_order( values: List["RowPlanCanonical"], @@ -954,10 +960,14 @@ def _to_upload_plan(rel_name: str, _self: "RowPlanCanonical"): ] all_headers = [*raw_headers, *to_one_headers, *to_many_headers] + def _is_anyrank_tree_relationship(name, value): + return name.lower() in SPECIFY_TREES and not isinstance(value, TreeRecord) + def _relationship_is_editable(name, value): return ( Func.is_not_empty(name, value) and name not in readonly_rels + and not _is_anyrank_tree_relationship(name, value) and not omit_relationships ) @@ -1011,7 +1021,7 @@ def run_batch_edit(collection, user, spquery, agent): recordsetid=spquery.get("recordsetid", None), fields=fields_from_json(spquery["fields"]), session_maker=models.session_context, - omit_relationships=True, + omit_relationships=False, treedefsfilter=spquery.get("treedefsfilter", None) ) (headers, rows, packs, json_upload_plan, visual_order) = run_batch_edit_query(props) @@ -1152,7 +1162,6 @@ def _get_orig_column(string_id: str): # Consider optimizing when relationships are not-editable? May not benefit actually # This permission just gets enforced here - # NOTE: Relationships disabled for issue-5413 branch to minimize scope of testing omit_relationships = props["omit_relationships"] or not has_target_permission( props["collection"].id, props["user"].id, diff --git a/specifyweb/stored_queries/tests/static/test_plan.py b/specifyweb/stored_queries/tests/static/test_plan.py index 451b192cea9..86a54eba7a0 100644 --- a/specifyweb/stored_queries/tests/static/test_plan.py +++ b/specifyweb/stored_queries/tests/static/test_plan.py @@ -222,6 +222,17 @@ "toOne": {}, "toMany": {}, }, + ], + "preparations": [ + { + "wbcols": { + "countamt": "Preparation countAmt", + "text1": "Preparation text1", + }, + "static": {}, + "toOne": {}, + "toMany": {}, + }, ] }, } diff --git a/specifyweb/stored_queries/tests/test_batch_edit.py b/specifyweb/stored_queries/tests/test_batch_edit.py index 5fa4beceff3..6d713e009fc 100644 --- a/specifyweb/stored_queries/tests/test_batch_edit.py +++ b/specifyweb/stored_queries/tests/test_batch_edit.py @@ -582,6 +582,8 @@ def test_duplicates_flattened(self): "Determination remarks #2", "Determination integer1 #3", "Determination remarks #3", + "Preparation countAmt", + "Preparation text1" ], ) @@ -618,6 +620,8 @@ def test_duplicates_flattened(self): "Determination remarks #2": None, "Determination integer1 #3": None, "Determination remarks #3": None, + "Preparation countAmt": None, + "Preparation text1": None, }, { "CollectionObject catalogNumber": "num-1", @@ -651,6 +655,8 @@ def test_duplicates_flattened(self): "Determination remarks #2": None, "Determination integer1 #3": None, "Determination remarks #3": None, + "Preparation countAmt": None, + "Preparation text1": None, }, { "CollectionObject catalogNumber": "num-2", @@ -684,6 +690,8 @@ def test_duplicates_flattened(self): "Determination remarks #2": None, "Determination integer1 #3": None, "Determination remarks #3": None, + "Preparation countAmt": None, + "Preparation text1": None, }, { "CollectionObject catalogNumber": "num-3", @@ -717,6 +725,8 @@ def test_duplicates_flattened(self): "Determination remarks #2": None, "Determination integer1 #3": None, "Determination remarks #3": None, + "Preparation countAmt": None, + "Preparation text1": None, }, { "CollectionObject catalogNumber": "num-4", @@ -750,6 +760,8 @@ def test_duplicates_flattened(self): "Determination remarks #2": "test remarks", "Determination integer1 #3": None, "Determination remarks #3": None, + "Preparation countAmt": None, + "Preparation text1": None, }, ] diff --git a/specifyweb/workbench/migrations/0008_spdataset_rolledback.py b/specifyweb/workbench/migrations/0008_spdataset_rolledback.py new file mode 100644 index 00000000000..431fd957b5e --- /dev/null +++ b/specifyweb/workbench/migrations/0008_spdataset_rolledback.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2025-04-17 15:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('workbench', '0007_spdatasetattachment'), + ] + + operations = [ + migrations.AddField( + model_name='spdataset', + name='rolledback', + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/specifyweb/workbench/models.py b/specifyweb/workbench/models.py index 378fa5b7e8a..9fa5caf09ce 100644 --- a/specifyweb/workbench/models.py +++ b/specifyweb/workbench/models.py @@ -145,6 +145,7 @@ class Spdataset(Dataset): rowresults = models.TextField(null=True) isupdate = models.BooleanField(default=False, null=True) + rolledback = models.BooleanField(default=False, null=True) # very complicated. Essentially, each batch-edit dataset gets backed by another dataset (for rollbacks). # This should be a one-to-one field, imagine the mess otherwise. @@ -163,6 +164,7 @@ def get_dataset_as_dict(self): "visualorder": self.visualorder, "rowresults": self.rowresults and json.loads(self.rowresults), "isupdate": self.isupdate == True, + "rolledback": self.rolledback == True, } ) return ds_dict diff --git a/specifyweb/workbench/tasks.py b/specifyweb/workbench/tasks.py index e4b0ee5eacb..61f9e34fe94 100644 --- a/specifyweb/workbench/tasks.py +++ b/specifyweb/workbench/tasks.py @@ -92,4 +92,5 @@ def progress(current: int, total: Optional[int]) -> None: unupload_dataset(ds, agent, progress) ds.uploaderstatus = None - ds.save(update_fields=['uploaderstatus']) \ No newline at end of file + ds.rolledback = True + ds.save(update_fields=['uploaderstatus', 'rolledback']) \ No newline at end of file diff --git a/specifyweb/workbench/upload/clone.py b/specifyweb/workbench/upload/clone.py index 614d2243e5f..007fff38d52 100644 --- a/specifyweb/workbench/upload/clone.py +++ b/specifyweb/workbench/upload/clone.py @@ -97,7 +97,7 @@ def _cloned(value, field, is_dependent): ).all() # Clone all records separatetly ] for (field, is_dependent) in marked - if is_dependent and not field.concrete + if is_dependent and not field.concrete and not (field.one_to_one or field.many_to_one) ] # Should be a relationship, but not on our side return inserted diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index e1cca803cfc..83fe69586ae 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -318,7 +318,7 @@ def _handle_multiple_or_no_treedefs(self, logger.warning(f"Multiple treedefs found in row: {targeted_treedefids}") error_col_name = ranks_columns[0].column_fullname - return self, WorkBenchParseFailure('Multiple tree definitions in row', {}, error_col_name) + return self, WorkBenchParseFailure('multipleTreeDefsInRow', {}, error_col_name) return None @@ -348,7 +348,7 @@ def get_cotype_tree_def(cotype_name: str): if len(treedefs_in_row) > 0 and cotype_treedef_id == list(treedefs_in_row)[0]: return None - return self, WorkBenchParseFailure('Invalid type for selected tree rank(s)', {}, self.cotype_column) + return self, WorkBenchParseFailure('invalidCotype', {}, self.cotype_column) def bind( self, diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index 3fd06e7b11c..14fd277c132 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -839,9 +839,10 @@ def delete_row(self, parent_obj=None) -> UploadResult: def _relationship_is_dependent(self, field_name) -> bool: django_model = self.django_model - # We could check to_one_fields, but we are not going to, because that is just redundant with is_one_to_one. - if field_name in self.toOne: - return self.toOne[field_name].is_one_to_one() + # One-to-ones can always be considered to be dependent + if field_name in self.toOne and self.toOne[field_name].is_one_to_one(): + return True + return django_model.specify_model.get_relationship(field_name).dependent @@ -973,10 +974,8 @@ def _do_upload( ) if self._has_scoping_changes(concrete_field_changes) and not self._is_scope_change_allowed(concrete_field_changes): - # I don't know what else to do. I don't think this will ever get raised. I don't know what I'll need to debug this, so showing everything. - raise Exception( - f"Attempting to change the scope of the record: {reference_record} at {self}. \n\n Diff: {concrete_field_changes}" - ) + scope_change_error = ParseFailures([WorkBenchParseFailure("scopeChangeError", {}, self.parsedFields[0].column)]) + return UploadResult(scope_change_error, {}, {}) to_one_changes = BoundUpdateTable._field_changed(reference_record, to_one_ids)