diff --git a/api/src/org/labkey/api/exp/ObjectProperty.java b/api/src/org/labkey/api/exp/ObjectProperty.java index 31233e375b0..3c7c178271a 100644 --- a/api/src/org/labkey/api/exp/ObjectProperty.java +++ b/api/src/org/labkey/api/exp/ObjectProperty.java @@ -1,437 +1,437 @@ -/* - * Copyright (c) 2005-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.exp; - -import org.apache.commons.beanutils.ConvertUtils; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.data.BeanObjectFactory; -import org.labkey.api.data.Container; -import org.labkey.api.data.MvUtil; - -import java.io.File; -import java.util.Collections; -import java.util.Date; -import java.util.Map; - -/** - * A single object-property-value triple. - * User: migra - * Date: Oct 25, 2005 - */ -public class ObjectProperty extends OntologyManager.PropertyRow -{ - private int hashCode = 0; - - // Object fields - private Container container; - private String objectURI; - private Integer objectOwnerId; - - // PropertyDescriptor - private String propertyURI; - private String name; - private String rangeURI; - private String conceptURI; - private String format; - - // ObjectProperty - private Identifiable objectValue; - private Map _childProperties; - - private AttachmentFile attachmentFile; - - // Don't delete this -- it's accessed via introspection - public ObjectProperty() - { - } - - public ObjectProperty(String objectURI, Container container, String propertyURI, String value) - { - init(objectURI, container, propertyURI, PropertyType.STRING); - this.stringValue = value; - } - - public ObjectProperty(String objectURI, Container container, String propertyURI, File value) - { - init(objectURI, container, propertyURI, PropertyType.FILE_LINK); - this.stringValue = value.getPath(); - } - - public ObjectProperty(String objectURI, Container container, String propertyURI, Date value) - { - init(objectURI, container, propertyURI, PropertyType.DATE_TIME); - this.dateTimeValue = value; - } - - public ObjectProperty(String objectURI, Container container, String propertyURI, Double value) - { - init(objectURI, container, propertyURI, PropertyType.DOUBLE); - this.floatValue = value; - } - - public ObjectProperty(String objectURI, Container container, String propertyURI, Integer value) - { - init(objectURI, container, propertyURI, PropertyType.INTEGER); - this.floatValue = value.doubleValue(); - } - - public ObjectProperty(String objectURI, Container container, String propertyURI, Identifiable value) - { - init(objectURI, container, propertyURI, PropertyType.RESOURCE); - this.stringValue = value.getLSID(); - this.objectValue = value; - } - - public ObjectProperty(String objectURI, Container container, String propertyURI, Object value) - { - this(objectURI, container, propertyURI, value, (String)null); - } - - public ObjectProperty(String objectURI, Container container, String propertyURI, Object value, String name) - { - this(objectURI, container, propertyURI, value, PropertyType.getFromClass(value.getClass()), name); - } - - public ObjectProperty(String objectURI, Container container, String propertyURI, Object value, PropertyType propertyType) - { - this(objectURI, container, propertyURI, value, propertyType, null); - } - - public ObjectProperty(String objectURI, Container container, PropertyDescriptor pd, Object value) - { - this(objectURI, container, pd.getPropertyURI(), value, pd.getPropertyType(), pd.getName()); - } - - public ObjectProperty(String objectURI, Container container, String propertyURI, Object value, PropertyType propertyType, String name) - { - init(objectURI, container, propertyURI, propertyType, value); - setName(name); - } - - - private void init(String objectURI, Container container, String propertyURI, PropertyType propertyType) - { - this.objectURI = objectURI; - this.container = container; - this.propertyURI = propertyURI; - this.typeTag = propertyType.getStorageType(); - //TODO: For resource, need to override with known type - this.rangeURI = propertyType.getTypeUri(); - } - - - // UNODNE: part of this is duplicate with PropertyRow() - private void init(String objectURI, Container container, String propertyURI, PropertyType propertyType, Object value) - { - this.objectURI = objectURI; - this.container = container; - this.propertyURI = propertyURI; - this.typeTag = propertyType.getStorageType(); - //TODO: For resource, need to override with known type - this.rangeURI = propertyType.getTypeUri(); - if (value instanceof MvFieldWrapper) - { - MvFieldWrapper wrapper = (MvFieldWrapper)value; - this.mvIndicator = wrapper.getMvIndicator(); - value = wrapper.getValue(); - } - switch (propertyType) - { - case STRING: - case MULTI_LINE: - this.stringValue = value == null ? null : value.toString(); - break; - case ATTACHMENT: - if (value instanceof AttachmentFile) - { - attachmentFile = (AttachmentFile)value; - this.stringValue = attachmentFile.getFilename(); - } - else - this.stringValue = value == null ? null : value.toString(); - break; - case FILE_LINK: - if (value instanceof File) - this.stringValue = ((File) value).getPath(); - else - this.stringValue = value == null ? null : value.toString(); - break; - case DATE: - case DATE_TIME: - if (value instanceof Date) - this.dateTimeValue = (Date) value; - else if (null != value) - this.dateTimeValue = (Date) ConvertUtils.convert(value.toString(), Date.class); - break; - case INTEGER: - if (value instanceof Integer) - this.floatValue = ((Integer) value).doubleValue(); - else if (null != value) - this.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - break; - case DOUBLE: - case FLOAT: - if (value instanceof Double) - this.floatValue = (Double) value; - else if (null != value) - this.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - break; - case BOOLEAN: - Boolean boolValue = null; - if (value instanceof Boolean) - boolValue = (Boolean)value; - else if (null != value) - boolValue = (Boolean) ConvertUtils.convert(value.toString(), Boolean.class); - this.floatValue = boolValue == Boolean.TRUE ? 1.0 : 0.0; - break; - case RESOURCE: - if (value instanceof Identifiable) - { - this.stringValue = ((Identifiable) value).getLSID(); - this.objectValue = (Identifiable) value; - } - else if (null != value) - this.stringValue = value.toString(); - - break; - case BIGINT: - if (value instanceof Long) - this.floatValue = ((Long) value).doubleValue(); - else if (null != value) - this.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - break; - case DECIMAL: - if (null != value) - this.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - break; - case BINARY: - if (null != value) - this.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - break; - default: - throw new IllegalArgumentException("Unknown property type: " + propertyType); - } - } - - public Object getValueMvAware() - { - Object value = value(); - if (mvIndicator == null) - return value; - return new MvFieldWrapper(MvUtil.getMvIndicators(container), value, mvIndicator); - } - - public Object value() - { - switch (getPropertyType()) - { - case STRING: - case MULTI_LINE: - return getStringValue(); - - case XML_TEXT: - return getStringValue(); - - case DATE: - case TIME: - case DATE_TIME: - return dateTimeValue; - - case ATTACHMENT: - return getStringValue(); - - case FILE_LINK: - String value = getStringValue(); - return value == null ? null : new File(value); - - case INTEGER: - return floatValue == null ? null : floatValue.intValue(); - - case BOOLEAN: - return floatValue == null ? null : floatValue.intValue() != 0 ? Boolean.TRUE : Boolean.FALSE; - - case DECIMAL: - case FLOAT: - case DOUBLE: - return floatValue; - - case RESOURCE: - if (null != objectValue) - return objectValue; - else - return getStringValue(); - } - - throw new IllegalStateException("Unknown data type: " + rangeURI); - } - - public PropertyType getPropertyType() - { - return PropertyType.getFromURI(getConceptURI(), getRangeURI()); - } - - public Container getContainer() - { - return container; - } - - public void setContainer(Container container) - { - this.container = container; - } - - public String getObjectURI() - { - return objectURI; - } - - public void setObjectURI(String objectURI) - { - this.objectURI = objectURI; - } - - public Integer getObjectOwnerId() - { - return objectOwnerId; - } - - public void setObjectOwnerId(Integer objectOwnerId) - { - this.objectOwnerId = objectOwnerId; - } - - public String getName() - { - return name; - } - - public void setName(String name) - { - this.name = name; - } - - public String getPropertyURI() - { - return propertyURI; - } - - public void setPropertyURI(String propertyURI) - { - this.propertyURI = propertyURI; - } - - public String getRangeURI() - { - return rangeURI; - } - - public void setRangeURI(String datatypeURI) - { - this.rangeURI = datatypeURI; - } - - public String getFormat() - { - return format; - } - - public void setFormat(String format) - { - this.format = format; - } - - public boolean equals(Object o) - { - if (null == o) - return false; - - if (!(o instanceof ObjectProperty)) - return false; - - ObjectProperty pv = (ObjectProperty) o; - if (pv.getObjectId() != objectId || !pv.getPropertyURI().equals(propertyURI)) - return false; - - Object value = value(); - - return value == null ? pv.value() == null : value.equals(pv.value()); - } - - public int hashCode() - { - if (0 == hashCode) - { - String hashString = objectURI + propertyURI + String.valueOf(value()); - hashCode = hashString.hashCode(); - } - - return hashCode; - } - - public String getConceptURI() - { - return conceptURI; - } - - public void setConceptURI(String conceptURI) - { - this.conceptURI = conceptURI; - } - - public void setChildProperties(Map childProperties) - { - _childProperties = childProperties; - } - - public Map retrieveChildProperties() - { - if (_childProperties == null) - { - if (getPropertyType() == PropertyType.RESOURCE) - { - _childProperties = OntologyManager.getPropertyObjects(getContainer(), getStringValue()); - } - else - { - _childProperties = Collections.emptyMap(); - } - } - return _childProperties; - } - - public AttachmentFile getAttachmentFile() - { - return attachmentFile; - } - - public static class ObjectPropertyObjectFactory extends BeanObjectFactory - { - public ObjectPropertyObjectFactory() - { - super(ObjectProperty.class); - } - - @Override - protected void fixupBean(ObjectProperty objProp) - { - super.fixupBean(objProp); - //Equality between Date and Timestamp doesn't work properly so make sure this is a date! - if (objProp.getPropertyType() == PropertyType.DATE_TIME && objProp.getDateTimeValue() instanceof java.sql.Timestamp) - objProp.setDateTimeValue(new Date(objProp.getDateTimeValue().getTime())); - } - } -} +/* + * Copyright (c) 2005-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.exp; + +import org.apache.commons.beanutils.ConvertUtils; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.data.BeanObjectFactory; +import org.labkey.api.data.Container; +import org.labkey.api.data.MvUtil; + +import java.io.File; +import java.util.Collections; +import java.util.Date; +import java.util.Map; + +/** + * A single object-property-value triple. + * User: migra + * Date: Oct 25, 2005 + */ +public class ObjectProperty extends OntologyManager.PropertyRow +{ + private int hashCode = 0; + + // Object fields + private Container container; + private String objectURI; + private Integer objectOwnerId; + + // PropertyDescriptor + private String propertyURI; + private String name; + private String rangeURI; + private String conceptURI; + private String format; + + // ObjectProperty + private Identifiable objectValue; + private Map _childProperties; + + private AttachmentFile attachmentFile; + + // Don't delete this -- it's accessed via introspection + public ObjectProperty() + { + } + + public ObjectProperty(String objectURI, Container container, String propertyURI, String value) + { + init(objectURI, container, propertyURI, PropertyType.STRING); + this.stringValue = value; + } + + public ObjectProperty(String objectURI, Container container, String propertyURI, File value) + { + init(objectURI, container, propertyURI, PropertyType.FILE_LINK); + this.stringValue = value.getPath(); + } + + public ObjectProperty(String objectURI, Container container, String propertyURI, Date value) + { + init(objectURI, container, propertyURI, PropertyType.DATE_TIME); + this.dateTimeValue = value; + } + + public ObjectProperty(String objectURI, Container container, String propertyURI, Double value) + { + init(objectURI, container, propertyURI, PropertyType.DOUBLE); + this.floatValue = value; + } + + public ObjectProperty(String objectURI, Container container, String propertyURI, Integer value) + { + init(objectURI, container, propertyURI, PropertyType.INTEGER); + this.floatValue = value.doubleValue(); + } + + public ObjectProperty(String objectURI, Container container, String propertyURI, Identifiable value) + { + init(objectURI, container, propertyURI, PropertyType.RESOURCE); + this.stringValue = value.getLSID(); + this.objectValue = value; + } + + public ObjectProperty(String objectURI, Container container, String propertyURI, Object value) + { + this(objectURI, container, propertyURI, value, (String)null); + } + + public ObjectProperty(String objectURI, Container container, String propertyURI, Object value, String name) + { + this(objectURI, container, propertyURI, value, PropertyType.getFromClass(value.getClass()), name); + } + + public ObjectProperty(String objectURI, Container container, String propertyURI, Object value, PropertyType propertyType) + { + this(objectURI, container, propertyURI, value, propertyType, null); + } + + public ObjectProperty(String objectURI, Container container, PropertyDescriptor pd, Object value) + { + this(objectURI, container, pd.getPropertyURI(), value, pd.getPropertyType(), pd.getName()); + } + + public ObjectProperty(String objectURI, Container container, String propertyURI, Object value, PropertyType propertyType, String name) + { + init(objectURI, container, propertyURI, propertyType, value); + setName(name); + } + + + private void init(String objectURI, Container container, String propertyURI, PropertyType propertyType) + { + this.objectURI = objectURI; + this.container = container; + this.propertyURI = propertyURI; + this.typeTag = propertyType.getStorageType(); + //TODO: For resource, need to override with known type + this.rangeURI = propertyType.getTypeUri(); + } + + + // UNODNE: part of this is duplicate with PropertyRow() + private void init(String objectURI, Container container, String propertyURI, PropertyType propertyType, Object value) + { + this.objectURI = objectURI; + this.container = container; + this.propertyURI = propertyURI; + this.typeTag = propertyType.getStorageType(); + //TODO: For resource, need to override with known type + this.rangeURI = propertyType.getTypeUri(); + if (value instanceof MvFieldWrapper) + { + MvFieldWrapper wrapper = (MvFieldWrapper)value; + this.mvIndicator = wrapper.getMvIndicator(); + value = wrapper.getValue(); + } + switch (propertyType) + { + case STRING: + case MULTI_LINE: + this.stringValue = value == null ? null : value.toString(); + break; + case ATTACHMENT: + if (value instanceof AttachmentFile) + { + attachmentFile = (AttachmentFile)value; + this.stringValue = attachmentFile.getFilename(); + } + else + this.stringValue = value == null ? null : value.toString(); + break; + case FILE_LINK: + if (value instanceof File) + this.stringValue = ((File) value).getPath(); + else + this.stringValue = value == null ? null : value.toString(); + break; + case DATE: + case DATE_TIME: + if (value instanceof Date) + this.dateTimeValue = (Date) value; + else if (null != value) + this.dateTimeValue = (Date) ConvertUtils.convert(value.toString(), Date.class); + break; + case INTEGER: + if (value instanceof Integer) + this.floatValue = ((Integer) value).doubleValue(); + else if (null != value) + this.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + break; + case DOUBLE: + case FLOAT: + if (value instanceof Double) + this.floatValue = (Double) value; + else if (null != value) + this.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + break; + case BOOLEAN: + Boolean boolValue = null; + if (value instanceof Boolean) + boolValue = (Boolean)value; + else if (null != value) + boolValue = (Boolean) ConvertUtils.convert(value.toString(), Boolean.class); + this.floatValue = boolValue == null ? null : boolValue == Boolean.TRUE ? 1.0 : 0.0; + break; + case RESOURCE: + if (value instanceof Identifiable) + { + this.stringValue = ((Identifiable) value).getLSID(); + this.objectValue = (Identifiable) value; + } + else if (null != value) + this.stringValue = value.toString(); + + break; + case BIGINT: + if (value instanceof Long) + this.floatValue = ((Long) value).doubleValue(); + else if (null != value) + this.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + break; + case DECIMAL: + if (null != value) + this.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + break; + case BINARY: + if (null != value) + this.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + break; + default: + throw new IllegalArgumentException("Unknown property type: " + propertyType); + } + } + + public Object getValueMvAware() + { + Object value = value(); + if (mvIndicator == null) + return value; + return new MvFieldWrapper(MvUtil.getMvIndicators(container), value, mvIndicator); + } + + public Object value() + { + switch (getPropertyType()) + { + case STRING: + case MULTI_LINE: + return getStringValue(); + + case XML_TEXT: + return getStringValue(); + + case DATE: + case TIME: + case DATE_TIME: + return dateTimeValue; + + case ATTACHMENT: + return getStringValue(); + + case FILE_LINK: + String value = getStringValue(); + return value == null ? null : new File(value); + + case INTEGER: + return floatValue == null ? null : floatValue.intValue(); + + case BOOLEAN: + return floatValue == null ? null : floatValue.intValue() != 0 ? Boolean.TRUE : Boolean.FALSE; + + case DECIMAL: + case FLOAT: + case DOUBLE: + return floatValue; + + case RESOURCE: + if (null != objectValue) + return objectValue; + else + return getStringValue(); + } + + throw new IllegalStateException("Unknown data type: " + rangeURI); + } + + public PropertyType getPropertyType() + { + return PropertyType.getFromURI(getConceptURI(), getRangeURI()); + } + + public Container getContainer() + { + return container; + } + + public void setContainer(Container container) + { + this.container = container; + } + + public String getObjectURI() + { + return objectURI; + } + + public void setObjectURI(String objectURI) + { + this.objectURI = objectURI; + } + + public Integer getObjectOwnerId() + { + return objectOwnerId; + } + + public void setObjectOwnerId(Integer objectOwnerId) + { + this.objectOwnerId = objectOwnerId; + } + + public String getName() + { + return name; + } + + public void setName(String name) + { + this.name = name; + } + + public String getPropertyURI() + { + return propertyURI; + } + + public void setPropertyURI(String propertyURI) + { + this.propertyURI = propertyURI; + } + + public String getRangeURI() + { + return rangeURI; + } + + public void setRangeURI(String datatypeURI) + { + this.rangeURI = datatypeURI; + } + + public String getFormat() + { + return format; + } + + public void setFormat(String format) + { + this.format = format; + } + + public boolean equals(Object o) + { + if (null == o) + return false; + + if (!(o instanceof ObjectProperty)) + return false; + + ObjectProperty pv = (ObjectProperty) o; + if (pv.getObjectId() != objectId || !pv.getPropertyURI().equals(propertyURI)) + return false; + + Object value = value(); + + return value == null ? pv.value() == null : value.equals(pv.value()); + } + + public int hashCode() + { + if (0 == hashCode) + { + String hashString = objectURI + propertyURI + String.valueOf(value()); + hashCode = hashString.hashCode(); + } + + return hashCode; + } + + public String getConceptURI() + { + return conceptURI; + } + + public void setConceptURI(String conceptURI) + { + this.conceptURI = conceptURI; + } + + public void setChildProperties(Map childProperties) + { + _childProperties = childProperties; + } + + public Map retrieveChildProperties() + { + if (_childProperties == null) + { + if (getPropertyType() == PropertyType.RESOURCE) + { + _childProperties = OntologyManager.getPropertyObjects(getContainer(), getStringValue()); + } + else + { + _childProperties = Collections.emptyMap(); + } + } + return _childProperties; + } + + public AttachmentFile getAttachmentFile() + { + return attachmentFile; + } + + public static class ObjectPropertyObjectFactory extends BeanObjectFactory + { + public ObjectPropertyObjectFactory() + { + super(ObjectProperty.class); + } + + @Override + protected void fixupBean(ObjectProperty objProp) + { + super.fixupBean(objProp); + //Equality between Date and Timestamp doesn't work properly so make sure this is a date! + if (objProp.getPropertyType() == PropertyType.DATE_TIME && objProp.getDateTimeValue() instanceof java.sql.Timestamp) + objProp.setDateTimeValue(new Date(objProp.getDateTimeValue().getTime())); + } + } +} diff --git a/api/src/org/labkey/api/study/assay/AbstractAssayTsvDataHandler.java b/api/src/org/labkey/api/study/assay/AbstractAssayTsvDataHandler.java index 5c6b4a43407..4a635179f31 100644 --- a/api/src/org/labkey/api/study/assay/AbstractAssayTsvDataHandler.java +++ b/api/src/org/labkey/api/study/assay/AbstractAssayTsvDataHandler.java @@ -1,1051 +1,1052 @@ -/* - * Copyright (c) 2008-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.study.assay; - -import org.apache.commons.beanutils.ConversionException; -import org.apache.log4j.Logger; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.Sets; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilterable; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.ForeignKey; -import org.labkey.api.data.ImportAliasable; -import org.labkey.api.data.MvUtil; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.UpdateableTableInfo; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.data.validator.ColumnValidator; -import org.labkey.api.data.validator.ColumnValidators; -import org.labkey.api.dataiterator.SimpleTranslator; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.MvColumn; -import org.labkey.api.exp.MvFieldWrapper; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.XarContext; -import org.labkey.api.exp.api.AbstractExperimentDataHandler; -import org.labkey.api.exp.api.DataType; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolApplication; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpSampleSet; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.ValidatorContext; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.qc.DataLoaderSettings; -import org.labkey.api.qc.ValidationDataHandler; -import org.labkey.api.query.PropertyValidationError; -import org.labkey.api.query.ValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.reader.ColumnDescriptor; -import org.labkey.api.reader.DataLoader; -import org.labkey.api.reader.TabLoader; -import org.labkey.api.security.User; -import org.labkey.api.study.ParticipantVisit; -import org.labkey.api.study.Study; -import org.labkey.api.study.StudyService; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.ViewBackgroundInfo; -import org.springframework.jdbc.BadSqlGrammarException; - -import java.io.File; -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; -import java.util.Set; - -/** - * User: jeckels - * Date: Jan 3, 2008 - */ -public abstract class AbstractAssayTsvDataHandler extends AbstractExperimentDataHandler implements ValidationDataHandler -{ - protected static final Object ERROR_VALUE = new Object() { - @Override - public String toString() - { - return "{AbstractAssayTsvDataHandler.ERROR_VALUE}"; - } - }; - - private static final Logger LOG = Logger.getLogger(AbstractAssayTsvDataHandler.class); - - protected abstract boolean allowEmptyData(); - - public void importFile(ExpData data, File dataFile, ViewBackgroundInfo info, Logger log, XarContext context) throws ExperimentException - { - ExpProtocolApplication sourceApplication = data.getSourceApplication(); - if (sourceApplication == null) - { - throw new ExperimentException("Cannot import a TSV without knowing its assay definition"); - } - ExpRun run = sourceApplication.getRun(); - ExpProtocol protocol = run.getProtocol(); - AssayProvider provider = AssayService.get().getProvider(protocol); - - DataLoaderSettings settings = new DataLoaderSettings(); - settings.setAllowLookupByAlternateKey(true); - - Map>> rawData = getValidationDataMap(data, dataFile, info, log, context, settings); - assert(rawData.size() <= 1); - try - { - importRows(data, info.getUser(), run, protocol, provider, rawData.values().iterator().next(), settings); - } - catch (ValidationException e) - { - throw new ExperimentException(e.toString(), e); - } - } - - public void importTransformDataMap(ExpData data, AssayRunUploadContext context, ExpRun run, List> dataMap) throws ExperimentException - { - try - { - DataLoaderSettings settings = new DataLoaderSettings(); - settings.setAllowLookupByAlternateKey(true); - importRows(data, context.getUser(), run, context.getProtocol(), context.getProvider(), dataMap, settings); - } - catch (ValidationException e) - { - throw new ExperimentException(e.toString(), e); - } - } - - @Override - public Map>> getValidationDataMap(ExpData data, File dataFile, ViewBackgroundInfo info, Logger log, XarContext context, DataLoaderSettings settings) throws ExperimentException - { - ExpProtocol protocol = data.getRun().getProtocol(); - AssayProvider provider = AssayService.get().getProvider(protocol); - - Domain dataDomain = provider.getResultsDomain(protocol); - - try (DataLoader loader = createLoaderForImport(dataFile, dataDomain, settings, true)) - { - Map>> datas = new HashMap<>(); - List> dataRows = loader.load(); - - // loader did not parse any rows - if (dataRows.isEmpty() && !settings.isAllowEmptyData() && dataDomain.getProperties().size() > 0) - throw new ExperimentException("Unable to load any rows from the input data. Please check the format of the input data to make sure it matches the assay data columns."); - if (!dataRows.isEmpty()) - adjustFirstRowOrder(dataRows, loader); - - datas.put(getDataType(), dataRows); - return datas; - } - catch (IOException ioe) - { - throw new ExperimentException("There was a problem loading the data file. " + (ioe.getMessage() == null ? "" : ioe.getMessage()), ioe); - } - } - - /** - * Creates a DataLoader that can handle missing value indicators if the columns on the domain - * are configured to support it. - * - * @throws ExperimentException - */ - public static DataLoader createLoaderForImport(File dataFile, @Nullable Domain dataDomain, DataLoaderSettings settings, boolean shouldInferTypes) throws ExperimentException - { - Map aliases = new HashMap<>(); - Set mvEnabledColumns = Sets.newCaseInsensitiveHashSet(); - Set mvIndicatorColumns = Sets.newCaseInsensitiveHashSet(); - - if (dataDomain != null) - { - List columns = dataDomain.getProperties(); - aliases = dataDomain.createImportMap(false); - for (DomainProperty col : columns) - { - if (col.isMvEnabled()) - { - // Check for all of the possible names for the column in the incoming data when deciding if we should - // check it for missing values - Set columnAliases = ImportAliasable.Helper.createImportMap(Collections.singletonList(col), false).keySet(); - mvEnabledColumns.addAll(columnAliases); - mvIndicatorColumns.add(col.getName() + MvColumn.MV_INDICATOR_SUFFIX); - } - } - } - - try - { - DataLoader loader = DataLoader.get().createLoader(dataFile, null, true, null, TabLoader.TSV_FILE_TYPE); - loader.setThrowOnErrors(settings.isThrowOnErrors()); - loader.setInferTypes(shouldInferTypes); - - for (ColumnDescriptor column : loader.getColumns()) - { - if (dataDomain != null) - { - if (mvEnabledColumns.contains(column.name)) - { - column.setMvEnabled(dataDomain.getContainer()); - } - else if (mvIndicatorColumns.contains(column.name)) - { - column.setMvIndicator(dataDomain.getContainer()); - column.clazz = String.class; - } - DomainProperty prop = aliases.get(column.name); - if (prop != null) - { - // Allow String values through if the column is a lookup and the settings allow lookups by alternate key. - // The lookup table unique indices or display column value will be used to convert the column to the lookup value. - if (!(settings.isAllowLookupByAlternateKey() && column.clazz == String.class && prop.getLookup() != null)) - { - // Otherwise, just use the expected PropertyDescriptor's column type - column.clazz = prop.getPropertyDescriptor().getPropertyType().getJavaType(); - } - } - else - { - // It's not an expected column. Is it an MV indicator column? - if (!settings.isAllowUnexpectedColumns() && !mvIndicatorColumns.contains(column.name)) - { - column.load = false; - } - } - } - - if (settings.isBestEffortConversion()) - column.errorValues = DataLoader.ERROR_VALUE_USE_ORIGINAL; - else - column.errorValues = ERROR_VALUE; - } - return loader; - - } - catch (IOException ioe) - { - throw new ExperimentException("There was a problem loading the data file. " + (ioe.getMessage() == null ? "" : ioe.getMessage()), ioe); - } - } - - /** - * Reorders the first row of the list of rows to be in original column order. This is usually enough - * to cause serializers for tsv formats to respect the original file column order. A bit of a hack but - * the way row maps are generated make it difficult to preserve order at row map generation time. - */ - private void adjustFirstRowOrder(List> dataRows, DataLoader loader) throws IOException - { - Map firstRow = dataRows.remove(0); - Map newRow = new LinkedHashMap<>(); - - for (ColumnDescriptor column : loader.getColumns()) - { - if (firstRow.containsKey(column.name)) - newRow.put(column.name, firstRow.get(column.name)); - } - dataRows.add(0, newRow); - } - - @Override - public void beforeDeleteData(List data) throws ExperimentException - { - for (ExpData d : data) - { - ExpProtocolApplication sourceApplication = d.getSourceApplication(); - if (sourceApplication != null) - { - ExpRun run = sourceApplication.getRun(); - if (run != null) - { - ExpProtocol protocol = run.getProtocol(); - AssayProvider provider = AssayService.get().getProvider(protocol); - - Domain domain; - if (provider != null) - { - domain = provider.getResultsDomain(protocol); - } - else - { - // Be tolerant of the AssayProvider no longer being available. See if we have the default - // results/data domain for TSV-style assays - try - { - domain = AbstractAssayProvider.getDomainByPrefix(protocol, ExpProtocol.ASSAY_DOMAIN_DATA); - } - catch (IllegalStateException ignored) - { - domain = null; - // Be tolerant of not finding a domain anymore, if the provider has gone away - } - } - - if (domain != null && domain.getStorageTableName() != null) - { - SQLFragment deleteSQL = new SQLFragment("DELETE FROM "); - deleteSQL.append(domain.getDomainKind().getStorageSchemaName()); - deleteSQL.append("."); - deleteSQL.append(domain.getStorageTableName()); - deleteSQL.append(" WHERE DataId = ?"); - deleteSQL.add(d.getRowId()); - - try - { - new SqlExecutor(DbSchema.get(domain.getDomainKind().getStorageSchemaName())).execute(deleteSQL); - } - catch (BadSqlGrammarException x) - { - // (18035) presumably this is an optimistic concurrency problem and the table is gone - // postgres returns 42P01 in this case... SQL Server? - if (SqlDialect.isObjectNotFoundException(x)) - { - // CONSIDER: unfortunately we can't swallow this exception, because Postgres leaves - // the connection in an unusable state - } - throw x; - } - } - } - } - } - } - - public void importRows(ExpData data, User user, ExpRun run, ExpProtocol protocol, AssayProvider provider, List> rawData) - throws ExperimentException, ValidationException - { - importRows(data, user, run, protocol, provider, rawData, null); - } - - public void importRows(ExpData data, User user, ExpRun run, ExpProtocol protocol, AssayProvider provider, List> rawData, @Nullable DataLoaderSettings settings) - throws ExperimentException, ValidationException - { - if (settings == null) - settings = new DataLoaderSettings(); - - try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) - { - Container container = data.getContainer(); - ParticipantVisitResolver resolver = createResolver(user, run, protocol, provider, container); - - Domain dataDomain = provider.getResultsDomain(protocol); - - if (rawData.size() == 0) - { - if (allowEmptyData() || dataDomain.getProperties().isEmpty()) - { - transaction.commit(); - return; - } - else - { - throw new ExperimentException("Data file contained zero data rows"); - } - } - - final ContainerFilterable dataTable = provider.createProtocolSchema(user, container, protocol, null).createDataTable(); - - Map inputMaterials = checkData(container, user, dataTable, dataDomain, rawData, settings, resolver); - - List> fileData = convertPropertyNamesToURIs(rawData, dataDomain); - - insertRowData(data, user, container, run, protocol, provider, dataDomain, fileData, dataTable); - - if (shouldAddInputMaterials()) - { - AbstractAssayProvider.addInputMaterials(run, user, inputMaterials); - } - - transaction.commit(); - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - catch (IOException e) - { - throw new ExperimentException(e); - } - } - - protected ParticipantVisitResolver createResolver(User user, ExpRun run, ExpProtocol protocol, AssayProvider provider, Container container) - throws IOException, ExperimentException - { - return AssayService.get().createResolver(user, run, protocol, provider, null); - } - - /** Insert the data into the database. Transaction is active. */ - protected void insertRowData(ExpData data, User user, Container container, ExpRun run, ExpProtocol protocol, AssayProvider provider, Domain dataDomain, List> fileData, TableInfo tableInfo) - throws SQLException, ValidationException - { - if (tableInfo instanceof UpdateableTableInfo) - { - OntologyManager.insertTabDelimited(tableInfo, container, user, new SimpleAssayDataImportHelper(data), fileData, LOG); - } - else - { - Integer id = OntologyManager.ensureObject(container, data.getLSID()); - OntologyManager.insertTabDelimited(container, user, id, - new SimpleAssayDataImportHelper(data), dataDomain, fileData, false); - } - } - - protected abstract boolean shouldAddInputMaterials(); - - // NOTE: Calls filterColumns which mutates rawData in-place - private void checkColumns(Domain dataDomain, Set actual, List missing, List unexpected, List> rawData, boolean strict) - { - Set checkSet = new CaseInsensitiveHashSet(); - List expected = dataDomain.getProperties(); - for (DomainProperty pd : expected) - { - checkSet.add(pd.getName()); - if (pd.isMvEnabled()) - checkSet.add((pd.getName() + MvColumn.MV_INDICATOR_SUFFIX)); - } - for (String col : actual) - { - if (!checkSet.contains(col)) - unexpected.add(col); - } - if (!strict) - { - if (unexpected.size() > 0) - filterColumns(dataDomain, actual, rawData); - unexpected.clear(); - } - - // Now figure out what's missing but required - Map importMap = dataDomain.createImportMap(true); - // Consider all of them initially - LinkedHashSet missingProps = new LinkedHashSet<>(expected); - - // Iterate through the ones we got - for (String col : actual) - { - // Find the property that it maps to (via name, label, import alias, etc) - DomainProperty prop = importMap.get(col); - if (prop != null) - { - // If there's a match, don't consider it missing any more - missingProps.remove(prop); - } - } - - for (DomainProperty pd : missingProps) - { - if ((pd.isRequired() || strict)) - missing.add(pd.getName()); - } - } - - // NOTE: Mutates the rawData list in-place - private void filterColumns(Domain domain, Set actual, List> rawData) - { - Map expectedKey2ActualKey = new HashMap<>(); - for (Map.Entry aliased : domain.createImportMap(true).entrySet()) - { - for (String actualKey : actual) - { - if (actualKey.equalsIgnoreCase(aliased.getKey())) - { - expectedKey2ActualKey.put(aliased.getValue().getName(), actualKey); - } - } - } - ListIterator> iter = rawData.listIterator(); - while (iter.hasNext()) - { - Map filteredMap = new HashMap<>(); - Map rawDataRow = iter.next(); - for (Map.Entry expectedAndActualKeys : expectedKey2ActualKey.entrySet()) - { - filteredMap.put(expectedAndActualKeys.getKey(), rawDataRow.get(expectedAndActualKeys.getValue())); - } - iter.set(filteredMap); - } - } - - /** - * TODO: Replace with a DataIterator pipeline - * NOTE: Mutates the rawData list in-place - * @return the set of materials that are inputs to this run - */ - private Map checkData(Container container, User user, ContainerFilterable dataTable, Domain dataDomain, List> rawData, DataLoaderSettings settings, ParticipantVisitResolver resolver) - throws ValidationException, ExperimentException - { - List missing = new ArrayList<>(); - List unexpected = new ArrayList<>(); - - Set columnNames = Collections.emptySet(); - if (rawData != null && !rawData.isEmpty() && rawData.get(0) != null) - columnNames = rawData.get(0).keySet(); - - // For now, we'll only enforce that required columns are present. In the future, we'd like to - // do a strict check first, and then present ignorable warnings. - checkColumns(dataDomain, columnNames, missing, unexpected, rawData, false); - if (!missing.isEmpty() || !unexpected.isEmpty()) - { - StringBuilder builder = new StringBuilder(); - if (!missing.isEmpty()) - { - builder.append("Expected columns were not found: "); - for (java.util.Iterator it = missing.iterator(); it.hasNext();) - { - builder.append(it.next()); - if (it.hasNext()) - builder.append(", "); - else - builder.append(". "); - } - } - if (!unexpected.isEmpty()) - { - builder.append("Unexpected columns were found: "); - for (java.util.Iterator it = unexpected.iterator(); it.hasNext();) - { - builder.append(it.next()); - if (it.hasNext()) - builder.append(", "); - } - } - throw new ValidationException(builder.toString()); - } - - DomainProperty participantPD = null; - DomainProperty specimenPD = null; - DomainProperty visitPD = null; - DomainProperty datePD = null; - DomainProperty targetStudyPD = null; - - Map sampleNameSampleSets = new HashMap<>(); - Map> sampleNamesBySampleSet = new LinkedHashMap<>(); - - Map sampleIdSampleSets = new HashMap<>(); - Map> sampleIdsBySampleSet = new LinkedHashMap<>(); - - Map> sampleNames = new HashMap<>(); - Map> sampleIds = new HashMap<>(); - - List columns = dataDomain.getProperties(); - Map> validatorMap = new HashMap<>(); - Map remapMap = new HashMap<>(); - - for (DomainProperty pd : columns) - { - // initialize the DomainProperty validator map - validatorMap.put(pd, ColumnValidators.create(null, pd)); - - if (pd.getName().equalsIgnoreCase(AbstractAssayProvider.PARTICIPANTID_PROPERTY_NAME) && - pd.getPropertyDescriptor().getPropertyType() == PropertyType.STRING) - { - participantPD = pd; - } - else if (pd.getName().equalsIgnoreCase(AbstractAssayProvider.SPECIMENID_PROPERTY_NAME) && - pd.getPropertyDescriptor().getPropertyType() == PropertyType.STRING) - { - specimenPD = pd; - } - else if (pd.getName().equalsIgnoreCase(AbstractAssayProvider.VISITID_PROPERTY_NAME) && - pd.getPropertyDescriptor().getPropertyType() == PropertyType.DOUBLE) - { - visitPD = pd; - } - else if (pd.getName().equalsIgnoreCase(AbstractAssayProvider.DATE_PROPERTY_NAME) && - pd.getPropertyDescriptor().getPropertyType() == PropertyType.DATE_TIME) - { - datePD = pd; - } - else if (pd.getName().equalsIgnoreCase(AbstractAssayProvider.TARGET_STUDY_PROPERTY_NAME) && - pd.getPropertyDescriptor().getPropertyType() == PropertyType.STRING) - { - targetStudyPD = pd; - } - else - { - ExpSampleSet ss = DefaultAssayRunCreator.getLookupSampleSet(pd, container, user); - if (ss != null) - { - if (pd.getPropertyType().getJdbcType().isText()) - { - sampleNameSampleSets.put(pd, ss); - sampleNamesBySampleSet.put(ss, new HashSet<>()); - } - else - { - sampleIdSampleSets.put(pd, ss); - sampleIdsBySampleSet.put(ss, new HashSet<>()); - } - } - else if (DefaultAssayRunCreator.isLookupToMaterials(pd)) - { - if (pd.getPropertyType().getJdbcType().isText()) - sampleNames.put(pd, new HashSet<>()); - else - sampleIds.put(pd, new HashSet<>()); - } - } - - if (dataTable != null && settings.isAllowLookupByAlternateKey()) - { - ColumnInfo column = dataTable.getColumn(pd.getName()); - ForeignKey fk = column != null ? column.getFk() : null; - if (fk != null && fk.allowImportByAlternateKey()) - { - remapMap.put(pd, new SimpleTranslator.RemapPostConvert(fk.getLookupTableInfo(), true, SimpleTranslator.RemapMissingBehavior.Error)); - } - } - } - - boolean resolveMaterials = specimenPD != null || visitPD != null || datePD != null || targetStudyPD != null; - - Set wrongTypes = new HashSet<>(); - - Map materialInputs = new LinkedHashMap<>(); - - Map aliasMap = dataDomain.createImportMap(true); - - // We want to share canonical casing between data rows, or we end up with an extra Map instance for each - // data row which can add up quickly - CaseInsensitiveHashMap caseMapping = new CaseInsensitiveHashMap<>(); - ValidatorContext validatorContext = new ValidatorContext(container, user); - - int rowNum = 0; - for (ListIterator> iter = rawData.listIterator(); iter.hasNext();) - { - rowNum++; - Collection errors = new ArrayList<>(); - - Map originalMap = iter.next(); - Map map = new CaseInsensitiveHashMap<>(caseMapping); - // Rekey the map, resolving aliases to the actual property names - for (Map.Entry entry : originalMap.entrySet()) - { - DomainProperty prop = aliasMap.get(entry.getKey()); - if (prop != null) - { - map.put(prop.getName(), entry.getValue()); - } - } - - String participantID = null; - String specimenID = null; - Double visitID = null; - Date date = null; - Container targetStudy = null; - - for (DomainProperty pd : columns) - { - Object o = map.get(pd.getName()); - if (o instanceof String) - { - o = ((String) o).trim(); - map.put(pd.getName(), o); - iter.set(map); - } - - // validate the data value - if (validatorMap.containsKey(pd)) - { - for (ColumnValidator validator : validatorMap.get(pd)) - { - String error = validator.validate(rowNum, o, validatorContext); - if (error != null) - errors.add(new PropertyValidationError(error, pd.getName())); - } - } - - if (participantPD == pd) - { - participantID = o instanceof String ? (String)o : null; - } - else if (specimenPD == pd) - { - specimenID = o instanceof String ? (String)o : null; - } - else if (visitPD == pd && o != null) - { - visitID = o instanceof Number ? ((Number)o).doubleValue() : null; - } - else if (datePD == pd && o != null) - { - date = o instanceof Date ? (Date) o : null; - } - else if (targetStudyPD == pd && o != null) - { - Set studies = StudyService.get().findStudy(o, null); - if (studies.isEmpty()) - { - errors.add(new PropertyValidationError("Couldn't resolve " + pd.getName() + " '" + o.toString() + "' to a study folder.", pd.getName())); - } - else if (studies.size() > 1) - { - errors.add(new PropertyValidationError("Ambiguous " + pd.getName() + " '" + o.toString() + "'.", pd.getName())); - } - if (!studies.isEmpty()) - { - Study study = studies.iterator().next(); - targetStudy = study != null ? study.getContainer() : null; - } - } - - boolean valueMissing; - if (o == null) - { - valueMissing = true; - } - else if (o instanceof MvFieldWrapper) - { - MvFieldWrapper mvWrapper = (MvFieldWrapper)o; - if (mvWrapper.isEmpty()) - valueMissing = true; - else - { - valueMissing = false; - if (!MvUtil.isValidMvIndicator(mvWrapper.getMvIndicator(), dataDomain.getContainer())) - { - String columnName = pd.getName() + MvColumn.MV_INDICATOR_SUFFIX; - wrongTypes.add(columnName); - errors.add(new PropertyValidationError(columnName + " must be a valid MV indicator.", columnName)); - } - } - - } - else - { - valueMissing = false; - } - - // If the column is a file link or attachment, resolve the value to a File object - String uri = pd.getType().getTypeURI(); - if (uri.equals(PropertyType.FILE_LINK.getTypeUri()) || uri.equals(PropertyType.ATTACHMENT.getTypeUri())) - { - if ("".equals(o)) - { - // Issue 36502: If the original input was an empty value, set it to null so we won't store an empty string in the database - o = null; - map.put(pd.getName(), null); - } - else - { - // File column values are stored as the absolute resolved path - File resolvedFile = AssayUploadFileResolver.resolve(o, container, pd); - if (resolvedFile != null) - { - o = resolvedFile; - map.put(pd.getName(), o); - iter.set(map); - } - } - } - - // If we have a String value for a lookup column, attempt to use the table's unique indices or display value to convert the String into the lookup value - // See similar conversion performed in SimpleTranslator.RemapPostConvertColumn - if (o instanceof String && remapMap.containsKey(pd)) - { - SimpleTranslator.RemapPostConvert remap = remapMap.get(pd); - Object remapped = null; - try - { - remapped = remap.mappedValue(o); - } - catch (ConversionException ex) - { - errors.add(new PropertyValidationError("Failed to convert '" + pd.getName() + "': " + ex.getMessage(), pd.getName())); - } - - if (o != remapped) - { - o = remapped; - map.put(pd.getName(), remapped); - iter.set(map); - } - } - - if (!valueMissing && o == ERROR_VALUE && !wrongTypes.contains(pd.getName())) - { - wrongTypes.add(pd.getName()); - errors.add(new PropertyValidationError(pd.getName() + " must be of type " + ColumnInfo.getFriendlyTypeName(pd.getPropertyDescriptor().getPropertyType().getJavaType()) + ".", pd.getName())); - } - - // Collect sample names or ids for each of the SampleSet lookup columns - ExpSampleSet byNameSS = sampleNameSampleSets.get(pd); - if (byNameSS != null && o instanceof String) - sampleNamesBySampleSet.get(byNameSS).add((String)o); - - ExpSampleSet byIdSS = sampleIdSampleSets.get(pd); - if (byIdSS != null && o instanceof Integer) - sampleIdsBySampleSet.get(byIdSS).add((Integer)o); - - if (DefaultAssayRunCreator.isLookupToMaterials(pd)) - { - if (sampleNames.containsKey(pd) && o instanceof String) - sampleNames.get(pd).add((String)o); - else if (sampleIds.containsKey(pd) && o instanceof Integer) - sampleIds.get(pd).add((Integer)o); - } - } - - if (!errors.isEmpty()) - throw new ValidationException(errors, rowNum); - - ParticipantVisit participantVisit = resolver.resolve(specimenID, participantID, visitID, date, targetStudy); - if (participantPD != null && map.get(participantPD.getName()) == null) - { - map.put(participantPD.getName(), participantVisit.getParticipantID()); - iter.set(map); - } - if (visitPD != null && map.get(visitPD.getName()) == null) - { - map.put(visitPD.getName(), participantVisit.getVisitID()); - iter.set(map); - } - if (datePD != null && map.get(datePD.getName()) == null) - { - map.put(datePD.getName(), participantVisit.getDate()); - iter.set(map); - } - if (targetStudyPD != null && participantVisit.getStudyContainer() != null) - { - // Original TargetStudy value may have been a container id, container path, or a study label. - // Store all TargetStudy values as Container ID string. - map.put(targetStudyPD.getName(), participantVisit.getStudyContainer().getId()); - iter.set(map); - } - - if (resolveMaterials) - { - materialInputs.put(participantVisit.getMaterial(), null); - } - } - - // Resolve sample lookups for each SampleSet - resolveSampleNames(container, user, sampleNameSampleSets, sampleNamesBySampleSet, materialInputs); - resolveSampleIds(sampleIdSampleSets, sampleIdsBySampleSet, materialInputs); - - // Resolve sample lookups to exp.Material - for (Map.Entry> entry : sampleNames.entrySet()) - { - resolveSampleNames(container, user, null, entry.getKey(), entry.getValue(), materialInputs); - } - for (Map.Entry> entry : sampleIds.entrySet()) - { - resolveSampleIds(null, entry.getKey(), entry.getValue(), materialInputs); - } - - return materialInputs; - } - - private void resolveSampleNames(Container container, User user, Map sampleSets, Map> sampleNamesBySampleSet, Map materialInputs) throws ExperimentException - { - for (Map.Entry> entry : sampleNamesBySampleSet.entrySet()) - { - // Default to looking in the current container - ExpSampleSet ss = entry.getKey(); - Set sampleNames = entry.getValue(); - - // Find the DomainProperty and use its name as the role - DomainProperty dp = null; - for (Map.Entry pair : sampleSets.entrySet()) - { - if (pair.getValue().equals(ss)) - { - dp = pair.getKey(); - break; - } - } - - resolveSampleNames(container, user, ss, dp, sampleNames, materialInputs); - } - } - - private void resolveSampleNames(Container container, User user, @Nullable ExpSampleSet ss, @Nullable DomainProperty dp, Set sampleNames, Map materialInputs) throws ExperimentException - { - // use DomainProperty name as the role - String role = dp == null ? null : dp.getName(); // TODO: More than one DomainProperty could be a lookup to the SampleSet - - Set searchContainers = ExpSchema.getSearchContainers(container, ss, dp, user); - - for (Container searchContainer : searchContainers) - { - List materials = ExperimentService.get().getExpMaterials(searchContainer, user, sampleNames, ss, false, false); - - for (ExpMaterial material : materials) - if (!materialInputs.containsKey(material)) - materialInputs.put(material, role); - } - } - - - private void resolveSampleIds(Map sampleSets, Map> sampleIdsBySampleSet, Map materialInputs) - { - for (Map.Entry> entry : sampleIdsBySampleSet.entrySet()) - { - ExpSampleSet ss = entry.getKey(); - Set sampleIds = entry.getValue(); - - // Find the DomainProperty and use its name as the role - DomainProperty dp = null; - for (Map.Entry pair : sampleSets.entrySet()) - { - if (pair.getValue().equals(ss)) - { - dp = pair.getKey(); - break; - } - } - - resolveSampleIds(ss, dp, sampleIds, materialInputs); - } - } - - private void resolveSampleIds(@Nullable ExpSampleSet ss, DomainProperty dp, Set sampleIds, Map materialInputs) - { - // use DomainProperty name as the role - String role = null; - if (dp != null) - { - role = dp.getName(); // TODO: More than one DomainProperty could be a lookup to the SampleSet - } - - List materials = ExperimentService.get().getExpMaterials(sampleIds); - - for (ExpMaterial material : materials) - { - // Ignore materials that aren't in the lookup sample set - if (ss != null && !ss.getLSID().equals(material.getCpasType())) - continue; - - if (!materialInputs.containsKey(material)) - materialInputs.put(material, role); - } - } - - /** Wraps each map in a version that can be queried based on on any of the aliases (name, property URI, import - * aliases, etc for a given property */ - // NOTE: Mutates the rawData list in place - protected List> convertPropertyNamesToURIs(List> dataMaps, Domain domain) - { - // Get the mapping of different names to the set of domain properties - final Map importMap = domain.createImportMap(true); - - // For a given property, find all the potential names it by which it could be referenced - final Map> propToNames = new HashMap<>(); - for (Map.Entry entry : importMap.entrySet()) - { - Set allNames = propToNames.get(entry.getValue()); - if (allNames == null) - { - allNames = new HashSet<>(); - propToNames.put(entry.getValue(), allNames); - } - allNames.add(entry.getKey()); - } - - // We want to share canonical casing between data rows, or we end up with an extra Map instance for each - // data row which can add up quickly - CaseInsensitiveHashMap caseMapping = new CaseInsensitiveHashMap<>(); - for (ListIterator> i = dataMaps.listIterator(); i.hasNext(); ) - { - Map dataMap = i.next(); - CaseInsensitiveHashMap newMap = new PropertyLookupMap(dataMap, caseMapping, importMap, propToNames); - - // Swap out the entry in the list with the transformed map - i.set(newMap); - } - return dataMaps; - } - - public void deleteData(ExpData data, Container container, User user) - { - OntologyManager.deleteOntologyObjects(container, data.getLSID()); - } - - public ActionURL getContentURL(ExpData data) - { - ExpRun run = data.getRun(); - if (run != null) - { - ExpProtocol protocol = run.getProtocol(); - return PageFlowUtil.urlProvider(AssayUrls.class).getAssayResultsURL(data.getContainer(), protocol, run.getRowId()); - } - return null; - } - - /** Wrapper around a row's key->value map that can find the values based on any of the DomainProperty's potential - * aliases, like the property name, URI, import aliases, etc */ - private static class PropertyLookupMap extends CaseInsensitiveHashMap - { - private final Map _importMap; - private final Map> _propToNames; - - public PropertyLookupMap(Map dataMap, CaseInsensitiveHashMap caseMapping, Map importMap, Map> propToNames) - { - super(dataMap, caseMapping); - _importMap = importMap; - _propToNames = propToNames; - } - - @Override - public Object get(Object key) - { - Object result = super.get(key); - - // If we can't find the value based on the name that was passed in, try any of its alternatives - if (result == null && key instanceof String) - { - // Find the property that's associated with that name - DomainProperty property = _importMap.get(key); - if (property != null) - { - // Find all of the potential synonyms - Set allNames = _propToNames.get(property); - if (allNames != null) - { - for (String name : allNames) - { - // Look for a value under that name - result = super.get(name); - if (result != null) - { - break; - } - } - } - } - } - return result; - } - } -} +/* + * Copyright (c) 2008-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.study.assay; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.Sets; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilterable; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.ForeignKey; +import org.labkey.api.data.ImportAliasable; +import org.labkey.api.data.MvUtil; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.UpdateableTableInfo; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.data.validator.ColumnValidator; +import org.labkey.api.data.validator.ColumnValidators; +import org.labkey.api.dataiterator.SimpleTranslator; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.MvColumn; +import org.labkey.api.exp.MvFieldWrapper; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.XarContext; +import org.labkey.api.exp.api.AbstractExperimentDataHandler; +import org.labkey.api.exp.api.DataType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolApplication; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpSampleSet; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.ValidatorContext; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.qc.DataLoaderSettings; +import org.labkey.api.qc.ValidationDataHandler; +import org.labkey.api.query.PropertyValidationError; +import org.labkey.api.query.ValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reader.ColumnDescriptor; +import org.labkey.api.reader.DataLoader; +import org.labkey.api.reader.TabLoader; +import org.labkey.api.security.User; +import org.labkey.api.study.ParticipantVisit; +import org.labkey.api.study.Study; +import org.labkey.api.study.StudyService; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.ViewBackgroundInfo; +import org.springframework.jdbc.BadSqlGrammarException; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; + +/** + * User: jeckels + * Date: Jan 3, 2008 + */ +public abstract class AbstractAssayTsvDataHandler extends AbstractExperimentDataHandler implements ValidationDataHandler +{ + protected static final Object ERROR_VALUE = new Object() { + @Override + public String toString() + { + return "{AbstractAssayTsvDataHandler.ERROR_VALUE}"; + } + }; + + private static final Logger LOG = Logger.getLogger(AbstractAssayTsvDataHandler.class); + + protected abstract boolean allowEmptyData(); + + public void importFile(ExpData data, File dataFile, ViewBackgroundInfo info, Logger log, XarContext context) throws ExperimentException + { + ExpProtocolApplication sourceApplication = data.getSourceApplication(); + if (sourceApplication == null) + { + throw new ExperimentException("Cannot import a TSV without knowing its assay definition"); + } + ExpRun run = sourceApplication.getRun(); + ExpProtocol protocol = run.getProtocol(); + AssayProvider provider = AssayService.get().getProvider(protocol); + + DataLoaderSettings settings = new DataLoaderSettings(); + settings.setAllowLookupByAlternateKey(true); + + Map>> rawData = getValidationDataMap(data, dataFile, info, log, context, settings); + assert(rawData.size() <= 1); + try + { + importRows(data, info.getUser(), run, protocol, provider, rawData.values().iterator().next(), settings); + } + catch (ValidationException e) + { + throw new ExperimentException(e.toString(), e); + } + } + + public void importTransformDataMap(ExpData data, AssayRunUploadContext context, ExpRun run, List> dataMap) throws ExperimentException + { + try + { + DataLoaderSettings settings = new DataLoaderSettings(); + settings.setAllowLookupByAlternateKey(true); + importRows(data, context.getUser(), run, context.getProtocol(), context.getProvider(), dataMap, settings); + } + catch (ValidationException e) + { + throw new ExperimentException(e.toString(), e); + } + } + + @Override + public Map>> getValidationDataMap(ExpData data, File dataFile, ViewBackgroundInfo info, Logger log, XarContext context, DataLoaderSettings settings) throws ExperimentException + { + ExpProtocol protocol = data.getRun().getProtocol(); + AssayProvider provider = AssayService.get().getProvider(protocol); + + Domain dataDomain = provider.getResultsDomain(protocol); + + try (DataLoader loader = createLoaderForImport(dataFile, dataDomain, settings, true)) + { + Map>> datas = new HashMap<>(); + List> dataRows = loader.load(); + + // loader did not parse any rows + if (dataRows.isEmpty() && !settings.isAllowEmptyData() && dataDomain.getProperties().size() > 0) + throw new ExperimentException("Unable to load any rows from the input data. Please check the format of the input data to make sure it matches the assay data columns."); + if (!dataRows.isEmpty()) + adjustFirstRowOrder(dataRows, loader); + + datas.put(getDataType(), dataRows); + return datas; + } + catch (IOException ioe) + { + throw new ExperimentException("There was a problem loading the data file. " + (ioe.getMessage() == null ? "" : ioe.getMessage()), ioe); + } + } + + /** + * Creates a DataLoader that can handle missing value indicators if the columns on the domain + * are configured to support it. + * + * @throws ExperimentException + */ + public static DataLoader createLoaderForImport(File dataFile, @Nullable Domain dataDomain, DataLoaderSettings settings, boolean shouldInferTypes) throws ExperimentException + { + Map aliases = new HashMap<>(); + Set mvEnabledColumns = Sets.newCaseInsensitiveHashSet(); + Set mvIndicatorColumns = Sets.newCaseInsensitiveHashSet(); + + if (dataDomain != null) + { + List columns = dataDomain.getProperties(); + aliases = dataDomain.createImportMap(false); + for (DomainProperty col : columns) + { + if (col.isMvEnabled()) + { + // Check for all of the possible names for the column in the incoming data when deciding if we should + // check it for missing values + Set columnAliases = ImportAliasable.Helper.createImportMap(Collections.singletonList(col), false).keySet(); + mvEnabledColumns.addAll(columnAliases); + mvIndicatorColumns.add(col.getName() + MvColumn.MV_INDICATOR_SUFFIX); + } + } + } + + try + { + DataLoader loader = DataLoader.get().createLoader(dataFile, null, true, null, TabLoader.TSV_FILE_TYPE); + loader.setThrowOnErrors(settings.isThrowOnErrors()); + loader.setInferTypes(shouldInferTypes); + + for (ColumnDescriptor column : loader.getColumns()) + { + if (dataDomain != null) + { + if (mvEnabledColumns.contains(column.name)) + { + column.setMvEnabled(dataDomain.getContainer()); + } + else if (mvIndicatorColumns.contains(column.name)) + { + column.setMvIndicator(dataDomain.getContainer()); + column.clazz = String.class; + } + DomainProperty prop = aliases.get(column.name); + if (prop != null) + { + // Allow String values through if the column is a lookup and the settings allow lookups by alternate key. + // The lookup table unique indices or display column value will be used to convert the column to the lookup value. + if (!(settings.isAllowLookupByAlternateKey() && column.clazz == String.class && prop.getLookup() != null)) + { + // Otherwise, just use the expected PropertyDescriptor's column type + column.clazz = prop.getPropertyDescriptor().getPropertyType().getJavaType(); + } + } + else + { + // It's not an expected column. Is it an MV indicator column? + if (!settings.isAllowUnexpectedColumns() && !mvIndicatorColumns.contains(column.name)) + { + column.load = false; + } + } + } + + if (settings.isBestEffortConversion()) + column.errorValues = DataLoader.ERROR_VALUE_USE_ORIGINAL; + else + column.errorValues = ERROR_VALUE; + } + return loader; + + } + catch (IOException ioe) + { + throw new ExperimentException("There was a problem loading the data file. " + (ioe.getMessage() == null ? "" : ioe.getMessage()), ioe); + } + } + + /** + * Reorders the first row of the list of rows to be in original column order. This is usually enough + * to cause serializers for tsv formats to respect the original file column order. A bit of a hack but + * the way row maps are generated make it difficult to preserve order at row map generation time. + */ + private void adjustFirstRowOrder(List> dataRows, DataLoader loader) throws IOException + { + Map firstRow = dataRows.remove(0); + Map newRow = new LinkedHashMap<>(); + + for (ColumnDescriptor column : loader.getColumns()) + { + if (firstRow.containsKey(column.name)) + newRow.put(column.name, firstRow.get(column.name)); + } + dataRows.add(0, newRow); + } + + @Override + public void beforeDeleteData(List data) throws ExperimentException + { + for (ExpData d : data) + { + ExpProtocolApplication sourceApplication = d.getSourceApplication(); + if (sourceApplication != null) + { + ExpRun run = sourceApplication.getRun(); + if (run != null) + { + ExpProtocol protocol = run.getProtocol(); + AssayProvider provider = AssayService.get().getProvider(protocol); + + Domain domain; + if (provider != null) + { + domain = provider.getResultsDomain(protocol); + } + else + { + // Be tolerant of the AssayProvider no longer being available. See if we have the default + // results/data domain for TSV-style assays + try + { + domain = AbstractAssayProvider.getDomainByPrefix(protocol, ExpProtocol.ASSAY_DOMAIN_DATA); + } + catch (IllegalStateException ignored) + { + domain = null; + // Be tolerant of not finding a domain anymore, if the provider has gone away + } + } + + if (domain != null && domain.getStorageTableName() != null) + { + SQLFragment deleteSQL = new SQLFragment("DELETE FROM "); + deleteSQL.append(domain.getDomainKind().getStorageSchemaName()); + deleteSQL.append("."); + deleteSQL.append(domain.getStorageTableName()); + deleteSQL.append(" WHERE DataId = ?"); + deleteSQL.add(d.getRowId()); + + try + { + new SqlExecutor(DbSchema.get(domain.getDomainKind().getStorageSchemaName())).execute(deleteSQL); + } + catch (BadSqlGrammarException x) + { + // (18035) presumably this is an optimistic concurrency problem and the table is gone + // postgres returns 42P01 in this case... SQL Server? + if (SqlDialect.isObjectNotFoundException(x)) + { + // CONSIDER: unfortunately we can't swallow this exception, because Postgres leaves + // the connection in an unusable state + } + throw x; + } + } + } + } + } + } + + public void importRows(ExpData data, User user, ExpRun run, ExpProtocol protocol, AssayProvider provider, List> rawData) + throws ExperimentException, ValidationException + { + importRows(data, user, run, protocol, provider, rawData, null); + } + + public void importRows(ExpData data, User user, ExpRun run, ExpProtocol protocol, AssayProvider provider, List> rawData, @Nullable DataLoaderSettings settings) + throws ExperimentException, ValidationException + { + if (settings == null) + settings = new DataLoaderSettings(); + + try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) + { + Container container = data.getContainer(); + ParticipantVisitResolver resolver = createResolver(user, run, protocol, provider, container); + + Domain dataDomain = provider.getResultsDomain(protocol); + + if (rawData.size() == 0) + { + if (allowEmptyData() || dataDomain.getProperties().isEmpty()) + { + transaction.commit(); + return; + } + else + { + throw new ExperimentException("Data file contained zero data rows"); + } + } + + final ContainerFilterable dataTable = provider.createProtocolSchema(user, container, protocol, null).createDataTable(); + + Map inputMaterials = checkData(container, user, dataTable, dataDomain, rawData, settings, resolver); + + List> fileData = convertPropertyNamesToURIs(rawData, dataDomain); + + insertRowData(data, user, container, run, protocol, provider, dataDomain, fileData, dataTable); + + if (shouldAddInputMaterials()) + { + AbstractAssayProvider.addInputMaterials(run, user, inputMaterials); + } + + transaction.commit(); + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + catch (IOException e) + { + throw new ExperimentException(e); + } + } + + protected ParticipantVisitResolver createResolver(User user, ExpRun run, ExpProtocol protocol, AssayProvider provider, Container container) + throws IOException, ExperimentException + { + return AssayService.get().createResolver(user, run, protocol, provider, null); + } + + /** Insert the data into the database. Transaction is active. */ + protected void insertRowData(ExpData data, User user, Container container, ExpRun run, ExpProtocol protocol, AssayProvider provider, Domain dataDomain, List> fileData, TableInfo tableInfo) + throws SQLException, ValidationException + { + if (tableInfo instanceof UpdateableTableInfo) + { + OntologyManager.insertTabDelimited(tableInfo, container, user, new SimpleAssayDataImportHelper(data), fileData, LOG); + } + else + { + Integer id = OntologyManager.ensureObject(container, data.getLSID()); + OntologyManager.insertTabDelimited(container, user, id, + new SimpleAssayDataImportHelper(data), dataDomain, fileData, false); + } + } + + protected abstract boolean shouldAddInputMaterials(); + + // NOTE: Calls filterColumns which mutates rawData in-place + private void checkColumns(Domain dataDomain, Set actual, List missing, List unexpected, List> rawData, boolean strict) + { + Set checkSet = new CaseInsensitiveHashSet(); + List expected = dataDomain.getProperties(); + for (DomainProperty pd : expected) + { + checkSet.add(pd.getName()); + if (pd.isMvEnabled()) + checkSet.add((pd.getName() + MvColumn.MV_INDICATOR_SUFFIX)); + } + for (String col : actual) + { + if (!checkSet.contains(col)) + unexpected.add(col); + } + if (!strict) + { + if (unexpected.size() > 0) + filterColumns(dataDomain, actual, rawData); + unexpected.clear(); + } + + // Now figure out what's missing but required + Map importMap = dataDomain.createImportMap(true); + // Consider all of them initially + LinkedHashSet missingProps = new LinkedHashSet<>(expected); + + // Iterate through the ones we got + for (String col : actual) + { + // Find the property that it maps to (via name, label, import alias, etc) + DomainProperty prop = importMap.get(col); + if (prop != null) + { + // If there's a match, don't consider it missing any more + missingProps.remove(prop); + } + } + + for (DomainProperty pd : missingProps) + { + if ((pd.isRequired() || strict)) + missing.add(pd.getName()); + } + } + + // NOTE: Mutates the rawData list in-place + private void filterColumns(Domain domain, Set actual, List> rawData) + { + Map expectedKey2ActualKey = new HashMap<>(); + for (Map.Entry aliased : domain.createImportMap(true).entrySet()) + { + for (String actualKey : actual) + { + if (actualKey.equalsIgnoreCase(aliased.getKey())) + { + expectedKey2ActualKey.put(aliased.getValue().getName(), actualKey); + } + } + } + ListIterator> iter = rawData.listIterator(); + while (iter.hasNext()) + { + Map filteredMap = new HashMap<>(); + Map rawDataRow = iter.next(); + for (Map.Entry expectedAndActualKeys : expectedKey2ActualKey.entrySet()) + { + filteredMap.put(expectedAndActualKeys.getKey(), rawDataRow.get(expectedAndActualKeys.getValue())); + } + iter.set(filteredMap); + } + } + + /** + * TODO: Replace with a DataIterator pipeline + * NOTE: Mutates the rawData list in-place + * @return the set of materials that are inputs to this run + */ + private Map checkData(Container container, User user, ContainerFilterable dataTable, Domain dataDomain, List> rawData, DataLoaderSettings settings, ParticipantVisitResolver resolver) + throws ValidationException, ExperimentException + { + List missing = new ArrayList<>(); + List unexpected = new ArrayList<>(); + + Set columnNames = Collections.emptySet(); + if (rawData != null && !rawData.isEmpty() && rawData.get(0) != null) + columnNames = rawData.get(0).keySet(); + + // For now, we'll only enforce that required columns are present. In the future, we'd like to + // do a strict check first, and then present ignorable warnings. + checkColumns(dataDomain, columnNames, missing, unexpected, rawData, false); + if (!missing.isEmpty() || !unexpected.isEmpty()) + { + StringBuilder builder = new StringBuilder(); + if (!missing.isEmpty()) + { + builder.append("Expected columns were not found: "); + for (java.util.Iterator it = missing.iterator(); it.hasNext();) + { + builder.append(it.next()); + if (it.hasNext()) + builder.append(", "); + else + builder.append(". "); + } + } + if (!unexpected.isEmpty()) + { + builder.append("Unexpected columns were found: "); + for (java.util.Iterator it = unexpected.iterator(); it.hasNext();) + { + builder.append(it.next()); + if (it.hasNext()) + builder.append(", "); + } + } + throw new ValidationException(builder.toString()); + } + + DomainProperty participantPD = null; + DomainProperty specimenPD = null; + DomainProperty visitPD = null; + DomainProperty datePD = null; + DomainProperty targetStudyPD = null; + + Map sampleNameSampleSets = new HashMap<>(); + Map> sampleNamesBySampleSet = new LinkedHashMap<>(); + + Map sampleIdSampleSets = new HashMap<>(); + Map> sampleIdsBySampleSet = new LinkedHashMap<>(); + + Map> sampleNames = new HashMap<>(); + Map> sampleIds = new HashMap<>(); + + List columns = dataDomain.getProperties(); + Map> validatorMap = new HashMap<>(); + Map remapMap = new HashMap<>(); + + for (DomainProperty pd : columns) + { + // initialize the DomainProperty validator map + validatorMap.put(pd, ColumnValidators.create(null, pd)); + + if (pd.getName().equalsIgnoreCase(AbstractAssayProvider.PARTICIPANTID_PROPERTY_NAME) && + pd.getPropertyDescriptor().getPropertyType() == PropertyType.STRING) + { + participantPD = pd; + } + else if (pd.getName().equalsIgnoreCase(AbstractAssayProvider.SPECIMENID_PROPERTY_NAME) && + pd.getPropertyDescriptor().getPropertyType() == PropertyType.STRING) + { + specimenPD = pd; + } + else if (pd.getName().equalsIgnoreCase(AbstractAssayProvider.VISITID_PROPERTY_NAME) && + pd.getPropertyDescriptor().getPropertyType() == PropertyType.DOUBLE) + { + visitPD = pd; + } + else if (pd.getName().equalsIgnoreCase(AbstractAssayProvider.DATE_PROPERTY_NAME) && + pd.getPropertyDescriptor().getPropertyType() == PropertyType.DATE_TIME) + { + datePD = pd; + } + else if (pd.getName().equalsIgnoreCase(AbstractAssayProvider.TARGET_STUDY_PROPERTY_NAME) && + pd.getPropertyDescriptor().getPropertyType() == PropertyType.STRING) + { + targetStudyPD = pd; + } + else + { + ExpSampleSet ss = DefaultAssayRunCreator.getLookupSampleSet(pd, container, user); + if (ss != null) + { + if (pd.getPropertyType().getJdbcType().isText()) + { + sampleNameSampleSets.put(pd, ss); + sampleNamesBySampleSet.put(ss, new HashSet<>()); + } + else + { + sampleIdSampleSets.put(pd, ss); + sampleIdsBySampleSet.put(ss, new HashSet<>()); + } + } + else if (DefaultAssayRunCreator.isLookupToMaterials(pd)) + { + if (pd.getPropertyType().getJdbcType().isText()) + sampleNames.put(pd, new HashSet<>()); + else + sampleIds.put(pd, new HashSet<>()); + } + } + + if (dataTable != null && settings.isAllowLookupByAlternateKey()) + { + ColumnInfo column = dataTable.getColumn(pd.getName()); + ForeignKey fk = column != null ? column.getFk() : null; + if (fk != null && fk.allowImportByAlternateKey()) + { + remapMap.put(pd, new SimpleTranslator.RemapPostConvert(fk.getLookupTableInfo(), true, SimpleTranslator.RemapMissingBehavior.Error)); + } + } + } + + boolean resolveMaterials = specimenPD != null || visitPD != null || datePD != null || targetStudyPD != null; + + Set wrongTypes = new HashSet<>(); + + Map materialInputs = new LinkedHashMap<>(); + + Map aliasMap = dataDomain.createImportMap(true); + + // We want to share canonical casing between data rows, or we end up with an extra Map instance for each + // data row which can add up quickly + CaseInsensitiveHashMap caseMapping = new CaseInsensitiveHashMap<>(); + ValidatorContext validatorContext = new ValidatorContext(container, user); + + int rowNum = 0; + for (ListIterator> iter = rawData.listIterator(); iter.hasNext();) + { + rowNum++; + Collection errors = new ArrayList<>(); + + Map originalMap = iter.next(); + Map map = new CaseInsensitiveHashMap<>(caseMapping); + // Rekey the map, resolving aliases to the actual property names + for (Map.Entry entry : originalMap.entrySet()) + { + DomainProperty prop = aliasMap.get(entry.getKey()); + if (prop != null) + { + map.put(prop.getName(), entry.getValue()); + } + } + + String participantID = null; + String specimenID = null; + Double visitID = null; + Date date = null; + Container targetStudy = null; + + for (DomainProperty pd : columns) + { + Object o = map.get(pd.getName()); + if (o instanceof String) + { + o = StringUtils.trimToNull((String) o); + map.put(pd.getName(), o); + iter.set(map); + } + + // validate the data value + if (validatorMap.containsKey(pd)) + { + for (ColumnValidator validator : validatorMap.get(pd)) + { + String error = validator.validate(rowNum, o, validatorContext); + if (error != null) + errors.add(new PropertyValidationError(error, pd.getName())); + } + } + + if (participantPD == pd) + { + participantID = o instanceof String ? (String)o : null; + } + else if (specimenPD == pd) + { + specimenID = o instanceof String ? (String)o : null; + } + else if (visitPD == pd && o != null) + { + visitID = o instanceof Number ? ((Number)o).doubleValue() : null; + } + else if (datePD == pd && o != null) + { + date = o instanceof Date ? (Date) o : null; + } + else if (targetStudyPD == pd && o != null) + { + Set studies = StudyService.get().findStudy(o, null); + if (studies.isEmpty()) + { + errors.add(new PropertyValidationError("Couldn't resolve " + pd.getName() + " '" + o.toString() + "' to a study folder.", pd.getName())); + } + else if (studies.size() > 1) + { + errors.add(new PropertyValidationError("Ambiguous " + pd.getName() + " '" + o.toString() + "'.", pd.getName())); + } + if (!studies.isEmpty()) + { + Study study = studies.iterator().next(); + targetStudy = study != null ? study.getContainer() : null; + } + } + + boolean valueMissing; + if (o == null) + { + valueMissing = true; + } + else if (o instanceof MvFieldWrapper) + { + MvFieldWrapper mvWrapper = (MvFieldWrapper)o; + if (mvWrapper.isEmpty()) + valueMissing = true; + else + { + valueMissing = false; + if (!MvUtil.isValidMvIndicator(mvWrapper.getMvIndicator(), dataDomain.getContainer())) + { + String columnName = pd.getName() + MvColumn.MV_INDICATOR_SUFFIX; + wrongTypes.add(columnName); + errors.add(new PropertyValidationError(columnName + " must be a valid MV indicator.", columnName)); + } + } + + } + else + { + valueMissing = false; + } + + // If the column is a file link or attachment, resolve the value to a File object + String uri = pd.getType().getTypeURI(); + if (uri.equals(PropertyType.FILE_LINK.getTypeUri()) || uri.equals(PropertyType.ATTACHMENT.getTypeUri())) + { + if ("".equals(o)) + { + // Issue 36502: If the original input was an empty value, set it to null so we won't store an empty string in the database + o = null; + map.put(pd.getName(), null); + } + else + { + // File column values are stored as the absolute resolved path + File resolvedFile = AssayUploadFileResolver.resolve(o, container, pd); + if (resolvedFile != null) + { + o = resolvedFile; + map.put(pd.getName(), o); + iter.set(map); + } + } + } + + // If we have a String value for a lookup column, attempt to use the table's unique indices or display value to convert the String into the lookup value + // See similar conversion performed in SimpleTranslator.RemapPostConvertColumn + if (o instanceof String && remapMap.containsKey(pd)) + { + SimpleTranslator.RemapPostConvert remap = remapMap.get(pd); + Object remapped = null; + try + { + remapped = remap.mappedValue(o); + } + catch (ConversionException ex) + { + errors.add(new PropertyValidationError("Failed to convert '" + pd.getName() + "': " + ex.getMessage(), pd.getName())); + } + + if (o != remapped) + { + o = remapped; + map.put(pd.getName(), remapped); + iter.set(map); + } + } + + if (!valueMissing && o == ERROR_VALUE && !wrongTypes.contains(pd.getName())) + { + wrongTypes.add(pd.getName()); + errors.add(new PropertyValidationError(pd.getName() + " must be of type " + ColumnInfo.getFriendlyTypeName(pd.getPropertyDescriptor().getPropertyType().getJavaType()) + ".", pd.getName())); + } + + // Collect sample names or ids for each of the SampleSet lookup columns + ExpSampleSet byNameSS = sampleNameSampleSets.get(pd); + if (byNameSS != null && o instanceof String) + sampleNamesBySampleSet.get(byNameSS).add((String)o); + + ExpSampleSet byIdSS = sampleIdSampleSets.get(pd); + if (byIdSS != null && o instanceof Integer) + sampleIdsBySampleSet.get(byIdSS).add((Integer)o); + + if (DefaultAssayRunCreator.isLookupToMaterials(pd)) + { + if (sampleNames.containsKey(pd) && o instanceof String) + sampleNames.get(pd).add((String)o); + else if (sampleIds.containsKey(pd) && o instanceof Integer) + sampleIds.get(pd).add((Integer)o); + } + } + + if (!errors.isEmpty()) + throw new ValidationException(errors, rowNum); + + ParticipantVisit participantVisit = resolver.resolve(specimenID, participantID, visitID, date, targetStudy); + if (participantPD != null && map.get(participantPD.getName()) == null) + { + map.put(participantPD.getName(), participantVisit.getParticipantID()); + iter.set(map); + } + if (visitPD != null && map.get(visitPD.getName()) == null) + { + map.put(visitPD.getName(), participantVisit.getVisitID()); + iter.set(map); + } + if (datePD != null && map.get(datePD.getName()) == null) + { + map.put(datePD.getName(), participantVisit.getDate()); + iter.set(map); + } + if (targetStudyPD != null && participantVisit.getStudyContainer() != null) + { + // Original TargetStudy value may have been a container id, container path, or a study label. + // Store all TargetStudy values as Container ID string. + map.put(targetStudyPD.getName(), participantVisit.getStudyContainer().getId()); + iter.set(map); + } + + if (resolveMaterials) + { + materialInputs.put(participantVisit.getMaterial(), null); + } + } + + // Resolve sample lookups for each SampleSet + resolveSampleNames(container, user, sampleNameSampleSets, sampleNamesBySampleSet, materialInputs); + resolveSampleIds(sampleIdSampleSets, sampleIdsBySampleSet, materialInputs); + + // Resolve sample lookups to exp.Material + for (Map.Entry> entry : sampleNames.entrySet()) + { + resolveSampleNames(container, user, null, entry.getKey(), entry.getValue(), materialInputs); + } + for (Map.Entry> entry : sampleIds.entrySet()) + { + resolveSampleIds(null, entry.getKey(), entry.getValue(), materialInputs); + } + + return materialInputs; + } + + private void resolveSampleNames(Container container, User user, Map sampleSets, Map> sampleNamesBySampleSet, Map materialInputs) throws ExperimentException + { + for (Map.Entry> entry : sampleNamesBySampleSet.entrySet()) + { + // Default to looking in the current container + ExpSampleSet ss = entry.getKey(); + Set sampleNames = entry.getValue(); + + // Find the DomainProperty and use its name as the role + DomainProperty dp = null; + for (Map.Entry pair : sampleSets.entrySet()) + { + if (pair.getValue().equals(ss)) + { + dp = pair.getKey(); + break; + } + } + + resolveSampleNames(container, user, ss, dp, sampleNames, materialInputs); + } + } + + private void resolveSampleNames(Container container, User user, @Nullable ExpSampleSet ss, @Nullable DomainProperty dp, Set sampleNames, Map materialInputs) throws ExperimentException + { + // use DomainProperty name as the role + String role = dp == null ? null : dp.getName(); // TODO: More than one DomainProperty could be a lookup to the SampleSet + + Set searchContainers = ExpSchema.getSearchContainers(container, ss, dp, user); + + for (Container searchContainer : searchContainers) + { + List materials = ExperimentService.get().getExpMaterials(searchContainer, user, sampleNames, ss, false, false); + + for (ExpMaterial material : materials) + if (!materialInputs.containsKey(material)) + materialInputs.put(material, role); + } + } + + + private void resolveSampleIds(Map sampleSets, Map> sampleIdsBySampleSet, Map materialInputs) + { + for (Map.Entry> entry : sampleIdsBySampleSet.entrySet()) + { + ExpSampleSet ss = entry.getKey(); + Set sampleIds = entry.getValue(); + + // Find the DomainProperty and use its name as the role + DomainProperty dp = null; + for (Map.Entry pair : sampleSets.entrySet()) + { + if (pair.getValue().equals(ss)) + { + dp = pair.getKey(); + break; + } + } + + resolveSampleIds(ss, dp, sampleIds, materialInputs); + } + } + + private void resolveSampleIds(@Nullable ExpSampleSet ss, DomainProperty dp, Set sampleIds, Map materialInputs) + { + // use DomainProperty name as the role + String role = null; + if (dp != null) + { + role = dp.getName(); // TODO: More than one DomainProperty could be a lookup to the SampleSet + } + + List materials = ExperimentService.get().getExpMaterials(sampleIds); + + for (ExpMaterial material : materials) + { + // Ignore materials that aren't in the lookup sample set + if (ss != null && !ss.getLSID().equals(material.getCpasType())) + continue; + + if (!materialInputs.containsKey(material)) + materialInputs.put(material, role); + } + } + + /** Wraps each map in a version that can be queried based on on any of the aliases (name, property URI, import + * aliases, etc for a given property */ + // NOTE: Mutates the rawData list in place + protected List> convertPropertyNamesToURIs(List> dataMaps, Domain domain) + { + // Get the mapping of different names to the set of domain properties + final Map importMap = domain.createImportMap(true); + + // For a given property, find all the potential names it by which it could be referenced + final Map> propToNames = new HashMap<>(); + for (Map.Entry entry : importMap.entrySet()) + { + Set allNames = propToNames.get(entry.getValue()); + if (allNames == null) + { + allNames = new HashSet<>(); + propToNames.put(entry.getValue(), allNames); + } + allNames.add(entry.getKey()); + } + + // We want to share canonical casing between data rows, or we end up with an extra Map instance for each + // data row which can add up quickly + CaseInsensitiveHashMap caseMapping = new CaseInsensitiveHashMap<>(); + for (ListIterator> i = dataMaps.listIterator(); i.hasNext(); ) + { + Map dataMap = i.next(); + CaseInsensitiveHashMap newMap = new PropertyLookupMap(dataMap, caseMapping, importMap, propToNames); + + // Swap out the entry in the list with the transformed map + i.set(newMap); + } + return dataMaps; + } + + public void deleteData(ExpData data, Container container, User user) + { + OntologyManager.deleteOntologyObjects(container, data.getLSID()); + } + + public ActionURL getContentURL(ExpData data) + { + ExpRun run = data.getRun(); + if (run != null) + { + ExpProtocol protocol = run.getProtocol(); + return PageFlowUtil.urlProvider(AssayUrls.class).getAssayResultsURL(data.getContainer(), protocol, run.getRowId()); + } + return null; + } + + /** Wrapper around a row's key->value map that can find the values based on any of the DomainProperty's potential + * aliases, like the property name, URI, import aliases, etc */ + private static class PropertyLookupMap extends CaseInsensitiveHashMap + { + private final Map _importMap; + private final Map> _propToNames; + + public PropertyLookupMap(Map dataMap, CaseInsensitiveHashMap caseMapping, Map importMap, Map> propToNames) + { + super(dataMap, caseMapping); + _importMap = importMap; + _propToNames = propToNames; + } + + @Override + public Object get(Object key) + { + Object result = super.get(key); + + // If we can't find the value based on the name that was passed in, try any of its alternatives + if (result == null && key instanceof String) + { + // Find the property that's associated with that name + DomainProperty property = _importMap.get(key); + if (property != null) + { + // Find all of the potential synonyms + Set allNames = _propToNames.get(property); + if (allNames != null) + { + for (String name : allNames) + { + // Look for a value under that name + result = super.get(name); + if (result != null) + { + break; + } + } + } + } + } + return result; + } + } +}